prevent awk from replacing ampersand's with matched text

#
# Render a zodiac page
#

BEGIN {
  action = "none"
  ext = "none"
  main = "_main"
  helpers_loaded = "no"
  layout = ""
}

{
  split(FILENAME, parts, ".")
  ext = parts[length(parts)]
  if (FILENAME == "-") {
    action = "config"
  } else if (ext == "meta") {
    action = "meta"
  } else if (ext == "layout") {
    action = "layout"
  } else {
    action = "page"
    filter_ext = ext
  }
}

# Process lines from meta files
action == "meta" {
  split($0, kv, ": ")
  data[kv[1]] = kv[2]
  next
}

# Done processing meta
# Since data is loaded now, load the helpers
action != "meta" && action != "config" && helpers_loaded == "no" && helpers == "yes" {
  load_helpers()
  helpers_loaded = "yes"
}

# Process lines from the page
action == "page" {
  if (!contents[main]) {
    contents[main] = bind_data($0)

    # save the extension for this content type
    # to find the appropriate filter to render it
    filter_exts[main] = ext
  } else {
    contents[main] = contents[main] "\n" bind_data($0)
  }
  next
}

# Process lines from the layout
action == "layout" {

  # replace yield with rendered content
  if (match($0, /{{{yield}}}/)) {
    sub(/{{{yield}}}/, escape_special_chars(render_content(main)))
  }

  if (layout == "") {
    layout = bind_data($0)
  } else {
    layout = layout "\n" bind_data($0)
  }
}

END {
  if (layout != "") {
    print layout
  } else {
    print render_content(main)
  }
}

function bind_data(txt,   tag, key) {
  if (match(txt, /{{([^}]*)}}/)) {
    tag = substr(txt, RSTART, RLENGTH)
    match(tag, /([[:alnum:]_]|[?]).*[^}]/)
    key = substr(tag, RSTART, RLENGTH)
    gsub(tag, data[key], txt)
    return bind_data(txt, data)
  } else {
    return txt
  }
}

function render_content(type,     ext_key, filter_ext, filter_cmd, txt) {
  ext_key = type "_ext"

  # Get the extension of the content type
  filter_ext = filter_exts[type]

  # Get the appropriate filter command for this extension
  filter_cmd = filter[filter_ext]

  # Get the text of the content for the given type
  txt = contents[type]

  if (filter_cmd != "none") {
    return run_filter(filter_cmd, txt)
  } else {
    return txt
  }
}

function run_filter(cmd, txt,   rand_date, tmpfile, rendered_txt, date_cmd, filter_cmd, line) {
  date_cmd = "date +%Y%m%d%H%M%S"
  date_cmd | getline rand_date
  close(date_cmd)

  tmpfile = "/tmp/awk_render" rand_date
  filter_cmd = cmd " > " tmpfile

  # pipe content to filter command
  print txt | filter_cmd
  close(filter_cmd)

  # pull out the filtered page
  while((getline line < tmpfile) > 0) {
    rendered_txt = rendered_txt "\n" line
  }
  close(tmpfile)

  system("rm " tmpfile)

  return rendered_txt
}

# Prevent awk from replacing ampersand's with matched text
function escape_special_chars(  txt) {
  gsub(/&/, "\\\\&", txt)
  return txt
}

#
# Render a zodiac page
#

BEGIN {
  action = "none"
  ext = "none"
  main = "_main"
  helpers_loaded = "no"
  layout = ""
}

{
  split(FILENAME, parts, ".")
  ext = parts[length(parts)]
  if (FILENAME == "-") {
    action = "config"
  } else if (ext == "meta") {
    action = "meta"
  } else if (ext == "layout") {
    action = "layout"
  } else {
    action = "page"
    filter_ext = ext
  }
}

# Process lines from meta files
action == "meta" {
  split($0, kv, ": ")
  data[kv[1]] = kv[2]
  next
}

# Done processing meta
# Since data is loaded now, load the helpers
action != "meta" && action != "config" && helpers_loaded == "no" && helpers == "yes" {
  load_helpers()
  helpers_loaded = "yes"
}

# Process lines from the page
action == "page" {
  if (!contents[main]) {
    contents[main] = bind_data($0)

    # save the extension for this content type
    # to find the appropriate filter to render it
    filter_exts[main] = ext
  } else {
    contents[main] = contents[main] "\n" bind_data($0)
  }
  next
}

# Process lines from the layout
action == "layout" {

  # replace yield with rendered content
  if (match($0, /{{{yield}}}/)) {
    sub(/{{{yield}}}/, render_content(main))
  }

  if (layout == "") {
    layout = bind_data($0)
  } else {
    layout = layout "\n" bind_data($0)
  }
}

END {
  if (layout != "") {
    print layout
  } else {
    print render_content(main)
  }
}

function bind_data(txt,   tag, key) {
  if (match(txt, /{{([^}]*)}}/)) {
    tag = substr(txt, RSTART, RLENGTH)
    match(tag, /([[:alnum:]_]|[?]).*[^}]/)
    key = substr(tag, RSTART, RLENGTH)
    gsub(tag, data[key], txt)
    return bind_data(txt, data)
  } else {
    return txt
  }
}

function render_content(type,     ext_key, filter_ext, filter_cmd, txt) {
  ext_key = type "_ext"

  # Get the extension of the content type
  filter_ext = filter_exts[type]

  # Get the appropriate filter command for this extension
  filter_cmd = filter[filter_ext]

  # Get the text of the content for the given type
  txt = contents[type]

  if (filter_cmd != "none") {
    return run_filter(filter_cmd, txt)
  } else {
    return txt
  }
}

function run_filter(cmd, txt,   rand_date, tmpfile, rendered_txt, date_cmd, filter_cmd, line) {
  date_cmd = "date +%Y%m%d%H%M%S"
  date_cmd | getline rand_date
  close(date_cmd)

  tmpfile = "/tmp/awk_render" rand_date
  filter_cmd = cmd " > " tmpfile

  # pipe content to filter command
  print txt | filter_cmd
  close(filter_cmd)

  # pull out the filtered page
  while((getline line < tmpfile) > 0) {
    rendered_txt = rendered_txt "\n" line
  }
  close(tmpfile)

  system("rm " tmpfile)

  return rendered_txt
}


Translate: fix getTranslation()

Translate: fix getTranslation()

#!/usr/bin/gawk -f

@include "lib/Commons"
@include "lib/PLTokenizer"
@include "lib/PLParser"

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist,
                        ####
                        altTranslations, article, ast, content,
                        explanation, group, i, il,
                        isPhonetic, j, original, phonetics,
                        r, rtl, saveSortedIn, segments,
                        temp, tokens, translation, translations,
                        word, words, wordClasses) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (getCode(tl) != "auto" && rtl = Locale[getCode(tl)]["rtl"]) {
        # Check if target language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(tl)]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    content = getResponse(text, sl, tl, hl)

    plTokenize(tokens, content)
    plParse(ast, tokens)

    if (!anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
    }

    # Debug mode
    if (Option["debug"]) {
        d(sprintf("content='%s'", content))
        da(tokens, "tokens[%s]='%s'")
        da(ast, "ast[%s]='%s'")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            segments[group[1]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # Identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$")
            append(il, literal(ast[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    if (!anything(il)) il[0] = sl

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            join(phonetics) :  # phonetic transcription
            s(translation, tl) # target language

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        r = AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
        if (anything(phonetics))
            r = r "\n" AnsiCode["bold"] join(phonetics) AnsiCode["no bold"] # phonetic transcription

        if (isarray(altTranslations[0]) && anything(altTranslations[0])) {
            # List alternative translations

            if (Locale[getCode(hl)]["rtl"] || Locale[getCode(il[0])]["rtl"])
                r = r "\n\n" s(sprintf(Locale[getCode(hl)]["message"], join(original))) # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
            else
                r = r "\n\n" sprintf(Locale[getCode(hl)]["message"], join(original))
            if (Locale[getCode(il[0])]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" s("(" Locale[getCode(il[0])]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
            else
                r = r "\n" "(" Locale[getCode(il[0])]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")"

            temp = segments[0] "(" join(altTranslations[0], "/") ")"
            for (i = 1; i < length(altTranslations); i++)
                temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
            if (Locale[getCode(il[0])]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
            else
                r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
        }

        if (isarray(wordClasses) && anything(wordClasses)) {
            # List dictionary entries

            for (i = 0; i < length(words); i++) {
                r = r "\n\n" s("[" wordClasses[i] "]", hl) # home language
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    explanation = join(words[i][j][1], ", ")
                    article = words[i][j][4]

                    if (rtl) {
                        r = r "\n" AnsiCode["bold"] sprintf("%" Option["width"] - 4 "s", s((article ?
                                                                                "(" article ")" :
                                                                                "") " " word, tl, Option["width"] - 4)) AnsiCode["no bold"] # target language
                        r = r "\n" s(explanation, il[0], Option["width"] - 8) # identified source language
                    } else {
                        r = r "\n" "    " AnsiCode["bold"] show((article ?
                                                                 "(" article ") " :
                                                                 "") word, tl) AnsiCode["no bold"] # target language
                        r = r "\n" "        " s(explanation, il[0], Option["width"] - 8) # identified source language
                    }
                }
            }
        }

        if (toSpeech) {
            if (index(Locale[getCode(hl)]["message"], "%s") > 2) {
                returnPlaylist[0]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il[0]
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il[0]
                returnPlaylist[1]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}

# Translate the source text (into all target languages).
function translate(text,
                   ####
                   i, j, playlist) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (getCode(Option["hl"]) != "auto" && Locale[getCode(Option["hl"])]["rtl"]) {
        # Check if home language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["hl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (getCode(Option["sl"]) != "auto" && Locale[getCode(Option["sl"])]["rtl"]) {
        # Check if source language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["sl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between translations
        if (!Option["interactive"]) {
            # Separator for multiple targets
            if (i > 1) {
                for (j = 0; j < Option["width"]; j++) printf "─"
                printf "\n"
            }
        }

        print getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"], playlist) > Option["output"]

        if (Option["play"])
            if (Option["player"])
                for (j in playlist)
                    play(playlist[j]["text"], playlist[j]["tl"])
            else if (SpeechSynthesizer)
                for (j in playlist)
                    print playlist[j]["text"] | SpeechSynthesizer
    }
}

#!/usr/bin/gawk -f

@include "lib/Commons"
@include "lib/PLTokenizer"
@include "lib/PLParser"

# Get the translation of a string.
function getTranslation(text, sl, tl, hl,
                        isVerbose, toSpeech, returnPlaylist,
                        ####
                        altTranslations, article, ast, content,
                        explanation, group, i, il,
                        isPhonetic, j, original, phonetics,
                        r, rtl, saveSortedIn, segments,
                        temp, tokens, translation, translations,
                        word, words, wordClasses) {
    isPhonetic = match(tl, /^@/)
    tl = substr(tl, 1 + isPhonetic)

    if (!getCode(tl)) {
        # Check if target language is supported
        w("[WARNING] Unknown target language code: " tl)
    } else if (getCode(tl) != "auto" && rtl = Locale[getCode(tl)]["rtl"]) {
        # Check if target language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(tl)]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    content = getResponse(text, sl, tl, hl)

    plTokenize(tokens, content)
    plParse(ast, tokens)

    if (!anything(ast)) {
        e("[ERROR] Oops! Something went wrong and I can't translate it for you :(")
        ExitCode = 1
    }

    # Debug mode
    if (Option["debug"]) {
        d(sprintf("content='%s'", content))
        da(tokens, "tokens[%s]='%s'")
        da(ast, "ast[%s]='%s'")
    }

    saveSortedIn = PROCINFO["sorted_in"]
    PROCINFO["sorted_in"] = "@ind_num_asc"
    for (i in ast) {
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "0$")
            append(translations, postprocess(literal(ast[i])))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "1$")
            append(original, literal(ast[i]))
        if (i ~ "^0" SUBSEP "0" SUBSEP "[[:digit:]]+" SUBSEP "2$")
            append(phonetics, literal(ast[i]))

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            wordClasses[group[1]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]][group[3]] = literal(ast[i])
        if (match(i, "^0" SUBSEP "1" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "1" SUBSEP "([[:digit:]]+)$", group))
            words[group[1]][group[2]]["1"][group[3]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            segments[group[1]] = literal(ast[i])

        if (match(i, "^0" SUBSEP "5" SUBSEP "([[:digit:]]+)" SUBSEP "2" SUBSEP "([[:digit:]]+)" SUBSEP "0$", group))
            altTranslations[group[1]][group[2]] = postprocess(literal(ast[i]))

        # Identified source languages
        if (i ~ "^0" SUBSEP "8" SUBSEP "0" SUBSEP "[[:digit:]]+$")
            append(il, literal(ast[i]))
    }
    PROCINFO["sorted_in"] = saveSortedIn

    translation = join(translations)

    if (!anything(il)) il[0] = sl

    # Generate output
    if (!isVerbose) {
        # Brief mode

        r = isPhonetic && anything(phonetics) ?
            join(phonetics) :  # phonetic transcription
            s(translation, tl) # target language

        if (toSpeech) {
            returnPlaylist[0]["text"] = translation
            returnPlaylist[0]["tl"] = tl
        }

    } else {
        # Verbose mode

        r = AnsiCode["bold"] s(translation, tl) AnsiCode["no bold"] # target language
        if (anything(phonetics))
            r = r "\n" AnsiCode["bold"] join(phonetics) AnsiCode["no bold"] # phonetic transcription

        if (anything(altTranslations[0])) {
            # List alternative translations

            if (Locale[getCode(hl)]["rtl"] || Locale[getCode(il[0])]["rtl"])
                r = r "\n\n" s(sprintf(Locale[getCode(hl)]["message"], join(original))) # caution: mixed languages, BiDi invoked must be implemented correctly (i.e. FriBidi is required)
            else
                r = r "\n\n" sprintf(Locale[getCode(hl)]["message"], join(original))
            if (Locale[getCode(il[0])]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" s("(" Locale[getCode(il[0])]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")") # caution: mixed languages
            else
                r = r "\n" "(" Locale[getCode(il[0])]["endonym"] " ➔ " Locale[getCode(tl)]["endonym"] ")"

            temp = segments[0] "(" join(altTranslations[0], "/") ")"
            for (i = 1; i < length(altTranslations); i++)
                temp = temp " " segments[i] "(" join(altTranslations[i], "/") ")"
            if (Locale[getCode(il[0])]["rtl"] || Locale[getCode(tl)]["rtl"])
                r = r "\n" AnsiCode["bold"] s(temp) AnsiCode["no bold"] # caution: mixed languages
            else
                r = r "\n" AnsiCode["bold"] temp AnsiCode["no bold"]
        }

        if (anything(wordClasses)) {
            # List dictionary entries

            for (i = 0; i < length(words); i++) {
                r = r "\n\n" s("[" wordClasses[i] "]", hl) # home language
                for (j = 0; j < length(words[i]); j++) {
                    word = words[i][j][0]
                    explanation = join(words[i][j][1], ", ")
                    article = words[i][j][4]

                    if (rtl) {
                        r = r "\n" AnsiCode["bold"] sprintf("%" Option["width"] - 4 "s", s((article ?
                                                                                "(" article ")" :
                                                                                "") " " word, tl, Option["width"] - 4)) AnsiCode["no bold"] # target language
                        r = r "\n" s(explanation, il[0], Option["width"] - 8) # identified source language
                    } else {
                        r = r "\n" "    " AnsiCode["bold"] show((article ?
                                                                 "(" article ") " :
                                                                 "") word, tl) AnsiCode["no bold"] # target language
                        r = r "\n" "        " s(explanation, il[0], Option["width"] - 8) # identified source language
                    }
                }
            }
        }

        if (toSpeech) {
            if (index(Locale[getCode(hl)]["message"], "%s") > 2) {
                returnPlaylist[0]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[0]["tl"] = hl
                returnPlaylist[1]["text"] = join(original)
                returnPlaylist[1]["tl"] = il[0]
            } else {
                returnPlaylist[0]["text"] = join(original)
                returnPlaylist[0]["tl"] = il[0]
                returnPlaylist[1]["text"] = sprintf(Locale[getCode(hl)]["message"], "")
                returnPlaylist[1]["tl"] = hl
            }
            returnPlaylist[2]["text"] = translation
            returnPlaylist[2]["tl"] = tl
        }
    }

    return r
}

# Translate the source text (into all target languages).
function translate(text,
                   ####
                   i, j, playlist) {

    if (!getCode(Option["hl"])) {
        # Check if home language is supported
        w("[WARNING] Unknown language code: " Option["hl"] ", fallback to English: en")
        Option["hl"] = "en" # fallback to English
    } else if (getCode(Option["hl"]) != "auto" && Locale[getCode(Option["hl"])]["rtl"]) {
        # Check if home language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["hl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    if (!getCode(Option["sl"])) {
        # Check if source language is supported
        w("[WARNING] Unknown source language code: " Option["sl"])
    } else if (getCode(Option["sl"]) != "auto" && Locale[getCode(Option["sl"])]["rtl"]) {
        # Check if source language is RTL
        if (!FriBidi)
            w("[WARNING] " Locale[getCode(Option["sl"])]["name"] " is a right-to-left language, but GNU FriBidi is not found on your system.\nText might be displayed incorrectly.")
    }

    for (i in Option["tl"]) {
        # Non-interactive verbose mode: separator between translations
        if (!Option["interactive"]) {
            # Separator for multiple targets
            if (i > 1) {
                for (j = 0; j < Option["width"]; j++) printf "─"
                printf "\n"
            }
        }

        print getTranslation(text, Option["sl"], Option["tl"][i], Option["hl"], Option["verbose"], Option["play"], playlist) > Option["output"]

        if (Option["play"])
            if (Option["player"])
                for (j in playlist)
                    play(playlist[j]["text"], playlist[j]["tl"])
            else if (SpeechSynthesizer)
                for (j in playlist)
                    print playlist[j]["text"] | SpeechSynthesizer
    }
}


chimeric script should ignore header lines in SAM files

#!/usr/bin/awk -f
##########
#The MIT License (MIT)
#
# Copyright (c) 2015 Aiden Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
##########
# Takes a SAM file, looks for chimeric reads and normal reads.  Outputs
# only those reads mapped to chromosomes 1-24 with MAPQ > 0.
#
# Output to fname1 is of the form:
# strand1 chromosome1 position1 frag1 strand2 chromosome2 position2 frag2
#  mapq1 cigar1 seq1 mapq2 cigar2 seq2
#   where chr1 is always <= chr2
#
# Output to fname2 retains SAM format.
#
# Chimeric reads are treated as follows:
# "Normal" chimeras (with 3 reads, two with the ligation junction), where the
# third read (w/o ligation junction) is within 20Kbp of one of the ligation
# junction reads, are sent to fname1.  All other chimeras (where either there
# are more than 3 reads, or they don't contain the ligation junction, or the
# one non-ligation junction end is far from the others, are sent to fname2.
#
# awk -f chimeric_blacklist.awk -v fname1="norm_chimera" fname2="abnorm_chimera" fname3="unmapped"
# Juicer version 1.5

# returns absolute value
function abs(value)
{
  return (value<0?-value:value);
}
# returns minimum of two values
function min(value1,value2)
{
  return (value1<value2?value1:value2);
}
# examines read1 (s1,c1,p1) versus read2 and returns true if
# the first read comes before the second read
# this is so duplicates can be found after sorting even when the strand and
# chromosome are the same
function less_than(s1,c1,p1,s2,c2,p2)
{

  if (c1 < c2) return 1;
  if (c1 > c2) return 0;
  # c1 == c2
  if (s1 < s2) return 1;
  if (s1 > s2) return 0;
  # s1 == s2 && c1 == c2
  if (p1 < p2) return 1;
  if (p1 > p2) return 0;
  # all are equal, doesn't matter
  return 1;
}
BEGIN{
  OFS="\t";
  tottot = -1; # will count first non-group
}
$0 !~ /^@/{
  # input file is sorted by read name.  Look at read name to group
  # appropriately
  split($1,a,"/");
  if(a[1]==prev){
    # move on to next record.  look below this block for actions that occur
    # regardless.
    count++;
  }
  else {
    # deal with read pair group
    tottot++;
    if (count==3) {
      # chimeric read
      for (j=1; j <= 3; j++) {
				split(c[j], tmp);
				split(tmp[1],readname,"/");
				read[j] = readname[2];
				name[j] = tmp[1];

				# strand; Bit 16 set means reverse strand
				str[j] = and(tmp[2],16);
				# chromosome
				chr[j] = tmp[3];
				# position
				pos[j] = tmp[4];
				# mapq score
				m[j] = tmp[5];
				# cigar string
				cigarstr[j] = tmp[6];
				# sequence
				seq[j] = tmp[10];
        qual[j] = tmp[11];

				# get rid of soft clipping to know correct position
				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				# get rid of hard clipping to know correct position
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					# can look like 15M10S20M, need to count all not just first
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					# add soft clipped bases in for proper position
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
          else if (tmp[6] ~ /[0-9]+H$/) {
              where = match(tmp[6],/[0-9]+H$/);
              cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
              pos[j] = pos[j] + cigloc;
          }
					# Mitochrondria loops around
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        # blacklist - if 3rd bit set (=4) it means unmapped
        mapped[j] = and(tmp[2],4) == 0;
      }

      dist[12] = abs(chr[1]-chr[2])*10000000 + abs(pos[1]-pos[2]);
      dist[23] = abs(chr[2]-chr[3])*10000000 + abs(pos[2]-pos[3]);
      dist[13] = abs(chr[1]-chr[3])*10000000 + abs(pos[1]-pos[3]);

      if (min(dist[12],min(dist[23],dist[13])) < 1000) {
				# The paired ends look like A/B...B.  Make sure we take A and B.
				if (read[1] == read[2]) {
					# take the unique one "B" for sure
					read2 = 3;
					# take the end of "A/B" that isn't close to "B"
					read1 = dist[13] > dist[23] ? 1:2;
				}
				else if (read[1] == read[3]) {
					read2 = 2;
					read1 = dist[12] > dist[23] ? 1:3;
				}
				else if (read[2] == read[3]) {
					read2 = 1;
					read1 = dist[12] > dist[13] ? 2:3;
				}
				else {
					printf("reads strange\n") > "/dev/stderr"
					exit 1
				}

				if (mapped[read1] && mapped[read2]) {
					count_norm++;
					if (less_than(str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2])) {
              print str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2],m[read1],cigarstr[read1],seq[read1],m[read2],cigarstr[read2],seq[read2],name[read1],name[read2] > fname1;
					}
					else {
              print str[read2],chr[read2],pos[read2],str[read1],chr[read1],pos[read1],m[read2],cigarstr[read2],seq[read2],m[read1],cigarstr[read1],seq[read1],name[read2],name[read1] > fname1;
					}
				}
				else {
					for (i in c) {
						print c[i] > fname3;
					}
					count_unmapped++;
				}
      }
			else {
				# chimeric read with the 3 ends > 1KB apart
				count_abnorm++;
				for (i in c) {
					print c[i] > fname2;
				}
      }
    }
    else if (count > 3) {
      # chimeric read > 3, too many to deal with
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    else if (count == 2) {
			# code here should be same as above, but it's a "normal" read
      j = 0;
      for (i in c) {
				split(c[i], tmp);
				split(tmp[1],readname,"/");
				str[j] = and(tmp[2],16);
				chr[j] = tmp[3];
				pos[j] = tmp[4];
				m[j] = tmp[5];
				cigarstr[j] = tmp[6];
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				name[j] = tmp[1];

        # blacklist - if 3rd bit set (=4) it means unmapped
        mapped[j] = and(tmp[2],4) == 0;

				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
				j++;
      }
      if (mapped[0] && mapped[1]) {
				count_reg++;
				if (less_than(str[0],chr[0],pos[0],str[1],chr[1],pos[1])) {
					# ideally we'll get rid of printing out cigar string at some point
            print str[0],chr[0],pos[0],str[1],chr[1],pos[1],m[0],cigarstr[0],seq[0],m[1],cigarstr[1],seq[1],name[0],name[1] > fname1;
				}
				else {
            print str[1],chr[1],pos[1],str[0],chr[0],pos[0],m[1],cigarstr[1],seq[1],m[0],cigarstr[0],seq[0],name[1],name[0] > fname1;
				}
      }
      else {
				for (i in c) {
					print c[i] > fname3;
				}
				count_unmapped++;
      }
    }
    else if (count == 1) {
      # this actually shouldn't happen, but it happens with alternate aligners on occasion
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    # reset variables
    delete c;
    count=1;
    prev=a[1];
  }
  # these happen no matter what, after the above processing
  c[count] = $0;
}
END{
    # deal with read pair group
    tottot++;
    if (count==3) {
      # chimeric read
      for (j=1; j <= 3; j++) {
				split(c[j], tmp);
				split(tmp[1],readname,"/");
				read[j] = readname[2];
				name[j] = tmp[1];

				# strand
				str[j] = and(tmp[2],16);
				# chromosome
				chr[j] = tmp[3];
				# position
				pos[j] = tmp[4];
				# mapq score
				m[j] = tmp[5];
				# cigar string
				cigarstr[j] = tmp[6];
				# sequence
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				# get rid of soft clipping to know correct position
				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					# can look like 15M10S20M, need to count all not just first
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					# add soft clipped bases in for proper position
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					# Mitochrondria loops around
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        mapped[j] = and(tmp[2],4) == 0;
      }

      dist[12] = abs(chr[1]-chr[2])*10000000 + abs(pos[1]-pos[2]);
      dist[23] = abs(chr[2]-chr[3])*10000000 + abs(pos[2]-pos[3]);
      dist[13] = abs(chr[1]-chr[3])*10000000 + abs(pos[1]-pos[3]);

      if (min(dist[12],min(dist[23],dist[13])) < 1000) {
				# The paired ends look like A/B...B.  Make sure we take A and B.
				if (read[1] == read[2]) {
					# take the unique one "B" for sure
					read2 = 3;
					# take the end of "A/B" that isn't close to "B"
					read1 = dist[13] > dist[23] ? 1:2;
				}
				else if (read[1] == read[3]) {
					read2 = 2;
					read1 = dist[12] > dist[23] ? 1:3;
				}
				else if (read[2] == read[3]) {
					read2 = 1;
					read1 = dist[12] > dist[13] ? 2:3;
				}
				else {
					printf("reads strange\n") > "/dev/stderr"
					exit 1
				}

				if (mapped[read1] && mapped[read2]) {
					count_norm++;
					if (less_than(str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2])) {
              print str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2],m[read1],cigarstr[read1],seq[read1],m[read2],cigarstr[read2],seq[read2],name[read1],name[read2] > fname1;
					}
					else {
              print str[read2],chr[read2],pos[read2],str[read1],chr[read1],pos[read1],m[read2],cigarstr[read2],seq[read2],m[read1],cigarstr[read1],seq[read1],name[read2],name[read1] > fname1;
					}
				}
				else {
					for (i in c) {
						print c[i] > fname3;
					}
					count_unmapped++;
				}
      }
			else {
				# chimeric read with the 3 ends > 1KB apart
				count_abnorm++;
				for (i in c) {
					print c[i] > fname2;
				}
      }
    }
    else if (count > 3) {
      # chimeric read > 3, too many to deal with
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    else if (count == 2) {
			# code here should be same as above, but it's a "normal" read
      j = 0;
      for (i in c) {
				split(c[i], tmp);
				split(tmp[1],readname,"/");
				str[j] = and(tmp[2],16);
				chr[j] = tmp[3];
				pos[j] = tmp[4];
				m[j] = tmp[5];
				cigarstr[j] = tmp[6];
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				name[j] = tmp[1];

				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        mapped[j] = and(tmp[2],4) == 0;
				j++;
      }
      if (mapped[0] && mapped[1]) {
				count_reg++;
				if (less_than(str[0],chr[0],pos[0],str[1],chr[1],pos[1])) {
					# ideally we'll get rid of printing out cigar string at some point
            print str[0],chr[0],pos[0],str[1],chr[1],pos[1],m[0],cigarstr[0],seq[0],m[1],cigarstr[1],seq[1],name[0],name[1] > fname1;
				}
				else {
            print str[1],chr[1],pos[1],str[0],chr[0],pos[0],m[1],cigarstr[1],seq[1],m[0],cigarstr[0],seq[0],name[1],name[0] > fname1;
				}
      }
      else {
				for (i in c) {
					print c[i] > fname3;
				}
				count_unmapped++;
      }
    }

  printf("%d %d %d %d %d\n", tottot, count_unmapped, count_reg, count_norm, count_abnorm) >> fname1".res.txt";
}

#!/usr/bin/awk -f
##########
#The MIT License (MIT)
#
# Copyright (c) 2015 Aiden Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
##########
# Takes a SAM file, looks for chimeric reads and normal reads.  Outputs
# only those reads mapped to chromosomes 1-24 with MAPQ > 0.
#
# Output to fname1 is of the form:
# strand1 chromosome1 position1 frag1 strand2 chromosome2 position2 frag2
#  mapq1 cigar1 seq1 mapq2 cigar2 seq2
#   where chr1 is always <= chr2
#
# Output to fname2 retains SAM format.
#
# Chimeric reads are treated as follows:
# "Normal" chimeras (with 3 reads, two with the ligation junction), where the
# third read (w/o ligation junction) is within 20Kbp of one of the ligation
# junction reads, are sent to fname1.  All other chimeras (where either there
# are more than 3 reads, or they don't contain the ligation junction, or the
# one non-ligation junction end is far from the others, are sent to fname2.
#
# awk -f chimeric_blacklist.awk -v fname1="norm_chimera" fname2="abnorm_chimera" fname3="unmapped"
# Juicer version 1.5

# returns absolute value
function abs(value)
{
  return (value<0?-value:value);
}
# returns minimum of two values
function min(value1,value2)
{
  return (value1<value2?value1:value2);
}
# examines read1 (s1,c1,p1) versus read2 and returns true if
# the first read comes before the second read
# this is so duplicates can be found after sorting even when the strand and
# chromosome are the same
function less_than(s1,c1,p1,s2,c2,p2)
{

  if (c1 < c2) return 1;
  if (c1 > c2) return 0;
  # c1 == c2
  if (s1 < s2) return 1;
  if (s1 > s2) return 0;
  # s1 == s2 && c1 == c2
  if (p1 < p2) return 1;
  if (p1 > p2) return 0;
  # all are equal, doesn't matter
  return 1;
}
BEGIN{
  OFS="\t";
  tottot = -1; # will count first non-group
}
{
  # input file is sorted by read name.  Look at read name to group
  # appropriately
  split($1,a,"/");
  if(a[1]==prev){
    # move on to next record.  look below this block for actions that occur
    # regardless.
    count++;
  }
  else {
    # deal with read pair group
    tottot++;
    if (count==3) {
      # chimeric read
      for (j=1; j <= 3; j++) {
				split(c[j], tmp);
				split(tmp[1],readname,"/");
				read[j] = readname[2];
				name[j] = tmp[1];

				# strand; Bit 16 set means reverse strand
				str[j] = and(tmp[2],16);
				# chromosome
				chr[j] = tmp[3];
				# position
				pos[j] = tmp[4];
				# mapq score
				m[j] = tmp[5];
				# cigar string
				cigarstr[j] = tmp[6];
				# sequence
				seq[j] = tmp[10];
        qual[j] = tmp[11];

				# get rid of soft clipping to know correct position
				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				# get rid of hard clipping to know correct position
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					# can look like 15M10S20M, need to count all not just first
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					# add soft clipped bases in for proper position
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
          else if (tmp[6] ~ /[0-9]+H$/) {
              where = match(tmp[6],/[0-9]+H$/);
              cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
              pos[j] = pos[j] + cigloc;
          }
					# Mitochrondria loops around
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        # blacklist - if 3rd bit set (=4) it means unmapped
        mapped[j] = and(tmp[2],4) == 0;
      }

      dist[12] = abs(chr[1]-chr[2])*10000000 + abs(pos[1]-pos[2]);
      dist[23] = abs(chr[2]-chr[3])*10000000 + abs(pos[2]-pos[3]);
      dist[13] = abs(chr[1]-chr[3])*10000000 + abs(pos[1]-pos[3]);

      if (min(dist[12],min(dist[23],dist[13])) < 1000) {
				# The paired ends look like A/B...B.  Make sure we take A and B.
				if (read[1] == read[2]) {
					# take the unique one "B" for sure
					read2 = 3;
					# take the end of "A/B" that isn't close to "B"
					read1 = dist[13] > dist[23] ? 1:2;
				}
				else if (read[1] == read[3]) {
					read2 = 2;
					read1 = dist[12] > dist[23] ? 1:3;
				}
				else if (read[2] == read[3]) {
					read2 = 1;
					read1 = dist[12] > dist[13] ? 2:3;
				}
				else {
					printf("reads strange\n") > "/dev/stderr"
					exit 1
				}

				if (mapped[read1] && mapped[read2]) {
					count_norm++;
					if (less_than(str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2])) {
              print str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2],m[read1],cigarstr[read1],seq[read1],m[read2],cigarstr[read2],seq[read2],name[read1],name[read2] > fname1;
					}
					else {
              print str[read2],chr[read2],pos[read2],str[read1],chr[read1],pos[read1],m[read2],cigarstr[read2],seq[read2],m[read1],cigarstr[read1],seq[read1],name[read2],name[read1] > fname1;
					}
				}
				else {
					for (i in c) {
						print c[i] > fname3;
					}
					count_unmapped++;
				}
      }
			else {
				# chimeric read with the 3 ends > 1KB apart
				count_abnorm++;
				for (i in c) {
					print c[i] > fname2;
				}
      }
    }
    else if (count > 3) {
      # chimeric read > 3, too many to deal with
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    else if (count == 2) {
			# code here should be same as above, but it's a "normal" read
      j = 0;
      for (i in c) {
				split(c[i], tmp);
				split(tmp[1],readname,"/");
				str[j] = and(tmp[2],16);
				chr[j] = tmp[3];
				pos[j] = tmp[4];
				m[j] = tmp[5];
				cigarstr[j] = tmp[6];
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				name[j] = tmp[1];

        # blacklist - if 3rd bit set (=4) it means unmapped
        mapped[j] = and(tmp[2],4) == 0;

				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
				j++;
      }
      if (mapped[0] && mapped[1]) {
				count_reg++;
				if (less_than(str[0],chr[0],pos[0],str[1],chr[1],pos[1])) {
					# ideally we'll get rid of printing out cigar string at some point
            print str[0],chr[0],pos[0],str[1],chr[1],pos[1],m[0],cigarstr[0],seq[0],m[1],cigarstr[1],seq[1],name[0],name[1] > fname1;
				}
				else {
            print str[1],chr[1],pos[1],str[0],chr[0],pos[0],m[1],cigarstr[1],seq[1],m[0],cigarstr[0],seq[0],name[1],name[0] > fname1;
				}
      }
      else {
				for (i in c) {
					print c[i] > fname3;
				}
				count_unmapped++;
      }
    }
    else if (count == 1) {
      # this actually shouldn't happen, but it happens with alternate aligners on occasion
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    # reset variables
    delete c;
    count=1;
    prev=a[1];
  }
  # these happen no matter what, after the above processing
  c[count] = $0;
}
END{
    # deal with read pair group
    tottot++;
    if (count==3) {
      # chimeric read
      for (j=1; j <= 3; j++) {
				split(c[j], tmp);
				split(tmp[1],readname,"/");
				read[j] = readname[2];
				name[j] = tmp[1];

				# strand
				str[j] = and(tmp[2],16);
				# chromosome
				chr[j] = tmp[3];
				# position
				pos[j] = tmp[4];
				# mapq score
				m[j] = tmp[5];
				# cigar string
				cigarstr[j] = tmp[6];
				# sequence
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				# get rid of soft clipping to know correct position
				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					# can look like 15M10S20M, need to count all not just first
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					# add soft clipped bases in for proper position
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					# Mitochrondria loops around
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        mapped[j] = and(tmp[2],4) == 0;
      }

      dist[12] = abs(chr[1]-chr[2])*10000000 + abs(pos[1]-pos[2]);
      dist[23] = abs(chr[2]-chr[3])*10000000 + abs(pos[2]-pos[3]);
      dist[13] = abs(chr[1]-chr[3])*10000000 + abs(pos[1]-pos[3]);

      if (min(dist[12],min(dist[23],dist[13])) < 1000) {
				# The paired ends look like A/B...B.  Make sure we take A and B.
				if (read[1] == read[2]) {
					# take the unique one "B" for sure
					read2 = 3;
					# take the end of "A/B" that isn't close to "B"
					read1 = dist[13] > dist[23] ? 1:2;
				}
				else if (read[1] == read[3]) {
					read2 = 2;
					read1 = dist[12] > dist[23] ? 1:3;
				}
				else if (read[2] == read[3]) {
					read2 = 1;
					read1 = dist[12] > dist[13] ? 2:3;
				}
				else {
					printf("reads strange\n") > "/dev/stderr"
					exit 1
				}

				if (mapped[read1] && mapped[read2]) {
					count_norm++;
					if (less_than(str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2])) {
              print str[read1],chr[read1],pos[read1],str[read2],chr[read2],pos[read2],m[read1],cigarstr[read1],seq[read1],m[read2],cigarstr[read2],seq[read2],name[read1],name[read2] > fname1;
					}
					else {
              print str[read2],chr[read2],pos[read2],str[read1],chr[read1],pos[read1],m[read2],cigarstr[read2],seq[read2],m[read1],cigarstr[read1],seq[read1],name[read2],name[read1] > fname1;
					}
				}
				else {
					for (i in c) {
						print c[i] > fname3;
					}
					count_unmapped++;
				}
      }
			else {
				# chimeric read with the 3 ends > 1KB apart
				count_abnorm++;
				for (i in c) {
					print c[i] > fname2;
				}
      }
    }
    else if (count > 3) {
      # chimeric read > 3, too many to deal with
      count_abnorm++;
      for (i in c) {
				print c[i] > fname2;
      }
    }
    else if (count == 2) {
			# code here should be same as above, but it's a "normal" read
      j = 0;
      for (i in c) {
				split(c[i], tmp);
				split(tmp[1],readname,"/");
				str[j] = and(tmp[2],16);
				chr[j] = tmp[3];
				pos[j] = tmp[4];
				m[j] = tmp[5];
				cigarstr[j] = tmp[6];
				seq[j] = tmp[10];
        qual[j] = tmp[11];
				name[j] = tmp[1];

				if (str[j] == 0 && tmp[6] ~/^[0-9]+S/) {
					split(tmp[6], cigar, "S");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 0 && tmp[6] ~/^[0-9]+H/) {
					split(tmp[6], cigar, "H");
					pos[j] = pos[j] - cigar[1];
					if (pos[j] <= 0) {
						pos[j] = 1;
					}
				}
				else if (str[j] == 16) {
					# count Ms,Ds,Ns,Xs,=s for sequence length
					seqlength=0;
					currstr=tmp[6];
					where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					while (where>0) {
						seqlength+=substr(currstr,where,RLENGTH-1)+0;
						currstr=substr(currstr,where+RLENGTH);
						where=match(currstr, /[0-9]+[M|D|N|X|=]/);
					}
					pos[j] = pos[j] + seqlength - 1;
					if (tmp[6] ~ /[0-9]+S$/) {
						where = match(tmp[6],/[0-9]+S$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (tmp[6] ~ /[0-9]+H$/) {
						where = match(tmp[6],/[0-9]+H$/);
						cigloc = substr(tmp[6],where,RLENGTH-1) + 0;
						pos[j] = pos[j] + cigloc;
					}
					if (chr[j] ~ /MT/ && pos[j] >= 16569) {
						pos[j] = pos[j] - 16569;
					}
				}
        mapped[j] = and(tmp[2],4) == 0;
				j++;
      }
      if (mapped[0] && mapped[1]) {
				count_reg++;
				if (less_than(str[0],chr[0],pos[0],str[1],chr[1],pos[1])) {
					# ideally we'll get rid of printing out cigar string at some point
            print str[0],chr[0],pos[0],str[1],chr[1],pos[1],m[0],cigarstr[0],seq[0],m[1],cigarstr[1],seq[1],name[0],name[1] > fname1;
				}
				else {
            print str[1],chr[1],pos[1],str[0],chr[0],pos[0],m[1],cigarstr[1],seq[1],m[0],cigarstr[0],seq[0],name[1],name[0] > fname1;
				}
      }
      else {
				for (i in c) {
					print c[i] > fname3;
				}
				count_unmapped++;
      }
    }

  printf("%d %d %d %d %d\n", tottot, count_unmapped, count_reg, count_norm, count_abnorm) >> fname1".res.txt";
}


Move content processing to 2018.

Move content processing to 2018.

#!/usr/bin/awk -f
# tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back],publisher:info[poller] *.tdb

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}
#$3>date
{
  # add a loop to add line only if either status is (wanted or testing) or ending year is gt or eq to contract year
  # increased to 2018 July 1 ish 2018
  current_year = 2018
  # end_year is the AU year, or the second half of a range, ie 2014 in 2013-2014
  end_year = 0
  incontract = 0
  # test_year is the contract year, or the back year if there is one.
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  # Is the AU year >= the contract year or the back year. And then decide what to do with the current year.
  if ($6 == "wanted" || $6 == "testing") {
      incontract = 1;
  } else if ((end_year >= test_year) && ((end_year < current_year) || (end_year == current_year && incl_cur == 1))) {
      incontract = 1;
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      if ($7 == "") {
        k[pn] = ".."
      } else {
        k[pn] = $7
      }
      d[pn] = $4
      r1[pn] = $5
      r2[pn] = $8
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "crawling"
  s[8] = "deepCrawl"
  s[9] = "frozen"
  s[10] = "ingNotReady"
  s[11] = "finished"
  s[12] = "down"
  s[13] = "superseded"
  s[14] = "zapped"
  s[15] = "doNotProcess"
  sn = 16

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "craw"
  sc[8] = "deep"
  sc[9] = "froz"
  sc[10] = "ingN"
  sc[11] = "fini"
  sc[12] = "down"
  sc[13] = "supe"
  sc[14] = "zapp"
  sc[15] = "doNP"
  scn = 16

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tP\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, poller, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r1[i], r2[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
        printf "\t.."
      } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tP\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}

#!/usr/bin/awk -f
# tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back],publisher:info[poller] *.tdb

BEGIN {
  FS="\t"
  pn = 0
    #str = "date +%m-%d-%Y";
    #str | getline date;
    #close str;
}
#$3>date
{
  # add a loop to add line only if either status is (wanted or testing) or ending year is gt or eq to contract year
  # increased to 2016 July 1 2016.
  current_year = 2017
  # end_year is the AU year, or the second half of a range, ie 2014 in 2013-2014
  end_year = 0
  incontract = 0
  # test_year is the contract year, or the back year if there is one.
  test_year = ""
  if ($7 == "") {
    test_year = $3
  } else {
    test_year = $7
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  #printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$7,end_year,test_year
  # Is the AU year >= the contract year or the back year. And then decide what to do with the current year.
  if ($6 == "wanted" || $6 == "testing") {
      incontract = 1;
  } else if ((end_year >= test_year) && ((end_year < current_year) || (end_year == current_year && incl_cur == 1))) {
      incontract = 1;
  }

  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      if ($7 == "") {
        k[pn] = ".."
      } else {
        k[pn] = $7
      }
      d[pn] = $4
      r1[pn] = $5
      r2[pn] = $8
      pn++
    }
    b[$1,lp2,$4]++
    c[$1,lp2,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "testing"
  s[5] = "notReady"
  s[6] = "ready"
  s[7] = "crawling"
  s[8] = "deepCrawl"
  s[9] = "frozen"
  s[10] = "ingNotReady"
  s[11] = "finished"
  s[12] = "down"
  s[13] = "superseded"
  s[14] = "zapped"
  s[15] = "doNotProcess"
  sn = 16

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "test"
  sc[5] = "notR"
  sc[6] = "read"
  sc[7] = "craw"
  sc[8] = "deep"
  sc[9] = "froz"
  sc[10] = "ingN"
  sc[11] = "fini"
  sc[12] = "down"
  sc[13] = "supe"
  sc[14] = "zapp"
  sc[15] = "doNP"
  scn = 16

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tP\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, poller, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r1[i], r2[i], b[p[i],n[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],d[i],s[j]] == 0) {
        printf "\t.."
      } else {
        printf "\t%d", c[p[i],n[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tP\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
  #printf "%s", date
  #printf "\n"
}


Edit awk path

Edit awk path

#!/usr/bin/awk -f

# Caller must set nbscript and path values on the command line.
# Example: compute_global_stats.awk -v path=2001-10-10@14:55:23/ -v nbscript=2 2001-10-10@14:55:23/stat_client0.html

BEGIN {
  printf "Computing global stats for "nbscript" scripts in "path" ...";
  outputFile = path"perf.html";
  footer = 0;
  headerRow = 0;
  for (i = 1 ; i < nbscript ; i++)
    scriptName[i] = path"stat_client"i".html";
}
{
  # Read each line from all files to be synchronized between them
  line[0] = $0;
  for (i = 1 ; i < nbscript ; i++)
    getline line[i] < scriptName[i];

  # Compute the global results for the statistics of each ramp
  if (headerRow == 1)
    {
      if ($0 ~ "

Total
")
	{ # This is the total line, compute the percentages and print the table
	  totalCount = 0;
	  totalErrors = 0;
	  totalTime = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "
");
	      gsub(/
||<\/B>/, "", splited[3]); # Get rid of the '
' and
	      totalCount += splited[3];
	      gsub(/
||<\/B>/, "", splited[4]); # Get rid of the '
' and
	      totalErrors += splited[4];
	    }
	  if (totalCount == 0)
	    totalCount = 1000000000000000000000000; # make totalCount infinite so that division result will give 0.
	  for (i = 0 ; (i < stateNb) && (stateNb != 0); i++)
	    {
	      if (count[i] != 0)
		{
		  printf "%s

%.2f %%

%d

%d

%d ms

%d ms

%.0f ms
\n", stateName[i], 100*count[i]/totalCount, count[i], errors[i], minTime[i], maxTime[i], avgTime[i]/stateNb >> outputFile;
		  totalTime += avgTime[i];
		}
	    }
	  # Display the total
	  if (stateNb == 0)
	    stateNb = 1000000000000000000000000; # make stateNb infinite so that division result will give 0.
	  printf "

Total

100 %

%d

%d

-

-

%.0f ms
", totalCount, totalErrors, totalTime/totalCount >> outputFile;
	}
      else if ($0 ~ "

Average throughput
")
	{ # Average throughput
	  throughput = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /|<\/B>/);
	      gsub(/ req\/s/, "", splited[4]); # Get rid of the '< req/s>'
	      throughput += splited[4];
	    }
	  print "

Average throughput

"throughput" req/s
" >> outputFile;
	}
      else if ($0 ~ "

Completed sessions
")
	{ # Completed sessions
	  sessions = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /
/);
	      gsub(/<\/div>/, "", splited[3]); # Get rid of the '
'
	      sessions += splited[3];
	    }
	  print "

Completed sessions

"sessions"
" >> outputFile;
	}
      else if ($0 ~ "

Total time
")
	{ # Total time
	  time = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /
/);
	      gsub(/ seconds<\/div>/, "", splited[3]); # Get rid of the '
'
	      time += splited[3];
	    }
	  print "

Total time

"time" seconds
" >> outputFile;
	}
      else if ($0 ~ "

Average session time
")
	{ # Average session time is the last stat of the table, we close the table here.
	  if (time == 0)
	    print "

Average session time

0 second
" >> outputFile;
	  else
	    print "

Average session time

"time/sessions" seconds
" >> outputFile;
	  print "

" >> outputFile;
	  headerRow = 0;
	}
     else if ($0 ~ "align=right>")
	{ # We have to process a stat line, let's go !
	  count[stateNb]   = 0;
	  errors[stateNb]  = 0;
	  minTime[stateNb] = 1000000000000000;
	  maxTime[stateNb] = 0;
	  avgTime[stateNb] = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "

");
	      if (i == 0)
		stateName[stateNb] = splited[1];
	      else if (splited[1] != stateName[stateNb])
		print "Error line "NR" in "path"stat_client"i".html: Bad state '"splited[1]"' does not match '"stateName[stateNb]"'";
	      # Skip percentage in splited[2], we'll have to compute that later
	      count[stateNb]    += splited[3];
	      gsub(/|<\/B>/, "", splited[4]); # Get rid of the '' and
	      errors[stateNb]   += splited[4];
	      sub(/ ms/, "", splited[5]); # Get rid of the ' ms'
	      if (minTime[stateNb] > splited[5])
		minTime[stateNb] = splited[5];
	      sub(/ ms/, "", splited[6]); # Get rid of the ' ms'
	      if (maxTime[stateNb] < splited[6])
		maxTime[stateNb] = splited[6];
	      sub(/ ms<\/div>/, "", splited[7]); # Get rid of the ' ms
'
	      avgTime[stateNb]  += splited[7];
	    }
	  stateNb++;
	}
    }

  # Print the headers for ramps
  if ($0 ~ /stat\"><\/A>|statistics<\/h3>
/)
    {
      print $0 >> outputFile;
      testTiming = 0;
    }

  # Print a summary of 'Test timing information' for all clients
  if ($0 ~ "")
    testTiming = 1;

  if (testTiming == 1)
    {
      if ($0 ~ "")
	{
	  printf $0 >> outputFile;
	  for (i = 1 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "");
	      printf ""splited[3] >> outputFile;
	    }
	  print "" >> outputFile;
	}
      else
	print $0 >> outputFile;
      if ($0 ~ "<TABLE")
	{
	  printf "

PhaseMain client" >> outputFile;
	  for (i = 1 ; i < nbscript ; i++)
	    printf "Remote client "i >> outputFile;
	  print "
" >> outputFile;
	}
    }

  if ($0 ~ /State name/)
    { # Table header recognized !
      headerRow = 1;
      stateNb = 0;
      print "
" >> outputFile;
      print $0 >> outputFile;
    }

  # Recopy the graph part which is common to every stat page
  if ($0 ~ /"cpu_graph"/)
    footer = 1;

  if (footer == 1)
    print $0 >> outputFile;

}
END {
  print " Done.";
}

#!/bin/awk -f

# Caller must set nbscript and path values on the command line.
# Example: compute_global_stats.awk -v path=2001-10-10@14:55:23/ -v nbscript=2 2001-10-10@14:55:23/stat_client0.html

BEGIN {
  printf "Computing global stats for "nbscript" scripts in "path" ...";
  outputFile = path"perf.html";
  footer = 0;
  headerRow = 0;
  for (i = 1 ; i < nbscript ; i++)
    scriptName[i] = path"stat_client"i".html";
}
{
  # Read each line from all files to be synchronized between them
  line[0] = $0;
  for (i = 1 ; i < nbscript ; i++)
    getline line[i] < scriptName[i];

  # Compute the global results for the statistics of each ramp
  if (headerRow == 1)
    {
      if ($0 ~ "

Total
")
	{ # This is the total line, compute the percentages and print the table
	  totalCount = 0;
	  totalErrors = 0;
	  totalTime = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "
");
	      gsub(/
||<\/B>/, "", splited[3]); # Get rid of the '
' and
	      totalCount += splited[3];
	      gsub(/
||<\/B>/, "", splited[4]); # Get rid of the '
' and
	      totalErrors += splited[4];
	    }
	  if (totalCount == 0)
	    totalCount = 1000000000000000000000000; # make totalCount infinite so that division result will give 0.
	  for (i = 0 ; (i < stateNb) && (stateNb != 0); i++)
	    {
	      if (count[i] != 0)
		{
		  printf "%s

%.2f %%

%d

%d

%d ms

%d ms

%.0f ms
\n", stateName[i], 100*count[i]/totalCount, count[i], errors[i], minTime[i], maxTime[i], avgTime[i]/stateNb >> outputFile;
		  totalTime += avgTime[i];
		}
	    }
	  # Display the total
	  if (stateNb == 0)
	    stateNb = 1000000000000000000000000; # make stateNb infinite so that division result will give 0.
	  printf "

Total

100 %

%d

%d

-

-

%.0f ms
", totalCount, totalErrors, totalTime/totalCount >> outputFile;
	}
      else if ($0 ~ "

Average throughput
")
	{ # Average throughput
	  throughput = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /|<\/B>/);
	      gsub(/ req\/s/, "", splited[4]); # Get rid of the '< req/s>'
	      throughput += splited[4];
	    }
	  print "

Average throughput

"throughput" req/s
" >> outputFile;
	}
      else if ($0 ~ "

Completed sessions
")
	{ # Completed sessions
	  sessions = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /
/);
	      gsub(/<\/div>/, "", splited[3]); # Get rid of the '
'
	      sessions += splited[3];
	    }
	  print "

Completed sessions

"sessions"
" >> outputFile;
	}
      else if ($0 ~ "

Total time
")
	{ # Total time
	  time = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, /
/);
	      gsub(/ seconds<\/div>/, "", splited[3]); # Get rid of the '
'
	      time += splited[3];
	    }
	  print "

Total time

"time" seconds
" >> outputFile;
	}
      else if ($0 ~ "

Average session time
")
	{ # Average session time is the last stat of the table, we close the table here.
	  if (time == 0)
	    print "

Average session time

0 second
" >> outputFile;
	  else
	    print "

Average session time

"time/sessions" seconds
" >> outputFile;
	  print "

" >> outputFile;
	  headerRow = 0;
	}
     else if ($0 ~ "align=right>")
	{ # We have to process a stat line, let's go !
	  count[stateNb]   = 0;
	  errors[stateNb]  = 0;
	  minTime[stateNb] = 1000000000000000;
	  maxTime[stateNb] = 0;
	  avgTime[stateNb] = 0;
	  for (i = 0 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "

");
	      if (i == 0)
		stateName[stateNb] = splited[1];
	      else if (splited[1] != stateName[stateNb])
		print "Error line "NR" in "path"stat_client"i".html: Bad state '"splited[1]"' does not match '"stateName[stateNb]"'";
	      # Skip percentage in splited[2], we'll have to compute that later
	      count[stateNb]    += splited[3];
	      gsub(/|<\/B>/, "", splited[4]); # Get rid of the '' and
	      errors[stateNb]   += splited[4];
	      sub(/ ms/, "", splited[5]); # Get rid of the ' ms'
	      if (minTime[stateNb] > splited[5])
		minTime[stateNb] = splited[5];
	      sub(/ ms/, "", splited[6]); # Get rid of the ' ms'
	      if (maxTime[stateNb] < splited[6])
		maxTime[stateNb] = splited[6];
	      sub(/ ms<\/div>/, "", splited[7]); # Get rid of the ' ms
'
	      avgTime[stateNb]  += splited[7];
	    }
	  stateNb++;
	}
    }

  # Print the headers for ramps
  if ($0 ~ /stat\"><\/A>|statistics<\/h3>
/)
    {
      print $0 >> outputFile;
      testTiming = 0;
    }

  # Print a summary of 'Test timing information' for all clients
  if ($0 ~ "")
    testTiming = 1;

  if (testTiming == 1)
    {
      if ($0 ~ "")
	{
	  printf $0 >> outputFile;
	  for (i = 1 ; i < nbscript ; i++)
	    {
	      split(line[i], splited, "");
	      printf ""splited[3] >> outputFile;
	    }
	  print "" >> outputFile;
	}
      else
	print $0 >> outputFile;
      if ($0 ~ "<TABLE")
	{
	  printf "

PhaseMain client" >> outputFile;
	  for (i = 1 ; i < nbscript ; i++)
	    printf "Remote client "i >> outputFile;
	  print "
" >> outputFile;
	}
    }

  if ($0 ~ /State name/)
    { # Table header recognized !
      headerRow = 1;
      stateNb = 0;
      print "
" >> outputFile;
      print $0 >> outputFile;
    }

  # Recopy the graph part which is common to every stat page
  if ($0 ~ /"cpu_graph"/)
    footer = 1;

  if (footer == 1)
    print $0 >> outputFile;

}
END {
  print " Done.";
}


Handle corrupted dmesg output better - sometimes, we get junk left around from the last boot, and we have more than one CPU: line. Sometimes (such as at Wisc.), the first instance contains garbage. So, instead of exiting after the first CPU: line, use a different method of handling unknown CPU speeds, modeled after chipset.awk .


Revise cppw2cpp

Revise cppw2cpp

#!/usr/bin/awk -f

# ----------------------------------------------------------------------
# cppw2cpp.awk
# ----------------------------------------------------------------------

# Count the leading spaces in a line
function count_leading_spaces(line,  leading_spaces) {
  leading_spaces = line
  sub(/[^ \t].*/, "", leading_spaces)
  gsub(/\t/, "  ", leading_spaces)
  return length(leading_spaces)
}

# Get the indent count offset at the current line
function get_indent_count_offset(line,  indent_count_for_line, offset) {
  indent_count_for_line = count_leading_spaces(line)
  if (indent_count_for_line == 0) {
    # If the line started at the left-hand margin, then keep it there in the output
    offset = 0
  }
  else {
    offset = indent_count_for_line - indent_count_at_begin
  }
  return offset
}

# Get the indent count for the current line
function get_indent_count(line,  indent_count_offset, indent_count) {
  indent_count_offset = get_indent_count_offset(line)
  indent_count = desired_indent_count + indent_count_offset
  if (indent_count < 0)
    indent_count = 0
  return indent_count
}

# Print indent spacing
function print_indent_spacing(indent_count,  i) {
  for (i = 1; i <= indent_count; ++i)
    printf(" ")
}

# Print the line content, i.e., everything but leading whitespace
function print_content(line,  content) {
  content = line
  sub(/^[ \t]*/, "", content)
  print content
}

# Print line at current indentation
function print_with_indent(line,  indent_count) {
  indent_count = get_indent_count(line)
  print_indent_spacing(indent_count)
  print_content(line)
}

# Are we in the current cpp file?
function in_current_cppfile() {
  return current_cppfile == "" || current_cppfile == cppfile
}

# Print a function argument
function print_function_arg(arg,  printed_arg) {
  printed_arg = arg
  sub(/ *\/\/.*$/, "", printed_arg)
  printf(printed_arg)
}

# Print a class function prototype
function print_class_function_prototype() {
  print_indent_spacing(desired_indent_count)
  print return_type " " class " ::"
  print_indent_spacing(desired_indent_count + 2)
  printf name
  if (num_args == 0)
    printf("(void)\n")
  else if (num_args == 1) {
    printf("(")
    print_function_arg(args[1])
    print ")"
  }
  else {
    printf("(\n")
    for (i = 1; i <= num_args; ++i) {
      print_indent_spacing(desired_indent_count + 6)
      printf(args[i])
      if (i < num_args)
        printf(",")
      print ""
    }
    print_indent_spacing(desired_indent_count + 2)
    print ")"
  }
  print_with_indent("{")
}

# Print a non-class function prototype
function print_non_class_function_prototype() {
  print_indent_spacing(desired_indent_count)
  if (static)
    printf("static ")
  printf("%s %s", return_type, name)
  if (num_args == 0)
    print "(void) {"
  else if (num_args == 1) {
    printf "("
    print_function_arg(args[1])
    print ") {"
  }
  else {
    printf("(\n")
    for (i = 1; i <= num_args; ++i) {
      print_indent_spacing(desired_indent_count + 4)
      printf(args[i])
      if (i < num_args)
        printf(",")
      print ""
    }
    print_with_indent ") {"
  }
}

BEGIN {
  cpp = 0
  fn = 0
  begin = 0
  desired_indent_count = 0
  indent_count_at_begin = 0
}

$1 == "@CPPFILE" {
  current_cppfile = $2
  gsub(/"/, "", current_cppfile)
}

$1 == "@CPP" {
  cpp = 1
}

$1 == "@END" && fn == 1 && in_current_cppfile() {
  print_with_indent("}")
}

$1 == "@END" {
  begin = 0
  cpp = 0
  fn = 0
}

cpp == 1 && begin == 1 && in_current_cppfile() {
  print
}

fn == 1 && begin == 1 && in_current_cppfile() {
  print_with_indent($0)
}

$1 == "@BEGIN" {
  indent_count_at_begin = count_leading_spaces($0)
}

$1 == "@BEGIN" && fn == 1 && in_current_cppfile() {
  print ""
  if (class != "")
    print_class_function_prototype()
  else
    print_non_class_function_prototype()
}

$1 == "@BEGIN" {
  begin = 1
}

$1 == "namespace" && begin == 0 {
  if (class != "") {
    print "cppw2cpp: namespace inside class at line " NR > "/dev/stderr"
    exit 1
  }
  print ""
  print_with_indent($0)
  desired_indent_count += 2
}

$1 == "class" && begin == 0 {
  if (class == "")
    class = $2
  else
    class = class "::" $2
}

$1 ~ "}" && begin == 0 && class != "" {
  if ($1 != "};") {
    print "cppw2cpp: missing semicolon at end of class in line " NR > "/dev/stderr"
    exit 1
  }
  if (class ~ /::/)
    sub(/::[^:]*$/, "", class)
  else
    class = ""
}

$1 == "}" && begin == 0 && class == "" {
  print ""
  desired_indent_count -= 2
  print_with_indent($0)
}

$1 == "@FUNCTION" {
  fn = 1
  static = 0
  const = 0
  num_args = 0
  return_type = "void"
}

$1 == "@NAME" { name = $2 }

$1 == "@RETURN" { return_type = $2  }

$1 == "@ARGUMENT" {
  arg = $0
  sub(/^.*@ARGUMENT */, "", arg)
  args[++num_args] = arg
}

$1 == "@STATIC" { static = 1 }

$1 == "@CONST" { const = 1 }

#!/usr/bin/awk -f

# ----------------------------------------------------------------------
# cppw2cpp.awk
# ----------------------------------------------------------------------

# Count the leading spaces in a line
function count_leading_spaces(line,  leading_spaces) {
  leading_spaces = line
  sub(/[^ \t].*/, "", leading_spaces)
  gsub(/\t/, "  ", leading_spaces)
  return length(leading_spaces)
}

# Get the indent count offset at the current line
function get_indent_count_offset(line,  indent_count_for_line, offset) {
  indent_count_for_line = count_leading_spaces(line)
  if (indent_count_for_line == 0) {
    # If the line started at the left-hand margin, then keep it there in the output
    offset = 0
  }
  else {
    offset = indent_count_for_line - indent_count_at_begin
  }
  return offset
}

# Get the indent count for the current line
function get_indent_count(line,  indent_count_offset, indent_count) {
  indent_count_offset = get_indent_count_offset(line)
  indent_count = desired_indent_count + indent_count_offset
  if (indent_count < 0)
    indent_count = 0
  return indent_count
}

# Print indent spacing
function print_indent_spacing(indent_count,  i) {
  for (i = 1; i <= indent_count; ++i)
    printf(" ")
}

# Print the line content, i.e., everything but leading whitespace
function print_content(line,  content) {
  content = line
  sub(/^[ \t]*/, "", content)
  print content
}

# Print line at current indentation
function print_with_indent(line,  indent_count) {
  indent_count = get_indent_count(line)
  print_indent_spacing(indent_count)
  print_content(line)
}

# Are we in the current cpp file?
function in_current_cppfile() {
  return current_cppfile == "" || current_cppfile == cppfile
}

# Print a class function prototype
function print_class_function_prototype() {
  print_with_indent("CLASS_FUNCTION_PROTOTYPE")
  # TODO
}

# Print a function argument
function print_function_arg(arg,  printed_arg) {
  printed_arg = arg
  sub(/ *\/\/.*$/, "", printed_arg)
  printf(printed_arg)
}

# Print a non-class function prototype
function print_non_class_function_prototype() {
  print_indent_spacing(desired_indent_count)
  if (static)
    printf("static ")
  printf("%s %s", return_type, name)
  if (num_args == 0)
    print "(void) {"
  else if (num_args == 1) {
    printf "("
    print_function_arg(args[1])
    print ") {"
  }
  else {
    printf("(\n")
    for (i = 1; i <= num_args; ++i) {
      print_indent_spacing(desired_indent_count + 4)
      printf(args[i])
      if (i < num_args)
        printf(",")
      print ""
    }
    print_with_indent ") {"
  }
}

BEGIN {
  cpp = 0
  fn = 0
  begin = 0
  desired_indent_count = 0
  indent_count_at_begin = 0
}

$1 == "@CPPFILE" {
  current_cppfile = $2
  gsub(/"/, "", current_cppfile)
}

$1 == "@CPP" {
  cpp = 1
}

$1 == "@END" && fn == 1 && in_current_cppfile() {
  print_with_indent("}")
}

$1 == "@END" {
  begin = 0
  cpp = 0
  fn = 0
}

cpp == 1 && begin == 1 && in_current_cppfile() {
  print
}

fn == 1 && begin == 1 && in_current_cppfile() {
  print_with_indent($0)
}

$1 == "@BEGIN" {
  indent_count_at_begin = count_leading_spaces($0)
}

$1 == "@BEGIN" && fn == 1 && in_current_cppfile() {
  print ""
  if (class != "")
    print_class_function_prototype()
  else
    print_non_class_function_prototype()
}

$1 == "@BEGIN" {
  begin = 1
}

$1 == "namespace" && begin == 0 {
  if (class != "") {
    print "cppw2cpp: namespace inside class at line " NR > "/dev/stderr"
    exit 1
  }
  print ""
  print_with_indent($0)
  desired_indent_count += 2
}

$1 == "class" && begin == 0 {
  if (class == "")
    class = $2
  else
    class = class "::" $2
}

$1 ~ "}" && begin == 0 && class != "" {
  if ($1 != "};") {
    print "cppw2cpp: missing semicolon at end of class in line " NR > "/dev/stderr"
    exit 1
  }
  if (class ~ /::/)
    sub(/::[^:]*$/, "", class)
  else
    class = ""
}

$1 == "}" && begin == 0 && class == "" {
  print ""
  desired_indent_count -= 2
  print_with_indent($0)
}

$1 == "@FUNCTION" {
  fn = 1
  static = 0
  const = 0
  num_args = 0
  return_type = "void"
}

$1 == "@NAME" { name = $2 }

$1 == "@RETURN" { return_type = $2  }

$1 == "@ARGUMENT" {
  arg = $0
  sub(/^.*@ARGUMENT */, "", arg)
  args[++num_args] = arg
}

$1 == "@STATIC" { static = 1 }

$1 == "@CONST" { const = 1 }