linkparser v0.3.1

linkparser v0.3.1

* Re-organize the code in a cleaner manner, which seems better
architectured.
* Get dependence file(.d, or .D) from cmd file(.cmd), replacing the
old algorithm.

BEGIN {
	version = "0.3.1"
	printf("link-parser, v%s\n", version)

	true = 1
	false = 0
	g_srcroot = ""
	g_instroot = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	count = 0

	if (arg ~ /ut/) {
		do_test();
		SKIP_END = true
		exit 0
	}

	if (arg ~ /pretend/) {
		pretend = true
	}

	if (arg ~ /dest=/) {
		destdir = gensub(/^.+dest=([^ \t;,]+)/, "\\1", "g", arg)
		if (system("test -d " destdir) != 0) {
			print "Directory [" destdir "] doesn't exist!"
			SKIP_END = true
			exit 0
		}
	}

	# Get g_srcroot
	getline line
	get_g_srcroot(line)
	printf("Parsing and dumping, this may take for a while...\n")
}

/^[^-]/ {
	# A link is a file name suffixed with '.o', '.lib' or '.obj' in a line.
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	# Parse .cmd files
	cmd_file = link".cmd"
	parse_cmd(cmd_file)
}

function get_g_srcroot(line) {
	if (g_srcroot)
		return

	patt = "\\/sw\\/vobs.+"
	if (line !~ patt)
		return

	gsub(patt, "", line)
	patt = ".+\\/"
	gsub(patt, "", line)
	g_srcroot = line
	"date +%Y%m%d-%H%M%S" | getline date
	g_instroot = g_srcroot"-"date

	if (destdir)
		g_instroot = destdir"/"g_instroot
}

function parse_cmd(cmd) {
	while ((getline line < cmd) > 0) {
		# Get dependence file, in the argument of option '-Wp,-MD'
		patt = "cd (.+);.+-MD,([^ \t]+)\\.d.+"
		if (line ~ patt) {
			dep = gensub(patt, "\\1/\\2.D", "g", line)
			# Parsing dependence file
			parse_dep(dep)
		} else if (pretend) {
			print RED line NC
		}

		# Get source file, suffixed with '.c', '.cpp', or '.cc'
		# ...

		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		else # relative path
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		# path transform
		#patt = "vobs\\/esam\\/objects\\/([^\\/]+\\/){2,2}sources\\/"
		patt = "vobs\\/esam\\/objects\\/.+\\/sources\\/(flat|src)"
		gsub(patt, "vobs/dsl/sw/flat", source)

		patt = "vobs\\/esam\\/objects\\/.+\\/sources\\/itf"
		gsub(patt, "vobs/dsl/sw/itf", source)

		patt = "\\/esam\\/.+\\/dummy\\.c"
		if (source ~ patt)
			continue;

		rel_install(source, g_srcroot, g_instroot)
	}
	close(cmd)
}

function parse_dep(dep) {
	# Get path prefix
	if ((getline < dep) < 0)
		return
	cur_path = $1
	gsub(/^#/, "", cur_path)

	while ((getline < dep) > 0) {
		gsub(/.+:/, " ")
		for (i = 1; i <= NF; i++) {
			#if ($i !~ /\.(h|hpp)$/)
			#	continue
			if ($i == "\\") continue

			header = cur_path "/" $i
			if (!buildroot_host_path) {
				patt = "(.+\\/esam\\/.+\\/output\\/host)\\/.+$"
				if (header ~ patt) {
					buildroot_host_path = gensub(patt, "\\1", "g", header)

					patt = ".+\\/"g_srcroot"\\/sw\\/vobs"
					gsub(patt, g_instroot"/sw/vobs", buildroot_host_path)
				}
			}

			rel_install(header, g_srcroot, g_instroot)
		}
	}
	close(dep)
}

END {
	if (SKIP_END)
		exit 0
	print "Total dumped files: " count

	if (buildroot_host_path && arg !~ /pretend/)
		system("mv " buildroot_host_path " " g_instroot"/sw/")

	if (arg ~ /tar/)
		make_tarball(arg)
}

function rel_install(source, rel_root, new_root) {
	count++

	dir = dirname(source)
	base = basename(source)
	patt = "^.+\\/"rel_root"\\/(sw.+)"
	rel_path = gensub(patt, "\\1", "g", dir)

	if (pretend) {
		print source > rel_root".source.list"
		print rel_path"/"base > rel_root".relative.list"
		return 0
	}

	system("mkdir -p " new_root "/" rel_path)

	cmdline = "if test -e " new_root "/" rel_path "/" base \
		";then echo yes;else echo no;fi"
	cmdline |getline ret
	close(cmdline)
	if (ret == "yes") {
		count--
		return 0
	}

	system("cp -fL " source " " new_root "/" rel_path)
}

function make_tarball(flag) {
	tarball = g_instroot ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)

	if (pretend || SKIP_END)
		exit 0

	system("tar zcf " tarball " " g_instroot)
	if (flag ~ /purge/)
		system("rm -r "g_instroot)
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}

BEGIN {
	version = "0.2.0"
	printf("link-parser, v%s\n", version)

	true = 1
	false = 0
	g_pat_srccommon = "\\/sw\\/vobs.+"
	g_sroot = ""
	inst_root = ""
	tar_root = ""

	# Colors
	RED="\033[0;31m"
	GRN="\033[0;32m"
	BLU="\033[0;34m"
	NC="\033[0m"
	BOL="\033[1m"

	count = 0

	if (arg ~ /ut/) {
		do_test();
		SKIP_END = true
		exit 0
	}

	if (arg ~ /pretend/) {
		pretend = true
	}

	if (arg ~ /dest=/) {
		destdir = gensub(/^.+dest=([^ \t;,]+)/, "\\1", "g", arg)
		if (system("test -d " destdir) != 0) {
			print "Directory [" destdir "] doesn't exist!"
			SKIP_END = true
			exit 0
		}
	}

	# Skip head lines
	getline
}

/^[^-]/ {
	link = $0
	# Remove the ending ' \'
	gsub(/[ \t]+\\$/, "", link)

	if (g_sroot == "") {
		printf("Grabbing sources and headers ...\n")
		g_sroot = get_sroot(link)
		"date +%Y%m%d-%H%M%S" | getline date
		inst_root = g_sroot"-"date
	}

	# ---------
	# Sources
	# ---------

	cmd_file = link".cmd"
	while ((getline line < cmd_file) > 0) {
		# abosolute path
		if (line ~ /[ \t]+-c[ \t]+\/.+$/)
			source = gensub(/^.+[ \t]+-c[ \t]+(.+)$/, "\\1", "g", line)
		else # relative path
			source = gensub(/^cd (.+);.+-c (.+)/, "\\1/\\2", "g", line)

		# path transform
		#patt = "vobs\\/esam\\/objects\\/([^\\/]+\\/){2,2}sources\\/"
		patt = "vobs\\/esam\\/objects\\/.+\\/sources\\/(flat|src)"
		gsub(patt, "vobs/dsl/sw/flat", source)

		patt = "vobs\\/esam\\/objects\\/.+\\/sources\\/itf"
		gsub(patt, "vobs/dsl/sw/itf", source)

		patt = "\\/esam\\/.+\\/dummy\\.c"
		if (source ~ patt)
			continue;

		rel_install(source, g_sroot, inst_root)
	}
	close(cmd_file)

	# ---------
	# Headers
	# ---------

	gsub(/\.(obj|o|lib)$/, "", link)
	dep_file = link ".D"

	if ((getline < dep_file) < 0)
		next

	# Get path prefix
	cur_path = $1
	gsub(/^#/, "", cur_path)

	while ((getline < dep_file) > 0) {
		gsub(/.+:/, " ")
		for (i = 1; i <= NF; i++) {
			#if ($i !~ /\.(h|hpp)$/)
			#	continue
			if ($i == "\\") continue

			header = cur_path "/" $i
			if (!buildroot_host_path) {
				patt = "(.+\\/esam\\/.+\\/output\\/host)\\/.+$"
				if (header ~ patt) {
					buildroot_host_path = gensub(patt, "\\1", "g", header)

					patt = ".+\\/"g_sroot"\\/sw\\/vobs"
					gsub(patt, inst_root"/sw/vobs", buildroot_host_path)
				}
			}

			rel_install(header, g_sroot, inst_root)
		}
	}
	close(dep_file)
}

END {
	if (SKIP_END)
		exit 0
	print "Total dumped files: " count

	if (buildroot_host_path && arg !~ /pretend/)
		system("mv " buildroot_host_path " " inst_root)

	if (arg ~ /tar/)
		make_tarball(arg)
}

function rel_install(source, rel_root, new_root) {
	count++

	dir = dirname(source)
	base = basename(source)
	patt = "^.+\\/"rel_root"\\/(sw.+)"
	rel_path = gensub(patt, "\\1", "g", dir)

	if (destdir)
		new_root = destdir "/" new_root

	if (pretend) {
		print source > rel_root".source.list"
		print rel_path"/"base > rel_root".relative.list"
		return 0
	}

	system("mkdir -p " new_root "/" rel_path)
	system("cp -fL " source " " new_root "/" rel_path)
}

function make_tarball(flag) {
	tarball = inst_root ".tar.gz"
	printf("Generating source tarball %s%s%s%s ...\n", RED, BOL, tarball, NC)

	if (pretend || SKIP_END)
		exit 0

	"pwd" |getline oldpwd
	if (destdir)
		system("cd " destdir)

	system("tar zcf " tarball " " inst_root)
	if (flag ~ /purge/)
		system("rm -r "inst_root)

	if (destdir)
		system("cd " oldpwd)
}

function get_sroot(line) {
	if (line !~ g_pat_srccommon)
		return ""

	gsub(g_pat_srccommon, "", line);
	gsub(/.+\//, "", line);
	return line;
}

# dirname
function dirname(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)

	# Remove basename
	gsub(/[^\/]+$/, "", path)
	if (path == "") return "."

	# Remove the ending '/' again
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	return path
}

# basename
function basename(path) {
	if (path == "")
		return ""

	# Remove the ending '/'
	gsub(/\/+$/, "", path)
	if (path == "")
		return "/"

	path = gensub(/^.+\/+(.+)$/, "\\1", "g", path)

	return path;
}

###### Test #########
function do_test() {
	printf("%s%s%s%s\n", BLU, BOL, "---- Test get_sroot ----", NC)
	test_get_sroot("/repo/lulinw/isam/sw/vobs/esam/build");
	test_get_sroot("/repo/lulinw/isam/sw/esam/build");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test dirname ----", NC)
	test_dirname("/home/abc");
	test_dirname("home///abc//");
	test_dirname("home///abc/de//");
	test_dirname("../abc//");
	test_dirname("abc");
	test_dirname("..");
	test_dirname("..///");

	printf("%s%s%s%s\n", BLU, BOL, "---- Test basename ----", NC)
	test_basename("/home//abc")
	test_basename("///")
	test_basename("/")
	test_basename("..")
	test_basename("../abc//")
	test_basename("/../abc//")
	test_basename("/../abc/efg/")
}

function test_get_sroot(line) {
	if ((ret = get_sroot(line)) == "") {
		printf("Unexpected line [PAT=%s]:\n  %s%s%s\n",
				g_pat_srccommon, RED, line, NC)
	} else
		printf("Root of %s is\n  %s%s%s\n", line, GRN, get_sroot(line), NC)
}

function test_dirname(path) {
	printf("dirname(%s) is \n  %s%s%s\n", path, GRN, dirname(path), NC)
}

function test_basename(path) {
	printf("basename(%s) is \n %s%s%s\n", path, GRN, basename(path), NC)
}


Initialize first to -1; this will create a (nearly) empty file on failure,
instead of looping until the disk is full.  This kind of failure can
especially happen when a version of awk that doesn't support POSIX
character classes is used.

Submitted by:	David Wolfskill <david@catwhisker.org>

#!/usr/bin/awk -f
#
# $FreeBSD$
#
# Merge two boot loader help files for FreeBSD 3.0
# Joe Abley <jabley@patho.gen.nz>

BEGIN \
{
  state = 0;
  first = -1;
  ind = 0;
}

# beginning of first command
/^###/ && (state == 0) \
{
  state = 1;
  next;
}

# entry header
/^# T[[:graph:]]+ (S[[:graph:]]+ )*D[[:graph:]][[:print:]]*$/ && (state == 1) \
{
  match($0, " T[[:graph:]]+");
  T = substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " S[[:graph:]]+");
  S = (RLENGTH == -1) ? "" : substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " D[[:graph:]][[:print:]]*$");
  D = substr($0, RSTART + 2);

  # find a suitable place to store this one...
  ind++;
  if (ind == 1)
  {
    first = ind;
    help[ind, "T"] = T;
    help[ind, "S"] = S;
    help[ind, "link"] = -1;
  } else {
    i = first; j = -1;
    while (help[i, "T"] help[i, "S"] < T S)
    {
      j = i;
      i = help[i, "link"];
      if (i == -1) break;
    }

    if (i == -1)
    {
      help[j, "link"] = ind;
      help[ind, "link"] = -1;
    } else {
      help[ind, "link"] = i;
      if (j == -1)
        first = ind;
      else
        help[j, "link"] = ind;
    }
  }
  help[ind, "T"] = T;
  help[ind, "S"] = S;
  help[ind, "D"] = D;

  # set our state
  state = 2;
  help[ind, "text"] = 0;
  next;
}

# end of last command, beginning of next one
/^###/ && (state == 2) \
{
  state = 1;
}

(state == 2) \
{
  sub("[[:blank:]]+$", "");
  if (help[ind, "text"] == 0 && $0 ~ /^[[:blank:]]*$/) next;
  help[ind, "text", help[ind, "text"]] = $0;
  help[ind, "text"]++;
  next;
}

# show them what we have (it's already sorted in help[])
END \
{
  node = first;
  while (node != -1)
  {
    printf "################################################################################\n";
    printf "# T%s ", help[node, "T"];
    if (help[node, "S"] != "") printf "S%s ", help[node, "S"];
    printf "D%s\n\n", help[node, "D"];
    for (i = 0; i < help[node, "text"]; i++)
      printf "%s\n", help[node, "text", i];
    node = help[node, "link"];
  }
  printf "################################################################################\n";
}

#!/usr/bin/awk -f
#
# $FreeBSD$
#
# Merge two boot loader help files for FreeBSD 3.0
# Joe Abley <jabley@patho.gen.nz>

BEGIN \
{
  state = 0;
  first = 0;
  ind = 0;
}

# beginning of first command
/^###/ && (state == 0) \
{
  state = 1;
  next;
}

# entry header
/^# T[[:graph:]]+ (S[[:graph:]]+ )*D[[:graph:]][[:print:]]*$/ && (state == 1) \
{
  match($0, " T[[:graph:]]+");
  T = substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " S[[:graph:]]+");
  S = (RLENGTH == -1) ? "" : substr($0, RSTART + 2, RLENGTH - 2);
  match($0, " D[[:graph:]][[:print:]]*$");
  D = substr($0, RSTART + 2);

  # find a suitable place to store this one...
  ind++;
  if (ind == 1)
  {
    first = ind;
    help[ind, "T"] = T;
    help[ind, "S"] = S;
    help[ind, "link"] = -1;
  } else {
    i = first; j = -1;
    while (help[i, "T"] help[i, "S"] < T S)
    {
      j = i;
      i = help[i, "link"];
      if (i == -1) break;
    }

    if (i == -1)
    {
      help[j, "link"] = ind;
      help[ind, "link"] = -1;
    } else {
      help[ind, "link"] = i;
      if (j == -1)
        first = ind;
      else
        help[j, "link"] = ind;
    }
  }
  help[ind, "T"] = T;
  help[ind, "S"] = S;
  help[ind, "D"] = D;

  # set our state
  state = 2;
  help[ind, "text"] = 0;
  next;
}

# end of last command, beginning of next one
/^###/ && (state == 2) \
{
  state = 1;
}

(state == 2) \
{
  sub("[[:blank:]]+$", "");
  if (help[ind, "text"] == 0 && $0 ~ /^[[:blank:]]*$/) next;
  help[ind, "text", help[ind, "text"]] = $0;
  help[ind, "text"]++;
  next;
}

# show them what we have (it's already sorted in help[])
END \
{
  node = first;
  while (node != -1)
  {
    printf "################################################################################\n";
    printf "# T%s ", help[node, "T"];
    if (help[node, "S"] != "") printf "S%s ", help[node, "S"];
    printf "D%s\n\n", help[node, "D"];
    for (i = 0; i < help[node, "text"]; i++)
      printf "%s\n", help[node, "text", i];
    node = help[node, "link"];
  }
  printf "################################################################################\n";
}


Add information about the payload format

git-svn-id: 392d0e5ce652024192c571322d40dcc23d6d83b0@146 8a65f31f-a56f-4bd6-98e8-2755f64920ec

# Hexadecimal to decimal convertor
function hex2dec(str) {
  hstr = "0123456789abcdef"
  res = 0
  n = split(tolower(str), digit, "")

  for(j = 1; j <= n; j++) {
    num = index(hstr, digit[j]) - 1
    res = res + (num*16^(n-j))
  }

  return res
}

BEGIN {
  first = 1
}

/^Frame [0-9]+: (.*)/ {
  # New frame, closing
 and html> from previous frame before compute new filename
  if (first == 0) {

    if (protocol == "DATA") {
      printf "\n" > filename
      printf "
Raw data
\n" > filename
      printf "
\n" > filename
      L = length(PL)
      for (i = 0; i <= L; i++) {
        print PL[i] > filename
      }
    }

    printf "
\n" >filename
    printf " \n" >filename
    printf "\n" >filename
    printf "\n" >filename
  } else {
    first = 0
  }

  nbr = $2
  sub (":", "", nbr)
  filename = destDir "/frames/Frame" nbr ".html"
  printf "\n" >filename
  printf " \n" >filename
  printf "  <link rel=\"stylesheet\" type=\"text/css\" href=\"callflow.css\" />\n" >filename
  # The style tag is actually not desired, but it is needed as otherwise the
  # SIP messages are displayed incorrectly in dynamic presentation mode.
  # The java script used in the dynamic presentation mode does not read the stylesheet.
  printf "  \n" >filename
  printf "  <title>callflow - frame %s</title>\n", nbr >filename
  printf " \n" >filename
  printf " \n" >filename
  printf "
\n" >filename
  print $0 >filename

  # Print Arrival Time
  getline
  gsub("^ *", "")
  print $0 >filename

  discard = 1
  protocol = ""
}

/^Internet Protocol,/ {
  print $0 > filename
  discard = 1
}
/^Internet Protocol$/ {
  discard = 0
}
/^User Datagram Protocol,/ {
  print $0 >filename
  discard = 1
}
/^User Datagram Protocol$/ {
  discard = 0
}
/^Transmission Control Protocol,/ {
  print $0 >filename
  discard = 1
}
/^Transmission Control Protocol$/ {
  discard = 0
}
/^Internet Control Message Protocol/ {
  discard = 0
}
/^Session Initiation Protocol/ {
  discard = 0
  protocol = "SIP"
  printf "\n" > filename
}

/^Diameter Protocol/ {
  discard = 0
}

/^WebSocket Protocol/ {
  discard = 0
}

/^Hypertext Transfer Protocol/ {
  discard = 0
}

/^Call Specification Language/ {
  discard = 0
}

/^Stream Control Transmission Protocol/ {
  discard = 0
  protocol = "SCTP"
  printf "\n" > filename
}

/^Data/ {
  discard = 0
  protocol = "DATA"
  delete PL
  id = 0
  printf "\n" > filename
  printf "
Fragmented data
\n" > filename
}

/^MEGACO/ {
  discard = 0
  protocol = "MEGACO"
  printf "\n" > filename
}

{
  if (discard == 0) {

    gsub("&","\\&")
    gsub(">","\\>")
    gsub("<","\\<")

    if ((protocol == "SIP") || (protocol == "MEGACO"))  {

      MARK = "no"
      # Abbreviated SIP messages
      if ($1 == "c:" ) MARK = "sip"
      if ($1 == "f:" ) MARK = "sip"
      if ($1 == "i:" ) MARK = "sip"
      if ($1 == "k:" ) MARK = "sip"
      if ($1 == "l:" ) MARK = "sip"
      if ($1 == "m:" ) MARK = "sip"
      if ($1 == "t:" ) MARK = "sip"
      if ($1 == "v:" ) MARK = "sip"

      # SIP messages
      if ($0 ~ "MEGACO" ) MARK = "sip"
      if ($0 ~ "Session Initiation Protocol" ) MARK = "sip"
      if ($1 ~ "Accept:" ) MARK = "sip"
      if ($1 ~ "Alert-Info:" ) MARK = "sip"
      if ($1 ~ "Allow:" ) MARK = "sip"
      if ($1 ~ "Allow-Events:" ) MARK = "sip"
      if ($0 ~ "Call-ID:" ) MARK = "sip"
      if ($1 ~ "Charge:" ) MARK = "sip"
      if ($1 ~ "Contact:" ) MARK = "sip"
      if ($1 ~ "Content-Disposition:" ) MARK = "sip"
      if ($1 ~ "Content-Length:" ) MARK = "sip"
      if ($1 ~ "Content-Type:" ) MARK = "sip"
      if ($1 ~ "CSeq:" ) MARK = "sip"
      if ($1 ~ "Date:" ) MARK = "sip"
      if ($1 ~ "Expires:" ) MARK = "sip"
      if ($1 ~ "From:" ) MARK = "sip"
      if ($1 ~ "Min-SE:" ) MARK = "sip"
      if ($1 ~ "P-[A-Z][a-z].*:" ) MARK = "sip"
      if ($1 ~ "Privacy:" ) MARK = "sip"
      if ($1 ~ "Proxy-Authenticate:" ) MARK = "sip"
      if ($1 ~ "RAck:" ) MARK = "sip"
      if ($1 ~ "Remote-Party-ID:" ) MARK = "sip"
      if ($1 ~ "Require:" ) MARK = "sip"
      if ($1 ~ "Request-Line:" ) MARK = "sip"
      if ($1 ~ "Request-Disposition:" ) MARK = "sip"
      if ($1 ~ "Route:" ) MARK = "sip"
      if ($1 ~ "RSeq:" ) MARK = "sip"
      if ($1 ~ "Server:" ) MARK = "sip"
      if ($1 ~ "Supported:" ) MARK = "sip"
      if ($1 ~ "Timestamp:" ) MARK = "sip"
      if ($1 ~ "To:" ) MARK = "sip"
      if ($1 ~ "User-Agent:" ) MARK = "sip"
      if ($1 ~ "Via:" ) MARK = "sip"

      # SDP part
      if ($0 ~ "Session Description Protocol") MARK = "media"
      if ($0 ~ "Connection Information") MARK = "media"
      # Use ... here.  Actually "(a)" should be used, but awk does not like the
      # parenthesis (), workaround it with the dots.
      if ($0 ~ "Media Attribute .a.:") MARK = "media"
      if ($0 ~ "Media Description, name and address") MARK = "media"
      if ($0 ~ "Owner/Creator, Session Id") MARK = "media"
      if ($0 ~ "Session Attribute .a.:") MARK = "media"
      if ($0 ~ "Session Description Protocol Version") MARK = "media"
      if ($0 ~ "Session Name") MARK = "media"
      if ($0 ~ "Time Description, active time") MARK = "media"

      if (MARK == "sip") {
        printf "
%s
\n", $0 > filename

      } else if (MARK == "media") {
        printf "
%s
\n", $0 > filename

      } else {
        print $0 > filename
      }

    } else if (protocol == "DATA") {

      # Memorize the current PayLoad (PL) line, so it can be dumped
      # in a raw data block later
      PL[id] = $0
      id++

      # The payload has the following format:
      # 0000  74 2d 4c 65 6e 67 74 68 3a 20 32 31 33 0d 0a 50   t-Length: 213..P
      # 0010  2d 41 73 73 65 72 74 65 64 2d 49 64 65 6e 74 69   -Asserted-Identi
      # ....
      # 0160  76 65 64 2d 55 73 65 72 3a 20 3c 73 69 70 3a 62   ved-User: <sip:b
      # The following code block displays the data in a human readable format
      if ((length($1) == 4) && ($1 != "Data")) {
        PAYLOAD = substr($0, 7, 50)
        L = split(PAYLOAD, CHARS, " ")
        for (i = 1; i <= L; i++) {
          dec = hex2dec(CHARS[i])
          printf "%c", dec > filename
        }
      }

    } else {
      print $0 > filename;
    }
  }
}
END {
  printf "
\n" >filename
  printf " \n" >filename
  printf "\n" >filename
  printf "\n" >filename
}

# Hexadecimal to decimal convertor
function hex2dec(str) {
  hstr = "0123456789abcdef"
  res = 0
  n = split(tolower(str), digit, "")

  for(j = 1; j <= n; j++) {
    num = index(hstr, digit[j]) - 1
    res = res + (num*16^(n-j))
  }

  return res
}

BEGIN {
  first = 1
}

/^Frame [0-9]+: (.*)/ {
  # New frame, closing
 and html> from previous frame before compute new filename
  if (first == 0) {

    if (protocol == "DATA") {
      printf "\n" > filename
      printf "
Raw data
\n" > filename
      printf "
\n" > filename
      L = length(PL)
      for (i = 0; i <= L; i++) {
        print PL[i] > filename
      }
    }

    printf "
\n" >filename
    printf " \n" >filename
    printf "\n" >filename
    printf "\n" >filename
  } else {
    first = 0
  }

  nbr = $2
  sub (":", "", nbr)
  filename = destDir "/frames/Frame" nbr ".html"
  printf "\n" >filename
  printf " \n" >filename
  printf "  <link rel=\"stylesheet\" type=\"text/css\" href=\"callflow.css\" />\n" >filename
  # The style tag is actually not desired, but it is needed as otherwise the
  # SIP messages are displayed incorrectly in dynamic presentation mode.
  # The java script used in the dynamic presentation mode does not read the stylesheet.
  printf "  \n" >filename
  printf "  <title>callflow - frame %s</title>\n", nbr >filename
  printf " \n" >filename
  printf " \n" >filename
  printf "
\n" >filename
  print $0 >filename

  # Print Arrival Time
  getline
  gsub("^ *", "")
  print $0 >filename

  discard = 1
  protocol = ""
}

/^Internet Protocol,/ {
  print $0 > filename
  discard = 1
}
/^Internet Protocol$/ {
  discard = 0
}
/^User Datagram Protocol,/ {
  print $0 >filename
  discard = 1
}
/^User Datagram Protocol$/ {
  discard = 0
}
/^Transmission Control Protocol,/ {
  print $0 >filename
  discard = 1
}
/^Transmission Control Protocol$/ {
  discard = 0
}
/^Internet Control Message Protocol/ {
  discard = 0
}
/^Session Initiation Protocol/ {
  discard = 0
  protocol = "SIP"
  printf "\n" > filename
}

/^Diameter Protocol/ {
  discard = 0
}

/^WebSocket Protocol/ {
  discard = 0
}

/^Hypertext Transfer Protocol/ {
  discard = 0
}

/^Call Specification Language/ {
  discard = 0
}

/^Stream Control Transmission Protocol/ {
  discard = 0
  protocol = "SCTP"
  printf "\n" > filename
}

/^Data/ {
  discard = 0
  protocol = "DATA"
  delete PL
  id = 0
  printf "\n" > filename
  printf "
Fragmented data
\n" > filename
}

/^MEGACO/ {
  discard = 0
  protocol = "MEGACO"
  printf "\n" > filename
}

{
  if (discard == 0) {

    gsub("&","\\&")
    gsub(">","\\>")
    gsub("<","\\<")

    if ((protocol == "SIP") || (protocol == "MEGACO"))  {

      MARK = "no"
      # Abbreviated SIP messages
      if ($1 == "c:" ) MARK = "sip"
      if ($1 == "f:" ) MARK = "sip"
      if ($1 == "i:" ) MARK = "sip"
      if ($1 == "k:" ) MARK = "sip"
      if ($1 == "l:" ) MARK = "sip"
      if ($1 == "m:" ) MARK = "sip"
      if ($1 == "t:" ) MARK = "sip"
      if ($1 == "v:" ) MARK = "sip"

      # SIP messages
      if ($0 ~ "MEGACO" ) MARK = "sip"
      if ($0 ~ "Session Initiation Protocol" ) MARK = "sip"
      if ($1 ~ "Accept:" ) MARK = "sip"
      if ($1 ~ "Alert-Info:" ) MARK = "sip"
      if ($1 ~ "Allow:" ) MARK = "sip"
      if ($1 ~ "Allow-Events:" ) MARK = "sip"
      if ($0 ~ "Call-ID:" ) MARK = "sip"
      if ($1 ~ "Charge:" ) MARK = "sip"
      if ($1 ~ "Contact:" ) MARK = "sip"
      if ($1 ~ "Content-Disposition:" ) MARK = "sip"
      if ($1 ~ "Content-Length:" ) MARK = "sip"
      if ($1 ~ "Content-Type:" ) MARK = "sip"
      if ($1 ~ "CSeq:" ) MARK = "sip"
      if ($1 ~ "Date:" ) MARK = "sip"
      if ($1 ~ "Expires:" ) MARK = "sip"
      if ($1 ~ "From:" ) MARK = "sip"
      if ($1 ~ "Min-SE:" ) MARK = "sip"
      if ($1 ~ "P-[A-Z][a-z].*:" ) MARK = "sip"
      if ($1 ~ "Privacy:" ) MARK = "sip"
      if ($1 ~ "Proxy-Authenticate:" ) MARK = "sip"
      if ($1 ~ "RAck:" ) MARK = "sip"
      if ($1 ~ "Remote-Party-ID:" ) MARK = "sip"
      if ($1 ~ "Require:" ) MARK = "sip"
      if ($1 ~ "Request-Line:" ) MARK = "sip"
      if ($1 ~ "Request-Disposition:" ) MARK = "sip"
      if ($1 ~ "Route:" ) MARK = "sip"
      if ($1 ~ "RSeq:" ) MARK = "sip"
      if ($1 ~ "Server:" ) MARK = "sip"
      if ($1 ~ "Supported:" ) MARK = "sip"
      if ($1 ~ "Timestamp:" ) MARK = "sip"
      if ($1 ~ "To:" ) MARK = "sip"
      if ($1 ~ "User-Agent:" ) MARK = "sip"
      if ($1 ~ "Via:" ) MARK = "sip"

      # SDP part
      if ($0 ~ "Session Description Protocol") MARK = "media"
      if ($0 ~ "Connection Information") MARK = "media"
      # Use ... here.  Actually "(a)" should be used, but awk does not like the
      # parenthesis (), workaround it with the dots.
      if ($0 ~ "Media Attribute .a.:") MARK = "media"
      if ($0 ~ "Media Description, name and address") MARK = "media"
      if ($0 ~ "Owner/Creator, Session Id") MARK = "media"
      if ($0 ~ "Session Attribute .a.:") MARK = "media"
      if ($0 ~ "Session Description Protocol Version") MARK = "media"
      if ($0 ~ "Session Name") MARK = "media"
      if ($0 ~ "Time Description, active time") MARK = "media"

      if (MARK == "sip") {
        printf "
%s
\n", $0 > filename

      } else if (MARK == "media") {
        printf "
%s
\n", $0 > filename

      } else {
        print $0 > filename
      }

    } else if (protocol == "DATA") {

      # Memorize the current PayLoad (PL) line, so it can be dumped
      # in a raw data block later
      PL[id] = $0
      id++

      if ((length($1) == 4) && ($1 != "Data")) {
        PAYLOAD = substr($0, 7, 50)
        L = split(PAYLOAD, CHARS, " ")
        for (i = 1; i <= L; i++) {
          dec = hex2dec(CHARS[i])
          printf "%c", dec > filename
        }
      }

    } else {
      print $0 > filename;
    }
  }
}
END {
  printf "
\n" >filename
  printf " \n" >filename
  printf "\n" >filename
  printf "\n" >filename
}


Fix the modules path in the xpi chrome.manifests

{ content = $1 ~ /^(content|skin|locale)$/ }
content && $NF ~ /^[a-z]/ { $NF = "/" name "/" $NF }
content {
    sub(/^\.\./, "", $NF);
    $NF = "jar:chrome/" name ".jar!" $NF
}
{
    sub("^\\.\\./common/", "", $NF)
    print
}

{ content = $1 ~ /^(content|skin|locale)$/ }
content && $NF ~ /^[a-z]/ { $NF = "/" name "/" $NF }
content {
    sub(/^\.\./, "", $NF);
    $NF = "jar:chrome/" name ".jar!" $NF
}
{
    sub("^\\.\\./liberator/", "", $NF)
    print
}


change whitespace

change whitespace

#! /usr/bin/gawk -f

# label_dot_edge.awk
# example:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries agains whr scigerph (neo4j)
# instance loaded with the ontologies we use.

# In the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1 $2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
    line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}

#! /usr/bin/gawk -f

# label_dot_edge.awk
# example:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries agains whr scigerph (neo4j)
# instance loaded with the ontologies we use.

# In the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1$2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
    line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}


Split debug output into 4 files

Split debug output into 4 files

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    # For debugging variations in seasons and episodes format
    str = "date +%y%m%d.%H%M%S"
    str | getline longdate
    close str
    for ( i = 1; i <= 4; i++ ) {
        outfile = "BBox-scrapes/c" i "-" longdate ".csv"
        print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > outfile
    }
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        # Season 3, Episode 8
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        outfile = "BBox-scrapes/c1-" longdate ".csv"
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        # 35 Hours, Episode 1
        seasonNumber = "99"
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 2
        outfile = "BBox-scrapes/c2-" longdate ".csv"
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        # Season 4, Episode 5
        seasonNumber = "98"
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 3
        outfile = "BBox-scrapes/c3-" longdate ".csv"
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else {
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        c4Num += 4
        outfile = "BBox-scrapes/c4-" longdate ".csv"
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > "debug.csv"
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        # Season 3, Episode 8
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,URL,showTitle,episodeTitle,Years) >>"debug.csv"
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        # 35 Hours, Episode 1
        seasonNumber = "99"
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 2
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,URL,showTitle,episodeTitle,Years) >>"debug.csv"
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        # Season 4, Episode 5
        seasonNumber = "98"
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 3
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,URL,showTitle,episodeTitle,Years) >>"debug.csv"
    } else {
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        c4Num += 4
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,URL,showTitle,episodeTitle,Years) >>"debug.csv"
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}


Improving c2 but a few specials remaining

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    # For debugging variations in seasons and episodes format
    str = "date +%y%m%d.%H%M%S"
    str | getline longdate
    close str
    for ( i = 1; i <= 4; i++ ) {
        outfile = "BBox-scrapes/c" i "-" longdate ".csv"
        print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > outfile
    }
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    shortURL = URL
    if (match (episodeTitle, /^Season [[:digit:]]+|^Series [[:digit:]]+/))
        episodeSeason = substr(episodeTitle,8,RLENGTH-7)
    else
        episodeSeason = ""

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        # Season 3, Episode 8
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        # Give precedence to season number in episodeTitle over season number in URL
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            revisedSeasons1 += 1
            sub (/^\/us\/episode\//,"",shortURL)
            printf ("==> Fixed c1 season: %02d-%02d\t%s\t%s\n", \
                    seasonNumber, episodeSeason, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        outfile = "BBox-scrapes/c1-" longdate ".csv"
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        # 35 Hours, Episode 1
        revisedSeasons2 += 1
        episodeSeason = Years
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            printf ("==> Fixed c2 season: %02d-%02d\t%s\t%s\n", \
                    seasonNumber, episodeSeason, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 1
        outfile = "BBox-scrapes/c2-" longdate ".csv"
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        # Season 4, Episode 5
        revisedSeasons3 += 1
        seasonNumber = "98"
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 1
        outfile = "BBox-scrapes/c3-" longdate ".csv"
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else {
        revisedSeasons4 += 1
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        c4Num += 1
        outfile = "BBox-scrapes/c4-" longdate ".csv"
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons1 > 0 ) {
        revisedSeasons1 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c1 %s revised in %s\n", revisedSeasons1, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons2 > 0 ) {
        revisedSeasons2 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c2 %s revised in %s\n", revisedSeasons2, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons3 > 0 ) {
        revisedSeasons3 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c3 %s revised in %s\n", revisedSeasons3, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons4 > 0 ) {
        revisedSeasons4 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c4 %s revised in %s\n", revisedSeasons4, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}

# Print the processed "Episode" lines from a WebScraper csv file saved in tsv format

# INVOCATION:
#       awk -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v ERROR_FILE=$ERROR_FILE \
#           -f getBBoxEpisodesFrom-webscraper.awk $EPISODES_SORTED_FILE >$EPISODES_SPREADSHEET_FILE

# Field numbers
#    1 web-scraper-start-url  2 SsnURL     3 SsnURL-href    4 URL        5 Program_Title
#    6 Episode_Title          7 Year       8 Duration       9 Rating    10 Description

BEGIN {
    FS="\t"
    OFS="\t"
    print "Sortkey\tTitle\tSeasons\tEpisodes\tDuration\tYear(s)\tRating\tDescription"
    # For debugging variations in seasons and episodes format
    str = "date +%y%m%d.%H%M%S"
    str | getline longdate
    close str
    for ( i = 1; i <= 4; i++ ) {
        outfile = "BBox-scrapes/c" i "-" longdate ".csv"
        print "Type\t#\tSortkey\tURL\tshowTitle\tepisodeTitle\tYear(s)" > outfile
    }
}

{
    for ( i = 1; i <= NF; i++ ) {
        if ($i == "null")
            $i = ""
    }
}

/\/us\/movie\/|\/us\/episode\// {
    baseURL = $1
    URL = $4
    showTitle = $5
    episodeTitle = $6
    Years = $7
    Duration = $8
    Rating = $9
    Description = $10
    showtype = "S"

    if (match (episodeTitle, /^Season [[:digit:]]+|^Series [[:digit:]]+/))
        episodeSeason = substr(episodeTitle,8,RLENGTH-7)
    else
        episodeSeason = ""

    # Extract sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_S[[:digit:]]*_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Days_S3_E8_23851
        # Season 3, Episode 8
        seasonNumber = substr(fld[nflds-2], 2)
        episodeNumber = substr(fld[nflds-1], 2)
        # Give precedence to season number in episodeTitle over season number in URL
        if (episodeSeason != "" && seasonNumber+0 != episodeSeason+0) {
            revisedSeasons1 += 1
            shortURL = URL
            sub (/^\/us\/episode\//,"",shortURL)
            printf ("==> Fixed c1 season: %02d-%02d\t%s\t%s\n", \
                    seasonNumber, episodeSeason, shortURL, episodeTitle) >> ERROR_FILE
            seasonNumber = episodeSeason
        }
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c1Num += 1
        outfile = "BBox-scrapes/c1-" longdate ".csv"
        printf ("c1\t%d\t%s\t%s\t%s\t%s\t%s\n",c1Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /_E[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/35_Hours_E1_26129
        # 35 Hours, Episode 1
        revisedSeasons2 += 1
        seasonNumber = "99"
        episodeNumber = substr(fld[nflds-1], 2)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c2Num += 1
        outfile = "BBox-scrapes/c2-" longdate ".csv"
        printf ("c2\t%d\t%s\t%s\t%s\t%s\t%s\n",c2Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else if (URL ~ /Episode_[[:digit:]]*_[[:digit:]]*$/) {
        # /us/episode/Episode_5_26089
        # Season 4, Episode 5
        revisedSeasons3 += 1
        seasonNumber = "98"
        episodeNumber = substr(fld[nflds-1], 1)
        sortkey = sprintf ("%s%02dE%03d", showtype, seasonNumber, episodeNumber)
        c3Num += 1
        outfile = "BBox-scrapes/c3-" longdate ".csv"
        printf ("c3\t%d\t%s\t%s\t%s\t%s\t%s\n",c3Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    } else {
        revisedSeasons4 += 1
        URL ~ /^\/us\/movie\// ? showtype = "M" : showtype = "E"
        # /us/episode/Orkneys_Stone_Age_Temple_7144
        # A History of Ancient Britain Special, Orkney's Stone Age Temple
        sortkey = sprintf ("%s%05d", showtype, fld[nflds])
        c4Num += 1
        outfile = "BBox-scrapes/c4-" longdate ".csv"
        printf ("c4\t%d\t%s\t%s\t%s\t%s\t%s\n",c4Num,sortkey,URL,showTitle,episodeTitle,Years) >> outfile
    }

    # Convert duration from minutes to HMS
    sub( / min/,"",Duration)
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (HMS == "00:00:00")
        HMS = ""

    # Special processing for some shows is needed in all getBBox*From-webscraper.awk scripts.
    # "Changed" message and counting revisedTitles are only in getBBoxProgramsFrom-webscraper.awk
    # so they are not repeated for each episode and each season.
    if (baseURL ~ /Maigret_15974$/) {
        showTitle = "Maigret (2016)"
    }
    if (baseURL ~ /Maigret_\(1992\)_15928$/) {
        showTitle = "Maigret (1992)"
    }
    if (baseURL ~ /Porridge_9509$/) {
        showTitle = "Porridge (1974-1977)"
    }
    if (baseURL ~ /Porridge_14747$/) {
        showTitle = "Porridge (2016-2017)"
    }

    # Get rid of redundant "Series #" or "Series #" from episodeTitle
    if (match (episodeTitle, /^Season [[:digit:]]*, |^Series [[:digit:]]*, /))
        episodeTitle = substr(episodeTitle,RLENGTH+1)

    # Get rid of initial/trailing quotes in showTitle
    # "BBC Three's ""Murdered By..."" Collection Trailer"
    if (showTitle ~ /^".*"$/) {
        revisedTitles += 1
        printf ("==> Fixed quoted title: www.britbox.com%s\n", URL) >> ERROR_FILE
        # printf ("    %s\n", showTitle) >> ERROR_FILE
        sub (/^"/,"",showTitle)
        sub (/"$/,"",showTitle)
        # printf ("    %s\n", showTitle) >> ERROR_FILE
    }

    # Non-existent fields in movies or episodes
    if (URL ~ /^\/us\/movie\//) {
        depth = "1"
        numSeasons = 1
        numEpisodes = 1
        fullTitle = showTitle
    } else {
        depth = "2"
        numSeasons = ""
        numEpisodes = ""
        fullTitle = showTitle ", " sortkey ", " episodeTitle
    }

    # Fix two consecutive double quotes in showTitle, but leave in fullTitle
    gsub ("\"\"","\"",showTitle)

    # Fix HEX quotes in Description
    gsub ("\xc2\x91","\"",Description)
    gsub ("\xc2\x92","\"",Description)

    # Create spreadsheet row in common format
    savedLine = sprintf \
        ("%s (%s) %s %s\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s\"\)\t%s\t%s\t%s\t%s\t%s\t%s",\
         showTitle, depth, Years, sortkey, URL, fullTitle, numSeasons, numEpisodes, \
         HMS, Years, Rating, Description)

    # Make sure line doesn't start with a single quote so it sorts correctly in Open Office
    sub (/^'/,"",savedLine)

    print savedLine
    next
}

/\/us\/show\// {
    # Shouldn't get here
    badEpisodes += 1
    URL = $4
    nflds = split (URL,URLflds,"/")
    printf ("==> Upstream error in %s\n", FILENAME)  >> ERROR_FILE
    print "==> Found /us/show/ in episodes: " URLflds[nflds]  >> ERROR_FILE
}

END {
    printf ("In getBBoxEpisodesFrom-webscraper.awk\n") > "/dev/stderr"
    if (revisedTitles > 0 ) {
        revisedTitles == 1 ? field = "title" : field = "titles"
        printf ("    %2d %s revised in %s\n", revisedTitles, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons1 > 0 ) {
        revisedSeasons1 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c1 %s revised in %s\n", revisedSeasons1, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons2 > 0 ) {
        revisedSeasons2 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c2 %s revised in %s\n", revisedSeasons2, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons3 > 0 ) {
        revisedSeasons3 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c3 %s revised in %s\n", revisedSeasons3, field, FILENAME) > "/dev/stderr"
    }
    if (revisedSeasons4 > 0 ) {
        revisedSeasons4 == 1 ? field = "season" : field = "seasons"
        printf ("    %3d c4 %s revised in %s\n", revisedSeasons4, field, FILENAME) > "/dev/stderr"
    }
    if (badEpisodes > 0 ) {
        badEpisodes == 1 ? field = "URL" : field = "URLs"
        printf ("    %2d extra /show/ %s in %s\n", badEpisodes, field, FILENAME) > "/dev/stderr"
    }
}


comments

comments

#! /usr/bin/gawk -f

# label_dot_edge.awk
# Attempts to augment opaque ontological terms with their current label
# example usage:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Arguments: two files
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries against our scigraph
# (neo4j) instance loaded with the ontologies we use.
# Reads the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1 $2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
    line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}

#! /usr/bin/gawk -f

# label_dot_edge.awk
# example:
# label_dot_edge.awk predicate_curie_term_label.tab  in.dot > out.dot
#
# Loads the first file as a map from terms to labels:
# (expects: [namespace:]<tab><term><tab><label>[<tab><whatever>])
# the first set was generated via cypher queries agains whr scigerph (neo4j)
# instance loaded with the ontologies we use.

# In the second Graphviz digraph dot format file:
# looks for <term> as edge-text
# and appends "! <label> (Iff <term> is found in map)
# note: This "<term> ! <label>" structure is a convention used by Ontologists.

BEGIN{FS="\t"; OFS="\t"}
# load labels
FNR == NR && $3 != "null"{label[$1 $2]=$3}

# output lines that do not get labels
FNR != NR && !/.* -> .*label=.*/ {print}

# output lines w/labels if they are available
FNR != NR && /.* -> .*label=.*/ {
	line="";
	split($0, row, / /)
	split(row[4], term, /"/)
	if (term[2] in label){
		term[2] = term[2] " ! " label[term[2]];
		row[4] = term[1];
		for(i=2;i<=length(term);i++)
			row[4] = row[4] "\"" term[i]
	}
    line = row[1]
	for(i=2;i<=length(row);i++)
		line = line " " row[i]
	print  line
}