fix condition in iteration getReply

fix condition in iteration getReply

@load "redis"
BEGIN{
  c=connectRedis()
  select(c,12)
  testListItem(c, 0)
  testPurchaseItem(c)
  testBenchmarkUpdateToken(c)
}

function testListItem(c, nested) {
  if (!nested){
     print "\n----- testListItem -----"
  }
  print "We need to set up just enough state so that a user can list an item"
  seller = "userX"
  item = "itemX"
  sadd(c,"inventory:"seller, item)
  delete(AR)
  smembers(c,"inventory:"seller,AR)
  print "The user's inventory has:"
  for(i in AR) {
    print "  "AR[i]
  }
  print
  print "Listing the item..."
  l = listItem(conn, item, seller, 10)
  print "Listing the item succeeded? "l
  delete(AR)
  zrangeWithScores(c,"market:",AR,0, -1)
  print "The market contains:"
  for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]", "AR[i+1]
  }
}

function testPurchaseItem(c) {
   print "\n----- testPurchaseItem -----"
   testListItem(c, 1)
   print "We need to set up just enough state so a user can buy an item"
   hset(c,"users:userY", "funds", "125")
   delete(AR)
   hgetall(c,"users:userY",AR)
   print "The user has some money:"
   for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]": "AR[i+1]
   }
   print
   print "Let's purchase an item"
   p = purchaseItem(c, "userY", "itemX", "userX", 10)
   print "Purchasing an item succeeded? "p
   delete(AR)
   hgetall(c,"users:userY",AR)
   print "Their money is now:"
   for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]": "AR[i+1]
   }
   buyer = "userY"
   delete(AR)
   smembers(c,"inventory:"buyer,AR)
   print "Their inventory is now:"
   for(member in AR) {
     print "  "AR[member]
   }
}

function testBenchmarkUpdateToken(c) {
   print "\n----- testBenchmarkUpdate -----"
   benchmarkUpdateToken(c, 5)
}

function listItem(c, itemId, sellerId, price) {
   inventory = "inventory:"sellerId
   item = itemId"."sellerId
   now=systime()*1000
   end=now+5000
   while (now < end) {
     watch(c,inventory)
     if(!sismember(c,inventory, itemId)){
       unwatch(c)
       return 0
     }
     multi(c)
     zadd(c,"market:", price, item)
     srem(c,inventory, itemId)
     if(exec(c,R)==0) {
       now=systime()*1000
       continue
     }
     return 1
   }
   return 0
}

function purchaseItem(c, buyerId, itemId, sellerId, lprice) {
   buyer = "users:"buyerId
   seller = "users:"sellerId
   item = itemId"."sellerId
   inventory = "inventory:"buyerId
   end=(systime()*1000)+10000
   while ((systime()*1000) < end){
     watch(c,"market:"buyer)
     price = zscore(c,"market:", item)
     funds = hget(c,buyer, "funds")
     if (price != lprice || price > funds){
       unwatch()
       return 0
     }
     multi(c)
     hincrby(c, seller, "funds", price)
     hincrby(c, buyer, "funds", -price)
     sadd(c, inventory, itemId)
     zrem(c, "market:", item)
     if(exec(c,R)==0) {
        continue
     }
     return 1
   }
   return 0
}

function benchmarkUpdateToken(c, duration) {
  for(i=1;i<=2;i++) {
     if(i==1){
        method="updateToken"
     }
     else {
        method="updateTokenPipeline"
	p=pipeline(c)
     }
     count = 0
     start=systime()*1000
     end = start + (duration * 1000)
     while((systime()*1000) < end){
        count++;
	if(i==1){
          updateToken(c, "token", "user", "item")
        }
        else {
          updateTokenPipeline(c, "token", "user", "item", p)
        }
     }
     delta=(systime()*1000) - start
     print method" "count" "(delta / 1000)" "(count / (delta / 1000))
  }
}

function updateToken(c, token, user, item) {
   timestamp=systime()
   hset(c,"login:", token, user)
   zadd(c,"recent:", timestamp, token)
   if (item) {
     zadd(c,"viewed:"token, timestamp, item)
     zremrangebyrank(c, "viewed:"token, 0, -26)
     zincrby(c, "viewed:", -1, item)
   }
}

function updateTokenPipeline(c, token, user, item, p) {
    timestamp=systime()
    hset(p,"login:", token, user)
    zadd(p,"recent:", timestamp, token)
    if (item) {
      zadd(p,"viewed:"token, timestamp, item)
      zremrangebyrank(p,"viewed:"token, 0, -26)
      zincrby(p,"viewed:", -1, item)
    }
    for(ERRNO="" ; ERRNO=="" ; getReply(p,REPLY))
      ;
}

@load "redis"
BEGIN{
  c=connectRedis()
  select(c,12)
  testListItem(c, 0)
  testPurchaseItem(c)
  testBenchmarkUpdateToken(c)
}

function testListItem(c, nested) {
  if (!nested){
     print "\n----- testListItem -----"
  }
  print "We need to set up just enough state so that a user can list an item"
  seller = "userX"
  item = "itemX"
  sadd(c,"inventory:"seller, item)
  delete(AR)
  smembers(c,"inventory:"seller,AR)
  print "The user's inventory has:"
  for(i in AR) {
    print "  "AR[i]
  }
  print
  print "Listing the item..."
  l = listItem(conn, item, seller, 10)
  print "Listing the item succeeded? "l
  delete(AR)
  zrangeWithScores(c,"market:",AR,0, -1)
  print "The market contains:"
  for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]", "AR[i+1]
  }
}

function testPurchaseItem(c) {
   print "\n----- testPurchaseItem -----"
   testListItem(c, 1)
   print "We need to set up just enough state so a user can buy an item"
   hset(c,"users:userY", "funds", "125")
   delete(AR)
   hgetall(c,"users:userY",AR)
   print "The user has some money:"
   for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]": "AR[i+1]
   }
   print
   print "Let's purchase an item"
   p = purchaseItem(c, "userY", "itemX", "userX", 10)
   print "Purchasing an item succeeded? "p
   delete(AR)
   hgetall(c,"users:userY",AR)
   print "Their money is now:"
   for(i=1;i<=length(AR);i+=2){
     print "  "AR[i]": "AR[i+1]
   }
   buyer = "userY"
   delete(AR)
   smembers(c,"inventory:"buyer,AR)
   print "Their inventory is now:"
   for(member in AR) {
     print "  "AR[member]
   }
}

function testBenchmarkUpdateToken(c) {
   print "\n----- testBenchmarkUpdate -----"
   benchmarkUpdateToken(c, 5)
}

function listItem(c, itemId, sellerId, price) {
   inventory = "inventory:"sellerId
   item = itemId"."sellerId
   now=systime()*1000
   end=now+5000
   while (now < end) {
     watch(c,inventory)
     if(!sismember(c,inventory, itemId)){
       unwatch(c)
       return 0
     }
     multi(c)
     zadd(c,"market:", price, item)
     srem(c,inventory, itemId)
     if(exec(c,R)==0) {
       now=systime()*1000
       continue
     }
     return 1
   }
   return 0
}

function purchaseItem(c, buyerId, itemId, sellerId, lprice) {
   buyer = "users:"buyerId
   seller = "users:"sellerId
   item = itemId"."sellerId
   inventory = "inventory:"buyerId
   end=(systime()*1000)+10000
   while ((systime()*1000) < end){
     watch(c,"market:"buyer)
     price = zscore(c,"market:", item)
     funds = hget(c,buyer, "funds")
     if (price != lprice || price > funds){
       unwatch()
       return 0
     }
     multi(c)
     hincrby(c, seller, "funds", price)
     hincrby(c, buyer, "funds", -price)
     sadd(c, inventory, itemId)
     zrem(c, "market:", item)
     if(exec(c,R)==0) {
        continue
     }
     return 1
   }
   return 0
}

function benchmarkUpdateToken(c, duration) {
  for(i=1;i<=2;i++) {
     if(i==1){
        method="updateToken"
     }
     else {
        method="updateTokenPipeline"
	p=pipeline(c)
     }
     count = 0
     start=systime()*1000
     end = start + (duration * 1000)
     while((systime()*1000) < end){
        count++;
	if(i==1){
          updateToken(c, "token", "user", "item")
        }
        else {
          updateTokenPipeline(c, "token", "user", "item", p)
        }
     }
     delta=(systime()*1000) - start
     print method" "count" "(delta / 1000)" "(count / (delta / 1000))
  }
}

function updateToken(c, token, user, item) {
   timestamp=systime()
   hset(c,"login:", token, user)
   zadd(c,"recent:", timestamp, token)
   if (item) {
     zadd(c,"viewed:"token, timestamp, item)
     zremrangebyrank(c, "viewed:"token, 0, -26)
     zincrby(c, "viewed:", -1, item)
   }
}

function updateTokenPipeline(c, token, user, item, p) {
    timestamp=systime()
    hset(p,"login:", token, user)
    zadd(p,"recent:", timestamp, token)
    if (item) {
      zadd(p,"viewed:"token, timestamp, item)
      zremrangebyrank(p,"viewed:"token, 0, -26)
      zincrby(p,"viewed:", -1, item)
    }
    for(ERRNO="" ; ERRNO="" ; var=getReply(p,REPLY)) ;
    ERRNO==""
}


Fix ofmtstrnum

Fix ofmtstrnum

This is a weird one: Go splitting on an empty regex gives the individual
characters in the string (like JavaScript). AWK and Python give the
original string. So change the regex to "x" to mimic the latter behavior.

BEGIN {
	split(" 1", f, "x")
	OFMT = "%.1f"
	print f[1]
	x = f[1] + 0
	print f[1]
}

BEGIN {
	split(" 1", f, "|")
	OFMT = "%.1f"
	print f[1]
	x = f[1] + 0
	print f[1]
}


Generalize series mismatch fix

Generalize series mismatch fix

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Acorn TV | 19-2</title>
#  ---
#        19-2
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>Acorn TV \| /,"")
    sub (/<.*/,"")
    gsub (/&/,"\\&")
    # gsub (/'/,"'")
    # gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if (numSeasons == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/'/,"'",description)
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, seriesTitle, episodeURL)  >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default christmasspecial episodeType to "X", don't increment seasonNumber
    if (episodeURL ~ /christmasspecial\//) {
        episodeType = "X"
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, and Vera have known problems
    if (episodeURL ~ \
           /\/series[0-9]{1,2}\/|\/murdoch\/season[0-9]{1,2}/) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}

# Create column files with lists of series titles, descriptions, number
# of seasons, and number of episodes, to be pasted into a spreadsheet.
# Note that IN_CANADA affects processing.
#
# Create a separate file with a line for each episode containing
# seriesNumber, episodeURL, seriesTitle, seasonNumber, episodeNumber,
# episodeTitle, & episodeDuration with the same columns as the primary
# spreadsheet so they can be combined into one.
#
# Create a separate file with a line for each series duration
#
# Return the total duration of all series
#
# INVOCATION:
#       curl -s https://acorn.tv/192/ \
#           | awk -v TITLE_FILE=$TITLE_FILE -v DESCRIPTION_FILE=$DESCRIPTION_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -v NUM_EPISODES_FILE=$NUM_EPISODES_FILE \
#           -v EPISODE_CURL_FILE=$EPISODE_CURL_FILE -v IN_CANADA=$IN_CANADA \
#           -v EPISODE_INFO_FILE=$EPISODE_INFO_FILE -v SERIES_NUMBER=$lastRow \
#           -v DURATION_FILE=$DURATION_FILE -v ERROR_FILE=$ERROR_FILE -f getAcornFrom-seriesPages.awk
#
# INPUT:
#        <title>Acorn TV | 19-2</title>
#  ---
#        19-2
#        <meta itemprop="numberOfEpisodes" content="30" />
#        <meta itemprop="numberOfSeasons" content="3" />
#  ---
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#        <meta itemprop="numberOfEpisodes" content="10" />
#  ---
#
"The writing is sublime" (New \
#        York Times) in this anything-but-a-procedural cop drama ... as reluctant partners \
#        patrolling the streets of Montreal. Not Available in Canada. CC Available.

#  ---
#
We're sorry, but Acorn TV is not available in this territory.

#  ---
#        <meta itemprop="seasonNumber" content="1" />
#  ---
#
#  ---
#        <meta itemprop="timeRequired" content="T44M06S" />
#  ---
#
Partners

#  ---
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

#
#
#
#  OUTPUT:
#       $EPISODE_INFO_FILE
#

# Extract the series title
/<title>/ {
    sub (/.*<title>Acorn TV \| /,"")
    sub (/<.*/,"")
    gsub (/&/,"\\&")
    # gsub (/'/,"'")
    # gsub (/"/,"\"\"")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    episodeLinesFound = 0
    numEpisodesStr = ""
    seriesTitle = $0
    print seriesTitle >> TITLE_FILE
    next
}

# Extract the number of episodes in the series
/itemprop="numberOfEpisodes"/ {
    split ($0,fld,"\"")
    episodeLinesFound += 1
    if (episodeLinesFound == 1)
        totalEpisodes = fld[4]
    if (episodeLinesFound != 1)
        numEpisodesStr = numEpisodesStr "+" fld[4]
    next
}

# Extract the number of seasons in the series
/itemprop="numberOfSeasons"/ {
    split ($0,fld,"\"")
    numSeasons = fld[4]
    if (numSeasons == 0)
        printf ("==> No seasons: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    print numSeasons >> NUM_SEASONS_FILE
    next
}

/Acorn TV is not available in this territory/ {
    printf ("==> Not available here: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
}

# Extract the series description
/id="franchise-description"/ {
    # get rid of boilerplate
    split ($0,fld,"[<>]")
    description = fld[3]
    # get rid of unnecessary characters and text
    gsub (/\\/,"",description)
    if (IN_CANADA != "yes") {
        sub (/Series [[:digit:]]* not available in Canada\./,"",description)
        sub (/Not [Aa]vailable in Canada\./,"",description)
        sub (/NOT AVAILABLE IN CANADA\./,"",description)
    }
    sub (/CC Available\. CC Available/,"CC Available",description)
    # fix sloppy input spacing
    sub (/\.CC Available/,". CC Available",description)
    gsub (/ \./,".",description)
    gsub (/  */," ",description)
    sub (/^ */,"",description)
    sub (/ *$/,"",description)
    # fix funky HTML characters
    gsub (/'/,"'",description)
    print description >> DESCRIPTION_FILE
    next
}

# Extract the seasonNumber
# Note that Acorn "season" numbers may not correspond to actual season numbers.
# They simply refer to the number of seasons available on Acorn.
/<meta itemprop="seasonNumber" content="/ {
    split ($0,fld,"\"")
    seasonNumber = fld[4]
    next
}

# Extract the episode URL
/> EPISODE_CURL_FILE
    next
}

# Extract the episode duration
/<meta itemprop="timeRequired"/ {
    split ($0,fld,"\"")
    split (fld[4],tm,/[TMS]/)
    secs = tm[3]
    mins = tm[2] + int(secs / 60)
    hrs =  int(mins / 60)
    secs %= 60; mins %= 60
    #
    seriesSecs += secs
    seriesMins += mins + int(seriesSecs / 60)
    seriesHrs += hrs + int(seriesMins / 60)
    seriesSecs %= 60; seriesMins %= 60
    #
    totalSecs += secs
    totalMins += mins + int(totalSecs / 60)
    totalHrs += hrs + int(totalMins / 60)
    totalSecs %= 60; totalMins %= 60
    #
    episodeDuration = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
    if (episodeDuration == "00:00:00")
        printf ("==> No duration: %d\t%s  %s\n", SERIES_NUMBER, seriesTitle, episodeURL)  >> ERROR_FILE
    next
}

# Extract the episode title
/
/ {
    split ($0,fld,"[<>]")
    episodeTitle = fld[3]
    gsub (/&/,"\\&", episodeTitle)
    next
}

# There are a number of variants in the "episodeNumber" string
#
#
Season 1: Episode 1

#
Season 9 Christmas Special: Episode 1

#
#
Series 1: Episode 1

#
Series 1 (Set 1): Episode 1

#
Series 10 (Sets 13 - 14): Episode 1

#
Series 19 (Part One): Episode 1

#
#
Set 1: Episode 1

#
#
Mini Series: Episode 1

#
Miniseries: Episode 1

#
#
Black Widows Season 1: Episode 1

#
#
A Dance to the Music of Time: Episode 1

#
Christmas Special : Episode 1

#
Feature Film: Episode 1

# Extract episode number
/
.*span itemprop="episodeNumber">/ {
    # If show is a Prequel, use "P"
    if (episodeURL ~ /\/prequel/)
        showType = "P"
    # If show is a Miniseries, use "M"
    if (episodeURL ~ /\/miniseries/)
        showType = "M"
    # If we don't know what kind of show it is, use "S"
    if (showType == "")
        showType = "S"
    # Default episodeType to "E"
    episodeType = "E"
    # Default christmasspecial episodeType to "X", don't increment seasonNumber
    if (episodeURL ~ /christmasspecial\//) {
        episodeType = "X"
        seasonNumber -= 1
    }
    #
    split ($0,fld,"[<>]")
    episodeNumber = fld[5]
    # Feature films don't have episodes
    if (episodeURL ~ /\/featurefilm\//)
        next
    # neither should single episode series
    if (totalEpisodes == 1)
        next
    # The season number should match that in the URL
    # Birds of a Feather, Doc Martin, Murdoch, Poirot, Rebus, and Vera have known problems
    if (episodeURL ~ \
           /\/birdsfeather\/series[0-9]{1,2}\/|\/docmartin\/series[0-9]{1,2}\/|\/murdoch\/season[0-9]{1,2}|\/poirot\/series[0-9]{1,2}\/|\/rebus\/series[0-9]{1,2}\/|\/vera\/series[0-9]{1,2}\//) {
        split (episodeURL, part, "/")
        URLseasonNumber = part[5]
        sub (/christmasspecial/,"",URLseasonNumber)
        sub (/[[:alpha:]]*/,"",URLseasonNumber)
        if (URLseasonNumber != seasonNumber) {
            printf ("==> Corrected mismatch: https://acorn.tv/%s/%s was series %d\n", \
               part[4], part[5], seasonNumber) >> ERROR_FILE
            seasonNumber = URLseasonNumber
        }
    }
    printf ("%d\t=HYPERLINK(\"%s\";\"%s, %s%02d%s%02d, %s\"\)\t\t\t%s\n", \
        SERIES_NUMBER, episodeURL, seriesTitle, showType, seasonNumber, episodeType, episodeNumber, \
        episodeTitle, episodeDuration) >> EPISODE_INFO_FILE
    showType = ""
    next
}

/-- Viewers Also Watched --/ {
    print (episodeLinesFound == 1 ? "=0" : "=" numEpisodesStr) >> NUM_EPISODES_FILE
    if (description == "")
        printf ("==> No description: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    description = ""
    #
    seriesDuration = sprintf ("%02d:%02d:%02d", seriesHrs, seriesMins, seriesSecs)
    print seriesDuration >> DURATION_FILE
    if (seriesDuration == "00:00:00")
        printf ("==> No duration: %d\t%s\n", SERIES_NUMBER, seriesTitle)  >> ERROR_FILE
    seriesSecs = 0
    seriesMins = 0
    seriesHrs = 0
    seriesDuration = ""
    #
    SERIES_NUMBER += 1
}

END {
    # Return the total duration of all series
    printf ("%02d:%02d:%02d\n", totalHrs, totalMins, totalSecs)
}


Replace GNU-specific character classes with MAWK-compatible expressions.

#!/bin/awk
#
# Copyright (c) 2014, Michal Policht. This file is dually licensed under terms of
# either WTFPL or BEER-WARE LICENSE. You may obtain the copy of WTFPL or BEER-WARE
# LICENSE in nearest drug store.
#
# THIS SOFTWARE IS ADDICTED TO DRUGS AND ALCOHOL AND IS A SUSPECT OF DRIVING UNDER THE
# INFLUENCE OF ALCOHOL. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS NOT" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS AND APPLICATION AND
# ACCOMODATION FOR A PARTICULAR PURPOSE AND MANY OTHERS NOT MENTIONED, NOR EVEN
# INVENTED WARRANTIES ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
# OR CONTRIBUTORS BE LIABLE FOR ANY TEMPORAL, PERMANENT, UNFORTUNATE, UNLUCKY,
# LUCKY, DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, ACCIDENTAL, MENTAL,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION;
# OR BEING KILLED BY A LIGHTNING; OR FLOOD; OR FIRE; OR ANTEATER (INCLUDING BUT NOT
# LIMITED TO, GIANT ANTEATER, ANTEATER WHO LIES ON A SOFA; ANTEATER WHO LISTENS HEAVY
# METAL MUSIC; ANTEATER WHO ATE HUMAN BEFORE) OR ANOTHER BEING.) HOWEVER CAUSED AND ON
# ANY, ANY THEORY OF LIABILITY, CONSPIRACY THEORY, ANY THEORY OF RELATIVITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, ARISING IN ANY WAY OUT OF THE SUN, EVEN IF PROMISSED
# WISHED OR ASSURED SUCH DAMAGE. THIS SOFTWARE CAN REALLY DO TERRIBLE THINGS.
#
# Script updates C/C++ include guards with macros based on header path (including directory structure).
# It seeks for "#ifndef, #define" pair and substitutes first encountered pair with generated values.
# Script does not distinguish include guards from other macros, so each HEADER FILE MUST CONTAIN
# INCLUDE GUARD AS ITS FIRST "#ifndef, #define" MACRO PAIR. Script only updates include guards;
# if file does not contain something that looks like include guard to the script it will not add one.
# Script will also eat everything after last "#endif", presumably include guard comment. It's a stupid
# beast, so always backup your data before use.
#
# usage: awk [-v prefix="PREFIX"] -f awkgward.awk header
# example: awk -v prefix="PROJECT_" -f awkgward.awk file.hpp
#
# parameters:
# header - header file.
#
# required tools: awk.

function create_guard_macro(prefix, filename)
{
    double_underscore = toupper(filename);
    # just in case, double single underscores
    gsub(/_/, "__", double_underscore);
    # \/ = slash (/), \. = dot (.), \\ = backslash (\) - replace them with underscore.
    gsub(/[\/\.\\]/, "_", double_underscore);
    return prefix double_underscore;
}

function create_guard(prefix, filename)
{
    guard_macro = create_guard_macro(prefix, filename);
    guard = "#ifndef " guard_macro ORS "#define " guard_macro;
    return guard;
}

function print_or_buf(line)
{
    if (global_buf_mode)
        global_buf = global_buf line ORS;
    else
        print line;
}

BEGIN {
    S_START = 0;
    S_FIRST_IFNDEF = 1;
    S_END = 2;

    if (!prefix)
        prefix = "";

    global_buf = "";
    global_buf_mode = 0;
    endif_line = "";

    ifndef_buf = "";
    # simple automata:
    # S_START            /^#ifndef\s.*/  -> S_FIRST_IFNDEF     /^#define\s.*/ -> S_END     // -> S_START
    # S_FIRST_IFNDEF     /^#define\s.*/ -> S_END
    # S_END              // -> S_END
    state = S_START;
}

/^#ifndef[\t ].*/ {
    if (state == S_START) {
        ifndef_buf = $0;
        state = S_FIRST_IFNDEF;
        next;
    }
}

/^#define[\t ].*/ {
    if (state == S_FIRST_IFNDEF) {
        #eat old guard and replace it with new one
        print_or_buf(create_guard(prefix, FILENAME));
        state = S_END;
        next;
    }
    if (state == S_START)
        state = S_END;
}

/^#endif[\t ].*/ {
    #flush old buffer and whatever was in endif line
    if (global_buf_mode) {
        print endif_line;
        printf global_buf;
        global_buf = "";
    }
    endif_line = $0;
    global_buf_mode = 1;
    next;
}

{
    if (state == S_END || state == S_START)
        print_or_buf($0);
    if (state == S_FIRST_IFNDEF) {
        print_or_buf(ifndef_buf);
        print_or_buf($0);
        state = S_END;
    }
}

END {
    if (global_buf_mode) {
        #it's last endif so do not print full endif_line
        print "#endif";
        printf global_buf; #flush whatever was after last endif
    }
}

#!/bin/awk
#
# Copyright (c) 2014, Michal Policht. This file is dually licensed under terms of
# either WTFPL or BEER-WARE LICENSE. You may obtain the copy of WTFPL or BEER-WARE
# LICENSE in nearest drug store.
#
# THIS SOFTWARE IS ADDICTED TO DRUGS AND ALCOHOL AND IS A SUSPECT OF DRIVING UNDER THE
# INFLUENCE OF ALCOHOL. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS NOT" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS AND APPLICATION AND
# ACCOMODATION FOR A PARTICULAR PURPOSE AND MANY OTHERS NOT MENTIONED, NOR EVEN
# INVENTED WARRANTIES ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
# OR CONTRIBUTORS BE LIABLE FOR ANY TEMPORAL, PERMANENT, UNFORTUNATE, UNLUCKY,
# LUCKY, DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, ACCIDENTAL, MENTAL,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION;
# OR BEING KILLED BY A LIGHTNING; OR FLOOD; OR FIRE; OR ANTEATER (INCLUDING BUT NOT
# LIMITED TO, GIANT ANTEATER, ANTEATER WHO LIES ON A SOFA; ANTEATER WHO LISTENS HEAVY
# METAL MUSIC; ANTEATER WHO ATE HUMAN BEFORE) OR ANOTHER BEING.) HOWEVER CAUSED AND ON
# ANY, ANY THEORY OF LIABILITY, CONSPIRACY THEORY, ANY THEORY OF RELATIVITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, ARISING IN ANY WAY OUT OF THE SUN, EVEN IF PROMISSED
# WISHED OR ASSURED SUCH DAMAGE. THIS SOFTWARE CAN REALLY DO TERRIBLE THINGS.
#
# Script updates C/C++ include guards with macros based on header path (including directory structure).
# It seeks for "#ifndef, #define" pair and substitutes first encountered pair with generated values.
# Script does not distinguish include guards from other macros, so each HEADER FILE MUST CONTAIN
# INCLUDE GUARD AS ITS FIRST "#ifndef, #define" MACRO PAIR. Script only updates include guards;
# if file does not contain something that looks like include guard to the script it will not add one.
# Script will also eat everything after last "#endif", presumably include guard comment. It's a stupid
# beast, so always backup your data before use.
#
# usage: awk [-v prefix="PREFIX"] -f awkgward.awk header
# example: awk -v prefix="PROJECT_" -f awkgward.awk file.hpp
#
# parameters:
# header - header file.
#
# required tools: awk.

function create_guard_macro(prefix, filename)
{
    # just in case, double single underscores
    double_underscore = toupper(filename);
    gsub(/_/, "__", double_underscore);
    # \/ = slash (/), \. = dot (.), \\ = backslash (\) - replace them with underscore.
    gsub(/[\/\.\\]/, "_", double_underscore);
    return prefix double_underscore;
}

function create_guard(prefix, filename)
{
    guard_macro = create_guard_macro(prefix, filename);
    guard = "#ifndef " guard_macro ORS "#define " guard_macro;
    return guard;
}

function print_or_buf(line)
{
    if (global_buf_mode)
        global_buf = global_buf line ORS;
    else
        print line;
}

BEGIN {
    S_START = 0;
    S_FIRST_IFNDEF = 1;
    S_END = 2;

    if (!prefix)
        prefix = "";

    global_buf = "";
    global_buf_mode = 0;
    endif_line = "";

    ifndef_buf = "";
    # simple automata:
    # S_START            /^#ifndef\s.*/  -> S_FIRST_IFNDEF     /^#define\s.*/ -> S_END     // -> S_START
    # S_FIRST_IFNDEF     /^#define\s.*/ -> S_END
    # S_END              // -> S_END
    state = S_START;
}

/^#ifndef[[:space:]].*/ {
    if (state == S_START) {
        ifndef_buf = $0;
        state = S_FIRST_IFNDEF;
        next;
    }
}

/^#define[[:space:]].*/ {
    if (state == S_FIRST_IFNDEF) {
        #eat old guard and replace it with new one
        print_or_buf(create_guard(prefix, FILENAME));
        state = S_END;
        next;
    }
    if (state == S_START)
        state = S_END;
}

/^#endif[[:space:]].*/ {
    #flush old buffer and whatever was in endif line
    if (global_buf_mode) {
        print endif_line;
        printf global_buf;
        global_buf = "";
    }
    endif_line = $0;
    global_buf_mode = 1;
    next;
}

{
    if (state == S_END || state == S_START)
        print_or_buf($0);
    if (state == S_FIRST_IFNDEF) {
        print_or_buf(ifndef_buf);
        print_or_buf($0);
        state = S_END;
    }
}

END {
    if (global_buf_mode) {
        #it's last endif so do not print full endif_line
        print "#endif";
        printf global_buf; #flush whatever was after last endif
    }
}


addr_utils.awk		-- added code to upcase 1st letter of words in street -- deals el Camino, de las Pulgas, etc

function AU_parse(options, addr, result, states, states_long, towns, st_types, dirs, st_ords,   nf, ary, i, f_st, f_twn, nf2, ary2, b1, e1, name, street, quals, town, state, work) {

	result["status"] = "B"
	result["emsg"  ] = ""
	result["name"  ] = ""
	result["street"] = ""
	result["quals" ] = ""
	result["town"  ] = ""
	result["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	# Some geocoders are US only so the last field is the state or state zip
	if(options["rply"]){
		if(!options["us_only"]){
			if(ary[nf] == "United States of America")
				nf--
			else{
				result["emsg"] = "not.US"
				return 1
			}
		}
	}

	if(!options["rply"]){
		# As a convenience to the user, given that nearly all the addresses
		# are in small set of nearby towns, the minimal query address can be
		#
		#	street, TOWN
		#
		# where TOWN is an abbreviation for town, state. Example: PA -> Palo Alto, CA
		if(nf < 2){
			result["emsg"] = "short.addr"
			return 1
		}
		# expand town if needed
		if(ary[nf] in towns){
			nf2 = split(towns[ary[nf]], ary2, ",")
			if(nf2 != 2){
				result["emsg"] = "bad.town"
				return  1
			}
			for(i = 1; i <= nf2; i++){
				sub(/^  */, "", ary2[i])
				sub(/  *$/, "", ary2[i])
			}
			ary[nf] = ary2[1]
			ary[nf+1] = ary2[2]
			nf++
		}
	}

	# At this point, all addresses must have at least three fields: street, town, state
	if(nf < 3){
		result["emsg"] = "short.addr"
		return 1
	}

	# check for a valid state, ignoring zip code
	if(substr(ary[nf], 1, 2) in states)
		state = ary[nf]
	else if(ary[nf] in states_long)
		state = ary[nf]
	else{
		result["emsg"] = sprintf("bad.US.state.%s", ary[nf])
		return 1
	}

	town = ary[nf-1]

	# find the street.
	# street is 1) 1st elt of ary[1:nf-2] that begins w/number & ends w/st_type or 2) last elt of ary[1:nf-2] that begins w/number
	f_st = 0
	for(i = 1; i < nf - 1; i++){
		if(ary[i] ~ /^[1-9]/){
			nf2 = split(ary[i], ary2, /  */)
			if(ary2[nf2] in st_types){
				f_st = i
				break
			}
			f_st = i
		}
	}
	if(f_st == 0){
		result["emsg"] = "no.street.num"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to user supplied default value
	if(f_st == 1)
		name = options["no_name"]
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf
	if(nf - f_st > 2){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf - 1; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		result["emsg"] = "short.street"
		return 1
	}

	# If a word in street begins w/lower case, upper case it.  Convert, say el Camino to El Camino, etc
	for(i = 2; i <= nf2; i++){
		if(substr(ary2[i], 1, 1) ~ /^[a-z]/)
			ary2[i] = toupper(substr(ary2[i], 1, 1)) substr(ary2[i], 2)
	}

	# Street should be num [ dir ] str [ st ] [ dir ]
	street = ary2[1]
	if(options["do_subs"]){
		# This is amusing.  What is South Court? Is it S. Court or South Ct.  No idea so leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			# handle any leading direction
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				b1 = 3
			}else
				b1 = 2
			# detect trailing direction
			e1 = ary2[nf2] in dirs ? nf2-1 : nf2
			for(i = b1; i < e1; i++){
				if(i == b1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" " ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[e1] in st_types) ? (" " st_types[ary2[e1]]) : (" " ary2[e1]))
			# handle any trailing direction
			if(e1 < nf2)
				street = street " " dirs[ary2[nf2]]
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	if(options["rply"]){
		if(town in towns)
			town = towns[town]
	}

	result["status"] = "G"
	result["name"  ] = name
	result["street"] = street
	result["quals" ] = quals
	result["town"  ] = town
	result["state" ] = state

	return 0
}
function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}
function AU_match(options, ref, cand,   nc_fields, c_fields, nr_fields, r_fields, i, nc_rtab, c_rtab, nr_rtab, r_rtab, nr_nwords, r_nwords, nc_nwords, c_nwords) {

	if(options["verbose"]){
		printf("ref.name    = %s\n", ref["name"]) > "/dev/stderr"
		printf("ref.street  = %s\n", ref["street"]) > "/dev/stderr"
		printf("ref.town    = %s\n", ref["town"]) > "/dev/stderr"
		printf("ref.state   = %s\n", ref["state"]) > "/dev/stderr"
		printf("cand.name   = %s\n", cand["name"]) > "/dev/stderr"
		printf("cand.street = %s\n", cand["street"]) > "/dev/stderr"
		printf("cand.town   = %s\n", cand["town"]) > "/dev/stderr"
		printf("cand.state  = %s\n", cand["state"]) > "/dev/stderr"
	}

	cand["emsg"] = ""
	if(cand["state"] != ref["state"]){
		if(options["ign_zip"]){
			if(substr(cand["state"], 1, 2) != substr(ref["state"], 1, 2)){
				cand["emsg"] = "states.ign_zip.diff"
				return 0
			}
		}else{
			cand["emsg"] = "states.diff"
			return 0
		}
	}

	if(cand["town"] != ref["town"]){
		cand["emsg"] = "towns.diff"
		return 0
	}

	# street numbers and/or ranges, etc is 1st field
	nc_fields = split(cand["street"], c_fields, /  */)
	nr_fields = split(ref["street"], r_fields, /  */)

	# check that non-number parts of streets agree
	if(nc_fields != nr_fields){
		cand["emsg"] = "num.st.fields.diff"
		return 0
	}
	for(i = 2; i <= nc_fields; i++){
		if(c_fields[i] != r_fields[i]){
			cand["emsg"] = sprintf("st.field.%d.diff|%s|%s", i, c_fields[i], r_fields[i])
			return 0
		}
	}

	nc_rtab = AU_get_rtab(c_fields[1], c_rtab)
	if(nc_rtab == 0){
		cand["emsg"] = "nc_rtab.is.zero"
		return 0
	}

	nr_rtab = AU_get_rtab(r_fields[1], r_rtab)
	if(nr_rtab == 0){
		cand["emsg"] = "nr_rtab.is.zero"
		return 0
	}

	if(!AU_rtabs_intersect(nc_rtab, c_rtab, nr_rtab, r_rtab)){
		cand["emsg"] = "rtab.no.ovlp"
		return 0
	}

	# deal with names
	if(cand["name"] == ref["name"]){
		return 3
	}else if(cand["name"] == ""){	# this is an unannotated address
		return 1
	}else if(ref["name"] == options["no_name"]){	# many destinations are just street, town, state
		return 1
	}else{	# check if 1st words match; but no matter what this is a match
		nc_nwords = split(cand["name"], c_nwords, /  */)
		nr_nwords = split(ref["name"], r_nwords, /  */)
		return c_nwords[1] == r_nwords[1] ? 2 : 1
	}
}
function AU_get_rtab(street, rtab,   n_rtab, i, nw, work, l_pfx, e_work2) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number

			if(rtab[i,1] > rtab[i,2]){
				# Is this a "delta" range where the 2d value shows only the different address suffix?
				# If so try to fix it
				# For example 945-51 would be expanded to 945-951 which is good, but
				# 945-37 would be expanded to 945-937 which is still wrong.
				l_pfx = length(work[1]) - length(work[2])
				e_work2 = ((l_pfx > 0) ? ((substr(work[1], 1, l_pfx) work[2])) : work[2]) + 0
				# did we fix it?
				if(rtab[i, 1] > e_work2){
					printf("ERROR: bad range 1st precedes 2nd: %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
					return 0
				}else
					rtab[i, 2] = e_work2
			}

			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}
function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}
function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}

function AU_parse(options, addr, result, states, states_long, towns, st_types, dirs, st_ords,   nf, ary, i, f_st, f_twn, nf2, ary2, b1, e1, name, street, quals, town, state, work) {

	result["status"] = "B"
	result["emsg"  ] = ""
	result["name"  ] = ""
	result["street"] = ""
	result["quals" ] = ""
	result["town"  ] = ""
	result["state" ] = ""

	nf = split(addr, ary, ",")
	for(i = 1; i <= nf; i++){
		sub(/^ */, "", ary[i])
		sub(/ *$/, "", ary[i])
	}

	# Some geocoders are US only so the last field is the state or state zip
	if(options["rply"]){
		if(!options["us_only"]){
			if(ary[nf] == "United States of America")
				nf--
			else{
				result["emsg"] = "not.US"
				return 1
			}
		}
	}

	# TODO: move this to a separate function, may AU_expand()
	if(!options["rply"]){
		# As a convenience to the user, given that nearly all the addresses
		# are in small set of nearby towns, the minimal query address can be
		#
		#	street, TOWN
		#
		# where TOWN is an abbreviation for town, state. Example: PA -> Palo Alto, CA
		if(nf < 2){
			result["emsg"] = "short.addr"
			return 1
		}
		# expand town if needed
		if(ary[nf] in towns){
			nf2 = split(towns[ary[nf]], ary2, ",")
			if(nf2 != 2){
				result["emsg"] = "bad.town"
				return  1
			}
			for(i = 1; i <= nf2; i++){
				sub(/^  */, "", ary2[i])
				sub(/  *$/, "", ary2[i])
			}
			ary[nf] = ary2[1]
			ary[nf+1] = ary2[2]
			nf++
		}
	}

	# At this point, all addresses must have at least three fields: street, town, state
	if(nf < 3){
		result["emsg"] = "short.addr"
		return 1
	}

	# check for a valid state, ignoring zip code
	if(substr(ary[nf], 1, 2) in states)
		state = ary[nf]
	else if(ary[nf] in states_long)
		state = ary[nf]
	else{
		result["emsg"] = sprintf("bad.US.state.%s", ary[nf])
		return 1
	}

	town = ary[nf-1]

	# find the street.
	# street is 1) 1st elt of ary[1:nf-2] that begins w/number & ends w/st_type or 2) last elt of ary[1:nf-2] that begins w/number
	f_st = 0
	for(i = 1; i < nf - 1; i++){
		if(ary[i] ~ /^[1-9]/){
			nf2 = split(ary[i], ary2, /  */)
			if(ary2[nf2] in st_types){
				f_st = i
				break
			}
			f_st = i
		}
	}
	if(f_st == 0){
		result["emsg"] = "no.street.num"
		return 1
	}

	# The name is what precedes f_st.  if f_st == 1, set name to user supplied default value
	if(f_st == 1)
		name = options["no_name"]
	else{
		name = ary[1]
		for(i = 2; i < f_st; i++)
			name = name ", " ary[i]
	}

	# Street qualifiers are those fields, if any, that follow f_st and precede nf
	if(nf - f_st > 2){
		quals = ary[f_st+1]
		for(i = f_st+2; i < nf - 1; i++)
			quals = quals ", " ary[i]
	}else
		quals = ""

	# clean up street
	nf2 = split(ary[f_st], ary2, /  */)
	if(nf2 < 2){
		result["emsg"] = "short.street"
		return 1
	}

	# Street Hacks: Probably should just compare ignore case but not ready for that step yet
	# change el Camino X to El Camino X
	# change el Monte X  to El Monte X
	if(options["do_subs"]){
		if(nf2 > 3){
			if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Camino")
				ary2[nf2-2] = "El"
			else if(ary2[nf2-2] == "el" && ary2[nf2-1] == "Monte")
				ary2[nf2-2] =  "El"
		}
	}

	# Street should be num [ dir ] str [ st ] [ dir ]
	street = ary2[1]
	if(options["do_subs"]){
		# This is amusing.  What is South Court? Is is S. Court or South Ct.  No idea so leave such streets in long form
		if(nf2 == 3 && (ary2[2] in dirs) && (ary2[3] in st_types)){
			for(i = 2; i <= nf2; i++)
				street = street " " ary2[i]
		}else{
			# handle any leading direction
			if(ary2[2] in dirs){
				street = street " " dirs[ary2[2]]
				b1 = 3
			}else
				b1 = 2
			# detect trailing direction
			e1 = ary2[nf2] in dirs ? nf2-1 : nf2
			for(i = b1; i < e1; i++){
				if(i == b1)
					street = street ((ary2[i] in st_ords) ? (" " st_ords[ary2[i]]) : (" " ary2[i]))
				else
					street = street " " ary2[i]
			}
			street = street ((ary2[e1] in st_types) ? (" " st_types[ary2[e1]]) : (" " ary2[e1]))
			# handle any trailing direction
			if(e1 < nf2)
				street = street " " dirs[ary2[nf2]]
		}
	}else{
		for(i = 2; i <= nf2; i++)
			street = street " " ary2[i]
	}

	# TODO: this belongs in a separate function
	if(options["rply"]){
		if(town in towns)
			town = towns[town]
	}

	result["status"] = "G"
	result["name"  ] = name
	result["street"] = street
	result["quals" ] = quals
	result["town"  ] = town
	result["state" ] = state

	return 0
}
function AU_get_addr_data(addr_info, key, data,   k, keys, nk, i) {

	n_data = 0
	for(k in addr_info){
		nk = split(k, keys, SUBSEP)
		if(keys[1] == key){
			data[keys[2]] = addr_info[k]
			n_data++
		}
	}
	return n_data
}
function AU_match(options, ref, cand,   nc_fields, c_fields, nr_fields, r_fields, i, nc_rtab, c_rtab, nr_rtab, r_rtab, nr_nwords, r_nwords, nc_nwords, c_nwords) {

	if(options["verbose"]){
		printf("ref.name    = %s\n", ref["name"]) > "/dev/stderr"
		printf("ref.street  = %s\n", ref["street"]) > "/dev/stderr"
		printf("ref.town    = %s\n", ref["town"]) > "/dev/stderr"
		printf("ref.state   = %s\n", ref["state"]) > "/dev/stderr"
		printf("cand.name   = %s\n", cand["name"]) > "/dev/stderr"
		printf("cand.street = %s\n", cand["street"]) > "/dev/stderr"
		printf("cand.town   = %s\n", cand["town"]) > "/dev/stderr"
		printf("cand.state  = %s\n", cand["state"]) > "/dev/stderr"
	}

	cand["emsg"] = ""
	if(cand["state"] != ref["state"]){
		if(options["ign_zip"]){
			if(substr(cand["state"], 1, 2) != substr(ref["state"], 1, 2)){
				cand["emsg"] = "states.ign_zip.diff"
				return 0
			}
		}else{
			cand["emsg"] = "states.diff"
			return 0
		}
	}

	if(cand["town"] != ref["town"]){
		cand["emsg"] = "towns.diff"
		return 0
	}

	# street numbers and/or ranges, etc is 1st field
	nc_fields = split(cand["street"], c_fields, /  */)
	nr_fields = split(ref["street"], r_fields, /  */)

	# check that non-number parts of streets agree
	if(nc_fields != nr_fields){
		cand["emsg"] = "num.st.fields.diff"
		return 0
	}
	for(i = 2; i <= nc_fields; i++){
		if(c_fields[i] != r_fields[i]){
			cand["emsg"] = sprintf("st.field.%d.diff|%s|%s", i, c_fields[i], r_fields[i])
			return 0
		}
	}

	nc_rtab = AU_get_rtab(c_fields[1], c_rtab)
	if(nc_rtab == 0){
		cand["emsg"] = "nc_rtab.is.zero"
		return 0
	}

	nr_rtab = AU_get_rtab(r_fields[1], r_rtab)
	if(nr_rtab == 0){
		cand["emsg"] = "nr_rtab.is.zero"
		return 0
	}

	if(!AU_rtabs_intersect(nc_rtab, c_rtab, nr_rtab, r_rtab)){
		cand["emsg"] = "rtab.no.ovlp"
		return 0
	}

	# deal with names
	if(cand["name"] == ref["name"]){
		return 3
	}else if(cand["name"] == ""){	# this is an unannotated address
		return 1
	}else if(ref["name"] == options["no_name"]){	# many destinations are just street, town, state
		return 1
	}else{	# check if 1st words match; but no matter what this is a match
		nc_nwords = split(cand["name"], c_nwords, /  */)
		nr_nwords = split(ref["name"], r_nwords, /  */)
		return c_nwords[1] == r_nwords[1] ? 2 : 1
	}
}
function AU_get_rtab(street, rtab,   n_rtab, i, nw, work, l_pfx, e_work2) {

	n_rtab = split(street, rtab, ";")
	for(i = 1; i <= n_rtab; i++){
		nw = split(rtab[i], work, "-")
		if(nw > 1){
			rtab[i, 1] = work[1] + 0	# force number
			rtab[i, 2] = work[2] + 0	# force number

			if(rtab[i,1] > rtab[i,2]){
				# Is this a "delta" range where the 2d value shows only the different address suffix?
				# If so try to fix it
				# For example 945-51 would be expanded to 945-951 which is good, but
				# 945-37 would be expanded to 945-937 which is still wrong.
				l_pfx = length(work[1]) - length(work[2])
				e_work2 = ((l_pfx > 0) ? ((substr(work[1], 1, l_pfx) work[2])) : work[2]) + 0
				# did we fix it?
				if(rtab[i, 1] > e_work2){
					printf("ERROR: bad range 1st precedes 2nd: %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
					return 0
				}else
					rtab[i, 2] = e_work2
			}

			rtab[i, "rng"] = 1
			if(rtab[i, 1] % 2 != rtab[i, 2] % 2){
				printf("ERROR: bad range: odd/even %d-%d\n", rtab[i, 1], rtab[i, 2]) > "/dev/stderr"
				return 0
			}
		}else{
			rtab[i] = work[1] + 0	# force number
			rtab[i, "rng"] = 0
		}
	}
	return n_rtab
}
function AU_rtabs_intersect(n_rtab1, rtab1, n_rtab2, rtab2,   i, j) {

	# the most common case: number v number
	if(n_rtab1 == 1 && !rtab1[1, "rng"] && n_rtab2 == 1 && !rtab2[1, "rng"])
		return rtab1[1] == rtab2[1]

	for(i = 1; i <= n_rtab1; i++){
		for(j = 1; j <= n_rtab2; j++){
			if(!rtab1[i, "rng"]){
				if(!rtab2[j, "rng"]){	# number v number
					if(rtab1[i] == rtab2[j])
						return 1
				}else{			# number v range
					if(rtab1[i] >= rtab2[j, 1] && rtab1[i] <= rtab2[j, 2])
						return 1
				}
			}else if(!rtab2[j, "rng"]){	# range v number
				if(rtab2[j] >= rtab1[i, 1] && rtab2[j] <= rtab1[i, 2])
					return 1
			}else{				# range v range
				if(!(rtab1[i, 2] < rtab2[j, 1] || rtab1[i, 1] > rtab2[j, 2]))
					return 1
			}
		}
	}
	return 0
}
function AU_rtab_dump(file, n_rtab, rtab,   i) {

	printf("n_rtab = %d {\n", n_rtab) > file
	for(i = 1; i <= n_rtab; i++){
		if(rtab[i, "rng"])
			printf("\trtab[%d, 1-2] = %d-%d\n", i, rtab[i, 1], rtab[i, 2]) > file
		else
			printf("\trtab[%d     ] = %d\n", i, rtab[i]) > file
	}
	printf("}\n") > file
}


parsegaspriceshtml.awk: Count nested <tr> for inGasPriceRecord. (Fix for UDF.)

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

// {
   sub("^.*
]*>","")
   sub("
.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && / <a href="/ {
   gsub("","Marathon")
   gsub("","BP")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   gsub("","Marathon")
   gsub("","UDF")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   # print "name", i, name[i], $0
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   # print "area", i, name[i], $0
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /
/ {
   inGasPriceRecord += 1
}

inGasPriceRecord && /<\/tr>/ {
   inGasPriceRecord -= 1
   if (!inGasPriceRecord) {
      i ++
      # print "EOR"
      # print $0
      inGasPriceRecord=False
   }
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}

BEGIN {
   False=0
   True=!False
   i=0
   inGasPriceRecord=False
   expectName=False
}

// {
   sub("^.*
]*>","")
   sub("
.*$","")
   # print
   price[i]=$0
}

inGasPriceRecord && /
/ {
   sub("^[ ]*
","")
   sub("
[ ]*$","")
   sub("&","\\&")
   address[i,ai[i]]=$0
   sub("^[ ]*","",address[i,ai[i]])
   # print "address["ai[i]"]="address[i,ai[i]]
   ai[i]++
}

inGasPriceRecord && /
/ {
   expectName=True
}

inGasPriceRecord && expectName && / <a href="/ {
   gsub("","Marathon")
   gsub("","BP")
   gsub("","Thorntons")
   gsub("","Thorntons")
   gsub("","Certified")
   gsub("","Marathon")
   # gsub("","UDF")
   expectName=False
   sub("[ ]*$","")
   sub("^.*>","")
   sub("&","\\&")
   name[i]=$0
   # print
}

inGasPriceRecord && / class="p_area">/ {
   sub("^.* class=\"p_area\">","")
   sub("[ ]*$","")
   area[i]=$0
   # print
}

inGasPriceRecord && /<a href="\/Profile[.]aspx[?]member=/ {
   sub("^.* ",",")
   sub("<.*$","")
   split($0,foo,",")
   reporter[i]=foo[1]
   time[i]=foo[2]
   timeAgo[i]=foo[3]
   # print
}

inGasPriceRecord && /<\/tr>/ {
   i ++
   # print "EOR"
   # print $0
   inGasPriceRecord=False
}

END {
   for (j=0;j<i && True;j++) {
      if (False) {
         print j
         print price[j]
         print name[j]
         print address[j,0]
         print area[j]
         print reporter[j]
         print time[j]
         print timeAgo[j]
      }

      delimiter="\t"
      print price[j] delimiter name[j] delimiter address[j,0] delimiter area[j] delimiter reporter[j] delimiter time[j] delimiter timeAgo[j]
   }
}


Do not use sort -s in lexicon.awk

Do not use sort -s in lexicon.awk

#!/usr/bin/awk -f
BEGIN {
    FS  = "\t";
}
TOLOWER {
    $3 = tolower($3);
}
{
    gsub(/, /, ORS, $3);
    print $3 | "sort --parallel=$(nproc) -S1G -u";
}

#!/usr/bin/awk -f
BEGIN {
    FS  = "\t";
}
TOLOWER {
    $3 = tolower($3);
}
{
    gsub(/, /, ORS, $3);
    print $3 | "sort --parallel=$(nproc) -S1G -us";
}


SImplify first awk script

SImplify first awk script

# Generate series URLs, Titles, Number of Seasons from MHz "Browse" page
#
# INVOCATION:
#       curl -s https://mhzchoice.vhx.tv/series https://mhzchoice.vhx.tv/series?page=2 \
#           | awk -v URL_FILE=$URL_FILE -v TITLE_FILE=$TITLE_FILE \
#           -v NUM_SEASONS_FILE=$NUM_SEASONS_FILE -f getMHzFrom-browsePage.awk
#
# INPUT:
#       A French Village
#   ---
#
6 seasons

#
# OUTPUT:
#       $URL_FILE, $TITLE_FILE, $NUM_SEASONS_FILE

/https:\/\/mhzchoice.vhx.tv\/.*strong title/ {
    split($0,fld,"\"")
    URL = fld[2]
    TITLE = fld[4]

    print URL "/" >> URL_FILE

    # Canonicalize Title
    gsub (/'/,"'",TITLE)
    if (match (TITLE, /^The /)) {
        TITLE = substr(TITLE, 5) ", The"
    }
    print TITLE >> TITLE_FILE
}

/h4 class="transparent.*media-count/ {
    sub (/.*media-count'>/,"")
    sub (/ season.*$/,"")
    print $0 >> NUM_SEASONS_FILE
}

# Given the output of:
#    curl -s https://mhzchoice.vhx.tv/series https://mhzchoice.vhx.tv/series?page=2
# create lists of titles, urls and seasons to be pasted into a spreadsheet
#
# e.g.
# A French Village
#
6 seasons

#
#
# Invoked with file parameters as follows:
# awk -v URL_FILE=$URL_FILE -v TITLE_FILE=$TITLE_FILE \
#     -v SEASONS_FILE=$SEASONS_FILE -f fetchMHz-series.awk

/https:\/\/mhzchoice.vhx.tv\/.*strong title/ {
    sub (/.*href="/,"")
    endTitle = match($0,/"/ ) -1
    print substr($0,1,endTitle) "/" >> URL_FILE

    sub (/.*strong title="/,"")
    sub (/".*$/,"")
    gsub (/'/,"'")
    if (match ($0, /^The /)) {
        $0 = substr($0, 5) ", The"
    }
    print >> TITLE_FILE
}

/h4 class="transparent.*media-count/ {
    sub (/.*media-count'>/,"")
    sub (/ season.*$/,"")
    print $0 >> SEASONS_FILE
}


Translators/BingTranslator: fix bingTTSUrl() (new API, #218)


Translators/BingTranslator: support phonetics (transliteration)


Allow testing of back content

Allow testing of back content

git-svn-id: 293778eaa97c8c94097d610b1bd5133a8f478f36@23463 4f837ed2-42f5-46e7-a7a5-fa17313484d4

#!/usr/bin/awk -f
# cat *.tdb | tdbout -t publisher,plugin,publisher:info[contract],year,publisher:info[tester],status,publisher:info[back]

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  end_year = 0
  incontract = 0
  if ($7 > 1900) {
    test_year = $7
  } else {
    test_year = $3
  }
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  if (end_year >= test_year) {
    incontract = 1
  }

  #if ((substr($4,1,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$3,$7,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      k[pn] = $7
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$3,$7,$4]++
    c[$1,lp2,$3,$7,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tContr\tBack\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, back, year, tester, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], k[i], d[i], r[i], b[p[i],n[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],t[i],k[i],d[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],t[i],k[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tBack\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}

#!/usr/bin/awk -f
# input: publisher\tplugin\tcontract_year\tyear\ttester\tstatus

BEGIN {
  FS="\t"
  pn = 0
}

{
  # add a loop to add line only if ending year is gt or eq to contract year
  end_year = 0
  incontract = 0
  if (length($4) > 3) {
    end_year = substr($4,length($4)-3,4)
  }
  if (end_year >= $3) {
    incontract = 1
  }

  #if ((substr($4,1,4) >= $3) || ((length($4) == 9) && (substr($4,6,4) >= $3))) {
  if (incontract == 1) {
      nn = split($2,na,/\./)
      lp2 = na[nn]
    if (!(($1,lp2,$3,$4) in b)) {
      p[pn] = $1
      n[pn] = lp2
  #    n[pn] = $2
      t[pn] = $3
      d[pn] = $4
      r[pn] = $5
      pn++
    }
    b[$1,lp2,$3,$4]++
    c[$1,lp2,$3,$4,$6]++
    x[$6]++
    tt++
  }
}

END {
  s[0] = "expected"
  s[1] = "exists"
  s[2] = "manifest"
  s[3] = "wanted"
  s[4] = "crawling"
  s[5] = "testing"
  s[6] = "notReady"
  s[7] = "released"
  s[8] = "down"
  s[9] = "superseded"
  s[10] = "zapped"
  sn = 11

  sc[0] = "expe"
  sc[1] = "exis"
  sc[2] = "mani"
  sc[3] = "want"
  sc[4] = "craw"
  sc[5] = "test"
  sc[6] = "notR"
  sc[7] = "rele"
  sc[8] = "down"
  sc[9] = "supe"
  sc[10] = "zapp"
  scn = 11

  #print out header
  printf "Publisher\tPlugin\tContr\tYear\tT\tTotal"
  for (j = 0 ; j < scn ; j++) {
    if (x[s[j]] > 0) {
    printf "\t%s", sc[j]
    }
  }
  printf "\n"

  #print out publisher, plugin, contract year, year, total aus
  for (i = 0 ; i < pn ; i++) {
    printf "%s\t%s\t%s\t%s\t%s\t%d", p[i], n[i], t[i], d[i], r[i], b[p[i],n[i],t[i],d[i]]
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0){
      if (c[p[i],n[i],t[i],d[i],s[j]] == 0) {
      printf "\t.."
    } else {
        printf "\t%d", c[p[i],n[i],t[i],d[i],s[j]]
      }
    }
    }
    printf "\n"
  }
    #print out bottom line sums
    printf "Publisher\tPlugin\tContr\tYear\tT\t%d", tt
    for (j = 0 ; j < sn ; j++) {
      if (x[s[j]] > 0) {
        printf "\t%d", x[s[j]]
      }
    }

  printf "\n"
}


improve utility when not processing dipper ingest files

#! /usr/bin/awk -f
# deltadot.awk <encumbant.gv> <candidate.gv>
# Use to illuminate the commonality and differences between dot files created
# by 'ntriples2dot.awk' but might work for other dot files
#
# To ignore counts
# deltadot.awk -v"NOCOUNTS=1" <encumbant.gv> <candidate.gv>
#
# expects two graphviz dot digraph files;
# by convention "old"  then "new"
# for color to mean;  orange lost in "new" and blue gained in "new"
# with simple node names ('C-lang' variable naming conventions)
# edge's unadorned predicate label
# concludes with a space then count in parens
#
# For example:
# NODE1 -> NODE2 [label=<curie:term ! label (42)>];
#
# where the " ! label " is optional, but the lable _could_ have parens
#
# filter for edges in each file associate the count with the edge
# if an edge is common between the files
# then change its "count" to the difference. (second minus first)
# otherwise leave count as is
# UNLESS there is a parameter NOCOUNTS not equal to 0
# in which case set all counts the same
# this is for use when the counts have no meaning
# such as with random samples

# associate the count with an edge
function parse(str, arr){
    # can't count on label being paren-free anymore
    n = match($0, / [(][[:digit:]]*[)]>];$/)
    if(n>0)
        arr[substr($0,1,n)] = substr($0, n+2, length($0)-n-5)
	else
		arr[$1 " " $2 " " $3 " "]++
}

# strip metadata from filename
function de_path_ext(pth){
	split(pth, a, "/")
	return substr(a[length(a)], 1, index(a[length(a)], ".")-1)
}

BEGIN { NOCOUNTS=0}

# collect the edges from both files
NR==FNR && /^[^ ]+ -> [^ ]+ / {
	label1 = FILENAME;
	parse($0, edge1)
}
NR!=FNR && /^[^ ]+ -> [^ ]+ / {
	label2 = FILENAME;
	parse($0, edge2)
}

END{
	for(e in edge1){
		if(!(e in edge2)) edges[e "(" edge1[e] ")>, color=\"orange\"];"];
		else {
			# edge is in both graphs
			if(NOCOUNTS)
				edges[e "()>, color=\"black\"];"]
			else {
				diff = edge2[e] - edge1[e];
				if(diff < 0)
					# diff = "" diff "" #  breaks svg
					edges[e "(" diff ")>, color=\"pink\"];"]
				else
					edges[e "(" diff ")>, color=\"black\"];"]
			}
			delete edge2[e]
		}
	}
	# only edges not in first file
	for(e in edge2)
		edges[e "(" edge2[e] ")>, color=\"blue\"];"];  # expects label=<tag> to close

	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	print "label=\"" de_path_ext(label1) " <=> " de_path_ext(label2) "\""

	n = asorti(edges,output)
	for(i=1;i<=n;i++ ){print output[i]}
	print "\n}\n"
}

#! /usr/bin/awk -f
# deltadot.awk <encumbant.gv> <candidate.gv>
# Use to illuminate the commonality and differences between dot files created
# by 'ntriples2dot.awk' but might work for other dot files
#
# To ignore counts
# deltadot.awk -v"NOCOUNTS=1" <encumbant.gv> <candidate.gv>
#
# expects two graphviz dot digraph files;
# by convention "old"  then "new"
# for color to mean;  orange lost in "new" and blue gained in "new"
# with simple node names ('C-lang' variable naming conventions)
# edge's unadorned predicate label
# concludes with a space then count in parens
#
# For example:
# NODE1 -> NODE2 [label=<curie:term ! label (42)>];
#
# where the " ! label " is optional, but the lable _could_ have parens
#
# filter for edges in each file associate the count with the edge
# if an edge is common between the files
# then change its "count" to the difference. (second minus first)
# otherwise leave count as is
# UNLESS there is a parameter NOCOUNTS not equal to 0
# in which case set all counts the same
# this is for use when the counts have no meaning
# such as with random samples

# associate the count with an edge
function parse(str, arr){
    # can't count on label being paren-free anymore
    n = match($0, / [(][[:digit:]]*[)]>];$/)
    if(n>0)
        arr[substr($0,1,n)] = substr($0, n+2, length($0)-n-5)
}

# strip metadata from filename
function de_path_ext(pth){
	split(pth, a, "/")
	return substr(a[length(a)], 1, index(a[length(a)], ".")-1)
}

BEGIN { NOCOUNTS=0}

# collect the edges from both files
NR==FNR && /^[^ ]+ -> [^ ]+ / {
	label1 = FILENAME;
	parse($0, edge1)
}
NR!=FNR && /^[^ ]+ -> [^ ]+ / {
	label2 = FILENAME;
	parse($0, edge2)
}

END{
	for(e in edge1){
		if(!(e in edge2)) edges[e "(" edge1[e] ")>, color=\"orange\"];"];
		else {
			# edge is in both graphs
			if(NOCOUNTS)
				edges[e "()>, color=\"black\"];"]
			else {
				diff = edge2[e] - edge1[e];
				if(diff < 0)
					# diff = "" diff "" #  breaks svg
					edges[e "(" diff ")>, color=\"pink\"];"]
				else
					edges[e "(" diff ")>, color=\"black\"];"]
			}
			delete edge2[e]
		}
	}
	# only edges not in first file
	for(e in edge2)
		edges[e "(" edge2[e] ")>, color=\"blue\"];"];

	print "digraph {"
	print "rankdir=LR;"
	print "charset=\"utf-8\";"
	print "LITERAL [shape=record];"
	print "labelloc=\"t\";"
	print "label=\"" de_path_ext(label1) " <=> " de_path_ext(label2) "\""

	n = asorti(edges,output)
	for(i=1;i<=n;i++ ){print output[i]}
	print "\n}\n"
}


INTEGRATION: CWS i18n28 (1.3.178); FILE MERGED 2006/12/08 15:53:16 er 1.3.178.1: #i64400# add getBreakIteratorRules_

#!/usr/bin/gawk -f
# Usage: gawk -f linkermapfile-check.awk *.map *.xml
# Order of *.map *.xml is important, otherwise all symbols are reported to be
# missing.
# Checks if all symbols of all locale data are present in the symbol scoping
# linker mapfiles.  Any output indicates a missing symbol, ../localedata.cxx is
# grep'ed to indicate the library to which mapfile the symbol should be added.
# Author: Eike Rathke <er@openoffice.org>

BEGIN {
    bAnyMissing = 0
    file = ""
    nMap = 0
    nMaps = 0
    nPublics = 0
    sPublic[nPublics++] = "getAllCalendars_"
    sPublic[nPublics++] = "getAllCurrencies_"
    sPublic[nPublics++] = "getAllFormats_"
    sPublic[nPublics++] = "getBreakIteratorRules_"
    sPublic[nPublics++] = "getCollationOptions_"
    sPublic[nPublics++] = "getCollatorImplementation_"
    sPublic[nPublics++] = "getContinuousNumberingLevels_"
    sPublic[nPublics++] = "getForbiddenCharacters_"
    sPublic[nPublics++] = "getLCInfo_"
    sPublic[nPublics++] = "getLocaleItem_"
    sPublic[nPublics++] = "getOutlineNumberingLevels_"
    sPublic[nPublics++] = "getReservedWords_"
    sPublic[nPublics++] = "getSearchOptions_"
    sPublic[nPublics++] = "getTransliterations_"
    sPublic[nPublics++] = "getIndexAlgorithm_"
    sPublic[nPublics++] = "getUnicodeScripts_"
    sPublic[nPublics++] = "getFollowPageWords_"
}

file != FILENAME {
    file = FILENAME
    if ( file ~ /\.map$/ )
    {
        sMapFile[nMaps] = file
        nMap = nMaps
        ++nMaps
    }
    else if ( file ~ /\.xml$/ )
    {
        bOut = 0
        n = split( file, arr, /[:\\\/.]/ )
        locale = arr[n-1]
        for ( i=0; i<nPublics; ++i )
        {
            symbol = sPublic[i] locale ";"
            bFound = 0
            for ( j=0; j<nMaps && !bFound; ++j )
            {
                if ( sSymbol[j,symbol] )
                    bFound = 1
            }
            if ( !bFound )
            {
                if ( !bOut )
                {
                    search = "\"" locale "\""
                    while ( !bOut && (getline <"../localedata.cxx") > 0 )
                    {
                        if ( $0 ~ search )
                        {
                            bOut = 1
                            print "../localedata.cxx says this should go into: " $0
                        }
                    }
                    close( "../localedata.cxx" )
                    if ( !bOut )
                        print "../localedata.cxx doesn't indicate to which lib this belongs to:"
                    bOut = 1
                }
                print symbol
            }
        }
        if ( bOut)
        {
            printf("\n")
            bAnyMissing = 1
        }
        nextfile
    }
    else
        nextfile
}

# only reached if .map file encountered, read in symbols
{
    if ( $1 ~ /;$/ )
        sSymbol[nMap,$1] = 1
}

END {
    if ( !bAnyMissing )
        print "All good." >>"/dev/stderr"
}

#!/usr/bin/gawk -f
# Usage: gawk -f linkermapfile-check.awk *.map *.xml
# Order of *.map *.xml is important, otherwise all symbols are reported to be
# missing.
# Checks if all symbols of all locale data are present in the symbol scoping
# linker mapfiles.  Any output indicates a missing symbol, ../localedata.cxx is
# grep'ed to indicate the library to which mapfile the symbol should be added.
# Author: Eike Rathke <er@openoffice.org>

BEGIN {
    bAnyMissing = 0
    file = ""
    nMap = 0
    nMaps = 0
    nPublics = 0
    sPublic[nPublics++] = "getAllCalendars_"
    sPublic[nPublics++] = "getAllCurrencies_"
    sPublic[nPublics++] = "getAllFormats_"
    sPublic[nPublics++] = "getCollationOptions_"
    sPublic[nPublics++] = "getCollatorImplementation_"
    sPublic[nPublics++] = "getContinuousNumberingLevels_"
    sPublic[nPublics++] = "getForbiddenCharacters_"
    sPublic[nPublics++] = "getLCInfo_"
    sPublic[nPublics++] = "getLocaleItem_"
    sPublic[nPublics++] = "getOutlineNumberingLevels_"
    sPublic[nPublics++] = "getReservedWords_"
    sPublic[nPublics++] = "getSearchOptions_"
    sPublic[nPublics++] = "getTransliterations_"
    sPublic[nPublics++] = "getIndexAlgorithm_"
    sPublic[nPublics++] = "getUnicodeScripts_"
    sPublic[nPublics++] = "getFollowPageWords_"
}

file != FILENAME {
    file = FILENAME
    if ( file ~ /\.map$/ )
    {
        sMapFile[nMaps] = file
        nMap = nMaps
        ++nMaps
    }
    else if ( file ~ /\.xml$/ )
    {
        bOut = 0
        n = split( file, arr, /[:\\\/.]/ )
        locale = arr[n-1]
        for ( i=0; i<nPublics; ++i )
        {
            symbol = sPublic[i] locale ";"
            bFound = 0
            for ( j=0; j<nMaps && !bFound; ++j )
            {
                if ( sSymbol[j,symbol] )
                    bFound = 1
            }
            if ( !bFound )
            {
                if ( !bOut )
                {
                    search = "\"" locale "\""
                    while ( !bOut && (getline <"../localedata.cxx") > 0 )
                    {
                        if ( $0 ~ search )
                        {
                            bOut = 1
                            print "../localedata.cxx says this should go into: " $0
                        }
                    }
                    close( "../localedata.cxx" )
                    if ( !bOut )
                        print "../localedata.cxx doesn't indicate to which lib this belongs to:"
                    bOut = 1
                }
                print symbol
            }
        }
        if ( bOut)
        {
            printf("\n")
            bAnyMissing = 1
        }
        nextfile
    }
    else
        nextfile
}

# only reached if .map file encountered, read in symbols
{
    if ( $1 ~ /;$/ )
        sSymbol[nMap,$1] = 1
}

END {
    if ( !bAnyMissing )
        print "All good." >>"/dev/stderr"
}


Add missing field

Add missing field

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields from XML
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
}

# if needed for debugging record placment, replace "/nosuchrecord/" below
/nosuchrecord/ {
    print ""
    print NR " - " $0
    for ( i = 1; i <= NF; i++ ) {
        if ($i ~ /\/us\// )
            print "field " i " = " $i
    }
}

# Example input lines
#
# <URL>/us/movie/70_Glorious_Years_13550</URL><Title>70 Glorious Years</Title><Subtitle></Subtitle>\
# <Duration>60</Duration><Year>1996</Year><Rating>TV-PG</Rating>
#
# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
#
# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>

/^<URL>\/us\// {
    URL = $3
    showTitle = $7

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

/^<URL>\/us\/episode\/|^<URL>\/us\/movie\// {
    Title = $11
    Duration = $15
    Year = $19
    Rating = $23
    Description = $27

    # Convert duration from minutes to HMS
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
}

/^<URL>\/us\/movie\// {
    # Extract movie sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_[[:digit:]]*$/) {
        sortkey = sprintf ("M%05d", fld[nflds])
        printf \
            ("%s %s - mv\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description)
    }
}

/^<URL>\/us\/episode\// {
    # Extract episode sortkey from URL
    nflds = split (URL,fld,"_")
    seasonNumber = fld[nflds - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nflds - 1]
    sub (/E/,"",episodeNumber)
    sortkey = sprintf ("S%02dE%02d", seasonNumber, episodeNumber)

    # Use X plus shownumber as sortkey for specials
    if (URL ~ /_Special_[[:digit:]]*$/)
        sortkey = sprintf ("X%05d", fld[nflds])

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "^Season " seasonNumber ", "
    sub (episodePrefix, "", Title)

    if (episodeNumber != "") {
        printf \
            ("%s %s - ep\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description)
    }
}

/^<URL>\/us\/season\// {
    seasonNumber = $11
    seasonYear = $15
    seasonEpisodes = $19
    seasonDescription = $23

    if (seasonNumber != "") {
        printf \
            ("%s S%02d - sn\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\n", \
                 showTitle, seasonNumber, URL, showTitle, seasonNumber, \
                 seasonEpisodes, seasonYear, seasonRating, seasonDescription)
    }
}

BEGIN {
    FS="[<>]"
}

# Eliminate unneeded constant fields from XML
/^<item>/ {
    sub (/^<item>/,"")
    sub (/<\/item>$/,"")
    gsub (/<value>/,"")
    gsub (/<\/value>/,"")
}

# if needed for debugging record placment, replace "/nosuchrecord/" below
/nosuchrecord/ {
    print ""
    print NR " - " $0
    for ( i = 1; i <= NF; i++ ) {
        if ($i ~ /\/us\// )
            print "field " i " = " $i
    }
}

# Example input lines
#
# <URL>/us/movie/70_Glorious_Years_13550</URL><Title>70 Glorious Years</Title><Subtitle></Subtitle>\
# <Duration>60</Duration><Year>1996</Year><Rating>TV-PG</Rating>
#
# <URL>/us/episode/Vera_S3_E1_13502</URL><Title>Vera</Title><Subtitle>Season 3, Castles in The Air</Subtitle>\
# <Duration>88 min</Duration><Year>2013</Year><Rating>TV-MA</Rating>
#
# <URL>/us/season/Vera_S8_15720</URL><Title>Vera</Title><SeasonNumber>8</SeasonNumber>\
# <Year>2018</Year><NumEpisodes>4</NumEpisodes>

/^<URL>\/us\// {
    URL = $3
    showTitle = $7

    # Titles starting with "The" should not sort based on "The"
    if (match (showTitle, /^The /)) {
        showTitle = substr(showTitle, 5) ", The"
    }
}

/^<URL>\/us\/episode\/|^<URL>\/us\/movie\// {
    Title = $11
    Duration = $15
    Year = $19
    Rating = $23
    Description = $27

    # Convert duration from minutes to HMS
    secs = 0
    mins = Duration % 60
    hrs = int(Duration / 60)
    HMS = sprintf ("%02d:%02d:%02d", hrs, mins, secs)
}

/^<URL>\/us\/movie\// {
    # Extract movie sortkey from URL
    nflds = split (URL,fld,"_")
    if (URL ~ /_[[:digit:]]*$/) {
        sortkey = sprintf ("M%05d", fld[nflds])
        printf \
            ("%s %s - mv\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description)
    }
}

/^<URL>\/us\/episode\// {
    # Extract episode sortkey from URL
    nflds = split (URL,fld,"_")
    seasonNumber = fld[nflds - 2]
    sub (/S/,"",seasonNumber)
    episodeNumber = fld[nflds - 1]
    sub (/E/,"",episodeNumber)
    sortkey = sprintf ("S%02dE%02d", seasonNumber, episodeNumber)

    # Use X plus shownumber as sortkey for specials
    if (URL ~ /_Special_[[:digit:]]*$/)
        sortkey = sprintf ("X%05d", fld[nflds])

    # Get rid of perfunctory "Season <n>, " from episode title
    episodePrefix = "^Season " seasonNumber ", "
    sub (episodePrefix, "", Title)

    if (episodeNumber != "") {
        printf \
            ("%s %s - ep\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, %s, %s\"\)\t\t%s\t%s\t%s\t%s\n", \
             showTitle, sortkey, URL, showTitle, sortkey, \
             Title, HMS, Year, Rating, Description)
    }
}

/^<URL>\/us\/season\// {
    seasonNumber = $11
    seasonYear = $15
    seasonEpisodes = $19

    if (seasonNumber != "") {
        printf \
            ("%s S%02d - sn\t=HYPERLINK(\"https://www.britbox.com%s\";\"%s, Season %d\"\)\t%s\t\t%s\t%s\t%s\n", \
                 showTitle, seasonNumber, URL, showTitle, seasonNumber, \
                 seasonEpisodes, seasonYear, seasonRating, seasonDescription)
    }
}


Fix an output formatting bug in the rewriter script.

BEGIN{
        now = systime()
        format = "%Y/%m/%d %H:%M:%S"

}
{
        split($1, DATE, "/")
        split($2, TIME, ":")
        t = mktime(DATE[1] " " DATE[2] " " DATE[3] " " TIME[1] " " TIME[2] " " TIME[3])
        if (delta == "") {
                delta = now - t
        }
        out = strftime(format, t + delta)
        for (i = 3; i <= NF; i++) {
                out = out OFS $i
        }
        print out
}

BEGIN{
        now = systime()
        format = "%Y/%m/%d %H:%M:%S"

}
{
        split($1, DATE, "/")
        split($2, TIME, ":")
        $1 = ""
        $2 = ""
        t = mktime(DATE[1] " " DATE[2] " " DATE[3] " " TIME[1] " " TIME[2] " " TIME[3])
        if (delta == "") {
                delta = now - t
        }
        print strftime(format, t + delta) $0
}