Make progress on parsing

Make progress on parsing

BEGIN {
    FS = "\t"
}

{
    web_scraper_start_url = $1
    Title = $2
    Years = $3
    Cast = $4
    Directors = $5
    Writers = $6
    person = ""
    role = ""
    num_episodes = ""
    episode_years = ""

    if (Cast == "" && Directors == "" && Writers == "")
        next

    if (NR == 1)
        next

    if (Cast != "") {
        type = "Actor"
        gsub (/[[:blank:]]{3,99}\(uncredited\)/," (uncredited)",Cast)
        gsub (/[[:blank:]]{3,99}\/ \.\.\./," / ...",Cast)
        uncredited = sub (/\(uncredited\) ?/,"",Cast)
        if (uncredited != 0)
            type = type " (uncredited)"
        nflds1 = split (Cast,fld,/[[:blank:]]{3,99}/)
        person = fld[1]
        role = fld[3]
        sub (/\302\240/,"",role)
        episodes = fld[4]
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Directors != "") {
        type = "Director"
        nflds1 = split (Directors,fld,/[[:blank:]]{3,99}/)
        person = fld[1]
        episodes = fld[3]
        uncredited = sub (/\(uncredited\) /,"",episodes)
        if (uncredited != 0)
            type = type " (uncredited)"
        gsub (/[()]/,"",episodes)
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Writers != "") {
        type = "Writer"
        nflds1 = split (Writers,fld,/[[:blank:]]{3,99}/)
        person = fld[1]
        parens = fld[3]
        nparens = split (parens,fld,/[()]/)
        # print "==> parens = " parens > "/dev/stderr"
        for ( i = 2; i < nparens-1; i+=2 ) {
            if (fld[i] != " ") {
                # print "==> i = " i " - |"fld[i] "|" > "/dev/stderr"
                type = type " (" fld[i] ")"
            }
        }
        episodes = fld[nparens-1]
        if (episodes !~ /episodes?/) {
            type = type " (" episodes ")"
        } else {
            nflds2 = split (episodes,fld,/,/)
            num_episodes = fld[1]
            sub (/ episodes?/,"",num_episodes)
            episode_years = fld[2]
            sub (/ /,"",episode_years)
        }
    }

    printf ("%s %s | %s | %s | %s | %s | %s\n",Title,Years,person,type,role,num_episodes,episode_years)

}

BEGIN {
    FS = "\t"
}

{
    web_scraper_start_url = $1
    Title = $2
    Years = $3
    Cast = $4
    Directors = $5
    Writers = $6

    if (Cast == "" && Directors == "" && Writers == "")
        next

    if (NR == 1)
        next

    if (Cast != "") {
        gsub (/[[:blank:]]{3,99}\(uncredited\)/," (uncredited)",Cast)
        nflds1 = split (Cast,fld,/[[:blank:]]{3,99}/)
        type = "Actor"
        person = fld[1]
        role = fld[3]
        sub (/\302\240/,"",role)
        episodes = fld[4]
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Directors != "") {
        nflds1 = split (Directors,fld,/[[:blank:]]{3,99}/)
        type = "Director"
        role = ""
        person = fld[1]
        episodes = fld[3]
        gsub (/[()]/,"",episodes)
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    if (Writers != "") {
        nflds1 = split (Writers,fld,/[[:blank:]]{3,99}/)
        type = "Writer"
        role = ""
        person = fld[1]
        episodes = fld[3]
        gsub (/[()]/,"",episodes)
        nflds2 = split (episodes,fld,/,/)
        num_episodes = fld[1]
        sub (/ episodes?/,"",num_episodes)
        episode_years = fld[2]
        sub (/ /,"",episode_years)
    }

    printf ("%d - %s %s\t%s\t%s\t%s\t%s\t%s\n",nflds1,Title,Years,person,type,role,num_episodes,episode_years)

}


set execute perm

set execute perm

#! /usr/bin/gawk -f
# spote_count.awk
# RDF ntriple counts for:
# subject predicate object triples entity
#
# spote_count.awk -v"RELEASE=YYYYMM" path/file.nt

function sum(a, t){
    for(i in a)
        t += a[i]
    return t
}

BEGIN{
    if(! RELEASE)
        "date +%Y%m" | getline RELEASE;
    FS = " "
    archive = "https://archive.monarchinitiative.org/"
}

# iri objects
/^.*> .$/ {
    s[$1]++;p[$2]++;o[$3]++
}

# entities  (typed subjects)
$2 == "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>" {
    e[$1]++
}

# literal objects
/^.*" .$/ {
    s[$1]++;p[$2]++;
    # not counting literals for now
    # $1="";$2="";l[$0]++
}

# not counting bnodes for now
# blank node subject
#/^[^ ]*\.well-known\/genid/ {
#    bs[$1]++
#    b[$1]++
#}

# blank node object
#/ <[^ ]*\.well-known\/genid/ {
#    bo[$3]++
#    b[$3]++
#}

END{
    split(FILENAME, path, "/")
    graphname = path[length(path)]
    # still has extension '.nt' might need to truncate

    subject_iri = "<" archive RELEASE "/rdf/" graphname ">"
    subject_curie = "MonarchArchive:" RELEASE "/rdf/" graphname

    # turtle like output for now b/c _datasets output is always turtle
    # not sure if or how this will get intregrated.

    print "@prefix MonarchArchive: <https://archive.monarchinitiative.org/> ."
    print "@prefix void: <http://rdfs.org/ns/void#> ."

    print subject_curie " a dctypes:Dataset ;"  # redundant typing
    print "\tvoid:distinctSubjects " length(s)  " / " sum(s) " ;"
    print "\tvoid:distinctObjects "  length(o)  " / " sum(o) " ;"
    print "\tvoid:entities "         length(e)  " / " sum(e) " ;"
    print "\tvoid:properties "       length(p)  " / " sum(p) " ;"

    # not aware of what predicates to use. (have not researched)
    # print "\texample:literals "      length(l)  " / " sum(l) " ;"
    # print "\texample:blanknode "     length(b)  " / " sum(b) " ;"
    # print "\texample:bnodesub "      length(bs) " / " sum(bs) " ;"
    # print "\texample:bnodeobj "      length(bo) " / " sum(bo) " ;"

    print "\tvoid:triples " FNR " ."

}


geo_utis.awk	-- new funcs for determining handed ness of polygons

function GU_pi() {
	return 4*atan2(1, 1)
}
function GU_d2r(d) {
	return (d+0)*GU_pi()/180.0
}
function GU_r2d(r) {
	return (r+0)*180.0/GU_pi()
}
function GU_is_righthanded(count, first, x, y,   area, p, x1, y1, x2, y2) {
	area = 0
	for(p = 0; p < p_count[poly]; p++){
		x1 = x[p_first[poly] + p]
		y1 = y[p_first[poly] + p]
		x2 = x[p_first[poly] + p + 1]
		y2 = y[p_first[poly] + p + 1]
		area += GU_d2r(x2 - x1) * (2.0 + sin(GU_d2r(y1)) + sin(GU_d2r(y2)))
	}
	return area >= 0
}
function GU_pr_header(title, n_points) {

	printf("{\n")
	printf("\"geojson\": {\n")
	printf("\"type\": \"FeatureCollection\",\n")
	printf("\"metadata\": {\n")
	printf("  \"generated\": \"%s\",\n", strftime("%Y%m%dT%H%M%S%Z"))
	printf("  \"title\": \"%s\",\n", title)
	printf("  \"count\": %d\n", n_points)
	printf("},\n")
	printf("\"features\": [\n")
}
function GU_pr_trailer() {
	printf("]\n")
	printf("}\n")
	printf("}\n")
}
function GU_find_pgroups(start, count, longs, lats, pg_starts, pg_counts,   n_pgroups, l_geo, geo, i) {

	n_pgroups = 1
	pg_starts[n_pgroups] = start
	pg_counts[n_pgroups] = 1
	l_geo[1] = longs[start]
	l_geo[2] = lats[start]
	for(i = 1; i < count; i++){
		geo[1] = longs[start + i]
		geo[2] = longs[start + i]
		if(GU_geo_equal(geo, l_geo)){
			pg_counts[n_pgroups]++
		}else{
			n_pgroups++
			pg_starts[n_pgroups] = start + i
			pg_counts[n_pgroups] = 1
		}
		l_geo[1] = geo[1]
		l_geo[2] = geo[2]
	}

	return n_pgroups
}
function GU_geo_equal(g1, g2) {
	return g1[1] == g2[1] && g1[2] == g2[2]
}
function GU_geo_isnull(g) {
	return g[1] == "" || g[2] == ""
}
function GU_geo_adjust(long, lat, n, long_adj, lat_adj,   PI, RAD, i, a) {

	PI = 4.0 * atan2(1, 1)
	RAD = 0.0001

	long_adj[1] = 0
	lat_adj[1] = 0
	if(n > 1){
		a = 2.0*PI/(n-1)
		for(i = 2; i <= n; i++){
			long_adj[i] = RAD * sin(a * (i-2))
			lat_adj[i] = RAD * cos(a * (i-2))
		}
	}
}
function GU_mk_point(file, color, style, long, lat, title, last,   h_color, h_style) {
 	h_color = color != "."
	h_style = style != "."
 	printf("{\n")									> file
 	printf("  \"type\": \"Feature\",\n")						> file
 	printf("  \"geometry\": {")							> file
 	printf("\"type\": \"Point\", ")							> file
 	printf("\"coordinates\": [%.5f, %.5f]", long, lat)				> file
 	printf("},\n")									> file
 	printf("  \"properties\": {\n")							> file
 	printf("    \"title\": \"%s\",\n", title)					> file
	if(h_color)
		printf("    \"marker-color\": \"%s\"%s\n", color, h_style ? "," : "")	> file
	if(h_style)
		printf("    %s\n", style)						> file
	printf("  }\n")									> file
	printf("}%s\n", !last ? "," : "")						> file
}
function GU_mk_line(file, lng_1, lat_1, lng_2, lat_2, last) {
	printf("{\n")																> file
	printf("  \"type\": \"Feature\",\n")													> file
	printf("  \"geometry\": {\"type\": \"LineString\", \"coordinates\": [[%.5f, %.5f], [%.5f, %.5f]]}\n", lng_1, lat_1, lng_2, lat_2)	> file
	printf("}%s\n", !last ? "," : "")													> file
}
function GU_gc_dist(lng_1, lat_1, lng_2, lat_2,   R_EARTH, D2R, ph1_1, phi_2, delta_phi, delta_lambda, a, c) {

	R_EARTH = 3.959e3	# mean radius in miles
	D2R = (2*atan2(0, -1))/360.0
	phi_1 = lat_1 * D2R
	phi_2 = lat_2 * D2R
	delta_phi = (lat_2 - lat_1) * D2R
	delta_lambda = (lng_2 - lng_1) * D2R
	a = sin(delta_phi/2.0) * sin(delta_phi/2.0) + cos(phi_1) * cos(phi_2) * sin(delta_lambda/2.0) * sin(delta_lambda/2.0)
	c = 2 * atan2(sqrt(a), sqrt(1.0-a))
	return R_EARTH * c
}


fix copyright headers

fix copyright headers

# Copyright (c) AT&T Corp. 1994, 1995.
# This code is licensed by AT&T Corp.  For the
# terms and conditions of the license, see
# http://www.research.att.com/orgs/ssr/book/reuse

BEGIN	{ s = ARGV[1]; gsub (".*/", "", s); gsub("\\.","_",s); printf("char *%s[] = {\n",s); }
/^#/	{ print $0; next; }
		{ gsub("\\\\","\\\\",$0); printf("\"%s\",\n",$0); }
END		{ printf("(char*)0 };\n"); }


disable __gmon_start__ symbol

disable __gmon_start__ symbol

/^[ 	]*\.endp/	 { need_endp = 1 }
/^[ 	]*\.end/	 { need_end = 1 }
/^[ 	]*\.align/ { if($2 > max) max = $2; }

END {
    if(need_endp)
    {
	print "#define END_INIT .endp _init";
	print "#define END_FINI .endp _fini";
    } else if(need_end)
    {
	print "#define END_INIT .end _init";
	print "#define END_FINI .end _fini";
    }
    else
    {
	print "#define END_INIT";
	print "#define END_FINI";
    }
    if(max)
	print "#define ALIGN .align", max;
    else
	print "#define ALIGN";

    print "#include <libc-symbols.h>";
    #print "weak_extern (__gmon_start__)";
}

/^[ 	]*\.endp/	 { need_endp = 1 }
/^[ 	]*\.end/	 { need_end = 1 }
/^[ 	]*\.align/ { if($2 > max) max = $2; }

END {
    if(need_endp)
    {
	print "#define END_INIT .endp _init";
	print "#define END_FINI .endp _fini";
    } else if(need_end)
    {
	print "#define END_INIT .end _init";
	print "#define END_FINI .end _fini";
    }
    else
    {
	print "#define END_INIT";
	print "#define END_FINI";
    }
    if(max)
	print "#define ALIGN .align", max;
    else
	print "#define ALIGN";

    print "#include <libc-symbols.h>";
    print "weak_extern (__gmon_start__)";
}