import AndroidDataPrivacy.Flow as Flow
import AndroidDataPrivacy.Result as Result
import AndroidDataPrivacy.Applications.AppDefault as AppDefault

type = ''
info = ''

urls = []

partialURLs = ['https://youtubei.googleapis.com']

userAgents = []

partialUserAgents = ['com.google.android.youtube']

def checkBehavior(flow, results):
	if (flow.requestType == 'GET'):
		analyzeGetRequest(flow, results)
	if (flow.requestType == 'POST'):
		analyzePostRequest(flow, results)
	if (flow.requestType == 'HEAD'):
		analyzePostRequest(flow, results)

def analyzeGetRequest(flow, results):
	checkGetURL(flow, results)
	checkRequestHeaders(flow, flow.requestHeaders, results)
	AppDefault.checkRequestHeadersDefault(flow, flow.requestHeaders, results)
	checkResponseHeaders(flow, flow.responseHeaders, results)
	AppDefault.checkResponseHeadersDefault(flow, flow.responseHeaders, results)
	AppDefault.analyzeGetRequestDefault(flow, results)

def analyzePostRequest(flow, results):
	checkPostURL(flow, results)
	checkRequestHeaders(flow, flow.requestHeaders, results)
	AppDefault.checkRequestHeadersDefault(flow, flow.requestHeaders, results)
	checkResponseHeaders(flow, flow.responseHeaders, results)
	AppDefault.checkResponseHeadersDefault(flow, flow.responseHeaders, results)
	AppDefault.analyzePostRequestDefault(flow, results)

def analyzeHeadRequest(flow, results):
	checkHeadURL(flow, results)
	checkRequestHeaders(flow, flow.requestHeaders, results)
	AppDefault.checkRequestHeadersDefault(flow, flow.requestHeaders, results)
	checkResponseHeaders(flow, flow.responseHeaders, results)
	AppDefault.checkResponseHeadersDefault(flow, flow.responseHeaders, results)
	AppDefault.analyzeHeadRequestDefault(flow, results)

def checkRequestHeaders(flow, headers, results):
	if ('User-Agent' in headers.keys()):
		if (headers['User-Agent'][:26] == 'com.google.android.youtube' and flow.source == ''):
			flow.source = 'Youtube'

	if ('x-goog-device-auth' in headers.keys()):
		type = 'System Info: Google API Device Authentication'
		info = headers['x-goog-device-auth']
		results.append(Result.Result(flow, type, info))

	if ('x-goog-visitor-id' in headers.keys()):
		type = 'User Info: Google Visitor ID'
		info = headers['x-goog-visitor-id']
		results.append(Result.Result(flow, type, info))

def checkResponseHeaders(flow, headers, results):
	return None

def checkGetURL(flow, results):
	if (flow.url.find('youtube.com') > -1):
		if (flow.requestContent.find('plid:') > -1):
			type = 'Youtube PLID'
			info = AppDefault.findFormEntry(flow.requestContent, 'plid')
			results.append(Result.Result(flow, type, info))

		if (flow.requestContent.find('cos:') > -1):
			type = 'System Info: OS'
			info = AppDefault.findFormEntry(flow.requestContent, 'cos')
			results.append(Result.Result(flow, type, info))

		if (flow.requestContent.find('docid:') > -1):
			type = 'Youtube Video ID'
			info = AppDefault.findFormEntry(flow.requestContent, 'docid')
			results.append(Result.Result(flow, type, info))
		elif (flow.requestContent.find('video_id:') > -1):
			type = 'Youtube Video ID'
			info = AppDefault.findFormEntry(flow.requestContent, 'video_id')
			results.append(Result.Result(flow, type, info))
		elif (flow.requestContent.find('content_v:') > -1):
			type = 'Youtube Video ID'
			info = AppDefault.findFormEntry(flow.requestContent, 'content_v')
			results.append(Result.Result(flow, type, info))

	if (flow.url.find('https://www.googleadservices.com/pagead/conversion') == 0):
		type = 'System Info: Youtube App Version'
		info = AppDefault.findFormEntry(flow.requestContent, 'appversion')
		results.append(Result.Result(flow, type, info))

		type = 'System Info: Android Version'
		info = AppDefault.findFormEntry(flow.requestContent, 'osversion')
		results.append(Result.Result(flow, type, info))

		type = 'User Info: Youtube Screen Opened'
		info = AppDefault.findFormEntry(flow.requestContent, 'data.screen_name')
		results.append(Result.Result(flow, type, info))

		type = 'User Info: Ad ID'
		info = AppDefault.findFormEntry(flow.requestContent, 'rdid')
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('upnphost/udhisapi.dll?content=uuid:') > -1):
		type = 'User Info: Youtube UUID'
		info = flow.requestContent[flow.requestContent.find('uuid:')+5:]
		info = info[:info.find('\n')]
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://www.youtube.com/csi_204') == 0):
		type = 'User Action: Youtube'
		info = AppDefault.findFormEntry(flow.requestContent, 'action')
		results.append(Result.Result(flow, type, info))

		type = 'System Info: Brand'
		info = AppDefault.findFormEntry(flow.requestContent, 'cbrand')
		results.append(Result.Result(flow, type, info))

		type = 'System Info: Model'
		info = AppDefault.findFormEntry(flow.requestContent, 'cmodel')
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://s.youtube.com/api/stats') == 0):
		if (flow.requestContent.find('state:') > -1):
			type = 'Youtube Video Status'
			info = AppDefault.findFormEntry(flow.requestContent, 'state')
			results.append(Result.Result(flow, type, info))

		if (flow.requestContent.find('referrer:') > -1):
			type = 'Youtube Video Referrer'
			info = AppDefault.findFormEntry(flow.requestContent, 'referrer')
			results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://www.youtube.com/api/stats/ads') == 0):
		type = 'Youtube Ad Video'
		info = AppDefault.findFormEntry(flow.requestContent, 'ad_v')
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://www.youtube.com/gen_204') == 0):
		type = 'Youtube Ad Video'
		info = AppDefault.findFormEntry(flow.requestContent, 'ad_vid')
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://suggestqueries.google.com/complete/search') == 0):
		type = 'User Action: Search Query'
		info = AppDefault.findFormEntry(flow.requestContent, 'q')
		results.append(Result.Result(flow, type, info))

		type = 'Youtube Search Suggestion'
		query = info
		if (len(info) > 0):
			for item in flow.responseContent.split('],['):
				info = item[item.find('\\\\u003e')+7:]
				info = info[:info.find('\\\\')]
				if (len(info) > 0):
					info = query + info
					results.append(Result.Result(flow, type, info))
		else:
			for item in flow.responseContent.split('}],['):
				info = item[item.find('youtube-android'):]
				info = info[info.find('\\\\u003d')+7:]
				info = info[:info.find('\\\\')]
				if (len(info) > 0):
					info = query + info
					results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://www.youtube.com/player_204') == 0):
		if (flow.requestContent.find('event:') > -1 and AppDefault.findFormEntry(flow.requestContent, 'event') == 'iv'):
			type = 'User Action: Youtube'
			info = 'Opened Video Info'
			results.append(Result.Result(flow, type, info))

def checkPostURL(flow, results):
	if (flow.url.find('https://youtubei.googleapis.com/youtubei') == 0 and flow.url.find('key=') > -1):
		type = 'User Info: Google API Key'
		info = flow.url[flow.url.find('key=')+4:]
		if (info.find('&') > -1):
			info = info[:info.find('&')]
		results.append(Result.Result(flow, type, info))

	if (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/search') == 0):
		type = 'User Action: Youtube Search'
		info = AppDefault.findJSONGroup(flow.requestContent, '16')
		info = info[info.find('4: ')+3:]
		info = info[:info.find('\n')]
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://www.youtube.com/error_204') == 0):
		source = 'Youtube Error'
		type = 'Youtube Error Message'
		info = AppDefault.findFormEntry(requestContent, 'exception.message')
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/browse/edit_playlist') == 0):
		flow.source = 'Youtube Playlist Edit'
		type = 'Youtube Video ID'
		if (flow.requestContent.find('2 {\n      2: ') > -1):
			info = flow.requestContent[flow.requestContent.find('2 {\n      2: '):]
			info = info[info.find('2: ')+3:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))
		elif (flow.requestContent.find('2 {\n      6: ') > -1):
			info = flow.requestContent[flow.requestContent.find('2 {\n      6: '):]
			info = info[info.find('17: ')+4:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

		if (flow.requestContent.find('        }\n      }\n    }\n    2 {') > -1):
			type = 'Youtube Playlist'
			info = flow.requestContent[flow.requestContent.find('        }\n      }\n    }\n    2 {')+40:]
			info = info[info.find('3: ')+3:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/browse') == 0):
		if (flow.requestContent.find('        }\n      }\n    }\n    2: ') > -1):
			type = 'User Action: Youtube Browsing'
			info = flow.requestContent[flow.requestContent.find('        }\n      }\n    }\n    2: ')+31:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/share/get_share_panel') == 0):
		type = 'User Action: Youtube'
		info = 'Opened share panel'
		results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/playlist/get_add_to_playlist') == 0):
		if (flow.requestContent.find('        }\n      }\n    }\n    2: ') > -1):
			type = 'User Action: Add Video to Playlist'
			info = flow.requestContent[flow.requestContent.find('        }\n      }\n    }\n    2: ')+31:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/playlist/create') == 0):
		if (flow.requestContent.find('        }\n      }\n    }\n    2: ') > -1):
			type = 'User Action: Create Playlist'
			info = flow.requestContent[flow.requestContent.find('        }\n      }\n    }\n    2: ')+31:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

	elif (flow.url.find('https://youtubei.googleapis.com/youtubei/v1/playlist/delete') == 0):
		if (flow.requestContent.find('        }\n      }\n    }\n    2: ') > -1):
			type = 'User Action: Delete Playlist'
			info = flow.requestContent[flow.requestContent.find('        }\n      }\n    }\n    2: ')+31:]
			info = info[:info.find('\n')]
			results.append(Result.Result(flow, type, info))

def checkHeadURL(flow, results):
	return None


origin = [37, 99, 73, 48, 47, 40, 40, 25, 99, 51]
newlist = sorted(origin)

def insert_sort(lst, i):
    low = 0
    high = len(lst)
    while low < high:
        mid = (low + high) // 2
        if lst[mid] < i:
            low = mid + 1
        else:
            high = mid
    lst.insert(low, i)

# for i in (40, 20, 41, 100):
#     insert_sort(newlist, i)
# print(newlist)

import bisect

def get_grade(score):
    breakpoints = [60, 70, 80, 90]
    grades = 'EDCBA'
    return grades[bisect.bisect(breakpoints, score)]

for x in (91, 82, 77, 65, 50, 60, 70):
    print('{}->{}'.format(x, get_grade(x)))


import re

class Solution:
    def solution(self, instructions):
        answer = 0

        for i in instructions:
            if i.isalpha():
                if i == "HALT":
                    return answer
                elif i == "LEFT":
                    answer -= 90
                elif i == "RIGHT":
                    answer += 90
            else:
                word = re.findall('[a-zA-z]+', i)
                number = re.findall("\d+", i)
                if word[0] == "LEFT":
                    answer -= int(number[0])
                elif word[0] == "RIGHT":
                    answer += int(number[0])
                elif i.startswith('T'):
                    answer += 180

        if answer < 0:
            answer += 360

        return answer

s1 = Solution()
result = s1.solution(["LEFT","LEFT", "LEFT"])
print(result)

'''
2/4 방위각 구하기
시간 제한 : 2초메모리 제한 : 256MB
문제 설명
로봇이 평면 위에서 다음과 같은 명령을 받아서 움직이고 있다.
LEFT – 왼쪽으로 90도 회전
RIGHT – 오른쪽으로 90도 회전
TURN AROUND – 반 바퀴 회전 (180도 회전)
LEFT X – 왼쪽으로 X도 회전 (X는 양의 정수)
RIGHT X – 오른쪽으로 X도 회전 (X는 양의 정수)
HALT – 명령어를 수행 중지, 앞으로 모든 명령을 수행하지 않는다

로봇의 명령어는 문자열의 배열 instructions 로 주어진다. 명령을 전부 수행 한 다음에 로봇의 방위각을 구하시오.
(방위각은 북쪽을 기존으로 로봇이 바라보는 각도이며, 방위각은 시계방향으로 측정이 된다. 즉, 로봇이 서쪽을 바라보고 있다면 방위각은 270이 된다.)

참고 / 제약 사항
명령어는 총 1개 이상, 50개 이하다.
각 명령어는 위에 언급한대로 유효한 명령어다
LEFT X, RIGHT X에서 1 <= X <= 179
테스트 케이스
instructions = ["RIGHT"]리턴(정답): 90
오른쪽으로 90도를 회전하면 방위각은 90이 된다.

instructions = ["LEFT","LEFT","TURN AROUND"]리턴(정답): 0
왼쪽으로 90도 회전하면 로봇은 서쪽을 바라보게 된다 (방위각 270). 다시 왼쪽으로 90도 회전하면 로봇은 남쪽을 바라보게 된다 (방위각 180). 마지막으로 반 바퀴를 회전하면 로봇은 북쪽을 바라보게 된다 (방위각 0).

instructions = ["LEFT 5","RIGHT 10","LEFT 15","RIGHT 20","LEFT 25","RIGHT 30","LEFT 35","RIGHT 40"]리턴(정답): 20
로봇은 왼쪽으로 5도 회전, 오른쪽으로 10도 회전 함으로써 결국 오른쪽으로 5도씩 4번 움직이게 된다.

instructions = ["RIGHT 59","RIGHT","RIGHT","HALT","LEFT","LEFT","LEFT"]리턴(정답): 239
네 번째 명령은 HALT다. 즉 다음에 등장하는 LEFT 명령어들은 실행되지 않는다.

instructions = ["TURN AROUND","HALT","LEFT 5","HALT","LEFT 5","HALT"]리턴(정답): 180
한번 이상의 HALT 명령이 주어질 수 있다.

'''


import re
from zzz.chapter03.knock20 import extract_text
from zzz.chapter03.knock25 import template
from zzz.chapter03.knock26 import del_mark

if __name__ == '__main__':
    filename = 'jawiki-country.json'
    text = extract_text(filename, 'イギリス')

    basic = template(text)

    pattern = r'(\'{3})(.*?)(\'{3})'
    basic = del_mark(basic, pattern, lambda x: x.group(2))
    # print(basic)

    # pattern = r'(\[\[.*?)(#\|.*?)(\]\])'
    # basic = del_mark(basic, pattern, 2)
    # print(basic)
    pattern = r'(\[\[)(.*?)(:?#.*?|)(\|)(.*?)(\]\])'                          # [[記事名|表示文字]], [[記事名#節名|表示文字]]
    basic = del_mark(basic, pattern, lambda x: x.group(5))
    pattern = r'(\[\[)(.*?)(\]\])'                                              # [[記事名]]
    basic = del_mark(basic, pattern, lambda x: x.group(2))

    for (key, value) in basic.items():
        print(key, value)


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.1.1
#   kernelspec:
#     display_name: Python (opioid_paper)
#     language: python
#     name: opioid_paper
# ---

# # Long Term Trends

# ## Load long term trends data

# +
import pandas as pd
import numpy as np
from ebmdatalab import bq

GBQ_PROJECT_ID = "620265099307"

q4 = """-- opioid long term data extraction
SELECT
  year,
  l.chem_substance,
  l.Is_LA,
  l.Is_High_LA,
  sum(itemsper1000) as items_per_1000,
  sum(quantityper1000) as quantity_per_1000,
  sum(quantityper1000*dose_per_unit*new_ome_multiplier) AS total_ome_per_1000,
  sum(Infl_corr_Cost_per1000) as cost_per_1000
FROM ebmdatalab.helen.trends_from_pca_final_2017 p
INNER JOIN (SELECT distinct drug_name, chem_substance, Is_LA, Is_High_LA, dose_per_unit, new_ome_multiplier FROM ebmdatalab.richard.opioid_converter) l on l.drug_name = p.drug_name

GROUP BY
  year,
  chem_substance,
  Is_LA,
  Is_High_LA"""

dfl = bq.cached_read(q4, csv_path="chemical_summary.zip").fillna(0)
dfl.head()
# -

# tidy data
dfl.loc[dfl["Is_High_LA"]!="TRUE","Is_High_LA"] = "Others"
dfl.loc[dfl["Is_High_LA"]=="TRUE","Is_High_LA"] = "High dose"
dfl.drop("Is_LA",axis=1).groupby(["year","Is_High_LA"]).sum()

# ## Summary results tables

# +
# Summary results table by year
tab = dfl.fillna(0)
tab.loc[tab["Is_High_LA"]=="High dose","OME per 1000_High Dose"] = tab["total_ome_per_1000"]
tab.loc[tab["Is_High_LA"]=="High dose","Items per 1000_High Dose"] = tab["items_per_1000"]
tab.loc[tab["Is_High_LA"]=="High dose","Cost per 1000_High Dose"] = tab["cost_per_1000"]

tab.loc[tab["Is_LA"]==True,"OME per 1000_Long Acting"] = tab["total_ome_per_1000"]
tab.loc[tab["Is_LA"]==True,"Items per 1000_Long Acting"] = tab["items_per_1000"]
tab.loc[tab["Is_LA"]==True,"Cost per 1000_Long Acting"] = tab["cost_per_1000"]

tab = tab.fillna(0)
tab = tab.groupby(["year"]).sum().drop(["Is_LA","quantity_per_1000"],axis=1)
tab = tab.rename(columns={"total_ome_per_1000":"OME per 1000_Total","items_per_1000":"Items per 1000_Total","cost_per_1000":"Cost per 1000_Total"})

tab["OME per 1000_% High Dose"] = (100*tab["OME per 1000_High Dose"]/tab["OME per 1000_Long Acting"])
tab["Items per 1000_% High Dose"] = (100*tab["Items per 1000_High Dose"]/tab["Items per 1000_Long Acting"])
tab["Cost per 1000_% High Dose"] = (100*tab["Cost per 1000_High Dose"]/tab["Cost per 1000_Long Acting"])

tab.columns = tab.columns.str.split('_',expand=True)
tab.sort_index(axis=1, ascending=False)

# +
# Summary by chemical substance
tab = dfl.copy().fillna(0)
tab.loc[tab["Is_High_LA"]=="High dose","OME per 1000_High Dose"] = tab["total_ome_per_1000"]
tab.loc[tab["Is_High_LA"]=="High dose","Items per 1000_High Dose"] = tab["items_per_1000"]
tab.loc[tab["Is_High_LA"]=="High dose","Cost per 1000_High Dose"] = tab["cost_per_1000"]

tab.loc[tab["Is_LA"]==True,"OME per 1000_Long Acting"] = tab["total_ome_per_1000"]
tab.loc[tab["Is_LA"]==True,"Items per 1000_Long Acting"] = tab["items_per_1000"]
tab.loc[tab["Is_LA"]==True,"Cost per 1000_Long Acting"] = tab["cost_per_1000"]

tab = tab.fillna(0)
tab = tab.groupby(["chem_substance"]).sum().drop(["year","Is_LA","quantity_per_1000"],axis=1).sort_values(by="total_ome_per_1000",ascending=False)
tab = tab.rename(columns={"total_ome_per_1000":"OME per 1000_Total","items_per_1000":"Items per 1000_Total","cost_per_1000":"Cost per 1000_Total"})

tab["OME per 1000_% High Dose"] = (100*tab["OME per 1000_High Dose"]/tab["OME per 1000_Long Acting"])
tab["Items per 1000_% High Dose"] = (100*tab["Items per 1000_High Dose"]/tab["Items per 1000_Long Acting"])
tab["Cost per 1000_% High Dose"] = (100*tab["Cost per 1000_High Dose"]/tab["Cost per 1000_Long Acting"])

tab.columns = tab.columns.str.split('_',expand=True)
tab.sort_index(axis=1, ascending=False).fillna(0).head()

# +
# Summary by chemical substance and year (OME only)
tab = dfl.fillna(0)
tab.loc[tab["Is_High_LA"]=="High dose","OME per 1000_High Dose"] = tab["total_ome_per_1000"]

tab.loc[tab["Is_LA"]==True,"OME per 1000_Long Acting"] = tab["total_ome_per_1000"]

grp = tab.loc[tab["year"]==2017]
grp = grp[["chem_substance","OME per 1000_Long Acting"]].groupby(["chem_substance"]).sum().fillna(0).sort_values(by="OME per 1000_Long Acting",ascending=False).reset_index()
grp["chemical"] = np.where(grp.index>5,"Other",grp.chem_substance)
grp = grp.drop(["OME per 1000_Long Acting"], axis=1)
tab2 = tab.reset_index().merge(grp,on="chem_substance")
tab2 = tab2[["year","chemical","total_ome_per_1000","OME per 1000_Long Acting","OME per 1000_High Dose"]].fillna(0)

tab2 = tab2.groupby(["year","chemical"]).sum().sort_values(by="total_ome_per_1000",ascending=False)
tab2 = tab2.rename(columns={"total_ome_per_1000":"OME per 1000_Total"})

tab2 = pd.DataFrame(tab2.stack()).reset_index().rename(columns={"level_2": 'measure', 0:"value"})
tab2 = tab2.set_index(["measure","year","chemical"]).unstack().fillna(0)
tab2["Total"] = tab2.sum(axis=1)
tab2
# -

# ## Plot data - all opioids

# +
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

sns.set_style("whitegrid",{'grid.color': '.9'})
sns.set_palette("tab20",n_colors=14)

dft = dfl.groupby("chem_substance")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dft.chem_substance.drop_duplicates())

grp = pd.DataFrame(sort_order).reset_index()
grp["chemical"] = np.where(grp.index>10,"Other",grp.chem_substance)
grp["other_flag"] = np.where(grp.index>10,0,1)
dfl2 = dfl.merge(grp,on="chem_substance")

dfs = dfl2.groupby(["other_flag","chemical"])["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"}).sort_values(by=["other_flag","total_ome_all_years"],ascending=False)
sort_order = pd.Series(dfs.chemical)

s=[(0,'(a)  Total prescriptions for all opioid-containing preparations','Prescriptions per 1000 population','items_per_1000'),
   (1,'(b)  Total Oral Morphine Equivalency (mg) for all opioid-containing preparations','Oral Morphine equivalency (mg) per 1000 population',"total_ome_per_1000"),
   (2,'(c)  Total cost of all opioid-containing preparations','Cost per 1000 population (2017 equivalent GBP)','cost_per_1000')]

fig = plt.figure(figsize=(15,20))

for i in s:
    ax = plt.subplot(3,1,i[0]+1)  # layout and position of subplot
    if i[0]==0:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==1:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==2:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    dfp.plot(ax=ax, kind='area',  linewidth=0)
    ax.set_xticks(np.arange(1998, 2017, step=1))
    ax.set_xlabel('Year', size="14")
    ax.set_ylabel(i[2], size="14")
    ax.tick_params(labelsize=12)
    ax.set_title(i[1], size="17")
    handles, labels = ax.get_legend_handles_labels()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    ax.legend(reversed(handles), reversed(labels),loc='center left', fontsize="12",bbox_to_anchor=(1, .62))

#plt.savefig("opioids_Figure1_revisedv2.pdf", transparent=True, dpi=300)
plt.show()

# -

# ## Plot data - long acting preparations

# +
la_df = dfl.loc[(dfl['Is_LA'] == True)]

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid",{'grid.color': '.9'})
sns.set_palette("Set3",n_colors=14)

dft = la_df.groupby("chem_substance")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dft.chem_substance.drop_duplicates())

grp = pd.DataFrame(sort_order).reset_index()
grp["chemical"] = np.where(grp.index>11,"Other",grp.chem_substance)
dfl2 = la_df.merge(grp,on="chem_substance")

dfs = dfl2.groupby("chemical")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dfs.chemical)

s=[(0,'(a)  Total prescriptions for long-acting opioid preparations','Prescriptions per 1000 population','items_per_1000'),
   (1,'(b)  Total Oral Morphine Equivalency (mg) for long-acting opioid preparations','Oral Morphine equivalency (mg) per 1000 population',"total_ome_per_1000"),
   (2,'(c)  Total cost of long-acting opioid preparations','Cost per 1000 population (2017 equivalent GBP)','cost_per_1000')]

fig = plt.figure(figsize=(15,20))

for i in s:
    ax = plt.subplot(3,1,i[0]+1)  # layout and position of subplot
    if i[0]==0:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==1:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==2:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    dfp.plot(ax=ax, kind='area',  linewidth=0)
    ax.set_xticks(np.arange(1998, 2017, step=1))
    ax.set_xlabel('Year', size="14")
    ax.set_ylabel(i[2], size="14")
    ax.tick_params(labelsize=12)
    ax.set_title(i[1], size="17")
    handles, labels = ax.get_legend_handles_labels()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    ax.legend(reversed(handles), reversed(labels),loc='center left', fontsize="12",bbox_to_anchor=(1, .62))

plt.show()

# -

# ## Plot data - High dose opioids

# +
high_df = dfl.loc[(dfl['Is_High_LA'] == "High dose")]

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid",{'grid.color': '.9'})
sns.set_palette("Set3",n_colors=14)

dft = high_df.groupby("chem_substance")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dft.chem_substance.drop_duplicates())

grp = pd.DataFrame(sort_order).reset_index()
grp["chemical"] = np.where(grp.index>11,"Other",grp.chem_substance)
dfl2 = high_df.merge(grp,on="chem_substance")

dfs = dfl2.groupby("chemical")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})

s=[(0,'(a)  Total prescriptions for high-dose, long-acting opioid preparations','Prescriptions per 1000 population','items_per_1000'),
   (1,'(b)  Total Oral Morphine Equivalency (mg) for high-dose, long-acting opioid preparations','Oral Morphine equivalency (mg) per 1000 population',"total_ome_per_1000"),
   (2,'(c)  Total cost of high-dose, long-acting opioid preparations','Cost per 1000 population (2017 equivalent GBP)', 'cost_per_1000')]

fig = plt.figure(figsize=(15,20))

for i in s:
    ax = plt.subplot(3,1,i[0]+1)  # layout and position of subplot
    if i[0]==1:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==0:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==2:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    dfp.plot(ax=ax, kind='area',  linewidth=0)
    ax.set_xticks(np.arange(1998, 2017, step=1))
    ax.set_xlabel('Year', size="14")
    ax.set_ylabel(i[2], size="14")
    ax.tick_params(labelsize=12)
    ax.set_title(i[1], size="17")
    handles, labels = ax.get_legend_handles_labels()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    ax.legend(reversed(handles), reversed(labels),loc='center left', fontsize="12",bbox_to_anchor=(1, .62))

#plt.savefig("opioids_Figure2_revised.pdf", transparent=True, dpi=300)
plt.show()

# -

# ## Plot data - less popular opioids (expand "Other" group)

# +
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid",{'grid.color': '.9'})
sns.set_palette("Set3",n_colors=14)

dft = dfl.groupby("chem_substance")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dft.chem_substance.drop_duplicates())

grp = pd.DataFrame(sort_order).reset_index()
grp = grp.loc[grp.index>=11].reset_index()
grp["chemical"] = np.where(grp.index>10,"Others",grp.chem_substance)
grp["other_flag"] = np.where(grp.index>10,0,1)
dfl2 = dfl.merge(grp,on="chem_substance")

dfs = dfl2.groupby(["other_flag","chemical"])["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"}).sort_values(by=["other_flag","total_ome_all_years"],ascending=False)
sort_order = pd.Series(dfs.chemical)

s=[(0,'(a)  Total prescriptions for opioid-containing preparations','Prescriptions per 1000 population','items_per_1000'),
   (1,'(b)  Total Oral Morphine Equivalency (mg) for opioid-containing preparations','Oral Morphine equivalency (mg) per 1000 population',"total_ome_per_1000"),
   (2,'(c)  Total cost of opioid-containing preparations','Cost per 1000 population (2017 equivalent GBP)','cost_per_1000')]

fig = plt.figure(figsize=(15,20))

for i in s:
    ax = plt.subplot(3,1,i[0]+1)  # layout and position of subplot
    if i[0]==0:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==1:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==2:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    dfp.plot(ax=ax, kind='area',  linewidth=0)
    ax.set_xticks(np.arange(1998, 2017, step=1))
    ax.set_xlabel('Year', size="14")
    ax.set_ylabel(i[2], size="14")
    ax.tick_params(labelsize=12)
    ax.set_title(i[1], size="17")
    handles, labels = ax.get_legend_handles_labels()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    ax.legend(reversed(handles), reversed(labels),loc='center left', fontsize="12",bbox_to_anchor=(1, .62))

plt.show()

# -

# ## Plot data - low-dose opioids

# +
low_df = dfl.loc[(dfl['Is_LA'] == True)&(dfl['Is_High_LA'] != "High dose")]

import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style("whitegrid",{'grid.color': '.9'})
sns.set_palette("Set3",n_colors=14)

dft = low_df.groupby("chem_substance")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})
sort_order = pd.Series(dft.chem_substance.drop_duplicates())

grp = pd.DataFrame(sort_order).reset_index()
grp["chemical"] = np.where(grp.index>11,"Other",grp.chem_substance)
dfl2 = low_df.merge(grp,on="chem_substance")

dfs = dfl2.groupby("chemical")["total_ome_per_1000"].sum().sort_values(ascending=False).reset_index().rename(columns={"total_ome_per_1000":"total_ome_all_years"})

s=[(0,'(a)  Total prescriptions for non-high-dose, long-acting opioid preparations','Prescriptions per 1000 population','items_per_1000'),
   (1,'(b)  Total Oral Morphine Equivalency (mg) for non-high-dose, long-acting opioid preparations','Oral Morphine equivalency (mg) per 1000 population',"total_ome_per_1000"),
   (2,'(c)  Total cost of non-high-dose, long-acting opioid preparations','Cost per 1000 population (2017 equivalent GBP)', 'cost_per_1000')]

fig = plt.figure(figsize=(15,20))

for i in s:
    ax = plt.subplot(3,1,i[0]+1)  # layout and position of subplot
    if i[0]==1:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==0:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    if i[0]==2:
        dfp = dfl2.groupby(['year','chemical'])[i[3]].sum().unstack()
        dfp = dfp.reindex(columns=sort_order)
    dfp.plot(ax=ax, kind='area',  linewidth=0)
    ax.set_xticks(np.arange(1998, 2017, step=1))
    ax.set_xlabel('Year', size="14")
    ax.set_ylabel(i[2], size="14")
    ax.tick_params(labelsize=12)
    ax.set_title(i[1], size="17")
    handles, labels = ax.get_legend_handles_labels()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    ax.legend(reversed(handles), reversed(labels),loc='center left', fontsize="12",bbox_to_anchor=(1, .62))

plt.show()

# -

# ## Find first appearances in prescribing data for each opioid (broken down by formulation)

# +

q = '''-- opioid long term data extraction with classification by formulation
WITH a AS (
SELECT
  year,
  l.chem_substance,
  REGEXP_EXTRACT(l.drug_name, r'[^_]+_(.*?)(?:[0-9]+.*)?$') AS mid_string,
  l.Is_LA,
  l.Is_High_LA,
  sum(itemsper1000) as items_per_1000,
  sum(quantityper1000) as quantity_per_1000,
  sum(quantityper1000*dose_per_unit*new_ome_multiplier) AS total_ome_per_1000,
  sum(Infl_corr_Cost_per1000) as cost_per_1000
FROM ebmdatalab.helen.trends_from_pca_final_2017 p
INNER JOIN (SELECT distinct drug_name, chem_substance, Is_LA, Is_High_LA, dose_per_unit, new_ome_multiplier FROM ebmdatalab.richard.opioid_converter) l on l.drug_name = p.drug_name

GROUP BY
  year,
  chem_substance,
  mid_string,
  Is_LA,
  Is_High_LA )

  SELECT DISTINCT chem_substance, --RTRIM(mid_string) AS mid_string,
  CASE  WHEN RTRIM(mid_string) IN ('Vag Crm','Vag Gel','Oily Crm','Crm','Gel','Lot','Gel','Soln','Gel Sach','Oint','Intrasite Gel') THEN 'Cream/Gel'    --this will only compare identical formulations --  PROBABLY WANT TO INCLUDE EQUIVALENT FORMULATIONS --
        WHEN RTRIM(mid_string) IN ('Tab Buccal','Buccal Film','Tab Subling','Disper Tab','Orodisper Tab','Loz','Tab Sublingual','Oral Lyophilisate') THEN 'Lozenge/Sublingual tab'
        WHEN RTRIM(mid_string) IN ('Tab Solb','Eff Tab','Solb Tab','Tab Eff','Pdr Sach','Eff Pdr Sach','Susp Gran Sach') THEN 'Soluble Tab/Powder'
        WHEN RTRIM(mid_string) LIKE '%Tab' OR RTRIM(mid_string) LIKE '%Cap' OR RTRIM(mid_string) IN ('Cap','Cap E/C','Capl') THEN 'Tab/Cap'
        WHEN RTRIM(mid_string) LIKE '%Susp%' OR
          RTRIM(mid_string) IN ('Liq Spec','Oral Soln','Sod Oral Soln','Elix','Liq','Oral Susp','Tinct','Liq Conc','Oral Dps','Oral Soln Conc','Oral Conc','Mix','Elix BPC Inc Duty','Methadone HCl Mix') THEN 'Liquid'
        WHEN RTRIM(mid_string) LIKE '%Inj' OR RTRIM(mid_string) IN ('I/V Inf','Syr','Inj','(S)','Lact Inj','P','Morph Sulf','Morph Sulph','Implant') THEN 'Injectible'
        WHEN RTRIM(mid_string) IN ('Patch','Patches','TransdermalPatch','Transdermal Patch','T/Derm Patch') THEN 'Patch'
        WHEN RTRIM(mid_string) IN ('Lact Suppos','Suppos') THEN 'Suppository'
        WHEN RTRIM(mid_string) LIKE '%Nsl Spy%' or RTRIM(mid_string) = 'Reefer' THEN 'Inhalation'
        ELSE RTRIM(mid_string) END AS formulation,
        MIN(year) AS min_year
   FROM a
   GROUP BY chem_substance,  formulation
   ORDER BY chem_substance'''

tbl = bq.cached_read(q, csv_path='by_formulation.zip')
tbl.head()
# -

#tbl[["chem_substance","min_year","formulation"]].sort_values(by=["chem_substance","min_year"]).set_index(["chem_substance","min_year"])
tab = pd.DataFrame(tbl.groupby(["chem_substance","min_year"])["formulation"].apply(lambda x:  "%s" % ', '.join(x)))
#tab.to_csv('first_year.csv')
tab

# # Practice and CCG level analysis

# +

#GBQ_PROJECT_ID = '620265099307'

# returns 1.2m rows - saved as 235MB of zipped csvs, loaded in next cell
q = '''
-- opioids - including practices open/prescribing but not prescribing opioids
WITH opioid_prescribing AS (
select p.*
FROM ebmdatalab.hscic.normalised_prescribing_standard p
INNER JOIN (SELECT DISTINCT bnf_code FROM `richard.opioid_converter`) o
ON p.bnf_code = o.bnf_code
),

-- create a table of all prescribing by practice by month.
A AS (
  SELECT practice, pct,
    CAST(month AS DATE) AS year_mon,
    sum(items) AS items
  FROM ebmdatalab.hscic.normalised_prescribing_standard p
  GROUP BY  practice, pct, year_mon),

-- inner join to practice list to filter for type 4 practices, and for months they had more than zero patients and more than zero total prescribing.
q2 AS (
  SELECT a.practice,
    a.pct,
    prac.status_code,
    a.year_mon,
    MAX(a.items) AS items,
    MAX(total_list_size) AS total_list_size
  FROM ebmdatalab.hscic.practice_statistics_all_years s
  LEFT JOIN A ON  a.practice = s.practice AND a.year_mon = CAST(s.month AS DATE)
  INNER JOIN  ebmdatalab.hscic.practices prac ON prac.code=a.practice AND prac.setting = 4
  WHERE total_list_size > 0 and items > 0
  GROUP BY practice, pct, status_code, year_mon )

-- join practices to opioid prescribing data
SELECT
  COALESCE(p.pct,q2.pct) AS pct,
  q2.practice,
  q2.status_code,
  q2.year_mon AS month,
  l.chem_substance,
  Is_LA,
  Is_High_LA,
  sum(p.items) as items,
  sum(quantity) as quantity,
  sum(quantity*dose_per_unit*new_ome_multiplier) AS total_ome,
  sum(net_cost) as net_cost,
  sum(actual_cost) as actual_cost
FROM q2
LEFT JOIN opioid_prescribing p ON p.practice = q2.practice AND CAST(p.month AS DATE) = q2.year_mon
LEFT JOIN (SELECT distinct bnf_code, chem_substance, Is_High_LA, Is_LA, dose_per_unit, new_ome_multiplier FROM ebmdatalab.richard.opioid_converter) l on l.bnf_code = p.bnf_code
GROUP BY
  pct,
  practice,
  status_code,
  month,
  chem_substance,
  Is_LA,
  Is_High_LA
'''

# +
import glob
import pandas as pd
import numpy as np

schema = {
    "pct": "category",
    "practice": "category",
    "status_code": "category",
    "chem_substance": "category",
    "Is_LA": "category",
    "Is_High_LA": "category",
    "items": pd.Int16Dtype(),
    "quantity": pd.Int64Dtype(),
    "total_ome": np.float16,
    "net_cost": np.float16,
    "actual_cost": np.float16,
}

def as_true_false(val):
    "Convert to true/false strings"
    if isinstance(val, str):
        if not val:
            val = None
        elif str(val).lower() == "true":
            val = True
        else:
            val = False
    else:
        if np.isnan(val):
            val = None
        else:
            if val:
                val = True
            else:
                val = False
    return val

converters = {"Is_LA": as_true_false, "Is_High_LA": as_true_false}
df1 = None
for path in sorted(glob.glob("opioid*gz")):
    print("loading {}....".format(path))
    if df1 is None:
        df1 = pd.read_csv(path, dtype=schema, parse_dates=["month"], date_parser=lambda col: pd.to_datetime(col, utc=True), converters=converters, low_memory=False)
    else:
        df1 = pd.concat([df1, pd.read_csv(path, dtype=schema, parse_dates=["month"], date_parser=lambda col: pd.to_datetime(col, utc=True), converters=converters, low_memory=False)], ignore_index=True)

#for col in ['pct', 'practice', 'status_code', 'chem_substance', 'Is_LA', 'Is_High_LA']:
#    # Because of the concat above, some of these are converted to objects where new categories are introduced
#    # between loop iterations
#    df1[col] = df1[col].astype("category")
for col in ['Is_LA', 'Is_High_LA']:
    # Because of the concat above, some of these are converted to objects where new categories are introduced
    # between loop iterations
    df1[col] = df1[col].astype("category")
# + {}
# import practice list size data

GBQ_PROJECT_ID = '620265099307'

q2 = '''SELECT practice,
    pct_id AS CCG,
    month,
    total_list_size
from ebmdatalab.hscic.practice_statistics_all_years
'''

pop = bq.cached_read(q2, csv_path='practice_list_size.zip', use_bqstorage_api=True)
pop['month'] = pd.to_datetime(pop['month'])
pop.head()
# -

df1.head()

# ### Load, organise and filter practice-level data

# +
import pandas as pd
import numpy as np

# convert dates to datetime format
#df1["month"] = pd.to_datetime(df1.month)

# tidy data and replace nulls with appropriate blanks
#df1["High_LA"] = False
#df1.loc[(df1["Is_High_LA"]=="TRUE")|(df1["Is_High_LA"].astype(str)=="True"),"High_LA"] = True
#df1["LA"] = False
#df1.loc[df1["Is_LA"].astype(str)=="True","LA"] = True

from pandas.api.types import is_numeric_dtype
for y in df1.columns:
    if is_numeric_dtype(df1[y].dtype):
          df1[y].fillna(0,inplace=True)

# count included practices and CCGs
print (df1.practice.nunique()," total practices")

# Filter to latest 6 months only and currently active practices with standard CCG codes (for regression and mapping)

dftest = df1.loc[(df1["month"]>df1["month"].max()-pd.DateOffset(months=6))]
print ("Latest ", dftest.month.nunique()," months only (" , df1["month"].max()-pd.DateOffset(months=5), " to ", df1["month"].max(), "):")
print (dftest.practice.nunique()," practices including those not active / without valid CCG")

dftestc = dftest.loc[dftest["status_code"] == "C"]
print (dftestc.practice.nunique()," practices closed")
dftestd = dftest.loc[dftest["status_code"] == "D"]
print (dftestd.practice.nunique()," practices dormant")

dftest = dftest.loc[(dftest["pct"].str.match(r"([0-9]{2})([A-Za-z])")) & (dftest["status_code"] == "A")]
print (dftest.practice.nunique()," practices included")
print (dftest.pct.nunique()," CCGs")
# -

# ## Practice data for regression analysis

# +
#
# -

# ## Practice trends analysis (monthly data, 2010-2018)

df1.info()

# ### (a) Create calculated fields e.g total OME per 1000 population

# +

df = df1.copy()

# tidy data
df = df.drop(["pct","quantity","net_cost"],axis=1).reset_index(drop=True)

# create columns for high dose and long acting OME
df.loc[df["Is_LA"]==True,"Items Long Acting"] = df["items"]
df.loc[df["Is_High_LA"]==True,"Items High Dose"] = df["items"]
df.loc[df["Is_LA"]==True,"OME Long Acting"] = df["total_ome"]
df.loc[df["Is_High_LA"]==True,"OME High Dose"] = df["total_ome"]
df.loc[df["Is_LA"]==True,"Cost Long Acting"] = df["actual_cost"]
df.loc[df["Is_High_LA"]==True,"Cost High Dose"] = df["actual_cost"]

# +
df2 = df.rename(columns={"total_ome":"Total OME",
                          "items":"Total Items",
                          "actual_cost":"Total Cost"})

df2 = df2.merge(pop[["practice","month","total_list_size"]], on=["practice","month"])
df2["Percent high dose (by OME)"] = 100*df2["OME High Dose"]/df2["OME Long Acting"]
df2["Percent high dose (by items)"] = 100*df2["Items High Dose"]/df2["Items Long Acting"]
df2["Total OME (per 1000)"] = 1000*df2["Total OME"]/df2.total_list_size
df2["Total items (per 1000)"] = 1000*df2["Total Items"]/df2.total_list_size
df2["Total cost (per 1000)"] = 1000*df2["Total Cost"]/df2.total_list_size
df2["Cost high dose opioids (per 1000)"] = 1000*df2["Cost High Dose"]/df2.total_list_size
df2["Cost high dose opioids (per item)"] = df2["Cost High Dose"]/df2["Items High Dose"]

df2["High dose items (per 1000)"] = 1000*df2["Items High Dose"]/df2.total_list_size
df2["Long acting items (per 1000)"] = 1000*df2["Items Long Acting"]/df2.total_list_size
df2["High dose OME (per 1000)"] = 1000*df2["OME High Dose"]/df2.total_list_size
df2["Long acting OME (per 1000)"] = 1000*df2["OME Long Acting"]/df2.total_list_size
df2.head()
# -

# ### (b) Calculate deciles

df3 = df2
x = np.arange(0.1, 1, 0.1) # set range of deciles required (0.1-0.9)
pc = df3.groupby('month').quantile(x)  # calculate deciles for each month
pc = pd.DataFrame(pc.stack()).reset_index().rename(columns={"level_1": 'percentile',"level_2": 'measure', 0:"value"}) # rearrange
pc["index"] = (pc.percentile*10).map(int) # create integer range of percentiles as integers are better for charts
pc.head(12)

# +

import matplotlib.pyplot as plt
import datetime
import matplotlib.gridspec as gridspec
import seaborn as sns

sns.set_style("whitegrid",{'grid.color': '.9'})
dfp = pc.sort_values(by=["month","measure"])
#dfp['month'] = dfp['month'].astype(str)
# set format for dates:
#dfp['dates'] = [datetime.datetime.strptime(date, '%Y-%m-%d').date() for date in dfp['month']]

# set sort order of measures manually, and add grid refs to position each subplot:
s = [(0,'Total OME (per 1000)',0,0,"Oral Morphine Equivalency (mg) per 1000 patients", "(a)  Total OME for all opioid-containing preparations"),
     (1,'High dose items (per 1000)',0,1,"Items per 1000 patients", "(b)  High dose items"),
     (2,'Percent high dose (by items)',1,0,"Percent",'(c)  Percent long-acting opioids prescribed as high dose (by items)'),
     (3,'Percent high dose (by OME)',1,1,"Percent",'(d)  Percent long-acting opioids prescribed as high dose (by OME)'),
     (4,'Total cost (per 1000)',2,0,"Cost per 1000 patients (GBP)", "(e)  Total cost for all opioid-containing preparations"),
     (5,'Cost high dose opioids (per 1000)',2,1,"Cost per 1000 patients (GBP)", "(f)  Cost for all high dose opioids")
    ]

fig = plt.figure(figsize=(18,20))
gs = gridspec.GridSpec(3,2)  # grid layout for subplots

# Plot each subplot using a loop
for i in s:
    ax = plt.subplot(gs[i[2], i[3]])  # position of subplot in grid using coordinates listed in s
    for decile in range(1,10):   # plot each decile line
        data = dfp.loc[(dfp['measure']==i[1]) & (dfp['index']==decile)]
        if decile == 5:
            ax.plot(data["month"],data['value'],'b-',linewidth=0.7)
        else:
            ax.plot(data["month"],data['value'],'b--',linewidth=0.4)
    ax.set_ylabel(i[4], size =14, alpha=0.6)
    ax.set_title(i[5],size = 17)
    ax.set_ylim([0,1.05*data["value"].max()])
    if  i[4]=="Percent":    # set y axis limit only for percentage measure
        ax.set_ylim([0, 70])
    ax.tick_params(labelsize=12)
    ax.set_xlim([dfp['month'].min(), dfp['month'].max()]) # set x axis range as full date range

plt.subplots_adjust(wspace = 0.13,hspace = 0.16)

#plt.savefig("opioids_Figure4_revised.pdf", transparent=True, dpi=300)
plt.show()

# -

# ## Variation by CCG (maps)

# ### (a) Aggregate data, create calculated fields

# +
# filter to latest 6 months only
df4 = dftest
df4 = df4.drop(["quantity","net_cost"],axis=1)

# group to ccg level and combine the 6 months
df4 = pd.DataFrame(df4.groupby(["pct","chem_substance","Is_LA","Is_High_LA"])["items","total_ome","actual_cost"].sum()).reset_index()

df4["fent_ome"] = np.where((df4["chem_substance"] == "Fentanyl")& (df4["Is_High_LA"]==True),df4["total_ome"],0)
df4["morph_ome"] = np.where((df4["chem_substance"] == "Morphine Sulfate")& (df4["Is_High_LA"]==True),df4["total_ome"],0)
df4["oxyco_ome"] = np.where((df4["chem_substance"] == "Oxycodone Hydrochloride")& (df4["Is_High_LA"]==True),df4["total_ome"],0)
df4["practice_count"] = 1

# aggregate chem substances
df = pd.DataFrame(df4.groupby(["pct","Is_LA","Is_High_LA"]).sum()).reset_index()

# create columns for high dose and long acting OME
df.loc[df["Is_LA"]==True,"Items Long Acting"] = df["items"]
df.loc[df["Is_High_LA"]==True,"Items High Dose"] = df["items"]
df.loc[df["Is_LA"]==True,"OME Long Acting"] = df["total_ome"]
df.loc[df["Is_High_LA"]==True,"OME High Dose"] = df["total_ome"]
df.loc[df["Is_LA"]==True,"Cost Long Acting"] = df["actual_cost"]
df.loc[df["Is_High_LA"]==True,"Cost High Dose"] = df["actual_cost"]

df = df.drop(["Is_LA","Is_High_LA"],axis=1)
df = df.groupby(["pct"]).sum().reset_index()

df3 = df.rename(columns={"total_ome":"Total OME",
                          "items":"Total Items",
                          "actual_cost":"Total Cost"})

#aggregate list sizes up to CCG level and get population sizes averaged over latest 6 months
popccg = pop.loc[pop["month"]> df1["month"].max()-pd.DateOffset(months=6)]
popccg = popccg.groupby(["CCG","month"])["total_list_size"].sum() # sum across CCGs
popccg = pd.DataFrame(popccg.groupby("CCG").mean()).reset_index() # average across months
df3 = df3.merge(popccg[["CCG","total_list_size"]], right_on="CCG", left_on="pct").drop("CCG",axis=1)

df3["Percent high dose (by items)"] = 100*df3["Items High Dose"]/df3["Items Long Acting"]
df3["Percent high dose (by OME)"] = 100*df3["OME High Dose"]/df3["OME Long Acting"]
df3["Total OME (per 1000)"] = 1000*df3["Total OME"]/df3.total_list_size
df3["Total items (per 1000)"] = 1000*df3["Total Items"]/df3.total_list_size
df3["High dose items (per 1000)"] = 1000*df3["Items High Dose"]/df3.total_list_size
df3["Long acting items (per 1000)"] = 1000*df3["Items Long Acting"]/df3.total_list_size
df3["High dose OME (per 1000)"] = 1000*df3["OME High Dose"]/df3.total_list_size
df3["Long acting OME (per 1000)"] = 1000*df3["OME Long Acting"]/df3.total_list_size
df3["Total cost (per 1000)"] = 1000*df3["Total Cost"]/df3.total_list_size
df3["Cost high dose opioids (per 1000)"] = 1000*df3["Cost High Dose"]/df3.total_list_size
df3.head()

df4 = df3.copy()

df4["% Fentanyl of high dose OME"] = 100*df4["fent_ome"]/df4["OME High Dose"]
df4["% Morphine of high dose OME"] = 100*df4["morph_ome"]/df4["OME High Dose"]
df4["% Oxycodone of high dose OME"] = 100*df4["oxyco_ome"]/df4["OME High Dose"]

df4.head()
# -

# ### (b) Summary table

table = df4[['Total items (per 1000)','Total OME (per 1000)','High dose items (per 1000)',
     'Percent high dose (by items)', 'Percent high dose (by OME)','Total cost (per 1000)',
         '% Fentanyl of high dose OME','% Morphine of high dose OME',
          '% Oxycodone of high dose OME']].agg([min, "median", max]).transpose()
table["fold-difference"] = table["max"]/table["min"]
table

# ### (c) Join to geographical data

# +

spending = df4.copy()
names = pd.read_csv('ccg_for_map.csv')
qc = '''
SELECT
  code, name
FROM  `hscic.ccgs` c
WHERE org_type = "CCG"
'''

names = bq.cached_read(qc, csv_path='ccg_names.csv')
spending2 = spending.merge(names[['code','name']],left_on="pct",right_on="code")
spending2 = spending2.drop(["code","fent_ome","morph_ome","oxyco_ome"],axis=1).set_index('name')
spending2 = spending2.round(0)

spending2.sort_values(by="Total OME (per 1000)") # 195 rows
# -

# ### (d) Plot maps

# +
import matplotlib.gridspec as gridspec
import geopandas as gpd
import matplotlib.pyplot as plt
import pandas as pd
#from matplotlib import verbose
#verbose.level = 'helpful'

# from our API https://openprescribing.net/api/1.0/org_location/?org_type=ccg
ccgs = gpd.read_file('ccgs.json').set_index('name')

ccgs = ccgs[~ccgs['geometry'].isnull()]  # remove ones without geometry - these are federations rather than individual CCGs
gdf = ccgs.join(spending2)

# set sort order of measures manually, and add grid refs to position each subplot:
s = [(0,'Total OME (per 1000)',0,0,'(a)  '),      (1,'Total cost (per 1000)',0,1,'(b)  '),
     (2,'High dose items (per 1000)',1,0,'(c)  '),      (3,'Percent high dose (by items)',1,1,'(d)  ')
    ]

fig = plt.figure(figsize=(16,30))
gs = gridspec.GridSpec(4,2)  # grid layout for subplots

for i in s:
    ax = plt.subplot(gs[i[2], i[3]])  # position of subplot in grid using coordinates listed in s
    gdf.plot(ax=ax,column=i[1],  edgecolor='black', linewidth=0.1, legend=True, cmap='OrRd')
    ax.set_aspect(1.63)
    ax.set_title(i[4]+i[1],size = 20)
    cb_ax = fig.axes[2*(i[0])+1] # extract legend labels from list of axis/legend labels
    cb_ax.tick_params(labelsize=14)
    plt.axis('off')

plt.subplots_adjust(wspace = 0.05,hspace = 0.05)

#plt.savefig("opioids_Figure3ad_revised.pdf", transparent=True, dpi=300)
plt.show()
# -

# ### (e) Plot additional maps for some percentage measures

# +
import matplotlib.gridspec as gridspec
import geopandas as gpd
import matplotlib.pyplot as plt
import pandas as pd

# set sort order of measures manually, and add grid refs to position each subplot:
s = [(0,'% Fentanyl of high dose OME',0,0,'(e)  ',False),      (1,'% Morphine of high dose OME',0,1,'(f)  ',False),
     (2,'% Oxycodone of high dose OME',0,2,'(g)  ',True)
    ]

fig = plt.figure(figsize=(17,6))
#plt.subplots(ncols=3)
gs = gridspec.GridSpec(1,3, width_ratios=[4,4,5])  # grid layout for subplots;
# adjust ratios to allow all plots to appear same size despite the third having a legend.

# set common value limits for colour scale
vmin = 0
vmax = gdf[["% Fentanyl of high dose OME","% Morphine of high dose OME","% Oxycodone of high dose OME"]].max().max()*1.05

for i in s:
    ax = plt.subplot(gs[0,i[0]])  # position of subplot in grid using coordinates listed in s
    gdf.plot(ax=ax,column=i[1],  edgecolor='black', linewidth=0.1, cmap='OrRd', legend = i[5], vmin=vmin, vmax=vmax)
    ax.set_title(i[4]+i[1],size = 20)
    ax.set_aspect(1.63)  # aspect for correct lat/long
    plt.axis('off')
cb_ax = fig.axes[3] # take legend label (4th item in list of axis/legend labels for all plots)
cb_ax.tick_params(labelsize=16) # set label size for legend

plt.tight_layout()
#plt.savefig("opioids_Figure3eg_revised.pdf", transparent=True, dpi=300)
plt.show()

# +
import matplotlib.gridspec as gridspec
import geopandas as gpd
import matplotlib.pyplot as plt
import pandas as pd
#from matplotlib import verbose
#verbose.level = 'helpful'

# from our API https://openprescribing.net/api/1.0/org_location/?org_type=ccg
ccgs = gpd.read_file('ccgs.json').set_index('name')

ccgs = ccgs[~ccgs['geometry'].isnull()]  # remove ones without geometry - these are federations rather than individual CCGs
gdf = ccgs.join(spending2)

# set sort order of measures manually, and add grid refs to position each subplot:
s = [(0,'Total OME (per 1000)',0,0,'(a)  '),      (1,'Total cost (per 1000)',0,1,'(b)  '),
     (2,'High dose items (per 1000)',1,0,'(c)  '),      (3,'Percent high dose (by items)',1,1,'(d)  ')
    ]

fig = plt.figure(figsize=(16,30))
gs1 = gridspec.GridSpec(4,2)  # grid layout for subplots

gs1.update(bottom=0.3, wspace=0.05,hspace = 0.05)

for i in s:
    ax = plt.subplot(gs1[i[2], i[3]])  # position of subplot in grid using coordinates listed in s
    gdf.plot(ax=ax,column=i[1],  edgecolor='black', linewidth=0.1, legend=True, cmap='OrRd')
    ax.set_aspect(1.63)
    ax.set_title(i[4]+i[1],size = 20)
    cb_ax = fig.axes[2*(i[0])+1] # extract legend labels from list of axis/legend labels
    cb_ax.tick_params(labelsize=14)
    plt.axis('off')

#plt.savefig("opioids_Figure3ad_revised.pdf", transparent=True, dpi=300)
#plt.show()

# set sort order of measures manually, and add grid refs to position each subplot:
s = [(0,'% Fentanyl of high dose OME',0,0,'(e)  ',False),      (1,'% Morphine of high dose OME',0,1,'(f)  ',False),
     (2,'% Oxycodone of high dose OME',0,2,'(g)  ',True)
    ]

#fig = plt.figure(figsize=(17,6))
#plt.subplots(ncols=3)
gs2 = gridspec.GridSpec(1,3, width_ratios=[4,4,5])  # grid layout for subplots;
# adjust ratios to allow all plots to appear same size despite the third having a legend.
gs2.update(top=0.3,wspace=0.05)

# set common value limits for colour scale
vmin = 0
vmax = gdf[["% Fentanyl of high dose OME","% Morphine of high dose OME","% Oxycodone of high dose OME"]].max().max()*1.05

for i in s:
    ax = plt.subplot(gs2[0,i[0]])  # position of subplot in grid using coordinates listed in s
    gdf.plot(ax=ax,column=i[1],  edgecolor='black', linewidth=0.1, cmap='OrRd', legend = i[5], vmin=vmin, vmax=vmax)
    ax.set_title(i[4]+i[1],size = 18)
    ax.set_aspect(1.63)  # aspect for correct lat/long
    plt.axis('off')
cb_ax = fig.axes[3] # take legend label (4th item in list of axis/legend labels for all plots)
cb_ax.tick_params(labelsize=16) # set label size for legend

plt.tight_layout()
#plt.savefig("opioids_Figure3eg_revised.png", format='png', dpi=300)
plt.show()
# -

# ## Potential cost savings

# +
# Extract lowest practice percentile for both high-dose items and cost per 1000 population for latest 6 months
lowest_dec = pc[["month","measure","value"]].loc[(pc["month"]>pc["month"].max() - pd.DateOffset(months=6))
       & (pc["percentile"] == 0.1)].fillna(0)
lowest_dec.loc[lowest_dec["measure"]=="High dose items (per 1000)","measure"] = "high dose items"
lowest_dec.loc[lowest_dec["measure"]=="Cost high dose opioids (per 1000)","measure"] = "high dose cost"
lowest_dec.loc[lowest_dec["measure"]=="Cost high dose opioids (per item)","measure"] = "high dose costperitem"

# extract data on high dose items
dfs = pd.DataFrame(df2[["practice","month","total_list_size","Items High Dose","Total Cost","Cost High Dose", "Cost high dose opioids (per item)"]].set_index(["practice","month","total_list_size"]).stack()).reset_index().rename(columns={"level_3":"measure",0:"actual_value"})
dfs.loc[dfs["measure"]=="Items High Dose","measure"] = "high dose items"
dfs.loc[dfs["measure"]=="Cost High Dose","measure"] = "high dose cost"
dfs.loc[dfs["measure"]=="Cost high dose opioids (per item)","measure"] = "high dose costperitem"

# merge with lowest decile data (inner join, this restricts data to latest 6 months)
savings = dfs.merge(lowest_dec,on=["month","measure"])

savings["target_value"] = savings["total_list_size"]*savings["value"]/1000
savings["potential_saving"] = savings["actual_value"] - savings["target_value"]
savings = savings.fillna(0)
savings.loc[savings["potential_saving"]<0,"potential_saving"] = 0

print("Practices at the lowest decile over the latest 6 months prescribed: ", lowest_dec.loc[lowest_dec["measure"]=="high dose items"].mean().round(2).value,
      "items / £", lowest_dec.loc[lowest_dec["measure"]=="high dose cost"].mean().round(2).value, " per 1,000 patients")

print(savings.groupby("measure")["actual_value","potential_saving"].sum())

# +
# Compare best cost-per-item prices
dfs2 = pd.DataFrame(df2[["practice","month","Items High Dose","Total Cost","Cost High Dose", "Cost high dose opioids (per item)"]])
#dfs2.loc[dfs2["measure"]=="Cost high dose opioids (per item)","measure"] = "high dose costperitem"
savings = dfs2.merge(lowest_dec.loc[lowest_dec.measure=="high dose costperitem"],on=["month"])
savings["target_value"] = savings["Items High Dose"]*savings["value"]
savings["potential_saving"] = savings["Cost High Dose"] - savings["target_value"]

savings = savings.fillna(0).drop("measure",axis=1)
savings.loc[savings["potential_saving"]<0,"potential_saving"] = 0

print("If each practice prescribed at the lowest decile cost-per-item over the latest six months (£",
      lowest_dec.loc[lowest_dec["measure"]=="high dose costperitem"].mean().round(2).value,
      "per item), in total they could have saved £", round(savings["potential_saving"].sum(),-3), " out of a total £",
      round(savings["Cost High Dose"].sum(),-3), "(",
      round(100*savings["potential_saving"].sum()/savings["Cost High Dose"].sum(),1), "%)" )

# -

# ## Change by CCG

# +
yr1 = df1.loc[(df1["pct"].str.match(r"([0-9]{2})([A-Za-z])"))& (df1['month'].dt.year>=2016)].reset_index()
yr1["year"] = yr1['month'].dt.year

yr1.head()
# -

df1.info()

yr1 = df1.loc[(df1["pct"].str.match(r"([0-9]{2})([A-Za-z])"))& (df1['month'].dt.year>=2016)].reset_index()
yr1["year"] = yr1['month'].dt.year
yr1.head()

# + {"active": ""}
#
# -

yr2 = yr1[["pct","practice","month","year","total_ome"]].groupby(["pct","practice","month","year"])['total_ome'].sum()
yr2 = yr2.reset_index().merge(pop[["practice","month","total_list_size"]], on=["practice","month"]).groupby(["pct","month","year"]).sum()
yr2.head()

yr2 = yr2.reset_index().groupby(['pct','year']).agg({'total_list_size':'mean','total_ome':"sum"})
yr2["OME_per_1000"] = 1000*yr2["total_ome"]/yr2.total_list_size
yr2 = yr2.drop(['total_list_size','total_ome'],axis=1).unstack().reset_index(col_level=1)
yr2.columns = yr2.columns.droplevel()
yr2 = yr2.drop(2018,axis=1)
yr2["change"] = yr2[2017]-yr2[2016]
yr2["%change"] = 100*(yr2["change"])/yr2[2016]
yr2 = yr2.sort_values(by="change").reset_index(drop=True)
yr2.head()

# +
change = yr2.copy()

change2 = change.merge(names[['code','name']],left_on="pct",right_on="code")
change2 = change2.drop([2016,2017,"code"],axis=1).set_index('name')

change2.sort_values(by="change") # 195 rows

# +
import matplotlib.gridspec as gridspec
import geopandas as gpd
import matplotlib.pyplot as plt
import pandas as pd
#from matplotlib import verbose
#verbose.level = 'helpful'
import matplotlib.colors as colors

cmap = 'coolwarm'
#bounds = np.array([-60000,-45000, -30000, -15000, 0, 0, 5000, 10000,15000, 20000])
#norm = colors.BoundaryNorm(boundaries=bounds, ncolors=256)

# from our API https://openprescribing.net/api/1.0/org_location/?org_type=ccg
ccgs = gpd.read_file('ccgs.json').set_index('name')

ccgs = ccgs[~ccgs['geometry'].isnull()]  # remove ones without geometry - these are federations rather than individual CCGs
gdf = ccgs.join(change2)

vmax = -gdf["change"].min() # set max of scale to reflect minimum such that zero falls in centre of scale.

fig, ax = plt.subplots(figsize=(12, 7))
g = gdf.plot(ax=ax,column="change",  edgecolor='black', linewidth=0.1, legend=True, cmap=cmap, vmax=vmax)
ax.set_aspect(1.63)
#cb_ax = fig.axes[2*(i[0])+1] # extract legend labels from list of axis/legend labels
#cb_ax.tick_params(labelsize=14)
plt.axis('off')
#fig.colorbar(g, ax=ax, extend='both', orientation='vertical')

plt.show()

# -

# # All practice types

# +
q5 = '''
-- opioids - including all practices

WITH q2 AS (
  SELECT
    s.practice,
    s.pct_id AS pct,
    CAST(s.month AS DATE) AS year_mon,
    CASE WHEN prac.setting = 4 THEN "GP" ELSE "other" END AS type,
    MAX(total_list_size) AS total_list_size
  FROM ebmdatalab.hscic.practice_statistics_all_years s
  INNER JOIN  ebmdatalab.hscic.practices prac ON prac.code=s.practice

  GROUP BY practice, pct, type, year_mon )

-- join practices to opioid prescribing data and group to CCGs
SELECT
  COALESCE(p.pct,q2.pct) AS pct,
  q2.year_mon AS month,
  type,
  l.chem_substance,
  Is_LA,
  Is_High_LA,
  sum(p.items) as items,
  sum(quantity) as quantity,
  sum(quantity*dose_per_unit*new_ome_multiplier) AS total_ome,
  sum(net_cost) as net_cost,
  sum(actual_cost) as actual_cost
FROM q2
LEFT JOIN ebmdatalab.helen.opioid_prescribing_2010_2018 p ON p.practice = q2.practice AND CAST(p.month AS DATE) = q2.year_mon
INNER JOIN (SELECT distinct bnf_code, chem_substance, Is_High_LA, Is_LA, dose_per_unit, new_ome_multiplier FROM ebmdatalab.richard.opioid_converter) l on l.bnf_code = p.bnf_code
INNER JOIN `hscic.ccgs` c ON c.code = p.pct AND org_type = "CCG"

GROUP BY
  pct,
  type,
  month,
  chem_substance,
  Is_LA,
  Is_High_LA
'''

'''opioid prescribing data extracted as follows:
select *
FROM ebmdatalab.hscic.normalised_prescribing_standard p
WHERE SUBSTR(p.bnf_code,1,6) IN ("040201","040702","100101")'''

ccg = bq.cached_read(q5, 'opiods_by_practice_setting.csv.gz', use_bqstorage_api=True)
ccg.head()
# -

c2 = ccg.copy()#.loc[ccg["Is_High_LA"]=="TRUE"]
c2["month"] = pd.to_datetime(c2.month)
c2 = c2.loc[c2["month"]>c2["month"].max() - pd.DateOffset(months=6)]
c2 = pd.DataFrame(c2.groupby(["pct","type"])["total_ome"].sum()).unstack() #.sort_values(by="total_ome").transpose()    # "pct","Is_LA","Is_High_LA"c
c2.columns = c2.columns.droplevel()
c2["ome_percent_nongp"] = 100*c2.other / (c2.other+c2.GP)
print(c2["ome_percent_nongp"].count(), 'CCGs have data on opioid prescribing from non-standard practices,')
print('accounting for a maximum of ',c2["ome_percent_nongp"].max().round(3), '% of CCG opioid prescribing')

# ### Which opioids are high-dose

# +
q = '''

SELECT
  chem_substance,
  bnf_code,
  drug_name,
  dose_per_unit as total_dose_per_unit_mg,
  LA_doseshourduration as duration_hrs,
  LA_daily_dose as daily_dose,
   new_ome_multiplier as ome_multiplier,
  LA_daily_OME as daily_OME

FROM ebmdatalab.richard.opioid_converter
WHERE Is_High_LA IN ("True","TRUE")

order by chem_substance, dose_per_unit desc

    '''

tbl = bq.cached_read(q, csv_path='opioids_high_dose_formulations.csv')
tbl.head()
# -


# Copyright (c) 2018 Yuhang Lu <luyuhang@iie.ac.cn>

from lib import gcn_utils,graph,gcn,mAP
import numpy as np
import os,time,datetime
import tensorflow as tf
from keras import backend as K
import keras
import keras.layers as layers
from keras.models import Model
from keras.callbacks import ModelCheckpoint,Callback
from keras.engine.topology import Layer

#GPU 控制
def build(txt_shape,img_shape,max_len=400,filters=250,act_1=None,act_2=None,loss_function=mAP.my_loss):
    K.clear_session()
    word_index,embedding_matrix = loading()
    input_text = layers.Input(shape=(None,),dtype='int32')
    input_image = layers.Input(shape=(img_shape,))
    embeddings = layers.Embedding(len(word_index)+1, 128,weights=[embedding_matrix],input_length=max_len,trainable=True)(input_text)
    filter_sizes,convs = [2,3,4,5],[]
    for fsz in filter_sizes:
        conv1 = layers.Conv1D(filters,kernel_size=fsz,activation='tanh')(embeddings)
        pool1 = layers.MaxPooling1D(max_len-fsz+1)(conv1)
        pool1 = layers.Flatten()(pool1)
        convs.append(pool1)
    merge = layers.Concatenate(1)(convs)
    text_embedding = layers.Dropout(0.5)(merge)
    text_dense = layers.Dense(512,activation=act_1)(text_embedding)
    image_dense = layers.Dense(512,activation=act_1)(input_image)
    mul = layers.Multiply()([text_dense,image_dense])
    pred = layers.Dense(1,activation=act_2)(mul)
    model = Model(inputs=[input_text,input_image], outputs=pred)
    model.compile(loss=loss_function, optimizer='adam', metrics=[mAP.auc])
    model.summary()
    return model

def loading():
    w_path = '../../myproject/data/'
    word_index,now = {},0
    for line in open(os.path.join(w_path, 'clean_vocab.txt')):
        line = line.strip()
        word = line.split()[0]
        if len(word)>0:
            word_index[word] = now
            now += 1
    embeddings_index = {}
    # f = open(os.path.join('../../myproject/data/', 'vectors.txt'))
    f = open(os.path.join(w_path, 'embeddings.txt'))
    for line in f:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs

    embedding_matrix = np.zeros((len(word_index) + 1, 128))
    for word, i in word_index.items():
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            # words not found in embedding index will be all-zeros.
            embedding_matrix[i] = embedding_vector

    return word_index,embedding_matrix

class Position_Embedding(Layer):

    def __init__(self, size=None, mode='sum', **kwargs):
        self.size = size #必须为偶数
        self.mode = mode
        super(Position_Embedding, self).__init__(**kwargs)

    def call(self, x):
        if (self.size == None) or (self.mode == 'sum'):
            self.size = int(x.shape[-1])
        batch_size,seq_len = K.shape(x)[0],K.shape(x)[1]
        position_j = 1. / K.pow(10000., \
                                 2 * K.arange(self.size / 2, dtype='float32' \
                               ) / self.size)
        position_j = K.expand_dims(position_j, 0)
        position_i = K.cumsum(K.ones_like(x[:,:,0]), 1)-1 #K.arange不支持变长，只好用这种方法生成
        position_i = K.expand_dims(position_i, 2)
        position_ij = K.dot(position_i, position_j)
        position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2)
        if self.mode == 'sum':
            return position_ij + x
        elif self.mode == 'concat':
            return K.concatenate([position_ij, x], 2)

    def compute_output_shape(self, input_shape):
        if self.mode == 'sum':
            return input_shape
        elif self.mode == 'concat':
            return (input_shape[0], input_shape[1], input_shape[2]+self.size)

class TextCNN(Layer):

    def __init__(self, x, **kwargs):
        super(TextCNN, self).__init__(**kwargs)

    def build(self, input_shape):
        self.WQ = self.add_weight(name='WQ',
                                  shape=(input_shape[0][-1], self.output_dim),
                                  initializer='glorot_uniform',
                                  trainable=True)
        self.WK = self.add_weight(name='WK',
                                  shape=(input_shape[1][-1], self.output_dim),
                                  initializer='glorot_uniform',
                                  trainable=True)
        self.WV = self.add_weight(name='WV',
                                  shape=(input_shape[2][-1], self.output_dim),
                                  initializer='glorot_uniform',
                                  trainable=True)
        super(Attention, self).build(input_shape)

    def Mask(self, inputs, seq_len, mode='mul'):
        if seq_len == None:
            return inputs
        else:
            mask = K.one_hot(seq_len[:,0], K.shape(inputs)[1])
            mask = 1 - K.cumsum(mask, 1)
            for _ in range(len(inputs.shape)-2):
                mask = K.expand_dims(mask, 2)
            if mode == 'mul':
                return inputs * mask
            if mode == 'add':
                return inputs - (1 - mask) * 1e12

    def call(self, x):
        #如果只传入Q_seq,K_seq,V_seq，那么就不做Mask
        #如果同时传入Q_seq,K_seq,V_seq,Q_len,V_len，那么对多余部分做Mask
        if len(x) == 3:
            Q_seq,K_seq,V_seq = x
            Q_len,V_len = None,None
        elif len(x) == 5:
            Q_seq,K_seq,V_seq,Q_len,V_len = x
        #对Q、K、V做线性变换
        Q_seq = K.dot(Q_seq, self.WQ)
        Q_seq = K.reshape(Q_seq, (-1, K.shape(Q_seq)[1], self.nb_head, self.size_per_head))
        Q_seq = K.permute_dimensions(Q_seq, (0,2,1,3))
        K_seq = K.dot(K_seq, self.WK)
        K_seq = K.reshape(K_seq, (-1, K.shape(K_seq)[1], self.nb_head, self.size_per_head))
        K_seq = K.permute_dimensions(K_seq, (0,2,1,3))
        V_seq = K.dot(V_seq, self.WV)
        V_seq = K.reshape(V_seq, (-1, K.shape(V_seq)[1], self.nb_head, self.size_per_head))
        V_seq = K.permute_dimensions(V_seq, (0,2,1,3))
        #计算内积，然后mask，然后softmax
        A = K.batch_dot(Q_seq, K_seq, axes=[3,3]) / self.size_per_head**0.5
        A = K.permute_dimensions(A, (0,3,2,1))
        A = self.Mask(A, V_len, 'add')
        A = K.permute_dimensions(A, (0,3,2,1))
        A = K.softmax(A)
        #输出并mask
        O_seq = K.batch_dot(A, V_seq, axes=[3,2])
        O_seq = K.permute_dimensions(O_seq, (0,2,1,3))
        O_seq = K.reshape(O_seq, (-1, K.shape(O_seq)[1], self.output_dim))
        O_seq = self.Mask(O_seq, Q_len, 'mul')
        return O_seq

    def compute_output_shape(self, input_shape):
        return (input_shape[0][0], input_shape[0][1], self.output_dim)


"""Second in a series of four pipelines that tell a story in a 'gaming' domain.

In addition to the concepts introduced in `user_score`, new concepts include:
windowing and element timestamps; use of `Filter`.

This pipeline processes data collected from gaming events in batch, building on
`user_score` but using fixed windows. It calculates the sum of scores per team,
for each window, optionally allowing specification of two timestamps before and
after which data is filtered out. This allows a model where late data collected
after the intended analysis window can be included, and any late-arriving data
prior to the beginning of the analysis window can be removed as well. By using
windowing and adding element timestamps, we can do finer-grained analysis than
with the `user_score` pipeline. However, our batch processing is high-latency,
in that we don't get results from plays at the beginning of the batch's time
period until the batch is processed.

To execute this pipeline using the static example input data, specify the
`--dataset=YOUR-DATASET` flag along with other runner specific flags. (Note:
BigQuery dataset you specify must already exist.)

Optionally include the `--input` argument to specify a batch input file. To
indicate a time after which the data should be filtered out, include the
`--stop_min` arg. E.g., `--stop_min=2015-10-18-23-59` indicates that any data
timestamped after 23:59 PST on 2015-10-18 should not be included in the
analysis. To indicate a time before which data should be filtered out, include
the `--start_min` arg. If you're using the default input
"gs://dataflow-samples/game/gaming_data*.csv", then
`--start_min=2015-11-16-16-10 --stop_min=2015-11-17-16-10` are good values.
"""

from __future__ import absolute_import

import argparse
import datetime
import logging

import apache_beam as beam
from apache_beam import typehints
from apache_beam.io import ReadFromText
from apache_beam.metrics import Metrics
from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.window import TimestampedValue
from apache_beam.typehints import with_input_types
from apache_beam.typehints import with_output_types
from apache_beam.utils.pipeline_options import GoogleCloudOptions
from apache_beam.utils.pipeline_options import PipelineOptions
from apache_beam.utils.pipeline_options import SetupOptions

class ParseEventFn(beam.DoFn):
  """Parses the raw game event info into GameActionInfo tuples.

  Each event line has the following format:
    username,teamname,score,timestamp_in_ms,readable_time

  e.g.:
    user2_AsparagusPig,AsparagusPig,10,1445230923951,2015-11-02 09:09:28.224

  The human-readable time string is not used here.
  """
  def __init__(self):
    super(ParseEventFn, self).__init__()
    self.num_parse_errors = Metrics.counter(self.__class__, 'num_parse_errors')

  def process(self, element):
    components = element.split(',')
    try:
      user = components[0].strip()
      team = components[1].strip()
      score = int(components[2].strip())
      timestamp = int(components[3].strip())
      yield {'user': user, 'team': team, 'score': score, 'timestamp': timestamp}
    except:  # pylint: disable=bare-except
      # Log and count parse errors.
      self.num_parse_errors.inc()
      logging.info('Parse error on %s.', element)

@with_input_types(ints=typehints.Iterable[int])
@with_output_types(int)
def sum_ints(ints):
  return sum(ints)

class ExtractAndSumScore(beam.PTransform):
  """A transform to extract key/score information and sum the scores.

  The constructor argument `field` determines whether 'team' or 'user' info is
  extracted.
  """
  def __init__(self, field):
    super(ExtractAndSumScore, self).__init__()
    self.field = field

  def expand(self, pcoll):
    return (pcoll
            | beam.Map(lambda info: (info[self.field], info['score']))
            | beam.CombinePerKey(sum_ints))

def configure_bigquery_write():

  def window_start_format(element, window):
    dt = datetime.datetime.fromtimestamp(int(window.start))
    return dt.strftime('%Y-%m-%d %H:%M:%S')

  return [
      ('team', 'STRING', lambda e, w: e[0]),
      ('total_score', 'INTEGER', lambda e, w: e[1]),
      ('window_start', 'STRING', window_start_format),
  ]

class WriteWindowedToBigQuery(beam.PTransform):
  """Generate, format, and write BigQuery table row information.

  This class may be used for writes that require access to the window
  information.
  """
  def __init__(self, table_name, dataset, field_info):
    """Initializes the transform.

    Args:
      table_name: Name of the BigQuery table to use.
      dataset: Name of the dataset to use.
      field_info: List of tuples that holds information about output table field
                  definitions. The tuples are in the
                  (field_name, field_type, field_fn) format, where field_name is
                  the name of the field, field_type is the BigQuery type of the
                  field and field_fn is a lambda function to generate the field
                  value from the element.
    """
    super(WriteWindowedToBigQuery, self).__init__()
    self.table_name = table_name
    self.dataset = dataset
    self.field_info = field_info

  def get_schema(self):
    """Build the output table schema."""
    return ', '.join(
        '%s:%s' % (entry[0], entry[1]) for entry in self.field_info)

  def get_table(self, pipeline):
    """Utility to construct an output table reference."""
    project = pipeline.options.view_as(GoogleCloudOptions).project
    return '%s:%s.%s' % (project, self.dataset, self.table_name)

  class BuildRowFn(beam.DoFn):
    """Convert each key/score pair into a BigQuery TableRow as specified."""
    def __init__(self, field_info):
      super(WriteWindowedToBigQuery.BuildRowFn, self).__init__()
      self.field_info = field_info

    def process(self, element, window=beam.DoFn.WindowParam):
      row = {}
      for entry in self.field_info:
        row[entry[0]] = entry[2](element, window)
      yield row

  def expand(self, pcoll):
    table = self.get_table(pcoll.pipeline)
    return (
        pcoll
        | 'ConvertToRow' >> beam.ParDo(
            WriteWindowedToBigQuery.BuildRowFn(self.field_info))
        | beam.io.Write(beam.io.BigQuerySink(
            table,
            schema=self.get_schema(),
            create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
            write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)))

def string_to_timestamp(datetime_str):
  dt = datetime.datetime.strptime(datetime_str, '%Y-%m-%d-%H-%M')
  epoch = datetime.datetime.utcfromtimestamp(0)
  return (dt - epoch).total_seconds() * 1000.0

class HourlyTeamScore(beam.PTransform):
  def __init__(self, start_min, stop_min, window_duration):
    super(HourlyTeamScore, self).__init__()
    self.start_min = start_min
    self.stop_min = stop_min
    self.window_duration = window_duration

  def expand(self, pcoll):
    start_min_filter = string_to_timestamp(self.start_min)
    end_min_filter = string_to_timestamp(self.stop_min)

    return (
        pcoll
        | 'ParseGameEvent' >> beam.ParDo(ParseEventFn())
        # Filter out data before and after the given times so that it is not
        # included in the calculations. As we collect data in batches (say, by
        # day), the batch for the day that we want to analyze could potentially
        # include some late-arriving data from the previous day. If so, we want
        # to weed it out. Similarly, if we include data from the following day
        # (to scoop up late-arriving events from the day we're analyzing), we
        # need to weed out events that fall after the time period we want to
        # analyze.
        | 'FilterStartTime' >> beam.Filter(
            lambda element: element['timestamp'] > start_min_filter)
        | 'FilterEndTime' >> beam.Filter(
            lambda element: element['timestamp'] < end_min_filter)
        # Add an element timestamp based on the event log, and apply fixed
        # windowing.
        # Convert element['timestamp'] into seconds as expected by
        # TimestampedValue.
        | 'AddEventTimestamps' >> beam.Map(
            lambda element: TimestampedValue(
                element, element['timestamp'] / 1000.0))
        # Convert window_duration into seconds as expected by FixedWindows.
        | 'FixedWindowsTeam' >> beam.WindowInto(FixedWindows(
            size=self.window_duration * 60))
        # Extract and sum teamname/score pairs from the event data.
        | 'ExtractTeamScore' >> ExtractAndSumScore('team'))

def run(argv=None):
  """Main entry point; defines and runs the hourly_team_score pipeline."""
  parser = argparse.ArgumentParser()

  # The default maps to two large Google Cloud Storage files (each ~12GB)
  # holding two subsequent day's worth (roughly) of data.
  parser.add_argument('--input',
                      dest='input',
                      default='gs://dataflow-samples/game/gaming_data*.csv',
                      help='Path to the data file(s) containing game data.')
  parser.add_argument('--dataset',
                      dest='dataset',
                      required=True,
                      help='BigQuery Dataset to write tables to. '
                           'Must already exist.')
  parser.add_argument('--table_name',
                      dest='table_name',
                      default='hourly_team_score',
                      help='The BigQuery table name. Should not already exist.')
  parser.add_argument('--window_duration',
                      type=int,
                      default=60,
                      help='Numeric value of fixed window duration, in minutes')
  parser.add_argument('--start_min',
                      dest='start_min',
                      default='1970-01-01-00-00',
                      help='String representation of the first minute after '
                           'which to generate results in the format: '
                           'yyyy-MM-dd-HH-mm. Any input data timestamped '
                           'prior to that minute won\'t be included in the '
                           'sums.')
  parser.add_argument('--stop_min',
                      dest='stop_min',
                      default='2100-01-01-00-00',
                      help='String representation of the first minute for '
                           'which to generate results in the format: '
                           'yyyy-MM-dd-HH-mm. Any input data timestamped '
                           'after to that minute won\'t be included in the '
                           'sums.')

  known_args, pipeline_args = parser.parse_known_args(argv)

  pipeline_options = PipelineOptions(pipeline_args)
  p = beam.Pipeline(options=pipeline_options)
  pipeline_options.view_as(SetupOptions).save_main_session = True

  (p  # pylint: disable=expression-not-assigned
   | ReadFromText(known_args.input)
   | HourlyTeamScore(
       known_args.start_min, known_args.stop_min, known_args.window_duration)
   | WriteWindowedToBigQuery(
       known_args.table_name, known_args.dataset, configure_bigquery_write()))

  result = p.run()
  result.wait_until_finish()

if __name__ == '__main__':
  logging.getLogger().setLevel(logging.INFO)
  run()