#!/usr/bin/env python
# coding: utf-8

# Importing the required libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import streamlit as st
import streamlit.components.v1 as components
from PIL import Image
import plotly.express as px
from streamlit_timeline import timeline
import plotly.graph_objects as go
import seaborn as sns
import plotly.figure_factory as ff

###########################

st.set_page_config(layout="wide")

st.title('How various factors have influenced the housing prices over the past 2 decades - A case Study 📔')

st.sidebar.markdown('''
# Contents
- [Introduction](#introduction)
- [Brief Timeline](#a-brief-timeline-of-major-economic-events)
- [The Past 2 Decades](#factors-and-how-they-influenced-the-past-2-decades)
    - [Demand](#demand)
        - [Population](#population)
        - [Unemployment](#unemployment)
        - [Income](#income)
        - [Mortgage Rate](#mortgage-rate)
        - [Debt](#household-debt)
    - [Supply](#supply)
        - [Permits](#new-single-family-unit-permits)
        - [Existing home sales](#existing-home-sales)
        - [Foreclosures](#foreclosures)
- [Most Influencial Factor](#determining-the-most-influencial-factor)
    - [Correlation Analysis](#correlation-analysis)
    - [Explanatory Regression Analysis](#explanatory-regression-analysis)
- [Conclusion](#conclusion)
- [Biblography](#biblography)

''', unsafe_allow_html=True)

###########################

st.header('Introduction')
col1,col2 = st.columns((2,1))
col1.write('''

Saying that the housing market is influenced by a myriad of factors could be an understatement.
The entire process of reverse engineering something as complex and as complicated as a housing market requires in-depth and
unbiased analysis which in turn produces credible predictive analytics.

The **S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index** measures the change in the value of the U.S.
residential housing market by tracking the purchase prices of single-family homes, which I have used as a proxy for home prices.

As of October 2021, there were 208.4 million single-family dwelling units in the United States and only 37.8 million multifamily units.
**The size of single-family housing units has steadily risen since the start of the 21st century.**

There is an upward trend in the average size of floor area in new single-family homes. In 1975, a new home had an average floor area of about 1,660 square feet.
By 2019, this figure had **increased by** about 900 square feet.

''')

labels = ['Multi-family dwelling unit', 'Single Family dewlling unit', 'Others']
sizes = [37.75, 208.36, 10.67]
explode = (0, 0.1, 0)  # only "explode" the 2nd slice

fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
ax1.set_title("The single-family units as a fraction of all the housing units in USA", pad = 20)

col2.pyplot(fig1)

st.write('''
The interavtive rgraph below shows how the S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index has changed in the past 2 years.
The data has been adjusted to show quaterly mean.
''')

col1,col2 = st.columns((2,1))

df = pd.read_csv('./data/cleaned/snp.csv')
df.rename(columns = {'CSUSHPISA':'Index'}, inplace = True)
fig = px.line(df, x='DATE', y='Index')
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
col1.plotly_chart(fig)

col2.markdown('''_
\n
The Case Shiller Home Price Index averaged 169.87 points from 2000 until 2021.
\n It reached an all-time high of 257.10 points in April 2021.
\nIt reached a record low of 100 points in January 2000.
''')

###########################
st.header('A Brief Timeline ⏱ of Major Economic Events')
with open('data.json', "r") as f:
    data = f.read()

# render timeline
timeline(data, height=800)

#####################
st.header('Factors and How They Influenced The Past 2 Decades 📊')
st.header('Population 🧑🏻🤝🧑🏾')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The U.S. population increased by 10 percent between 2000 and 2010 and is projected to increase by 8 percent between 2010 and 2020, from 309 million to 333 million. An 8 percent gain would be the smallest percentage increase in the U.S. population between censuses since the 1930s.
\nThe population-housing relationship is a two-way street. On the one hand, population growth causes a shift in housing demand. Population growth, and especially an increase in the number of households, leads to an increase in housing demand.
\nPopulation decline might lead to a decrease in housing demand. This will, however, only happen in the long run, after not only the number of people but also the number of households has started to decline. The danger of population decline is greatest in remote rural areas and in areas with lower-quality housing.
At the same time, the supply of housing influences the opportunities for population increase through migration.

''')

df = pd.read_csv('./data/cleaned/population_q_cleaned.csv')
df.rename(columns = {'POPTHM':'Population'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

col1,col2 = st.columns((3,4))
df = pd.read_csv('./data/cleaned/pops.csv')
df.rename(columns = {'CNP16OV_TTLHH':'No. of members in a household'}, inplace = True)
fig = px.line(df, x='Date', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
with col2:
    st.plotly_chart(fig)

with col1:
    st.write('**Poplulation level/Total household** 🧑🏻🤝🧑🏾/🏠')
    st.write('''
    The graph shows that the number of people in each household has decreased in the longer time period. So the number of households in the US has increased faster than population, which means that any measure divided by population grows faster than one divided by number of households.
    ''')

#####################
st.header('Unemployment')
colm1,colm2 = st.columns((3,4))
df = pd.read_csv('./data/cleaned/unemp.csv')
df.rename(columns = {'UNRATE':'Unemployment Rate in %'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

with colm1:
    components.iframe("https://d3fy651gv2fhd3.cloudfront.net/embed/?s=usurtot&v=202203041337V20200908&d1=19970314&type=type=line&title=false&url2=/united-states/case-shiller-home-price-index&h=300&w=600",width=600, height=300, scrolling=False)
    st.write("Unemployment tracks the business cycle. Recessions are part of that cycle and can cause high unemployment. Businesses often lay off workers and, without an income, those jobless workers have less money to spend.")

st.write('''
Lower consumer spending reduces business revenue, which forces companies to cut more payroll. This downward cycle can be devastating to individuals and the economy.
When unemployment is rising, fewer people will be able to afford a house. But, even the fear of unemployment may discourage people from entering the property market.
An increase in jobs, employees and wages consequently results in an increase in income for all workers in a country. With that, an increase in income also increases the demand for affordable housing.
\nDuring the Great Recession, unemployment reached 10% in October 2009.
In 2020, it reached double digits again at 14.7% in April when the U.S. was dealing with a pandemic and recession.
The US unemployment rate edged down to 3.8 percent in February of 2022 from 4 percent in the previous month, a new pandemic low and below market expectations of 3.9 percent. The number of unemployed persons edged down by 243 thousand to 6.270 million.

''')
############################
st.header('Income 💵')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The shortfall in household income is attributable in part to two recessions since 2000. The first recession, lasting from March 2001 to November 2001, was relatively short-lived.7 Yet household incomes were slow to recover from the 2001 recession and it was not until 2007 that the median income was restored to about its level in 2000.

\nBut 2007 also marked the onset of the Great Recession, and that delivered another blow to household incomes. This time it took until 2015 for incomes to approach their pre-recession level. Indeed, the median household income in 2015 – $70,200 – was no higher than its level in 2000, marking a 15-year period of stagnation, an episode of unprecedented duration in the past five decades.8

\nMore recent trends in household income suggest that the effects of the Great Recession may finally be in the past. From 2015 to 2018, the median U.S. household income increased from 70,200 to 74,600 dollars, at an annual average rate of 2.1%.
''')

col1,col2 = st.columns((3,4))
with col2:
    img = Image.open("aff.png")
    st.image(img)

with col1:
    st.write('**Affordability of Homes**')
    st.write('''

Steady job growth and an increase in the median wage continue to fuel housing demand. Aside from rising wages, another good sign of financial health is spending less disposable income on repaying debts like credit cards, auto loans, and personal loans.

Today, household budgets continue to be the strongest they’ve been in over 40 years and a far cry from 2008, suggesting consumers have more disposable income to buy a home.

    ''')

df = pd.read_csv('./data/cleaned/incomenew.csv')
df.rename(columns = {'MEHOINUSA672N':'Median Income'}, inplace = True)
df.rename(columns = {'HOUSTCB1FQ_HOUSTOB1FQ':'New Privately Owned Homes'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
###########################

st.header('Mortgage Rate')
colm1,colm2 = st.columns((3,4))
colm1.write('''
Monetary policy affects interest rates, which affect mortgages, which affect housing market decisions. That may be simple to grasp, but the housing statistics may not follow such clear-cut patterns.

\nThe graph's red line represents the average 30-year fixed-rate mortgage for the last 20 years. For the same time period, the blue line in the graph represents the ratio of house starts built by contractors to housing starts built by owners.

\nFrom 2000 to 2007, this ratio remained largely steady, hovering around 1.5, meaning that contractors built roughly 60% of new starts. However, during periods of macroeconomic turmoil, the ratio has deviated from its historical average.
\nBut during the Great Recession, this ratio increased sharply, to over 2.0, peaking at 2.6 in 2016, which implies contractors built 72% of housing starts. In both cases, GDP declined and unemployment rose, but this housing measure behaved differently.
''')

st.write('''
A clear difference between these two episodes is the level of mortgage rates: Rates were much higher in the Great Recession. As mortgage rates go up, the ratio goes down and vice versa.
A potential reason is that, as the price of mortgages increases, the cost of purchasing a new home from a contractor increases relative to the cost of building one’s own home. And, if the costs are basically the same,
many would-be homeowners might choose to build their own home rather than purchase one that someone else built.
''')

df = pd.read_csv('./data/cleaned/dd.csv')
df.rename(columns = {'MORTGAGE30US':'Mortgage Rate'}, inplace = True)
df.rename(columns = {'HOUSTCB1FQ_HOUSTOB1FQ':'New Privately Owned Homes'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

#####################

st.header('Household Debt')
colm1,colm2 = st.columns((3,4))
colm1.write('''
There are many types of debt, including household debt, and many specific types of household debt as well.
\nThe graph on the right shows Mortgage Debt Service Payments as a Percent of Disposable Personal Income.
\nThe financial burdens from mortgages debt vary quite a bit. Let’s consider two reasons for this: The larger
the debt, the larger the burden, as households need to pay more interest on a larger principal. And changes
in interest rates obviously influence how much is paid to service loans.\n The mortgage debt increased from the early 1990s until the past recession,
when it decreased. This decrease is the result of the combination of the two effects noted above: the amount of debt and interest rates.
With one exception (in the fourth quarter of 2012), total debt obligations are at the lowest they’ve been since these data were first
collected. And this is especially true of mortgage debt.

''')

df = pd.read_csv('./data/cleaned/debt_inc.csv')
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)

labels = ['Mortgage','Student','Others']
values = [10.44, 1.58, 3.22]
# Use `hole` to create a donut-like pie chart
fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.3)])

col1,col2 = st.columns((1,2))
with col2:
    st.plotly_chart(fig)

with col1:
    st.write('**Mortgage debt as a fraction of total household debt**')
    st.write('''
    Mortgage Debt Service Payments as a Percent of Disposable Personal Income was 3.81% in April of 2021, according to the United States Federal Reserve.\nHistorically, United States - Mortgage Debt Service Payments as a Percent of Disposable Personal Income reached a record high of 7.18 in October of 2007 and a record low of 3.51 in January of 2021.
    By the end of 2021, a record level of household debt was reached. Mortgage being the biggest contributor to it being 68.5% at 10.44 Trillion Dollars. Student debt was the second highest contributor with 21.1% at 1.58 Trillion Dollars.

    ''')

##############################

st.header('New Single Family Unit Permits 📋')
colm1,colm2 = st.columns((3,4))
colm1.write('''
\nSingle-family housing starts were pridicted to total 1 million in 2020, the highest since 2007. However,
There were only approximately 978,000 building permits for single-family housing units granted in the United States in 2020, which is
an increase of around 116,000 on the previous year. Units have steadily increased since 2011, but the numbers remain considerably
lower than the 1.68 million authorized by building permits in 2005.
\nThat was about 3 years prior to the housing market meltdown that spurred a global financial rout.
\nThe rising cost of land, labor, building materials, and regulations, are some of the reasons why we can see significantly more blue (people who may need homes) in the graph below, than orange (total homes built) since 2008. You can also see that few of the new homes built were “starter homes” focused on first-time buyers.
''')

df = pd.read_csv('./data/cleaned/permit.csv')
df.rename(columns = {'PERMIT1':'No. of permits issued in thousands'}, inplace = True)
df.rename(columns = {'HSN1F':'No. of one family homes sold in thousands'}, inplace = True)
fig = px.line(df, x='DATE', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
col1,col2 = st.columns((1,1))
img = Image.open('new_perms.png')
with col1:
    st.image(img)

with col2:
    st.write('''
    \n Even though the new homes built by 2020 were the fastest rate since 2007, Single-family home construction is running at the slowest pace since 1995.
    \nThe U.S. is short 5.24 million homes, an increase of 1.4 million from the 2019 gap of 3.84 million.
    \nSingle-family home construction has suffered from a severe labor shortage that began well before the pandemic but was then exacerbated by it. Supply chain disruptions in the past year have pushed prices for building materials higher, and as pandemic-induced demand soared, prices for land increased as well. While new household formation is actually slower than it was before the pandemic, homebuilders would have to double their recent new home production pace to close the gap in five to six years. A new household can be either owner-occupied or rented.
    ''')

################################

st.header('Existing Home Sales')
colm1,colm2 = st.columns((3,4))
colm1.write('''
The steady rise in sales after the sharp drop in 2008 is indicative of the general consensus that the housing market is recovering. Construction is showing positive signs, consumers are growing in confidence and are becoming freer with their spending and the market is entering new periods of growth.
\n
This is a far cry from the dire situation in the not too distant past, in the run up to the bursting of the U.S. housing bubble. Interest rates were very low at that time, making credit cheap and abundantly available. Banks and lending institutions led people to believe that it was okay to buy multiple properties with little money and that real estate was just about the safest investment anyone could make. More and more people decided to take the risk and invest in the market. This coupled with the increased number of people descending on the market caused prices to soar; it seemed like an easy way to make cash fast. But this is how the bubble formed and it was this bubble, upon bursting, that would set into motion a chain of events that would bring the global economy to its knees; plunging the world into an economic depression of which it has not seen the likes since the Great Depression of the 1930s.
''')

df = pd.read_csv('./data/cleaned/existinghomessold.csv')
fig = px.line(df, x='Year', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
col1,col2 = st.columns((2,1))

labels = ['Exisiting Homes Sold','New Homes Sold']
values = [5640000, 822000]

fig = go.Figure(data=[go.Pie(labels=labels, values=values, textinfo='label+percent',
                             insidetextorientation='radial'
                            )])
col1.plotly_chart(fig)

with col2:
    st.write('''
    **U.S. existing home sales 2005-2023**
    \nIn 2021, the U.S. home sales in the United States surged, reaching the highest value observed since 2006. A total of 6.1 million housing transactions were completed in that year, up from 5.6 million in 2020. According to the forecast, sales activity is expected to slow down slightly in 2022 and increase again in 2023.
    ''')

##################################

st.header('Foreclosures 🏦')
colm1,colm2 = st.columns((3,4))
colm1.write('''
Despite a deep recession, the U.S. housing market in 2020 set a record for the fewest foreclosures ever.

Just 214,323 properties were in some stage of foreclosure last year, well below the previous low set in 2019, according to ATTOM Data Solutions. Properties with foreclosure filings in 2020 represented a scant 0.16 percent of all U.S. homes, down from a foreclosure rate of 0.36 percent in 2019. The record high was 2.23 percent, in 2010.

In a separate study released Friday, mortgage data firm Black Knight said foreclosure starts plunged by 67 percent from 2019, while foreclosure sales dropped by 70 percent compared to the previous year.

However, the data isn’t all rosy. Fully 2.15 million American homeowners were 90 days past due on their mortgages, a number that rose by 1.7 million during the year of the coronavirus pandemic, Black Knight said.

Foreclosures fell not because all was well economically, but because lenders essentially stopped taking back properties in 2020. After the coronavirus pandemic struck in March, federal and state officials hit the pause button on default filings by lenders. And the CARES Act called for mortgage forbearance plans designed to keep struggling workers in their homes.
''')

df = pd.read_csv('./data/cleaned/foreclosures.csv')
fig = px.line(df, x='Year', y=df.columns)
fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=list([
            dict(count=6, label="6m", step="month", stepmode="backward"),
            dict(count=1, label="YTD", step="year", stepmode="todate"),
            dict(count=1, label="1y", step="year", stepmode="backward"),
            dict(step="all")
        ])
    )
)
colm2.plotly_chart(fig)
##
col1, col2 = st.columns((2,1))

img = Image.open('subprime_mort.png')
with col1:
    st.image(img)

col2.write('''
    \n The mortgage delinquency rate is the share of the total number of mortgaged home loans in the U.S. where payment is overdue by 30 days or more. Many borrowers are eventually able to service their loan though, with foreclosure rates being generally 50-75 percent lower than delinquency rates. Total home mortgage debt in the U.S. stood at 10.94 trillion U.S. dollars in 2020.
    \n ‘Subprime’ loans, being targeted at high-risk borrowers and generally coupled with higher interest rates to compensate for the risk, have far higher delinquency rates than conventional loans. Defaulting on such loans was one of the triggers for the 2007-2010 financial crisis, with subprime delinquency rates reaching almost 26 percent around this time. These higher delinquency rates translate into higher foreclosure rates, which peaked at just under 15 percent of all subprime mortgages in 2011.
    ''')
st.write("In the second quarter of 2020, under the effects of the coronavirus crisis, the mortgage delinquency rate in the United States spiked at 8.22 percent, just one percent down from its peak of 9.3 percent during the subprime mortgage crisis of 2007-2010. Following the drastic increase directly after the outbreak of the pandemic, delinquency rates started gradually declining and reached 4.65 percent as of the fourth quarter of 2021. ")

##############################
st.header('Determining the most influencial factor')
st.write('''
To Determine the most influencial factor of all the forementioned factors,
all the data for the factors were resampled to a similar time intervals and combined into a single dataframe:
''')
df = pd.read_csv('finalcombined.csv')
st.dataframe(df)

st.write('Here CUSUSHPISA is the S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index')

desc = df.describe()
st.dataframe(desc)

corr = df.corr()
corr_inf = pd.DataFrame(corr.CSUSHPISA)

st.header("Correlation Analysis")
col1,col2 = st.columns((1,1))
col1.write('''

''')
col1.dataframe(corr_inf,  height=700)

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
fig = plt.subplots()

sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
plt.savefig('corr_dia1.png')
img = Image.open('corr_dia1.png')
col2.image(img)

col11,col12 = st.columns((2,1))
corr_inf_new = corr_inf.head(10)
fig = px.bar(corr_inf_new)
col11.plotly_chart(fig)
col12.title("_")
col12.write('''
This analysis shows how the factors are correlated with the index and amongst themselves. Higher the bars in the graph
(in magnitude) on the left, the more correlated the factors are with the index.
\n For all the plots above the 0 value, there is a positive correlation and vice-versa.
''')

col1, col2 = st.columns((2,1))
col1.write('''
The absolute values indicate the level of correlation and the sign shows the relation, i.e either positive or negative correlation.
\n The most correlated factors(in order) are:
''')
data = abs(corr_inf_new)
sorted_data = data.sort_values(by=['CSUSHPISA'], ascending=False)
sorted_data_new = corr_inf_new.sort_values(by=['CSUSHPISA'], ascending=False)
col2.dataframe(sorted_data, height=700)
top_5 = sorted_data.head(5)

st.header("Explanatory Regression Analysis")

st.write('''
Next, a multiple regression analysis was performed on the cleaned and combined data. Given in the case, in which we would like to predict the prices
of the index, a multiple regression model would have been created and only the factor(s) influencing it the most would be chosen. Here, only the analysis
is performed in order to examine whether:\n 1) Is it better to take all the factors into consideration?
\n2) Is it better to take only the top 5 highest correlated data into consideration?
\n 3) Is it better to just take one factor into consideration?
\n Each of the cases is examined and the output is explained below.
''')
##
col1,col2 = st.columns((1,1))

col1.write('''
Case 1) Building a multiple regression model taking all the factors into consideration:
''')

col1.dataframe(corr_inf_new, height=700)

img_mlr_2 = Image.open('MLR_1.png')
col2.image(img_mlr_2)
##
col1.write('''
Case 2) The asbolute values of the correlation table have been sorted and the 5 factors with the highest correlation are represented in the table
below. When these factors(only) are taken into consideration:
''')

col1.dataframe(top_5)

img_mlr_2 = Image.open('MLR_5.png')
col2.image(img_mlr_2)
##
col1.write('''
Case 3) Only the most correlated factor i.e. the population was taken into consideration:
''')

data = abs(corr_inf_new)
sorted_data = data.sort_values(by=['CSUSHPISA'], ascending=False)
top_1 = sorted_data.head(1)
col1.dataframe(top_1)

img_mlr_2 = Image.open('MLR_O.png')
col2.image(img_mlr_2)

##
st.header("Conclusion")
st.write('''
It is known that for a regression model the R-Squared value always increases and never decres incase the number of variables are increased.
To make up for this, the Adjusted R-Squared penalises the R-Squared based on the number of variables. A very low Adjusted R-Square may indicate that some
of the variables are not contributing. And as we take more variables the R-Squared value rises, which is represented in the above cases.
However, in the case 1, the Adjusted R-Squared value is very low, indicating that there is no variable that is not contributing at all.
\n The probablistic F-Static value for case 3 is lesser than that of case 1, indicating that the most correlated field, "Population",
does have great influence on the index. This also allows us to conclude, that the results of the correlation analysis are true and are proven here.
Thus, we can say that the most import factors in order influencing the *S&P CoreLogic Case-Shiller U.S. National Home Price NSA Index* are the most
correlated factors. And that Multiple regression analysis is only useful in quantifying relationships in factors when there is a need to do predective
modelling and not singling out one most important variable out of multiple variables.
''')

##############################
st.header('Biblography 📚')
html_string = '''<html>
    <head>
        <title> Biblography </title>
    </head>
    <body>
        <table style="border-width: 1;">
            <th style="background-color:lightgray;"> Refernces</th>
            <tr>
                <td>
                    <a href="http://fred.stlouisfed.org">fred.stlouisfed.org</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://www.statista.com">statista.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="www.opendoor.com">opendoor.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="http://nar.realtor">nar.realtor</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://tradingeconomics.com/">tradingeconomics.com</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://data.worldbank.org">data.worldbank.org</a>
                </td>
            </tr>
            <tr>
                <td>
                    <a href="https://www.cnbc.com">cnbc.com</a>
                </td>
            </tr>
        </table>
    </body>
</html>'''
st.markdown(html_string, unsafe_allow_html=True)


#!/usr/bin/env python
# coding: utf-8

# # Text Generation using LSTMs

# ## 1. Import the libraries

# In[ ]:

# keras module for building LSTM
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Embedding, LSTM, Dense, Dropout
from keras.preprocessing.text import Tokenizer
from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
import keras.utils as ku

# set seeds for reproducability
from tensorflow import set_random_seed
from numpy.random import seed
set_random_seed(2)
seed(1)

import pandas as pd
import numpy as np
import string, os

import warnings
warnings.filterwarnings("ignore")
warnings.simplefilter(action='ignore', category=FutureWarning)

# ## 2. Load the dataset

# In[ ]:

curr_dir = "../../../input/kingburrito666_shakespeare-plays/"
play_df = pd.read_csv(curr_dir + "Shakespeare_data.csv")

all_lines = [h for h in play_df.PlayerLine]

print(len(all_lines))

# ## 3. Dataset preparation

# First, we will clean the data.

# In[ ]:

def clean_text(txt):
    txt = "".join(v for v in txt if v not in string.punctuation).lower()
    txt = txt.encode("utf8").decode("ascii",'ignore')
    return txt

corpus = [clean_text(x) for x in all_lines]
corpus[:10]

# Next we will generate sequence of N-gram tokens using Keras' Tokenizer.

# In[ ]:

tokenizer = Tokenizer()

def get_sequence_of_tokens(corpus):
    ## tokenization
    corpus = corpus[:7000]
    tokenizer.fit_on_texts(corpus)
    total_words = len(tokenizer.word_index) + 1

    ## convert data to sequence of tokens
    input_sequences = []
    for line in corpus:
        token_list = tokenizer.texts_to_sequences([line])[0]
        for i in range(1, len(token_list)):
            n_gram_sequence = token_list[:i+1]
            input_sequences.append(n_gram_sequence)
    return input_sequences, total_words

inp_sequences, total_words = get_sequence_of_tokens(corpus)
inp_sequences[:10]

# Next we will generate padded sequences.

# In[ ]:

def generate_padded_sequences(input_sequences):
    max_sequence_len = max([len(x) for x in input_sequences])
    input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))

    predictors, label = input_sequences[:,:-1],input_sequences[:,-1]
    label = ku.to_categorical(label, num_classes=total_words)
    return predictors, label, max_sequence_len

predictors, label, max_sequence_len = generate_padded_sequences(inp_sequences)
predictors.shape, label.shape

# ## 4. Using LSTM for text generation

# In[ ]:

def create_model(max_sequence_len, total_words):
    input_len = max_sequence_len - 1
    model = Sequential()

    # Add Input Embedding Layer
    model.add(Embedding(total_words, 10, input_length=input_len))

    # Add Hidden Layer 1 - LSTM Layer
    model.add(LSTM(512))
    model.add(Dropout(0.4))

    # Add Output Layer
    model.add(Dense(total_words, activation='softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam')

    return model

model = create_model(max_sequence_len, total_words)
model.summary()

# In[ ]:

model.fit(predictors, label, epochs=2, verbose=1)

# In[ ]:

model.fit(predictors, label, epochs=20, verbose=2)

# In[ ]:

model.fit(predictors, label, epochs=20, verbose=0)

# ## 5. Generating the text

# In[ ]:

def generate_text(seed_text, next_words, model, max_sequence_len):
    for _ in range(next_words):
        token_list = tokenizer.texts_to_sequences([seed_text])[0]
        token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
        predicted = model.predict_classes(token_list, verbose=0)

        output_word = ""
        for word,index in tokenizer.word_index.items():
            if index == predicted:
                output_word = word
                break
        seed_text += " "+output_word
    return seed_text.title()

# In[ ]:

print ("1. ",generate_text("Julius", 20, model, max_sequence_len))
print ("2. ",generate_text("Thou", 20, model, max_sequence_len))
print ("3. ",generate_text("King is", 20, model, max_sequence_len))
print ("4. ",generate_text("Death of", 20, model, max_sequence_len))
print ("5. ",generate_text("The Princess", 20, model, max_sequence_len))
print ("6. ",generate_text("Thanos", 20, model, max_sequence_len))

# In[ ]:


#  Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import numpy as np
import unittest
import sys

sys.path.append("..")
from op_test import OpTest, skip_check_grad_ci
import paddle
import paddle.fluid as fluid

paddle.enable_static()

SEED = 2021
EPOCH = 100

class TestDropoutOp(OpTest):

    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64)).astype('uint8')
        }

    def init_dtype(self):
        self.dtype = np.float32

    def set_npu(self):
        self.__class__.use_npu = True
        self.place = paddle.NPUPlace(0)

    def test_check_output(self):
        self.check_output_with_place(self.place)

    def test_check_grad_normal(self):
        self.check_grad_with_place(self.place, ['X'], 'Out')

class TestDropoutOpInput1d(TestDropoutOp):
    # change input shape
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((3, 62)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((3, 62)).astype('uint8')
        }

class TestDropoutOpInput1d_1(TestDropoutOp):
    # the input is 1-D
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((2000)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((2000)).astype('uint8')
        }

class TestDropoutOp2(TestDropoutOp):
    # the dropout_prob is 1.0
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 1.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': np.zeros((32, 64)).astype('float32'),
            'Mask': np.zeros((32, 64)).astype('uint8')
        }

class TestDropoutOp3(TestDropoutOp):
    # the input dim is 3
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((32, 64, 2)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64, 2)).astype('uint8')
        }

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOpInference(OpTest):
    # is_test = True
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((32, 64)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.35,
            'fix_seed': True,
            'is_test': True,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {'Out': self.inputs['X']}

    def init_dtype(self):
        self.dtype = np.float32

    def set_npu(self):
        self.__class__.use_npu = True
        self.place = paddle.NPUPlace(0)

    def test_check_output(self):
        self.check_output_with_place(self.place)

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOpInference2(TestDropoutOpInference):

    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {'X': np.random.random((32, 64, 3)).astype(self.dtype)}
        self.attrs = {
            'dropout_prob': 0.75,
            'is_test': True,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {'Out': self.inputs['X']}

class TestDropoutOpWithSeed(TestDropoutOp):
    # the seed is a Tensor
    def setUp(self):
        self.op_type = "dropout"
        self.set_npu()
        self.init_dtype()
        self.inputs = {
            "X": np.random.random((32, 64)).astype(self.dtype),
            "Seed": np.asarray([125], dtype="int32")
        }
        self.attrs = {
            'dropout_prob': 0.0,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64)).astype('uint8')
        }

class TestDropoutOpFp16(TestDropoutOp):
    # float16
    def init_dtype(self):
        self.dtype = np.float16

    def set_npu(self):
        self.__class__.use_npu = True
        self.__class__.no_need_check_grad = True
        self.place = paddle.NPUPlace(0)

class TestDropoutAPI(unittest.TestCase):

    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace(), paddle.NPUPlace(0)]

    def check_static_result(self, place):
        with fluid.program_guard(fluid.Program(), fluid.Program()):
            input = fluid.data(name="input", shape=[40, 40], dtype="float32")
            res1 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                training=False,
                                                mode='upscale_in_train')
            res2 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                axis=0,
                                                training=True,
                                                mode='upscale_in_train')
            res3 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                axis=0,
                                                training=False,
                                                mode='upscale_in_train')
            res4 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                axis=[0, 1],
                                                training=True,
                                                mode='upscale_in_train')
            res5 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                axis=[0, 1],
                                                training=False,
                                                mode='upscale_in_train')
            res6 = paddle.nn.functional.dropout(x=input,
                                                p=1.,
                                                training=True,
                                                mode='upscale_in_train')
            res7 = paddle.fluid.layers.dropout(
                x=input,
                dropout_prob=0.,
                dropout_implementation='upscale_in_train')
            res8 = paddle.nn.functional.dropout(x=input,
                                                p=0.,
                                                axis=(0, 1),
                                                training=False,
                                                mode='upscale_in_train')

            in_np = np.random.random([40, 40]).astype("float32")
            res_np = in_np
            res_np2 = np.zeros_like(in_np)

            exe = fluid.Executor(place)
            res_list = [res1, res2, res3, res4, res5, res7, res8]
            for res in res_list:
                fetches = exe.run(fluid.default_main_program(),
                                  feed={"input": in_np},
                                  fetch_list=[res])
                self.assertTrue(np.allclose(fetches[0], res_np))
            fetches2 = exe.run(fluid.default_main_program(),
                               feed={"input": in_np},
                               fetch_list=[res6])
            self.assertTrue(np.allclose(fetches2[0], res_np2))

    def test_static(self):
        for place in self.places:
            self.check_static_result(place=place)

if __name__ == '__main__':
    unittest.main()


# SPDX-License-Identifier: Apache-2.0

"""
What is the opset number?
=========================

.. index:: opset, target opset, version

Every library is versioned. :epkg:`scikit-learn` may change
the implementation of a specific model. That happens
for example with the `SVC <https://scikit-learn.org/stable/
modules/generated/sklearn.svm.SVC.html>`_ model where
the parameter *break_ties* was added in 0.22. :epkg:`ONNX`
does also have a version called *opset number*.
Operator *ArgMin* was added in opset 1 and changed in opset
11, 12, 13. Sometimes, it is updated to extend the list
of types it supports, sometimes, it moves a parameter
into the input list. The runtime used to deploy the model
does not implement a new version, in that case, a model
must be converted by usually using the most recent opset
supported by the runtime, we call that opset the
*targeted opset*. An ONNX graph only contains
one unique opset, every node must be described following
the specifications defined by the latest opset below the
targeted opset.

This example considers an `IsolationForest
<https://scikit-learn.org/stable/modules/generated/
sklearn.ensemble.IsolationForest.html>`_ and digs into opsets.

.. contents::
    :local:

Data
++++

A simple example.
"""
from onnx.defs import onnx_opset_version
from skl2onnx import to_onnx
import numpy
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest
from sklearn.datasets import make_blobs

X, y = make_blobs(n_samples=100, n_features=2)

model = IsolationForest(3)
model.fit(X)
labels = model.predict(X)

fig, ax = plt.subplots(1, 1)
for k in (0, 1):
    ax.plot(X[labels == k, 0], X[labels == k, 1], 'o', label="cl%d" % k)
ax.set_title("Sample")

#######################################
# ONNX
# ++++

onx = to_onnx(model, X[:1].astype(numpy.float32))
print(onx)

##########################
# The last line shows the opsets.
# Let's extract it.

domains = onx.opset_import
for dom in domains:
    print("domain: %r, version: %r" % (dom.domain, dom.version))

###################################
# There are two opsets, one for standard operators,
# the other for machine learning operators.

########################################
# ONNX and opset
# ++++++++++++++
#
# The converter can convert a model to an older opset
# than the default one, from 1 to the last available one.

def get_domain_opset(onx):
    domains = onx.opset_import
    res = [{'domain': dom.domain, 'version': dom.version}
           for dom in domains]
    return {d['domain']: d['version'] for d in res}

for opset in range(1, onnx_opset_version() + 1):
    try:
        onx = to_onnx(model, X[:1].astype(numpy.float32), target_opset=opset)
    except RuntimeError as e:
        print('target: %r error: %r' % (opset, e))
        continue
    nodes = len(onx.graph.node)
    print('target: %r --> %s %d' % (opset, get_domain_opset(onx), nodes))

########################################
# It shows that the model cannot be converted for opset
# below 5. Operator `Reshape <https://github.com/onnx/
# onnx/blob/master/docs/Operators.md#Reshape>`_ changed in
# opset 5: a parameter became an input. The converter
# does not support *opset < 5* because runtimes usually do not.
#
# Other opsets
# ++++++++++++
#
# The previous example changed the opset of the main domain
# ``''`` but the other opset domain can be changed as well.

for opset in range(9, onnx_opset_version() + 1):
    for opset_ml in range(1, 3):
        tops = {'': opset, 'ai.onnx.ml': opset_ml}
        try:
            onx = to_onnx(
                model, X[:1].astype(numpy.float32), target_opset=tops)
        except RuntimeError as e:
            print('target: %r error: %r' % (opset, e))
            continue
        nodes = len(onx.graph.node)
        print('target: %r --> %s %d' % (opset, get_domain_opset(onx), nodes))


"""
    This file is copied/apdated from https://github.com/berkeleydeeprlcourse/homework/tree/master/hw3
"""
import numpy as np
import random

def set_global_seeds(i):
    try:
        import torch
    except ImportError:
        pass
    else:
        torch.manual_seed(i)
    np.random.seed(i)
    random.seed(i)


# This class fournishes methods to plot your simulation relevant data
import matplotlib.pyplot as plt
import numpy as np
plt.ion() # activating interactive plotting
# import time

class SimPlot:

    # Constructor method
    # -- Input
    #   - filterType: 1 for KF, 2 for EKF, 3 for PF
    def __init__(self, filterType):

        # Instantiating useful variables
        self.ax = [None]
        self.fig = [None]
        self.lines = [None]
        self.scatters = [None]

        # Instantiating the correctly filter plot style
        if filterType == 1:
            self.kf_initiate()
        elif filterType == 2:
            self.ekf_initiate()
        elif filterType == 3:
            self.pf_initiate()
        else:
            print('SimPlot constructed with unknown arguments.')

    def __del__(self):

        # plt.waitforbuttonpress()
        pass

    # ===== Method - Kalman filter plot initiation
    def kf_initiate(self):

        # self.fig[0] = plt.figure()
        #
        # self.ax[0] = self.fig[0].add_subplot(1, 1, 1)
        #
        # # defining some plot characteristics

        # # self.ax[0].set_xlim(-6, 6)
        # # self.ax[0].set_ylim(-6, 6)
        # self.ax[0].grid(True)
        #
        # # Displays the plot
        # self.fig[0].show()

        # creating the figure
        self.fig[0] = plt.figure()

        # creating the axes region
        self.ax[0] = self.fig[0].add_subplot(1, 1, 1)

        # defining some plot parameters
        self.ax[0].set_xlim(-6, 6)
        self.ax[0].set_ylim(-6, 6)
        self.ax[0].set_xlabel('pos x [m]')
        self.ax[0].set_ylabel('pos y [m]')
        self.ax[0].set_title('Kalman Filter')
        self.ax[0].grid(True)

        # creating robot real position line
        self.lines[0], = self.ax[0].plot([], [], '--r')

        # creating measurement plot line
        self.lines.extend(self.ax[0].plot([], [], 'og'))

        # creating estimated plot line
        self.lines.extend(self.ax[0].plot([], [], '-b'))

    def kf_draw(self, x_real, x_estimated, z_t):

        # updating  robot_realposition plot
        self.lines[0].set_xdata(x_real[0, :])
        self.lines[0].set_ydata(x_real[1, :])

        # updating reading plot
        self.lines[1].set_xdata(z_t[0])
        self.lines[1].set_ydata(z_t[1])

        # updating estimation plot
        self.lines[2].set_xdata(x_estimated[0, :])
        self.lines[2].set_ydata(x_estimated[1, :])

        # self.ax[0].relim()
        # self.ax[0].autoscale_view()

        # updating the figure
        self.fig[0].canvas.draw()
        self.fig[0].canvas.flush_events()

        plt.pause(0.05)

    # ===== Method - Extended Kalman filter plot initiation
    def ekf_initiate(self):

        # creating the figure
        self.fig[0] = plt.figure()

        # creating the axes region
        self.ax[0] = self.fig[0].add_subplot(1, 1, 1)

        # defining some plot parameters
        self.ax[0].set_xlim(-6, 6)
        self.ax[0].set_ylim(-6, 6)
        self.ax[0].set_xlabel('pos x [m]')
        self.ax[0].set_ylabel('pos y [m]')
        self.ax[0].set_title('Extended Kalman Filter')
        self.ax[0].grid(True)

        # creating robot real position line
        self.lines[0], = self.ax[0].plot([], [], '--r')

        # creating measurement plot line
        self.lines.extend(self.ax[0].plot([], [], 'og'))

        # creating estimated plot line
        self.lines.extend(self.ax[0].plot([], [], '-b'))

    def ekf_draw(self, x_real, x_estimated):

        # updating  robot_realposition plot
        self.lines[0].set_xdata(x_real[0, :])
        self.lines[0].set_ydata(x_real[1, :])

        # updating estimation plot
        self.lines[2].set_xdata(x_estimated[0, :])
        self.lines[2].set_ydata(x_estimated[1, :])

        # self.ax[0].relim()
        # self.ax[0].autoscale_view()

        # updating the figure
        self.fig[0].canvas.draw()
        self.fig[0].canvas.flush_events()

        plt.pause(0.05)

    # ===== Method - Particle filter plot initiation
    def pf_initiate(self):

        # creating the figure
        self.fig[0] = plt.figure()

        # creating the axes region
        self.ax[0] = self.fig[0].add_subplot(1, 1, 1)

        # defining some plot parameters
        self.ax[0].set_xlim(-6, 6)
        self.ax[0].set_ylim(-6, 6)
        self.ax[0].set_xlabel('pos x [m]')
        self.ax[0].set_ylabel('pos y [m]')
        self.ax[0].set_title('Particle Filter')
        self.ax[0].grid(True)

        # creating particles plot
        self.lines.extend(self.ax[0].plot([], [], 'xg', linewidth=0.5))

        # creating robot real position line
        self.lines[0], = self.ax[0].plot([], [], 'og')

    def pf_draw(self, x_real, xCal, color):

        # corrects xCal, if it is not an numpy array
        if xCal is not None:
            if type(xCal) is not np.ndarray:
                aux_v1 = xCal[0]
                for aux_i in range(len(xCal)-1):
                    aux_v1 = np.hstack((aux_v1, xCal[aux_i+1]))
                xCal = aux_v1

        # updates the robot real position data
        if x_real is not None:

            # updating  robot_realposition plot
            self.lines[0].set_xdata(x_real[0])
            self.lines[0].set_ydata(x_real[1])

        # updates the particles data
        if xCal is not None:

            # plotting the particles
            self.lines[1].set_xdata(xCal[0, :])
            self.lines[1].set_ydata(xCal[1, :])

        # updates the color
        if color is not None:
            self.lines[1].set_color(color)

        # updating the figure
        self.fig[0].canvas.draw()
        self.fig[0].canvas.flush_events()

        plt.pause(0.05)

# for class testing purposes
if __name__ == '__main__':

    import numpy as np

    test = SimPlot(3)

    test.pf_initiate()

    test.pf_draw(np.array([[1],[2]]), np.array([[1, 1, 1],[2, 3, 4]]))


# -*- coding: utf-8 -*-

from ...syntax import macros, nb

def test():
    with nb:
        assert _ is None
        2 + 3          # top-level expressions autoprint, and auto-assign result to _
        assert _ == 5  # ...and only expressions do that, so...
        _ * 42         # ...here _ still has the value from the first line.
        assert _ == 210

    try:
        from sympy import symbols, pprint
    except ImportError:
        print("*** SymPy not installed, skipping symbolic math test ***")
    else:
        with nb(pprint):  # you can specify a custom print function (first positional arg)
            assert _ is None
            x, y = symbols("x, y")
            x * y
            assert _ == x * y
            3 * _
            assert _ == 3 * x * y

    print("All tests PASSED")

if __name__ == '__main__':
    test()


import streamlit as st
from tensorflow.keras.models import load_model
import numpy as np
import scipy.io
# import pathlib
# from tensorflow import keras
from src.visualization import plot_ecg

#---------------------------------#
# Page layout
## Page expands to full width
st.set_page_config(
    page_title='🫀 ECG Classification',
    # anatomical heart favicon
    page_icon="https://api.iconify.design/openmoji/anatomical-heart.svg?width=500",
    layout='wide'
)

# PAge Intro
st.write("""
# 🫀 ECG Classification

For this app, we trained a model to detect heart anomalies based on the [Physionet 2017 Cardiology Challenge](https://physionet.org/content/challenge-2017/1.0.0/)
dataset.

**Possible Predictions:** Atrial Fibrillation, Normal, Other Rhythm, or Noise

### Authors:

- Simon E. Sanchez Viloria
- Andres Ruiz Calvo
- Daniel De Las Cuevas Turel
- Enrique Botía Barbera
- Zijun He

**Try uploading your own ECG!**

-------
""".strip())

#---------------------------------#
# Data preprocessing and Model building

@st.cache(allow_output_mutation=True)
def read_ecg_preprocessing(uploaded_ecg):

      FS = 300
      maxlen = 30*FS

      uploaded_ecg.seek(0)
      mat = scipy.io.loadmat(uploaded_ecg)
      mat = mat["val"][0]

      uploaded_ecg = np.array([mat])

      X = np.zeros((1,maxlen))
      uploaded_ecg = np.nan_to_num(uploaded_ecg) # removing NaNs and Infs
      uploaded_ecg = uploaded_ecg[0,0:maxlen]
      uploaded_ecg = uploaded_ecg - np.mean(uploaded_ecg)
      uploaded_ecg = uploaded_ecg/np.std(uploaded_ecg)
      X[0,:len(uploaded_ecg)] = uploaded_ecg.T # padding sequence
      uploaded_ecg = X
      uploaded_ecg = np.expand_dims(uploaded_ecg, axis=2)
      return uploaded_ecg

model_path = 'models/weights-best.hdf5'
classes = ['Normal','Atrial Fibrillation','Other','Noise']

@st.cache(allow_output_mutation=False,ttl=24*60*60)
def get_model(model_path):
    model = load_model(f'{model_path}')
    return model

@st.cache(allow_output_mutation=True,show_spinner=False)
def get_prediction(data,model):
    prob = model(data)
    ann = np.argmax(prob)
    #true_target =
    #print(true_target,ann)

    #confusion_matrix[true_target][ann] += 1
    return classes[ann],prob #100*prob[0,ann]

# Visualization --------------------------------------
@st.cache(allow_output_mutation=True,show_spinner=False)
def visualize_ecg(ecg,FS):
    fig = plot_ecg(uploaded_ecg=ecg, FS=FS)
    return fig

#Formatting ---------------------------------#

hide_streamlit_style = """
        <style>
        #MainMenu {visibility: hidden;}
        footer {
            visibility: hidden;
        }
        footer:after {
            content:'Made for Machine Learning in Healthcare with Streamlit';
            visibility: visible;
            display: block;
            position: relative;
            #background-color: red;
            padding: 5px;
            top: 2px;
        }
"""
st.markdown(hide_streamlit_style, unsafe_allow_html=True)

#---------------------------------#
# Sidebar - Collects user input features into dataframe
with st.sidebar.header('1. Upload your ECG'):
    uploaded_file = st.sidebar.file_uploader("Upload your ECG in .mat format", type=["mat"])

st.sidebar.markdown("")

file_gts = {
    "A00001" : "Normal",
    "A00002" : "Normal",
    "A00003" : "Normal",
    "A00004" : "Atrial Fibrilation",
    "A00005" : "Other",
    "A00006" : "Normal",
    "A00007" : "Normal",
    "A00008" : "Other",
    "A00009" : "Atrial Fibrilation",
    "A00010" : "Normal",
    "A00015" : "Atrial Fibrilation",
    "A00205" : "Noise",
    "A00022" : "Noise",
    "A00034" : "Noise",
}
valfiles = [
    'None',
    'A00001.mat','A00010.mat','A00002.mat','A00003.mat',
    "A00022.mat", "A00034.mat",'A00009.mat',"A00015.mat",
    'A00008.mat','A00006.mat','A00007.mat','A00004.mat',
    "A00205.mat",'A00005.mat'
]

if uploaded_file is None:
    with st.sidebar.header('2. Or use a file from the validation set'):
        pre_trained_ecg = st.sidebar.selectbox(
            'Select a file from the validation set',
            valfiles,
            format_func=lambda x: f'{x} ({(file_gts.get(x.replace(".mat","")))})' if ".mat" in x else x,
            index=1,

        )
        if pre_trained_ecg != "None":
            f = open("data/validation/"+pre_trained_ecg, 'rb')
            if not uploaded_file:
                uploaded_file = f
        st.sidebar.markdown("Source: Physionet 2017 Cardiology Challenge")
else:
    st.sidebar.markdown("Remove the file above to demo using the validation set.")

st.sidebar.markdown("---------------")
st.sidebar.markdown("Check the [Github Repository](https://github.com/simonsanvil/ECG-classification-MLH) of this project")
#---------------------------------#
# Main panel

model = get_model(f'{model_path}')

if uploaded_file is not None:
    #st.write(uploaded_file)
    col1,_,col2 = st.columns((0.5,.05,0.45))

    with col1: # visualize ECG
        st.subheader('1.Visualize ECG')
        ecg = read_ecg_preprocessing(uploaded_file)

        fig = visualize_ecg(ecg, FS=300)
        st.pyplot(fig, use_container_width=True)

    with col2: # classify ECG
        st.subheader('2. Model Predictions')
        with st.spinner(text="Running Model..."):
            pred,conf = get_prediction(ecg,model)
        mkd_pred_table = """
        | Rhythm Type | Confidence |
        | --- | --- |
        """ + "\n".join([f"| {classes[i]} | {conf[0][i]*100:.2f}% |" for i in range(len(classes))])

        st.write("ECG classified as **{}**".format(pred))
        pred_confidence = conf[0,np.argmax(conf)]*100
        st.write("Confidence of the prediction: **{:3.1f}%**".format(pred_confidence))
        st.write(f"**Likelihoods:**")
        st.markdown(mkd_pred_table, unsafe_allow_html=False)

    # st.line_chart(np.concatenate(ecg).ravel().tolist())