"""Cisco devices (Switch/Router) configuration parser directive.

local variables::
	List down all commands requires to be parsed here in dictionary of tuples.
	where index-0 refers to command and index-1 refers to filters/arguments if any

	cisco_cmds_list = OrderedDict([
		('sh lldp nei', {'dsr': True}),			# dsr = domain suffix removal
		('sh cdp nei', {'dsr': True}),			# dsr = domain suffix removal
		('sh int status', {}),
		('sh int desc', {}),
		('show mac address-table', {}),
		('sh ip arp', {}),
		('sh run', {}),
		('sh ver', {}),
		## ADD More as grow ## ])

	List down all commands: belonging level here in dictionary

	cisco_cmds_op_hierachy_level = {
		'sh lldp nei': 'Interfaces',
		'sh cdp nei': 'Interfaces',
		'sh int status': 'Interfaces',
		'sh int desc': 'Interfaces',
		'show mac address-table': 'arp',
		'sh ip arp': 'arp',
		'sh run': ('Interfaces', 'system', 'bgp neighbor'),
		'sh ver': 'system',
		## ADD More as grow ## }

Returns:
	None: None
"""
# ------------------------------------------------------------------------------
from collections import OrderedDict

from facts_finder.common import read_file
from facts_finder.device import DevicePapa
from facts_finder.cisco import get_cdp_neighbour
from facts_finder.cisco import get_lldp_neighbour
from facts_finder.cisco import get_interface_status
from facts_finder.cisco import get_interface_description
from facts_finder.cisco import get_mac_address_table
from facts_finder.cisco import get_arp_table
from facts_finder.cisco import get_interfaces_running, get_system_running
from facts_finder.cisco import get_version
from facts_finder.cisco import get_bgp_running
# ------------------------------------------------------------------------------

CMD_LINE_START_WITH = "output for command: "
LEN_CMD_LINE = len(CMD_LINE_START_WITH)

# ------------------------------------------------------------------------------
# // Cisco //
# ------------------------------------------------------------------------------
# COMMANDS LIST DICTIONARY, DEFINE **kwargs as dictionary in command value     #
# ------------------------------------------------------------------------------
cisco_cmds_list = OrderedDict([
	('sh lldp nei', {'dsr': True}),			# dsr = domain suffix removal
	('sh cdp nei', {'dsr': True}),			# dsr = domain suffix removal
	('sh int status', {}),
	('sh int desc', {}),
	('show mac address-table', {}),
	('sh ip arp', {}),
	('sh run', {}),
	('sh ver', {}),
	## ADD More as grow ##
])
# ------------------------------------------------------------------------------
# COMMAND OUTPUT HIERARCHY LEVEL ( key need to match with 'cisco_cmds_list' )
# ------------------------------------------------------------------------------
cisco_cmds_op_hierachy_level = {
	'sh lldp nei': 'Interfaces',
	'sh cdp nei': 'Interfaces',
	'sh int status': 'Interfaces',
	'sh int desc': 'Interfaces',
	'show mac address-table': 'arp',
	'sh ip arp': 'arp',
	'sh run': ('Interfaces', 'system', 'bgp neighbor'),
	'sh ver': 'system',
	## ADD More as grow ##
}
# ------------------------------------------------------------------------------
# Dict of cisco commands, %full commands in keys mapped with parser func.
# ------------------------------------------------------------------------------
cisco_commands_parser_map = {
	'show lldp neighbors': get_lldp_neighbour,
	'show cdp neighbors': get_cdp_neighbour,
	'show interfaces status': get_interface_status,
	'show interfaces description': get_interface_description,
	'show mac address-table': get_mac_address_table,
	'show ip arp': get_arp_table,
	'show running-config': (get_interfaces_running, get_system_running, get_bgp_running),
	'show version': get_version,
}

# ------------------------------------------------------------------------------
def absolute_command(cmd, cmd_parser_map):
	"""returns absolute full command for shorteened cmd
	if founds an entry in cmd_parser_map keys.

	Args:
		cmd (str): executed/ captured command ( can be trunked or full )
		cmd_parser_map (dict, set): containing cisco standard full length (absolute command) equivalent command

	Returns:
		str: cisco command - full untrunked
	"""
	spl_cmd = cmd.split()
	for c_cmd in cmd_parser_map:
		spl_c_cmd = c_cmd.split()
		for i, word in enumerate(spl_cmd):
			try:
				word_match = spl_c_cmd[i].startswith(word)
				if not word_match: break
			except:
				word_match = False
				break
		if word_match: break
	if word_match:  return c_cmd
	return cmd

def get_op_cisco(file, abs_cmd, cmd_parser_map):
	"""returns output of a command in list format found from capture file.
	this is differ from general get_op() in a way that it has to get absolute command
	in case if provided trunked while capture. Juniper does auto-complete at terminal
	so it does not require that step.

	Either cisco output should either be captured thru package/utility - capture_it,
	or output should be in below format.

	! ========================================

	! output for command: <show command>

	! ========================================

	<< output of show command >>

	Args:
		file (str): capture file name
		abs_cmd (str): cisco absolute command string
		cmd_parser_map (dict, set): containing cisco standard full length (absolute command) equivalent command

	Returns:
		list: capture of the output in list format
	"""
	file_lines = read_file(file)
	toggle, op_lst = False, []
	for l in file_lines:
		if l.find(CMD_LINE_START_WITH)>0:
			cmd_line_cmd = absolute_command(l[LEN_CMD_LINE+1:].strip(), cmd_parser_map)
			toggle = abs_cmd == cmd_line_cmd
			continue
		if toggle:
			op_lst.append(l.rstrip())
	return op_lst

# ------------------------------------------------------------------------------
class Cisco(DevicePapa):
	"""class defining cisco parser directives.

	Args:
		DevicePapa (type): Common shared methods/properties definitions under this parent class.
	"""

	def __init__(self, file):
		"""Initialize the object by providing the capture file

		Args:
			file (str): capture file
		"""
		super().__init__(file)

	def parse(self, cmd, *arg, **kwarg):
		"""start command output parsing from provided capture.
		provide any additional arg, kwargs for dynamic filter purpose.

		Args:
			cmd (str): cisco command for which output to be parsed

		Returns:
			dict: dictionary with the details captured from the output
		"""
		abs_cmd = absolute_command(cmd, cisco_commands_parser_map)
		parse_func = cisco_commands_parser_map[abs_cmd]
		if isinstance(parse_func, tuple):
			parsed_op = [self.run_parser(pf, abs_cmd, *arg, **kwarg) for pf in parse_func]
		else:
			parsed_op = self.run_parser(parse_func, abs_cmd, *arg, **kwarg)
		return parsed_op

	def run_parser(self, parse_func, abs_cmd, *arg, **kwarg):
		"""derives the command output list for the provided absolute command.
		and runs provided parser function on to it to get the necessary details.
		provide any additional arg, kwargs for dynamic filter purpose.

		Args:
			parse_func (func): function
			abs_cmd (str): cisco absolute command for which output to be parsed

		Returns:
			dict: dictionary with the details captured from the output
		"""
		op_list = get_op_cisco(self.file, abs_cmd, cisco_commands_parser_map)
		if not parse_func: return None
		po = parse_func(op_list,*arg, **kwarg)
		return po
# ------------------------------------------------------------------------------


################### MT19AIE323 #################

## The program is for showing different option for entering user's details for predicting whether the person
# is having desease or not

import pickle
import streamlit as st
from streamlit_option_menu import option_menu
import re
import preprocess_kgptalkie as ps
from sklearn.feature_extraction.text import TfidfVectorizer
import tweepy as tw
import pandas as pd
from gensim.models import KeyedVectors
import ftfy
from nltk.corpus import stopwords
from nltk import PorterStemmer
import nltk
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
import numpy as np

backgroundColor = '#273346'

# loading the saved models

# Load ML model
survey_model = pickle.load(open('D:/MTech/Semester VI/MTP/saved models/mt19aie323_mentaldisorderprediction_survey_model.pkl', 'rb'))
diabetes_model = pickle.load(open('D:/MTech/Semester VI/MTP/saved models/mt19aie323_diabetes_model.sav', 'rb'))
heart_disease_model = pickle.load(open('D:/MTech/Semester VI/MTP/saved models/mt19aie323_heart_disease_model.sav','rb'))
suicidal_prediction_model = pickle.load(open('D:/MTech/Semester VI/MTP/saved models/mt19aie323-twitter-suicidal-model.pkl', 'rb'))

# sidebar for navigation
with st.sidebar:

    selected = option_menu('Mental Disorder, Suicidal,Diabetes and Heart Desease Prediction System',
                          ['Mental Disorder Prediction on Survey Data',
                           'Suicidal Thought on Twitter Profile',
                           'Depression on Twitter Profile',
                           'Diabetes Prediction',
                           'Heart Disease Prediction'],
                          icons=['person','twitter','twitter','activity','heart'],
                          default_index=0)

# Mental Disorder Prediction on Survey Data
if (selected == 'Mental Disorder Prediction on Survey Data'):
    # page title
    st.title('Mental Disorder Prediction on Survey Data')
    # getting the input data from the user
    col1, col2, col3,col4 = st.columns(4,gap="large")

    with col1:

         FHMDoption=('Yes', 'No', 'Do not know')
         FHMD = st.selectbox("Do you have Mental Illness family History?", range(len(FHMDoption)), format_func=lambda x: FHMDoption[x])

    with col2:

         CZoption=('1-5', '6-25', '26-100','101-500','501-1000','More than 1000','Do not know')
         CZ = st.selectbox("What is your company size?", range(len(CZoption)), format_func=lambda x: CZoption[x])
    with col3:
         SurveyYoption=('2014', '2016', '2017','2018','2019')
         SurveyY = st.selectbox("What is year of the Survey?", range(len(SurveyYoption)), format_func=lambda x: SurveyYoption[x])

    with col4:
        Age = st.text_input('What is your Age?')

    with col1:
        if Age<'20':
            AgeG = st.text('0-20')
            AgeG1=1
        elif Age>'20' and Age<='30':
            AgeG = st.text('21-30')
            AgeG1=2
        elif Age>'30' and Age<='40':
            AgeG = st.text('31-40')
            AgeG1=3
        elif Age>'40' and Age<='65':
            AgeG = st.text('41-65')
            AgeG1=4
        elif Age>'65' and Age<='100':
            AgeG = st.text('66-100')
            AgeG1=5
        else:
            AgeG = st.text('66-100')
            AgeG1=6

    with col2:
         Genoption=('Male', 'Female', 'Do not disclosed')
         Gen = st.selectbox("What is your gender?", range(len(Genoption)), format_func=lambda x: Genoption[x])
    with col3:

         PreferAnonymityoption=('Yes', 'No','Do not know')
         PreferAnonymity = st.selectbox("Do you Prefer Anonymity?", range(len(PreferAnonymityoption)), format_func=lambda x: PreferAnonymityoption[x])

    with col4:

         RtPoption=('Above Average', 'Below Average', 'Do not know')
         RtP = st.selectbox("How do you rate Reaction to Problems?", range(len(RtPoption)), format_func=lambda x: RtPoption[x])

    with col1:

         NegConseqoption=('Yes', 'No', 'May be','Do not know')
         NegConseq = st.selectbox("Do you feel  Negative Consequences?", range(len(NegConseqoption)), format_func=lambda x: NegConseqoption[x])

    with col2:

         AcessToInfooption=('Yes', 'No')
         AcessToInfo = st.selectbox("Do you feel problem in accessing Information?", range(len(AcessToInfooption)), format_func=lambda x: AcessToInfooption[x])

    with col3:
         Insuranceoption=('Yes', 'No','Do not know')
         Insurance = st.selectbox("Do you have Insurance?", range(len(Insuranceoption)), format_func=lambda x: Insuranceoption[x])

    with col4:
         Diagnosisoption=('Yes', 'No','Sometime','Do not know')
         Diagnosis = st.selectbox("Did you diagnosis Mental Disorder?", range(len(Diagnosisoption)), format_func=lambda x: Diagnosisoption[x])

    with col1:
         DMHPoption=('Yes', 'No','Maybe','Do not know')
         DMHP = st.selectbox("Is your company is responsible employer?", range(len(DMHPoption)), format_func=lambda x: DMHPoption[x])

    with col2:
        ResEmpoption=('Yes', 'No','Maybe','Self-Employed','Do not know')
        ResEmp = st.selectbox("Is your company is responsible employer?", range(len(ResEmpoption)), format_func=lambda x: ResEmpoption[x])

    with col3:
        Disorderoption=('Yes', 'No')
        Disorder = st.selectbox("Do you have Disorder?", range(len(Disorderoption)), format_func=lambda x: Disorderoption[x])
    with col4:
        PriTechCompoption=('Yes', 'No','Do not know')
        PriTechComp = st.selectbox("Are you working in Sofware Company?", range(len(PriTechCompoption)), format_func=lambda x: PriTechCompoption[x])

    print(AgeG1)
    # creating a button for Prediction
    survey_diagnosis = ''

    print(CZ)
    print(SurveyY)
    print(Age)
    print(AgeG1)
    print(Gen)
    print(PreferAnonymity)
    print(RtP)

    if st.button('Mental Disorder Prediction on SurveyData Test Result'):
        survey_diagnosis = survey_model.predict([[FHMD, CZ, SurveyY, Age, AgeG1, Gen,PreferAnonymity,RtP,NegConseq,AcessToInfo,Insurance,Diagnosis,DMHP,ResEmp,Disorder,PriTechComp]])
        print("predicted figure")
        print(survey_diagnosis)
        if (survey_diagnosis[0] == 1):
          survey_diagnosis = 'The person is having mental Disorder'
        else:
          survey_diagnosis = 'The person is not having mental Disorder'

    st.success(survey_diagnosis)

# Suicidal Prediction on Tweets
if (selected == 'Suicidal Thought on Twitter Profile'):
    # page title
    st.title('Suicidal Thought on Twitter Profile')
    # getting the input data from the user
    col1, col2, col3 = st.columns(3,gap="large")

    with col1:
        screen_name = st.text_input('Screen name')
    with col2:
        consumer_key = st.text_input('consumer key')
    with col3:
        consumer_secret = st.text_input('consumer secret')
    with col1:
        access_token = st.text_input('access token')
    with col2:
        access_token_secret = st.text_input('access token secret')

    # code for Prediction
    ## Data cleaning function
    def fn_get_clean(x):
        x = str(x).lower().replace('\\', '').replace('_', ' ')
        x = ps.cont_exp(x)
        x = ps.remove_emails(x)
        x = ps.remove_urls(x)
        x = ps.remove_html_tags(x)
        x = ps.remove_rt(x)
        x = ps.remove_accented_chars(x)
        x = ps.remove_special_chars(x)
        x = re.sub("(.)\\1{2,}", "\\1", x)
        return x

    tweets_diagnosis = ''
    if st.button('Suicidal Prediction result on tweets'):
        # # Authenticate to Twitter
        auth = tw.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token,access_token_secret)
        api = tw.API(auth)
        # Get the User object for twitter...
        #user = api.get_user(screen_name='SushilDwid')
        posts = api.user_timeline(screen_name=screen_name,count=100, tweet_mode="extended")

        tfidf=TfidfVectorizer(max_features=1000,ngram_range=(1,3),analyzer='char')
        df1=pd.DataFrame([tweet.full_text for tweet in posts] , columns=['Tweets'])
        ## Clean the tweets
        df1['Tweets']=df1['Tweets'].apply(lambda x:fn_get_clean(x))
        X1=tfidf.fit_transform(df1['Tweets'])
        #Prediction on tweets
        y_pred1=suicidal_prediction_model.predict(X1)
        num_correct = 0
        for i in range(len(y_pred1)):
            if y_pred1[i] == 1:
                num_correct += 1
        accuracy=(num_correct/len(y_pred1)*100)

        if (accuracy>8):
          tweets_diagnosis = 'Person is having suicidal thought'
        else:
          tweets_diagnosis = 'Person is not having suicidal thought'

    st.success(tweets_diagnosis)

# Depression Prediction on Tweets
if (selected == 'Depression on Twitter Profile'):
    # page title
    st.title('Depression on Twitter Profile')
    # getting the input data from the user
    col1, col2, col3 = st.columns(3,gap="large")

    with col1:
        screen_name = st.text_input('Screen name')
    with col2:
        consumer_key = st.text_input('consumer key')
    with col3:
        consumer_secret = st.text_input('consumer secret')
    with col1:
        access_token = st.text_input('access token')
    with col2:
        access_token_secret = st.text_input('access token secret')

    # code for Prediction
    MAX_SEQUENCE_LENGTH = 140 # Max tweet size
    MAX_NB_WORDS = 2000
    EMBEDDING_FILE = 'D:/MTech/Semester VI/MTP/GoogleNews-vectors-negative300.bin.gz'
    #user = api.get_user(screen_name='SushilDwid')
    word2vec = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
    # Expand Contraction
    cList = {
      "ain't": "am not",
      "aren't": "are not",
      "can't": "cannot",
      "can't've": "cannot have",
      "'cause": "because",
      "could've": "could have",
      "couldn't": "could not",
      "couldn't've": "could not have",
      "didn't": "did not",
      "doesn't": "does not",
      "don't": "do not",
      "hadn't": "had not",
      "hadn't've": "had not have",
      "hasn't": "has not",
      "haven't": "have not",
      "he'd": "he would",
      "he'd've": "he would have",
      "he'll": "he will",
      "he'll've": "he will have",
      "he's": "he is",
      "how'd": "how did",
      "how'd'y": "how do you",
      "how'll": "how will",
      "how's": "how is",
      "I'd": "I would",
      "I'd've": "I would have",
      "I'll": "I will",
      "I'll've": "I will have",
      "I'm": "I am",
      "I've": "I have",
      "isn't": "is not",
      "it'd": "it had",
      "it'd've": "it would have",
      "it'll": "it will",
      "it'll've": "it will have",
      "it's": "it is",
      "let's": "let us",
      "ma'am": "madam",
      "mayn't": "may not",
      "might've": "might have",
      "mightn't": "might not",
      "mightn't've": "might not have",
      "must've": "must have",
      "mustn't": "must not",
      "mustn't've": "must not have",
      "needn't": "need not",
      "needn't've": "need not have",
      "o'clock": "of the clock",
      "oughtn't": "ought not",
      "oughtn't've": "ought not have",
      "shan't": "shall not",
      "sha'n't": "shall not",
      "shan't've": "shall not have",
      "she'd": "she would",
      "she'd've": "she would have",
      "she'll": "she will",
      "she'll've": "she will have",
      "she's": "she is",
      "should've": "should have",
      "shouldn't": "should not",
      "shouldn't've": "should not have",
      "so've": "so have",
      "so's": "so is",
      "that'd": "that would",
      "that'd've": "that would have",
      "that's": "that is",
      "there'd": "there had",
      "there'd've": "there would have",
      "there's": "there is",
      "they'd": "they would",
      "they'd've": "they would have",
      "they'll": "they will",
      "they'll've": "they will have",
      "they're": "they are",
      "they've": "they have",
      "to've": "to have",
      "wasn't": "was not",
      "we'd": "we had",
      "we'd've": "we would have",
      "we'll": "we will",
      "we'll've": "we will have",
      "we're": "we are",
      "we've": "we have",
      "weren't": "were not",
      "what'll": "what will",
      "what'll've": "what will have",
      "what're": "what are",
      "what's": "what is",
      "what've": "what have",
      "when's": "when is",
      "when've": "when have",
      "where'd": "where did",
      "where's": "where is",
      "where've": "where have",
      "who'll": "who will",
      "who'll've": "who will have",
      "who's": "who is",
      "who've": "who have",
      "why's": "why is",
      "why've": "why have",
      "will've": "will have",
      "won't": "will not",
      "won't've": "will not have",
      "would've": "would have",
      "wouldn't": "would not",
      "wouldn't've": "would not have",
      "y'all": "you all",
      "y'alls": "you alls",
      "y'all'd": "you all would",
      "y'all'd've": "you all would have",
      "y'all're": "you all are",
      "y'all've": "you all have",
      "you'd": "you had",
      "you'd've": "you would have",
      "you'll": "you you will",
      "you'll've": "you you will have",
      "you're": "you are",
      "you've": "you have"
    }

    c_re = re.compile('(%s)' % '|'.join(cList.keys()))

    def expandContractions(text, c_re=c_re):
        def replace(match):
            return cList[match.group(0)]
        return c_re.sub(replace, text)

    def clean_tweets(tweets):
        cleaned_tweets = []
        for tweet in tweets:
            tweet = str(tweet)
            # if url links then dont append to avoid news articles
            # also check tweet length, save those > 10 (length of word "depression")
            if re.match("(\w+:\/\/\S+)", tweet) == None and len(tweet) > 10:
                #remove hashtag, @mention, emoji and image URLs
                tweet = ' '.join(re.sub("(@[A-Za-z0-9]+)|(\#[A-Za-z0-9]+)|(<.>)|(pic\.twitter\.com\/.*)", " ", tweet).split())

                #fix weirdly encoded texts
                tweet = ftfy.fix_text(tweet)

                #expand contraction
                tweet = expandContractions(tweet)

                #remove punctuation
                tweet = ' '.join(re.sub("([^0-9A-Za-z \t])", " ", tweet).split())

                #stop words
                stop_words = set(stopwords.words('english'))
                word_tokens = nltk.word_tokenize(tweet)
                filtered_sentence = [w for w in word_tokens if not w in stop_words]
                tweet = ' '.join(filtered_sentence)

                #stemming words
                tweet = PorterStemmer().stem(tweet)

                cleaned_tweets.append(tweet)

        return cleaned_tweets

    depression_diagnosis = ''

    # creating a button for Prediction

    if st.button('Depression on Twitter Profile'):
        print("Depression Test")
        # Authenticate to Twitter
        auth = tw.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_token,access_token_secret)
        api = tw.API(auth)
        posts = api.user_timeline(screen_name=screen_name,count=100, tweet_mode="extended")
        df1=pd.DataFrame([tweet.full_text for tweet in posts] , columns=['Tweets'])
        depressive_tweets_arr = [x for x in df1['Tweets']]
        X_d = clean_tweets(depressive_tweets_arr)
        tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
        tokenizer.fit_on_texts(X_d)
        sequences_d = tokenizer.texts_to_sequences(X_d)
        data_d = pad_sequences(sequences_d, maxlen=MAX_SEQUENCE_LENGTH)
        # Load ML model
        model = pickle.load(open('D:/MTech/Semester VI/MTP/saved models/mt19aie323-twitter-depression-model.pkl', 'rb'))

        labels_pred1 = model.predict(data_d)
        #labels_pred1 = np.round(labels_pred1.flatten())
        num_correct = 0
        for i in range(len(labels_pred1)):
            if labels_pred1[i] == 1:
                num_correct += 1
        accuracy=(num_correct/len(labels_pred1)*100)

        if (accuracy>8):
          depression_diagnosis = 'Person''s'' tweets are showing depression thought'
        else:
          depression_diagnosis = 'Person''s'' tweets are not showing depression thought'

    st.success(depression_diagnosis)

# Diabetes Prediction Page
if (selected == 'Diabetes Prediction'):

    # page title
    st.title('Diabetes Disease Prediction')

    # getting the input data from the user
    col1, col2, col3 = st.columns(3)

    with col1:
        Pregnancies = st.text_input('Number of Pregnancies')

    with col2:
        Glucose = st.text_input('Glucose Level')

    with col3:
        BloodPressure = st.text_input('Blood Pressure value')

    with col1:
        SkinThickness = st.text_input('Skin Thickness value')

    with col2:
        Insulin = st.text_input('Insulin Level')

    with col3:
        BMI = st.text_input('BMI value')

    with col1:
        DiabetesPedigreeFunction = st.text_input('Diabetes Pedigree Function value')

    with col2:
        Age = st.text_input('Age of the Person')

    # code for Prediction
    diab_diagnosis = ''

    # creating a button for Prediction

    if st.button('Diabetes Test Result'):
        diab_prediction = diabetes_model.predict([[Pregnancies, Glucose, BloodPressure, SkinThickness, Insulin, BMI, DiabetesPedigreeFunction, Age]])

        if (diab_prediction[0] == 1):
          diab_diagnosis = 'The person is diabetic'
        else:
          diab_diagnosis = 'The person is not diabetic'

    st.success(diab_diagnosis)

# Heart Disease Prediction Page
if (selected == 'Heart Disease Prediction'):

    # page title
    st.title('Heart Disease Prediction')

    col1, col2, col3 = st.columns(3)

    with col1:
        age = st.text_input('Age')

    with col2:
        sex = st.text_input('Sex')

    with col3:
        cp = st.text_input('Chest Pain types')

    with col1:
        trestbps = st.text_input('Resting Blood Pressure')

    with col2:
        chol = st.text_input('Serum Cholestoral in mg/dl')

    with col3:
        fbs = st.text_input('Fasting Blood Sugar > 120 mg/dl')

    with col1:
        restecg = st.text_input('Resting Electrocardiographic results')

    with col2:
        thalach = st.text_input('Maximum Heart Rate achieved')

    with col3:
        exang = st.text_input('Exercise Induced Angina')

    with col1:
        oldpeak = st.text_input('ST depression induced by exercise')

    with col2:
        slope = st.text_input('Slope of the peak exercise ST segment')

    with col3:
        ca = st.text_input('Major vessels colored by flourosopy')

    with col1:
        thal = st.text_input('thal: 0 = normal; 1 = fixed defect; 2 = reversable defect')

    # code for Prediction
    heart_diagnosis = ''

    # creating a button for Prediction

    if st.button('Heart Disease Test Result'):
        heart_prediction = heart_disease_model.predict([[age, sex, cp, trestbps, chol, fbs, restecg,thalach,exang,oldpeak,slope,ca,thal]])

        if (heart_prediction[0] == 1):
          heart_diagnosis = 'The person is having heart disease'
        else:
          heart_diagnosis = 'The person does not have any heart disease'

    st.success(heart_diagnosis)


import pathlib
from typing import Iterable
from src.utilis import list2sqlclass

class FileReader:

    def __init__(self, endpoint, end_signal=None, point_for_signal=None, signal=None,
                 data_class_for_converting=None):
        self._endpoint = endpoint
        self.point_for_signal = point_for_signal
        self.signal = signal
        self._end_signal = end_signal
        self.dataclass_for_converting = data_class_for_converting
        self._is_signal_sent = False

    def _send_maybe_signal(self, line: str):
        if self.point_for_signal is not None and line == self.point_for_signal:
            self._is_signal_sent = True
            yield self.signal
            return

        self._is_signal_sent = False
        return

    def read_generator_file(self, file_path: str):
        with pathlib.Path(file_path).open(mode="r") as f:
            line = f.readline().replace("\n", "")
            while line != self._endpoint:
                yield from self._send_maybe_signal(line)
                if not self._is_signal_sent:
                    values = line.split()
                    obj = list2sqlclass(values, self.dataclass_for_converting)
                    yield obj
                line = f.readline().replace("\n", "")
        yield self._end_signal
        return


#!/usr/bin/env python3
from setuptools import setup
# To use a consistent encoding
from codecs import open
from os import path

import pytemscript
from pytemscript import __version__

here = path.abspath(path.dirname(__file__))

# Long description
with open(path.join(here, "README.rst"), "r", encoding="utf-8") as fp:
    long_description = fp.read()

setup(name='pytemscript',
      version=__version__,
      description='TEM Scripting adapter for FEI/TFS microscopes',
      author='Tore Niermann, Grigory Sharov',
      author_email='tore.niermann@tu-berlin.de, gsharov@mrc-lmb.cam.ac.uk',
      long_description=long_description,
      packages=['pytemscript'],
      platforms=['any'],
      license="GNU General Public License v3 (GPLv3)",
      python_requires='>=3.4',
      classifiers=[
          'Development Status :: 3 - Alpha',
          'Intended Audience :: Science/Research',
          'Intended Audience :: Developers',
          'Operating System :: OS Independent',
          'Programming Language :: Python :: 3',
          'Topic :: Scientific/Engineering',
          'Topic :: Software Development :: Libraries',
          'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
          'Operating System :: OS Independent'
      ],
      install_requires=['comtypes'],
      entry_points={'console_scripts': ['pytemscript-server = pytemscript.server:run_server']},
      url="https://github.com/azazellochg/pytemscript",
      project_urls={
          "Source": "https://github.com/azazellochg/pytemscript",
          "Documentation": "https://pytemscript.readthedocs.io/"
      }
      )


import bpy
import bmesh
import numpy as np
from math import *
from mathutils import *

### Using blender 2.82a
## How to use:
### ./blender --background --python fiber_bundle_IJRR.py &&
###  convert -trim ~/image.png ~/image.png && display ~/image.png

diameterState = 0.45
mobiusStripThickness = 0.1
pathThickness = 0.3
circleThickness = 0.3

cameraFocusPoint = Vector((0,-8,-7))
cameraLocation = Vector((-15,-8,+10))
distanceCamera = 35
offsetAnnulusLeft = Vector((-3, +1, -15))
offsetAnnulusRight = Vector((-3,-15, -15))

torusMinorRadius = 2.3
torusMajorRadius = 4.1
torusLocation = (0,-14,0)

colorStartState = (0.0, 1.0, 0.0, 1.0)
colorGoalState = (0.7, 0.0, 0.0, 1.0)

minor_radius = 2
major_radius = 5

materialGreen = bpy.data.materials.new(name="Green")
materialGreen.diffuse_color = (0.0, 1.0, 0.0, 1.0)
materialGreen.metallic = 0.0
materialGreen.specular_intensity = 0.0

materialMagenta = bpy.data.materials.new(name="Magenta")
materialMagenta.diffuse_color = (0.45, 0.05, 0.72, 1.0)
materialMagenta.diffuse_color = (0.575, 0.12, 1.0, 1.0)
materialMagenta.diffuse_color = (0.205, 0.0, 0.37, 1.0)
materialMagenta.metallic = 0.9
materialMagenta.specular_intensity = 0.9

def arrow_mesh(position, length, width=-1, headlength=-1, headwidth = -1):
    verts = []
    faces = []

    if width < 0:
      width = 0.15*length
    if headlength < 0:
      headlength = 0.3*length
    if headwidth < 0:
      headwidth = 2*width

    v1 = position
    v2 = Vector((position[0], position[1] - 0.5*headwidth, position[2] + headlength))
    v3 = Vector((position[0], position[1] - 0.5*width, position[2] + headlength))
    v4 = Vector((position[0], position[1] - 0.5*width, position[2] + length))

    v5 = Vector((position[0], position[1] + 0.5*headwidth, position[2] + headlength))
    v6 = Vector((position[0], position[1] + 0.5*width, position[2] + headlength))
    v7 = Vector((position[0], position[1] + 0.5*width, position[2] + length))

    #   3---6
    #   |   |
    #   |   |
    #1--2---5--4
    # \       /
    #  \     /
    #   \   /
    #    \ /
    #     0

    # verts.extend([v1, v2])
    verts.extend([v1, v2, v3, v4, v5, v6, v7])
    faces.append([0, 1, 2])
    faces.append([0, 2, 5])
    faces.append([0, 5, 4])
    faces.append([2, 3, 6])
    faces.append([2, 6, 5])

    mesh = bpy.data.meshes.new("arrow")
    mesh.from_pydata(verts, [], faces)

    for p in mesh.polygons:
        p.use_smooth=True

    return mesh

def annulus_mesh(resolution, inner_radius, outer_radius):
    verts = []
    faces = []

    for i in range(resolution):
        theta = 2*pi * i/resolution
        rot = Matrix.Rotation(theta, 3, [0,0,1])
        v1 = rot @ Vector([inner_radius, 0, 0])
        v2 = rot @ Vector([outer_radius, 0, 0])

        i1 = len(verts)
        verts.extend([v1,v2])

        if i+1<resolution:
            ia = i1+2
            ib = i1+3
        else:
            ia = 0
            ib = 1

        faces.append( [i1,i1+1,ib,ia])

    mesh = bpy.data.meshes.new("annulus")
    mesh.from_pydata(verts, [], faces)

    for p in mesh.polygons:
        p.use_smooth=True

    return mesh

def mobius_mesh(resolution, thick):

    verts = []
    faces = []

    for i in range(resolution):

        theta = 2*pi * i/resolution
        phi = pi * i/resolution

        rot1 = Matrix.Rotation(phi, 3, [0,1,0])
        rot2 = Matrix.Rotation(theta, 3, [0,0,1])
        c1 = Vector([major_radius, 0, 0])
        v1 = rot2 @ (c1 + rot1 @ Vector([-thick / 2, 0, minor_radius]) )
        v2 = rot2 @ (c1 + rot1 @ Vector([thick / 2, 0, minor_radius]) )
        v3 = rot2 @ (c1 + rot1 @ Vector([thick / 2, 0, -minor_radius]) )
        v4 = rot2 @ (c1 + rot1 @ Vector([-thick / 2, 0, -minor_radius]) )

        i1 = len(verts)
        verts.extend([v1,v2,v3,v4])

        if i+1<resolution:
            ia = i1+4
            ib = i1+5
            ic = i1+6
            id = i1+7
        else:
            ia = 2
            ib = 3
            ic = 0
            id = 1

        # faces.append( [i1+j for j in range(4) ])
        faces.append( [i1,i1+1,ib,ia])
        faces.append( [i1+1,i1+2,ic,ib])
        faces.append( [i1+2,i1+3,id,ic])
        faces.append( [i1+3,i1,ia,id])

    mesh = bpy.data.meshes.new("mobius")
    mesh.from_pydata(verts, [], faces)

    for p in mesh.polygons:
        p.use_smooth=True

    return mesh

def addPath():
    ## create curve OBJ
    curve = bpy.data.curves.new(name="L1FF", type='CURVE')
    curve.dimensions = '3D'
    path_obj = bpy.data.objects.new("path", curve)
    path_obj.location = (0,0,0)

    ### create milestones
    poly = curve.splines.new('POLY')
    X = np.arange(0.45,-0.01,-0.01)
    poly.points.add(2 + len(X) - 1)
    poly.points[0].co = getVectorOnStrip4D(0.45, 0.1)
    poly.points[1].co = getVectorOnStrip4D(0.45, 0.9)
    ctr = 2
    for x in X:
      poly.points[ctr].co = getVectorOnStrip4D(x, 0.9)
      ctr = ctr+1

    bpy.context.collection.objects.link(path_obj)

    curve.fill_mode = 'FULL'
    curve.bevel_depth = pathThickness

    path_obj.data.materials.append(materialGreen)

    # mesh = bpy.data.meshes.new("path_mesh")
    # mesh.from_pydata(verts, edges, [])

def addPathAnnulus():
    ## create curve OBJ
    curve = bpy.data.curves.new(name="PATH_ANNULUS", type='CURVE')
    curve.dimensions = '3D'
    path_obj = bpy.data.objects.new("path", curve)
    path_obj.location = (0,0,0)

    ### create milestones
    poly = curve.splines.new('POLY')
    X = np.arange(0.45,-0.01,-0.01)
    poly.points.add(len(X) - 1)
    ctr = 0
    for x in X:
      theta = x*(2*pi)
      rot = Matrix.Rotation(theta, 3, [0,0,1])
      v = (rot @ Vector([major_radius, 0, 0])) + offsetAnnulusLeft
      poly.points[ctr].co = (v[0],v[1],v[2],1)
      ctr = ctr+1

    bpy.context.collection.objects.link(path_obj)

    curve.fill_mode = 'FULL'
    curve.bevel_depth = pathThickness

    path_obj.data.materials.append(materialGreen)

def addPathTorusAnnulus():
    curve = bpy.data.curves.new(name="PATH_TORUS_ANNULUS", type='CURVE')
    curve.dimensions = '3D'
    path_obj = bpy.data.objects.new("path_torus_annulus", curve)
    path_obj.location = (0,0,0)

    ### create milestones
    poly = curve.splines.new('POLY')
    X = np.arange(0.55,0.85,+0.01)
    poly.points.add(len(X) - 1)
    ctr = 0
    for x in X:
      theta = x*(2*pi)
      rot = Matrix.Rotation(theta, 3, [0,0,1])
      v = (rot @ Vector([major_radius, 0, 0])) + offsetAnnulusRight
      poly.points[ctr].co = (v[0],v[1],v[2],1)
      ctr = ctr+1

    bpy.context.collection.objects.link(path_obj)

    curve.fill_mode = 'FULL'
    curve.bevel_depth = pathThickness

    path_obj.data.materials.append(materialGreen)

def update_camera(camera, focus_point=Vector((0.0, 0.0, 0.0)), distance=10.0):
    """
    Focus the camera to a focus point and place the camera at a specific distance from that
    focus point. The camera stays in a direct line with the focus point.

    :param camera: the camera object
    :type camera: bpy.types.object
    :param focus_point: the point to focus on (default=``Vector((0.0, 0.0, 0.0))``)
    :type focus_point: Vector
    :param distance: the distance to keep to the focus point (default=``10.0``)
    :type distance: float
    """
    looking_direction = camera.location - focus_point
    rot_quat = looking_direction.to_track_quat('Z', 'Y')

    camera.rotation_mode = 'XYZ'
    camera.rotation_euler = rot_quat.to_euler()
    camera.location = camera.location + rot_quat @ Vector((0.0, 0.0, distance))

def getVectorOnStrip4D(x_1, x_2):
    v = getVectorOnStrip(x_1, x_2)
    return (v[0],v[1],v[2],1)

def getVectorOnStrip(x_1, x_2):
    theta = x_1*(2*pi)
    phi = x_1*pi
    dx = -minor_radius+x_2*(2*minor_radius)

    rot1 = Matrix.Rotation(phi, 3, [0,1,0])
    rot2 = Matrix.Rotation(theta, 3, [0,0,1])
    c1 = Vector([major_radius, 0, 0])
    return rot2 @ (c1 + rot1 @ Vector([0, 0, dx]) )

def addSphere(pos, name, color=(1.0, 1.0, 1.0, 1.0)):
    xi_mesh = bpy.data.meshes.new("init_state_mesh_"+name)
    xi_obj = bpy.data.objects.new("init_state_"+name, xi_mesh)
    xi_obj.location = xi_obj.location + pos

    bpy.context.collection.objects.link(xi_obj)

    ## select active object
    bpy.context.view_layer.objects.active = xi_obj
    xi_obj.select_set(True)

    ## add mesh
    bm = bmesh.new()
    bmesh.ops.create_uvsphere(bm, u_segments=32, v_segments=16,
        diameter=diameterState)
    bm.to_mesh(xi_mesh)
    bm.free()

    bpy.ops.object.modifier_add(type='SUBSURF')
    bpy.ops.object.shade_smooth()

    ## add color
    mat = bpy.data.materials.new("PKHG")
    mat.diffuse_color = color
    mat.metallic = 0.0
    mat.specular_intensity = 0.0
    xi_obj.active_material = mat

def addState(x_1, x_2, name, color=(1.0, 1.0, 1.0, 1.0)):

    pos = getVectorOnStrip(x_1, x_2)
    addSphere(pos, name, color)

def addStateTorus(u, v, color=(1.0,1.0,1.0,1.0)):
    r0 = torusMajorRadius
    r1 = torusMinorRadius
    x = ((r0 + r1*cos(2*pi*v))*cos(2*pi*u), \
         (r0 + r1*cos(2*pi*v))*sin(2*pi*u), \
         r1*sin(2*pi*v))
    pos = Vector((x[0]+torusLocation[0],x[1]+torusLocation[1],x[2]+torusLocation[2]))
    addSphere(pos, "x_on_torus", color)

def addStateAnnulusLeft(x_1, name, color=(1.0, 1.0, 1.0, 1.0)):
    addStateAnnulus(x_1, name, color, offsetAnnulusLeft)

def addStateAnnulusRight(x_1, name, color=(1.0, 1.0, 1.0, 1.0)):
    addStateAnnulus(x_1, name, color, offsetAnnulusRight)

def addStateAnnulus(x_1, name, color=(1.0, 1.0, 1.0, 1.0), offset =
    offsetAnnulusLeft):

    theta = x_1*(2*pi)
    rot = Matrix.Rotation(theta, 3, [0,0,1])
    pos = (rot @ Vector([major_radius, 0, 0])) + offset
    addSphere(pos, name, color)

def addLightSource():

    light_data = bpy.data.lights.new(name="light_2.80", type='SUN')
    light_data.energy = 5
    light_object = bpy.data.objects.new(name="light_2.80",
      object_data=light_data)
    bpy.context.collection.objects.link(light_object)
    light_object.location = (0, 0, 20)

    light_2 = bpy.data.lights.new(name="light_source", type='AREA')
    light_2.energy = 400
    light_2.size = 13
    light_2_object = bpy.data.objects.new(name="light_source",
      object_data=light_2)
    bpy.context.collection.objects.link(light_2_object)
    light_2_object.location = offsetAnnulusLeft + Vector((0, 0, +1))

    light_3 = bpy.data.lights.new(name="light_source", type='AREA')
    light_3.energy = 400
    light_3.size = 13
    light_3_object = bpy.data.objects.new(name="light_source",
      object_data=light_2)
    bpy.context.collection.objects.link(light_3_object)
    light_3_object.location = offsetAnnulusRight + Vector((0, 0, +1))

def addCamera(location):
    cam = bpy.data.cameras.new("Camera")
    cam_ob = bpy.data.objects.new("Camera", cam)
    bpy.context.collection.objects.link(cam_ob)
    cam_ob.location = location
    bpy.context.scene.camera = cam_ob
    bpy.context.view_layer.objects.active = cam_ob
    update_camera(cam_ob, focus_point=cameraFocusPoint, distance=distanceCamera)

def torusRestriction_mesh(torusLocation):
    r0 = torusMajorRadius
    r1 = torusMinorRadius
    U = np.arange(0.55, 0.86, 0.01)
    V = np.arange(0,1.01,0.01)
    ctr = 0
    verts = []
    faces = []
    for u in U:
      M = len(verts)
      for v in V:
        x = ((r0 + r1*cos(2*pi*v))*cos(2*pi*u), \
             (r0 + r1*cos(2*pi*v))*sin(2*pi*u), \
             r1*sin(2*pi*v))
        x = Vector((x[0]+torusLocation[0],x[1]+torusLocation[1],x[2]+torusLocation[2]))
        verts.extend([x])

      N = len(V)
      if ctr < len(U) - 1:
          for k in range(M,M + N - 1):
            faces.append( [k,k+1,k+N])
            faces.append( [k+1,k+1+N, k+N])

      ctr = ctr + 1

    mesh = bpy.data.meshes.new("trestrict")
    mesh.from_pydata(verts, [], faces)

    for p in mesh.polygons:
        p.use_smooth=True

    return mesh

def addTorus(position):

    torus_mesh = bpy.ops.mesh.primitive_torus_add(major_radius = torusMajorRadius,
        minor_radius = torusMinorRadius-0.05,
        major_segments = 64, minor_segments = 32, location=position)
    torus_obj = bpy.context.object
    torus_obj.name = 'Torus'
    torus_obj.data.materials.append(materialMagenta)

def drawScene(scn, resolution, thick):

    ###########################################################################
    ### DELETE ALL OBJECTS
    ###########################################################################
    bpy.ops.object.select_all(action='SELECT')
    bpy.ops.object.delete()

    ###########################################################################
    ### Add Light
    ###########################################################################
    addLightSource()

    ###########################################################################
    ### Add Camera
    ###########################################################################
    addCamera(cameraLocation)

    ###########################################################################
    ### MOBIUS STRIP
    ###########################################################################

    mobius_strip_mesh = mobius_mesh(resolution, thick)
    mobius_strip_obj = bpy.data.objects.new("mobius strip", mobius_strip_mesh)
    bpy.context.collection.objects.link(mobius_strip_obj)
    bpy.context.view_layer.objects.active = mobius_strip_obj
    mod = mobius_strip_obj.modifiers.new("edge split", 'EDGE_SPLIT')

    mobius_strip_obj.data.materials.append(materialMagenta)
    # mobius_strip_obj.show_transparent = True

    ###########################################################################
    ### ANNULUS LEFT
    ###########################################################################
    annulus_strip_mesh = annulus_mesh(resolution, major_radius -
        circleThickness, major_radius + circleThickness)
    annulus_strip_obj = bpy.data.objects.new("annulus", annulus_strip_mesh)
    annulus_strip_obj.location = offsetAnnulusLeft

    bpy.context.collection.objects.link(annulus_strip_obj)
    bpy.context.view_layer.objects.active = annulus_strip_obj
    mod = annulus_strip_obj.modifiers.new("edge split", 'EDGE_SPLIT')

    annulus_strip_obj.data.materials.append(materialMagenta)

    ###########################################################################
    ### ANNULUS RIGHT
    ###########################################################################
    annulus_r_strip_mesh = annulus_mesh(resolution, major_radius -
        circleThickness, major_radius + circleThickness)
    annulus_r_strip_obj = bpy.data.objects.new("annulus", annulus_r_strip_mesh)
    annulus_r_strip_obj.location = offsetAnnulusRight

    bpy.context.collection.objects.link(annulus_r_strip_obj)
    bpy.context.view_layer.objects.active = annulus_r_strip_obj
    mod = annulus_r_strip_obj.modifiers.new("edge split", 'EDGE_SPLIT')

    annulus_r_strip_obj.data.materials.append(materialMagenta)

    ###########################################################################
    ### ARROW
    ###########################################################################
    arrow_strip_mesh = arrow_mesh(Vector((0,0,+7.5)), length = 2.0, width = 0.5)
    arrow_strip_obj = bpy.data.objects.new("arrow", arrow_strip_mesh)
    arrow_strip_obj.location = offsetAnnulusLeft + Vector((0,-0.8,0))
    bpy.context.collection.objects.link(arrow_strip_obj)
    bpy.context.view_layer.objects.active = arrow_strip_obj

    arrow_r_strip_mesh = arrow_mesh(Vector((0,0,+8.0)), length = 2.0, width = 0.5)
    arrow_r_strip_obj = bpy.data.objects.new("arrow", arrow_r_strip_mesh)
    arrow_r_strip_obj.location = offsetAnnulusRight + Vector((0,0.8,0))
    bpy.context.collection.objects.link(arrow_r_strip_obj)
    bpy.context.view_layer.objects.active = arrow_r_strip_obj

    addTorus(torusLocation)

    restriction_torus_mesh = torusRestriction_mesh(torusLocation)
    restriction_torus_obj = bpy.data.objects.new("arrow", restriction_torus_mesh)
    # restriction_torus_obj.location = offsetAnnulusLeft + Vector((0,-0.8,0))
    bpy.context.collection.objects.link(restriction_torus_obj)
    restriction_torus_obj.data.materials.append(materialGreen)

    ###########################################################################
    ### INITIAL STATE
    ###########################################################################

    addState(0.45, 0.1, "xi", color=colorStartState)
    addState(0, 0.9, "xg", color=colorGoalState)

    addStateAnnulusLeft(0.45, "xi", color=colorStartState)
    addStateAnnulusLeft(0, "xg", color=colorGoalState)

    addStateAnnulusRight(0.55, "xi", color=colorStartState)
    addStateAnnulusRight(0.85, "xg", color=colorGoalState)

    addStateTorus(0.55, 0.1, color=colorStartState)
    addStateTorus(0.85, 0.4, color=colorGoalState)

    addPath()
    addPathAnnulus()

    addPathTorusAnnulus()

    ### RESET ACTIVE OBJECT TO MOBIUS STRIP
    bpy.context.view_layer.objects.active = mobius_strip_obj

    ### SAVE IMAGE
    bpy.context.scene.render.film_transparent = True
    bpy.context.scene.render.image_settings.color_mode = 'RGBA'
    bpy.context.scene.render.filepath = '/home/aorthey/image.png'
    bpy.ops.render.render(write_still = True)

    print(bpy.context.scene.render.filepath)

drawScene(bpy.context.scene, 72, mobiusStripThickness)


from django.http import HttpResponse
from django.shortcuts import redirect, render

from .forms import ProjectForm
from .models import Project

def projects(request):
    projects = Project.objects.all()
    context = {'projects': projects}
    return render(request, 'projects/projects.html', context)

def project(request, pk):
    projectObj = Project.objects.get(id=pk)
    #tags = projectObj.tags.all()
    print('projectObj :', projectObj)
    return render(request, 'projects/project.html', {'project': projectObj})

    # return render(request, 'projects/project.html', {'project': projectObj, 'tags': tags})

def createProject(request):
    form = ProjectForm()

    if request.method == 'POST':
        # print(request.POST)
        form = ProjectForm(request.POST, request.FILES)
        if form.is_valid():
            form.save()
            return redirect('projects')

    context = {'form': form}
    return render(request, 'projects/project_form.html', context)

def updateProject(request, pk):
    project = Project.objects.get(id=pk)
    form = ProjectForm(instance=project)

    if request.method == 'POST':
        # print(request.POST)
        form = ProjectForm(request.POST, request.FILES, instance=project)
        if form.is_valid():
            form.save()
            return redirect('projects')

    context = {'form': form}
    return render(request, 'projects/project_form.html', context)

def deleteProject(request, pk):
    project = Project.objects.get(id=pk)
    if request.method == 'POST':
        project.delete()
        return redirect('projects')
    context = {'object': project}
    return render(request, 'projects/delete_template.html', context)

# projectList = [
#
#    {
#        'id': '1',
#        'title': 'Ecommerce Website',
#        'description': 'Fully functional Ecommerce website'
#    },
#
#    {
#        'id': '2',
#        'title': 'Portfolio Website',
#        'description': 'A personal website to write articles and display work'
#    },
#
#    {
#        'id': '3',
#        'title': 'Social Network',
#        'description': 'An open source project built by the community'
#    },
#
# ]
#
# projectList = [
#    {
#        'id': '1',
#        'title': 'Ecommerce Website',
#        'description': 'Fully functional Ecommerce website',
#    },
#    {
#        'id': '2',
#        'title': 'Portfolio Website',
#        'description': 'Fully functional Portfolio website',
#    },
#    {
#        'id': '3',
#        'title': 'Social Website',
#        'description': 'Fully functional Social website',
#    },
# ]


from PyQt5 import QtWidgets, QtCore, QtGui, QtMultimedia
from PyQt5.QtWidgets import QGraphicsItem
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *

class GraphicsScene(QGraphicsScene):
    nodes = []
    edges = []
    show_attribute_signal = pyqtSignal(int)
    add_attribute_signal = pyqtSignal(int)
    remove_attribute_signal = pyqtSignal(int)

    def __init__(self, parent=None):
        super().__init__(parent)

    def sendMsg(self, index):
        if len(self.selectedItems()) == 1:
            if isinstance(self.selectedItems()[0], NodeItem or QGraphicsTextItem):
                self.show_attribute_signal.emit(index)

    def addNode(self, id, pos):
        index = len(self.nodes)
        node = NodeItem(index, id)
        self.nodes.append(node)
        self.nodes[index].setPos(pos)
        self.addItem(self.nodes[index])
        self.add_attribute_signal.emit(id)

    def addEdge(self, start_node, end_node):
        index = len(self.edges)
        start_node.connected = True
        end_node.connected = True
        start_node.edge_index = index
        end_node.edge_index = index
        edge = GraphicsPathItem(index, start_node, end_node)
        self.edges.append(edge)
        self.addItem(self.edges[index])
        self.edges[index].setEdge()

    def removeNode(self, node):
        for edge in self.edges:
            if edge.start_node.index == node.index:
                self.nodes[edge.end_node.index].leftPort.connected = False
                self.nodes[edge.end_node.index].leftPort.edge_index = -1
                self.removeItem(edge)
                self.edges.remove(edge)
        for edge in self.edges:
            if edge.end_node.index == node.index:
                self.nodes[edge.start_node.index].rightPort.connected = False
                self.nodes[edge.start_node.index].rightPort.edge_index = -1
                self.removeItem(edge)
                self.edges.remove(edge)
        self.remove_attribute_signal.emit(node.index)
        self.removeItem(node)
        self.nodes.remove(node)
        self.updateNode()
        self.updateEdge()

    def removeEdge(self, edge):
        self.nodes[edge.end_node.index].leftPort.connected = False
        self.nodes[edge.start_node.index].rightPort.connected = False
        self.nodes[edge.end_node.index].leftPort.edge_index = -1
        self.nodes[edge.start_node.index].rightPort.edge_index = -1
        self.removeItem(edge)
        self.edges.remove(edge)
        self.updateEdge()

    def updateNode(self):
        for node in self.nodes:
            for edge in self.edges:
                if edge.start_node.index == node.index:
                    edge.start_node.index = self.nodes.index(node)
                if edge.end_node.index == node.index:
                    edge.end_node.index = self.nodes.index(node)
            node.index = self.nodes.index(node)
            if hasattr(node, 'leftPort'):
                node.leftPort.index = node.index
            if hasattr(node, 'rightPort'):
                node.rightPort.index = node.index

    def updateEdge(self):
        for edge in self.edges:
            for node in self.nodes:
                if hasattr(node, 'rightPort'):
                    if node.rightPort.edge_index == edge.index:
                        node.rightPort.edge_index = self.edges.index(edge)
                if hasattr(node, 'leftPort'):
                    if node.leftPort.edge_index == edge.index:
                        node.leftPort.edge_index = self.edges.index(edge)
            edge.index = self.edges.index(edge)

class GraphicsView(QGraphicsView):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.parent = parent
        self.setAcceptDrops(True)
        self.setMouseTracking(True)
        self.setRenderHints(QPainter.Antialiasing |
                            QPainter.HighQualityAntialiasing |
                            QPainter.TextAntialiasing |
                            QPainter.SmoothPixmapTransform |
                            QPainter.LosslessImageRendering)
        self.graphics_scene = GraphicsScene(self)
        self.setScene(self.graphics_scene)

        self.drag_edge = False

        self.setDragMode(self.RubberBandDrag)
        self.setTransformationAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
        # self.setResizeAnchor(QtWidgets.QGraphicsView.AnchorUnderMouse)
        # self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
        # self.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)

    def getClickedItem(self, pos):
        return self.itemAt(pos)

    def mousePressEvent(self, e):
        super().mousePressEvent(e)
        if e.buttons() == Qt.RightButton:
            item = self.getClickedItem(e.pos())
            if isinstance(item, PortItem):
                if item.id == 1 and item.connected == False:
                    self.start_item = item
                    self.end = self.mapToScene(e.pos())
                    self.fake_edge = GraphicsPathItem(-1,
                                                      self.start_item, self.end)
                    self.fake_edge.start(self.start_item.pos(
                    )+QPointF(self.start_item.rect().width()/2, self.start_item.rect().height()/2)+self.graphics_scene.nodes[self.start_item.index].pos())
                    self.graphics_scene.addItem(self.fake_edge)
                    self.drag_edge = True

    def mouseMoveEvent(self, e):
        super().mouseMoveEvent(e)
        if e.buttons() == Qt.RightButton and self.drag_edge:
            self.end = self.mapToScene(e.pos())
            self.fake_edge.end_node = self.end
            self.fake_edge.setEdge()
            # self.fake_edge.edge_path.clear()

    def mouseReleaseEvent(self, e):
        if self.drag_edge:
            item = self.getClickedItem(e.pos()+QPoint(3, 0))
            item1 = self.getClickedItem(e.pos()+QPoint(-3, 0))
            if isinstance(item, PortItem):
                if item.index != self.start_item.index and item.id == 0 and item.connected == False:
                    self.graphics_scene.addEdge(self.start_item, item)
            elif isinstance(item1, PortItem):
                if item1.index != self.start_item.index and item1.id == 0 and item1.connected == False:
                    self.graphics_scene.addEdge(self.start_item, item1)

            self.graphics_scene.removeItem(self.fake_edge)
            self.fake_edge = None
            self.drag_edge = False
        else:
            super().mouseReleaseEvent(e)

    def dragEnterEvent(self, e):
        e.accept()

    def dragMoveEvent(self, e):
        e.accept()

    def dropEvent(self, e):
        if e.mimeData().text() == "conv2D":
            e.accept()
            self.graphics_scene.addNode(
                1, (self.mapToScene(e.pos())).toPoint())
        elif e.mimeData().text() == "pooling2D":
            e.accept()
            self.graphics_scene.addNode(
                2, (self.mapToScene(e.pos())).toPoint())
        elif e.mimeData().text() == "flatten":
            e.accept()
            self.graphics_scene.addNode(
                3, (self.mapToScene(e.pos())).toPoint())
        elif e.mimeData().text() == "dense":
            e.accept()
            self.graphics_scene.addNode(
                4, (self.mapToScene(e.pos())).toPoint())
        elif e.mimeData().text() == "input":
            e.accept()
            self.graphics_scene.addNode(
                5, (self.mapToScene(e.pos())).toPoint())
        elif e.mimeData().text() == "output":
            e.accept()
            self.graphics_scene.addNode(
                6, (self.mapToScene(e.pos())).toPoint())

class GraphicsPathItem(QGraphicsPathItem):
    def __init__(self, index, start_node, end_node, parent=None):
        super().__init__(parent)
        self.index = index
        self.start_node = start_node
        self.end_node = end_node
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsSelectable)
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsFocusable, True)
        pen = QPen()
        pen.setColor(QColor("#0614B2"))
        pen.setWidth(3)
        self.setPen(pen)

        self.edge_path = QPainterPath()

    def start(self, pos):
        self.edge_path.moveTo(pos)

    def setEdge(self):
        start_pos = self.start_node.pos(
        )+QPointF(self.start_node.rect().width()/2, self.start_node.rect().height()/2)+self.scene().nodes[self.start_node.index].pos()
        end_pos = QPointF()
        if isinstance(self.end_node, PortItem):
            end_pos = self.end_node.pos(
            )+QPointF(self.end_node.rect().width()/2, self.end_node.rect().height()/2)+self.scene().nodes[self.end_node.index].pos()
        else:
            end_pos = self.end_node
        mid_pos = (start_pos+end_pos)/2
        self.edge_path.moveTo(start_pos)
        self.edge_path.cubicTo(
            QPointF(mid_pos.x(), start_pos.y()), mid_pos, end_pos)
        self.setPath(self.edge_path)
        self.edge_path.clear()

    def keyPressEvent(self, e):
        if e.key() == Qt.Key.Key_Delete:
            self.scene().removeEdge(self)

class NodeItem(QGraphicsRectItem):

    def __init__(self, index, id, parent=None):
        super().__init__(parent)
        self.index = index
        self.id = id
        self.name = " "
        if self.id == 1:
            self.name = "Conv2D"
        elif self.id == 2:
            self.name = "pooling2D"
        elif self.id == 3:
            self.name = "Flatten"
        elif self.id == 4:
            self.name = "Dense"
        elif self.id == 5:
            self.name = "Input"
        elif self.id == 6:
            self.name = "Output"
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsMovable)
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsSelectable)
        self.setFlag(
            QGraphicsItem.GraphicsItemFlag.ItemSendsGeometryChanges, True)
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsFocusable, True)
        self.width = 160
        self.height = 80
        self.setRect(0, 0, self.width, self.height)
        self.setBrush(QColor(6, 178, 178, 140))

        self.text = QGraphicsSimpleTextItem(self.name, self)
        self.text.setFont(QFont("Consolas", 12))
        self.text.setPos(self.width/2-self.text.boundingRect().width()/2,
                         self.height/2-self.text.boundingRect().height()/2)
        self.text.setBrush(QColor(255, 0, 0))

        if self.id != 5:
            self.leftPort = PortItem(self.index, 0, self)
            self.leftPort.setPos(
                5, self.height/2-self.leftPort.boundingRect().height()/2)

        if self.id != 6:
            self.rightPort = PortItem(self.index, 1, self)
            self.rightPort.setPos(self.width-25, self.height /
                                  2-self.rightPort.boundingRect().height()/2)

        # self.pixmap = QPixmap("image/bubble.png")
        # self.pixmap = self.pixmap.scaled(30, 30)
        # self.gpixmap = QGraphicsPixmapItem(self.pixmap, self)
        # self.gpixmap.setPos(30, self.height/2-self.gpixmap.boundingRect().height()/2)

    def mousePressEvent(self, e):
        super().mousePressEvent(e)
        if self.isSelected():
            self.scene().sendMsg(self.index)

    def mouseMoveEvent(self, e):
        super().mouseMoveEvent(e)
        if self.isSelected():
            for edge in self.scene().edges:
                edge.setEdge()

    def keyPressEvent(self, e):
        if e.key() == Qt.Key.Key_Delete:
            self.scene().removeNode(self)

class PortItem(QGraphicsEllipseItem):
    connected = False
    edge_index = -1

    def __init__(self, index, id, parent=None):
        super(PortItem, self).__init__(parent)
        self.index = index
        self.id = id
        self.portDiam = 20
        self.setFlag(QGraphicsItem.GraphicsItemFlag.ItemIsSelectable)
        self.setBrush(QColor(255, 128, 128, 180))
        self.setRect(0, 0, self.portDiam, self.portDiam)

class ViewWidget(QWidget):
    def __init__(self):
        super(ViewWidget, self).__init__()
        self.vlayout = QVBoxLayout()
        self.setLayout(self.vlayout)
        self.gv = GraphicsView(self)
        self.vlayout.addWidget(self.gv)
        self.toggle_btn = QPushButton("Toggle Drag Mode")
        self.toggle_btn.clicked.connect(self.toggle_drag_mode)
        self.vlayout.addWidget(self.toggle_btn)
        self.edge_btn = QPushButton("Store Setting")
        self.vlayout.addWidget(self.edge_btn)
        self.edgeLabel = QLabel("Edge: \n")
        self.edgeLabel.setObjectName("edge")
        self.edgeLabel.setVisible(False)
        self.vlayout.addWidget(self.edgeLabel)
        self.setLayout(self.vlayout)
        with open("./stylesheet/model.qss", "r") as f:
            self.setStyleSheet(f.read())

    def PlaySound(self):
        audio_url = QtCore.QUrl.fromLocalFile("./sound/toggle.wav")
        audio_content = QtMultimedia.QMediaContent(audio_url)
        self.player = QtMultimedia.QMediaPlayer()
        self.player.setVolume(30.0)
        self.player.setMedia(audio_content)
        self.player.play()

    def toggle_drag_mode(self):
        self.PlaySound()
        if self.gv.dragMode() == QGraphicsView.ScrollHandDrag:
            self.gv.setDragMode(QtWidgets.QGraphicsView.RubberBandDrag)
        else:
            self.gv.setDragMode(QtWidgets.QGraphicsView.ScrollHandDrag)

    def showEdge(self):
        now_node = None
        gs = self.gv.graphics_scene
        self.edgeLabel.setText("Edge: \n")
        for node in gs.nodes:
            if node.id == 5:
                now_node = node
        if now_node == None:
            print("No Input Layer")
            return

        for i in range(len(gs.nodes)):
            if now_node.id != 6 and now_node.rightPort.edge_index != -1:
                self.edgeLabel.setText(self.edgeLabel.text() +
                                       "index: " + str(now_node.index) +
                                       ", id: " + str(now_node.id) +
                                       ", name: " + now_node.name +
                                       ", next edge's index: " + str(now_node.rightPort.edge_index)+"\n")
                now_node = gs.nodes[gs.edges[now_node.rightPort.edge_index].end_node.index]
            else:
                self.edgeLabel.setText(self.edgeLabel.text() +
                                       "index: " + str(now_node.index) +
                                       ", id: " + str(now_node.id) +
                                       ", name: " + now_node.name +
                                       ", next edge's index: The End\n")
                break


import os
import time
import torch.backends.cudnn as cudnn
from skimage.metrics import peak_signal_noise_ratio
from torch import nn
from torch.utils.tensorboard import SummaryWriter
from datasets import SRDataset
from model import SRResNet
from utils import *

crop_size = 291
scaling_factor = 3

large_kernel_size = 9
small_kernel_size = 3
n_channels = 64
n_blocks = 16
checkpoint_dir = 'models'
if not os.path.isdir(checkpoint_dir):
    os.mkdir(checkpoint_dir)

epoch = 0
checkpoint = checkpoint_dir
batch_size = 32
start_epoch = 0
iterations = 1e5
workers = 4
lr = 1e-3
grad_clip = True

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
writer = SummaryWriter()

cudnn.benchmark = True

def main():
    """
    Training.
    """
    global start_epoch, epoch, checkpoint

    print('Find checkpoint')
    checkpoint = torch.load('./models/best_checkpoint_srresnet.pth.tar')
    start_epoch = checkpoint['epoch'] + 1
    model = checkpoint['model']
    optimizer = checkpoint['optimizer']

    model = model.to(device)
    criterion = nn.MSELoss().to(device)

    train_dataset = SRDataset(split='train',
                              crop_size=crop_size,
                              scaling_factor=scaling_factor,
                              lr_img_type='imagenet-norm',
                              hr_img_type='[-1, 1]')

    val_dataset = SRDataset(split='val',
                            crop_size=0,
                            scaling_factor=scaling_factor,
                            lr_img_type='imagenet-norm',
                            hr_img_type='[-1, 1]')
    print('train_loader loading')
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=workers,
                                               pin_memory=True)
    print('val_loader loading')

    val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, shuffle=True, num_workers=workers,
                                             pin_memory=True)

    epochs = int(iterations // len(train_loader) + 1)

    max_psnr = 0
    for epoch in range(start_epoch, epochs):
        train(train_loader=train_loader,
              model=model,
              criterion=criterion,
              optimizer=optimizer,
              epoch=epoch)
        psnr = val(val_loader=val_loader,
                   model=model,
                   epoch=epoch)
        if psnr.avg > max_psnr:
            max_psnr = psnr.avg
            torch.save({'epoch': epoch,
                        'model': model,
                        'optimizer': optimizer},os.path.join(checkpoint_dir, 'best_checkpoint_srresnet.pth.tar'))

        torch.save({'epoch': epoch,
                    'model': model,
                    'optimizer': optimizer},os.path.join(checkpoint_dir, 'checkpoint_srresnet.pth.tar'))

def train(train_loader, model, criterion, optimizer, epoch):

    epochs = str(int(iterations // len(train_loader) + 1))

    model.train()

    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    PSNRs = AverageMeter()

    start = time.time()

    for i, (lr_imgs, hr_imgs) in enumerate(train_loader):
        data_time.update(time.time() - start)

        lr_imgs = lr_imgs.to(device)
        hr_imgs = hr_imgs.to(device)

        sr_imgs = model(lr_imgs)

        sr_imgs_y = convert_image(sr_imgs, source='[-1, 1]', target='y-channel').squeeze(0)
        hr_imgs_y = convert_image(hr_imgs, source='[-1, 1]', target='y-channel').squeeze(0)
        psnr = peak_signal_noise_ratio(hr_imgs_y.cpu().detach().numpy(), sr_imgs_y.cpu().detach().numpy(), data_range=255.)
        PSNRs.update(psnr, lr_imgs.size(0))

        loss = criterion(sr_imgs, hr_imgs)

        optimizer.zero_grad()
        loss.backward()

        if grad_clip is not None:
            clip_gradient(optimizer, grad_clip)

        optimizer.step()

        losses.update(loss.item(), lr_imgs.size(0))

        batch_time.update(time.time() - start)

        start = time.time()

    writer.add_scalar('Loss/train', losses.avg, epoch)
    writer.add_scalar('PSNR/train', PSNRs.avg, epoch)
    print(
        f'Epoch: {epoch} / ' + epochs + ' -- '
        f'Batch Time: {batch_time.avg:.3f} -- '
        f'Loss: {losses.avg:.4f} -- '
        f'PSNR: {PSNRs.avg:.4f}'
    )
    del lr_imgs, hr_imgs, sr_imgs

def val(val_loader, model, epoch):
    model.eval()
    PSNRs = AverageMeter()
    with torch.no_grad():
        for i, (lr_imgs, hr_imgs) in enumerate(val_loader):

            lr_imgs = lr_imgs.to(device)
            hr_imgs = hr_imgs.to(device)

            sr_imgs = model(lr_imgs)

            sr_imgs_y = convert_image(sr_imgs, source='[-1, 1]', target='y-channel').squeeze(0)
            hr_imgs_y = convert_image(hr_imgs, source='[-1, 1]', target='y-channel').squeeze(0)
            psnr = peak_signal_noise_ratio(hr_imgs_y.cpu().numpy(), sr_imgs_y.cpu().numpy(), data_range=255.)

            PSNRs.update(psnr, lr_imgs.size(0))
    print(f'Epoch: {epoch}, PSNR: {PSNRs.avg}')
    writer.add_scalar('PSNR/val', PSNRs.avg, epoch)
    del lr_imgs, hr_imgs, sr_imgs

    return PSNRs

if __name__ == '__main__':
    main()


#!/usr/bin/env python
#-*- coding:utf-8 -*-

import cv2
import numpy as np

# 鼠标按下是转为true
drawing = False
# true 绘制矩形 键盘 m 控制
mode = True
ix, iy=-1, -1

def draw_circle_rect(event, x, y, flags, param):
    global ix, iy, drawing, mode

    # 左键
    if event == cv2.EVENT_LBUTTONDOWN:
        drawing = True
        ix, iy = x, y

    if event == cv2.EVENT_MOUSEMOVE and flags == cv2.EVENT_FLAG_LBUTTON:
        if drawing == True:
            if mode == True:
                cv2.rectangle(img, (ix, iy), (x, y), (0,255,0), 1)
            else:
                cv2.circle(img, (x,y), 3, (0,0,255), 1)
    if event == cv2.EVENT_LBUTTONUP:
        drawing = False

# 创建图像窗口并将窗口与回调函数绑定
img = np.zeros((400, 600, 3), np.uint8)
cv2.namedWindow('canvas')
cv2.setMouseCallback('canvas', draw_circle_rect)

while(True):
    cv2.imshow('canvas', img)
    key = cv2.waitKey(10) & 0xFF
    if key == ord('m'):
        mode = not mode
    if key == 27:
        break

cv2.destroyAllWindows()


from functools import partial
from math import log
from multiprocessing import cpu_count
from multiprocessing.pool import Pool
from operator import itemgetter
from typing import Callable, List, Any, Tuple
from sklearn.ensemble import RandomForestClassifier
import numpy as np
import pandas as pd
from pandas import DataFrame
from scipy.stats import chi2_contingency
from constants.dataset import CATEGORICAL_COLS

def parallel_map_df(func: Callable[[DataFrame, Any], DataFrame], num_cores=cpu_count()):
    """
    Python decorator that must be wrapped to a function that receive as first input a pandas Dataframe.
    Apply a trasformation on all the dataframe in parallel using the multiprocessing module

    Example:
        @parallel_map_df
        def sum_of_cols(df):
            df['c'] = df['a'] + df['b']
            return df

        @parallel_map_df(num_cores=16)
        def sum_of_cols(df):
            df['c'] = df['a'] + df['b']
            return df
    """

    def wrapper(df: DataFrame, *args, **kwargs):
        partial_func = partial(func, *args, **kwargs)
        partitions = np.linspace(0, len(df) - 1, num_cores + 1).round().astype('int')
        df_split = [df.iloc[partitions[i]:partitions[i + 1]] for i in range(len(partitions) - 1)]
        pool = Pool(num_cores)
        df = pd.concat(pool.map(partial_func, df_split))
        pool.close()
        pool.join()
        return df

    return wrapper

def _take_first_k(seq: np.array, k, keyf=lambda x: x) -> np.array:
    n = len(seq)
    if k > log(n):
        return sorted(seq, key=keyf)[:k]
    first_k = []
    for _ in range(k):
        max_el = max(seq, key=keyf)
        first_k.append(max_el)
        seq[max_el[0]][1] = -1  # to avoid remotion
    return first_k

def adversial_validation(train: DataFrame, test: DataFrame, perc_val=0.3, model=RandomForestClassifier()) -> Tuple[
    DataFrame, DataFrame]:
    """
    Do an adversial validation on the input datasets. Basically the adversial validation consists on finding the most similar
    validation set to the test test. You can find more info here: http://manishbarnwal.com/blog/2017/02/15/introduction_to_adversarial_validation/
    Attention: the columns on train and test must be in a numeric form
    :return a tuple (train, validation)
    """
    from constants.dataset import TARGETVAR
    train_orig = train
    if TARGETVAR in train.columns:
        train = train.drop(TARGETVAR, axis=1)
    if TARGETVAR in test.columns:
        test = test.drop(TARGETVAR, axis=1)
    train['__target'] = 0
    test['__target'] = 1
    dataset = pd.concat([train, test])
    X, y = dataset.drop('__target', axis=1), dataset['__target']
    model.fit(X, y)
    del train['__target']
    del test['__target']
    p = list(enumerate(map(itemgetter(1), model.predict_proba(train))))
    k = int(round(perc_val * len(train)))
    higher_proba = _take_first_k(p, k, itemgetter(1))
    newtest = train_orig.iloc[list(map(itemgetter(0), higher_proba))]
    newtrain = train_orig.drop(newtest.index)
    return newtrain, newtest

def _cramerv(col1: np.array, col2: np.array):
    confusion_matrix = pd.crosstab(col1, col2)
    chi2, p_value = chi2_contingency(confusion_matrix)[0:2]
    n = confusion_matrix.sum().sum()
    phi2 = chi2 / n
    r, k = confusion_matrix.shape
    phi2corr = max(0, phi2 - ((k - 1) * (r - 1)) / (n - 1))
    rcorr = r - ((r - 1) ** 2) / (n - 1)
    kcorr = k - ((k - 1) ** 2) / (n - 1)
    return np.sqrt(phi2corr / min((kcorr - 1), (rcorr - 1))) if not phi2corr == 0 else 0, p_value

def cramerv(dataset: DataFrame, cols: List[str] = CATEGORICAL_COLS) -> np.matrix:
    """
    Apply the cramerv test on each pair of columns. The cramerv test is used to
    measure in [0,1] the correlation between categorical variables
    :param dataset: Dataframe where the correlation is measured
    :param cols: the columns of dataset where measure the correlation
    :return: the correlation matrix of cramerv on the specified columns of dataset
    """
    matr = np.matrix([[0] * len(cols)] * len(cols), dtype='float')
    for i in range(len(cols)):
        for j in range(i, len(cols)):
            if i == j:
                matr[i, j] = 1
            else:
                corr, pvalue = _cramerv(dataset[cols[i]], dataset[cols[j]])
                matr[i, j] = corr
                matr[j, i] = corr
    return matr

def _fill_na(df, column, model):
    na_rows = df[column].isna()
    train, test = df[~na_rows], df[na_rows].drop(columns=column)
    try:
        model.fit(train.drop(columns=column).values, train[column].values)
    except ValueError as e:
        if str(e) == "Unknown label type: 'continuous'":
            model.fit(train.drop(columns=column).values, train[column].values.astype('int'))
    del train
    new_values = model.predict(test.values)
    del model
    test[column] = new_values
    df.loc[test.index, column] = test[column]
    del test
    return df

def _cat_to_int(df, targetcol):
    strcols = df.select_dtypes(include=['object', 'category']).columns.tolist()
    for strcol in strcols:
        df[strcol] = pd.to_numeric(df[strcol].astype('category').cat.codes, downcast='integer')
    if targetcol in strcols:
        df[targetcol] = df[targetcol].replace(-1, np.NaN)
    return df

def _fill_quantitative_with_mean(df, quant_cols):
    df[quant_cols] = df[quant_cols].fillna(df[quant_cols].mean())
    return df

def fill_na_as_prediction(df: pd.DataFrame, cols_na: List[str], model=None):
    """
    Fill na of column as prediction task.
    df category columns must be of type
    string, bool or category, while numeric of
    any other type (int*, float*, uint*).
    When predict the other columns with na
    fill temporally the others quantitative/
    category columns
    :param df: a pandas dataframe
    :param cols_na: The columns to fill
    :param model: By default it is a sklearn RandomForestClassifier
    :return: a new dataframe with na filled
    """
    quantitative_na_cols = df.loc[:, df.isna().any()] \
        .select_dtypes(exclude=['category', 'object', 'bool']) \
        .columns.tolist()
    if model is None:
        model = RandomForestClassifier(n_jobs=-1)
    for col_na in cols_na:
        if col_na in quantitative_na_cols:
            quantitative_na_cols.remove(col_na)
        tmp = df.copy()
        tmp = _fill_quantitative_with_mean(tmp, quantitative_na_cols)
        tmp = _cat_to_int(tmp, col_na)
        tmp = _fill_na(tmp, col_na, model)
        if col_na in df.select_dtypes(include=['category', 'object']).columns:
            df[col_na] = tmp[col_na].astype('category')
        else:
            df[col_na] = tmp[col_na]
        del tmp
    return df

__all__ = ['fill_na_as_prediction', 'cramerv', 'adversial_validation', 'parallel_map_df']


import torch
import gpytorch
import time
import numpy as np

from meta_bo.models.models import LearnedGPRegressionModel, NeuralNetwork, AffineTransformedDistribution
from meta_bo.models.util import _handle_input_dimensionality, DummyLRScheduler
from meta_bo.models.abstract import RegressionModelMetaLearned
from config import device

class PACOH_MAP_GP(RegressionModelMetaLearned):

    def __init__(self, input_dim, learning_mode='both', weight_decay=0.0, feature_dim=2, num_iter_fit=10000,
                 covar_module='NN', mean_module='NN', mean_nn_layers=(32, 32), kernel_nn_layers=(32, 32),
                 task_batch_size=5, lr=1e-3, lr_decay=1.0, normalize_data=True,
                 normalization_stats=None, random_state=None):

        super().__init__(normalize_data, random_state)

        assert learning_mode in ['learn_mean', 'learn_kernel', 'both', 'vanilla']
        assert mean_module in ['NN', 'constant', 'zero'] or isinstance(mean_module, gpytorch.means.Mean)
        assert covar_module in ['NN', 'SE'] or isinstance(covar_module, gpytorch.kernels.Kernel)

        self.input_dim = input_dim
        self.lr, self.weight_decay, self.feature_dim = lr, weight_decay, feature_dim
        self.num_iter_fit, self.task_batch_size, self.normalize_data = num_iter_fit, task_batch_size, normalize_data

        """ Setup prior and likelihood """
        self._setup_gp_prior(mean_module, covar_module, learning_mode, feature_dim, mean_nn_layers, kernel_nn_layers)
        self.likelihood = gpytorch.likelihoods.GaussianLikelihood(
            noise_constraint=gpytorch.likelihoods.noise_models.GreaterThan(1e-3)).to(device)
        self.shared_parameters.append({'params': self.likelihood.parameters(), 'lr': self.lr})
        self._setup_optimizer(lr, lr_decay)

        """ ------- normalization stats & data setup  ------- """
        self._normalization_stats = normalization_stats
        self.reset_to_prior()

        self.fitted = False

    def meta_fit(self, meta_train_tuples, meta_valid_tuples=None, verbose=True, log_period=500, n_iter=None):

        assert (meta_valid_tuples is None) or (all([len(valid_tuple) == 4 for valid_tuple in meta_valid_tuples]))
        self.likelihood.train()

        task_dicts = self._prepare_meta_train_tasks(meta_train_tuples)

        t = time.time()
        cum_loss = 0.0
        n_iter = self.num_iter_fit if n_iter is None else n_iter

        for itr in range(1, n_iter + 1):
            # actual meta-training step
            task_dict_batch = self._rds.choice(task_dicts, size=self.task_batch_size)
            loss = self._step(task_dict_batch)
            cum_loss += loss

            # print training stats stats
            if itr == 1 or itr % log_period == 0:
                duration = time.time() - t
                avg_loss = cum_loss / (log_period if itr > 1 else 1.0)
                cum_loss = 0.0
                t = time.time()

                message = 'Iter %d/%d - Loss: %.6f - Time %.2f sec' % (itr, self.num_iter_fit, avg_loss, duration)

                # if validation data is provided  -> compute the valid log-likelihood
                if meta_valid_tuples is not None:
                    self.likelihood.eval()
                    valid_ll, valid_rmse, calibr_err, calibr_err_chi2 = self.eval_datasets(meta_valid_tuples)
                    self.likelihood.train()
                    message += ' - Valid-LL: %.3f - Valid-RMSE: %.3f - Calib-Err %.3f' % (valid_ll, valid_rmse, calibr_err)

                if verbose:
                    print(message)

        self.fitted = True

        # set gpytorch modules to eval mode and set gp to meta-learned gp prior
        assert self.X_data.shape[0] == 0 and self.y_data.shape[0] == 0, "Data for posterior inference can be passed " \
                                                                        "only after the meta-training"
        for task_dict in task_dicts:
            task_dict['model'].eval()
        self.likelihood.eval()
        self.reset_to_prior()
        return loss

    def predict(self, test_x: np.ndarray, return_density: bool = False, include_obs_noise: bool = True, **kwargs):
        if test_x.ndim == 1:
            test_x = np.expand_dims(test_x, axis=-1)

        with torch.no_grad():
            test_x_normalized = self._normalize_data(test_x)
            test_x_tensor = torch.from_numpy(test_x_normalized).float().to(device)

            pred_dist = self.gp(test_x_tensor)
            if include_obs_noise:
                pred_dist = self.likelihood(pred_dist)
            pred_dist_transformed = AffineTransformedDistribution(pred_dist, normalization_mean=self.y_mean,
                                                                  normalization_std=self.y_std)
            if return_density:
                return pred_dist_transformed
            else:
                pred_mean = pred_dist_transformed.mean.cpu().numpy()
                pred_std = pred_dist_transformed.stddev.cpu().numpy()
                return pred_mean, pred_std

    def meta_predict(self, context_x, context_y, test_x, return_density=False):
        """
        Performs posterior inference (target training) with (context_x, context_y) as training data and then
        computes the predictive distribution of the targets p(y|test_x, test_context_x, context_y) in the test points

        Args:
            context_x: (ndarray) context input data for which to compute the posterior
            context_y: (ndarray) context targets for which to compute the posterior
            test_x: (ndarray) query input data of shape (n_samples, ndim_x)
            return_density: (bool) whether to return result as mean and std ndarray or as MultivariateNormal pytorch object

        Returns:
            (pred_mean, pred_std) predicted mean and standard deviation corresponding to p(t|test_x, test_context_x, context_y)
        """

        context_x, context_y = _handle_input_dimensionality(context_x, context_y)
        test_x = _handle_input_dimensionality(test_x)
        assert test_x.shape[1] == context_x.shape[1]

        # normalize data and convert to tensor
        context_x, context_y = self._prepare_data_per_task(context_x, context_y)

        test_x = self._normalize_data(X=test_x, Y=None)
        test_x = torch.from_numpy(test_x).float().to(device)

        with torch.no_grad():
            # compute posterior given the context data
            gp_model = LearnedGPRegressionModel(context_x, context_y, self.likelihood,
                                                learned_kernel=self.nn_kernel_map, learned_mean=self.nn_mean_fn,
                                                covar_module=self.covar_module, mean_module=self.mean_module)
            gp_model.eval()
            self.likelihood.eval()
            pred_dist = self.likelihood(gp_model(test_x))
            pred_dist_transformed = AffineTransformedDistribution(pred_dist, normalization_mean=self.y_mean,
                                                                  normalization_std=self.y_std)

        if return_density:
            return pred_dist_transformed
        else:
            pred_mean = pred_dist_transformed.mean
            pred_std = pred_dist_transformed.stddev
            return pred_mean.cpu().numpy(), pred_std.cpu().numpy()

    def reset_to_prior(self):
        """Clears the training data that was added via add_data(X, y) and resets the posterior to the prior."""
        self._reset_data()
        self.gp = lambda x: self._prior(x)

    def _reset_posterior(self):
        x_context = torch.from_numpy(self.X_data).float().to(device)
        y_context = torch.from_numpy(self.y_data).float().to(device)
        self.gp = LearnedGPRegressionModel(x_context, y_context, self.likelihood,
                                      learned_kernel=self.nn_kernel_map, learned_mean=self.nn_mean_fn,
                                      covar_module=self.covar_module, mean_module=self.mean_module)
        self.gp.eval()
        self.likelihood.eval()

    def state_dict(self):
        state_dict = {
            'optimizer': self.optimizer.state_dict(),
            'model': self.task_dicts[0]['model'].state_dict()
        }
        for task_dict in self.task_dicts:
            for key, tensor in task_dict['model'].state_dict().items():
                assert torch.all(state_dict['model'][key] == tensor).item()
        return state_dict

    def load_state_dict(self, state_dict):
        for task_dict in self.task_dicts:
            task_dict['model'].load_state_dict(state_dict['model'])
        self.optimizer.load_state_dict(state_dict['optimizer'])

    def _prior(self, x):
        if self.nn_kernel_map is not None:
            projected_x = self.nn_kernel_map(x)
        else:
            projected_x = x

            # feed through mean module
        if self.nn_mean_fn is not None:
            mean_x = self.nn_mean_fn(x).squeeze()
        else:
            mean_x = self.mean_module(projected_x).squeeze()

        covar_x = self.covar_module(projected_x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)

    def _prepare_meta_train_tasks(self, meta_train_tuples):
        self._check_meta_data_shapes(meta_train_tuples)

        if self._normalization_stats is None:
            self._compute_meta_normalization_stats(meta_train_tuples)
        else:
            self._set_normalization_stats(self._normalization_stats)

        task_dicts = [self._dataset_to_task_dict(x, y) for x, y in meta_train_tuples]
        return task_dicts

    def _dataset_to_task_dict(self, x, y):
        # a) prepare data
        x_tensor, y_tensor = self._prepare_data_per_task(x, y)
        task_dict = {'train_x': x_tensor, 'train_y': y_tensor}

        # b) prepare model
        task_dict['model'] = LearnedGPRegressionModel(task_dict['train_x'], task_dict['train_y'], self.likelihood,
                                                      learned_kernel=self.nn_kernel_map, learned_mean=self.nn_mean_fn,
                                                      covar_module=self.covar_module, mean_module=self.mean_module)
        task_dict['mll_fn'] = gpytorch.mlls.ExactMarginalLogLikelihood(self.likelihood, task_dict['model'])
        return task_dict

    def _step(self, task_dicts):
        assert len(task_dicts) > 0
        loss = 0.0
        self.optimizer.zero_grad()

        for task_dict in task_dicts:
            output = task_dict['model'](task_dict['train_x'])
            mll = task_dict['mll_fn'](output, task_dict['train_y'])
            loss -= mll

        loss.backward()
        self.optimizer.step()
        self.lr_scheduler.step()
        return loss.item()

    def _setup_gp_prior(self, mean_module, covar_module, learning_mode, feature_dim, mean_nn_layers, kernel_nn_layers):

        self.shared_parameters = []

        # a) determine kernel map & module
        if covar_module == 'NN':
            assert learning_mode in ['learn_kernel', 'both'], 'neural network parameters must be learned'
            self.nn_kernel_map = NeuralNetwork(input_dim=self.input_dim, output_dim=feature_dim,
                                          layer_sizes=kernel_nn_layers).to(device)
            self.shared_parameters.append(
                {'params': self.nn_kernel_map.parameters(), 'lr': self.lr, 'weight_decay': self.weight_decay})
            self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel(ard_num_dims=feature_dim)).to(device)
        else:
            self.nn_kernel_map = None

        if covar_module == 'SE':
            self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel(ard_num_dims=self.input_dim)).to(device)
        elif isinstance(covar_module, gpytorch.kernels.Kernel):
            self.covar_module = covar_module.to(device)

        # b) determine mean map & module

        if mean_module == 'NN':
            assert learning_mode in ['learn_mean', 'both'], 'neural network parameters must be learned'
            self.nn_mean_fn = NeuralNetwork(input_dim=self.input_dim, output_dim=1, layer_sizes=mean_nn_layers).to(device)
            self.shared_parameters.append(
                {'params': self.nn_mean_fn.parameters(), 'lr': self.lr, 'weight_decay': self.weight_decay})
            self.mean_module = None
        else:
            self.nn_mean_fn = None

        if mean_module == 'constant':
            self.mean_module = gpytorch.means.ConstantMean().to(device)
        elif mean_module == 'zero':
            self.mean_module = gpytorch.means.ZeroMean().to(device)
        elif isinstance(mean_module, gpytorch.means.Mean):
            self.mean_module = mean_module.to(device)

        # c) add parameters of covar and mean module if desired

        if learning_mode in ["learn_kernel", "both"]:
            self.shared_parameters.append({'params': self.covar_module.hyperparameters(), 'lr': self.lr})

        if learning_mode in ["learn_mean", "both"] and self.mean_module is not None:
            self.shared_parameters.append({'params': self.mean_module.hyperparameters(), 'lr': self.lr})

    def _setup_optimizer(self, lr, lr_decay):
        self.optimizer = torch.optim.AdamW(self.shared_parameters, lr=lr, weight_decay=self.weight_decay)
        if lr_decay < 1.0:
            self.lr_scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, 1000, gamma=lr_decay)
        else:
            self.lr_scheduler = DummyLRScheduler()

    def _vectorize_pred_dist(self, pred_dist):
        return torch.distributions.Normal(pred_dist.mean, pred_dist.stddev)

if __name__ == "__main__":
    from meta_bo.meta_environment import RandomMixtureMetaEnv

    meta_env = RandomMixtureMetaEnv(random_state=np.random.RandomState(29))
    meta_train_data = meta_env.generate_uniform_meta_train_data(num_tasks=20, num_points_per_task=10)
    meta_test_data = meta_env.generate_uniform_meta_valid_data(num_tasks=50, num_points_context=10, num_points_test=160)

    NN_LAYERS = (32, 32)
    plot = True
    from matplotlib import pyplot as plt

    if plot:
        for x_train, y_train in meta_train_data:
            plt.scatter(x_train, y_train)
        plt.title('sample from the GP prior')
        plt.show()

    """ 2) Classical mean learning based on mll """

    print('\n ---- GPR mll meta-learning ---- ')

    torch.set_num_threads(2)

    weight_decay = 0.01
    gp_model = PACOH_MAP_GP(input_dim=meta_env.domain.d, num_iter_fit=20000,
                            weight_decay=weight_decay, task_batch_size=2,
                            covar_module='NN', mean_module='NN', mean_nn_layers=NN_LAYERS,
                            kernel_nn_layers=NN_LAYERS)
    itrs = 0
    print("---- weight-decay =  %.4f ----"%weight_decay)
    for i in range(10):
        gp_model.meta_fit(meta_train_data, meta_valid_tuples=meta_test_data, log_period=1000, n_iter=500)
        itrs += 500

        x_plot = np.linspace(meta_env.domain.l, meta_env.domain.u, num=150)
        x_context, t_context, x_test, y_test = meta_test_data[0]
        pred_mean, pred_std = gp_model.meta_predict(x_context, t_context, x_plot)
        ucb, lcb = (pred_mean + 2 * pred_std).flatten(), (pred_mean - 2 * pred_std).flatten()

        plt.scatter(x_test, y_test)
        plt.scatter(x_context, t_context)

        plt.plot(x_plot, pred_mean)
        plt.fill_between(x_plot.flatten(), lcb, ucb, alpha=0.2)
        plt.title('GPR meta mll (weight-decay =  %.4f) itrs = %i' % (weight_decay, itrs))
        plt.show()