from selenium import webdriver
from bs4 import BeautifulSoup
import time
import re

class LimitNotMatched(Exception):
    def __init__(self, error_msg):
        super().__init__(error_msg)

class GooglePlayStoreReviewCrawler:
    """
    Google play에서 리뷰를 크롤링하는 class
    """
    def __init__(self, app_name:str, chrome_driver_url:str, url:str, conn, limit_scroll:bool = False, limit_num:int= -1, scroll_pause:int= 2):
        """

        :param app_name: 크롤링 할 어플리케이션 이름
        :param chrome_driver_url: 크롬 드라이버의 위치
        :param url: 크롤링 할 페이지 url
        :param conn: DB 연결
        :param limit_scroll: 스크롤 제한 설정
        :param limit_num: 스크롤 제한 수
        :param scroll_pause: 스크롤 동작 시 잠시 멈추는 시간

        """
        self.app_name = app_name
        self.__chrome_driver_url = chrome_driver_url
        self.url = url
        self.limit_scroll = limit_scroll
        self.limit_num = limit_num
        self.conn = conn
        self.scroll_pause = scroll_pause
        self.__make_table()

    def __make_table(self):
        """
        Table이 존재하지 않으면 생성
        :return: None
        """
        cur = self.conn.cursor()
        sql = '''
        CREATE TABLE IF NOT EXISTS {} (
            user_id VARCHAR(50),
            total_score TINYINT,
            review_score TINYINT,
            write_date DATE,
            useful_cnt SMALLINT,
            review_txt LONGTEXT
        );
        '''.format(self.app_name)
        cur.execute(sql)
        self.conn.commit()
        cur.close()

    def start(self):
        """
        크롤링을 진행하는 함수
        :return: None
        """
        if self.limit_scroll is False and self.limit_num != -1:
            raise LimitNotMatched("limit_scroll is False, but you enter limit_num.")

        cur = self.conn.cursor()

        driver = webdriver.Chrome(self.__chrome_driver_url)
        driver.get(self.url)

        last_height = driver.execute_script("return document.body.scrollHeight")
        scroll_cnt = 0

        while True:
            if self.limit_scroll is True and scroll_cnt >= self.limit_num:
                break

            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(self.scroll_pause)
            new_height = driver.execute_script("return document.body.scrollHeight")

            if new_height == last_height:
                try:
                    driver.find_element_by_xpath("//*[contains(@class,'U26fgb O0WRkf oG5Srb C0oVfc n9lfJ')]").click()
                except:
                    break

            last_height = new_height
            scroll_cnt += 1

        page = driver.page_source

        bs = BeautifulSoup(page, "lxml")
        rs = bs.find_all('div',{'class':'d15Mdf bAhLNe'})

        for rs_i in rs:
            name = rs_i.find('div', {'class': 'bAhLNe kx8XBd'}).find('span', {'class': "X43Kjb"}).text
            score = rs_i.find('div', {'class': 'pf5lIe'}).find('div').get('aria-label')
            total_score, review_score = re.findall('\d', score)
            date = rs_i.find('span', {'class': "p2TkOb"}).text
            date = '/'.join(re.findall('[0-9]+', date))
            useful_cnt = rs_i.find('div', {'class': 'YCMBp GVFJbb'}).find('div', {'class': "XlMhZe"}).find('div', {
                'class': "jUL89d y92BAb"}).text
            review_txt = rs_i.find('div', {'class': 'UD7Dzf'}).find('span', {'jsname': 'fbQN7e'}).text

            if len(review_txt) == 0:
                review_txt = rs_i.find('div', {'class': 'UD7Dzf'}).find('span', {'jsname': 'bN97Pc'}).text

            review_txt = review_txt.replace("'","\\'")

            sql = 'INSERT INTO {} VALUES (\'{}\',{},{},\'{}\',{},\'{}\')'.format(self.app_name, name, total_score,
                                                                     review_score, date, useful_cnt, review_txt)
            try:
                cur.execute(sql)
            except Exception as e:
                print('Error Occured in {} {} \n:{}'.format(name,date,e))
                print(sql)
        self.conn.commit()
        driver.close()
        print('Finished')


import os
import sys
from PIL import Image
# os is inbuilt module in python
# It is used to perform many tasks
currDir = sys.argv[1]
newDir = sys.argv[2]

allFiles = os.listdir(os.getcwd())
newDirPath = os.getcwd()+"/" + newDir
currPath = os.getcwd()+"/"+currDir
if currDir not in allFiles:
    raise Exception(currDir + " does not exists")

if newDir not in allFiles:
    os.mkdir(newDirPath)

for file_ in os.listdir(currPath):
    if file_.endswith('.jpg'):
        img = Image.open(currPath+"/"+file_)
        file_ = file_.replace('.jpg', '.png')
        img.save(newDirPath+"/"+file_, 'png')


import sys
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras
import tensorflow as tf
from tensorflow.keras import backend as K

def custom_AF(n):
	def custom_func(x):
		return(n*K.tanh(x))
	return custom_func

my_data = np.load(sys.argv[1])

training_inputs = my_data['train_samples']
training_targets = my_data['train_targets']
validation_inputs = my_data['val_samples']
validation_targets = my_data['val_targets']

width, channels = training_inputs[0].shape

epochs=30

training_regimes = []
for AF1 in [custom_AF(3.5),'relu']:
	for AF2 in [custom_AF(3.5),'relu']:
		for AF3 in [custom_AF(3.5),'relu']:
			training_regimes.append((AF1,AF2,AF3))

training_names = []
for AF1 in ['tanh3.5','relu']:
	for AF2 in ['tanh3.5','relu']:
		for AF3 in ['tanh3.5','relu']:
			training_names.append((AF1,AF2,AF3))

plt.style.use('dark_background')
fig, axes = plt.subplots(ncols=3, nrows=3, figsize=(20,20))

ax_count = 0
for i in range(len(training_regimes)):
	training_regime = training_regimes[i]
	training_name = training_names[i]
	model = keras.Sequential()
	model.add(keras.layers.Conv1D(filters=100, kernel_size=5, strides=1, padding='VALID', input_shape=(width, channels), activation=training_regime[0], use_bias=True))
	model.add(keras.layers.MaxPooling1D(pool_size=2, strides=None, padding='VALID'))
	model.add(keras.layers.Conv1D(filters=50, kernel_size=4, strides=4, padding='VALID', activation=training_regime[1], use_bias=True))
	model.add(keras.layers.Flatten())
	model.add(keras.layers.Dense(100, activation=training_regime[2], use_bias=True))
	model.add(keras.layers.Dense(1, activation='sigmoid', use_bias=True))

	model.compile(optimizer=tf.train.AdamOptimizer(0.001),
				  loss='mse',
				  metrics=['mae'])

	model_history = model.fit(training_inputs, training_targets,
			 				  epochs=epochs,
			 				  batch_size=100,
							  shuffle=True,
							  validation_data=(validation_inputs, validation_targets),
							  verbose=1)

	predicted_validation = model.predict(validation_inputs)

	sorted_data = sorted(list(zip(validation_targets, predicted_validation)), key = lambda x: x[0])

	sorted_true = [x[0] for x in sorted_data]
	sorted_pred = [float(x[1]) for x in sorted_data]

	train_error = model_history.history['mean_absolute_error']
	val_error = model_history.history['val_mean_absolute_error']

	axes.flat[ax_count].scatter(x=range(len(sorted_pred)), y=sorted_pred, s=4, alpha=0.3, color='#1f78b3', label='Predicted Cscore data', zorder=5)
	axes.flat[ax_count].plot(sorted_true, color='#fe7f02', label='True Cscore data', zorder=10)
	axes.flat[ax_count].set_xticklabels([])
	axes.flat[ax_count].set_xlabel('Sorted Validation Data')
	axes.flat[ax_count].set_ylabel('Scaled Cscore')
	axes.flat[ax_count].set_title(str(training_name))

	ax_count += 1
	axes.flat[8].plot([x for x in range(1,epochs+1)], val_error, label=str(training_name))

for ax in axes.flat:
	ax.legend()
axes.flat[8].set_ylabel('Validation Mean Absolute Error')
axes.flat[8].set_xlabel('Epoch')
plt.tight_layout()
fig.savefig(sys.argv[2])
plt.close()


# coding=gbk
import json
import re

FileName = ""
ethernet = []
hw_id = []

with open('config.boot', 'r') as config:
    for line in config:
        #print(line)
        if(re.match('^[a-zA-Z].*', line)):
            FileName = re.match('^[a-zA-Z].*|^}', line).group().split()[0]
            print(FileName)
        elif(re.search('.*{', line)):
            key = re.search('.*{', line).group().split()[1]
            print(key)
            ethernet.append(key)
            if(key == "lo"):
                ethernet.pop()
        elif (re.search('.*}', line)):
            continue
        else:
            val = line.split()[1]
            print(val)
            hw_id.append(val)

result = []
for i in range(len(ethernet)):
    dic = {'ethernet':ethernet[i],'hw_id':hw_id[i]}
    result.append(dic)

print("###"+FileName+"###")
newFile = FileName
with open(newFile, 'w') as name:
    for i in range(len(result)):
        jsonData = json.dumps(result[i])
        name.write(jsonData+','+'\n')


import os
import json
import sys

__config_file_path = './config.json'
__config = {}

def get_configuration():
    # Get configuration file
    if os.path.isfile(__config_file_path):
        config_file = open(__config_file_path, 'r')
        __config.update(json.loads(config_file.read()))
        config_file.close()
    else:
        print("Configuration file does not exist.")
        sys.exit(0)

def main():

    get_configuration()

    count = 0

    hosts = os.listdir(__config['out_dir_path'] + __config['hosts_dir_path'])

    for host in hosts:
        info_json = {}
        info_file_path = __config['out_dir_path'] + __config['hosts_dir_path'] + host + '/' +  __config['host_json_filename']
        if os.path.isfile(info_file_path):
            info_file = open(info_file_path, 'r')
            info_json.update(json.loads(info_file.read()))
            info_file.close()
        else:
            print('ERROR. Info file does not exist. (' + info_file_path + ')')

        try:
            gsb_dic = info_json['GSB']
            if len(gsb_dic) > 0:
                count += 1
        except:
            continue

    print('\nGSB tagged {0} hosts of {1}\n'.format(count, len(hosts)))

if __name__ == '__main__':
    main()


#%%
from ast import Suite
#from termios import VDISCARD
from textwrap import dedent
from time import sleep
from typing import Sequence, Text
from numpy import NAN, dtype, expand_dims, log, unicode_
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import pandas as pd
import time, datetime, os
from csv import writer
import cloudscraper #faz a requisição pra receber os dados da pagina
from bs4 import BeautifulSoup #faz o tratamento do texto e extração por css
import re #para expressoes regulares
import timeit
from sqlalchemy import true
import unidecode
import re
import numpy as np
import math as mat

user="Pupe"
url = "https://www.dfimoveis.com.br/venda/df/brasilia/asa-norte/apartamento"
base_exportar="venda aps asa norte - dfimoveis - base limpa pelo python"

driver = webdriver.Chrome(executable_path='./chromedriver100')

imoveis_link_list = []

driver.get(url)
driver.maximize_window()

try:
    element = WebDriverWait(driver, 5).until(
       EC.presence_of_element_located((By.CLASS_NAME, "btn-cookie"))
    )
    element.click()
except:
    print("Sem conferência de coockies")

def iniciar_coleta():
    time.sleep(1)
    resultados = driver.find_element_by_class_name("property-list")
    imoveis = resultados.find_elements_by_class_name("gtm-imovel")
    numlinks= range(1,len(imoveis)+1)
    for i in numlinks:
        PAGE_URL = driver.find_element_by_xpath('/html/body/div[1]/section[3]/div[2]/ul/li[{}]/div/div[2]/div[2]/h3[1]/a'.format(i)).get_attribute("href")
        imoveis_link_list.append(PAGE_URL)

    time.sleep(2)
    #return driver.find_element_by_link_text("Próxima Página >").is_enabled()

iniciar_coleta()

for i in range(300):
    try:
        proxima_pagina = driver.find_element_by_link_text("Próximo")
        proxima_pagina.click()
        time.sleep(5)
        iniciar_coleta()
    except:
        print("Sem próxima página")
        break
    time.sleep(5)

print(imoveis_link_list)
print(len(imoveis_link_list))

links=imoveis_link_list

scraper = cloudscraper.create_scraper()

imoveis_list=[]

##%% -> caso o scraping pare por falta de internet, ativar essa célula e rodar de novo
links=imoveis_link_list[len(imoveis_list):len(imoveis_link_list)]
scraper = cloudscraper.create_scraper()

for link in range(0,len(links)):
    content = scraper.get(links[link]).text
    soup = BeautifulSoup(content, 'html.parser')

    try:
        PAGE_URL = links[link]
    except:
        PAGE_URL = ""
    try:
        ENDERECO_COMP = soup.find("div",class_="col-9").text
    except:
        ENDERECO_COMP = ""
    try:
        GERAL = soup.find_all("div",class_="r-computador-dados")[1].text
    except:
        GERAL = ""
    try:
        DETALHES1 = soup.find_all("div",class_="col-md-12 bg-white shadow mt-2 pb-2")[0].text
    except:
        DETALHES1 = ""
    try:
        DETALHES = soup.find_all("div",class_="col-md-12 bg-white shadow mt-2 pb-2")[0].text
    except:
        DETALHES = ""
    try:
        ANUNCIANTE = soup.find_all("div",class_="col-md-8 mt-2 ml-2")[0].text
    except:
        ANUNCIANTE= ""
    try:
        FOTO1 = str(soup.find("div",class_="CarroselFotos"))
    except:
        FOTO1 = ""
    try:
        DESCRICAO_COMP = soup.find("p",class_="texto-descricao").text
    except:
        DESCRICAO_COMP = ''

    SCRAPING_TIME = datetime.datetime.now().strftime("%D %H:%M:%S")

    imoveis_dic = {
        'ENDERECO_COMP': ENDERECO_COMP,
        'GERAL': GERAL,
        'DETALHES1':DETALHES1,
        'DETALHES':DETALHES,
        'ANUNCIANTE': ANUNCIANTE,
        'FOTO1': FOTO1,
        'DESCRICAO_COMP': DESCRICAO_COMP,
        'SCRAPING_TIME': SCRAPING_TIME,
        'PAGE_URL': PAGE_URL,
    }

    imoveis_list.append(imoveis_dic)
    time.sleep(3)
    print("Pegou o imóvel {}: {}".format(len(imoveis_list),ENDERECO_COMP))

dds2 = pd.DataFrame(imoveis_list)

dds2_teste = dds2

# Criando a coluna DATA_ID
dds2_teste["DATA_ID"] = range(0,len(dds2_teste))

# Reordenando as colunas
cols = dds2_teste.columns.tolist()
cols = cols[-1:] + cols[:-1]
dds2_teste = dds2_teste[cols]

# Renoameando o nome da coluna PAGE_URL para LINK
dds2_teste = dds2_teste.rename(columns={"PAGE_URL": "LINK"})
dds2_teste["LINK"] = dds2_teste["LINK"].str.lower()

for i in range(0,len(dds2_teste)):
    for j in range(0,len(dds2_teste.columns)):
        dds2_teste.iloc[i,j]=unidecode.unidecode(str(dds2_teste.iloc[i,j])).upper()

# Criando as colunas TIPO_IMOVEL e TIPO_ANUNCIO
TIPO = dds2_teste["DETALHES1"].str.split(" DE ", expand = True)
ANUNCIO = TIPO.iloc[:,0]
ANUNCIO = ANUNCIO.str.replace(" ","")
ANUNCIO = ANUNCIO.str.split("\\\r\\\n", expand = True).iloc[:,1].str.strip()
dds2_teste["TIPO_ANUNCIO"] = ANUNCIO

IMOVEL = TIPO.iloc[:,1].str.split("CODIGO", expand = True).iloc[:,0]
IMOVEL = IMOVEL.str.replace("(\\\n|\\\r)","").str.strip()
dds2_teste["TIPO_IMOVEL"] = IMOVEL

# Criando a coluna RUA_OU_QUADRA
dds2_teste["RUA_OU_QUADRA"] = dds2_teste["ENDERECO_COMP"].str.replace("\\\n", "").str.strip()

# Criando as colunas BAIRRO e CIDADE
CIDADE_BAIRRO = dds2_teste["GERAL"].str.split("CIDADE",expand = True).iloc[:,1]
CIDADE_BAIRRO = CIDADE_BAIRRO.str.split("\\\n\\\n\\\n\\\n\\\n", expand = True).iloc[:,0]
CIDADE_BAIRRO = CIDADE_BAIRRO.str.replace("(\\\n|\\\r|\\:)","").str.strip()
CIDADE_BAIRRO = CIDADE_BAIRRO.str.split(" - ", expand = True)
dds2_teste["CIDADE"] = CIDADE_BAIRRO.iloc[:,0]
dds2_teste["BAIRRO"] = CIDADE_BAIRRO.iloc[:, 1]

# Criando a coluna ESTADO
dds2_teste["ESTADO"] = pd.DataFrame([url])[0].str.split("/",expand = True).iloc[0,4]
dds2_teste["ESTADO"] = dds2_teste["ESTADO"].str.upper()

# Criando a coluna AREA
AREA = dds2_teste["GERAL"].str.split("AREA UTIL",expand = True)
AREA = AREA.iloc[:,1].str.split("(M2|HA)", expand = True).iloc[:,0]
AREA = AREA.str.replace("(\\\r|\\\n|\\:|R|\\$|\\/)", "").str.strip()
AREA = AREA.str.split(" A ", expand = True).iloc[:,0]
AREA = AREA.str.replace(".", "")
AREA = AREA.str.replace(",",".")
dds2_teste["AREA"] = AREA
dds2_teste["AREA"] = pd.to_numeric(dds2_teste["AREA"])
dds2_teste = dds2_teste.dropna(subset=["AREA"])

# LEMBRAR DE PEGAR AS AREAS > 0
dds2_teste = dds2_teste[ dds2_teste.AREA >= 1 ]
dds2_teste.index = range(0,len(dds2_teste))

# Criando a coluna VALOR e VALOR_LN
VALOR = dds2_teste["GERAL"].str.split("R\\$\\\r\\\n", expand = True).iloc[:,1]

VALOR = VALOR.str.split("AREA UTIL\\:\\\r\\\n", expand = True).iloc[:,0]

VALOR = VALOR.str.replace("(A PARTIR|SIMULAR CREDITO|SOB CONSULTA|\\\r|\\\n|\\:)", "").str.strip()
VALOR = VALOR.str.replace(".", "")
VALOR = VALOR.str.replace(",",".")

dds2_teste["VALOR"] = VALOR
dds2_teste["VALOR"] = pd.to_numeric(dds2_teste["VALOR"])
dds2_teste["VALOR_LN"] = log(dds2_teste["VALOR"])

# Criando a coluna VALOR_M2 e VALOR_M2_LN
dds2_teste["VALOR_M2"] = dds2_teste["VALOR"]/dds2_teste["AREA"]
dds2_teste["VALOR_M2_LN"] = log(dds2_teste["VALOR_M2"])

# CRIANDO A COLUNA DE QUARTO
QUARTO = dds2_teste["GERAL"].str.split("AREA UTIL", expand = True).iloc[:,1]
QUARTO = QUARTO.str.split("CIDADE", expand = True).iloc[:,0]
QUARTO = QUARTO.str.replace("(\\\n|\\\r)"," ")
QUARTO = QUARTO.str.extract("(...QUARTO)")[0].str.replace("QUARTO","").str.strip()
dds2_teste["QUARTO"] = QUARTO
dds2_teste["QUARTO"] = pd.to_numeric(dds2_teste["QUARTO"])

# CRIANDO A COLUNA DE SUITE
SUITE = dds2_teste["GERAL"].str.split("AREA UTIL", expand = True).iloc[:,1]
SUITE = SUITE.str.split("CIDADE", expand = True).iloc[:,0]
SUITE = SUITE.str.replace("(\\\n|\\\r)"," ")
SUITE = SUITE.str.extract("(...SUITE)")[0].str.replace("SUITE","").str.strip()
dds2_teste["SUITE"] = SUITE
dds2_teste["SUITE"] = pd.to_numeric(dds2_teste["SUITE"])

# CRIANDO A COLUNA DE VAGA
VAGA = dds2_teste["GERAL"].str.split("AREA UTIL", expand = True).iloc[:,1]
VAGA = VAGA.str.split("CIDADE", expand = True).iloc[:,0]
VAGA = VAGA.str.replace("(\\\n|\\\r)"," ")
VAGA = VAGA.str.extract("(...VAGA)")[0].str.replace("VAGA","").str.strip()
dds2_teste["VAGA"] = VAGA
dds2_teste["VAGA"] = pd.to_numeric(dds2_teste["VAGA"])

# Criando coluna BANHEIRO
dds2_teste["BANHEIRO"] = 0

# Criando coluna IPTU
dds2_teste["IPTU"] = 0

# Criando coluna CONDOMINIO
CONDOMINIO = dds2_teste["GERAL"]
#CONDOMINIO = CONDOMINIO.str.split("AREA UTIL", expand = True).iloc[:,1]
#CONDOMINIO = CONDOMINIO.str.split("CIDADE", expand = True).iloc[:,0]
#CONDOMINIO = CONDOMINIO.str.replace("\\\r\\\n"," ")
#CONDOMINIO = CONDOMINIO.str.extract("(CONDOMINIO.*CIDADE)")[0].str.replace("CONDOMINIO","").str.strip()
#CONDOMINIO = CONDOMINIO.str.replace("(\\\n|\\\r)"," ").str.strip()
#dds2_teste["CONDOMINIO"] = CONDOMINIO
#dds2_teste["CONDOMINIO"] = pd.to_numeric(dds2_teste["CONDOMINIO"])

dds2_teste["CONDOMINIO"]=0

# Criando a coluna DESCRICAO
dds2_teste["DESCRICAO"] = dds2_teste["DESCRICAO_COMP"].str.replace("\\\n", "").str.strip()

# Criando a coluna DATA_SCRAPING
dds2_teste["DATA_SCRAPING"] = dds2_teste["SCRAPING_TIME"].str.split(" ",expand = True).iloc[:,0]

# Estruturando coluna SCRAPING_TIME
dds2_teste["SCRAPING_TIME"] = dds2_teste["SCRAPING_TIME"].str.split(" ",expand = True).iloc[:,1]

# Criando a coluna SITE
dds2_teste["SITE"] = "DFIMOVEIS"

# Criando a coluna TITULO
dds2_teste["TITULO"] = dds2_teste["ENDERECO_COMP"].str.replace("\\\n", "").str.strip()

# Renomeando o nome da coluna FOTO1 para FOTOS
dds2_teste = dds2_teste.rename(columns={"FOTO1": "FOTOS"})
dds2_teste["FOTOS"] = dds2_teste["FOTOS"].str.lower()

# Criando a coluna ANUNCIANTE
dds2_teste["ANUNCIANTE"] = dds2_teste["ANUNCIANTE"].str.replace("\\\n"," ")

# Retirando as duplicadas
dds2_teste = dds2_teste.drop_duplicates(subset=["LINK"])
dds2_teste = dds2_teste.drop_duplicates(subset=["VALOR","AREA","BAIRRO"])

# Retirando os NA's
dds2_teste = dds2_teste.dropna(subset=["VALOR"])
dds2_teste = dds2_teste.dropna(subset=["AREA"])
dds2_teste = dds2_teste.dropna(subset=["RUA_OU_QUADRA"])

# Reordenando as colunas do banco
dds3 = dds2_teste[["DATA_ID","TITULO","TIPO_ANUNCIO","TIPO_IMOVEL","RUA_OU_QUADRA","ESTADO","CIDADE","BAIRRO","AREA","QUARTO","BANHEIRO","SUITE","VAGA","CONDOMINIO","IPTU","VALOR","VALOR_LN","VALOR_M2","VALOR_M2_LN","ANUNCIANTE","DESCRICAO","DATA_SCRAPING","SCRAPING_TIME","FOTOS","LINK"]]

if user == "Pupe":
    dds3.to_excel('C:/Users/Admin/Ermida/Ícaro Costa - Quadra Urbana/03 - Clientes [Projetos]/QU/{}.xlsx'.format(base_exportar))

if user == "Rafael":
    dds3.to_excel('C:/Users/Ermida/OneDrive - Ermida/Quadra Urbana/03 - Clientes [Projetos]/QU/{}.xlsx'.format(base_exportar))

# %%
len(imoveis_list)
# %%
dds2_teste["GERAL"][653]
# %%


# Cgroup tune script
# Example:
# python tuneCgroups.py --cgName "usr_xpus-sched2.scope" --cgController "memory" --memory.limit_in_bytes 40466547916 --memory.memsw.limit_in_bytes 49056482508 --memory.soft_limit_in_bytes 36419893125

import os
import os.path
import shutil
import psutil
import json
import time
import argparse
import sys

from xcalar.external.LegacyApi.XcalarApi import XcalarApi
from xcalar.external.LegacyApi.WorkItem import WorkItem
from xcalar.compute.util.config import build_config
from xcalar.compute.coretypes.Status.ttypes import StatusT
from xcalar.external.LegacyApi.Env import XcalarConfigPath
from xcalar.external.client import Client
from xcalar.external.exceptions import XDPException
from xcalar.compute.services.Cgroup_xcrpc import Cgroup, CgRequest
from xcalar.external.LegacyApi.XcalarApi import XcalarApiStatusException

# Args parsing
parser = argparse.ArgumentParser()
parser.add_argument("--cgName", type=str, help="Cgroup name", dest='cgname')
parser.add_argument("--cgController", type=str, help="Cgroup controller name", dest='controller')
parser.add_argument("--memory.limit_in_bytes", type=int, help="Cgroup controller memory.limit_in_bytes", dest='limit_in_bytes')
parser.add_argument("--memory.memsw.limit_in_bytes", type=int, help="Cgroup controller memory.memsw.limit_in_bytes", dest='swlimit_in_bytes')
parser.add_argument("--memory.soft_limit_in_bytes", type=int, help="Cgroup controller memory.soft_limit_in_bytes", dest='soft_limit_in_bytes')
args = parser.parse_args()

def listCgroupParams():
    client = Client(bypass_proxy=True)
    cgserv = Cgroup(client)
    inputObj = {}
    inputObj["func"] = "listCgroupsConfigs"
    cgreq = CgRequest()
    cgreq.jsonInput = json.dumps(inputObj)
    cgres = cgserv.process(cgreq)
    return json.loads(cgres.jsonOutput)

def getCgroupParams(cgName, cgController):
    client = Client(bypass_proxy=True)
    cgserv = Cgroup(client)
    inputObj = {}
    inputObj["func"] = "getCgroup"
    inputObj["cgroupName"] = cgName
    inputObj["cgroupController"] = cgController
    cgreq = CgRequest()
    cgreq.jsonInput = json.dumps(inputObj)
    cgres = cgserv.process(cgreq)
    cgParamDict = json.loads(json.loads(
        cgres.jsonOutput)[0][0])[cgController][0]
    for k, v in cgParamDict.items():
        cgParamDict[k] = str(v).rstrip()
    print("\n\nGet cgName: {}, cgController: {}, Params {}".format(cgName, cgController, cgParamDict))
    return cgParamDict

def setCgroupParams(cgName, cgController, cgParams):
    client = Client(bypass_proxy=True)
    cgserv = Cgroup(client)
    inputObj = {}
    inputObj["func"] = "setCgroup"
    inputObj["cgroupName"] = cgName
    inputObj["cgroupController"] = cgController
    inputObj["cgroupParams"] = cgParams
    cgreq = CgRequest()
    cgreq.jsonInput = json.dumps(inputObj)
    cgres = cgserv.process(cgreq)
    print("\n\nSet cgName: {}, cgController: {}, Params {}".format(cgName, cgController, cgParams))

# List Cgroup Params
listOut = listCgroupParams()
print("***\n\t\tList Cgroup Params: {}".format(listOut))

# Get all XPU Cgroup params
print("***\n\t\tGet XPUs current config")
xpuMemSysCgSched0 = getCgroupParams("sys_xpus-sched0.scope", "memory")
xpuCpuSysCgSched0 = getCgroupParams("sys_xpus-sched0.scope", "cpu")
#xpuCpusetSysCgSched0 = getCgroupParams("sys_xpus-sched0.scope", "cpuset")
xpuCpuacctSysCgSched0 = getCgroupParams("sys_xpus-sched0.scope", "cpuacct")

xpuMemSysCgSched1 = getCgroupParams("sys_xpus-sched1.scope", "memory")
xpuCpuSysCgSched1 = getCgroupParams("sys_xpus-sched1.scope", "cpu")
#xpuCpusetSysCgSched1 = getCgroupParams("sys_xpus-sched1.scope", "cpuset")
xpuCpuacctSysCgSched1 = getCgroupParams("sys_xpus-sched1.scope", "cpuacct")

xpuMemSysCgSched2 = getCgroupParams("sys_xpus-sched2.scope", "memory")
xpuCpuSysCgSched2 = getCgroupParams("sys_xpus-sched2.scope", "cpu")
#xpuCpusetSysCgSched2 = getCgroupParams("sys_xpus-sched2.scope", "cpuset")
xpuCpuacctSysCgSched2 = getCgroupParams("sys_xpus-sched2.scope", "cpuacct")

xpuMemUsrCgSched0 = getCgroupParams("usr_xpus-sched0.scope", "memory")
xpuCpuUsrCgSched0 = getCgroupParams("usr_xpus-sched0.scope", "cpu")
#xpuCpusetUsrCgSched0 = getCgroupParams("usr_xpus-sched0.scope", "cpuset")
xpuCpuacctUsrCgSched0 = getCgroupParams("usr_xpus-sched0.scope", "cpuacct")

xpuMemUsrCgSched1 = getCgroupParams("usr_xpus-sched1.scope", "memory")
xpuCpuUsrCgSched1 = getCgroupParams("usr_xpus-sched1.scope", "cpu")
#xpuCpusetUsrCgSched1 = getCgroupParams("usr_xpus-sched1.scope", "cpuset")
xpuCpuacctUsrCgSched1 = getCgroupParams("usr_xpus-sched1.scope", "cpuacct")

xpuMemUsrCgSched2 = getCgroupParams("usr_xpus-sched2.scope", "memory")
xpuCpuUsrCgSched2 = getCgroupParams("usr_xpus-sched2.scope", "cpu")
#xpuCpusetUsrCgSched2 = getCgroupParams("usr_xpus-sched2.scope", "cpuset")
xpuCpuacctUsrCgSched2 = getCgroupParams("usr_xpus-sched2.scope", "cpuacct")

# Set given config
print("\n\t\t***Setting Config")
if ("xpus" not in args.cgname or args.controller != "memory" or args.limit_in_bytes == 0 or args.swlimit_in_bytes == 0 or args.soft_limit_in_bytes == 0):
    print("Invalid input params")
    sys.exit(-1)
curParams = getCgroupParams(args.cgname, args.controller)
newCgParams = {}
newCgParams["memory.limit_in_bytes"] = args.limit_in_bytes
newCgParams["memory.memsw.limit_in_bytes"] = args.swlimit_in_bytes
newCgParams["memory.soft_limit_in_bytes"] = args.soft_limit_in_bytes
setCgroupParams(args.cgname, args.controller, newCgParams)
print("\n\t\t***Getting Config")
getCgroupParams(args.cgname, args.controller)


import sys
sys.stdout = open("output.txt","w")

import re
import requests

url = "http://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
response = requests.get(url)
print(response.ok)

string = response.text
string = string.replace("\n", " ")

compiler = re.compile(r'
',)

result = re.findall(compiler, string)
print(len(result))
print(result[0])

Web Crawling/find_specific category of book.py


#!/usr/bin/env python
import os
import sys
import numpy as np
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation, Dropout, TimeDistributed, LSTM
from keras.optimizers import Adam
from sklearn.preprocessing import MinMaxScaler
from keras.callbacks import Callback, ModelCheckpoint
import codecs

def load_cfg(cfg):
    # TO
    # We should use argparser instead hard code

    with codecs.open(cfg, "r", encoding = "utf8") as f:
        data = f.readlines()
        params = {}
        for line in data:
            line = line.strip().split("=")
            params[line[0]] = int(line[1])

        return params["batch_size"], params["num_steps"], params["input_dim"]

class KerasBatchGenerator(object):

    def __init__(self, data, batch_size, num_steps, input_dim):
        self.data = data
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.input_dim = input_dim
        self.cur_idx = 0

    def generate(self):

        scaler = MinMaxScaler(feature_range=(0,1))

        xx = np.zeros((self.batch_size, self.num_steps, self.input_dim))
        yy = np.zeros((self.batch_size))

        while True:
            for i in range(self.batch_size):
                if self.cur_idx >= len(self.data):
                    self.cur_idx = 0
                xt = self.data[self.cur_idx: self.cur_idx+self.num_steps]
                xt = scaler.fit_transform(xt)
                xx[i,:,:] = xt
                self.cur_idx += self.num_steps
                yy[i] = self.data[self.cur_idx][0]
                self.cur_idx += 1
            yield xx, yy

def init_lstm(num_steps, input_dim, hidden_size = 50):

    model = Sequential()
    model.add(LSTM(hidden_size, input_shape = (num_steps, input_dim), return_sequences = True))
    model.add(LSTM(hidden_size, input_shape = (num_steps, input_dim), return_sequences = False))
    model.add(Dense(1))
    model.add(Activation("linear"))
    optimizer = Adam
    model.compile(loss="mean_absolute_error", optimizer = "rmsprop")

    print(model.summary())

    return model

def analyze_data(data, num_steps):

    length = len(data)

    return length//(num_steps+1)

def train(fp, batch_size, num_steps, input_dim):

    data = np.loadtxt(fp, "float")
    print(len(data))
    samples_tot = analyze_data(data, num_steps)
    print(samples_tot)
    spe = samples_tot//batch_size
    print(spe)
    train_data_generator = KerasBatchGenerator(data, batch_size, num_steps, input_dim)
    md = init_lstm(num_steps, input_dim, 50)
    #checkpoint = ModelCheckpoint(filepath = "./", monitor = "val_acc", verbose = 1, save_best_only = True, mode = "max")
    #callback_list = [checkpoint]

    md.fit_generator(train_data_generator.generate(), steps_per_epoch = spe, epochs = 30)
    #md.fit_generator(train_data_generator.generate(), steps_per_epoch = spe, callbacks = callback_list, epochs = 30)
    md.save_weights("final_model_weights.h5")
def test(fp, batch_size, num_steps, input_dim):
    md = init_lstm(num_steps, input_dim, 50)
    md.load_weights("final_model_weights")

    return

def main(feats, stage):

    batch_size, num_steps, input_dim = load_cfg("../conf/nn.conf")
    if stage == "0":
        print("Training starts...")
        fp = feats+"/"+"train"
        train(fp, batch_size, num_steps, input_dim)
    elif stage == "1":
        print("Testing starts...")
        fp = feats+"/"+"test"
        test(fp, batch_size, num_steps, input_dim)
    else:
        print("Invalid command.")
        exit(0)

if __name__ == "__main__":

    if len(sys.argv) != 3:
        print("Usage: {} input stage<0: training, 1: test>".format(sys.argv[0]))
        exit(0)
    main(sys.argv[1], sys.argv[2])


#!/usr/bin/env python
import pytest
import xarray
from pytest import approx
from pathlib import Path
import georinex as gr
import numpy as np

from datetime import datetime
#
R = Path(__file__).parent / 'data'

def test_blank(tmp_path):
    fn = R/'blank.10o'
    obs = gr.load(fn)
    assert obs is None

    outdir = tmp_path
    gr.load(fn, outdir)
    assert not (outdir / (fn.name + '.nc')).is_file()

    times = gr.gettime(fn)
    assert times is None

def test_minimal(tmp_path):
    fn = R/'minimal.10o'
    obs = gr.load(fn)
    assert isinstance(obs, xarray.Dataset), f'{type(obs)} should be xarray.Dataset'

    outdir = tmp_path
    gr.load(fn, outdir)

    outfn = (outdir / (fn.name + '.nc'))
    assert outfn.is_file()
    assert obs.equals(gr.load(outfn)), f'{outfn}  {fn}'

    times = gr.gettime(fn)
    assert np.isnan(times.interval)

    assert obs.fast_processing

def test_fast_slow():
    fn = R/'minimal.10o'
    fobs = gr.load(fn, fast=True)
    sobs = gr.load(fn, fast=False)

    assert fobs.equals(sobs)

    assert fobs.fast_processing
    assert not sobs.fast_processing

def test_meas_continuation():
    """
    tests OBS2 files with more than 10 types of observations
    """
    fn = R/'ab430140.18o.zip'
    obs = gr.load(fn, verbose=True)

    assert len(obs.data_vars) == 20
    for v in ['L1', 'L2', 'C1', 'P2', 'P1', 'S1', 'S2', 'C2', 'L5', 'C5', 'S5',
              'L6', 'C6', 'S6', 'L7', 'C7', 'S7', 'L8', 'C8', 'S8']:
        assert v in obs

    times = obs.time.values.astype('datetime64[us]').astype(datetime)
    assert times.size == 9

    assert obs.fast_processing

def test_meas_one():
    """
    test specifying specific measurements (usually only a few of the thirty or so are needed)
    """
    fn = R/'demo.10o'
    obs = gr.load(fn)
    for v in ['L1', 'L2', 'P1', 'P2', 'C1', 'S1', 'S2']:
        assert v in obs
    assert len(obs.data_vars) == 7

    sv = obs.sv.values
    assert len(sv) == 14
    for s in ['G13', 'R19', 'G32', 'G07', 'R23', 'G31', 'G20', 'R11',
              'G12', 'G26', 'G09', 'G21', 'G15', 'S24']:
        assert s in sv

    assert obs['C1'].sel(sv='G07').values == approx([22227666.76, 25342359.37])
# %% one measurement
    obs = gr.load(fn, meas='C1')
    assert 'L1' not in obs

    C1 = obs['C1']
    assert C1.shape == (2, 14)  # two times, 14 SVs overall for all systems in this file
    assert C1.sel(sv='G07').values == approx([22227666.76, 25342359.37])

    assert obs.fast_processing

def test_meas_two():
    fn = R/'demo.10o'

    C1 = gr.load(fn, meas='C1')['C1']
# %% two NON-SEQUENTIAL measurements
    obs = gr.load(fn, meas=['L1', 'S1'])
    assert 'L2' not in obs

    L1 = obs['L1']
    assert L1.shape == (2, 14)
    assert L1.sel(sv='G07').values == approx([118767195.32608, 133174968.81808])

    S1 = obs['S1']
    assert S1.shape == (2, 14)

    assert (S1.sel(sv='R23') == approx([39., 79.])).all()

    assert not C1.equals(L1)

    assert obs.fast_processing

def test_meas_miss():
    fn = R/'demo.10o'
# %% measurement not in some systems
    obs = gr.load(fn, meas=['S2'])
    assert 'L2' not in obs

    S2 = obs['S2']
    assert S2.shape == (2, 10)  # all NaN SVs are dropped
    assert (S2.sel(sv='G13') == approx([40., 80.])).all()

    with pytest.raises(KeyError):
        S2.sel(sv='R23')

    assert obs.fast_processing
# %% measurement not in any system
    obs = gr.load(fn, meas='nonsense')
    assert obs is None
# %% wildcard
    obs = gr.load(fn, meas='P')
    assert 'L1' not in obs
    assert 'P1' in obs and 'P2' in obs
    assert len(obs.data_vars) == 2

    assert obs.fast_processing

def test_mangled_data():
    fn = R/'14601736.18o'

    obs = gr.load(fn)

    times = obs.time.values.astype('datetime64[us]').astype(datetime)

    assert (times == (datetime(2018, 6, 22, 6, 17, 30), datetime(2018, 6, 22, 6, 17, 45), datetime(2018, 6, 22, 6, 18))).all()

    assert not obs.fast_processing

def test_mangled_times():
    fn = R/'badtime.10o'

    obs = gr.load(fn)

    times = obs.time.values.astype('datetime64[us]').astype(datetime)

    assert times

def test_Z_lzw():
    pytest.importorskip('unlzw')

    fn = R/'ac660270.18o.Z'

    obs = gr.load(fn)

    hdr = gr.rinexheader(fn)

    assert hdr['t0'] <= obs.time[0].values.astype('datetime64[us]').astype(datetime)

    assert not obs.fast_processing

def test_tlim():
    pytest.importorskip('unlzw')

    obs = gr.load(R/'ac660270.18o.Z', tlim=('2018-01-27T00:19', '2018-01-27T00:19:45'))

    times = obs.time.values.astype('datetime64[us]').astype(datetime).tolist()

    assert times == [datetime(2018, 1, 27, 0, 19),
                     datetime(2018, 1, 27, 0, 19, 15),
                     datetime(2018, 1, 27, 0, 19, 30),
                     datetime(2018, 1, 27, 0, 19, 45)]

    assert not obs.fast_processing

def test_one_sv():
    obs = gr.load(R / 'rinex2onesat.10o')

    assert len(obs.sv) == 1
    assert obs.sv.item() == 'G13'

    times = gr.gettime(R/'rinex2onesat.10o').values.astype('datetime64[us]').astype(datetime)

    assert (times == [datetime(2010, 3, 5, 0, 0), datetime(2010, 3, 5, 0, 0, 30)]).all()

    assert obs.fast_processing

def test_all_systems(tmp_path):
    """
    ./ReadRinex.py tests/demo.10o -o r2all.nc
    ./ReadRinex.py tests/demo.10n -o r2all.nc
    """
    pytest.importorskip('netCDF4')

    truth = xarray.open_dataset(R / 'r2all.nc', group='OBS', autoclose=True)
# %% test reading all satellites
    for u in (None, ' ', '', ['G', 'R', 'S']):
        print('use', u)
        obs = gr.load(R/'demo.10o', use=u)
        assert obs.equals(truth)
        assert obs.fast_processing

    assert obs.position == pytest.approx([4789028.4701, 176610.0133, 4195017.031])
    try:
        assert obs.position_geodetic == approx([41.38871005, 2.11199932, 166.25085213])
    except AttributeError:  # no pymap3d
        pass
# %% test read .nc
    obs = gr.rinexobs(R / 'r2all.nc')
    assert obs.equals(truth)
# %% test write .nc
    outdir = tmp_path
    outfn = outdir / 'testout.nc'
    gr.rinexobs(R/'demo.10o', outfn=outdir / 'testout.nc')
    assert outfn.is_file() and 50000 > outfn.stat().st_size > 30000

def test_one_system():
    """./ReadRinex.py tests/demo.10o -u G -o r2G.nc
    """
    pytest.importorskip('netCDF4')

    truth = xarray.open_dataset(R / 'r2G.nc', group='OBS', autoclose=True)

    for u in ('G', ['G']):
        obs = gr.load(R/'demo.10o', use=u)
        assert obs.equals(truth)
        assert obs.fast_processing

def test_multi_system():
    """./ReadRinex.py tests/demo.10o -u G R -o r2GR.nc
    """
    pytest.importorskip('netCDF4')

    truth = xarray.open_dataset(R / 'r2GR.nc', group='OBS', autoclose=True)

    obs = gr.load(R/'demo.10o', use=('G', 'R'))
    assert obs.equals(truth)
    assert obs.fast_processing

def test_all_indicators():
    """
    ./ReadRinex.py tests/demo.10o -useindicators  -o r2all_indicators.nc
    """
    pytest.importorskip('netCDF4')

    obs = gr.load(R/'demo.10o', useindicators=True)
    truth = gr.rinexobs(R/'r2all_indicators.nc', group='OBS')

    assert obs.equals(truth)
    assert obs.fast_processing

def test_meas_indicators():
    """
    ./ReadRinex.py tests/demo.10o -useindicators -m C1 -o r2_C1_indicators.nc
    """
    pytest.importorskip('netCDF4')

    obs = gr.load(R/'demo.10o', meas='C1', useindicators=True)
    truth = gr.rinexobs(R/'r2_C1_indicators.nc', group='OBS')

    assert obs.equals(truth)
    assert obs.fast_processing

def test_meas_onesys_indicators():
    obs = gr.load(R/'demo.10o', use='G', meas='C1', useindicators=True)

    C1 = obs['C1']

    assert C1.sel(sv='G07').values == approx([22227666.76, 25342359.37])
    assert obs.fast_processing

def test_time_system_determination():
    obs = gr.load(R/"demo.10o")
    assert obs.attrs['time_system'] == 'GPS'

    obs = gr.load(R/'default_time_system2.10o')
    assert obs.attrs['time_system'] == 'GLO'

if __name__ == '__main__':
    pytest.main(['-xrsv', __file__])


from pathlib import Path
import re
from axi_renamer.renamer_starter import Renamer, code_rename_map
from axi_renamer.constant_maps import pkg_map_for_folders
import os

class FolderHandler:
    def __init__(self, in_folder, out_folder, rename=False, valid_suffixes:list=None, process_files:list=None,
                       delete_sourcefile=False, target='CODE'):
        self.renamer = Renamer(target=target)
        self.root_folder = in_folder
        self.root_out_folder = out_folder
        self.valid_suffixes = valid_suffixes
        self.process_files = process_files
        self.delete_sourcefile = delete_sourcefile
        if rename:
            if delete_sourcefile:
                print('Rename & remove mode')
            else:
                print('Rename & copy mode')

        elif delete_sourcefile:
            print('Move mode')
        else:
            print('Copy mode')

    def simple_replace(self):
        self.replace_in_file_tree(self.root_folder)

    def make_renamed_dir(self, current_folder):
        relative_path = Path(current_folder).relative_to(self.root_folder)
        relative_path_str = str(relative_path)
        for row in code_rename_map:
            relative_path_str = re.sub(row[0], row[1], relative_path_str)
        relative_path_str = relative_path_str.replace('\\', '/')
        for row in pkg_map_for_folders:
            relative_path_str = re.sub(row[0], row[1], relative_path_str)
        os.path.normpath(relative_path_str)
        out_path = os.path.join(self.root_out_folder, relative_path_str)
        if not os.path.exists(out_path):
            os.makedirs(out_path)
        return out_path

    def need_to_process_file(self, file_fullname):
        if self.valid_suffixes and os.path.splitext(file_fullname)[1] not in self.valid_suffixes:
            return False

        if not self.process_files:
            return True
        b = False
        for process_file in self.process_files:
            try:
                file_fullname.index(process_file)
                b = True
                break
            except:
                continue
        return b

    def replace_in_file_tree(self, current_folder):
        files = os.listdir(current_folder)

        for file in files:
            full_name = os.path.join(current_folder, file)
            if os.path.isfile(full_name):

                if self.need_to_process_file(full_name):
                    print('Process File: ' + full_name)
                    infile = open(full_name, 'rt', encoding='UTF-8')
                    data = infile.read()
                    new_filename = str(file)

                    #Переименование файла
                    new_filename = self.renamer.get_axi_val(new_filename)

                    # Переименование других вхождений в тексте кода
                    data = self.renamer.get_axi_val(data, content_type='code')

                    # Создание переименованной директории
                    out_folder = self.make_renamed_dir(current_folder)

                    # Запись переименованнного файла
                    outfile = open(os.path.join(out_folder, new_filename), 'wt', encoding='UTF-8')
                    outfile.write(data)
                    infile.close()
                    outfile.close()

                    # Удаление исходного файла при включенной опции удаления (в т.ч. для удаления неиспользуемых файлов)
                    if self.delete_sourcefile:
                        os.remove(full_name)
            else:
                self.replace_in_file_tree(full_name)

    def free_find(self, reg_list):
        self.free_find_in_file_tree(reg_list, self.root_folder)

    def free_find_in_file_tree(self, reg_list, current_folder):
        files = os.listdir(current_folder)

        for file in files:
            full_name = os.path.join(current_folder, file)
            if os.path.isfile(full_name):

                if self.need_to_process_file(full_name):

                    infile = open(full_name, 'rt', encoding='UTF-8')
                    data = infile.read()
                    for reg in reg_list:
                        occurences = re.findall(reg, data, flags=re.IGNORECASE)
                        if bool(occurences):
                            occurences = list(set(occurences))
                            print('File: ' + full_name)
                            print(occurences)
            else:
                self.free_find_in_file_tree(reg_list, full_name)


"""This module contains the detection code for potentially insecure low-level
calls."""

from mythril.analysis import solver
from mythril.analysis.potential_issues import (
    PotentialIssue,
    get_potential_issues_annotation,
)
from mythril.laser.ethereum.state.constraints import Constraints
from mythril.laser.ethereum.transaction.symbolic import ACTORS
from mythril.analysis.module.base import DetectionModule, EntryPoint
from mythril.laser.smt import UGT, symbol_factory, Or, BitVec
from mythril.laser.ethereum.natives import PRECOMPILE_COUNT
from mythril.laser.ethereum.state.global_state import GlobalState
from mythril.exceptions import UnsatError
import logging

from mythril.laser.smt import simplify
from copy import deepcopy
from mythril.analysis.report import Issue
from mythril.analysis.issue_annotation import IssueAnnotation
from mythril.laser.ethereum.util import get_concrete_int
from mythril.laser.smt.bool import And
import re

TRANSACTION_ORDER_DEPENDENCY = 144

log = logging.getLogger(__name__)

DESCRIPTION = '''
MODULE DESCRIPTION:
This module finds the existance of transaction order dependence vulnerabilities.
The following webpage contains an extensive description of the vulnerability:
https://consensys.github.io/smart-contract-best-practices/known_attacks/#transaction-ordering-dependence-tod-front-running
'''

class TransactionOrderDependency(DetectionModule):
    """This module searches for low level calls (e.g. call.value()) that
    forward all gas to the callee."""

    name = "Transaction Order Dependency"
    swc_id = TRANSACTION_ORDER_DEPENDENCY
    description = DESCRIPTION
    entry_point = EntryPoint.CALLBACK
    pre_hooks = ["CALL"]

    def _execute(self, state: GlobalState) -> None:
        """

        :param state:
        :return:
        """

        potential_issues = self._analyze_state(state)

        annotation = get_potential_issues_annotation(state)
        annotation.potential_issues.extend(potential_issues)

    def _analyze_state(self, state: GlobalState):
        """

        :param state:
        :return:
        """
        try:

            interesting_storages = list(_get_influencing_storages(state))
            changing_sstores = list(_get_influencing_sstores(state, interesting_storages))

            if len(changing_sstores) > 0:

                to = state.mstate.stack[-2]

                constraints = (
                    state.world_state.constraints
                    + [to == ACTORS.attacker]
                )

                transaction_sequence = solver.get_transaction_sequence(
                    state, constraints
                )

                instruction = state.get_current_instruction()
                issue = Issue(
                    contract=state.environment.active_account.contract_name,
                    function_name=state.environment.active_function_name,
                    address=instruction["address"],
                    swc_id=TRANSACTION_ORDER_DEPENDENCY,
                    bytecode=state.environment.code.bytecode,
                    title="Unprotected Transaction Order",
                    severity="High",
                    description_head="description_head",
                    description_tail="description_tail",
                    transaction_sequence=transaction_sequence,
                    gas_used=(state.mstate.min_gas_used, state.mstate.max_gas_used),
                )

                state.annotate(
                    IssueAnnotation(
                        conditions=[And(*constraints)], issue=issue, detector=self
                    )
                )

                return [issue]
        except:
            log.debug("No model found")
        return []

def _get_influencing_storages(state: GlobalState):
    gas = state.mstate.stack[-1]
    to = state.mstate.stack[-2]
    value = state.mstate.stack[-3]
    storages = []

    if to.symbolic is False:
        storages += _dependent_on_storage(to.value)
    if value.symbolic is False:
        storages += _dependent_on_storage(value.value)

    for storage in storages:
        variable = _get_storage_variable(storage, state)
        can_change = _can_change(state.world_state.constraints, variable, gas)
        if can_change:
            yield storage

def _dependent_on_storage(expression):

    pattern = re.compile(r"storage_[a-z0-9_&^]+")
    return pattern.findall(str(simplify(expression)))

def _get_storage_variable(storage, state):

    index = int(re.search('[0-9]+', storage).group())

    try:
        return state.environment.active_account.storage(index)
    except:
        return None

def _can_change(constraints, variable, gas):
    _constraints = deepcopy(constraints)

    try:
        model = solver.get_model(_constraints + [UGT(gas, symbol_factory.BitVecVal(2300, 256))])

        try:
            initial_value = int(str(model.eval(variable, model_completion=True)))
            _constraints.append(variable != initial_value)
            solver.get_model(_constraints)
            return True

        except UnsatError:
            return False

    except UnsatError:
        return False

def _get_influencing_sstores(state, interesting_storages):
    for sstore_state, node in _get_states_with_opcode(state, "SSTORE"):
        index, value = sstore_state.mstate.stack[-1], sstore_state.mstate.stack[-2]
        try:
            index = get_concrete_int(index)
        except AttributeError:
            index = str(index)
        if "storage_{}".format(index) not in interesting_storages:
            continue
        yield sstore_state, node

# sstore가 들어가 잇는 state가져오기
# 예전과 방식이 바뀐 느낌.. 가져오느게 다름.
def _get_states_with_opcode(state, opcode):
    for k in state.node:
        node = state.nodes[k]
        for _state in node.states:
            if state.get_current_instruction()["opcode"] == opcode:
                yield state, node

detector = TransactionOrderDependency()


# coding: utf-8

# # Deep eXtreme MultiLabel (DXML) Classification
#
# A classification task is defined as 'extreme' when the number of classes considered grows considerably (from a dozen to millions of classes). The 'Multi-label' problem is similar to a classification task, but each sample may have more than one associated label. Furthermore, any sample may have a different number of associated labels.
#
# This notebook aims to prove that naive approaches may not be neither performant nor mathematically correct for this problem, and to show a simplified version of the XML method developed in paper [**Deep Extreme Multi-label Learning**](https://arxiv.org/pdf/1704.03718.pdf) by Zhang et al.

# In[1]:

# Imports
get_ipython().magic('matplotlib inline')
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestRegressor

# # Theory: label independance
#
# A multi-label problem is different from a multi-class one as the latter only attributes the most probable class for a sample, whereas the multi-label problem needs to attribute the most probable subset of classes.
#
# For instance, in visual recognition, a multi-class problem would be to recognise the main subject of a picture whereas a multi-label problem would be the recognition of all recognizable objects in the picture.
#
# At first one could think that the subset of the most probable classes would form the most probable subset of classes, but it is not true on the general case. It can only be true if each class probability is independent from the others (or $P(A\,|\, B) = P(A) $ for each A,B classes in our problem). **This hypothesis can't possibly be satisfied in most of our problems**.
#
# That's what makes mult-label so tricky. Indeed the number of possible subsets is much larger than the number of classes (growing as $2^N$, with N classes. That means multilabelling with 100 labels makes a little more than a 1 million classes classification problem. Scaling to 100 labels would have us have to choose between $ 1,26 . 10^{30}$ possible subsets.
#
# Thus there's a need for a reasonable approach.

# # Birds chant recognition
#
# We'll follow through this reasoning using data from [The 9th annual MLSP competition: New methods for acoustic classification of multiple simultaneous bird species in a noisy environment]( http://vintage.winklerbros.net/Publications/mlsp2013ma.pdf ).
#
# This data represents audio recordings of birds chants, with the objective of recognising up to 19 different birds. It has been split for the sake of this exercise.

# In[19]:

# Get data
X_train, X_test = np.loadtxt('samples/X_train_birds.txt'), np.loadtxt('samples/X_test_birds.txt')
y_train, y_test = np.loadtxt('samples/y_train_birds.txt'), np.loadtxt('samples/y_test_birds.txt')
birds_list = ["Brown Creeper","Pacific Wren","Pacific-slope Flycatcher","Red-breasted Nuthatch",
              "Dark-eyed Junco","Olive-sided Flycatcher","Hermit Thrush","Chestnut-backed Chickadee",
              "Varied Thrush","Hermit Warbler","Swainson's Thrush","Hammond's Flycatcher",
              "Western Tanager","Black-headed Grosbeak","Golden Crowned Kinglet","Warbling Vireo",
              "MacGillivray's Warbler","Stellar's Jay","Common Nighthawk"]
print(X_train.shape)

#
Let's look at the data to picture the multi-label problem. Observe and comment on the data: check that this is a multi-label problem. What problems could arise if we applied a global "blind" method?

# In[3]:

print(len(X_train), ' train samples and ', len(X_test), ' test sample')
print('###')
for i in range(len(birds_list)):
    print(' Species {} appears {} times'.format(birds_list[i], np.sum(y_train[:,i]) + np.sum(y_test[:,i])))
print('###')
all_y = np.concatenate([y_train, y_test])
all_sum_y = np.sum(all_y, axis=1)
print('As a mean {} species appear on each recording (standard deviation {})'.format(np.mean(all_sum_y), np.std(all_sum_y)))

# Remember that the repartion of occurence between species is quite unequal, which would tend to bias black-block approaches

#
(Optional) Implement a Multi-Layer Perceptron (MLP) classifier to check the difficulty of the task.

#
# ## Mathematically false approach: parallel classifiers
#
#
The intuitive approach that supposes independance of different labels would be to implement parallel classifiers. Let's look at their performance.

# In[4]:

# CORRECTION 1 : SKLEARN
mlpc = MLPClassifier(hidden_layer_sizes=(64,32,32), activation='relu')
mlpc.fit(X_train, y_train)
mlpc.score(X_test, y_test), mlpc.score(X_train, y_train)

# In[5]:

# CORRECTION 2 : KERAS

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import Adam

mlpk = Sequential()

#model.add(BatchNormalization())
#model.add(Activation('relu'))

mlpk.add(Dense(32, activation='relu', input_shape=(X_train.shape[1],)))
mlpk.add(Dense(32, activation='relu'))
mlpk.add(Dense(len(birds_list), activation='tanh'))

mlpk.compile(loss='categorical_crossentropy', metrics=['categorical_accuracy'], optimizer=Adam())

h = mlpk.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=20, verbose=0)

# In[6]:

# CORRECTION
plt.figure()
plt.plot(h.history['categorical_accuracy'], c='blue', label='train')
plt.plot(h.history['val_categorical_accuracy'], c='red', label='test')
plt.legend(loc='best')
plt.title('Accuracy of the MLP')
plt.show()

# # Definitely, MLPs are not very convincing!

# ## Baseline: parallel classifiers
#
#
The intuitive approach (which you now know is inexact!): Individually predict each label (for example, use Naive Bayes classifiers).

# In[8]:

# CORRECTION
classif_list = []
train_score_list, test_score_list = [], []

# Training
for i in range(len(birds_list)):
    y_train_for_this_bird = y_train[:,i]
    new_classifier = GaussianNB()
    new_classifier.fit(X_train, y_train_for_this_bird)
    classif_list.append(new_classifier)

# Test & display results
for i in range(len(birds_list)):
    classif = classif_list[i]
    train_score = float('{0:.3f}'.format(classif.score(X_train, y_train[:,i])))
    test_score = float('{0:.3f}'.format(classif.score(X_test, y_test[:,i])))
    train_score_list.append(train_score)
    test_score_list.append(test_score)
    print('Detecting {} with {}% accuracy (training {}%)'.format(birds_list[i], 100*test_score, 100*train_score))

predict_train = np.zeros_like(y_train)
predict_test = np.zeros_like(y_test)
for i in range(len(birds_list)):
    classif = classif_list[i]
    predict_train[:,i] = classif.predict(X_train)
    predict_test[:,i] = classif.predict(X_test)
acc_train = 1 - np.sum(np.abs(predict_train - y_train))/(y_train.shape[0]*y_train.shape[1])
acc_test = 1 - np.sum(np.abs(predict_test - y_test))/(y_test.shape[0]*y_test.shape[1])
print('###')
print('Global accuracy: testing {}, training {}'.format(acc_test, acc_train))

well_labeled = 0
for i in range(len(y_train)):
    if np.sum(np.abs(y_train[i,:] - predict_train[i,:])) == 0:
        well_labeled +=1
print('Overall {} out of the {} training samples were well labeled'.format(well_labeled,len(y_train)))

well_labeled = 0
for i in range(len(y_test)):
    if np.sum(np.abs(y_test[i,:] - predict_test[i,:])) == 0:
        well_labeled +=1
print('Overall {} out of the {} testing samples were well labeled'.format(well_labeled,len(y_test)))

# Of course, in addition to being mathematically frowned upon, this method is cumbersome,
# imagine having to build a million classifiers! It is not at all extensible to extreme classification

#
Remember that those results derive from an inadequate method. The Multi - Label classification and its XL version, the eXtreme MultiLabel Classification (XML) are difficult to treat problems on which the classical methods can not be used as they are.

# ## Embedding 101
#
# Embedding is a projection of data in another space. As we can specifically design this destination space, we'll be able to highlight specific relation between data points that may have been hidden before.
#
# For instance:
#
#

#

#
#         Dog
#         Cat
#         Elephant
#         Turtle
#         Fox
#

#

#         Dog
#         1
#         0
#         0
#         0
#         0
#

#

#         Cat
#         0
#         1
#         0
#         0
#         0
#

#

#         Fox
#         0
#         0
#         0
#         0
#         1
#

#

#
# could be projected as:
#
#

#

#
#         Dimension 1
#         Dimension 2
#         Dimension 3
#         Dimension 4
#

#

#         Dog
#         0.123
#         0.452
#         0.865
#         0.852
#

#

#         Cat
#         0.458
#         0.741
#         0.852
#         0.963
#

#

#         Fox
#         0.789
#         0.456
#         0.845
#         0.654
#

#

#
# Embedding is used a lot in Natural Language Processing to create mathematically just syntaxical or lexical relations, but in this problem, it will enable us to relate one labels to another.

# # Zhang et. al. method : Interaction between labels Embedding
#
# Their method can be decomposed in 4 very simple steps, as we'll dispense from the 'deep' part thanks to our relatively small problem.
#
# ### First step: compute labels adjacency
#
# Let's look at our labels to see how close they are to each other. We'll draw the adjacency matrix and adjacency graph. In the latter, each node is a label and a vertex symbolizes the number of conjoint appearances of two labels.
#
# ### Second step: learning adjacency-based label embedding
#
# The cited paper uses DeepWalk to form an embedding based on the adjacency graph.
#
# In our example here we'll do with a linear projection using the adjacency matrix, which despite being much more simple will prove its value.
#
# ### Third step : prediction in embedded space
#
# Instead of doing a classification problem from the features to the labels, we'll train a multidimensional regression agent from the features to the embedded space, which will yield coordinates for a data sample in this continuous space.
#
# ### Fourth step : return to labels
#
# From our embedded space coordinates, we'll run a k-Nearest Neighbors (kNN) to find the closest labels and return them as a prediction.
#
#
# Here's the original, full-scale full-deep method as proposed in the original article:
#

# ## On our example
#
# ### Compute adjacency matrix
#
#
#
#
 Let's build the labels proximity matrix as $M_{[i,j] \in [|1,19|]^2}$ so that $m_{i,j}$ is the joint apparition frequency of labels $i$ and $j$.

# In[20]:

# Trick
#Visualizing a 19 * 19 matrix is not necessarily obvious.
#You will find below a simple method to represent a matrix as an image.

my_matrix = np.random.normal(loc=1, scale=0.3, size=(25,25)) # matrix to represent
plt.figure()
plt.imshow(my_matrix,cmap="hot")
plt.colorbar()
plt.title('Visualization of the matrix')
plt.show()

# In[10]:

#We'll rather visualize an image than a 19*19 matrix.

matrix = np.zeros((y_train.shape[1],y_train.shape[1]))
edges =[]
for row in y_train :
    for i in range(len(row)):
        for j in range(len(row)):

            if row[i]*row[j] == 1: # both labels are activated
                matrix[i,j] += 1
                if ([i,j] not in edges) and ([j,i] not in edges) and not i==j:
                    edges.append([i,j])

#to get frequency we'll divide by number of appearances
norm_matrix = np.copy(matrix)
# np.save(norm_matrix,'norm_matrix.npm')
for i in range(matrix.shape[0]):
    d = norm_matrix[i,i]
    norm_matrix[:,i] = norm_matrix[:,i]/d
    norm_matrix[i,:] = norm_matrix[i,:]/d
    norm_matrix[i,i] = 1

# Visualize
plt.figure()
plt.imshow(norm_matrix,cmap="hot")
plt.colorbar()
plt.title('Labels proximity (absolute)')
plt.show()

# Diagonal coefficients are not interesting, let's ditch them
norm_matrix -= np.identity(norm_matrix.shape[0])
plt.figure()
plt.imshow(norm_matrix,cmap="hot")
plt.colorbar()
plt.title('Label proximity (without diagonal)')
plt.show()

# ### Adjacency graph
#
# This graph is actually used in Zhang et al's method, but for us it will only be a data visualization tool.
#
# Here's the **draw_graph** function that could be used in other projects too.
#
# Another way of representing the proximity between labels is to construct a weighted graph. This is necessary for Zhang et al, who use this graph for their embedding, but ours will only require the adjacency matrix. We recommend that you build this graph at a minimum in order to visualize the data.
#
# We put at your disposal the function draw_graph which will automatically build it from a list of connected labels and the proximity matrix you just calculated. The threshold option allows you to choose from how many occurrences a link will be represented on the graph.

# In[11]:

import networkx as nx #For those among you that use Anaconda dist, you already got this

def draw_graph(edges, weights_matrix=None, threshold=0, figsize=(8,8)):

    '''
    edges : lists of connected labels
    weights_matrix : labels proximity matrix
    threshold : number of occurence required to draw a connection
    figsize : size of the plot to display
    '''

    edges = [edge for edge in edges if weights_matrix[edge[0],edge[1]] > threshold]

    # additional settings (you can mess around here)
    node_size = 1600
    node_color = 'blue'
    node_alpha = 0.2
    node_text_size = 12
    edge_color = 'blue'
    edge_alpha= 0.3
    edge_tickness = 1
    edge_text_pos = 0.3
    text_font = 'sans-serif'

    plt.figure(figsize=figsize)
    # create networkx graph
    G=nx.Graph()

    # add edges
    for edge in edges:
        G.add_edge(edge[0], edge[1])

    # select shell autolocation
    graph_pos=nx.shell_layout(G)

    # draw graph
    nx.draw_networkx_nodes(G,
                           graph_pos,
                           node_size=node_size,
                           alpha=node_alpha,
                           node_color=node_color)
    nx.draw_networkx_edges(G, graph_pos,width=edge_tickness, alpha=edge_alpha,edge_color=edge_color)
    nx.draw_networkx_labels(G, graph_pos,font_size=node_text_size, font_family=text_font)

    # construct weights dict
    weights={}
    for i in range(len(edges)):
        weights[tuple(edges[i])] = weights_matrix.astype(int)[edges[i][0],edges[i][1]]

    # draw weights
    edge_labels = weights
    nx.draw_networkx_edge_labels(G, graph_pos, edge_labels=edge_labels,
                                 label_pos=edge_text_pos)

    # show graph
    plt.title('Proximity graph between labels with a proximity threshold at {} occurrences'.format(threshold))
    plt.show()

# Exemple
graph = [(0, 1), (1, 4), (2, 3), (0, 4), (1, 3)]
weights_matrix = np.random.randint(1,high=5,size=(5,5))
draw_graph(graph,weights_matrix)
draw_graph(graph,weights_matrix, threshold=2)

#
 Let's look at some graphs

# In[12]:

# CORRECTION

draw_graph(edges, matrix, figsize=(13,13), threshold=0)
draw_graph(edges, matrix, figsize=(8,8), threshold=5)
draw_graph(edges, matrix, figsize=(8,8), threshold=9)

# ## Design the embedding
#
# Zhang et al use [**Deepwalk**](https://github.com/phanein/deepwalk) on the adjacency graph to build their embeding.
#
# Once the graph is constructed, Zhang et al use Deepwalk to reduce the size of their labels, that is, project them into a continuous space of smaller size.
#
# At our scale, we can content ourselves with making a simple multiplication of our labels vectors by the standardized proximity matrix that we have constructed previously, which will project our labels in a continuous space of dimension 19. It is in this space Continuing that we are going to do a regression learning.
#
# In our problem, multiplying the label vectors by the normalized adjacency matrix will project our labels into a continuous space of 19 dimensions. Our regression will be conducted in this space.
#
# For the moment, we are going to set up the Embedding in this continuous space. Construct a ** embedding_encode ** function that multiplies one or more samples by the matrix to the continuous space (adjacency matrix) and then centers and collapses; apply it to y_train and y_test.
#
# For example, we want to create a function such that: f ([0,0,0,0,0,0,1,0,0,1]) = [0.25, 0.32, 0.87]

# In[13]:

def embedding_encode(y, matrix, scalor=None, return_scalor=False):
    y_conti = np.dot(y, norm_matrix)
    if scalor:
        y_conti=scalor.transform(y_conti)
    else:
        scalor = StandardScaler()
        y_conti = scalor.fit_transform(y_conti)

    if return_scalor:
        return y_conti, scalor
    else:
        return y_conti

y_conti_train, scalor = embedding_encode(y_train, norm_matrix, return_scalor=True)
y_conti_test = embedding_encode(y_test, norm_matrix, scalor=scalor)

# The decoding part will be performed by a kNN research in the continuous space.
#
#
 The * KNearestNeighborsClassifier * class you can import from * sklearn.neighbors * has a * k neighbors * method that returns the nearest n $ $ indices. Use it to create a ** encoding_decode ** method that lets you move from a sample of continuous space to the label space.

#
# (If you prefer to implement directly or otherwise the KNN algorithm, it's to your taste!)
#
# For example, we want to create a function such that: f ([0.25, 0.32, 0.87]) = [0,0,0,0,0,0,1,1,0,0,1]

# In[14]:

knn = KNeighborsClassifier()
knn.fit(y_conti_train, y_train)

def embedding_decode(y, knn, neighbors_number=2, algorithm='brute', y_train=y_train):
    indexes_of_neighbors = knn.kneighbors(y,n_neighbors=neighbors_number, return_distance=False)
    number_of_samples = indexes_of_neighbors.shape[0]
    labels = np.zeros((number_of_samples, y_train.shape[1]))

    for i in range(number_of_samples):
        for j in range(neighbors_number):
            labels[i, :] += y_train[indexes_of_neighbors[i,j],:]
        for j in range(labels.shape[1]):
            if labels[i,j]: labels[i,j] = 1

    return labels

# small optimization on neighbors_number
err_list=[]
err_list_t=[]
for n in range(2,30):
    y_train_decoded = embedding_decode(y_conti_train, knn, n )
    y_test_decoded = embedding_decode(y_conti_test, knn , n)
    error = np.sum(np.abs(y_train_decoded - y_train))
    error_t = np.sum(np.abs(y_test_decoded - y_test))
    err_list.append(error/(y_train.shape[0]*y_train.shape[1]))
    err_list_t.append(error_t/(y_test.shape[0]*y_test.shape[1]))
plt.figure()
plt.plot(err_list, label='training data')
plt.plot(err_list_t, label='testing data', color='red')
plt.legend(loc='best')
plt.title('Error from number of selected neighbours')
plt.show()

# ## Multidimensional Regression
#
# We now have a way to move from the space of the labels to their continuous representation and vice versa, it remains for us to implement the regression in the continuous space to complete the method!
#
#
 Create a regressor (for example, use RandomForestRegressor) predicting continuous values (* y_conti *) from features (* X_train, X_test *), and then use the ** embedding_decode ** function previously coded to return to the label space. Compare the accuracy obtained with that of your parallel classifiers.

#
# We'll implement a simple model to predict coordinates in our embedded space from features, such as a random forest.

# In[15]:

get_ipython().run_cell_magic('time', '', '# mini-optimization\n# This estimator is the one who realizes the true "prediction", a good precision is more than desirable\n# We thus realize a mini-optimization on the number of arbs of our Random Forest\nlist_train, list_test = [],[]\nfor n in range(100,1101,200):\n    regressor = RandomForestRegressor(n_estimators=n, n_jobs=-1)\n    regressor.fit(X_train, y_conti_train)\n    list_train.append(regressor.score(X_train,y_conti_train))\n    list_test.append(regressor.score(X_test,y_conti_test))\nplt.figure()\nplt.plot(list_train, label=\'score training\')\nplt.plot(list_test, label=\'score testing\', color=\'red\')\nplt.legend(loc=\'best\')\nplt.show()')

# In[16]:

# actual regression
regressor = RandomForestRegressor(n_estimators = 400, n_jobs=-1)
regressor.fit(X_train, y_conti_train)
y_conti_predict = regressor.predict(X_test)

# In[17]:

# Conversion in labels
y_predict = embedding_decode(y_conti_predict, knn)

# In[18]:

# Evaluate the full method precision
well_labeled = 0
for i in range(len(y_test)):
    if np.array_equal(y_test[i,:] , y_predict[i,:]):
        well_labeled +=1
print('Overall {} out of the {} testing samples were well labeled'.format(well_labeled,len(y_test)))

#
Even though our embedding is dramatically simple and the precision of our regressor is more than questionable, taking the labels proximity into account took our global precision from 24 to 146 well labeled samples out of 323. That's a way to show mathemaical correctness is also a practical help!

#
#
 Although our embedding is reduced to its simplest expression and our regressor is a questionable prediction, taking into account the adjacency of the labels has made it possible to pass from 24 to 141 well-ranked samples on 323 compared to our baseline of 19 parallel naive-bayes classifiers. The added value of the method is undeniable both mathematically and practically!

# The main limitation of the method is related to the function passing from the embedding space to the space of the labels. Indeed, the k-Nearest-Neighbors function takes as attribute the number of neighbors to be selected. This number is therefore fixed for all the samples. This implies that the same number of labels will be predicted for all samples.
#
# # For further details:
#
# Here are more XML datasets to have fun with:
#     - http://mulan.sourceforge.net/datasets-mlc.html
#     - http://manikvarma.org/downloads/XC/XMLRepository.html
#
# You can try on these to build the adjacency graph and use [**Deepwalk**](https://github.com/phanein/deepwalk) on it !
#
# Deepwalk is an algorithm that uses random walks to learn a representation of the nodes of a graph. Deepwalk uses the information obtained after random walks by considering the sequence of nodes encountered. This method has an advantage, it is adaptable. It is not necessary to restart the process from the beginning to add new nodes and relationships.


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
#from src.utils import get_max_lengths, get_evaluation
from han_in_dataset_bert import HanInDataset
from han_hierarchical_attn_bert import HierAttn

from tensorboardX import SummaryWriter
from nltk import word_tokenize, sent_tokenize
from sklearn import metrics
from sklearn.metrics import classification_report, roc_auc_score
import argparse
from tqdm.notebook import tqdm
import csv
import shutil
import numpy as np
import pandas as pd

# In[3]:

from text_with_topic_words_w_3_context_w_demog import handleTask, handleMergingCaseControlDataset

# In[4]:

def get_args():
    # for amazon_review_polarity dataset
    parser= argparse.ArgumentParser()
    parser.add_argument("--batch_size", type= int, default=16)
    parser.add_argument("--num_epochs", type= int, default= 100)
    parser.add_argument("--lr", type=float, default=0.001)
    parser.add_argument("--momentum", type=float, default=0.9)
    parser.add_argument("--word_hidden_size", type=int, default= 128)
    parser.add_argument("--sent_hidden_size", type=int, default= 128)
    parser.add_argument("--es_min_delta", type=float, default=0.0,
                        help="Early stopping's parameter: minimum change loss to qualify as an improvement")
    parser.add_argument("--es_patience", type=int, default=35,
                        help="Early stopping's parameter: number of epochs with no improvement after which training will be stopped. Set to 0 to disable this technique.")
    parser.add_argument("--train_set", type=str, default="./../../dataset/data/amazon_review_polarity_csv/train.csv")
    parser.add_argument("--test_set", type=str, default="./../../dataset/data/amazon_review_polarity_csv/test.csv")
    parser.add_argument("--test_interval", type=int, default=1, help="Number of epoches between testing phases")
    parser.add_argument("--word2vec_path", type=str, default="./../../dataset/data/glove.6B.50d.txt")
    parser.add_argument("--log_path", type=str, default="tensorboard/han_voc")
    parser.add_argument("--saved_path", type=str, default="./../../checkpoints/saved_models")

    args, unknown = parser.parse_known_args()

    return args

# In[5]:

def get_max_lengths(data_path):
    word_length_list = []
    sent_length_list = []
    with open(data_path) as csv_file:
        reader = csv.reader(csv_file, quotechar='"')
        for idx, line in tqdm(enumerate(reader)):
            text = ""
            for tx in line[1:]:
                text += tx.lower()
                text += " "
            sent_list = sent_tokenize(text)
            sent_length_list.append(len(sent_list))

            for sent in sent_list:
                word_list = word_tokenize(sent)
                word_length_list.append(len(word_list))

        sorted_word_length = sorted(word_length_list)
        sorted_sent_length = sorted(sent_length_list)

    return sorted_word_length[int(0.8*len(sorted_word_length))], sorted_sent_length[int(0.8*len(sorted_sent_length))]

# In[6]:

def get_evaluation(y_true, y_prob, list_metrics):
    y_pred = np.argmax(y_prob, -1)
    output = {}
    if 'accuracy' in list_metrics:
        output['accuracy'] = metrics.accuracy_score(y_true, y_pred)
    if 'loss' in list_metrics:
        try:
            output['loss'] = metrics.log_loss(y_true, y_prob)
        except ValueError:
            output['loss'] = -1
    if 'confusion_matrix' in list_metrics:
        output['confusion_matrix'] = str(metrics.confusion_matrix(y_true, y_pred))

    try:
        print(classification_report(y_true, y_pred))
        print("AUC= ",roc_auc_score(y_true, y_pred, average="macro", multi_class="ovo", labels=[0,1,2]))
    except:
        print('An exception occurred while generating classification report')

    return output

# In[16]:

import nltk
def handleTopicSent(dataset: pd.DataFrame, topic_set: pd.DataFrame):
    all_topic_words= topic_set[["0","1","2"]].values.tolist()
    all_topic_words=[word for subset in all_topic_words for word in subset ]
    ori_len=[]
    short_len=[]
    for i,doc in tqdm(enumerate(dataset["text"])):
        temp_doc=[]
        #print("--------->",doc)
        doc=nltk.sent_tokenize(doc)
        ori_len.append(len(doc))
        for each_sent in doc:
            found=[word for word in nltk.word_tokenize(each_sent) if word in set(all_topic_words)]
            if len(found) !=0:
                #print(found)
                #print(each_sent)
                temp_doc.append(each_sent)
        short_len.append(len(temp_doc))
        #print("==========>", " ".join(temp_doc))
        dataset.loc[i, "text"]=(" ".join(temp_doc))
    ori_avg_len=sum(ori_len)/len(ori_len)
    short_avg_len=sum(short_len)/len(short_len)
    print(ori_avg_len)
    print(short_avg_len)
    return dataset

# In[17]:

def handleDataSupport(all_dataset:pd.DataFrame,topic_set:pd.DataFrame,with_context:bool,with_demog:bool, all:bool, window_len:int, cluster_num:int):
    #with context
    full_dataset=handleTopicSent(all_dataset, topic_set) # imported from text_with_topic_words.ipynb file
    full_dataset["text"].replace(to_replace=[r"-"," +"], value=[" "," "],inplace=True, regex=True)
    #pure_dataset=full_dataset.dropna().reset_index(drop=True) # dropping null value
    #full_dataset=all_dataset
    # full_dataset.rename(columns={"TEXT_REPROT":"text"}, inplace=True)
    # transformed_label_list=[1 if i== cluster_num else 0 for i in full_dataset["case"].tolist()]
    # full_dataset["label"]=transformed_label_list
    #full_dataset.head(5)

    print(full_dataset.shape)
    full_dataset.to_csv("./temp_full_dataset.csv")
    full_dataset=pd.read_csv("./temp_full_dataset.csv")
    full_dataset=full_dataset.iloc[:,1:]
    full_dataset=full_dataset.dropna().reset_index(drop=True)
    print(full_dataset.shape)

    df0=[]
    df1=[]
    df2=[]
    for i in tqdm(range(len(full_dataset))):
        #print(int(full_dataset.at[i,"GAF"]))
        if int(full_dataset.at[i,"label"])==0:
            #print("hu")
            df0.append(full_dataset.iloc[i,:])
        elif int(full_dataset.at[i,"label"])==1:
            df1.append(full_dataset.iloc[i,:])
            #print("ku")
        else:
            df2.append(full_dataset.iloc[i,:])
    df0=pd.concat(df0, axis=1, ignore_index=True).T
    df1=pd.concat(df1, axis=1, ignore_index=True).T
    df2=pd.concat(df2, axis=1, ignore_index=True).T
    print(df0.shape, df1.shape, df2.shape)

    eq_val=2000

    df_train=pd.concat([df0.iloc[:eq_val,:],df1.iloc[:eq_val,:],df2.iloc[:eq_val,:]], ignore_index=True)
    df_valid=pd.concat([df0.iloc[eq_val:,:],df1.iloc[eq_val:,:],df2.iloc[eq_val:,:]], ignore_index=True)

    df_train = df_train.sample(frac=1, random_state=42)
    df_valid = df_valid.sample(frac=1, random_state=42)

    df_train.to_csv("./temp_42_tr3-.csv")
    df_train=pd.read_csv("./temp_42_tr3-.csv")
    df_train=df_train.iloc[:,1:]
    df_train=df_train.dropna().reset_index(drop=True)
    print(df_train.shape)

    df_valid.to_csv("./temp_42_val3-.csv")
    df_valid=pd.read_csv("./temp_42_val3-.csv")
    df_valid=df_valid.iloc[:,1:]
    df_valid=df_valid.dropna().reset_index(drop=True)
    print(df_valid.shape)

    #df_test=df_test.reset_index(drop=True)

    #print(df_test.head(5))
    return df_train, df_valid

# In[9]:

def train(opt, train_set, valid_set, embd_size, model_name):
    if torch.cuda.is_available():
        torch.cuda.manual_seed(123)
    else:
        torch.manual_seed(123)

    output_file= open(opt.saved_path+ os.sep + "logs.txt", "w")
    output_file.write("Model Parameters: {}".format(vars(opt)))
    training_params={"batch_size":opt.batch_size, "shuffle": True, "drop_last": True}
    test_params={"batch_size":opt.batch_size, "shuffle": False, "drop_last": True}
    #max_word_length, max_sent_length= get_max_lengths(opt.train_set)
    max_word_length=30
    max_sent_length=60
    num_classes= 3
    training_set= HanInDataset(train_set, opt.word2vec_path, max_sent_length, max_word_length, model_name) # tuple(ndarray(encoded_doc), label)
    print(training_set[0])
    training_generator= DataLoader(training_set, **training_params)

    test_set = HanInDataset(valid_set, opt.word2vec_path, max_sent_length, max_word_length, model_name)
    test_generator = DataLoader(test_set, **test_params)

    # defining the model
    model = HierAttn(opt.word_hidden_size, opt.sent_hidden_size, opt.batch_size, num_classes,
                       opt.word2vec_path, max_sent_length, max_word_length, embd_size, model_name)

    if os.path.isdir(opt.log_path):
        shutil.rmtree(opt.log_path)
    os.makedirs(opt.log_path)
    writer = SummaryWriter(opt.log_path)
    # writer.add_graph(model, torch.zeros(opt.batch_size, max_sent_length, max_word_length))

    if torch.cuda.is_available():
        model.cuda()

    criterion = nn.CrossEntropyLoss() # can handle both binary and multiclass classification
    optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.lr, momentum=opt.momentum)
    best_loss = 1e5
    best_epoch = 0
    model.train()   # model.train() tells your model that we are training the model.
                    # This helps inform layers such as Dropout and BatchNorm, which are designed to behave differently during training and evaluation.
                    # For instance, in training mode, BatchNorm updates a moving average on each new batch;
                    # whereas, for evaluation mode, these updates are frozen.
    num_iter_per_epoch = len(training_generator)
    for epoch in tqdm(range(opt.num_epochs)):
        for iter, (feature, label) in enumerate(training_generator):
            if torch.cuda.is_available():
                feature = feature.cuda()
                label = label.cuda()
            optimizer.zero_grad()
            model._init_hidden_state()
            # here, feature is numpy array
            predictions = model(feature)    # predicting from the model

            loss = criterion(predictions, label)
            loss.backward()
            optimizer.step()
            training_metrics = get_evaluation(label.cpu().numpy(), predictions.cpu().detach().numpy(), list_metrics=["accuracy"])
            print("Epoch: {}/{}, Iteration: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
                epoch + 1,
                opt.num_epochs,
                iter + 1,
                num_iter_per_epoch,
                optimizer.param_groups[0]['lr'],
                loss, training_metrics["accuracy"]))
            writer.add_scalar('Train/Loss', loss, epoch * num_iter_per_epoch + iter)
            writer.add_scalar('Train/Accuracy', training_metrics["accuracy"], epoch * num_iter_per_epoch + iter)
        if epoch % opt.test_interval == 0:
            model.eval()
            loss_ls = []
            te_label_ls = []
            te_pred_ls = []
            for te_feature, te_label in test_generator:
                num_sample = len(te_label)
                if torch.cuda.is_available():
                    te_feature = te_feature.cuda()
                    te_label = te_label.cuda()
                with torch.no_grad():
                    model._init_hidden_state(num_sample)
                    te_predictions = model(te_feature)
                te_loss = criterion(te_predictions, te_label)
                loss_ls.append(te_loss * num_sample)
                te_label_ls.extend(te_label.clone().cpu())
                #te_predictions = F.softmax(te_predictions)
                te_pred_ls.append(te_predictions.clone().cpu())
                #print(te_predictions)

            te_loss = sum(loss_ls) / test_set.__len__()
            te_pred = torch.cat(te_pred_ls, 0)
            te_label = np.array(te_label_ls)
            test_metrics = get_evaluation(te_label, te_pred.numpy(), list_metrics=["accuracy", "confusion_matrix"])

            output_file.write(
                "Epoch: {}/{} \nTest loss: {} Test accuracy: {} \nTest confusion matrix: \n{}\n\n".format(
                    epoch + 1, opt.num_epochs,
                    te_loss,
                    test_metrics["accuracy"],
                    test_metrics["confusion_matrix"]))
            print("Epoch: {}/{}, Lr: {}, Loss: {}, Accuracy: {}".format(
                epoch + 1,
                opt.num_epochs,
                optimizer.param_groups[0]['lr'],
                te_loss, test_metrics["accuracy"]))
            writer.add_scalar('Test/Loss', te_loss, epoch)
            writer.add_scalar('Test/Accuracy', test_metrics["accuracy"], epoch)
            model.train()
            if te_loss + opt.es_min_delta < best_loss:
                best_loss = te_loss
                best_epoch = epoch
                torch.save(model, opt.saved_path + os.sep + "whole_model_han")

            # Early stopping
            if epoch - best_epoch > opt.es_patience > 0:
                print("Stop training at epoch {}. The lowest loss achieved is {}".format(epoch, te_loss))
                break

# In[19]:

if __name__== "__main__":
    # with joined atc mapping (10,25)
    with_context=True
    with_demog=False
    all=True
    window_len=2
    model_name= "prajjwal1/bert-mini"
    embd_size= 256
    opt = get_args()

    all_dataset= pd.read_csv("./../../../generated_files/bipolar_ori_partitioned_atc_mapped_data.csv")

    topic_set= pd.read_csv("./../../../generated_files/bipolar_topic_words.csv")

    #print(topic_set)

    all_dataset= all_dataset.iloc[:,1:]

    all_dataset.rename(columns={"TEXT_REPROT": "text", "GAF": "label"}, inplace=True)
    all_datasett=all_dataset.iloc[:, lambda df: [4, 5]]
    all_datasett= all_datasett.sample(frac=1, random_state=42).iloc[:,:].reset_index(drop=True)
    df_train, df_valid=handleDataSupport(all_datasett,topic_set.iloc[:25,:], with_context,with_demog, all, window_len, 0)

    train(opt, df_train,df_valid,embd_size, model_name)

# In[ ]:

# In[11]:

df_train.head()

# In[12]:

df_train.at[0,"text"]

# In[13]:

#------------------------ P L A Y G R O U N D -------------------------#
# a=torch.Tensor([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])
# b=torch.tensor([[11,22,33],[44,55,66],[77,88,99],[100,200,300]])

# aa=a.unsqueeze(1)
# print(aa.shape)
# bb=b.unsqueeze(1)
# c=torch.cat((aa,bb),1)
# print(c)
# print(c.size())

# d,e=torch.split(c,[1,1],1)
# print(d.squeeze(1).shape)


from flask import Flask
#creates the application object as an instance of class Flask imported from the flask package
from config import Config
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager
import logging
from logging.handlers import SMTPHandler
from logging.handlers import RotatingFileHandler
import os
from flask_bootstrap import Bootstrap
from flask_moment import Moment
from flask_flatpages import FlatPages
#from flask_babel import _, lazy_gettext as _l

db = SQLAlchemy()
migrate = Migrate()
login = LoginManager()
login.login_view = 'auth.login'
login.login_message = 'Please log in to access this page.'
#mail = Mail()
bootstrap = Bootstrap()
moment = Moment()
pages = FlatPages()

def create_app(config_class=Config):
    app = Flask(__name__) #__name__ is Python predifined var, which is
                      #set to the name of the module in which it is used
    app.config.from_object(config_class)

    db.init_app(app)
    migrate.init_app(app, db)
    login.init_app(app)
    bootstrap.init_app(app)
    moment.init_app(app)
    pages.init_app(app)

    from app.errors import bp as errors_bp
    app.register_blueprint(errors_bp)

    from app.auth import bp as auth_bp
    app.register_blueprint(auth_bp, url_prefix='/auth')

    from app.main import bp as main_bp
    app.register_blueprint(main_bp)

    if not app.debug:
        if app.config['MAIL_SERVER']:
            auth = None
            if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']:
                auth = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])
            secure = None
            if app.config['MAIL_USE_TLS']:
                secure = ()
            mail_handler = SMTPHandler(
                mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']),
                fromaddr='no-reply@' + app.config['MAIL_SERVER'],
                toaddrs=app.config['ADMINS'], subject='Microblog Failure',
                credentials=auth, secure=secure)
            mail_handler.setLevel(logging.ERROR)
            app.logger.addHandler(mail_handler)
            if not os.path.exists('logs'):
                os.mkdir('logs')
            file_handler = RotatingFileHandler('logs/microblog.log', maxBytes=10240,
                                               backupCount=10)
            file_handler.setFormatter(logging.Formatter(
                '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
            file_handler.setLevel(logging.INFO)
            app.logger.addHandler(file_handler)

            app.logger.setLevel(logging.INFO)
            app.logger.info('Microblog startup')

    return app

from app import models


import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
import os, json
import pdb
import matplotlib.ticker as ticker
import glob2, argparse
from os.path import expanduser
import visualization_module as viz

if __name__ == '__main__' :

    parser = argparse.ArgumentParser(description='data viz', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # reference directories
    parser.add_argument('--sd', default='/home/mattiacarletti/Desktop/plots/', help='Save directory')
    parser.add_argument('--base', default='/mnt/DATA/Mattia_data/results/', help='Base directory')
    # the following 3 parameters define the sub-directories from which we load data
    parser.add_argument('--exp', required=True, help='Experiment to be considered')
    parser.add_argument('--arch', required=True, nargs='*', help='List of architectures to be considered ("resnet18", "resnet34")')
    parser.add_argument('--datasets', required=True, nargs='*', help='List of datasets to be considered ("cifar10", "cifar100", "imagenet_lt")')
    # quantity to plot on the y-axis (plot) and phase
    parser.add_argument('--plot', required=True, help='Plot to be created ("loss", "top1", "class_count")')
    parser.add_argument('--phase', help='Phase to plot ("train", "val", "train_val")')
    parser.add_argument('--epochs', nargs='*', help='List of epochs to be considered (for class count hist)')
    # comparison at plot level (grouping variables)
    parser.add_argument('--hue', default=None, type=json.loads, help='Hue grouping variable for the seaborn plot')
    parser.add_argument('--style', default=None, type=json.loads, help='Style grouping variable for the seaborn plot (ignored when using phase="train_val")')
    parser.add_argument('--size', default=None, type=json.loads, help='Size grouping variable for the seaborn plot')
    # comparison at file level
    parser.add_argument('--cfl', default=None, type=json.loads, help='List of variables for comparisons at file level')
    opt = vars(parser.parse_args())

    # when phase='train_val', we use plain line for val and dashed line for train
    if opt['phase'] == 'train_val':
        opt['style'] = None
        print('\n"style" argument ignored since "phase"=="train_val"\n')
    # when plot='class_count', opt['style'] and opt['size'] cannot be used
    if opt['plot'] == 'class_count' and opt['style'] is not None and opt['size'] is not None:
        opt['style'] = None
        opt['size'] = None
        print('\n"style" and "size" arguments cannot be used since "plot"=="class_count"\n')
    # if 'arch' and 'dataset' are not grouping variables,
    # information about the architecture and dataset must be included in 'cfl'
    options = [list(opt[name].keys())[0] for name in ['hue','style','size'] if opt[name]]
    if 'arch' not in options:
        assert opt['cfl'] is not None, "'arch' must be added to the variables for comparison at file level"
        assert 'arch' in opt['cfl'].keys(), "'arch' must be added to the variables for comparison at file level"
    if 'dataset' not in options:
        assert opt['cfl'] is not None, "'dataset' must be added to the variables for comparison at file level"
        assert 'dataset' in opt['cfl'].keys(), "'dataset' must be added to the variables for comparison at file level"
    # PLOTS
    if opt['plot'] in ['loss', 'top1']:
        if opt['phase'] in ['train','val']:
            viz.compared_plot(opt)
        elif opt['phase'] == 'train_val':
            viz.compared_plot_train_val(opt)
    elif opt['plot'] == 'class_count':
        viz.compared_class_count_hist(opt)
    else:
        NotImplementedError


from absl.testing import absltest
from jaxonnxruntime import call_onnx
from jaxonnxruntime.experimental import custom_ops  # pylint: disable=unused-import
import numpy as np

import onnx
from onnx import helper as onnx_helper

class ZerosLikeTest(absltest.TestCase):

  def test_basic(self):
    # Create input and output tensors.
    input_tensor = onnx_helper.make_tensor_value_info(
        "X", onnx.TensorProto.FLOAT, [1, 3, 224, 224]
    )
    output_tensor = onnx_helper.make_tensor_value_info(
        "Y", onnx.TensorProto.FLOAT, [1, 3, 224, 224]
    )

    # Create the ZerosLike node
    node = onnx_helper.make_node(
        "ZerosLike",
        inputs=["X"],
        outputs=["Y"],
        dtype=onnx.TensorProto.FLOAT,
        domain="jaxonnxruntime",
    )

    # Create the graph with the node
    graph_def = onnx_helper.make_graph(
        [node],
        "ZerosLike_Model",
        [input_tensor],
        [output_tensor],
    )

    # Create the model
    onnx_model = onnx_helper.make_model(
        graph_def,
        producer_name="JAX-ONNX",
        opset_imports=[
            onnx_helper.make_opsetid(
                onnx.defs.ONNX_DOMAIN, onnx.defs.onnx_opset_version()
            ),
            onnx_helper.make_opsetid("jaxonnxruntime", 1),
        ],
    )
    x = np.random.randn(3, 4, 5).astype(np.float32)
    y = np.zeros_like(x).astype(np.float32)
    inputs = [x]
    jax_model_func, jax_model_params = call_onnx.call_onnx_model(
        onnx_model, inputs
    )
    outputs = jax_model_func(jax_model_params, inputs)
    expect_outputs = [y]
    self.assertLen(outputs, 1, f"output is {outputs}")
    np.testing.assert_allclose(outputs[0], expect_outputs[0], atol=1e-6)

if __name__ == "__main__":
  absltest.main()


import sys
from collections import deque

input = sys.stdin.readline

M, N = map(int, input().split())
board = [list(input().rstrip()) for _ in range(M)]

visited = []
dx = [1, -1, 0, 0]
dy = [0, 0, 1, -1]
result_v = 0
result_o = 0

for m in range(M):
    for n in range(N):
        queue = deque()
        temp = []
        if board[m][n] in ["v", "o", "."] and [m, n] not in visited:
            queue.append([m, n])
            temp.append(board[m][n])
            visited.append([m, n])
            while queue:
                m_, n_ = queue.popleft()
                for i in range(4):
                    nx = dx[i] + n_
                    ny = dy[i] + m_
                    if nx < 0 or N <= nx or ny < 0 or M <= ny:
                        continue
                    if board[ny][nx] == '#':
                        continue
                    if [ny, nx] not in visited:
                        queue.append([ny, nx])
                        temp.append(board[ny][nx])
                        visited.append([ny, nx])
        if temp:
            if temp.count('v') < temp.count('o'):
                result_o += temp.count('o')
            else:
                result_v += temp.count('v')
print(result_o, result_v)