from docx import Document
from urllib.request import urlretrieve
import glob, os
import time
import locale
import datetime
import pymysql.cursors
import comtypes.client
import winsound

##pratiklerin word dosyasındaki tablo sırası
pt_1 = 14
pt_2 = 21
pt_3 = 26
pt_4 = 36
pt_5 = 41

##ders listelerinin word dosyasındaki tablo sırası
dt_1 = 18
dt_2 = 23
dt_3 = 33
dt_4 = 38
dt_5 = 43

##Ders programlarının en güncel halini indirir
def DosyaIndir():  # download file from internet and save same file
    print(" ")
    print(" ________________________________________")
    print(" / 3. sınıf - doc indirme başladı...     \\")
    url = "http://www.ctf.edu.tr/egitim_ogretim/ders/2019_2020/19_20_dersprog3.doc"
    urlretrieve(url, "19_20_dersprog3.doc")
    print(" | indirme tamamlandı                    |")

## .doc dosyasını .docx dosyasına çevirmek gerekiyor
def DosyayıCevir():  # convert .doc file to .docx
    print(" |- - - - - - - - - - - - - - - - - - - -|")
    print(" | çevirme başladı...                    |")
    wdFormatPDF = 16
    TEST_FILENAME = os.path.join(os.path.dirname(__file__), '19_20_dersprog3.doc')
    word = comtypes.client.CreateObject('Word.Application')
    for file in glob.glob(TEST_FILENAME):
        doc = word.Documents.Open(file)

    TEST_FILENAME2 = os.path.join(os.path.dirname(__file__), '19_20_dersprog3.docx')
    doc.SaveAs(TEST_FILENAME2, FileFormat=wdFormatPDF)
    doc.Close()
    word.Quit()
    print(" | 3. sınıf - çevirme tamamlandı         |")
    print(" \_______________________________________/")

## Güncellemeden önce eski veritabanını temizlemek gerekli
def TabloyuSil():
    print("Silme Başladı...")
    connection = pymysql.connect(host='cerrahapp.cf',
                                 user='****',
                                 password='****',
                                 db='****',
                                 charset='utf8mb4',
                                 cursorclass=pymysql.cursors.DictCursor)
    try:
        with connection.cursor() as cursor:
            # Create a new record
            sql_sil = "TRUNCATE TABLE `ders_listesi`";
            cursor.execute(sql_sil)
        connection.commit()
    finally:
        connection.close()
        print("Silme tamamlandı")
        print("________________")

##eski pratik tablosunu silmek için
def TabloyuSilpratik():
    print("Silme Başladı...")
    connection = pymysql.connect(host='cerrahapp.cf',
                                 user='*',
                                 password='****',
                                 db='****',
                                 charset='utf8mb4',
                                 cursorclass=pymysql.cursors.DictCursor)
    try:
        with connection.cursor() as cursor:
            # Create a new record
            sql_sil = "TRUNCATE TABLE `uygulama_ucuncu_sinif`";
            cursor.execute(sql_sil)
        connection.commit()
    finally:
        connection.close()
        print("Silme tamamlandı")
        print("________________")

##Veritabanına ders programını ekleme fonksiyonu, tek tek ekliyor
def DB_Ekle_ders(_amfi,_dersno,_tarih,_konu,_hoca,_ders,_sureilk,_prog,_i):
    connection = pymysql.connect(host='cerrahapp.cf',
                                 user='****',
                                 password='****',
                                 db='****',
                                 charset='utf8mb4',
                                 cursorclass=pymysql.cursors.DictCursor)
    try:
        with connection.cursor() as cursor:
            # Create a new record
            sql = "INSERT INTO `ders_listesi` (`Amfi`, `Ders No`, `Tarih/saat`, `Konu`, `Öğretim Üyesi`, `Anabilim Dalı`) VALUES (%s,%s,%s,%s,%s,%s);";
            cursor.execute(sql, (_amfi, _dersno, _tarih, _konu.replace("'", " "), _hoca.replace("'", " "), _ders.replace("'", " ")))
        connection.commit()
    finally:
        connection.close()

##veritabanına pratik ekleme fonksiyonu, tek tek
def DB_Ekle_pratik(_tarih,_sayi,_harf,_deger,_sureilk,_prog,_i):
    connection = pymysql.connect(host='cerrahapp.cf',
                                 user='****',
                                 password='****',
                                 db='****',
                                 charset='utf8mb4',
                                 cursorclass=pymysql.cursors.DictCursor)
    try:
        with connection.cursor() as cursor:
            # Create a new record
            sql = "INSERT INTO `uygulama_ucuncu_sinif` (`tarih`, `sayi`, `harf`, `deger`) VALUES (%s,%s,%s,%s);";
            cursor.execute(sql, (_tarih, _sayi,_harf,_deger))
        connection.commit()
    finally:
        connection.close()

##Ders listesini word dosyasındaki tablolardan çekme fonksiyonu, tüm büyü burada
def Yukle(_i):  #EKU
    yenigun=False
    document = Document('19_20_dersprog3.docx')
    table = document.tables[_i]  # i need second table
    keys = None
    data = []
    gun = ""
    dersnoEKU = 0
    dersnoING = 0
    dersnobir = 0
    dersnoiki = 0
    dersnoing = 0
    yariyil = "iki"

    ##her bir dersin kendisine ait kodu var, burada o yapılıyor, 3. kuruun 1. amfi için  1. kurul dersinin nosu, 311 ile başlıyor
    if (_i == dt_1):
        yariyil = "iki"
        dersnobir = 311001
        dersnoiki = 321001
        dersnoing = 331001
    elif (_i == dt_2):
        yariyil = "iki"
        dersnobir = 312001
        dersnoiki = 322001
        dersnoing = 332001
    elif (_i == dt_3):
        yariyil = "iki" #iki
        dersnobir = 313001
        dersnoiki = 323001
        dersnoing = 333001
    elif (_i == dt_4):
        yariyil = "iki"
        dersnobir = 314001
        dersnoiki = 324001
        dersnoing = 334001
    elif (_i == dt_5):
        yariyil = "iki"
        dersnobir = 315001
        dersnoiki = 325001
        dersnoing = 335001

    ##her bir satır için işlem yapıyor
    for i, row in enumerate(table.rows):

        sureilk = time.time()
        text = (cell.text for cell in row.cells)
        if i == 0:
            keys = tuple(text)
            continue

        row_data = tuple(text)
        ########################
        if (yariyil=="bir"):
            saat = row_data[0]
            ekuders = row_data[1]  # Periferik ve Merkezi Sinir Sistemi, Tanımlamalar (Doç. Dr. Selman Demirci, Anatomi)
            ingders = row_data[2]

            if ("2019" in saat):
                gun = saat.replace(" Pazartesi", "")
                gun = gun.replace(" Salı", "")
                gun = gun.replace(" Çarşamba", "")
                gun = gun.replace(" Perşembe", "")
                gun = gun.replace(" Cuma", "")
            if ("2020" in saat):
                gun = saat.replace(" Pazartesi", "")
                gun = gun.replace(" Salı", "")
                gun = gun.replace(" Çarşamba", "")
                gun = gun.replace(" Perşembe", "")
                gun = gun.replace(" Cuma", "")
            tarihX = gun + " " + saat
            locale.setlocale(locale.LC_ALL, 'turkish')

            try:
                tarihX = datetime.datetime.strptime(tarihX, '%d %B %Y %H:%M')
            except ValueError:
                tarihX = "--"

            data.append([tarihX, ekuders])

            str1 = ekuders.rsplit('(', 1)
            str2 = ingders.rsplit('(', 1)

            hocaEKU = repr(str1[len(str1) - 1])
            hocaEKU = hocaEKU.split(", ")
            hocaING = repr(str2[len(str2) - 1])
            hocaING = hocaING.split(", ")

            konuEKU = str1[0].replace("'", "")
            konuING = str2[0].replace("'", "")

            ogretim_uyesiEKU = hocaEKU[0].replace("'", "")
            ogretim_uyesiING = hocaING[0].replace("'", "")
            try:
                dersEKU = hocaEKU[-1].replace(")'", "")
            except IndexError:
                dersEKU = "---"
            try:
                dersING = hocaING[-1].replace(")'", "")
            except IndexError:
                dersING = "---"

            if (konuEKU != "Uygulamalar" and konuEKU != "Uygulamalar " and konuEKU != "" and konuEKU != " "):
                dersnoEKU = dersnoEKU + 1
                print("")
                print('\033[91m' + "##########################################" + '\033[0m')
                print('\033[93m' + "ders id: " + '\033[0m' + str(dersnoEKU))
                print('\033[93m' + "ders: " + '\033[0m' + str(tarihX))
                print('\033[93m' + "konu: " + '\033[0m' + konuEKU)
                print('\033[93m' + "hoca: " + '\033[0m' + ogretim_uyesiEKU)
                print('\033[93m' + "ders: " + '\033[0m' + dersEKU)
                print('\033[91m' + "##########################################" + '\033[0m')
                print("")
                DB_Ekle_ders("EKU", dersnoEKU, tarihX, konuEKU, ogretim_uyesiEKU, dersEKU, sureilk, 0, i)

            if (konuING != "Practices" and konuING != "Practices " and konuING != "" and konuING != " "):
                dersnoING = dersnoING + 1
                print("")
                print('\033[91m' + "##########################################" + '\033[0m')
                print('\033[93m' + "ders idING: " + '\033[0m' + str(dersnoING))
                print('\033[93m' + "dersING: " + '\033[0m' + str(tarihX))
                print('\033[93m' + "konuING: " + '\033[0m' + konuING)
                print('\033[93m' + "hocaING: " + '\033[0m' + ogretim_uyesiING)
                print('\033[93m' + "dersING: " + '\033[0m' + dersING)
                print('\033[91m' + "##########################################" + '\033[0m')
                print("")
                DB_Ekle_ders("İng3", dersnoING, tarihX, konuING, ogretim_uyesiING, dersING, sureilk, 0, i)
                ######################################################################################
        elif (yariyil=="iki"):
            saat = row_data[0]
            birders = row_data[1]  # Periferik ve Merkezi Sinir Sistemi, Tanımlamalar (Doç. Dr. Selman Demirci, Anatomi) örnek ders satırı böyle
            ikiders = row_data[2]
            ingders = row_data[3]

            if ("2019" in saat):
                gun = saat.replace(" Pazartesi", "")
                gun = gun.replace(" Salı", "")
                gun = gun.replace(" Çarşamba", "")
                gun = gun.replace(" Perşembe", "")
                gun = gun.replace(" Cuma", "")
            if ("2020" in saat):
                gun = saat.replace(" Pazartesi", "")
                gun = gun.replace(" Salı", "")
                gun = gun.replace(" Çarşamba", "")
                gun = gun.replace(" Perşembe", "")
                gun = gun.replace(" Cuma", "")
            tarihX = gun + " " + saat
            locale.setlocale(locale.LC_ALL, 'turkish')
            try:
                tarihX = datetime.datetime.strptime(tarihX, '%d %B %Y %H:%M') ##tarih formati vs
            except ValueError:
                tarihX = "--"

            str1 = birders.rsplit('(', 1)
            str2 = ikiders.rsplit('(', 1)
            str3 = ingders.rsplit('(', 1)

            hocabir = repr(str1[len(str1) - 1])
            hocabir = hocabir.split(", ")

            hocaiki = repr(str2[len(str2) - 1])
            hocaiki = hocaiki.split(", ")

            hocaing = repr(str3[len(str3) - 1])
            hocaing = hocaing.split(", ")

            konubir = str1[0].replace("'", "")
            konuiki = str2[0].replace("'", "")
            konuing = str3[0].replace("'", "")

            ogretim_uyesibir = hocabir[0].replace("'", "")
            ogretim_uyesiiki = hocaiki[0].replace("'", "")
            ogretim_uyesiing = hocaing[0].replace("'", "")
            try:
                dersbir = hocabir[-1].replace(")'", "")
            except IndexError:
                dersbir = "---"
            try:
                dersiki = hocaiki[-1].replace(")'", "")
            except IndexError:
                dersiki = "---"
            try:
                dersing = hocaing[-1].replace(")'", "")
            except IndexError:
                dersing = "---"

            if (konubir != "Uygulamalar" and konubir != "Uygulamalar " and konubir != "" and konubir != " "):
                dersnobir = dersnobir + 1
                print("")
                print('\033[91m' + "##########################################" + '\033[0m')
                print('\033[93m' + "ders id 1: " + '\033[0m' + str(dersnobir))
                print('\033[93m' + "tarih 1: " + '\033[0m' + str(tarihX))
                print('\033[93m' + "konu 1: " + '\033[0m' + konubir)
                print('\033[93m' + "hoca 1: " + '\033[0m' + ogretim_uyesibir)
                print('\033[93m' + "ders 1: " + '\033[0m' + dersbir)
                print('\033[91m' + "##########################################" + '\033[0m')
                print("")
                DB_Ekle_ders("1", dersnobir, tarihX, konubir, ogretim_uyesibir, dersbir, sureilk, 0, i)

            if (konuiki != "Uygulamalar" and konuiki != "Uygulamalar " and konuiki != "" and konuiki != " " ):
                dersnoiki = dersnoiki + 1
                print("")
                print('\033[91m' + "##########################################" + '\033[0m')
                print('\033[93m' + "ders id 2: " + '\033[0m' + str(dersnoiki))
                print('\033[93m' + "tarih 2: " + '\033[0m' + str(tarihX))
                print('\033[93m' + "konu 2: " + '\033[0m' + konuiki)
                print('\033[93m' + "hoca 2: " + '\033[0m' + ogretim_uyesiiki)
                print('\033[93m' + "ders 2: " + '\033[0m' + dersiki)
                print('\033[91m' + "##########################################" + '\033[0m')
                print("")
                DB_Ekle_ders("2", dersnoiki, tarihX, konuiki, ogretim_uyesiiki, dersiki, sureilk, 0, i)
            if (konuing != "Practices" and konuing != "Practices " and konuing != "" and konuing != " "):
                dersnoing = dersnoing + 1
                print("")
                print('\033[91m' + "##########################################" + '\033[0m')
                print('\033[93m' + "ders id ing: " + '\033[0m' + str(dersnoing))
                print('\033[93m' + "tarih ing: " + '\033[0m' + str(tarihX))
                print('\033[93m' + "konu ing: " + '\033[0m' + konuing)
                print('\033[93m' + "hoca ing: " + '\033[0m' + ogretim_uyesiing)
                print('\033[93m' + "ders ing: " + '\033[0m' + dersing)
                print('\033[91m' + "##########################################" + '\033[0m')
                print("")
                DB_Ekle_ders("İng3", dersnoing, tarihX, konuing, ogretim_uyesiing, dersing, sureilk, 0, i)

##pratikler word dosyasından çekiliyor
def Pratik(_i):
    document = Document('19_20_dersprog3.docx')
    table = document.tables[_i]  # i need second table
    keys = None
    data = []

    # pratik: 13 , 19 , 23 , 32 , 36

    for i, row in enumerate(table.rows):
        a=2
        sureilk = time.time()
        text = (cell.text for cell in row.cells)
        if i == 0:
            keys = tuple(text)
            continue
        row_data = tuple(text)
        ########################
        #No/Tarih/Anatomi/Fizyoloji/Biyokimya/Histoloji/Patoloji/Klinik
        print("")
        print('\033[91m' + "##########################################" + '\033[0m')
        while a < len(keys):
            pratiktarih=row_data[1]

            pratiktarih = pratiktarih.replace("** ","")
            pratiktarih = pratiktarih.replace("**","")
            pratiktarih = pratiktarih.replace("* ","")
            pratiktarih = pratiktarih.replace("*","")

            locale.setlocale(locale.LC_ALL, 'turkish')
            try:
                pratiktarih = str(datetime.datetime.strptime(pratiktarih, '%d.%m.%Y'))
            except ValueError:
                pratiktarih = "---"

            if("," in row_data[a]):
                b = 0
                #pratik içinde virgül var
                tumgrup=row_data[a].split(",")    #1,2 / 2,4 / 2a,2/ 1b,2a
                grup1=len(tumgrup)
                while b < grup1:
                    sayiB = tumgrup[b]  #2/2a/1b
                    pratik = keys[a]
                    pratikb = pratik.rsplit(' / ')
                    if "sınav" in sayiB:
                        pratikb[0] = pratikb[0] + " (Sınav)"
                        pratikb[1] = pratikb[1] + " (exam)"
                        sayiB = sayiB.replace(" (sınav)", "")
                    if ("1" in sayiB):
                        print("tarih: " + pratiktarih + " " + "sayı: " + sayiB + " pratik: " + pratikb[1])
                        if (("a" in sayiB) or ("b" in sayiB)):
                            DB_Ekle_pratik(pratiktarih, sayiB[0], (sayiB[1]).upper(), pratikb[1], sureilk, 0, i)
                        else:
                            DB_Ekle_pratik(pratiktarih, sayiB, "A", pratikb[1], sureilk, 0, i)
                            DB_Ekle_pratik(pratiktarih, sayiB, "B", pratikb[1], sureilk, 0, i)
                    else:
                        print("tarih: " + pratiktarih + " " + "sayı: " + sayiB + " pratik: " + pratikb[0])
                        if (("a" in sayiB) or ("b" in sayiB)):
                            DB_Ekle_pratik(pratiktarih, sayiB[0], (sayiB[1]).upper(), pratikb[0], sureilk, 0, i)
                        else:
                            DB_Ekle_pratik(pratiktarih, sayiB, "A", pratikb[0], sureilk, 0, i)
                            DB_Ekle_pratik(pratiktarih, sayiB, "B", pratikb[0], sureilk, 0, i)
                    b=b+1
            else:
                # pratik içinde virgül yok tek değer
                sayi = row_data[a]   #1 / 2 / 2a / 1b
                if sayi != "-" and sayi != "":

                    pratik = keys[a]
                    if pratik == "Biyoistatistik/ Biostatistics":
                        pratik = "Biyoistatistik / Biostatistics"
                    pratikb = pratik.rsplit(' / ')
                    if "sınav" in sayi:
                        pratikb[0] = pratikb[0] + " (Sınav)"
                        pratikb[1] = pratikb[1] + " (exam)"
                        sayi = sayi.replace(" (sınav)", "")
                    try:
                        if ("1" in sayi):
                            print("tarih: " + pratiktarih + " " + "sayı: " + sayi + " pratik: " + pratikb[1])
                            if (("a" in sayi) or ("b" in sayi)):
                                DB_Ekle_pratik(pratiktarih, sayi[0], (sayi[1]).upper(), pratikb[1], sureilk, 0, i)
                            else:
                                DB_Ekle_pratik(pratiktarih, sayi, "A", pratikb[1], sureilk, 0, i)
                                DB_Ekle_pratik(pratiktarih, sayi, "B", pratikb[1], sureilk, 0, i)
                        else:
                            print("tarih: " + pratiktarih + " " + "sayı: " + sayi + " pratik: " + pratikb[0])
                            if (("a" in sayi) or ("b" in sayi)):
                                DB_Ekle_pratik(pratiktarih, sayi[0], (sayi[1]).upper(), pratikb[0], sureilk, 0, i)
                            else:
                                DB_Ekle_pratik(pratiktarih, sayi, "A", pratikb[0], sureilk, 0, i)
                                DB_Ekle_pratik(pratiktarih, sayi, "B", pratikb[0], sureilk, 0, i)
                    except IndexError:
                        print("")
            a=a+1
        print('\033[91m' + "##########################################" + '\033[0m')
        print("")

        #DB_Ekle("EKU", dersnoEKU, tarihX, konuEKU, ogretim_uyesiEKU, dersEKU, sureilk, 0, i)

##tablo numaralarının eşleşmesi doğru mu kontrol ediliyor
def Tablolar(_i):
    document = Document('19_20_dersprog3.docx')
    table = document.tables[_i]  # i need second table
    keys = None
    data = []

    # pratik: 13 , 19 , 23 , 32 , 36

    for i, row in enumerate(table.rows):
        a=2
        sureilk = time.time()
        text = (cell.text for cell in row.cells)
        if i == 0:
            keys = tuple(text)
            continue
        row_data = tuple(text)
        data.append(row_data)
    print(str(data)[:30])
        #DB_Ekle("EKU", dersnoEKU, tarihX, konuEKU, ogretim_uyesiEKU, dersEKU, sureilk, 0, i)

def DerslistesiniSil():
    TabloyuSil()
def ders_guncelle():
            Yukle(dt_1)
            Yukle(dt_2)
            Yukle(dt_3)
            Yukle(dt_4)
            Yukle(dt_5)
def pratik_guncelle():
    TabloyuSilpratik()
    Pratik(pt_1)
    Pratik(pt_2)
    Pratik(pt_3)
    Pratik(pt_4)
    Pratik(pt_5)
def indir():
    DosyaIndir()
    DosyayıCevir()

def ses ():
    duration = 100  # millisecond
    freq1 = 440  # Hz
    freq2 = 540  # Hz
    freq3 = 640  # Hz
    winsound.Beep(freq1, duration)
    winsound.Beep(freq2, duration)
    winsound.Beep(freq3, duration)
    winsound.Beep(freq1, duration)
    winsound.Beep(freq1, duration)
    winsound.Beep(freq2, duration)
    winsound.Beep(freq3, duration)
    winsound.Beep(freq1, duration)
def TabloPratiklerYaz():
    print("******2. Sınıf Pratik Programları ********")
    Tablolar(pt_1)
    Tablolar(pt_2)
    Tablolar(pt_3)
    Tablolar(pt_4)
    Tablolar(pt_5)
    print("*********************")
def TabloDerslerYaz():
    print("******3. Sınıf Ders Programları *********")
    Tablolar(dt_1)
    Tablolar(dt_2)
    Tablolar(dt_3)
    Tablolar(dt_4)
    Tablolar(dt_5)
    print("*********************")


# main.py
# 2021.08.28 ~
# Shin, SangYun

import sys
import json
import threading
from BucketBot import *
from CatchBot import *
from OrderInfo import *
from loguru import logger
from TelegramBot import *

def readJsonSetting(jsonFile):
    with open(jsonFile, "r") as file:
        return json.load(file)

def parseJsOption(jsOption):
    return OrderInfo(
        targetDI = jsOption['targetDI'],
        marginRatio = jsOption['marginRatio'],
        profitPercent = jsOption['profitPercent'],
        stoplossTriggerPercent = jsOption['stoplossTriggerPercent'],
        stoplossPercent = 0.001)

def startBucketBot(symbol, option):
    BucketBot(symbol, option).start()

def startCatchBot(symbol, option):
    CatchBot(symbol, option).start()

def getStartMsg():
    return "Start!!\nCatch: {} | Bucket: {} | Symbols: {}\n\
    BucketBot\n\
        - TargetDI: {:.1f}%\n\
        - profit: {:.1f}%\n\
        - stoplossTrigger: {:.1f}%\n\
        - marginRatio: {:.1f}%\n\
    CatchBot\n\
        - TargetDI: {:.1f}%\n\
        - profit: {:.1f}%\n\
        - stoplossTrigger: {:.1f}%\n\
        - marginRatio: {:.1f}%".format(js['useCatchBot'], js['useBucketBot'], len(js['symbols']),
            bucketOption.targetDI*100, bucketOption.profitPercent*100, bucketOption.stoplossTriggerPercent*100, bucketOption.marginRatio*100,
            catchOption.targetDI*100, catchOption.profitPercent*100, catchOption.stoplossTriggerPercent*100, catchOption.marginRatio*100)

def panicSellAlarm(symbols, deltaForAlarm):
    prevPriceList = {}

    for symbol in symbols:
        try:
            prevPriceList[symbol] = Lib.getCurrentPrice(symbol)
        except Exception as ex:
            logger.warning("{:10} | Raised an exception. {}", symbol, repr(ex))
            continue

    while True:
        time.sleep(5)

        for symbol in symbols:
            try:
                curPrice = Lib.getCurrentPrice(symbol)
                prevPrice = prevPriceList[symbol]
                deltaPercent = ((curPrice - prevPrice) / prevPrice) * 100

                if deltaPercent <= deltaForAlarm:
                    logger.info("{:10} | {:10.2f}% 급락", symbol, deltaPercent)
                    TelegramBot.sendMsg("{:10} | {:10.2f}% 급락".format(symbol, deltaPercent))

                prevPriceList[symbol] = curPrice

            except Exception as ex:
                logger.warning("{:10} | Raised an exception. {}", symbol, repr(ex))
                continue

#################### main ####################
js = readJsonSetting("trade_setting.json")
Lib.init(js['binanceApi']['key'], js['binanceApi']['secret'])
catchOption = parseJsOption(js['catch'][0])   # 1번 DI 타겟
bucketOption = parseJsOption(js['bucket'])

logger.remove()
logger.add('logs/log.log', level = js['log']['fileLevel'])
logger.add(sys.stderr, level = js['log']['consoleLevel'])
logger.info("CatchBot: {} | BucketBot: {} | Symbols: {}", js['useCatchBot'], js['useBucketBot'], len(js['symbols']))
TelegramBot.init(js['telegramToken'])

if js['useCatchBot']:
    for coin in js["symbols"]:
        threading.Thread(target = startCatchBot, args = (coin, catchOption)).start()

if js['useBucketBot']:
    for coin in js["symbols"]:
        threading.Thread(target = startBucketBot, args=(coin, bucketOption)).start()

msg = getStartMsg()
logger.info("TelegramOn: {}", TelegramBot.isChatOn)
logger.info(msg)
TelegramBot.sendMsg(msg)

if js['panicSellAlarm']['use']:
    threading.Thread(target = panicSellAlarm, args=(js["symbols"], js['panicSellAlarm']['deltaPercent'])).start()


'''
Test the handling & generation of missing values
'''

# Standard library imports
import unittest
from collections import namedtuple
from unittest.mock import Mock, patch

# External library imports
import pandas as pd
import numpy as np
from pandas.testing import assert_frame_equal, assert_series_equal

# Exhibit imports
from exhibit.db import db_util
from exhibit.core.constants import MISSING_DATA_STR, ORIGINAL_VALUES_PAIRED
from exhibit.core.tests.test_reference import temp_exhibit

# Module under test
from exhibit.core.generate import missing as tm

class missingDataTests(unittest.TestCase):
    '''
    Doc string
    '''

    @classmethod
    def tearDownClass(cls):
        '''
        Clean up local exhibit.db from temp tables
        '''

        db_util.purge_temp_tables()

    def test_feeding_data_to_missing_generator(self):
        '''
        Doc string
        '''

        test_df = pd.DataFrame()

        path = "exhibit.core.generate.missing.MissingDataGenerator.__init__"
        with patch(path) as mock_init:
            mock_init.return_value = None
            generatorMock = tm.MissingDataGenerator(Mock(), Mock())

        setattr(generatorMock, "data", test_df)

        self.assertTrue(
            isinstance(generatorMock.data,
            pd.DataFrame))

    def test_never_null_indices_are_identified(self):
        '''
        Some cells can't ever have nulls due to custom constraints.
        Filter and Partition fields are optional when defining custom
        constraints.
        '''

        test_dict = {
            "constraints" : {
                "custom_constraints": {
                    "cc1" : {
                        "filter"  : "~Test.isna()",
                        "targets" : {
                            "Num" : "make_not_null"
                        }
                    },
                }
            }
        }

        test_data = pd.DataFrame(data={
            "Test" : [1, 2, 3, pd.NA, 5],
            "Num"  : [1, 2, 3, 4, 5]
        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)

        not_null_idx = test_gen._find_not_null_idx()

        result = not_null_idx[0]

        assert_series_equal(
            test_data.loc[result],
            test_data.loc[[0, 1, 2, 4], "Num"])

    def test_paired_columns_with_missing_data_identified(self):
        '''
        Doc string
        '''

        test_dict = {
            "columns" : {
                "A" : {
                    "type"            : "categorical",
                    "paired_columns"  : ["B"],
                    "miss_probability": 0.5,
                    "original_values" : ORIGINAL_VALUES_PAIRED
                },
                "B" : {
                    "type"            : "categorical",
                    "paired_columns" : ["A"],
                    "miss_probability" : 0.5,
                    "original_values" : pd.DataFrame()
                },
                "C" : {
                    "type"            : "categorical",
                    "paired_columns" : ["D"],
                    "miss_probability" : 0.6,
                    "original_values" : pd.DataFrame()
                },
                "D" : {
                    "type"            : "categorical",
                    "paired_columns" : ["C"],
                    "miss_probability" : 0.7,
                    "original_values" : ORIGINAL_VALUES_PAIRED
                }
            },
            "constraints" : {
                "custom_constraints" : {},

                },
            "linked_columns" : []
        }

        expected = [
            {"A", "B"},
        ]

        test_gen = tm.MissingDataGenerator(test_dict, Mock())
        result = test_gen._find_columns_with_linked_missing_data()

        self.assertCountEqual(expected, result)

    def test_linked_columns_with_missing_data_identified(self):
        '''
        Doc string
        '''

        test_dict = {
            "columns" : {
                "A" : {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0.5,
                    "original_values" : pd.DataFrame()
                    },
                "B" : {
                    "type"            : "categorical",
                    "paired_columns" : [],
                    "miss_probability" : 0.5,
                    "original_values" : pd.DataFrame()
                    },
                "C" : {
                    "type"            : "categorical",
                    "paired_columns" : [],
                    "miss_probability" : 0.6,
                    "original_values" : pd.DataFrame()
                    },
                "D" : {
                    "type"            : "categorical",
                    "paired_columns" : [],
                    "miss_probability" : 0.5,
                    "original_values" : pd.DataFrame()
                    }
            },
            "constraints" : {
                "custom_constraints" : {},
                },
            "linked_columns" : [
                (1, ["A", "B"]),
                (2, ["C", "D"])
                ]
        }

        expected = [
            {"A", "B"},
        ]

        test_gen = tm.MissingDataGenerator(test_dict, Mock())
        result = test_gen._find_columns_with_linked_missing_data()

        self.assertCountEqual(expected, result)

    def test_linked_and_paired_columns_with_missing_data_identified(self):
        '''
        Doc string
        '''

        test_dict = {
            "columns" : {
                "A" : {
                    "type"            : "categorical",
                    "paired_columns"  : ["B"],
                    "miss_probability": 0.5,
                    "original_values" : pd.DataFrame()
                    },
                "B" : {
                    "type"            : "categorical",
                    "paired_columns" : ["A"],
                    "miss_probability" : 0.5,
                    "original_values" : ORIGINAL_VALUES_PAIRED
                    },
                "C" : {
                    "type"            : "categorical",
                    "paired_columns" : [],
                    "miss_probability" : 0.5,
                    "original_values" : pd.DataFrame()
                    }
            },
            "constraints" : {
                "custom_constraints" : {},
                },
            "linked_columns" : [
                (0, ["A", "C"]),
                ]
        }

        expected = [
            {"A", "B", "C"},
        ]

        test_gen = tm.MissingDataGenerator(test_dict, Mock())
        result = test_gen._find_columns_with_linked_missing_data()

        self.assertTrue(expected[0], result[0])

    def test_make_null_constraint_in_columns(self):
        '''
        When we're adding nulls to categorical columns, the non-null
        numerical values must be re-calulcated and re-scaled because
        Missing data (null identifier in categorical columns) can have
        vastly different weights compared to the old values. However,
        we shouldn't rescaled the whole column anew, just the added values.
        '''

        Weights = namedtuple("Weights", ["weight", "equal_weight"])

        #demo weights table
        weights_df = pd.DataFrame(
            data=[
                ("C", "A", "spam", Weights(0.5, 0.5)),
                ("C", "A", "eggs", Weights(0.5, 0.5)),
                ("C", "B", "bacon", Weights(0.5, 0.5)),
                ("C", "A", MISSING_DATA_STR, Weights(0.5, 0.5)),
                ("C", "B", MISSING_DATA_STR, Weights(0.5, 0.5)),
            ],
            columns=["num_col", "cat_col", "cat_value", "weights"])

        #reformat into dictionary
        weights = (
            weights_df
                .set_index(["num_col", "cat_col", "cat_value"])
                .to_dict(orient="index")
        )

        test_dict = {
            "_rng" : np.random.default_rng(seed=0),
            "metadata" : {
                "categorical_columns": ["A", "B"],
                "numerical_columns" : ["C"]
            },
            "columns": {
                "A": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0,
                    "original_values" : pd.DataFrame()

                },
                "B": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0,
                    "original_values" : pd.DataFrame()

                },
                "C": {
                    "type"            : "continuous",
                    "precision"       : "integer",
                    "distribution"    : "weighted_uniform",
                    "distribution_parameters": {
                        "dispersion": 0,
                        "target_sum" : 10,
                    },
                    "miss_probability": 0
                },

            },
            "constraints" : {
                "custom_constraints": {
                    "cc1" : {
                        "filter"  : "A == 'spam'",
                        "targets" : {
                            "B" : "make_null"
                        }
                    }
                }
            },
            "linked_columns" : [],
            "weights_table" : weights,
            "weights_table_target_cols": ["A", "B"]
        }

        test_data = pd.DataFrame(data={
            "A" : ["spam", "spam", "eggs", "eggs"],
            "B" : ["bacon"] * 4,
            "C" : [10, 20, 4, 4],
        })

        expected = pd.DataFrame(data={
            "A" : ["spam", "spam", "eggs", "eggs"],
            "B" : [pd.NA, pd.NA, "bacon", "bacon"],
            "C" : [1, 1, 4, 4],
        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)
        result = test_gen.add_missing_data()

        assert_frame_equal(result, expected, check_dtype=False)

    def test_not_null_constraint_in_columns(self):
        '''
        Doc string
        '''

        test_dict = {
            "_rng" : np.random.default_rng(seed=0),
            "metadata" : {
                "categorical_columns": ["C", "D"],
                "numerical_columns" : []
            },
            "columns": {
                "C": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0.2,
                    "original_values" : pd.DataFrame()

                },
                "D": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0.5,
                    "original_values" : pd.DataFrame()

                }
            },
            "constraints" : {
                "custom_constraints": {
                    "cc1" : {
                        "filter"  : "~C.isna()",
                        "targets" : {
                            "D" : "make_not_null"
                        }
                    }
                }
            },
            "linked_columns" : []
        }

        test_data = pd.DataFrame(data={
            "C" : np.random.random(1000), #pylint: disable=no-member
            "D" : np.random.random(1000), #pylint: disable=no-member

        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)
        result = test_gen.add_missing_data()

        self.assertTrue(result["C"].isna().any())
        self.assertTrue(result["D"].isna().any())
        self.assertFalse(result.loc[~result["C"].isna(), "D"].isna().any())

    def test_paired_columns_are_respected_for_missing_data(self):
        '''
        Doc string
        '''

        test_dict = {
            "_rng" : np.random.default_rng(seed=0),
            "metadata" : {
                "categorical_columns": ["A", "B"],
                "numerical_columns" : []
            },
            "columns": {
                "A": {
                    "type"            : "categorical",
                    "paired_columns"  : ["B"],
                    "miss_probability": 0.5,
                    "original_values" : pd.DataFrame()
                },
                "B": {
                    "type"            : "categorical",
                    "paired_columns"  : ["A"],
                    "miss_probability": 0.5,
                    "original_values" : ORIGINAL_VALUES_PAIRED

                },
            },
            "constraints" : {
                "custom_constraints" : {},
            },
            "linked_columns" : [],
        }

        test_data = pd.DataFrame(data={
            "A" : np.random.random(1000), #pylint: disable=no-member
            "B" : np.random.random(1000), #pylint: disable=no-member
        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)
        result = test_gen.add_missing_data()

        self.assertTrue(result["A"].isna().any())
        self.assertTrue(result["B"].isna().any())
        assert_series_equal(result["B"].isna(), result["A"].isna(), check_names=False)

    def test_missing_data_added_to_standalone_categorical_column(self):
        '''
        Doc string
        '''

        test_dict = {
            "_rng" : np.random.default_rng(seed=0),
            "metadata" : {
                "categorical_columns": ["A", "B"],
                "numerical_columns" : []
            },
            "columns": {
                "A": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 1,
                    "original_values" : pd.DataFrame()
                },
                "B": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0,
                    "original_values" : pd.DataFrame()

                },
            },
            "constraints" : {
                "custom_constraints" : {}
            },
            "linked_columns" : [],
        }

        test_data = pd.DataFrame(data={
            "A" : list("ABCDE"),
            "B" : list("ABCDE")
        })

        expected = pd.DataFrame(data={
            "A" : [pd.NA] * 5,
            "B" : list("ABCDE")
        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)
        result = test_gen.add_missing_data()

        assert_frame_equal(expected, result, check_dtype=False)

    def test_continuous_column_adjusted_to_categorical_missing_data(self):
        '''
        Remember that continuous columns depend on values in categorical columns
        in the same row for their weights, including for Missing data values.
        Adding Missing data also changes the target_sum of the continuous column
        so we need to re-scale the whole column after adding missing data either
        to it or to the categorical columns.

        We rely on np.random to generate reasonable number of NAs with 0.5 prob,
        but that can sometimes fail so we ensure that the seed is constant.
        '''

        Weights = namedtuple("Weights", ["weight", "equal_weight"])

        #demo weights table
        weights_df = pd.DataFrame(
            data=[
                ("C2", "C1", "A", Weights(0.1, 0.5)),
                ("C2", "C1", "B", Weights(0.9, 0.5)),
                ("C2", "C1", MISSING_DATA_STR, Weights(0.2, 0.5)),
            ],
            columns=["num_col", "cat_col", "cat_value", "weights"])

        #reformat into dictionary
        weights = (
            weights_df
                .set_index(["num_col", "cat_col", "cat_value"])
                .to_dict(orient="index")
        )

        test_dict = {
            "_rng" : np.random.default_rng(seed=0),
            "metadata": {
                "categorical_columns": [
                    "C1"
                ],
                "numerical_columns" : [
                    "C2"
                ]
            },
            "columns": {
                "C1": {
                    "type"            : "categorical",
                    "paired_columns"  : [],
                    "miss_probability": 0.5,
                    "original_values" : pd.DataFrame()
                },
                "C2": {
                    "type"            : "continuous",
                    "precision"       : "integer",
                    "distribution"    : "weighted_uniform",
                    "distribution_parameters": {
                        "uniform_base_value" : 100,
                        "dispersion": 0,
                        "target_sum" : 200, # factor of two
                    },
                    "miss_probability": 0
                },
            },
            "constraints" : {
                "custom_constraints" : {}
            },
            "linked_columns" : [],
            "weights_table" : weights,
            "weights_table_target_cols": ["C1"]
        }

        test_data = pd.DataFrame(data={
            "C1" : ["A", "A", "A", "B", "B"] * 20,
            "C2" : [1] * 100
        })

        test_gen = tm.MissingDataGenerator(test_dict, test_data)
        result = test_gen.add_missing_data()

        self.assertTrue(result["C1"].isna().any())
        self.assertEqual(result["C2"].sum(), 200)

    def test_user_linked_columns_having_missing_data(self):
        '''
        Because user linked columns can have complex relationships, we
        need to make sure missing data is handled correctly.
        '''

        test_df = pd.DataFrame(data={
            "A": ["spam", "spam", "eggs", "eggs", "spam"],
            "B": ["bacon", "spamspam", np.nan, "parrot", "bacon"],
            "C": range(5)
        })

        test_dict = {
            "metadata" : {
                "number_of_rows" : 1000
            }
        }

        fromdata_test = {
            "linked_columns" : ["A", "B"]
        }

        _, df = temp_exhibit(
            filename=test_df, fromdata_namespace=fromdata_test,
            test_spec_dict=test_dict, return_spec=False)

        self.assertTrue(df.query("A == 'eggs'")["B"].isna().any())

if __name__ == "__main__" and __package__ is None:
    #overwrite __package__ builtin as per PEP 366
    __package__ = "exhibit"
    unittest.main(warnings="ignore")


import cv2 as cv
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data
import torchvision.transforms as transforms
import torchvision.utils
import os
from PIL import Image
from torch.autograd import Variable
from TextureTrain import texture
from Net import ContentNet
from Net import CoderGan

overlap_pred = 4
device = torch.device("cuda:0")
n_epoches = 25
image_size = 128
image_size_raw = 512
batch_size = 1
beta_1 = 0.5
lr = 0.0002
wtl2 = 0.998
wtlD = 0.001
niter = 25
overlapL2Weight = 10
dir = 'result/'
content_path = 'test/3.jpg'
transform = transforms.Compose([transforms.Scale(image_size),

                                transforms.CenterCrop(image_size),
                                transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
transform1 = transforms.Compose([transforms.Scale(image_size_raw),
                                 transforms.CenterCrop(image_size_raw),
                                 transforms.ToTensor(),
                                 transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))])
denorm_transform = transforms.Normalize(mean=(-1, -1, -1), std=(2, 2, 2))
denorm_transform1 = transforms.Normalize(mean=(-2.12, -2.04, -1.80), std=(4.37, 4.46, 4.44))
transform3 = transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))
pic = content_path
if not os.path.exists(dir):
    os.mkdir(dir)

content_image_ori = cv.imread(pic)
print(pic)
# print(content_image_ori)
content_image_ori = cv.cvtColor(content_image_ori, cv.COLOR_BGR2RGB)
content_image_ori_PIL = Image.fromarray(content_image_ori)
content_image = transform(content_image_ori_PIL).unsqueeze(0)
content_images = content_image
device = torch.device("cuda:0")
content_raw = transform1(content_image_ori_PIL).unsqueeze(0).to(device)

torchvision.utils.save_image(denorm_transform1(content_raw[0]), dir + '/real.jpg')

content_raw[:, :, int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2),
int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2)] = 0.0
content_raw[:, :, int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2),
int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2)] = torch.mean(content_raw)

torchvision.utils.save_image(denorm_transform1(content_raw[0]), dir + '/cropped.jpg')

photos = ["test/2.jpg", "test/3.jpg"]
for path in photos:
    content_image_ori1 = cv.imread(path)
    content_image_ori1 = cv.cvtColor(content_image_ori1, cv.COLOR_BGR2RGB)
    content_image_ori_PIL1 = Image.fromarray(content_image_ori1)
    content_image1 = transform(content_image_ori_PIL1).unsqueeze(0)
    content_images = torch.cat((content_images, content_image1), 0)

input_cropped = torch.FloatTensor(batch_size, 3, image_size, image_size)

input_cropped = Variable(input_cropped).to(device)

input_cropped.resize_(content_images.size()).copy_(content_images)

input_cropped.data[:, 0,

int(image_size / 4 + overlap_pred):int(image_size / 4 + image_size / 2 - overlap_pred),
int(image_size / 4 + overlap_pred):int(
    image_size / 4 + image_size / 2 - overlap_pred)] = 2 * 117.0 / 255.0 - 1.0
input_cropped.data[:, 1,
int(image_size / 4 + overlap_pred):int(image_size / 4 + image_size / 2 - overlap_pred),
int(image_size / 4 + overlap_pred):int(
    image_size / 4 + image_size / 2 - overlap_pred)] = 2 * 104.0 / 255.0 - 1.0
input_cropped.data[:, 2,
int(image_size / 4 + overlap_pred):int(image_size / 4 + image_size / 2 - overlap_pred),
int(image_size / 4 + overlap_pred):int(
    image_size / 4 + image_size / 2 - overlap_pred)] = 2 * 123.0 / 255.0 - 1.0

content_net = CoderGan().to(device)
content_net.load_state_dict(torch.load("GAN_cifar10.pth", map_location=lambda storage, location: storage)['state_dict'])
resume_epoch = torch.load("GAN_cifar10.pth")['epoch']
print(content_net)
synthesis = content_net(input_cropped)
recon_image = input_cropped.clone()
recon_image.data[:, :, int(image_size / 4):int(image_size / 4 + image_size / 2),
int(image_size / 4):int(image_size / 4 + image_size / 2)] = synthesis.data
torchvision.utils.save_image(denorm_transform(recon_image.data[0]), dir + '/input.jpg')
torchvision.utils.save_image(denorm_transform(synthesis.data[0]), dir + '/output.jpg')
content_result = denorm_transform(synthesis.data[0])
content_result = transform3(content_result)
content_result = content_result.unsqueeze(0)
content_result = F.interpolate(content_result, [int(image_size_raw / 2), int(image_size_raw / 2)],
                               mode="bilinear")
content_result.cuda()
result = texture(content_raw, content_result, dir)
content_raw = denorm_transform1(content_raw.data[0])
for i in range(3):
    content_raw.data[i, int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2), \
    int(image_size_raw / 4):int(image_size_raw / 4 + image_size_raw / 2)] = result.data[0][i]
torchvision.utils.save_image(content_raw, dir + '/result.jpg')


import os

import torch.nn as nn
from utils import *

class dynamical_model(nn.Module):

    def __init__(self,
                input_length: int,
                in_channels: int,
                residual_channels: int,
                pred_var_dims: int=3,
                activation: str='relu',
                diagonal: bool=False,
                ):
        super(dynamical_model, self).__init__()
        self.in_channels = in_channels
        self.input_feat = 3*in_channels*input_length
        self.pred_var_dims = pred_var_dims
        self.diagonal = diagonal
        self.res_channels = residual_channels
        bias = True

        assert self.input_feat//8 > self.res_channels*3
        self.encoder = nn.Sequential(
            nn.Linear(in_features=self.input_feat, out_features=self.input_feat//2, bias=bias),
            activation_func(activation),
            #nn.Linear(in_features=self.input_feat, out_features=self.input_feat//2, bias=bias),
            #activation_func(activation),
            #nn.Linear(in_features=self.input_feat//2, out_features=self.input_feat//4, bias=bias),
            #activation_func(activation),
            #nn.Linear(in_features=self.input_feat//4, out_features=self.input_feat//8, bias=bias),
            #activation_func(activation),
            nn.Linear(in_features=self.input_feat//2, out_features=self.res_channels*3),
            #activation_func(activation),
            #nn.Linear(in_features=self.input_feat//2, out_features=self.input_feat//4),
            #activation_func(activation),
            #nn.Linear(in_features=self.input_feat//4, out_features=self.input_feat//8),
            #activation_func(activation),
        )
        self.dropout = nn.Dropout(0.5)

        self.enc_to_mean = nn.Linear(self.res_channels*3, self.pred_var_dims)
        if self.diagonal:
            self.enc_to_var = nn.Linear(self.res_channels*3, self.pred_var_dims)
        else:
            self.alpha = int(self.pred_var_dims*(1+ (self.pred_var_dims-1)/2)) # n*(n-1)/2
            self.enc_to_var = nn.Linear(self.res_channels*3, self.alpha)

    def forward(self, history_input, target_x, target_vel):
        #INPUT: batchsize x 1 x k history x nmodes x 3
        #TARGET: batchsize x 1 x nmodes x 3
        # encoding
        #self.history_input = history_input
        #self.history_input = history_input.squeeze(1).squeeze(2)
        self.rot_history = self.Rotate(history_input)
        out = self.encoder(self.rot_history.view( self.rot_history.size(0),-1))

        out_drop = self.dropout(out)
        rot_mean, logvar, encoded = self.encode(out_drop)

        #target_x = target_x.squeeze(1)
        #target_vel = target_vel.squeeze(1)
        rot_target_vel = torch.bmm(self.rotation_matrix.transpose(1,2),target_vel.swapaxes(1,2)).squeeze(-1) #target rotation to frame (t-1,t-2,t-3)
        rot_target_x = torch.bmm(self.rotation_matrix.transpose(1,2),target_x.swapaxes(1,2)).swapaxes(1,2)

        # Loss evaluation
        dyn_loss = self.NLL(rot_mean, logvar, rot_target_vel)
        loss = dyn_loss
        return loss, [rot_mean,logvar], encoded

    def encode(self, out):

        means = self.enc_to_mean(out)
        if self.diagonal:
           logvarsquared = self.enc_to_var(out)
           return means, logvarsquared
        else:
            elements = self.enc_to_var(out)
            lower = torch.zeros(elements.size(0), self.pred_var_dims, self.pred_var_dims).to(elements)
            batched_eye = torch.eye(self.pred_var_dims).unsqueeze(0).expand(elements.size(0),-1,-1).to(elements)
            lower = lower + batched_eye
            index = np.cumsum([n for n in range(0,self.pred_var_dims)])
            for row in range(1,self.pred_var_dims):
                lower[:,row,:row] = elements[:,index[row-1]:index[row]]

            diag = torch.exp(elements[:,self.alpha-self.pred_var_dims:]).unsqueeze(1)*batched_eye
            partial_prod1 = torch.bmm(lower, diag)
            self.sigma = torch.bmm(partial_prod1, lower.transpose(1,2))
            return means, [diag, lower], out

    def NLL(self, mean,logvar, target, diagonal=False):
        diag, lower = logvar
        partial_prod1 = torch.bmm(lower, diag)
        self.sigma = torch.bmm(partial_prod1, lower.transpose(1,2))
        sigma_inverse = torch.inverse(self.sigma)
        partial_prod2  = torch.bmm((mean-target).unsqueeze(1), sigma_inverse)
        logprob = -(1/2)*torch.diagonal(diag, dim1=1, dim2=2).sum(dim=1) - torch.bmm(partial_prod2, (mean-target).unsqueeze(-1)).squeeze(-1).squeeze(-1)/2
        return - logprob.mean()

    def get_parameters(self, history_input):
        #INPUT: batchsize x 1 x k history x nmodes x 3
        #TARGET: batchsize x 1 x nmodes x 3
        #self.history_input = history_input
        #self.history_input = history_input.squeeze(1).squeeze(2)
        self.rot_history = self.Rotate(history_input)
        out = self.encoder(self.rot_history.view( self.rot_history.size(0),-1))
        out_drop = self.dropout(out)
        rot_mean, logvar, _ = self.encode(out_drop)

        #rot_history = self.Rotate(history_input )
        #rot_mean, logvar, _ = self.encode(rot_history)
        return rot_mean, logvar

    def RotationMatrix(self):
        channels = self.history_input.size(2)
        rotation_matrix = torch.zeros(self.history_input.size(0), channels, channels).to(self.history_input)
        v_t1, norm_t1 = self.history_input[:,-1,:],  self.history_input[:,-1,:].norm(dim=1)
        v_t2 = self.history_input[:,-2,:]
        v_t3 = self.history_input[:,-3,:]
        orthonormal_b1 = v_t1/norm_t1.unsqueeze(-1).expand(-1,channels)
        aux_b2 = v_t2 - torch.bmm(v_t2.unsqueeze(1), orthonormal_b1.unsqueeze(-1)).squeeze(-1)*orthonormal_b1
        orthonormal_b2 = aux_b2/aux_b2.norm(dim=1).unsqueeze(-1).expand(-1,channels)
        rotation_matrix[:,:,0] = orthonormal_b1
        rotation_matrix[:,:,1] = orthonormal_b2
        orthonormal_b3 = torch.linalg.cross(orthonormal_b1, orthonormal_b2)
        sign_flip = torch.bmm(v_t3.unsqueeze(1), orthonormal_b3.unsqueeze(-1)) > 0
        mask = 2*sign_flip -1
        orthonormal_b3 = mask.squeeze(-1)*orthonormal_b3
        rotation_matrix[:,:,2] = orthonormal_b3
        return rotation_matrix

    def Rotate(self, history_input):
        self.history_input = history_input
         #INPUT: batchsize x k history x nmodes x 3dims
        self.rotation_matrix = self.RotationMatrix()
        #assert (torch.diagonal(torch.bmm(rotation_matrix, rotation_matrix.transpose(1,2)),dim1=1,dim2=2).sum() == float(self.history_input.size(0)*self.history_input.size(-1))).item()
        rot_traj = torch.zeros_like(self.history_input).to(self.history_input)
        for k in range(self.history_input.size(1)):
            rot_traj[:,k,:] = torch.bmm(self.rotation_matrix.transpose(1,2),self.history_input[:,k,:].unsqueeze(-1)).squeeze(-1)
        return rot_traj

    def __getitem__(self):
        return self.sigma

    def __getitem__(self):
        return self.rot_history


import xml.etree.ElementTree as ET
import copy
import os
import json

possessive_XML_filenames = os.listdir('../BuNaMo/possessive')

def main():

    default_possessive_objects = {}
    possessive_variations_objects = []

    count = 1
    for fname in possessive_XML_filenames:
        print(str(count) + ' ')
        object_from_one_file = parse_one_xml_file('../BuNaMo/possessive/' + fname)
        default = object_from_one_file[0].pop('default')
        default_possessive_objects[default] = object_from_one_file[0]
        possessive_variations_objects.extend(object_from_one_file[1])
        count += 1

    with open('../converted-JSON-data/possessives/possessives_default.json', 'w') as outfile:
        json.dump(default_possessive_objects, outfile, ensure_ascii=False)

    with open('../converted-JSON-data/possessives/possessives.json', 'w') as outfile:
        json.dump(possessive_variations_objects, outfile, ensure_ascii=False)

def parse_one_xml_file(filename):
    mytree = ET.parse(filename)
    root = mytree.getroot()

    default_json_object = copy.deepcopy(root.attrib)
    json_variations = []
    for el in root:
        el_json = {'default': default_json_object['default']}

        if 'fem' in filename:
            el_json['gender'] = 'fem'
            el_json['number'] = 'sg'
        elif 'masc' in filename:
            el_json['gender'] = 'masc'
            el_json['number'] = 'sg'
        elif 'pl' in filename or 'ár'in filename or 'bhur' in filename:
            el_json['number'] = 'pl'
        else:
            el_json['number'] = 'sg'

        for item in el.items():
            el_json['type'] = "possessive"
            el_json['word'] = item[1]
            el_json['form'] = el.tag.lower()
            default_json_object[el.tag] = item[1]
        json_variations.append(el_json)

    # print('default_json_object:', default_json_object)
    # print('json_variations:', json_variations)
    return [default_json_object, json_variations]

if __name__ == "__main__":
    main()
# parse_one_xml_file('../BuNaMo/possessive/bhur_poss.xml')


import logging
import hashlib
from pathlib import Path

def match_patterns_in_dirs(dirs: list[str], glob_patterns: list[str]) -> dict[str, dict[str, str]]:
    """Search each given dir using each given "glob" pattern, return matched files, with content digests, per dir."""
    matches = {}
    for dir in dirs:
        dir_matches = {}
        for glob_pattern in glob_patterns:
            dir_glob_matches = match_pattern_in_dir(dir, glob_pattern)
            dir_matches.update(dir_glob_matches)
        if dir_matches:
            matches[dir] = dir_matches
    return matches

def match_pattern_in_dir(dir: str, glob_pattern: str) -> dict[str, str]:
    """Search the given dir using the given "glob" pattern, return matched files with their content digests."""
    matches = Path(dir).glob(glob_pattern)
    file_matches = [match for match in matches if match.is_file()]
    return {path.relative_to(dir).as_posix(): hash_contents(path) for path in file_matches}

def hash_contents(path: Path, algorithm: str = "sha256") -> str:
    """Hash the file contents at the given path, return hex-encoded digest prefixed with the algorignm name."""
    logging.info(f"Computing content hash ({algorithm}) for file: {path.as_posix()}")
    with open(path, "rb") as f:
        digest = hashlib.file_digest(f, algorithm)
    return f"{digest.name}:{digest.hexdigest()}"

def count_matches(matches: dict[str, dict[str, str]]) -> int:
    return sum(len(dir_matches) for dir_matches in matches.values())

def flatten_matches(matches: dict[str, dict[str, str]], **kwargs) -> list[dict[str, str]]:
    flattened = []
    for volume, file_info in matches.items():
        for path, digest in file_info.items():
            flattened.append(file_summary(volume=volume, path=path, digest=digest, **kwargs))
    return flattened

def file_summary(volume: str = "", path: str = "", digest: str = "", **kwargs) -> dict[str, str]:
    return {
        'file_volume': volume,
        'file_path': path,
        'file_digest': digest,
        **kwargs
    }


import json
import re
import time

def current_milli_time():
    return round(time.time() * 1000)

def parse_html(description, content):
    pattern = r'($\\r\\n|$\\n|$\\r)+'
    pattern2 = r'<[^>]*>'

    if description != None and description != "" and description != "None":
        parsed_description = description.replace('\n', '. ').replace('\r', '')
        parsed_description = re.sub(pattern2, ' ', parsed_description, re.M)
        parsed_description = parse_pontuation(parsed_description)
    else:
        parsed_description = ""

    if content != None and content != "" and content != "None":
        parsed_content = content.replace('\n', '. ').replace('\r', '')
        parsed_content = re.sub(pattern2, ' ', parsed_content, re.M)
        parsed_content = parse_pontuation(parsed_content)
    else:
        parsed_content = ""

    if parsed_content == [] or parsed_content == "":
        parsed_text = parsed_description
    elif parsed_description == [] or parsed_description == "":
        parsed_text = parsed_content
    elif parsed_description in parsed_content:
        parsed_text = parsed_content
    else:
        parsed_text = parsed_description + " " + parsed_content

    return parsed_text

def parse_pontuation(s):
    if s[-1] != "." and s[-1] != "!" and s[-1] != "?" and s[-1] != ";":
        ponctuation = [s.rfind('. '), s.rfind(
            '! '), s.rfind('? '), s.rfind('; ')]
        later = ponctuation.index(max(ponctuation))

        if later == 0:
            pont = ". "
        elif later == 1:
            pont = "! "
        elif later == 2:
            pont = "? "
        elif later == 3:
            pont = "; "

        parsed_s = s.split(pont)[:-1]
        if parsed_s != []:
            parsed_s = pont.join(parsed_s)+"."

        return parsed_s
    return s

def check_if_exists(article_title, news_collection):
    info = news_collection.document(article_title).get()
    if info.exists:
        return True
    else:
        return False

def add_to_database(final_json, news_collection):
    try:
        news_collection.document(final_json["title"]).set(final_json)
        return json.dumps({'success': True}), 200
    except Exception as e:
        return f"An Error Occured: {e}"

def parse_title(title):
    parsed_title = title.split(" - ")[:-1]
    if parsed_title != []:
        parsed_title = " - ".join(parsed_title)
        parsed_title = parsed_title.replace("/", "-")
        return parsed_title

    title = title.replace("/", "-")
    return title


from django.db import models
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import (
    ValidationError,
)

class Account(models.Model):
    ACCOUNT_FULL_NAME_MAX_LENGTH = 20
    user = models.OneToOneField(
        User,
        on_delete = models.CASCADE
    )
    full_name = models.CharField(max_length = ACCOUNT_FULL_NAME_MAX_LENGTH,verbose_name = 'Аккаунт')
    description = models.TextField()

    def __str__(self):
        return f'Аккаунт : {self.user.id} / {self.full_name}'

    class Meta:
        ordering = (
            'full_name',
        )
        verbose_name = 'Акакаунт'
        verbose_name_plural = 'Аккаунты'

class Group(models.Model):
    GROUP_NAME_MAX_LENGTH = 10

    name = models.CharField(
        max_length = GROUP_NAME_MAX_LENGTH
        )
    def __str__(self) -> str:
        return f'Группа : {self.name}'

    class Meta:
        ordering = (
            'name',
        )
        verbose_name = 'Группа'
        verbose_name_plural = 'Группы'

class Student(models.Model):
    MAX_AGE = 27
    account = models.OneToOneField(
        Account,
        on_delete = models.CASCADE,
        verbose_name = 'Аккаунт'
    )
    age = models.IntegerField(
        'Возраст студета'
    )
    group = models.ForeignKey(
        Group,
        on_delete = models.PROTECT,
        verbose_name = 'Группа'
    )
    gpa = models.FloatField(
        'Средняя оценка'
    )
    def __str__(self) -> str:
        return f'Имя : {self.account.full_name} - Возраст : {self.age} - {self.group} - Оценка : {self.gpa}'

    def save(
        self,
        *args: tuple,
        **kwargs: dict
    ) -> None:
        if self.age > self.MAX_AGE:
            #v1
        #     self.age = self.MAX_AGE

        #v2
            raise ValidationError(
                f'Допустимый возраст : {self.MAX_AGE}'
            )
        super().save(*args,**kwargs)

    class Meta:
        ordering = (
            'account',
        )
        verbose_name = 'Студент'
        verbose_name_plural = 'Студенты'

class Proffessor(models.Model):
    FULL_NAME_MAX_LENGTH = 20

    TOPIC_JAVA = 'Java'
    TOPIC_PYTHON = 'python'
    TOPIC_TYPESCRIPT = 'Typescript'
    TOPIC_JAVASCRIPT = 'Javascript'
    TOPIC_RUBY = 'Ruby'
    TOPIC_GOLAND = 'Goland'
    TOPIC_SQL = 'Sql'
    TOPIC_SWIFT = 'swift'

    TOPIC_CHOICES = (
        {TOPIC_JAVA,'JAVA'},
        {TOPIC_PYTHON,'Python'},
        {TOPIC_TYPESCRIPT,'JavaScript'},
        {TOPIC_RUBY,'TypeScript'},
        {TOPIC_GOLAND,'Ruby'},
        {TOPIC_SQL,'Golang'},
        {TOPIC_SWIFT,'SQL'},
        {TOPIC_SWIFT,'Swift'},
    )
    full_name = models.CharField(
        verbose_name = 'полное имя',
        max_length = FULL_NAME_MAX_LENGTH,
    )
    topic = models.CharField(
        verbose_name = 'предмет',
        choices = TOPIC_CHOICES,
        default = TOPIC_JAVA,
        max_length = FULL_NAME_MAX_LENGTH,
    )
    students = models.ManyToManyField(
        Student
    )

    def __str__(self) -> str:
        return f'Имя : {self.full_name}  -- Организация : {self.topic}'

    def save(
        self,
        *args: tuple,
        **kwargs:dict,
    ) -> None:
        if self.topic is None:
            raise ValidationError(
                f"Вы не выбрали свою организвцию"
            )
        super().save(*args,**kwargs)

    class Meta:
        ordering = (
            'full_name',
        )
        verbose_name = 'Проффессор'
        verbose_name_plural = 'Проффессоры'


# Partition to K Equal Sum Subsets
from typing import List

class Solution:
    def canPartitionKSubsets(self, nums: List[int], k: int) -> bool:
        target = sum(nums) / k
        nums.sort(reverse=True)
        goodIdx = []
        def getSumIdx(left: int, curIdxs: [int], idx: int):
            if left == 0: return curIdxs
            if left < 0: return []
            while idx < len(nums) and nums[idx] > 0:
                tmp = list(curIdxs)
                tmp.append(idx)
                flag = getSumIdx(left - nums[idx], tmp, idx + 1)
                if flag:
                    goodIdx.append(flag)
                idx += 1
            return
        getSumIdx(target, [], 0)
        goodIdx = set(tuple(x) for x in goodIdx)
        print(goodIdx)
        def getK(current, curIdxs):
            if current == k: return True
            for s in goodIdx:
                flag = True
                for n in s:
                    if n in curIdxs:
                        flag = False
                        break
                if flag:
                    tmp = set(curIdxs)
                    tmp.update(s)
                    if getK(current + 1, tmp): return True
            return False
        a = set()
        return getK(0, a)

s = Solution()
print(s.canPartitionKSubsets([4, 3, 2, 3, 5, 2, 1], 4))


import torch
import torch.nn as nn
import torch.nn.functional as F
import os
import glob
import json
import tqdm
import functools
import numpy as np

from torch.utils.data.distributed import DistributedSampler
from einops import rearrange
from .dist_utils import all_gather

SMALL_NUM = np.log(1e-45)

def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
    return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device))

def clip_loss(similarity: torch.Tensor) -> torch.Tensor:
    caption_loss = contrastive_loss(similarity)
    image_loss = contrastive_loss(similarity.t())
    return (caption_loss + image_loss) / 2.0

def compute_mlm(pl_module, ret, mode):
    mlm_logits = ret[f"{mode}_logits"]
    if "self" in mode:
        mlm_labels = ret["encoder_text_labels_mlm"]
    else:
        mlm_labels = ret["decoder_text_labels_mlm"]

    mlm_loss = F.cross_entropy(
        mlm_logits.view(-1, pl_module.hparams.config["vocab_size"]),
        mlm_labels.view(-1),
        ignore_index=-100,
    )

    new_ret = {
        f"{mode}_mlm_loss": mlm_loss
    }

    phase = "train" if pl_module.training else "val"
    loss_mlm = getattr(pl_module, f"{phase}_{mode}_loss")(mlm_loss)
    pl_module.log(f"{mode}/{phase}/{mode}_loss", loss_mlm)
    return new_ret

def FLOAP(batch_rep):
    return torch.sum(torch.mean(torch.abs(batch_rep), dim=0) ** 2)

def compute_contrastive(pl_module, ret):
    # Query
    if pl_module.training_mode == "both":
        text_reps = F.normalize(ret["text_bottleneck_repre"][1])
        image_reps = F.normalize(ret["image_bottleneck_repre"][1])
    else:
        text_reps = F.normalize(ret["text_bottleneck_repre"])
        image_reps = F.normalize(ret["image_bottleneck_repre"])

    FLOAP_loss = 0.002 * (FLOAP(text_reps) + FLOAP(image_reps))

    all_text_reps = pl_module.gather(text_reps)
    all_image_reps = pl_module.gather(image_reps)

    # in-batch contrastive
    # Cross Entropy
    logits_per_text = torch.einsum("nc,ck->nk", [all_text_reps, all_image_reps.transpose(-2, -1)]) / pl_module.T
    contrastive_loss = clip_loss(logits_per_text)

    new_ret = {
        "contrastive_loss": contrastive_loss + FLOAP_loss,
    }

    phase = "train" if pl_module.training else "val"
    loss = getattr(pl_module, f"{phase}_contrastive_loss")(new_ret["contrastive_loss"])
    pl_module.log(f"contrastive/{phase}/loss", loss)

    return new_ret

@torch.no_grad()
def compute_irtr_recall(pl_module):
    text_dset = pl_module.trainer.datamodule.dms[0].make_val_dset()
    text_dset.tokenizer = pl_module.trainer.datamodule.dms[0].tokenizer
    text_loader = torch.utils.data.DataLoader(
        text_dset,
        batch_size=64,
        num_workers=pl_module.hparams.config["num_workers"],
        pin_memory=True,
        collate_fn=functools.partial(
            text_dset.collate,
            mlm_collator=pl_module.trainer.datamodule.dms[0].mlm_collator,
        ),
    )

    image_dset = pl_module.trainer.datamodule.dms[0].make_val_dset(
        image_only=True
    )
    image_dset.tokenizer = pl_module.trainer.datamodule.dms[0].tokenizer
    dist_sampler = DistributedSampler(image_dset, shuffle=False)
    image_loader = torch.utils.data.DataLoader(
        image_dset,
        batch_size=1,
        num_workers=pl_module.hparams.config["num_workers"],
        sampler=dist_sampler,
        pin_memory=True,
        collate_fn=functools.partial(
            image_dset.collate,
            mlm_collator=pl_module.trainer.datamodule.dms[0].mlm_collator,
        ),
    )

    text_preload = list()
    for _b in tqdm.tqdm(text_loader, desc="text prefetch loop"):
        text_ids = _b["text_ids"].to(pl_module.device)
        text_masks = _b["text_masks"].to(pl_module.device)
        text_preload.append(
            {
                "img_index": _b["img_index"],
                "text_reps": pl_module.encode_text(
                text_ids, text_masks)[1]
            }
        )

    tiids = list()
    for pre in text_preload:
        tiids += pre["img_index"]
    tiids = torch.tensor(tiids)

    image_preload = dict()
    image_preload_reps = list()
    for _b in tqdm.tqdm(image_loader, desc="image prefetch loop"):
        img_index = _b["img_index"][0]
        if img_index not in image_preload:

            image_features = _b["image_features"].to(pl_module.device)
            img_reps = pl_module.encode_image(image_features) # [bsz, 768]
            image_preload[img_index] = 1
            image_preload_reps.append((img_reps, _b["img_index"]))

    rank_scores = list()
    rank_iids = list()

    for img_batch in tqdm.tqdm(image_preload_reps, desc="rank loop"):
        _img_reps, _iid = img_batch # [bsz, 768]
        _img_reps = _img_reps / torch.norm(_img_reps, dim=-1, keepdim=True)

        img_batch_score = list()
        for txt_batch in text_preload:
            _text_reps = txt_batch["text_reps"] # [bsz, 768]
            _text_reps = _text_reps / torch.norm(_text_reps, dim=-1, keepdim=True)
            with torch.cuda.amp.autocast():
                score = torch.einsum('nc,cm->nm', [_img_reps, _text_reps.transpose(-1, -2)])
            img_batch_score.append(score)
        img_batch_score = torch.cat(img_batch_score, dim=-1) # [bsz, num_texts]
        rank_scores.append(img_batch_score.cpu().tolist())
        rank_iids += _iid

    ###
    torch.distributed.barrier()
    gather_rank_scores = all_gather(rank_scores)
    gather_rank_iids = all_gather(rank_iids)

    ################################
    tmp = []
    for rank_iids in gather_rank_iids:
        tmp += rank_iids
    gather_rank_iids = tmp

    tmp = []
    for rank_scores in gather_rank_scores:
        tmp += rank_scores
    gather_rank_scores = tmp
    ###############################

    iids = torch.tensor(gather_rank_iids)
    iids = iids.view(-1)
    scores = torch.tensor(gather_rank_scores)
    scores = scores.view(len(iids), -1)

    topk5 = scores.topk(5, dim=0)
    topk5_iids = iids[topk5.indices] # [5, 25010]

    topk10 = scores.topk(10, dim=1)
    topk5 = scores.topk(5, dim=1)
    topk1 = scores.topk(1, dim=1)
    topk10_iids = tiids[topk10.indices] # [5000, 10]
    topk5_iids = tiids[topk5.indices] # [5000, 5]
    topk1_iids = tiids[topk1.indices] # [5000, 1]

    tr_r10 = (iids.unsqueeze(1) == topk10_iids).float().max(dim=1)[0].mean()
    tr_r5 = (iids.unsqueeze(1) == topk5_iids).float().max(dim=1)[0].mean()
    tr_r1 = (iids.unsqueeze(1) == topk1_iids).float().max(dim=1)[0].mean()

    topk10 = scores.topk(10, dim=0)
    topk5 = scores.topk(5, dim=0)
    topk1 = scores.topk(1, dim=0)
    topk10_iids = iids[topk10.indices] # [10, 25010]
    topk5_iids = iids[topk5.indices] # [5, 25010]
    topk1_iids = iids[topk1.indices] # [1, 25010]
    # tiids [25010]

    ir_r10 = (tiids.unsqueeze(0) == topk10_iids).float().max(dim=0)[0].mean()
    ir_r5 = (tiids.unsqueeze(0) == topk5_iids).float().max(dim=0)[0].mean()
    ir_r1 = (tiids.unsqueeze(0) == topk1_iids).float().max(dim=0)[0].mean()
    # print((ir_r1, ir_r5, ir_r10, tr_r1, tr_r5, tr_r10))

    return (ir_r1, ir_r5, ir_r10, tr_r1, tr_r5, tr_r10)

def init_weights(module):
    if isinstance(module, (nn.Linear, nn.Embedding)):
        module.weight.data.normal_(mean=0.0, std=0.02)
    elif isinstance(module, nn.LayerNorm):
        module.bias.data.zero_()
        module.weight.data.fill_(1.0)

    if isinstance(module, nn.Linear) and module.bias is not None:
        module.bias.data.zero_()


import cv2
import numpy as np

'''
import numpy as np
from scipy.signal import cwt
from skimage.io import imread

# Read the input image
img = imread('input_image.png')

# Perform the continuous wavelet transform (CWT) on the image
# using the 'poisson' wavelet
cwt_result = cwt(img, 'poisson')

# The CWT returns a 2D array, with the first dimension representing the time
# (or in this case, the rows of the image) and the second dimension representing
# the scale (or columns of the image)
# We will use the absolute value of the CWT to identify noisy pixels
cwt_result = np.abs(cwt_result)

# Now we will threshold the CWT result to identify noisy pixels
# First, we need to determine a suitable threshold value
# We can use the median absolute deviation (MAD) to find this value
threshold = np.median(cwt_result) + 0.6745 * np.median(np.abs(cwt_result - np.median(cwt_result)))

# Now we can threshold the CWT result to identify the noisy pixels
noisy_pixels = cwt_result > threshold

# The 'noisy_pixels' array will be a boolean array, with 'True'
# values indicating the presence of noisy pixels in the image
'''

def poisson_noise(img):
    from skimage.io import imread
    import matplotlib.pyplot as plt
    import numpy as np
    import scipy.signal as sig
    from skimage import data

    import numpy as np
    from functools import lru_cache

    def poisson2d_wavelet(omega_x, omega_y, sigma):
        r = np.sqrt(omega_x ** 2 + omega_y ** 2)
        return -sigma ** 2 * np.exp(-sigma ** 2 * r ** 2) * (1 - 2 * sigma ** 2 * r ** 2)

    wavelets = {}
    wavelets['poisson'] = poisson2d_wavelet

    def _get_wavelet_mask(wavelet: str, omega_x: np.array, omega_y: np.array, **kwargs):
        assert omega_x.shape == omega_y.shape
        return wavelets[wavelet](omega_x, omega_y, **kwargs)

    @lru_cache(5)
    def _create_frequency_plane(image_shape: tuple):
        assert len(image_shape) == 2

        h, w = image_shape
        w_2 = (w - 1) // 2
        h_2 = (h - 1) // 2

        w_pulse = 2 * np.pi / w * np.hstack((np.arange(0, w_2 + 1), np.arange(w_2 - w + 1, 0)))
        h_pulse = 2 * np.pi / h * np.hstack((np.arange(0, h_2 + 1), np.arange(h_2 - h + 1, 0)))

        xx, yy = np.meshgrid(w_pulse, h_pulse, indexing='xy')
        dxx_dyy = abs((xx[0, 1] - xx[0, 0]) * (yy[1, 0] - yy[0, 0]))

        return xx, yy, dxx_dyy

    def cwt_2d(x, scales, wavelet, **wavelet_args):
        assert isinstance(x, np.ndarray) and len(x.shape) == 2, 'x should be 2D numpy array'

        x_image = np.fft.fft2(x)
        xx, yy, dxx_dyy = _create_frequency_plane(x_image.shape)

        cwt = []
        wav_norm = []

        for scale_val in scales:
            mask = scale_val * _get_wavelet_mask(wavelet, scale_val * xx, scale_val * yy, **wavelet_args)
            mask = mask.T
            cwt.append(np.fft.ifft2(x_image * mask.T))
            wav_norm.append((np.sum(abs(mask)**2)*dxx_dyy)**(0.5 / (2 * np.pi)))

        cwt = np.stack(cwt, axis=2)
        wav_norm = np.array(wav_norm)

        return cwt, wav_norm

    scales = np.arange(1, 10)
    # img = cv2.imread(img_path,0)
    # img = img_path.copy()
    # print(img.shape)
    # cwtmatr, freqs = sig.cwt2d(image, erlang2d_wavelet, scales, theta=5, scale=2)
    cwtmatr, freqs = cwt_2d(img,scales,'poisson',sigma=1)
    cwt_result = np.abs(cwtmatr)

    # Now we will threshold the CWT result to identify noisy pixels
    # First, we need to determine a suitable threshold value
    # We can use the median absolute deviation (MAD) to find this value
    threshold = np.median(cwt_result) + 0.6745 * np.median(np.abs(cwt_result - np.median(cwt_result)))

    # Now we can threshold the CWT result to identify the noisy pixels
    noisy_pixels_orig = cwt_result > threshold

    img_with_noises = []
    for i in range(9):
        noisy_pixels = noisy_pixels_orig[:img.shape[0],:img.shape[1],i]
        # The 'noisy_pixels' array will be a boolean array, with 'True'
        # values indicating the presence of noisy pixels in the image
        # img = cv2.imread(img_path,1)
        img_with_noise = np.zeros(img.shape)
        img_with_noise[noisy_pixels] = 255
        img_with_noises.append(img_with_noise)
        # cv2.imwrite(f'img_with_noise-{i}.jpg',img_with_noise)
        # print("reached here!")
    # return img_with_noise
    # return cwtmatr
    return img_with_noises

# img_path = r"C:\Users\gprak\Downloads\Github Repos\watermark.png"
# img_path = r"C:\Users\gprak\Downloads\Research Papers\dog\images.jpg"
# img_path = r"C:\Users\gprak\Downloads\Research Papers\label_poisoned_dataset - Copy\aadhar\Aadhaar_letter_large.png"
# img_path = r"C:\Users\gprak\Downloads\Research Papers\label_poisoned_dataset - Copy\invoice\Screenshot 2023-03-19 223303.png"
# poisson_noise(img_path)
# cv2.imwrite('img_with_noise.jpg',img_with_noise)


import numpy as np, cv2

image = cv2.imread("../../resources/images07/morph.jpg", cv2.IMREAD_GRAYSCALE)
if image is None: raise Exception("영상파일 읽기 오류")

mask = np.array([[0,1,0],
                 [1,1,1],
                 [0,1,0]]).astype('uint8')
th_img = cv2.threshold(image, 128, 255, cv2.THRESH_BINARY)[1]

dst = cv2.dilate(th_img, mask)
# dst = cv2.morphologyEx(th_img, cv2.MORPH_DILATE, mask)

cv2.imshow("image", image)
cv2.imshow("erode", dst)
cv2.waitKey(0)


from django.shortcuts import render, redirect, reverse
from .models import Producto
from .forms import Productoform

# Create your views here.

def productos(request):
    Productos = Producto.objects.all()
    return render(request, "crud_productos/productos.html",{'Productos':Productos})

def productos_user(request):
    Productos = Producto.objects.all()
    return render(request, "crud_productos/productos_user.html",{'Productos':Productos})

def add_producto(request):
    if request.method == 'POST':
        form_producto = Productoform(request.POST or None, request.FILES or None)
        if form_producto.is_valid():
            idProducto = form_producto.cleaned_data.get("idProducto")
            nombre = form_producto.cleaned_data.get("nombre")
            descripcion = form_producto.cleaned_data.get("descripcion")
            image_producto = form_producto.cleaned_data.get("image_producto")
            precio = form_producto.cleaned_data.get("precio")
            stock = form_producto.cleaned_data.get("stock")
            marca = form_producto.cleaned_data.get("marca")
            obj = Producto.objects.create(
                idProducto = idProducto,
                nombre = nombre,
                descripcion = descripcion,
                image_producto = image_producto,
                precio = precio,
                stock = stock,
                marca = marca,
            )
            obj.save()
            return redirect(reverse('add_producto') + "?OK")
        else:
            return redirect(reverse('add_producto') + "?FAIL")
    else:
        form_producto = Productoform
    return render(request,"crud_productos/add_productos.html",{'form':form_producto})

# Vista modificar Producto
def update_producto(request, idProducto):
    udt_producto = Producto.objects.get(idProducto=idProducto)
    form_producto = Productoform(instance = udt_producto)

    if request.method == 'POST':
        form_producto = Productoform(request.POST or None, request.FILES or None, instance=udt_producto)
        if form_producto.is_valid():
            form_producto.save()
            return redirect(reverse('productos') + "?OK")
        else:
            return redirect(reverse('update_producto') + idProducto)

    return render(request, "crud_productos/update_productos.html", {'form':form_producto})

# Vista Eliminar Producto
def delete_producto(request,idProducto):
    dlt_producto = Producto.objects.get(idProducto=idProducto)
    dlt_producto.delete()
    return redirect(to='productos')


import json
import requests

def geocode(mapquest_key, inx):
    result = {}
    url = 'http://open.mapquestapi.com/geocoding/v1/address?key={0}&location={1}'

    request = url.format(mapquest_key, inx)
    maps = json.loads(requests.get(request).text)

    if len(maps['results']) > 0:
        result = maps['results'][0]
    else:
        result={}
    return result


import asyncio
import datetime
import time

from cdislogging import get_logger

from fence.config import config
from fence.models import User
from fence.resources.openid.ras_oauth2 import RASOauth2Client as RASClient

logger = get_logger(__name__, log_level="debug")

class Visa_Token_Update(object):
    def __init__(
        self,
        chunk_size=None,
        concurrency=None,
        thread_pool_size=None,
        buffer_size=None,
        logger=logger,
    ):
        """
        args:
            chunk_size: size of chunk of users we want to take from each iteration
            concurrency: number of concurrent users going through the visa update flow
            thread_pool_size: number of Docker container CPU used for jwt verifcation
            buffer_size: max size of queue
        """
        self.chunk_size = chunk_size or 10
        self.concurrency = concurrency or 5
        self.thread_pool_size = thread_pool_size or 3
        self.buffer_size = buffer_size or 10
        self.n_workers = self.thread_pool_size + self.concurrency
        self.logger = logger

        # This job runs without an application context, so it cannot use the
        # current_app.jwt_public_keys cache.
        # This is a simple dict with the same lifetime as the job.
        # When there are many visas from many issuers it will make sense to
        # implement a more persistent cache.
        self.pkey_cache = {}

        self.visa_types = config.get("USERSYNC", {}).get("visa_types", {})

        # Initialize visa clients:
        oidc = config.get("OPENID_CONNECT", {})
        if "ras" not in oidc:
            self.logger.error("RAS client not configured")
            self.ras_client = None
        else:
            self.ras_client = RASClient(
                oidc["ras"],
                HTTP_PROXY=config.get("HTTP_PROXY"),
                logger=logger,
            )

    async def update_tokens(self, db_session):
        """
        Initialize a producer-consumer workflow.

        Producer: Collects users from db and feeds it to the workers
          Worker: Takes in the users from the Producer and passes it to the Updater to
                  update the tokens and passes those updated tokens for JWT validation
         Updater: Updates refresh_tokens and visas by calling the update_user_authorization
                  from the correct client

        """
        start_time = time.time()
        self.logger.info("Initializing Visa Update Cronjob . . .")
        self.logger.info("Total concurrency size: {}".format(self.concurrency))
        self.logger.info("Total thread pool size: {}".format(self.thread_pool_size))
        self.logger.info("Total buffer size: {}".format(self.buffer_size))
        self.logger.info("Total number of workers: {}".format(self.n_workers))

        queue = asyncio.Queue(maxsize=self.buffer_size)
        updater_queue = asyncio.Queue(maxsize=self.n_workers)
        loop = asyncio.get_event_loop()

        producers = loop.create_task(self.producer(db_session, queue, chunk_idx=0))
        workers = [
            loop.create_task(self.worker(j, queue, updater_queue))
            for j in range(self.n_workers)
        ]
        updaters = [
            loop.create_task(self.updater(i, updater_queue, db_session))
            for i in range(self.concurrency)
        ]

        await asyncio.gather(producers)
        self.logger.info("Producers done producing")
        await queue.join()

        await asyncio.gather(*workers)
        await updater_queue.join()  # blocks until everything in updater_queue is complete

        for u in updaters:
            u.cancel()

        self.logger.info(
            "Visa cron job completed in {}".format(
                datetime.timedelta(seconds=time.time() - start_time)
            )
        )

    async def get_user_from_db(self, db_session, chunk_idx):
        """
        Window function to get chunks of data from the table
        """
        start, stop = self.chunk_size * chunk_idx, self.chunk_size * (chunk_idx + 1)
        users = db_session.query(User).slice(start, stop).all()
        return users

    async def producer(self, db_session, queue, chunk_idx):
        """
        Produces users from db and puts them in a queue for processing
        """
        chunk_size = self.chunk_size
        while True:
            users = await self.get_user_from_db(db_session, chunk_idx)

            if users == None:
                break
            for user in users:
                self.logger.info("Producer producing user {}".format(user.username))
                await queue.put(user)
            if len(users) < chunk_size:
                break
            chunk_idx += 1

    async def worker(self, name, queue, updater_queue):
        """
        Create tasks to pass to updater to update visas.
        """
        while not queue.empty():
            user = await queue.get()
            await updater_queue.put(user)
            queue.task_done()

    async def updater(self, name, updater_queue, db_session):
        """
        Update visas in the updater_queue.
        Note that only visas which pass validation will be saved.
        """
        while True:
            user = await updater_queue.get()
            try:
                client = self._pick_client(user)
                if client:
                    self.logger.info(
                        "Updater {} updating authorization for user {}".format(
                            name, user.username
                        )
                    )
                    # when getting access token, this persists new refresh token,
                    # it also persists validated visa(s) in the database
                    client.update_user_authorization(
                        user,
                        pkey_cache=self.pkey_cache,
                        db_session=db_session,
                    )
                else:
                    self.logger.debug(
                        f"Updater {name} NOT updating authorization for "
                        f"user {user.username} because no client was found for IdP: {user.identity_provider}"
                    )
            except Exception as exc:
                self.logger.error(
                    f"Updater {name} could not update authorization "
                    f"for {user.username}. Error: {exc}. Continuing."
                )
                pass

            updater_queue.task_done()

    def _pick_client(self, user):
        """
        Pick oidc client according to the identity provider
        """
        client = None
        if (
            user.identity_provider
            and getattr(user.identity_provider, "name") == self.ras_client.idp
        ):
            client = self.ras_client
        return client

    def _pick_client_from_visa(self, visa):
        """
        Pick oidc client according to the visa provider
        """
        client = None
        if visa.type in self.visa_types["ras"]:
            client = self.ras_client
        else:
            raise Exception(
                "Visa type {} not configured in fence-config".format(visa.type)
            )
        if not client:
            raise Exception(
                "Visa Client not set up or not available for type {}".format(visa.type)
            )
        return client


"""
Module with helper functions for end points
"""
import string
import logging
from functools import wraps
from typing import Callable
from collections import Counter

from fastapi.responses import JSONResponse
from gitflow_learning_api.stringproc\
    import stringproc_request_classes

logging.basicConfig(filename="log.txt", filemode="w",
                    format="[%(asctime)s] [PID:%(process)d ] "
                           "[%(module)s: %(lineno)d] "
                           " [%(levelname)s] %(message)s",
                    level=logging.DEBUG)

def validator_custom(fnc: Callable):
    """
    Decorator which catches all ValueError within
    called function and   returns a JSONResponse with
    code 200, fit answer+ logging

    :param fnc: function
    :return: JSONResponse with code 200, content is the return
        from the fnc or error(if that is the case)
    """

    @wraps(fnc)
    def inner(*args, **kwargs) -> JSONResponse:

        try:
            logging.info("Function %s called",fnc.__name__)
            return JSONResponse(status_code=200, content=fnc(*args, **kwargs))
        except ValueError as error_e:
            logging.error("Error when called function %s  "
                            "-   %s",fnc.__name__, error_e)
            return JSONResponse(status_code=200,
                                content={"Error": f"{error_e}"})

    return inner

@validator_custom
def get_char_occ(word: str):
    """
      :param word: string
      :return: JSONResponse will send code  200
      and a dict with frequence of each char
      """
    if not set(word) <= set(string.ascii_letters):
        raise ValueError("Word must contain only ascii")
    return Counter(word)

@validator_custom
def char_rmv(word: stringproc_request_classes.CustomString):
    """
    :param word: Pydantic class which contains a string formed
    by letters and numbers, where the letters are stripped and
    if anoter atribute of class is true then it sums remaining
    numbers and returns
    :return: JSONResponse
    """
    if (len(word.string) > 255 or len(word.string) < 1
            or (not set(word.string) <= set(string.ascii_letters
                                            + string.digits))):
        raise ValueError("Word must be up to 255 characters "
                         "and must be alphanumeric")

    statement = "".join(filter(lambda x: not x.isalpha() , word.string))

    sum1 = None
    if word.calculate_sum:
        sum1 = sum(int(x) for x in statement)

    return {"stripped_strings": "".join(statement),
            "numeric_sum": sum1
            }


import numpy as np
import tensorflow as tf
from model import MyModel
from loss import BiTemeredLoss
from sklearn.preprocessing import OneHotEncoder

def get_training_data(salt_percentage):
  # get minst data
  mnist = tf.keras.datasets.mnist
  (x_train, y_train), (x_test, y_test) = mnist.load_data()
  x_train, x_test = x_train / 255.0, x_test / 255.0

  # add salt
  if salt_percentage > 0:
    np.random.seed(2021)
    shaffle_len = int(salt_percentage*len(y_train))
    shaffle_idxs = np.random.choice(range(len(y_train)), size=shaffle_len, replace=False)
    y_train[shaffle_idxs] = np.random.randint(0, 10, shaffle_len)

  # onehot
  onehot = OneHotEncoder()
  onehot.fit(np.array([y_train.tolist() + y_test.tolist()]).reshape(-1, 1))
  y_train, y_test = onehot.transform(y_train.reshape(-1, 1)).toarray(), onehot.transform(y_test.reshape(-1, 1)).toarray()

  # Add a channels dimension
  x_train = x_train[..., tf.newaxis].astype("float32")
  x_test = x_test[..., tf.newaxis].astype("float32")
  train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(10000).batch(32)
  test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)
  return train_ds, test_ds

model = MyModel()
optimizer = tf.keras.optimizers.Adam()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy')

# define training and testing steps
@tf.function
def train_step(images, labels, loss_object):
  with tf.GradientTape() as tape:
    predictions = model(images, training=True)
    loss = loss_object(labels, predictions)
  gradients = tape.gradient(loss, model.trainable_variables)
  optimizer.apply_gradients(zip(gradients, model.trainable_variables))
  train_loss(loss)
  train_accuracy(labels, predictions)

@tf.function
def test_step(images, labels, loss_object):
  predictions = model(images, training=False)
  t_loss = loss_object(labels, predictions)
  test_loss(t_loss)
  test_accuracy(labels, predictions)

def train_model(train_ds, test_ds, loss_object_train, loss_object_test, EPOCHS=20):
  for epoch in range(EPOCHS):
    train_loss.reset_states()
    train_accuracy.reset_states()
    test_loss.reset_states()
    test_accuracy.reset_states()
    for images, labels in train_ds:
      train_step(images, labels, loss_object_train)
    for test_images, test_labels in test_ds:
      test_step(test_images, test_labels, loss_object_test)
    print(
      f'Epoch {epoch + 1}, '
      f'Loss: {train_loss.result()}, '
      f'Accuracy: {train_accuracy.result() * 100}, '
      f'Test Loss: {test_loss.result()}, '
      f'Test Accuracy: {test_accuracy.result() * 100}'
    )

if __name__ == '__main__':
  import argparse
  parser = argparse.ArgumentParser()
  parser.add_argument("t1", help='t1, between 0 to 1', type=float)
  parser.add_argument("t2", help='t2, between 1 to 10', type=float)
  parser.add_argument("salt_percentage", help='salt_percentage, between 0 to 1', type=float)
  args =  parser.parse_args()

  EPOCHS = 15
  train_ds, test_ds = get_training_data(args.salt_percentage)
  loss_object_ce = tf.keras.losses.CategoricalCrossentropy(from_logits=True) # from logits: without softmax
  if (args.t1 == 1.0) and (args.t2 == 1.0):
    train_model(train_ds, test_ds, loss_object_ce, loss_object_ce, EPOCHS=EPOCHS)
  else:
    loss_object_bi = BiTemeredLoss(t1=args.t1, t2=args.t2, multi=True)
    train_model(train_ds, test_ds, loss_object_bi, loss_object_ce, EPOCHS=EPOCHS)


import os

import pytest

def test_context(monkeypatch, tmp_path):
    initial_wd = os.getcwd()
    with monkeypatch.context() as monkey:  # context mocks works only under its code block
        monkey.setattr(os, "getcwd", lambda: str(tmp_path))
        assert os.getcwd() == str(tmp_path)
    assert os.getcwd() == initial_wd

def test_undo(monkeypatch, tmp_path):
    monkeypatch.delattr("os.getcwd")
    with pytest.raises(AttributeError):
        print(os.getcwd())
    monkeypatch.undo()  # undo previous monkeypatch
    print(os.getcwd())