import mock
from odoo import exceptions

from .common import TestSeBackendCaseBase
from .models import (
    BindingResPartnerFake,
    ResPartnerFake,
    SeAdapterFake,
    SeBackendFake,
)

class TestBindingIndexBase(TestSeBackendCaseBase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        BindingResPartnerFake._test_setup_model(cls.env)
        ResPartnerFake._test_setup_model(cls.env)
        cls._load_fixture("ir_exports_test.xml")
        cls.exporter = cls.env.ref(
            "connector_search_engine.ir_exp_partner_test"
        )

    @classmethod
    def tearDownClass(cls):
        ResPartnerFake._test_teardown_model(cls.env)
        BindingResPartnerFake._test_teardown_model(cls.env)
        super().tearDownClass()

    @classmethod
    def _prepare_index_values(cls, backend=None):
        backend = backend or cls.backend
        return {
            "name": "Partner Index",
            "backend_id": backend.id,
            "model_id": cls.env.ref(
                "connector_search_engine.model_res_partner_binding_fake"
            ).id,
            "lang_id": cls.env.ref("base.lang_en").id,
            "exporter_id": cls.exporter.id,
        }

    @classmethod
    def setup_records(cls, backend=None):
        backend = backend or cls.backend
        # create an index for partner model
        cls.se_index = cls.se_index_model.create(
            cls._prepare_index_values(backend)
        )
        # create a binding + partner alltogether
        cls.binding_model = cls.env[BindingResPartnerFake._name]
        cls.partner_binding = cls.binding_model.create(
            {
                "name": "Marty McFly",
                "country_id": cls.env.ref("base.us").id,
                "email": "marty.mcfly@future.com",
                "child_ids": [
                    (
                        0,
                        0,
                        {"name": "Doc Brown", "email": "docbrown@future.com"},
                    )
                ],
                "index_id": cls.se_index.id,
            }
        )
        cls.partner = cls.partner_binding.record_id

class TestBindingIndexBaseFake(TestBindingIndexBase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        SeAdapterFake._build_component(cls._components_registry)
        SeBackendFake._test_setup_model(cls.env)
        cls.fake_backend_model = cls.env[SeBackendFake._name]
        cls.backend_specific = cls.fake_backend_model.create(
            {"name": "Fake SE"}
        )
        cls.backend = cls.backend_specific.se_backend_id
        cls.setup_records()

    @classmethod
    def tearDownClass(cls):
        SeBackendFake._test_teardown_model(cls.env)
        super().tearDownClass()

class TestBindingIndex(TestBindingIndexBaseFake):

    # TODO: the following `test_backend*` methods
    # should be splitted to a smaller test case.
    # ATM is not possible since the teardown of this class
    # is going to drop the fake models and make any subsequent test case fail.
    # We should find a way to run tear down at the end of ALL test cases.
    def test_backend_specific_select(self):
        self.assertIn(
            self.fake_backend_model._name,
            [x[0] for x in self.env["se.backend"]._select_specific_model()],
        )

    def test_backend_specific_records(self):
        sel = "{},{}".format(
            self.fake_backend_model._name, self.backend_specific.id
        )
        self.assertIn(
            (sel, self.backend_specific.name),
            self.env["se.backend"]._select_specific_backend(),
        )

    def test_backend_defaults(self):
        self.assertEqual(
            self.backend.specific_model, self.backend_specific._name
        )

    def test_backend_unlink(self):
        self.assertTrue(self.backend_specific.exists())
        self.assertTrue(self.backend.exists())
        self.backend_specific.unlink()
        self.assertFalse(self.backend_specific.exists())
        self.assertFalse(self.backend.exists())

    def test_changing_name_with_index_raise_warning(self):
        res = self.backend_specific.onchange_backend_name()
        self.assertIn("warning", res)
        self.backend_specific.index_ids.unlink()
        res = self.backend_specific.onchange_backend_name()
        self.assertIsNone(res)

    def test_changing_model_remove_exporter(self):
        res = self.se_index.onchange_model_id()
        self.assertEqual(len(self.se_index.exporter_id), 0)
        self.assertIn("domain", res)
        self.assertIn("exporter_id", res["domain"])

    def test_get_model_domain(self):
        result = self.se_index._get_model_domain()
        models = result[0][2]
        self.assertIn("res.partner.binding.fake", models)

    def test_recompute_all_indexes(self):
        # on creation indexes are computed and external data stored
        expected = {
            "id": self.partner_binding.id,
            "active": True,
            "lang": "en_US",
            "name": "Marty McFly",
            "credit_limit": 0.0,
            "country_id": {"code": "US", "name": "United States"},
            "color": 0,
            "child_ids": [
                {
                    "id": self.partner.child_ids[0].id,
                    "name": "Doc Brown",
                    "email": "docbrown@future.com",
                    "child_ids": [],
                    "country_id": {"code": "US", "name": "United States"},
                }
            ],
        }
        self.assertEqual(self.partner_binding.get_export_data(), expected)
        self.assertEqual(self.partner_binding.sync_state, "to_update")
        # on index recompute external data must be updated
        self.partner.name = "George McFly"
        expected["name"] = self.partner.name
        self.env["se.index"].recompute_all_index()
        self.assertEqual(self.partner_binding.get_export_data(), expected)
        self.assertEqual(self.partner_binding.sync_state, "to_update")

    def test_force_recompute_all_binding(self):
        with mock.patch.object(
            type(self.se_index), "recompute_all_binding"
        ) as mocked:
            self.se_index.force_recompute_all_binding()
        mocked.assert_called_with(force_export=True)

    def test_force_batch_export(self):
        self.partner_binding.sync_state = "new"
        with mock.patch.object(type(self.se_index), "batch_export") as mocked:
            self.se_index.force_batch_export()
        self.assertEqual(self.partner_binding.sync_state, "to_update")
        mocked.assert_called()

    def test_generate_batch_export_per_index(self):
        with mock.patch.object(type(self.se_index), "batch_export") as mocked:
            self.env["se.index"].generate_batch_export_per_index()
        mocked.assert_called()

    def test_get_domain_for_exporting_binding(self):
        expected = [
            ("index_id", "=", self.se_index.id),
            ("sync_state", "=", "to_update"),
        ]
        self.assertEqual(
            self.se_index._get_domain_for_exporting_binding(), expected
        )

    def test_batch_export(self):
        # state = new, nothing to export
        self.partner_binding.sync_state = "new"
        with mock.patch.object(
            type(self.partner_binding), "synchronize"
        ) as mocked:
            self.se_index.batch_export()
        self.assertEqual(self.partner_binding.sync_state, "new")
        mocked.assert_not_called()
        # now it should find the bindings marked for update and schedule them
        self.partner_binding.sync_state = "to_update"
        with mock.patch.object(
            type(self.partner_binding), "synchronize"
        ) as mocked:
            self.se_index.batch_export()
        self.assertEqual(self.partner_binding.sync_state, "scheduled")
        mocked.assert_called()
        # even if the binding is inactive it should schedule them
        self.partner_binding.sync_state = "to_update"
        self.partner_binding.active = False
        with mock.patch.object(
            type(self.partner_binding), "synchronize"
        ) as mocked:
            self.se_index.batch_export()
        self.assertEqual(self.partner_binding.sync_state, "scheduled")
        mocked.assert_called()

    def test_clear_index(self):
        with SeAdapterFake.mocked_calls() as calls:
            self.se_index.clear_index()
            self.assertEqual(len(calls), 1)
            self.assertEqual(calls[0]["work_ctx"]["index"], self.se_index)
            self.assertEqual(calls[0]["method"], "clear")

    def test_synchronize_active_binding(self):
        # when binding is active it should update it
        with SeAdapterFake.mocked_calls() as calls:
            self.partner_binding.synchronize()
            self.assertEqual(len(calls), 1)
            self.assertEqual(calls[0]["work_ctx"]["index"], self.se_index)
            self.assertEqual(
                calls[0]["work_ctx"]["records"], self.partner_binding
            )
            self.assertEqual(calls[0]["method"], "index")

    def test_synchronize_inactive_binding(self):
        # when binding is inactive it should delete it
        self.partner_binding.active = False
        with SeAdapterFake.mocked_calls() as calls:
            self.partner_binding.synchronize()
            self.assertEqual(len(calls), 1)
            self.assertEqual(calls[0]["work_ctx"]["index"], self.se_index)
            self.assertEqual(
                calls[0]["work_ctx"]["records"], self.partner_binding
            )
            self.assertEqual(calls[0]["method"], "delete")

    def test_unlink_new_binding(self):
        self.partner_binding.sync_state = "new"
        self.partner_binding.unlink()
        self.assertEqual(len(self.partner_binding.exists()), 0)

    def test_unlink_done_inactive_binding(self):
        self.partner_binding.active = False
        self.partner_binding.sync_state = "done"
        self.partner_binding.unlink()
        self.assertEqual(len(self.partner_binding.exists()), 0)

    def test_unlink_to_update_inactive_binding(self):
        self.partner_binding.active = False
        self.partner_binding.sync_state = "to_update"
        with self.assertRaises(exceptions.UserError) as err:
            self.partner_binding.unlink()
        self.assertIn("wait until it's synchronized.", err.exception.name)

    def test_unlink_to_upgrade_active_binding(self):
        self.partner_binding.sync_state = "to_update"
        with self.assertRaises(exceptions.UserError) as err:
            self.partner_binding.unlink()
        self.assertIn("unactivate it first", err.exception.name)

    def test_unlink_done_active_binding(self):
        self.partner_binding.sync_state = "done"
        with self.assertRaises(exceptions.UserError) as err:
            self.partner_binding.unlink()
        self.assertIn("unactivate it first", err.exception.name)

    def test_inactive_binding_change_state(self):
        self.partner_binding.sync_state = "done"
        self.partner_binding.active = False
        self.assertEqual(self.partner_binding.sync_state, "to_update")

    def test_inactive_new_binding_do_not_change_state(self):
        self.partner_binding.sync_state = "new"
        self.partner_binding.active = False
        self.assertEqual(self.partner_binding.sync_state, "new")

    def test_resynchronize_all_bindings(self):
        # The iter method in the mock will return the item id 42 that do not
        # exist as we have dropped all binding
        # the resync method will call the delete to remove this obsolete item
        bindings = self.binding_model.search([])
        bindings.write({"sync_state": "new"})
        bindings.unlink()
        with SeAdapterFake.mocked_calls() as calls:
            self.se_index.resynchronize_all_bindings()
            self.assertEqual(len(calls), 2)
            self.assertEqual(calls[0]["work_ctx"]["index"], self.se_index)
            self.assertEqual(calls[0]["method"], "iter")
            self.assertEqual(calls[1]["work_ctx"]["index"], self.se_index)
            self.assertEqual(calls[1]["method"], "delete")
            self.assertEqual(calls[1]["args"], [42])


from abc import abstractmethod
from dataclasses import dataclass
from typing import Any, List, Optional, Protocol, Sequence, Tuple

@dataclass
class User:
    id: int
    telegram_id: int
    banned: bool
    available_for_broadcast: bool

@dataclass
class Game:
    id: int
    game_id: str
    group_id: int
    message_id: int
    owner_id: int
    owner_name: str
    word: str
    created_at: int
    finished: bool

class Database(Protocol):
    @abstractmethod
    async def _async__init__(self: "Database") -> "Database":
        """Async initializer"""
        raise NotImplementedError

    def __await__(self):
        return self._async__init__().__await__()

    async def open(self: "Database") -> None:
        """Create connection pool"""
        await self

    @abstractmethod
    async def close(self: "Database") -> None:
        """Close connection pool"""
        raise NotImplementedError

    @abstractmethod
    async def sql(
        self: "Database", sql: str, params: Optional[Sequence[Any]] = None
    ) -> Any:
        """Execute plain sql"""
        raise NotImplementedError

    @abstractmethod
    async def get_user_or_create(self: "Database", user_id: int) -> Tuple[User, bool]:
        """Get or create user"""
        raise NotImplementedError

    async def get_users(self: "Database") -> List[User]:
        """Get all users"""
        raise NotImplementedError

    async def create_game(
        self: "Database",
        game_id: str,
        group_id: int,
        owner_id: int,
        owner_name: str,
        word: str,
    ) -> Game:
        """Create new game"""
        raise NotImplementedError

    async def get_game(self: "Database", game_id: str) -> Optional[Game]:
        """Get game"""
        raise NotImplementedError

    async def get_group_game(self: "Database", group_id: int) -> Optional[Game]:
        """Get current group game"""
        raise NotImplementedError

    async def update_game_message(
        self: "Database", game_id: int, new_message_id: int
    ) -> None:
        """Update game message id"""
        raise NotImplementedError

    async def game_finished(
        self: "Database", game_id: int
    ) -> None:
        """Update finished game: delete"""
        raise NotImplementedError

    async def delete_games(
        self: "Database", group_id: int
    ) -> None:
        """Delete all games in group"""
        raise NotImplementedError


from requests import Session
import requests
import os

# NLP_ENGINE_URL = "http://127.0.0.1:5001"
# INTENTS_MANAGMENT_URL = "http://127.0.0.1:5002"
# CONTEXT_MANAGMENT_URL = "http://127.0.0.1:5003"

NLP_ENGINE_URL = os.environ.get('NLP_ENGINE_URL')
INTENTS_MANAGMENT_URL = os.environ.get('INTENTS_MANAGMENT_URL')
CONTEXT_MANAGMENT_URL = os.environ.get('CONTEXT_MANAGMENT_URL')

session = Session()
session.trust_env = False
session.verify = False
session.headers["Accept"] = "application/json"
session.headers["Content-Type"] = "application/json"

def discover_intent(agent, text):
    """
    This method connects to the microservice *NLP Engine* in order to retrieve the intents from a raw text.

    :param agent: The agent name of the chatbot

    :param text: The raw input to discover the intent

    :return: The uri of a intent
    """
    url = NLP_ENGINE_URL + "/intents"
    try:
        r = session.post(url, json={"agent": agent, "sentence": text})
        if r.status_code == 200:
            response = r.json()
            if "intent" in response and len(response["intent"]) > 0:
                return response["intent"][0]["prediction"]
            else:
                return None
    except requests.exceptions.RequestException as e:
        print(e)
        return None

def discover_entities(agent, text):
    """
        This method connects to the microservice *NLP Engine* in order to retrieve the entities from a raw text.

        :param agent: The agent name of the chatbot

        :param text: The raw input to discover the intent

        :return: A list with the URIS of the entities and type of entities.
    """
    url = NLP_ENGINE_URL + "/entities"
    try:
        r = session.post(url, json={"agent": agent, "sentence": text})
        if r.status_code == 200:
            response = r.json()
            entities = []
            if len(response["entities"]) > 0:
                for entity in response["entities"]:
                    entities.append({
                        "type": entity["entity"],
                        "value": entity["prediction"],
                    })
            return entities
    except requests.exceptions.RequestException as e:
        print(e)
        return []

def get_requirements(intent):
    """
    This service connects to the microservice *Intents Managment* in order to retrieve requirements of an intent

    :param intent: The intent from where requirements will be retrieved

    :return: A list of URIS of the different entities needed to complete an intent
    """
    url = INTENTS_MANAGMENT_URL + "/intent/requires"
    try:
        r = session.get(url, json={"intent": intent})
        if r.status_code == 200:
            response = r.json()
            if "requires" in response:
                return response["requires"]
            else:
                return None
    except requests.exceptions.RequestException as e:
        print(e)
        return None

def get_options(entity):
    """
        This service connects to the microservice *Intents Management* in order to retrieve options of an entity type

        :param entity: The entity from where options will be retrieved

        :return: A list of options to complete an entity

        .. todo:: give a list of entities
        """
    url = INTENTS_MANAGMENT_URL + "/entity/options"
    try:
        # print({"entity": entity})
        r = session.get(url, json={"entity": entity})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None

def find_in_context(user, entities):
    """
    This method looks for information in the conversation thread.
     :param user: The id of the user to find information

     :param entities: The entities to be found in context.
    """
    url = CONTEXT_MANAGMENT_URL + "/context/entities"
    try:
        r = session.get(url, json={"user": user, "entities": entities})
        if r.status_code == 200:
            response = r.json()
            if "entities" in response:
                entities = response["entities"]
                return entities
            else:
                return []
    except requests.exceptions.RequestException as e:
        print(e)
        return []

def get_answer(intent, entities):
    """
            This service connects to the microservice *Intents Management* in order to retrieve the answer of an intent.

            :param intent: The intent from where answer will be retrieved

            :param entities: A list of entities used to retrieve answer

            :return: A list of options to complete an entity
            """
    url = INTENTS_MANAGMENT_URL + "/intent/answer"
    try:
        r = session.get(url, json={"intent": intent, "entities": entities})
        if r.status_code == 200:
            response = r.json()
            return response
    except requests.exceptions.RequestException as e:
        print(e)

def get_agent_data(agent):
    """
        This service connects to the microservice *Intents Management* in order to information about and agent

        :param agent: A valid agent name

        :return: A dict containing agent, a description and the different intents.

        """
    url = INTENTS_MANAGMENT_URL + "/agent/info"
    try:
        r = session.get(url, json={"agent": agent})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None

def get_intent_rq(intent, entity):
    """
        This service connects to the microservice *Intents Management* in order to get the resolution question of an intent

        :param intent: A valid intent

        :param entity: A valid entity

        :return: A dict containing intent, a entity and the resolution question.

        """
    url = INTENTS_MANAGMENT_URL + "/intent/rq"
    try:
        r = session.get(url, json={"intent": intent, "entity": entity})
        if r.status_code == 200:
            response = r.json()
            return response

    except requests.exceptions.RequestException as e:
        print(e)
        return None


import requests
from typing import Dict
from bs4 import BeautifulSoup
from time import sleep
import json

baseurl = "https://wiki.lesswrong.com/wiki/RAZ_Glossary"

def download_html() -> str:
    """
    Download the html of the relative url
    """
    response = requests.get(baseurl)
    return response.text

def extract_keyword_def_map(html: str) -> Dict[str, str]:
    """
    Create a keyword -> definition dictionary from the html
    """

    soup = BeautifulSoup(html, 'html.parser')
    ret = {}
    for ul in soup.find_all('ul'):
        for li in ul.find_all('li'):
            title = li.find('b')
            if not title:
                continue
            ret[title.string] = li.decode_contents()
    return ret

def save_as_json(dic: Dict[str, str]) -> None:
    with open('keyword_def_map.json', 'w') as fp:
        json.dump(dic, fp)

if __name__ == "__main__":
    html = download_html()
    dic = extract_keyword_def_map(html)
    save_as_json(dic)


# -*- coding: utf-8 -*-
import tensorflow as tf
import os
import sys
from dataset import Dataset, Vocab
from model import Model
from config import Config
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.logging.set_verbosity(tf.logging.ERROR)

def main():
    config = Config(sys.argv)
    model = Model(config)
    model.build_graph()
    model.initialize_session()

    if config.trn:
        trn = Dataset(config.trn, config.voc_src, config.voc_tgt, config.seq_size, config.max_sents, config.p_unpair, config.p_delete, config.p_extend, config.p_replace, do_shuffle=True)
        dev = Dataset(config.dev, config.voc_src, config.voc_tgt, 0,               0,                config.p_unpair, config.p_delete, config.p_extend, config.p_replace, do_shuffle=False)
        model.learn(trn, dev, config.n_epochs)
    if config.tst:
        tst = Dataset(config.tst, config.voc_src, config.voc_tgt, seq_size=0,      max_sents=0,      p_unpair=0.0,    p_delete=0.0   , p_extend=0.0,    p_replace=0.0,    do_shuffle=False)
        model.inference(tst)

    model.close_session()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# coding: utf-8

# In[278]:

#import pyspark
import pandas as pd
#from pyspark.sql import SparkSession,functions
#from pyspark.sql.types import *
#from pyspark.sql import DataFrame
import re
#spark=SparkSession.builder.appName('learn').getOrCreate()
#from pyspark.sql import regexp_replace
#spark

#!/usr/bin/env python
# coding: utf-8

# In[81]:

from bs4 import BeautifulSoup
from selenium import webdriver
import pandas as pd
import time
import datetime
import calendar

# In[92]:

products=[] #List to store name of the product
prices=[] #List to store price of the product
ratings=[] #List to store rating of the product
from requests_html import HTMLSession
session = HTMLSession()

driver = webdriver.Chrome(r"/Users/muralitulluri/Documents/Dalplex_webscraping/chromedriver")
driver.get("https://www.bestbuy.ca/en-ca/collection/laptops-on-sale/46082?icmp=computing_20211112_laptops_category_feature_slideshow_shopby_save_on_laptops")
time.sleep(10)
for i in range(100):
    driver.find_element_by_class_name("loadMore_3AoXT").click()
    time.sleep(5)

# In[93]:

content = driver.page_source
soup = BeautifulSoup(content,"html.parser")
#soup=BeautifulSoup(response.content, 'html.parser')
#soup
name=[]
price=[]
name=[a.get_text() for a in soup.findAll("div", {"data-automation":"productItemName"})]
price=[a.get_text() for a in soup.findAll("span", {"class":"screenReaderOnly_3anTj large_3aP7Z"})]
save=[a.get_text() for a in soup.findAll("span", {"data-automation":"product-saving"})]

df = pd.DataFrame({'Product_Name':name,'Price':price,'Save':save})
df.to_csv('/Users/muralitulluri/Documents/Best_Buy_Laptops_Black_Friday_Deals.csv', index=False, encoding='utf-8')

#df_spark=spark.read.option('header','true').csv("/Users/muralitulluri/Documents/Best_Buy_Laptops.csv",inferSchema=True)

laptops = pd.read_csv("/Users/muralitulluri/Documents/Best_Buy_Laptops_Black_Friday_Deals.csv")

# In[286]:

#laptops.Product_Name
laptops.Product_Name=laptops.Product_Name.str.replace('2-in-1', '2in1')
laptops.Price=laptops.Price.str.replace('\$', '')
laptops.Save=laptops.Save.str.replace('SAVE \$', '')
laptops.Save=laptops.Save.str.replace(',', '')
laptops['Price']=pd.to_numeric(laptops['Price'])
laptops['Save']=pd.to_numeric(laptops['Save'])
#print(laptop_list)
laptops1=laptops[laptops['Product_Name'].str.contains("Lenovo|HP|Dell")]

Certified_refurbished_laptops1=laptops1[(laptops1['Product_Name'].str.lower()).str.contains("certified refurbished")]
refurbished_laptops1=laptops1[(~(laptops1['Product_Name'].str.lower()).str.contains("certified refurbished")) & ((laptops1['Product_Name'].str.lower()).str.contains("refurbished"))]
OpenBox_laptops1=laptops1[(laptops1['Product_Name'].str.lower()).str.contains("open box")]
new_laptops1=laptops1[(~(laptops1['Product_Name'].str.lower()).str.contains("certified refurbished")) & (~(laptops1['Product_Name'].str.lower()).str.contains("refurbished")) & (~laptops1['Product_Name'].str.contains("Open Box"))]
new_laptops2=new_laptops1[(new_laptops1['Product_Name'].str.contains("8GB RAM|12GB RAM|16GB RAM")) & (new_laptops1['Product_Name'].str.contains("i5|i7")) & (new_laptops1['Product_Name'].str.contains("256GB SSD|512GB SSD"))]

new_openbox_laptops2=OpenBox_laptops1[(OpenBox_laptops1['Product_Name'].str.contains("8GB RAM|12GB RAM|16GB RAM")) & (OpenBox_laptops1['Product_Name'].str.contains("i5|i7")) & (OpenBox_laptops1['Product_Name'].str.contains("256GB SSD|512GB SSD"))]
#new_openbox_laptops2
final_openbox_laptop_list=new_openbox_laptops2[new_openbox_laptops2['Price']<=1100]
final_new_laptop_list=new_laptops2[new_laptops2['Price']<=1100]
x = str(datetime.datetime.now())
filename1='/Users/muralitulluri/Documents/New_Laptops_Best_Buy_'+x+'.xlsx'
final_new_laptop_list.to_excel(filename1,index = False)
filename2='/Users/muralitulluri/Documents/OpenBox_Laptops_Best_Buy_'+x+'.xlsx'
final_openbox_laptop_list.to_excel(filename2,index = False)

# In[245]:

import smtplib,ssl

# Import the email modules we'll need
from email.message import EmailMessage
from os.path import basename
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate

# In[289]:

smtp_server = 'smtp.gmail.com'
smtp_port = 587
#Replace with your own gmail account
gmail = 'tulluri.murali@gmail.com'
password = '$password'
message = MIMEMultipart('mixed')
message['From'] = 'Contact <{sender}>'.format(sender = gmail)
message['To'] = 'jayalex7001@gmail.com'
message['CC'] = 'tulluri.murali@gmail.com'
message['Subject'] = 'Bestbuy NewLaptops and Open Box laptops under $1100 as of '+x

# In[290]:

try:
	with open(filename1, "rb") as attachment:
		p = MIMEApplication(attachment.read(),_subtype="csv")
		p.add_header('Content-Disposition', "attachment; filename= %s" % filename1.split("\\")[-1])
		message.attach(p)
except Exception as e:
	print(str(e))

try:
	with open(filename2, "rb") as attachment:
		p = MIMEApplication(attachment.read(),_subtype="csv")
		p.add_header('Content-Disposition', "attachment; filename= %s" % filename2.split("\\")[-1])
		message.attach(p)
except Exception as e:
	print(str(e))

msg_full = message.as_string()
context = ssl.create_default_context()
with smtplib.SMTP(smtp_server, smtp_port) as server:
	server.ehlo()
	server.starttls(context=context)
	server.ehlo()
	server.login(gmail, password)
	server.sendmail(gmail, message['To'].split(";") + (message['CC'].split(";") if message['CC'] else []),msg_full)
	server.quit()

print("email sent out successfully")

# In[ ]:


import os

import web3
from etherscan.contracts import Contract as EtherscanContract

w3_infura = web3.Web3(
    web3.Web3.HTTPProvider(
        f"https://mainnet.infura.io/v3/{os.environ['WEB3_INFURA_PROJECT_ID']}",
    ),
)

def init_contract(address: str):

    address = web3.Web3.toChecksumAddress(address)
    contract_abi = EtherscanContract(
        address=address,
        api_key=os.environ["ETHERSCAN_API_KEY"],
    ).get_abi()
    contract = w3_infura.eth.contract(address=address, abi=contract_abi)
    return contract


import os
import sys
import zipfile

#获取脚本路径, 如果是在IDE下运行，只能用此方式获取
def get_cur_dir():
    path = sys.path[0]
    #判断为脚本文件还是py2exe编译后的文件，如果是脚本文件，则返回的是脚本的目录，如果是py2exe编译后的文件，则返回的是编译后的文件路径
    if os.path.isdir(path):
        return path

    elif os.path.isfile(path):
        return os.path.dirname(path)

def recurse_files(input_path, result):
    files = os.listdir(input_path)

    for file in files:
        if os.path.isdir(os.path.join(input_path,file)):
            if file != 'users': #过滤文件夹
                recurse_files(os.path.join(input_path,file), result)
        else:
            ext = os.path.splitext(file)[1]
            if input_path == cur_dir:#根目录只添加特定类型文件
                if ext == '.dll' or ext == '.exe' or ext == '.qm':
                    result.append(os.path.join(input_path,file))
            else:
                result.append(os.path.join(input_path,file))

if __name__ == '__main__':
    cur_dir = get_cur_dir()

    zip_file_name = os.path.join(cur_dir, 'debug.zip');
    if os.path.isfile(zip_file_name): os.remove(zip_file_name)
    zip = zipfile.ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED)

    filelists = []
    recurse_files(cur_dir, filelists)

    for file in filelists:
        arcName = file #在压缩包内名字
        if os.path.dirname(file) == cur_dir:
            arcName = os.path.basename(file) #文件在要压缩目录下，则只为文件名
        else:
            dir = os.path.dirname(file)
            dir = dir.replace(cur_dir + '\\', '') #只从要压缩的目录下当前目录名开始
            arcName = dir + '\\' + os.path.basename(file)

        zip.write(file, arcName)
        #zip.write(file) #会出现全路径

    # 调用了close方法才会保证完成压缩
    zip.close()


import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import entropy
from scipy.integrate import simps
from sklearn.neighbors import KernelDensity
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import LeaveOneOut, KFold

# error pdfs #
if False:
    #d1 = pd.read_csv('./d1.dat')
    #d2 = pd.read_csv('./collected.csv')
    #d3 = pd.read_csv('./timedsamples.dat')
    #dat = pd.concat([d1, d2, d3])
    hfDs, cvDs = [], []
    dat = pd.read_csv('./collected.csv')
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    f, ax = plt.subplots(2)
    plt.subplots_adjust(hspace=0.5)
    TEST = 50
    LF = 300
    bigBS = int(dat.shape[0] / (LF))
    dats = np.array_split(dat, bigBS)
    samplesx = [125]
    print('*', bigBS)
    while samplesx[-1]*2 < bigBS:
        samplesx.append(samplesx[-1] * 2)
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(samplesx)))
    print(samplesx)
    bss = []
    for ll, BS in enumerate(samplesx):
        bss.append(BS)
        LW = .1
        ALPHA = 0.5
        x = np.array([int(s) for s in np.arange(10, TEST+1, 2)])
        epsh = np.zeros((x.size, BS))
        epscv = np.zeros((x.size, BS))
        for _ in range(BS):
            dat = dats[_]
            def mch(x):
                 return (np.mean(dat.hf[TEST:]) - np.mean(dat.hf.values[:x])) ** 2, np.var(dat.hf.values[:x]) / x, np.mean(dat.hf.values[:x]), np.mean(dat.hf[TEST:])
            mch = np.vectorize(mch)

            def cv2(x):
                alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
                m = (np.mean(dat.hf[TEST:]) - (
                    np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf[:LF]) - np.mean(dat.lf.values[:x])))) ** 2
                std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
                return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
            cv2 = np.vectorize(cv2)

            ev_mch = mch(x)
            ev_cv2 = cv2(x)
            epsh[:, _] = np.sqrt(ev_mch[0])
            epscv[:, _] = np.sqrt(ev_cv2[0])
            if len(ev_cv2[0][np.isnan(ev_cv2[0])]) > 0: hey

        xs, ys = [], []
        y2s = []
        lastpdfhf, lastpdfcv = None, None
        for ii in [np.where(x==TEST)[0][0]]:
            thesex, thesey, vesey = [], [], []
            for jj in range(epsh.shape[1]):
                ys.append(epsh[ii, jj])
                y2s.append(epscv[ii, jj])
                xs.append(x[ii])
                thesey.append(y2s[-1])
                vesey.append(ys[-1])
            np.save('thesey%i'%BS, thesey)
            thesey = np.array(thesey)
            vesey = np.array(vesey)
            xx = np.linspace(np.min(vesey), max(vesey), 100)
            #xx = np.linspace(min(vesey), max(vesey), 100)
            print('---->', min(thesey), max(thesey))
            kde2 = KernelDensity(kernel='gaussian', bandwidth=.05)
            kde2.fit(vesey.reshape(-1, 1))
            pdff = np.exp(kde2.score_samples(xx.reshape(-1, 1)))
            if lastpdfcv: cvDs.append(entropy(pdff, laspdfcv))
            lastpdfcv = pdff.copy()
            ax[0].plot(xx, pdff / simps(pdff, xx), label=BS, c=colors[ll])
            #xx = np.linspace(0, 2 * max(thesey), 1000)
            xx = np.linspace(min(thesey), max(thesey), 100)
            kde = KernelDensity(kernel='gaussian', bandwidth=.01)
            kde.fit(thesey.reshape(-1, 1))
            pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
            if lastpdfhf: hfDs.append(entropy(pdff, laspdfhf))
            lastpdfhf = pdff.copy()
            ax[1].plot(xx, pdff / simps(pdff, xx), label=BS, c=colors[ll])
    ax[0].hist(vesey, 200, alpha=0.3, normed=True)
    ax[1].hist(thesey, 200, alpha=0.3, normed=True)
    ax[0].set_xlabel(r'$\epsilon(Q^{HF})$')
    ax[1].set_xlabel(r'$\epsilon(Q^{CV})$')
    l = ax[0].legend(ncol=3)
    l.set_title('Sample Sets')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    f.suptitle('%i sets, %i HF samples, %i LF samples'%(bigBS, TEST, LF))
    f.savefig('sampleContours.pdf', bbox_inches='tight') ; plt.clf()
    plt.clf()
    plt.close()

    plt.plot(bss[1:], lastpdfhf, label='HF')
    plt.plot(bss[1:], lastpdfcv, label='CV')
    plt.yscale('log')
    plt.xscale('log')
    plt.legend()
    plt.ylabel('D')
    plt.xlabel('n')
    plt.show()

# raw values
if True:
    d1 = pd.read_csv('./d1.dat')
    d2 = pd.read_csv('./collected.csv')
    d3 = pd.read_csv('./timedsamples.dat')
    dat = pd.concat([d1, d2, d3])
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    #dat = dat.convert_objects(convert_numeric=True)
    f, ax = plt.subplots(2, sharex=True)
    #plt.subplots_adjust(bottom=-2, hspace=1)
    SAMPS = [50]
    while SAMPS[-1] < 30000: SAMPS.append(SAMPS[-1] * 2)
    BINS = 100
    HFENTROPIES = []
    LFENTROPIES = []
    lflastp = None
    hflastp = None
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(SAMPS)))
    for ll, n in enumerate(SAMPS):
        HF = dat.hf[:n]
        LF = dat.lf[:n]
        xx = np.linspace(10, 26, 1000)
        #bandwidths = 10 ** np.linspace(-1, 1, 100)
        #print(n)
        #grid = GridSearchCV(KernelDensity(kernel='cosine'),
        #            {'bandwidth': bandwidths},
        #            cv=KFold(5))
        #grid.fit(HF[:, None])
        kde = KernelDensity(kernel='gaussian', bandwidth=.1)
        kde.fit(HF[:, None])
        myfit = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        myfit /= simps(myfit, xx)
        if hflastp is not None: HFENTROPIES.append(entropy(myfit, hflastp))
        hflastp = myfit.copy()
        xx = np.linspace(11, 19, 1000)
        ax[0].plot(xx, np.exp(kde.score_samples(xx.reshape(-1, 1))), c=colors[ll], label=n)
        #xx = np.linspace(min(LF), max(LF), 100)
        kde = KernelDensity(kernel='gaussian', bandwidth=.1)
        kde.fit(LF.reshape(-1, 1))
        pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        pdff /= simps(pdff, xx)
        ax[1].plot(xx, pdff, c=colors[ll])
        if lflastp is not None: LFENTROPIES.append(entropy(pdff, lflastp))
        lflastp = pdff.copy()
    l = ax[0].legend(ncol=4, bbox_to_anchor=[.9, 1.5])
    l.set_title('Number of Samples')
    ax[0].hist(HF, 50, alpha=0.6, normed=True)
    ax[1].hist(LF, 50, alpha=0.6, normed=True)
    ax[1].set_xlabel(r'$\max(\nabla u)_{400 Cells}$')
    ax[0].set_xlabel(r'$\max(\nabla u)_{7000 Cells}$')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    plt.savefig('./rawPDFs.pdf', bbox_inches='tight')
    plt.clf()
    plt.close()

    plt.plot(SAMPS[1:], np.abs(HFENTROPIES))
    plt.plot(SAMPS[1:], np.abs(LFENTROPIES))
    plt.ylabel('D')
    plt.ylabel('n')
    plt.yscale('log')
    plt.xscale('log')
    plt.savefig('./pdfDees.pdf')

# expected values #
if False:

    d1 = pd.read_csv('./d1.dat')
    d2 = pd.read_csv('./collected.csv')
    d3 = pd.read_csv('./timedsamples.dat')
    dat = pd.concat([d1, d2, d3])
    dat.rename(columns={'nx7000':'hf', 'nx400':'lf', 'nu':'visc'}, inplace=1)
    TEST = 10
    LF = 109
    bigBS = int(dat.shape[0] / (TEST * LF))
    dats = np.array_split(dat, bigBS)
    f, ax = plt.subplots(2, sharex=True)
    SAMPS = [50]
    while SAMPS[-1] * 2 < len(dats): SAMPS.append(SAMPS[-1]*2)
    #plt.subplots_adjust(bottom=-2, hspace=1)
    colors = plt.cm.coolwarm(np.linspace(0, 1, len(SAMPS)))
    for ll, n in enumerate(SAMPS):
        exps = []
        CC = []
        for ii in range(n):
            exps.append(dats[ii].hf[:TEST].mean())
            dat = dats[ii]
            def cv2(x):
                alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
                m = (np.mean(dat.hf[TEST:]) - (
                    np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf[:LF*x]) - np.mean(dat.lf.values[:x])))) ** 2
                std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
                return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
            CC.append(cv2(50)[-1])
        xx = np.linspace(14, 18, 1000)
        #xx = np.linspace(.9 * np.min(exps), 1.1 * np.max(exps), 1000)
        kde = KernelDensity(kernel='gaussian', bandwidth=1e-1)
        kde.fit(np.array(exps).reshape(-1, 1))
        myfit = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        myfit /= simps(myfit, xx)
        ax[0].plot(xx, myfit, c=colors[ll], label=n, zorder=3)
        #xx = np.linspace(min(LF), max(LF), 100)
        kde = KernelDensity(kernel='gaussian', bandwidth=5e-2)
        kde.fit(np.array(CC).reshape(-1, 1))
        pdff = np.exp(kde.score_samples(xx.reshape(-1, 1)))
        ax[1].plot(xx, pdff / simps(pdff, xx), c=colors[ll], zorder=4)
    leg = ax[0].legend(ncol=2, bbox_to_anchor=[1, 1])
    leg.set_title("Number of sample sets")
    ax[0].hist(exps, 30, normed=True, alpha=0.8, zorder=1)
    ax[1].hist(CC, 30, normed=True, alpha=0.8, zorder=2)
    ax[0].set_xlabel(r'$Q^{HF}$')
    ax[1].set_xlabel(r'$Q^{CV}$')
    ax[0].set_ylabel('probability density')
    ax[1].set_ylabel('probability density')
    plt.suptitle('%i sets, %i LF samples'%(bigBS, dat.shape[0]), y=1.)
    plt.savefig('./qPDFs.pdf', bbox_inches='tight')
    plt.clf()
    plt.close()

    exps = []
    CC = []
    NSAMPS = []
    for kk in [10, 20, 50]:
      for ii in range(n):
        exps.append(dats[ii].hf[:kk].mean())
        dat = dats[ii]
        def cv2(x):
            alpha = np.corrcoef(dat.hf.values[:x], dat.lf.values[:x])[0][1] * np.sqrt(np.var(dat.hf.values[:x]) / np.var(dat.lf.values[:x]))
            m = (np.mean(dat.hf[TEST:]) - ( np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x])))) ** 2
            std = np.var(dat.hf.values[:x]) / x + (1. / float(x) - 1. /float(len(dat.lf))) * (np.var(dat.lf) * alpha ** 2 - 2 * alpha * np.corrcoef(dat.lf.values[:x], dat.hf.values[:x])[0][1] * np.sqrt(np.var(dat.lf) * np.var(dat.hf[:x])))
            return m, std, alpha, np.mean(dat.hf.values[:x]) + alpha * (np.mean(dat.lf) - np.mean(dat.lf.values[:x]))
        CC.append(cv2(kk)[-1])
        NSAMPS.append(kk)

    vdat = pd.DataFrame()
    vdat['NSAMPS'] = NSAMPS
    vdat['HF'] = exps
    vdat['CV'] = CC
    f, ax = plt.subplots(2)
    plt.subplots_adjust(right=1.5)
    sns.violinplot(x='NSAMPS', y='HF', data=vdat, ax=ax[0], color='white')
    sns.violinplot(x='NSAMPS', y='CV', data=vdat, ax=ax[1], color='white')
    for ii in range(2):
        ax[ii].set_xlabel("Number of High Fidelity Samples")
        ax[ii].set_ylim(vdat.HF.min(), vdat.HF.max())
    ax[0].set_ylabel(r'$Q^{HF}$')
    ax[1].set_ylabel(r'$Q^{CV}$')
    plt.suptitle('%i sets, %i LF samples'%(bigBS, dat.shape[0]), y=1.)
    plt.tight_layout()
    plt.savefig('expectedViolins.pdf', bbox_inches='tight')


from __future__ import annotations

from typing import TYPE_CHECKING, Iterable

from public import public

import ibis.expr.operations as ops
from ibis.expr.types.generic import Column, Scalar, Value, literal
from ibis.expr.types.typing import V

if TYPE_CHECKING:
    import ibis.expr.datatypes as dt
    import ibis.expr.types as ir

import ibis.common.exceptions as com

@public
class ArrayValue(Value):
    def length(self) -> ir.IntegerValue:
        """Compute the length of an array.

        Returns
        -------
        IntegerValue
            The integer length of `self`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2, 3])
        >>> a.length()
        ArrayLength((1, 2, 3))
        """
        return ops.ArrayLength(self).to_expr()

    def __getitem__(
        self,
        index: int | ir.IntegerValue | slice,
    ) -> ir.Value:
        """Extract one or more elements of `self`.

        Parameters
        ----------
        index
            Index into `array`

        Returns
        -------
        Value
            - If `index` is an [`int`][int] or
              [`IntegerValue`][ibis.expr.types.IntegerValue] then the return
              type is the element type of `self`.
            - If `index` is a [`slice`][slice] then the return type is the same
              type as the input.

        Examples
        --------
        Extract a single element

        >>> import ibis
        >>> x = ibis.array([1, 2, 3, 4])
        >>> x[2]
        ArrayIndex((1, 2, 3, 4), index=2)

        Extract a range of elements

        >>> import ibis
        >>> x = ibis.array([1, 2, 3, 4])
        >>> x[1:3]
        ArraySlice((1, 2, 3, 4), start=1, stop=3)
        """
        if isinstance(index, slice):
            start = index.start
            stop = index.stop
            step = index.step

            if step is not None and step != 1:
                raise NotImplementedError('step can only be 1')

            op = ops.ArraySlice(self, start if start is not None else 0, stop)
        else:
            op = ops.ArrayIndex(self, index)
        return op.to_expr()

    def __add__(self, other: ArrayValue) -> ArrayValue:
        """Concatenate this array with another.

        Parameters
        ----------
        other
            Array to concat with `self`

        Returns
        -------
        ArrayValue
            `self` concatenated with `other`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2])
        >>> b = ibis.array([3, 4, 5])
        >>> a + b
        ArrayConcat(left=(1, 2), right=(3, 4, 5))
        """
        return ops.ArrayConcat(self, other).to_expr()

    def __radd__(self, other: ArrayValue) -> ArrayValue:
        """Concatenate this array with another.

        Parameters
        ----------
        other
            Array to concat with `self`

        Returns
        -------
        ArrayValue
            `self` concatenated with `other`

        Examples
        --------
        >>> import ibis
        >>> a = ibis.array([1, 2])
        >>> b = ibis.array([3, 4, 5])
        >>> b + a
        ArrayConcat(left=(3, 4, 5), right=(1, 2))
        """
        return ops.ArrayConcat(self, other).to_expr()

    def __mul__(self, n: int | ir.IntegerValue) -> ArrayValue:
        """Repeat this array `n` times.

        Parameters
        ----------
        n
            Number of times to repeat `self`.

        Returns
        -------
        ArrayValue
            `self` repeated `n` times

        Examples
        --------
        >>> import ibis
        >>> x = ibis.array([1, 2])
        >>> x * 2
        ArrayRepeat((1, 2), times=2)
        """
        return ops.ArrayRepeat(self, n).to_expr()

    def __rmul__(self, n: int | ir.IntegerValue) -> ArrayValue:
        """Repeat this array `n` times.

        Parameters
        ----------
        n
            Number of times to repeat `self`.

        Returns
        -------
        ArrayValue
            `self` repeated `n` times

        Examples
        --------
        >>> import ibis
        >>> x = ibis.array([1, 2])
        >>> 2 * x
        ArrayRepeat((1, 2), times=2)
        """
        return ops.ArrayRepeat(self, n).to_expr()

    def unnest(self) -> ir.Value:
        """Unnest an array.

        Returns
        -------
        ir.Value
            Unnested array
        """
        expr = ops.Unnest(self).to_expr()
        try:
            return expr.name(self.get_name())
        except com.ExpressionError:
            return expr

@public
class ArrayScalar(Scalar, ArrayValue):
    pass

@public
class ArrayColumn(Column, ArrayValue):
    pass

@public
def array(
    values: Iterable[V],
    type: str | dt.DataType | None = None,
) -> ArrayValue:
    """Create an array expression.

    If the input expressions are all column expressions, then the output will
    be an `ArrayColumn`. The input columns will be concatenated row-wise to
    produce each array in the output array column. Each array will have length
    _n_, where _n_ is the number of input columns. All input columns should be
    of the same datatype.

    If the input expressions are Python literals, then the output will be a
    single `ArrayScalar` of length _n_, where _n_ is the number of input
    values. This is equivalent to

    ```python
    values = [1, 2, 3]
    ibis.literal(values)
    ```

    Parameters
    ----------
    values
        An iterable of Ibis expressions or a list of Python literals
    type
        An instance of `ibis.expr.datatypes.DataType` or a string indicating
        the ibis type of `value`.

    Returns
    -------
    ArrayValue
        An array column (if the inputs are column expressions), or an array
        scalar (if the inputs are Python literals)

    Examples
    --------
    Create an array column from column expressions
    >>> import ibis
    >>> t = ibis.table([('a', 'int64'), ('b', 'int64')], name='t')
    >>> result = ibis.array([t.a, t.b])

    Create an array scalar from Python literals
    >>> import ibis
    >>> result = ibis.array([1.0, 2.0, 3.0])
    """
    if all([isinstance(value, Column) for value in values]):
        return ops.ArrayColumn(values).to_expr()
    elif any([isinstance(value, Column) for value in values]):
        raise com.IbisTypeError(
            'To create an array column using `array`, all input values must '
            'be column expressions.'
        )
    else:
        try:
            return literal(list(values), type=type)
        except com.IbisTypeError as e:
            raise com.IbisTypeError(
                'Could not create an array scalar from the values provided '
                'to `array`. Ensure that all input values have the same '
                'Python type, or can be casted to a single Python type.'
            ) from e


# python
import datetime

# django
from django.core import mail

from djangopeople.unit_tests.testbase import TestCase
from welcome.models import WelcomeEmail

class ModelTestCase(TestCase):

    def test_basic_creation(self):
        user, person = self._create_person('bob', 'bob@example.com')
        welcome_email = WelcomeEmail.objects.create(user=user,
                                                    subject='Test',
                                                    body='Test body')
        self.assertEqual(welcome_email.email, 'bob@example.com')
        self.assertEqual(welcome_email.send_date, None)
        welcome_email.send()

        welcome_email = WelcomeEmail.objects.get(user=user)
        self.assertNotEqual(welcome_email.send_date, None)
        self.assertTrue(welcome_email.send_date <= datetime.datetime.now())


from typing import List

import fvcore.nn.weight_init as weight_init
import torch
from torch import nn
from torch.nn import functional as F
from einops import rearrange
from detectron2.config import configurable
from detectron2.layers import Conv2d, ShapeSpec, cat, get_norm
from detectron2.modeling import ROI_MASK_HEAD_REGISTRY
from detectron2.modeling.roi_heads.mask_head import BaseMaskRCNNHead
from detectron2.structures import Instances
# from .mask_encoding_V2 import DctMaskEncodingV2
from .mask_encoding import DctMaskEncoding

@ROI_MASK_HEAD_REGISTRY.register()
class MaskRCNNPatchDCTHead_NSTAGE(BaseMaskRCNNHead):
    """
    A mask head with several conv layers, plus an upsample layer (with `ConvTranspose2d`).
    Predictions are made with a final 1x1 conv layer.
    """

    @configurable
    def __init__(self, input_shape: ShapeSpec, *, num_classes, conv_dims, conv_norm="",
                 dct_vector_dim, mask_size, pooler_resolution, hidden_features, fine_features_resolution,
                 mask_size_assemble, patch_size, patch_dct_vector_dim, mask_loss_para, dct_loss_type,
                 num_stage, mask_loss_para_each_stage, patch_threshold, eval_gt,
                 **kwargs):
        """
        NOTE: this interface is experimental.

        Args:
            input_shape (ShapeSpec): shape of the input feature
            num_classes (int): the number of classes. 1 if using class agnostic prediction.For COCO,num_classes=80
            conv_dims (list[int]): a list of N>0 integers representing the output dimensions
                of N-1 conv layers and the last upsample layer.
            conv_norm (str or callable): normalization for the conv layers.
                See :func:`detectron2.layers.get_norm` for supported types.
            dct_vector_dim: dct vector dim in DCT_MASK(default=300)
            mask_size: resolution of mask to be refined(default=112)
            hidden_features: feature dim of linear layer(default=1024)
            fine_features_resolution: feature map in PatchDCT(default=42)
            mask_size_assemble: mask size in PatchDCT(default=112)
            patch_size: patch size(default=8)
            patch_dct_vector_dim: DCT vector dim for each patch
            mask_loss_para: coefficient of total loss(default=1)
            dct_loss_type: loss type of DCT vector regressor(default=l1, option=[l1, sl1,l2])
            num_stage: number of segmentation stage, equals to number of PatchDCT blocks+1(default=2)
            mask_loss_para_each_stage: coefficient of loss for each segmentation stage
            patch_threshold: threshold used for classifier(default=0.3),
            eval_gt: use for calculate the upper bound of the model

        """
        super().__init__(**kwargs)
        assert len(conv_dims) >= 1, "conv_dims have to be non-empty!"
        assert num_stage == len(mask_loss_para_each_stage)
        self.patch_dct_vector_dim = patch_dct_vector_dim
        self.mask_size_assemble = mask_size_assemble
        self.patch_size = patch_size
        self.num_classes = num_classes
        self.hidden_features = hidden_features
        self.dct_vector_dim = dct_vector_dim
        self.mask_size = mask_size
        self.dct_loss_type = dct_loss_type
        self.mask_loss_para = mask_loss_para
        self.scale = self.mask_size // self.patch_size
        self.ratio = fine_features_resolution // self.scale
        self.num_stage = num_stage - 1  # num_stage below is the number of PatchDCT block
        self.loss_para = mask_loss_para_each_stage
        self.patch_threshold = patch_threshold
        self.eval_gt = eval_gt
        self.dct_encoding = DctMaskEncoding(vec_dim=self.dct_vector_dim, mask_size=self.mask_size)
        self.patch_dct_encoding = DctMaskEncoding(vec_dim=self.patch_dct_vector_dim, mask_size=self.patch_size)

        self.conv_norm_relus = []

        cur_channels = input_shape.channels
        for k, conv_dim in enumerate(conv_dims[:-1]):
            conv = Conv2d(
                cur_channels,
                conv_dim,
                kernel_size=3,
                stride=1,
                padding=1,
                bias=not conv_norm,
                norm=get_norm(conv_norm, conv_dim),
                activation=F.relu,
            )
            self.add_module("mask_fcn{}".format(k + 1), conv)
            self.conv_norm_relus.append(conv)
            cur_channels = conv_dim

        self.predictor = nn.Sequential(
            nn.Linear(pooler_resolution ** 2 * conv_dim, self.hidden_features),
            nn.ReLU(),
            nn.Linear(self.hidden_features, self.hidden_features),
            nn.ReLU(),
            nn.Linear(self.hidden_features, self.dct_vector_dim)
        )
        self.reshape = Conv2d(
            1,
            conv_dim,
            kernel_size=3,
            stride=1,
            padding=1,
            activation=F.relu
        )
        self.fusion = nn.Sequential(
            Conv2d(cur_channels,
                   conv_dim,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu),
            Conv2d(cur_channels,
                   conv_dim,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu)
        )

        self.downsample = nn.Sequential(
            Conv2d(
                cur_channels,
                self.hidden_features,
                kernel_size=self.ratio,
                stride=self.ratio,
                padding=0,
                bias=not conv_norm,
                norm=get_norm(conv_norm, conv_dim),
                activation=F.relu, ),
            Conv2d(self.hidden_features,
                   self.hidden_features,
                   kernel_size=3,
                   stride=1,
                   padding=1,
                   bias=not conv_norm,
                   norm=get_norm(conv_norm, conv_dim),
                   activation=F.relu),
        )

        self.predictor1 = Conv2d(self.hidden_features,
                                 self.patch_dct_vector_dim * self.num_classes,
                                 kernel_size=1,
                                 stride=1,
                                 padding=0,
                                 )
        self.predictor_bfg = Conv2d(self.hidden_features,
                                    3 * self.num_classes,
                                    kernel_size=1,
                                    stride=1,
                                    padding=0,
                                    )

        for layer in self.conv_norm_relus:
            weight_init.c2_msra_fill(layer)

    @classmethod
    def from_config(cls, cfg, input_shape):
        ret = super().from_config(cfg, input_shape)
        conv_dim = cfg.MODEL.ROI_MASK_HEAD.CONV_DIM
        num_conv = cfg.MODEL.ROI_MASK_HEAD.NUM_CONV
        ret.update(
            conv_dims=[conv_dim] * (num_conv + 1),  # +1 for ConvTranspose
            conv_norm=cfg.MODEL.ROI_MASK_HEAD.NORM,
            input_shape=input_shape,
            hidden_features=cfg.MODEL.ROI_MASK_HEAD.HIDDEN_FEATURES,
            pooler_resolution=cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION,
            dct_vector_dim=cfg.MODEL.ROI_MASK_HEAD.DCT_VECTOR_DIM,
            mask_loss_para=cfg.MODEL.ROI_MASK_HEAD.MASK_LOSS_PARA,
            mask_size=cfg.MODEL.ROI_MASK_HEAD.MASK_SIZE,
            dct_loss_type=cfg.MODEL.ROI_MASK_HEAD.DCT_LOSS_TYPE,
            fine_features_resolution=cfg.MODEL.ROI_MASK_HEAD.FINE_FEATURES_RESOLUTION,
            mask_size_assemble=cfg.MODEL.ROI_MASK_HEAD.MASK_SIZE_ASSEMBLE,
            patch_size=cfg.MODEL.ROI_MASK_HEAD.PATCH_SIZE,
            patch_dct_vector_dim=cfg.MODEL.ROI_MASK_HEAD.PATCH_DCT_VECTOR_DIM,
            num_stage=cfg.MODEL.ROI_MASK_HEAD.NUM_STAGE,
            mask_loss_para_each_stage=cfg.MODEL.ROI_MASK_HEAD.MASK_LOSS_PARA_EACH_STAGE,
            patch_threshold=cfg.MODEL.ROI_MASK_HEAD.PATCH_THRESHOLD,
            eval_gt=cfg.MODEL.ROI_MASK_HEAD.EVAL_GT,
        )

        if cfg.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK:
            ret["num_classes"] = 1
        else:
            ret["num_classes"] = cfg.MODEL.ROI_HEADS.NUM_CLASSES
        return ret

    def layers(self, x, fine_mask_features, instances):
        """

        Args:
            x: feature map used in DCT-Mask
            fine_mask_features: feature map used in PatchDCT

        Returns:
            x (Tensor): [B, D]. D is dct-dim. [B, D]. DCT_Vector in DCT-Mask.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
        """
        for layer in self.conv_norm_relus:
            x = layer(x)
        # DCT_Mask
        x = self.predictor(x.flatten(start_dim=1))
        if not self.training:
            num_masks = x.shape[0]
            if num_masks == 0:
                return x, {}, {}

        # reverse transform to obtain high-resolution masks
        # if use DctMaskEncodingV2, plese use: masks = self.dct_encoding.decode(x)
        masks = self.dct_encoding.decode(x).real
        masks = masks[:, None, :, :]

        # PatchDCT
        if self.training:
            classes = self.get_gt_classes(instances)
        else:
            classes = instances[0].pred_classes
        num_instance = classes.size()[0]
        indices = torch.arange(num_instance)

        bfg, patch_vector = self.patchdct(fine_mask_features, masks, classes, indices)

        bfg_dict = {}
        patch_dict = {}

        bfg_dict[0] = bfg
        patch_dict[0] = patch_vector

        for i in range(1, self.num_stage):
            masks = self.patch2mask_nstage(bfg, patch_vector)
            bfg, patch_vector = self.patchdct(fine_mask_features, masks, classes, indices)
            bfg_dict[i] = bfg
            patch_dict[i] = patch_vector

        return x, bfg_dict, patch_dict

    def patchdct(self, fine_mask_features, masks, classes, indices):
        """
        PatchDCT block
        Args:
            fine_mask_features: feature map cropped from FPN P2
            masks: masks to be refined
            classes: a tensor of shape [B], classes of each instance
            indices: a tensor of shape [B]

        Returns:
            bfg and patch_vector of each PatchDCT block
        """
        masks = F.interpolate(masks, size=(self.scale * self.ratio, self.scale * self.ratio))
        masks = self.reshape(masks)
        fine_mask_features = masks + fine_mask_features
        fine_mask_features = self.fusion(fine_mask_features)
        fine_mask_features = self.downsample(fine_mask_features)
        patch_vector = self.predictor1(fine_mask_features)
        bfg = self.predictor_bfg(fine_mask_features)

        bfg = rearrange(bfg, "b (n c) h w -> b n c h w", n=self.num_classes)
        patch_vector = rearrange(patch_vector, "b (n c) h w -> b n c h w", n=self.num_classes)

        bfg = rearrange(bfg[indices, classes], "b c h w -> (b h w) c")
        patch_vector = rearrange(patch_vector[indices, classes], "b c h w -> (b h w) c")
        return bfg, patch_vector

    def patch2mask_nstage(self, bfg, patch_vector):
        """

        Args:
            bfg: A tensor of shape [B*scale*scale,3]
            patch_vector: A tensor of shape [B*scale*scale,patch_dct_vector_dim]

        Returns:
            masks: A tensor of shape [B,1,mask_size,mask_size]
        """
        index = torch.argmax(bfg, dim=1)
        bg = torch.zeros_like(patch_vector, device=bfg.device)
        bg[index == 1] = 1
        fg = torch.zeros_like(patch_vector, device=bfg.device)
        fg[index == 2, 0] = self.patch_size
        patch_vector = patch_vector * bg + fg
        # if use DctMaskEncodingV2, plese use: patch_masks = self.patch_dct_encoding.decode(patch_vector)
        patch_masks = self.patch_dct_encoding.decode(patch_vector).real
        masks = rearrange(patch_masks, "(b s1 s2) p1 p2 -> b (s1 p1) (s2 p2)", s1=self.scale, s2=self.scale)
        return masks[:, None, :, :]

    def forward(self, x, fine_mask_features, instances: List[Instances]):
        """
        Args:
            x: input region feature(s) provided by :class:`ROIHeads`.
            fine_mask_features: features cropped from FPN-P2
            instances (list[Instances]): contains the boxes & labels corresponding
                to the input features.
                Exact format is up to its caller to decide.
                Typically, this is the foreground instances in training, with
                "proposal_boxes" field and other gt annotations.
                In inference, it contains boxes that are already predicted.

        Returns:
            A dict of losses in training. The predicted "instances" in inference.
        """
        x, bfg_dict, patch_dict = self.layers(x, fine_mask_features, instances)
        if self.training:
            return {"loss_mask": self.mask_rcnn_dct_loss(x, bfg_dict, patch_dict, instances, self.vis_period)}
        else:
            pred_instances = self.mask_rcnn_dct_inference(x, bfg_dict, patch_dict, instances)
            return pred_instances

    def mask_rcnn_dct_loss(self, pred_mask_logits, bfg_dict, patch_dict, instances, vis_period=0):
        """
        Compute the mask prediction loss defined in the PatchDCT paper.

        Args:
            pred_mask_logits (Tensor): [B, D]. D is dct-dim. [B, D]. DCT_Vector in DCT-Mask.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
            instances (list[Instances]): A list of N Instances, where N is the number of images
                in the batch. These instances are in 1:1
                correspondence with the pred_mask_logits. The ground-truth labels (class, box, mask,
                ...) associated with each instance are stored in fields.
            vis_period (int): the period (in steps) to dump visualization.

        Returns:
            mask_loss (Tensor): A scalar tensor containing the loss.
        """
        if self.dct_loss_type == "l1":
            loss_func = F.l1_loss
        elif self.dct_loss_type == "sl1":
            loss_func = F.smooth_l1_loss
        elif self.dct_loss_type == "l2":
            loss_func = F.mse_loss
        else:
            raise ValueError("Loss Type Only Support : l1, l2; yours: {}".format(self.dct_loss_type))

        gt_masks, gt_masks_coarse, gt_bfg = self.get_gt_mask(instances, pred_mask_logits)

        mask_loss = loss_func(pred_mask_logits, gt_masks_coarse)
        mask_loss = mask_loss * self.loss_para[0]
        for i in range(self.num_stage):
            bfg = bfg_dict[i]
            patch_vector = patch_dict[i]
            patch_vector = patch_vector[gt_bfg == 1]
            mask_loss += self.loss_para[i + 1] * (F.cross_entropy(bfg, gt_bfg) + loss_func(patch_vector, gt_masks))

        mask_loss = self.mask_loss_para * mask_loss
        return mask_loss

    def mask_rcnn_dct_inference(self, pred_mask_logits, bfg_dict, patch_dict, pred_instances):
        """
        Convert pred_mask_logits to estimated foreground probability masks while also
        extracting only the masks for the predicted classes in pred_instances. For each
        predicted box, the mask of the same class is attached to the instance by adding a
        new "pred_masks" field to pred_instances.

        Args:
            pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask)
                for class-specific or class-agnostic, where B is the total number of predicted masks
                in all images, C is the number of foreground classes, and Hmask, Wmask are the height
                and width of the mask predictions. The values are logits.
            bfg_dict: A dict includes results of the three-class classifier for each PatchDCT blocks
                bfg for each PatchDCT: A tensor of shape [B*scale*scale,3].
                A NxN masks is divided into scale x scale patches.
                bfg demonstrates results of three-class classifier in PatchDCT
                0 for foreground,1 for mixed,2 for background
            patch_dict : A dict includes results of the regressor for each PatchDCT blocks
                patch_vector for each PatchDCT: A tensor of shape:[B*scale*scale,patch_dct_vector_dim].
                DCT vector for each patch (only calculate loss for mixed patch)
            pred_instances (list[Instances]): A list of N Instances, where N is the number of images
                in the batch. Each Instances must have field "pred_classes".

        Returns:
            None. pred_instances will contain an extra "pred_masks" field storing a mask of size (Hmask,
                Wmask) for predicted class. Note that the masks are returned as a soft (non-quantized)
                masks the resolution predicted by the network; post-processing steps, such as resizing
                the predicted masks to the original image resolution and/or binarizing them, is left
                to the caller.
        """

        num_patch = pred_mask_logits.shape[0]
        device = pred_mask_logits.device
        if num_patch == 0:
            pred_instances[0].pred_masks = torch.empty([0, 1, self.mask_size, self.mask_size]).to(device)
            return pred_instances
        else:
            with torch.no_grad():
                last_block = self.num_stage - 1
                bfg = bfg_dict[last_block]
                patch_vector = patch_dict[last_block]
                bfg = F.softmax(bfg, dim=1)
                bfg[bfg[:, 0] > self.patch_threshold, 0] = bfg[bfg[:, 0] > self.patch_threshold, 0] + 1
                bfg[bfg[:, 2] > self.patch_threshold, 2] = bfg[bfg[:, 2] > self.patch_threshold, 2] + 1
                index = torch.argmax(bfg, dim=1)

                # --------------only for upper bound evaluation------------
                if self.eval_gt:
                    gt_masks, gt_bfg = self.get_gt_mask_inference(pred_instances, pred_mask_logits)
                    index = gt_bfg
                    patch_vector[gt_bfg == 1] = gt_masks
                # ---------------------------------------------------------

                patch_vector[index == 0, ::] = 0
                patch_vector[index == 2, ::] = 0
                patch_vector[index == 2, 0] = self.patch_size

                pred_mask_rc = self.patch_dct_encoding.decode(patch_vector)
                # assemble patches to obtain an entire mask
                pred_mask_rc = rearrange(pred_mask_rc,
                                         "(b s1 s2) p1 p2 -> b (s1 p1) (s2 p2)", s1=self.scale, s2=self.scale)

            pred_mask_rc = pred_mask_rc[:, None, :, :]
            pred_instances[0].pred_masks = pred_mask_rc
            return pred_instances

    def get_gt_classes(self, instances):
        gt_classes = []
        for instances_per_image in instances:
            if len(instances_per_image) == 0:
                continue
            gt_classes_per_image = instances_per_image.gt_classes.to(dtype=torch.int64)
            gt_classes.append(gt_classes_per_image)
        gt_classes = cat(gt_classes, dim=0)  # [N_instance]
        return gt_classes

    def get_gt_mask(self, instances, pred_mask_logits):
        gt_masks = []
        gt_masks_coarse = []
        for instances_per_image in instances:

            if len(instances_per_image) == 0:
                continue

            gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                instances_per_image.proposal_boxes.tensor, self.mask_size)
            gt_masks_coarse.append(gt_masks_per_image)

            gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                instances_per_image.proposal_boxes.tensor, self.mask_size_assemble)
            # divided masks into scale x scale patch,patch size=8
            gt_masks_per_image = rearrange(gt_masks_per_image,
                                           "b (s1 p1) (s2 p2) -> (b s1 s2) p1 p2", s1=self.scale, s2=self.scale)

            gt_masks.append(gt_masks_per_image)

        if len(gt_masks) == 0:
            return pred_mask_logits.sum() * 0
        gt_masks = cat(gt_masks, dim=0)
        gt_masks = self.patch_dct_encoding.encode(gt_masks)  # [N, dct_v_dim]
        gt_masks = gt_masks.to(dtype=torch.float32)  # [N_instance,patchdct_vector_dim]

        gt_masks_coarse = cat(gt_masks_coarse, dim=0)
        gt_masks_coarse = self.dct_encoding.encode(gt_masks_coarse).to(dtype=torch.float32)

        gt_masks, gt_bfg = self.get_gt_bfg(gt_masks)
        return gt_masks, gt_masks_coarse, gt_bfg

    def get_gt_mask_inference(self,instances,pred_mask_logits):
        gt_masks = []

        for instances_per_image in instances:
            if len(instances_per_image) == 0:
                continue
            if instances_per_image.has("gt_masks"):
                gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize(
                    instances_per_image.pred_boxes.tensor, self.mask_size_assemble)
            else:
                # print("gt_mask is empty")
                shape = instances_per_image.pred_boxes.tensor.shape[0]
                device = instances_per_image.pred_boxes.tensor.device
                gt_masks_per_image = torch.zeros((shape, self.mask_size_assemble, self.mask_size_assemble),
                                                 dtype=torch.bool).to(device)

            gt_masks_per_image = rearrange(gt_masks_per_image,
                                           "b (s1 p1) (s2 p2) -> (b s1 s2) p1 p2", s1=self.scale, s2=self.scale)

            gt_masks.append(gt_masks_per_image)

        if len(gt_masks) == 0:
            return pred_mask_logits.sum() * 0

        gt_masks = cat(gt_masks, dim=0)
        gt_masks = self.patch_dct_encoding.encode(gt_masks)
        gt_masks = gt_masks.to(dtype=torch.float32)

        gt_masks, gt_bfg = self.get_gt_bfg(gt_masks)
        return gt_masks, gt_bfg

    def get_gt_bfg(self, gt_masks):
        gt_bfg = gt_masks[:, 0].clone()
        gt_bfg[(gt_bfg > 0) & (gt_bfg < self.patch_size)] = 1.
        gt_bfg[gt_bfg == self.patch_size] = 2
        gt_bfg = gt_bfg.to(dtype=torch.int64)
        gt_masks = gt_masks[gt_bfg == 1, :]
        return gt_masks, gt_bfg


import numpy as np
import pandas as pd

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.colors as colors

# simulations = ["binary","unary"]
simulations = ["restest"]

with open("../local_config/skirt_data_dir") as f:
    skirt_data_dir = f.readline()

for sim in simulations:
    print(f"Summarising {sim}")
    # fname = f"{skirt_data_dir}/{sim}_lowres_allprobe_cellprops.dat"
    fname = f"{skirt_data_dir}/test_data/big_{sim}_allprobe_cellprops.dat"
    df = pd.read_csv(fname,sep=' ',header=None,skiprows=9,index_col=0,
                     names = ["x","y","z","V","tau","dust","e","H"])
    fname = f"{skirt_data_dir}/test_data/big_{sim}_tdust_T.dat"
    df_tdust = pd.read_csv(fname,sep=' ',header=None,skiprows=3,index_col=0,
                     names = ["Tdust"])
    df = pd.merge(df,df_tdust,left_index=True,right_index=True)

    # print(df)

    for key in df.keys():
        if np.sum(np.nonzero(df[key].values))==0:
            print(f"{key} is zero throughout run")
        else:
            print(f"{key}: Min:{np.nanmin(df[key].values)} Mean:{np.nanmean(df[key].values)} Median:{np.nanmedian(df[key].values)} Max:{np.nanmax(df[key].values)} ")
            print(f"{key}: Non-zero:{np.count_nonzero(df[key].values)/len(df)} Finite:{np.sum(np.isfinite(df[key].values))/len(df)}")

    print("Dust 2D histograms")

    df["r2d"] = np.sqrt(df["x"] ** 2 + df["y"] ** 2)
    for key in ["dust","Tdust","tau"]:
        # dust mean properties plots
        # H,xedges,yedges = np.histogram2d(df["z"],df["r2d"],weights=df[key],bins=1024)

        for coords,prefix in [[["y","x"],"yx"],[["z","r2d"],"rz"]]:

            H,yedges,xedges = np.histogram2d(df[coords[0]],df[coords[1]],weights=df[key],bins=128)
            fig,sp = plt.subplots(figsize=(6,6),dpi=200,constrained_layout=True)
            mp = sp.pcolormesh(xedges,yedges,H,norm=colors.LogNorm())
            fig.colorbar(mp)
            fig.savefig(f"../quick_out/{prefix}_{key}_{sim}.png")
            # fig.savefig(f"../quick_out/xy_{key}_{sim}.png")
            plt.close(fig)


import contextlib
from django.contrib import messages
from django.shortcuts import render, get_object_or_404
from django.http import HttpResponseRedirect
from django.urls import reverse
from libvirt import libvirtError
from computes.models import Compute
from interfaces.forms import AddInterface
from vrtManager.interface import wvmInterface, wvmInterfaces
from admin.decorators import superuser_only

@superuser_only
def interfaces(request, compute_id):
    """
    :param request:
    :param compute_id:
    :return:
    """

    ifaces_all = []
    compute = get_object_or_404(Compute, pk=compute_id)

    try:
        conn = wvmInterfaces(
            compute.hostname,
            compute.login,
            compute.password,
            compute.type
        )
        ifaces = conn.get_ifaces()

        netdevs = ["eth0", "eth1"]
        with contextlib.suppress(Exception):
            netdevs = conn.get_net_devices()

        for iface in ifaces:
            interf = wvmInterface(
                compute.hostname,
                compute.login,
                compute.password,
                compute.type,
                iface
            )
            ifaces_all.append(interf.get_details())

        if request.method == "POST":
            if "create" in request.POST:
                form = AddInterface(request.POST)
                if form.is_valid():
                    data = form.cleaned_data
                    conn.create_iface(
                        data["name"],
                        data["itype"],
                        data["start_mode"],
                        data["netdev"],
                        data["ipv4_type"],
                        data["ipv4_addr"],
                        data["ipv4_gw"],
                        data["ipv6_type"],
                        data["ipv6_addr"],
                        data["ipv6_gw"],
                        data["stp"],
                        data["delay"],
                    )
                    return HttpResponseRedirect(request.get_full_path())
                else:
                    for msg_err in form.errors.values():
                        messages.error(request, msg_err.as_text())
        conn.close()
    except libvirtError as lib_err:
        messages.error(request, lib_err)

    return render(request, "interfaces.html", locals())

@superuser_only
def interface(request, compute_id, iface):
    """
    :param request:
    :param compute_id:
    :param iface:
    :return:
    """

    ifaces_all = []
    compute = get_object_or_404(Compute, pk=compute_id)

    try:
        conn = wvmInterface(
            compute.hostname,
            compute.login,
            compute.password,
            compute.type,
            iface
        )
        start_mode = conn.get_start_mode()
        state = conn.is_active()
        mac = conn.get_mac()
        itype = conn.get_type()
        ipv4 = conn.get_ipv4()
        ipv4_type = conn.get_ipv4_type()
        ipv6 = conn.get_ipv6()
        ipv6_type = conn.get_ipv6_type()
        bridge = conn.get_bridge()
        slave_ifaces = conn.get_bridge_slave_ifaces()

        if request.method == "POST":
            if "stop" in request.POST:
                conn.stop_iface()
                return HttpResponseRedirect(request.get_full_path())
            if "start" in request.POST:
                conn.start_iface()
                return HttpResponseRedirect(request.get_full_path())
            if "delete" in request.POST:
                conn.delete_iface()
                return HttpResponseRedirect(reverse("interfaces", args=[compute_id]))
        conn.close()
    except libvirtError as lib_err:
        messages.error(request, lib_err)

    return render(request, "interface.html", locals())


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

from imageio import imread, imwrite
import matplotlib.pyplot as plt
import numpy as np
import scipy.ndimage

Img = imread('C:/Users/fc48286/Downloads/lena.tif')

dim = Img.shape
tipo = Img.dtype
npix = Img.size

plt.figure
plt.imshow(Img)

#Decompor as bandas R,G,B

plt.figure(figsize=(12,3))
plt.subplot(141); plt.imshow(Img)
plt.title('Inicial'); plt.axis('off')
plt.subplot(142); plt.imshow(Img[:,:,0], 'gray')
plt.title('R'); plt.axis('off')
plt.subplot(143); plt.imshow(Img[:,:,1], 'gray')
plt.title('G'); plt.axis('off')
plt.subplot(144); plt.imshow(Img[:,:,2], 'gray')
plt.title('B'); plt.axis('off')

#Gravar Img no disco
imwrite('C:/Users/fc48286/Downloads/' + 'img1_r' + '.tif', Img[:,:, 0])
imwrite('C:/Users/fc48286/Downloads/' + 'img1_g' + '.tif', Img[:,:, 1])
imwrite('C:/Users/fc48286/Downloads/' + 'img1_b' + '.tif', Img[:,:, 2])

val = Img[25,31,0]

linha_cse = 125
coluna_cse = 118
linha_cid = 140
coluna_cid = 180
cropImg = Img[linha_cse:linha_cid,coluna_cse:coluna_cid,:]
plt.figure(); plt.imshow(cropImg);
plt.title('Crop'); plt.axis('off')

Img1 = Img*1
Img1[linha_cse:linha_cid,coluna_cse:coluna_cid,:] = 10
plt.figure(); plt.imshow(Img1)
plt.title(u'Incognita'); plt.axis('off')

#Plot de perfis

#Construir uma linha "num" pontos entre os pontos de coordenadas
#(linha, coluna): (y0, x0) e (y1, x2)
y0, x0 = 100, 50
y1, x1 = 200, 200
num = 100
y, x = np.linspace(y0, y1, num), np.linspace(x0, x1, num)

#Extrair os valores dos pixels ao longo da linha
d = np.vstack((y,x))
perfil = scipy.ndimage.map_coordinates(Img[:,:,0], 'gray')

plt.figure(figsize=(15, 3))
plt.subplot(121); plt.imshow(Img[:,:,0], 'gray')
plt.title['Inicial']; plt.axis('off')
plt.plot([x0, x1], [y0, y1], 'ro=')
plt.subplot(122); plt.plot(perfil, 'ro==')
plt.title('Perfil')


import json
import logging
import pandas
import configparser
import sqlite3
from sqlite3 import Error

if __name__ == '__main__':
    config = configparser.ConfigParser()
    config.read('../../config/config.cfg')
    db_filename = config['DEFAULT']['db_filename']

    json_dir = '../../output/repos_with_topics/'
    languages = ['c', 'c++', 'c#', 'go', 'java', 'javascript', 'python', 'php', 'ruby', 'shell']
    languages = ['java']

    log_filename = '../../log/extract_and_load_repos_with_topics_from_json_to_db.log'
    logging.basicConfig(handlers=[logging.FileHandler(log_filename, 'w+', 'utf-8')], level=20)
    logging.getLogger().addHandler(logging.StreamHandler())

    df = pandas.DataFrame(columns=['repo_full_name', 'topics'])
    for l in languages:
        for i in range(1, 11):
            filename = '{0}_{1}.txt'.format(l, i)
            logging.info('Processing {0}'.format(filename))
            with open(json_dir + filename, 'r', encoding='utf-8') as curr_file:
                json_str = curr_file.read()
                json_objects = json.loads(json_str)
                for j_obj in json_objects['items']:
                    repo_full_name = j_obj['full_name']
                    topics = ','.join(j_obj['topics'])
                    df = df.append({'repo_full_name': repo_full_name, 'topics': topics}, ignore_index=True)
    exit()
    conn = sqlite3.connect(db_filename)
    try:
        # drop duplicates (possible if position of one repository shift to next page between queries
        df.drop_duplicates(subset='repo_full_name', keep='first', inplace=True)
        df.to_sql(name='tag_recommender_ground_truth_repo_topics', con=conn, if_exists='replace', index=False)
    except Error as e:
        logging.exception(e)
    except Exception as e:
        logging.exception(e)
    finally:
        conn.close()

    logging.info('Total repositories loaded: {0}'.format(len(df.index)))


#Práctica número 3 de la Asignatura
#Desarrollo de aplicaciones para internet.
#Autor: Samuel Medina Gutiérrez

from flask import Flask, render_template, request, flash, redirect, url_for, session, jsonify
from bson import ObjectId
from pickleshare import *
from ejercicios.ejercicio2 import burbuja
from ejercicios.ejercicio3 import criba
from ejercicios.ejercicio4 import fibonacci
from model import dataBase
from pymongo import MongoClient

app = Flask(__name__)
app.secret_key = 'claveSuperSecreta'

bd = dataBase()

#Conectamos con el servicio "mongo en su puerto estándar"
client = MongoClient("mongo", 27017)
db = client.SampleCollections

@app.route('/')
@app.route('/index')
def index():
    if not session.get('visited'):
        session['visited'] = []
    if not session.get('logeado'):
        session['logeado'] = False
        if not session.get('usuActual'):
            session['usuActual'] = None

    return render_template('index.html', urls = session['visited'], logeado = session['logeado'])

@app.route('/login', methods =['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        name = request.form['username']
        pssw = request.form['password']
        if  name in bd.getKeys() and bd.getPssw(name) == pssw:
            flash('Te has logeado correctamente')
            session['logeado'] = True
            session['usuActual'] = name
            return redirect (url_for('index'))
        else:
            error = 'Credenciales incorrectas'
    return render_template('login.html', error = error)

@app.route('/signup', methods = ["GET" , "POST"])
def singup():
    error = None
    if request.method == 'POST':
        name = request.form['username']
        pssw = request.form['password']
        if not name or not pssw:
            error = "Has introducido una cadena vacia"
        elif name in bd.getKeys():
            error = "Lo siento, ya existe este usuario"
        else:
            bd.aniade(name,pssw)
            flash('Te has registrado correctamente, por favor intente acceder')
            return redirect (url_for('index'))

    return render_template ('signup.html', error = error)

##Cuandio un usuario se desloguee redirigimos al index

@app.route('/logout')
def logout():
    session['logeado'] = False
    return redirect (url_for('index'))

@app.route('/adddata', methods = ["GET" , "POST"])
def aniade():
    error = None

    usuarioAct = session['usuActual']
    infor = bd.getInformacion(usuarioAct)
    name = infor[0]
    surname = infor[1]
    age = infor[2]
    if request.method == 'POST':
        usuarioAct = session['usuActual']
        name = request.form['name']
        surname = request.form['surname']
        age = request.form['age']

        bd.aniadirInformacion(usuarioAct,name,surname,age)

        return redirect(url_for('index'))
    return render_template('aniade.html', error = error, nombre = name, apellido = surname, edad = age)

@app.route('/userdata')
def mostrar():
    usuarioAct = session['usuActual']
    infor = bd.getInformacion(usuarioAct)
    name = infor[0]
    surname = infor[1]
    age = infor[2]
    return render_template('userdata.html', nombre = name, apellido = surname, edad = age)

@app.route('/ejer1' , methods=['GET','POST'])
def mostrarEjer1():
    if request.method == 'POST':
        numeros = request.form['numbers']
        lista = numeros.split(',')
        ordenado =burbuja(lista)
        return render_template('ejer1.html', ordenado=True, lista=ordenado)
    return render_template('ejer1.html', ordenado=False)

@app.route('/ejer2' , methods=['GET','POST'])
def mostrarEjer2():
    if request.method == 'POST':
        numero = request.form['number']
        lista =criba(numero)
        return render_template('ejer2.html', ordenado=True, lista=lista)
    return render_template('ejer2.html', ordenado=False)

@app.route('/ejer3' , methods=['GET','POST'])
def mostrarEjer3():
    if request.method == 'POST':
        numero = request.form['number']
        numero = fibonacci(numero)
        return render_template('ejer3.html', ordenado=True, lista=numero)
    return render_template('ejer3.html', ordenado=False)

@app.after_request
def visited_url(response):
    if not session.get('visited'):
        session['visited'] = []
    session['visited'].append(request.url)
    if len(session['visited']) > 3:
        session['visited'].pop(0)
    session.modified = True

    return (response)

@app.context_processor
def aniade_url():
    if session.get('visited'):
        lurls = session['visited']
    else:
        lurls=[]
    return dict(urls = lurls)

@app.context_processor
def log():
    logeados = session['logeado']
    return dict(logeado=logeados)

############### PRACTICA 4 ###############################

@app.route('/mongo',  methods=['GET','POST'])
def mongo():
    prueba = db.samples_friends

    if request.method == 'POST':
        titulo = request.form['title']

        episodios = prueba.find({'name': { "$regex": titulo, "$options": 'i'}})

        titulo = []

        for episodio in episodios:
            sumary = episodio['summary'].replace("<p>" , " ").replace("</p>" , " ")

            if episodio['image'] != None :
                tit = [episodio['name'], episodio['airdate'], episodio['airtime'], sumary,
                    episodio['image']['medium']]
            else:
                tit = [episodio['name'], episodio['airdate'], episodio['airtime'], sumary, None]
            titulo.append(tit)

        return render_template('list.html', buscado = True, match=titulo)

    ##return str(lista_episodios)
    return render_template('list.html', buscado = False)

##Práctica 5
@app.route('/api/capitulos', methods = ['GET', 'POST'])
def api_1():
    if request.method == 'GET':
        lista = []
        capitulos = db.samples_friends.find().sort('name')
        for capitulo in capitulos:
            lista.append({
                  'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
                  'title': str(capitulo.get('name')),
                  'airTime':  str(capitulo.get('airtime')),
                  'date':  str(capitulo.get('airdate'))
                })
        return jsonify(lista)

    if request.method == 'POST':
        name = request.form['title']
        result = db.samples_friends.insert_one({'name': name})
        capitulo = db.samples_friends.find_one({'_id':ObjectId(result.inserted_id)})
        return jsonify({
            'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
            'title': capitulo.get('name'),
            'airTime':  capitulo.get('airtime'),
            'date':  capitulo.get('airdate')
        })

@app.route('/api/capitulos/<id>', methods = ['GET', 'PUT', 'DELETE'])
def api_2(id):
    if request.method == 'GET':
        capitulo = db.samples_friends.find_one({'_id':ObjectId(id)})
        return jsonify({
            'id':    str(capitulo.get('_id')), # pasa a string el ObjectId
            'title': capitulo.get('name'),
            'airTime':  capitulo.get('airrime'),
            'date':  capitulo.get('airdate')
        })
    if request.method == 'DELETE':
        db.samples_friend.remove({'_id':ObjectId(id)})
        return jsonify('deleted')

    if requested.method == 'PUT':
        name = request.form['name']


import scrapy
from bankiru_crawl.items import BankiruCrawlItem
# from scrapy.shell import inspect_response

class bankiru_spider(scrapy.Spider):
    handle_httpstatus_list = [404]
    name = "bankiru_spider"
    allowed_domains = ['banki.ru']
    URL = "http://www.banki.ru/services/responses/bank/{}/?page="
    # header = {'Connection': 'keep-alive', 'Accept-Encoding': 'gzip, deflate',
    #          'Accept': '*/*', 'User-Agent': 'python-requests/2.12.4'}
    xpaths = {'review_text': ('div[@class="responses__item__message '
                              'markup-inside-small '
                              'markup-inside-small--bullet"]/text()'),
              'review_url': 'div/a[@itemprop="url"]/@href',
              'review_header': 'div/a[@itemprop="url"]/text()',
              'review_author_url': 'footer//a[@itemprop="author"]/@href',
              'review_author_name': 'footer//a[@itemprop="author"]/span[@itemprop="name"]/text()',
              'review_datetime': 'footer//time/@datetime',
              'review_rating': 'div//meta[@itemprop="ratingValue"]/@content',
              'bank_answer_datetime': 'div//time/@datetime',
              'bank_answer_text': 'div//div[@class="thread-item__text"]/text()'}
    check_404 = False

    def start_requests(self):
        page_number = int(self.start_page)
        while True:
            if self.check_404:
                break
            page_number += 1
            url = self.URL.format(self.bank_name)
            yield scrapy.Request(url=url + str(page_number), callback=self.parse)

    def parse(self, response):
        def extract_item(article, x_path):
            content = article.xpath(x_path).extract()
            item = content[0] if len(content) > 0 else "empty"
            return item
        # inspect_response(response, self)
        if response.status == 404:
            self.check_404 = True
        for article in response.xpath('//article'):
            item = BankiruCrawlItem()
            item['bank_name'] = self.bank_name
            item["review_text"] = " ".join(article.xpath(self.xpaths['review_text'])
                                           .extract())
            item["review_url"] = extract_item(article, self.xpaths['review_url'])
            item["review_header"] = extract_item(article, self.xpaths['review_header'])
            item["review_author_url"] = extract_item(article,
                                                     self.xpaths['review_author_url'])
            item["review_author_name"] = extract_item(article, self.xpaths['review_author_name'])
            item['review_datetime'] = extract_item(article, self.xpaths['review_datetime'])
            item['review_rating'] = extract_item(article, self.xpaths['review_rating'])
            item['bank_answer_datetime'] = extract_item(article,
                                                        self.xpaths['bank_answer_datetime'])
            item['bank_answer_text'] = " ".join(article.xpath(self.xpaths['bank_answer_text']).extract())
            if item['bank_answer_text'] != '':
                item['bank_answer_check'] = True
            else:
                item['bank_answer_check'] = False
            yield item