"""

Copyright (C) 2021 Hoichan JUNG <hoichanjung@korea.ac.kr> - All Rights Reserved

"""

import time
from tqdm import tqdm
from copy import deepcopy

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
import torch.optim as optim
from torch.utils.data import DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingWarmRestarts
from warmup_scheduler import GradualWarmupScheduler
from IQA_pytorch import SSIM
from ptflops import get_model_complexity_info

from config import getConfig
from DatasetGenerator import DatasetGenerator
from models import *
from utils import *
from losses import *

cfg = getConfig()

class Trainer():
    def train(pathDirData, noiseType, nnArchitecture, batchSize, trMaxEpoch, imgtransResize, model_name, logger, run):

        # -------------------- SETTINGS: NETWORK ARCHITECTURE
        if nnArchitecture == 'UNET':
            model = Denoising_UNet(in_channels=1, n_classes=1, padding=True).cuda()
        elif nnArchitecture == 'REDNET':
            model = REDNet20(in_channels=1, num_layers=6).cuda()
        elif nnArchitecture == 'UNET_REDNET':
            model = UNet_REDNet().cuda()
        elif nnArchitecture == 'VDSR':
            model = VDSR().cuda()
        elif nnArchitecture == 'HINET':
            model = HINet(in_chn = 1).cuda()
        elif nnArchitecture == 'MPRNET':
            model = MPRNet(in_c = 1, out_c=1).cuda()
        elif nnArchitecture == 'UFORMER':
            model = Uformer(img_size=imgtransResize, in_chans=1, token_mlp='ffn').cuda()
        elif nnArchitecture == 'RESTORMER':
            model = Restormer(inp_channels=1, out_channels=1, LayerNorm_type='BiasFree').cuda()

        macs, params = get_model_complexity_info(model, (1, imgtransResize, imgtransResize), as_strings=True,
                                                print_per_layer_stat=False, verbose=True)
        print('MACs(G): ', macs)
        print('Params: ', params)

        logger.info(f'MACs(G): {macs}')
        logger.info(f'Params: {params}')

        run['MACs(G)'] = macs
        run['Params'] = params

        model = torch.nn.DataParallel(model).cuda()
        pathModel = f'results/{model_name}/model.pth.tar'

        # -------------------- SETTINGS: DATA TRANSFORMS
        transformList = []
        # transformList.append(transforms.RandomHorizontalFlip())
        # transformList.append(transforms.RandomVerticalFlip())
        # transformList.append(transforms.RandomAffine((-20, 20)))
        # transformList.append(transforms.RandomRotation(90))
        transformList.append(transforms.Resize(imgtransResize))
        transformList.append(transforms.ToTensor())
        # transformList.append(transforms.Normalize([0.5], [0.5]))
        transform = transforms.Compose(transformList)

        # -------------------- SETTINGS: DATASET BUILDERS
        datasetTrain = DatasetGenerator(pathDirData, dataset='train', noise_type=noiseType, transform=transform, logger=logger)
        datasetValid = DatasetGenerator(pathDirData, dataset='valid', noise_type=noiseType, transform=transform, logger=logger)

        dataLoaderTrain = DataLoader(dataset = datasetTrain, batch_size = batchSize , shuffle = True, pin_memory = True)
        dataLoaderValid = DataLoader(dataset = datasetValid, batch_size = batchSize , shuffle = False, pin_memory = True)
        run['Batch'] = batchSize
        # -------------------- SETTINGS: OPTIMIZER & SCHEDULER
        if cfg.op == 'adam':
            optimizer = optim.Adam(model.parameters(), lr=cfg.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-5)

        if cfg.scheduler == 'ReduceLROnPlateau':
            scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=cfg.patience, mode='min')
        elif cfg.scheduler == 'CosineAnnealingWarmRestarts':
            warmup_epochs = 3
            scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, trMaxEpoch-warmup_epochs, eta_min=1e-8)
            scheduler = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=warmup_epochs, after_scheduler=scheduler_cosine)

        # -------------------- SETTINGS: Loss Function
        # criterion_mse = nn.MSELoss()
        criterion_char = CharbonnierLoss()
        criterion_edge = EdgeLoss()
        # criterion_psnr = PSNRLoss()

        # -------------------- TRAIN / VALIDATION
        lossMIN = 10000

        for epochID in range(trMaxEpoch):
            timestampTime = time.strftime("%H%M%S")
            timestampDate = time.strftime("%d%m%Y")
            timestampSTART = timestampDate + '-' + timestampTime
            print('current lr : {:.7f}'.format(optimizer.param_groups[0]['lr']))

            if nnArchitecture == 'UNET' or nnArchitecture == 'REDNET' or nnArchitecture == 'UNET_REDNET'\
                or nnArchitecture == 'VDSR' or nnArchitecture == 'UFORMER' or nnArchitecture == 'RESTORMER':
                epochTrainUNET(model, dataLoaderTrain, optimizer, criterion_char, criterion_edge)
                lossVal, losstensor = epochValidUNET(model, dataLoaderValid, criterion_char, criterion_edge, epochID, model_name)

            elif nnArchitecture == 'HINET' or nnArchitecture == 'MPRNET':
                epochTrainHINET(model, dataLoaderTrain, optimizer, criterion_char, criterion_edge)
                lossVal, losstensor = epochValidHINET(model, dataLoaderValid, criterion_char, criterion_edge, epochID, model_name)

            timestampTime = time.strftime("%H%M%S")
            timestampDate = time.strftime("%d%m%Y")
            timestampEND = timestampDate + '-' + timestampTime

            scheduler.step(losstensor.data)

            if lossVal < lossMIN:
                lossMIN = lossVal

                torch.save({'epoch': epochID + 1, 'state_dict': model.state_dict(), 'best_loss': lossMIN,
                            'optimizer': optimizer.state_dict()}, pathModel)
                print('Epoch [' + str(epochID + 1) + '] [save] [' + timestampEND + '] loss= ' + str(lossVal))
                logger.info('Epoch [' + str(epochID + 1) + '] [save] [' + timestampEND + '] loss= ' + str(lossVal))
                run['Valid loss'].log(lossVal)

            else:
                print('Epoch [' + str(epochID + 1) + '] [----] [' + timestampEND + '] loss= ' + str(lossVal))
                logger.info('Epoch [' + str(epochID + 1) + '] [----] [' + timestampEND + '] loss= ' + str(lossVal))

    def test(pathDirData, noiseType, nnArchitecture, batchSize, imgtransResize, model_name, logger, run):
        cudnn.benchmark = True

        # -------------------- SETTINGS: NETWORK ARCHITECTURE
        if nnArchitecture == 'UNET':
            model = Denoising_UNet(in_channels=1, n_classes=1, padding=True).cuda()
        elif nnArchitecture == 'REDNET':
            model = REDNet20(in_channels=1, num_layers=6).cuda()
        elif nnArchitecture == 'UNET_REDNET':
            model = UNet_REDNet().cuda()
        elif nnArchitecture == 'VDSR':
            model = VDSR().cuda()
        elif nnArchitecture == 'HINET':
            model = HINet(in_chn = 1).cuda()
        elif nnArchitecture == 'MPRNET':
            model = MPRNet(in_c = 1, out_c=1).cuda()
        elif nnArchitecture == 'UFORMER':
            model = Uformer(img_size=imgtransResize, in_chans=1, token_mlp='ffn').cuda()
        elif nnArchitecture == 'RESTORMER':
            model = Restormer(inp_channels=1, out_channels=1, LayerNorm_type='BiasFree').cuda()

        model = torch.nn.DataParallel(model).cuda()
        pathModel = f'results/{model_name}/model.pth.tar'
        load_pretrained(model, pathModel)

        # -------------------- SETTINGS: DATA TRANSFORMS
        transformList = []
        transformList.append(transforms.Resize(imgtransResize))
        transformList.append(transforms.ToTensor())
        # transformList.append(transforms.Normalize([0.5], [0.5]))
        transform = transforms.Compose(transformList)

        # -------------------- SETTINGS: DATASET BUILDERS
        datasetTest = DatasetGenerator(pathDirData, dataset = 'test', noise_type = noiseType, transform = transform, logger=logger)
        dataLoaderTest = DataLoader(dataset = datasetTest, batch_size = batchSize , shuffle = False, pin_memory = True)

        # -------------------- TEST
        with torch.no_grad():

            model.eval()

            label = torch.FloatTensor().cuda()
            noise = torch.FloatTensor().cuda()
            pred = torch.FloatTensor().cuda()

            img_number = 0
            for batchID, data in enumerate(tqdm(dataLoaderTest)):

                img_og, img_noise = data['original'].cuda(), data['noise'].cuda()
                img_output = model(img_noise)

                label = torch.cat((label, img_og), 0)
                noise = torch.cat((noise, img_noise), 0)

                if nnArchitecture == 'UNET' or nnArchitecture == 'REDNET' or nnArchitecture == 'UNET_REDNET'\
                    or nnArchitecture == 'VDSR' or nnArchitecture == 'UFORMER' or nnArchitecture == 'RESTORMER':
                    pred = torch.cat((pred, img_output), 0)
                elif nnArchitecture == 'HINET':
                    pred = torch.cat((pred, img_output[1]), 0)
                elif nnArchitecture == 'MPRNET':
                    pred = torch.cat((pred, img_output[0]), 0)

                for imgID in range(img_og.shape[0]):

                    save_image(img_og[imgID], f'./results/{model_name}/original/{img_number}.png')
                    save_image(img_noise[imgID], f'./results/{model_name}/noise/{img_number}.png')

                    if nnArchitecture == 'UNET' or nnArchitecture == 'REDNET' or nnArchitecture == 'UNET_REDNET'\
                        or nnArchitecture == 'VDSR' or nnArchitecture == 'UFORMER' or nnArchitecture == 'RESTORMER':
                        save_image(img_output[imgID], f'./results/{model_name}/output/{img_number}.png')
                    elif nnArchitecture == 'HINET':
                        save_image(img_output[1][imgID], f'./results/{model_name}/output/{img_number}.png')
                    elif nnArchitecture == 'MPRNET':
                        save_image(img_output[0][imgID], f'./results/{model_name}/output/{img_number}.png')

                    img_number += 1

            noisePSNR = torchPSNR(label, noise)
            predPSNR = torchPSNR(label, pred)
            print(f"Noise PSNR : {noisePSNR: .2f}")
            print(f"Pred PSNR : {predPSNR: .2f}")

            logger.info(f"Noise PSNR : {noisePSNR: .2f}")
            logger.info(f"Pred PSNR : {predPSNR: .2f}")

            run['Noise PSNR'] = f"{noisePSNR: .2f}"
            run['Pred PSNR'] = f"{predPSNR: .2f}"

            ssim = SSIM()
            noiseSSIM = torch.mean(ssim(label, noise, as_loss=False))
            predSSIM = torch.mean(ssim(label, pred, as_loss=False))

            print(f"Noise SSIM : {noiseSSIM: .3f}")
            print(f"Pred SSIM : {predSSIM: .3f}")

            logger.info(f"Noise SSIM : {noiseSSIM: .3f}")
            logger.info(f"Pred SSIM : {predSSIM: .3f}")

            run['Noise SSIM'] = f"{noiseSSIM: .3f}"
            run['Pred SSIM'] = f"{predSSIM: .3f}"

            save_results(label, noise, pred, f'./results/{model_name}/results.csv')


from __future__ import (
    absolute_import, print_function, division, unicode_literals
)

import mock
import re
import requests
import responses
import pytest

from inspect import getargspec
from requests.exceptions import ConnectionError, HTTPError

def assert_reset():
    assert len(responses._default_mock._urls) == 0
    assert len(responses.calls) == 0

def assert_response(resp, body=None):
    assert resp.status_code == 200
    assert resp.headers['Content-Type'] == 'text/plain'
    assert resp.text == body

def test_response():
    @responses.activate
    def run():
        responses.add(responses.GET, 'http://example.com', body=b'test')
        resp = requests.get('http://example.com')
        assert_response(resp, 'test')
        assert len(responses.calls) == 1
        assert responses.calls[0].request.url == 'http://example.com/'
        assert responses.calls[0].response.content == b'test'

        resp = requests.get('http://example.com?foo=bar')
        assert_response(resp, 'test')
        assert len(responses.calls) == 2
        assert responses.calls[1].request.url == 'http://example.com/?foo=bar'
        assert responses.calls[1].response.content == b'test'

    run()
    assert_reset()

def test_connection_error():
    @responses.activate
    def run():
        responses.add(responses.GET, 'http://example.com')

        with pytest.raises(ConnectionError):
            requests.get('http://example.com/foo')

        assert len(responses.calls) == 1
        assert responses.calls[0].request.url == 'http://example.com/foo'
        assert type(responses.calls[0].response) is ConnectionError

    run()
    assert_reset()

def test_match_querystring():
    @responses.activate
    def run():
        url = 'http://example.com?test=1&foo=bar'
        responses.add(
            responses.GET, url,
            match_querystring=True, body=b'test')
        resp = requests.get('http://example.com?test=1&foo=bar')
        assert_response(resp, 'test')
        resp = requests.get('http://example.com?foo=bar&test=1')
        assert_response(resp, 'test')

    run()
    assert_reset()

def test_match_querystring_error():
    @responses.activate
    def run():
        responses.add(
            responses.GET, 'http://example.com/?test=1',
            match_querystring=True)

        with pytest.raises(ConnectionError):
            requests.get('http://example.com/foo/?test=2')

    run()
    assert_reset()

def test_match_querystring_regex():
    @responses.activate
    def run():
        """Note that `match_querystring` value shouldn't matter when passing a
        regular expression"""

        responses.add(
            responses.GET, re.compile(r'http://example\.com/foo/\?test=1'),
            body='test1', match_querystring=True)

        resp = requests.get('http://example.com/foo/?test=1')
        assert_response(resp, 'test1')

        responses.add(
            responses.GET, re.compile(r'http://example\.com/foo/\?test=2'),
            body='test2', match_querystring=False)

        resp = requests.get('http://example.com/foo/?test=2')
        assert_response(resp, 'test2')

    run()
    assert_reset()

def test_match_querystring_error_regex():
    @responses.activate
    def run():
        """Note that `match_querystring` value shouldn't matter when passing a
        regular expression"""

        responses.add(
            responses.GET, re.compile(r'http://example\.com/foo/\?test=1'),
            match_querystring=True)

        with pytest.raises(ConnectionError):
            requests.get('http://example.com/foo/?test=3')

        responses.add(
            responses.GET, re.compile(r'http://example\.com/foo/\?test=2'),
            match_querystring=False)

        with pytest.raises(ConnectionError):
            requests.get('http://example.com/foo/?test=4')

    run()
    assert_reset()

def test_accept_string_body():
    @responses.activate
    def run():
        url = 'http://example.com/'
        responses.add(
            responses.GET, url, body='test')
        resp = requests.get(url)
        assert_response(resp, 'test')

    run()
    assert_reset()

def test_throw_connection_error_explicit():
    @responses.activate
    def run():
        url = 'http://example.com'
        exception = HTTPError('HTTP Error')
        responses.add(
            responses.GET, url, exception)

        with pytest.raises(HTTPError) as HE:
            requests.get(url)

        assert str(HE.value) == 'HTTP Error'

    run()
    assert_reset()

def test_callback():
    body = b'test callback'
    status = 400
    headers = {'foo': 'bar'}
    url = 'http://example.com/'

    def request_callback(request):
        return (status, headers, body)

    @responses.activate
    def run():
        responses.add_callback(responses.GET, url, request_callback)
        resp = requests.get(url)
        assert resp.text == "test callback"
        assert resp.status_code == status
        assert 'foo' in resp.headers
        assert resp.headers['foo'] == 'bar'

    run()
    assert_reset()

def test_regular_expression_url():
    @responses.activate
    def run():
        url = re.compile(r'https?://(.*\.)?example.com')
        responses.add(responses.GET, url, body=b'test')

        resp = requests.get('http://example.com')
        assert_response(resp, 'test')

        resp = requests.get('https://example.com')
        assert_response(resp, 'test')

        resp = requests.get('https://uk.example.com')
        assert_response(resp, 'test')

        with pytest.raises(ConnectionError):
            requests.get('https://uk.exaaample.com')

    run()
    assert_reset()

def test_custom_adapter():
    @responses.activate
    def run():
        url = "http://example.com"
        responses.add(responses.GET, url, body=b'test')

        class DummyAdapter(requests.adapters.HTTPAdapter):
            pass

        # Test that the adapter is actually used
        adapter = mock.Mock(spec=DummyAdapter())
        session = requests.Session()
        session.mount("http://", adapter)

        resp = session.get(url)
        assert adapter.build_response.called == 1

        # Test that the response is still correctly emulated
        session = requests.Session()
        session.mount("http://", DummyAdapter())

        resp = session.get(url)
        assert_response(resp, 'test')

    run()

def test_responses_as_context_manager():
    def run():
        with responses.mock:
            responses.add(responses.GET, 'http://example.com', body=b'test')
            resp = requests.get('http://example.com')
            assert_response(resp, 'test')
            assert len(responses.calls) == 1
            assert responses.calls[0].request.url == 'http://example.com/'
            assert responses.calls[0].response.content == b'test'

            resp = requests.get('http://example.com?foo=bar')
            assert_response(resp, 'test')
            assert len(responses.calls) == 2
            assert (responses.calls[1].request.url ==
                    'http://example.com/?foo=bar')
            assert responses.calls[1].response.content == b'test'

    run()
    assert_reset()

def test_activate_doesnt_change_signature():
    def test_function(a, b=None):
        return (a, b)

    decorated_test_function = responses.activate(test_function)
    assert getargspec(test_function) == getargspec(decorated_test_function)
    assert decorated_test_function(1, 2) == test_function(1, 2)
    assert decorated_test_function(3) == test_function(3)

def test_activate_doesnt_change_signature_for_method():
    class TestCase(object):

        def test_function(self, a, b=None):
            return (self, a, b)

    test_case = TestCase()
    argspec = getargspec(test_case.test_function)
    decorated_test_function = responses.activate(test_case.test_function)
    assert argspec == getargspec(decorated_test_function)
    assert decorated_test_function(1, 2) == test_case.test_function(1, 2)
    assert decorated_test_function(3) == test_case.test_function(3)


from logging import DEBUG
from unittest import TestCase, main
from requests import delete

from SCIM.helpers import set_up_logger
from SCIM.tests.common import TestHelper, BASE_URL

logger = set_up_logger(__name__, level=DEBUG)

ENDPOINT_URI = '/Groups'

test_helper = TestHelper(ENDPOINT_URI, logger)

logger.info("These tests use the sample data in SCIM/examples/group-no-members.csv and and expect a flushed db")

class DeleteGroupsTests(TestCase):
    def test_delete_group(self) -> None:
        response = delete(BASE_URL + ENDPOINT_URI + '/')
        if response.status_code != 204: logger.error('Response from Connector: %s' % str(response.json()))
        self.assertEqual(response.status_code, 204)
        logger.info('Group with ID %s was deleted successfully')

if __name__ == '__main__':
    main()


from mock import (
    Mock, patch, call
)

from kiwi_boxed_plugin.utils.dir_files import DirFiles

class TestDirFiles:
    @patch('kiwi_boxed_plugin.utils.dir_files.Path')
    def setup(self, mock_Path):
        self.dir_manager = DirFiles('box_dir')
        mock_Path.wipe.assert_called_once_with('box_dir.tmp')

    @patch('kiwi_boxed_plugin.utils.dir_files.Path')
    def setup_method(self, cls, mock_Path):
        self.setup()

    @patch('kiwi_boxed_plugin.utils.dir_files.NamedTemporaryFile')
    def test_register(self, mock_NamedTemporaryFile):
        tmpfile = Mock()
        tmpfile.name = 'tmp_a'
        mock_NamedTemporaryFile.return_value = tmpfile
        self.dir_manager.register('/some/path/to/file_a')
        assert self.dir_manager.collection == {
            'file_a': 'tmp_a'
        }

    @patch('kiwi_boxed_plugin.utils.dir_files.Path')
    @patch('kiwi_boxed_plugin.utils.dir_files.Command.run')
    def test_commit(self, mock_Command_run, mock_Path):
        self.dir_manager.collection = {
            'file_a': 'tmp_a'
        }
        self.dir_manager.commit()
        mock_Path.create.assert_called_once_with('box_dir.tmp')
        assert mock_Command_run.call_args_list == [
            call(
                ['bash', '-c', 'cp -a box_dir/* box_dir.tmp'],
                raise_on_error=False
            ),
            call(['mv', 'tmp_a', 'box_dir.tmp/file_a']),
            call(['mv', 'box_dir', 'box_dir.wipe']),
            call(['mv', 'box_dir.tmp', 'box_dir']),
            call(['rm', '-rf', 'box_dir.wipe'])
        ]


import boto3

START_RULE_NAME = 'start-every-day'
STOP_RULE_NAME = 'stop-every-day'

def lambda_handler(event, context):
    rule_name = event['ruleName']
    ec2 = boto3.resource('ec2')

    if rule_name == START_RULE_NAME:
        try:
            stopped_instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['stopped']}])
            for instance in stopped_instances:
                check_instance_start(instance)
        except Exception as e:
            print(e.args[-1])

    elif rule_name == STOP_RULE_NAME:
        try:
            running_instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
            for instance in running_instances:
                check_instance_stop(instance)
        except Exception as e:
            print(e.args[-1])

    else:
        print("Unknown rule name.")

def check_instance_start(instance):
    if instance.tags:
        for tag in instance.tags:
            if tag['Key'].lower() == "shutdown" and tag['Value'].lower() == "true":
                print("Instance with ID \"{0}\" will be started.".format(instance.instance_id))
                instance.start()

def check_instance_stop(instance):
    if instance.tags:
        for tag in instance.tags:
            if tag['Key'].lower() == "shutdown" and tag['Value'].lower() == "true":
                print("Instance with ID \"{0}\" will be stopped.".format(instance.instance_id))
                instance.stop()


from mlxtend.frequent_patterns import fpgrowth
import pandas as pd

import time
import cProfile

def get_closed_itemsets(baskets, threshold):
    print('========== Collecting Closed Itemsets ==========')
    # Each itemset has minimum possible support 'threshold', assuming it appears in the database
    print(f'Finding all frequent itemsets with support above: {threshold}')
    start_time = time.time()
    itemsets = fpgrowth(baskets, min_support=threshold, use_colnames=True)
    print(f'Time to run fpgrowth with min_sup 0: {time.time() - start_time}')

    su = itemsets.support.unique()

    fredic = {}
    for i in range(len(su)):
        inset = list(itemsets.loc[itemsets.support == su[i]]['itemsets'])
        fredic[su[i]] = inset

    start_time = time.time()
    cl = []
    print(itemsets.shape)
    for index, row in itemsets.iterrows():
        isclose = True
        cli = row['itemsets']
        cls = row['support']
        checkset = fredic[cls]
        for i in checkset:
            if (cli != i):
                if (frozenset.issubset(cli, i)):
                    isclose = False
                    break

        if isclose:
            cl.append((row['itemsets'], row['support']))

    print("Stage 1 done", len(cl))
    closed_itemset_dict = dict()
    for c, s in cl:
        # c = frozenset([int(c_i) for c_i in c])
        closed_itemset_dict[c] = s

    print(f'Time to find closed itemsets: {time.time() - start_time}')
    print(f'{itemsets.shape[0]} itemsets reduced to {len(cl)} closed itemsets')
    print('================================================\n')
    return closed_itemset_dict, itemsets

def itemsets_from_closed_itemsets(closed_itemsets, possible_itemsets):
    # profile = cProfile.Profile()
    # profile.enable()
    supports = []
    for itemset in possible_itemsets:
        max_supp = 0
        for closed_itemset, supp in closed_itemsets.items():
            # closed_itemset = frozenset([str(c_i) for c_i in closed_itemset])
            if itemset <= closed_itemset:
                max_supp = max(max_supp, supp)
        supports.append(max_supp)

    df = pd.DataFrame(data={'support': supports, 'itemsets': possible_itemsets})

    # profile.disable()
    # profile.print_stats()
    return df


# ///////////////////////////////////////////////////////////////
#
# CREATED: NIZAR IZZUDDIN Y.F
# V: 1.0.1
# GITHUB: https://github.com/nizariyf
#
# This project is free for everyone, if there is an error please contribute in this
#
# ///////////////////////////////////////////////////////////////

import time
import json
import math
import re
import requests

url = 'https://10minutemail.com'
r = requests.Session()

def runEmail():
  try:
    r.get(url)
    return True
  except:
    return False

def getEmail():
  try:
    page = r.get(f'{url}/session/address')
    email = page.json()
    return email['address']
  except:
    return False

def getCountSeconds():
  try:
    page = r.get(f'{url}/session/secondsLeft')
    count = page.json()
    minutes = math.floor(int(count['secondsLeft'])/60)
    seconds = int(count['secondsLeft'])-minutes*60
    time = f'{minutes}:{seconds} Minute'
    return time
  except:
    return False

def getCountBox():
  try:
    page = r.get(f'{url}/messages/messageCount')
    count = page.json()
    return count['messageCount']
  except:
    return False

def getCodeVerifTwitter():
  try:
    page = r.get(f'{url}/messages/')
    box = page.json()
    count = getCountBox()
    count = int(count)-1
    body = box[count]['bodyPlainText']
    code = re.findall(r'\d+', body)[0]
    return code
  except:
    return False

def longActive():
  try:
    r.get(f'{url}/session/reset')
    return True
  except:
    return False

def clearCookie():
  try:
    r.cookies.clear()
    return True
  except:
    return False


import bs4
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup
import pandas as pd
import numpy as np
import re
import datetime
import requests

tmp_url = "https://www.surfline.com/surf-report/terra-mar-point/5842041f4e65fad6a77088a6"
tamarack = 'https://www.surfline.com/surf-report/tamarack/5842041f4e65fad6a7708837'
oside_pier_south_url = "https://www.surfline.com/surf-report/oceanside-pier-southside/584204204e65fad6a7709435"
oside_pier_north_url = "https://www.surfline.com/surf-report/oceanside-pier-northside/5842041f4e65fad6a7708835"
oside_harbor_north_url = "https://www.surfline.com/surf-report/oceanside-harbor-north-jetty/5842041f4e65fad6a7708832"
grandview_url = "https://www.surfline.com/surf-report/grandview/5842041f4e65fad6a770889f"
seaside_reef_url = "https://www.surfline.com/surf-report/seaside-reef/5842041f4e65fad6a77088b3"
d_street_url = "https://www.surfline.com/surf-report/d-street/5842041f4e65fad6a77088b7"
la_jolla_url = "https://www.surfline.com/surf-report/la-jolla-shores/5842041f4e65fad6a77088cc"
swamis_url = "https://www.surfline.com/surf-report/swami-s/5842041f4e65fad6a77088b4"
pacific_beach = 'https://www.surfline.com/surf-report/pacific-beach/5842041f4e65fad6a7708841'
beacons='https://www.surfline.com/surf-report/beacons/5842041f4e65fad6a77088a0'
moonlight='https://www.surfline.com/surf-report/moonlight-state-beach/5842041f4e65fad6a77088a3'
pipes = 'https://www.surfline.com/surf-report/pipes/5c008f5313603c0001df5318'
blacks='https://www.surfline.com/surf-report/blacks/5842041f4e65fad6a770883b'
windansea='https://www.surfline.com/surf-report/windansea/5842041f4e65fad6a770883c'
la_jolla_shores='https://www.surfline.com/surf-report/la-jolla-shores/5842041f4e65fad6a77088cc'
tourmaline='https://www.surfline.com/surf-report/old-man-s-at-tourmaline/5842041f4e65fad6a77088c4'
torrey_pines='https://www.surfline.com/surf-report/torrey-pines-state-beach/584204204e65fad6a7709994'
del_mar='https://www.surfline.com/surf-report/del-mar/5d7687fdb4c559000112e666'
san_elijo='https://www.surfline.com/surf-report/san-elijo-state-beach/5842041f4e65fad6a77088b8'
trails = "https://www.surfline.com/surf-report/trails/5842041f4e65fad6a7708885"
the_point = "https://www.surfline.com/surf-report/the-point-at-san-onofre/5842041f4e65fad6a7708831"
upper_trestles = "https://www.surfline.com/surf-report/upper-trestles/5842041f4e65fad6a7708887"
lower_trestles = "https://www.surfline.com/surf-report/lower-trestles/5842041f4e65fad6a770888a"
san_onofre = "https://www.surfline.com/surf-report/san-onofre-state-beach/584204204e65fad6a77099d4"

urls = [
    beacons,
    blacks,
    d_street_url,
    del_mar,
    grandview_url,
    la_jolla_url,
    moonlight,
    oside_harbor_north_url,
    oside_pier_north_url,
    oside_pier_south_url,
    pacific_beach,
    pipes,
    san_elijo,
    san_onofre,
    seaside_reef_url,
    swamis_url,
    tamarack,
    the_point,
    tmp_url,
    torrey_pines,
    tourmaline,
    trails,
    upper_trestles,
    lower_trestles,
    windansea
]

def get_surf_data():

    dictionary = {}

    for url in urls:

        data = {}

        url_page = requests.get(url)
        url_soup = soup(url_page.text, "html.parser")

        url_loc = url_soup.h1.text.replace(" Report & Forecast", "")

        # current conditions (i.e. top table on website)
        current_cond = url_soup.find("div", {"class": "quiver-spot-report"})
        url_cond = current_cond.div.text.lower()
        data['condition'] = url_cond

        # takes current wave height
        current_height = url_soup.find("span", {"class": "quiver-surf-height"})
        url_height = current_height.text.lower()
        data['wave_height'] = url_height

        # takes current tide
        current_tide = url_soup.find("div", {
            "class": "quiver-spot-forecast-summary__stat-container quiver-spot-forecast-summary__stat-container--tide"})
        url_tide = current_tide.next_element.next_element.next_element.text
        data['tide'] = url_tide

        # takes wind speed
        current_wind = url_soup.find("div", {
            "class": "quiver-spot-forecast-summary__stat-container quiver-spot-forecast-summary__stat-container--wind"})
        url_wind = current_wind.next_element.next_element.next_element.text
        data['wind'] = url_wind

        # # swells
        # data['swells'] = [swells_line_1, swells_line_2, swells_line_3]

        # outside temp
        current_weather = url_soup.find("div", {"class": "quiver-weather-stats"})
        url_weather = current_weather.next_element.next_element.next_element.next_element.next_element
        data['weather'] = url_weather

        # water temp
        current_H20temp = url_soup.find("div", {"class": "quiver-water-temp"})
        H20temp1 = current_H20temp.next_element.next_element.next_element.next_element.next_element
        H20temp2 = current_H20temp.next_element.next_element.next_element.next_element.next_element.next_element.next_element.next_element.next_element
        data['H20temp'] = H20temp1 + "-" + H20temp2

        # ----------------- API --------------------

        # WEATHER DATA
        spot_id = url.split('/')[-1]
        api_url = 'https://services.surfline.com/kbyg/spots/forecasts/weather?spotId={}&days=6&intervalHours=1'.format(spot_id)
        api_data = requests.get(api_url).json()

        # first light
        first_light_timestamp = api_data['data']['sunlightTimes'][0]['dawn']
        first_light = datetime.datetime.fromtimestamp(first_light_timestamp)
        data['first_light'] = first_light

        # last light
        last_light_timestamp = api_data['data']['sunlightTimes'][0]['dusk']
        last_light = datetime.datetime.fromtimestamp(last_light_timestamp)
        data['last_light'] = last_light

        dictionary[url_loc] = data

    return dictionary


import math

import torch
import torch.nn as nn
import torch.utils.checkpoint as cp
from mmcv.cnn import build_conv_layer, build_norm_layer, constant_init, ConvModule, xavier_init, kaiming_init
from mmdet.utils import get_root_logger
from mmcv.runner import load_checkpoint

from ..builder import BACKBONES
from .resnet import Bottleneck as _Bottleneck
from .resnet import ResNet
import torch.utils.model_zoo as model_zoo
import torch.nn.functional as F
from torch.nn.modules.batchnorm import _BatchNorm

class Bottle2neck(_Bottleneck):
    expansion = 4

    def __init__(self,
                 inplanes,
                 planes,
                 scales=4,
                 base_width=26,
                 base_channels=64,
                 stage_type='normal',
                 sac=None,
                 **kwargs):
        """Bottle2neck block for Res2Net.

        If style is "pytorch", the stride-two layer is the 3x3 conv layer, if
        it is "caffe", the stride-two layer is the first 1x1 conv layer.
        """
        super(Bottle2neck, self).__init__(inplanes, planes, **kwargs)
        assert scales > 1, 'Res2Net degenerates to ResNet when scales = 1.'
        assert sac is None or isinstance(sac, dict)
        self.sac = sac
        self.with_sac = sac is not None

        width = int(math.floor(self.planes * (base_width / base_channels)))

        self.norm1_name, norm1 = build_norm_layer(
            self.norm_cfg, width * scales, postfix=1)
        self.norm3_name, norm3 = build_norm_layer(
            self.norm_cfg, self.planes * self.expansion, postfix=3)

        self.conv1 = build_conv_layer(
            self.conv_cfg,
            self.inplanes,
            width * scales,
            kernel_size=1,
            stride=self.conv1_stride,
            bias=False)
        self.add_module(self.norm1_name, norm1)

        if stage_type == 'stage' and self.conv2_stride != 1:
            self.pool = nn.AvgPool2d(
                kernel_size=3, stride=self.conv2_stride, padding=1)
        convs = []
        bns = []

        fallback_on_stride = False
        if self.with_dcn:
            fallback_on_stride = self.dcn.pop('fallback_on_stride', False)

        if self.with_sac:
            for i in range(scales - 1):
                convs.append(
                    build_conv_layer(
                        self.sac,
                        width,
                        width,
                        kernel_size=3,
                        stride=self.conv2_stride,
                        padding=self.dilation,
                        dilation=self.dilation,
                        bias=False))
                bns.append(
                    build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1])
            self.convs = nn.ModuleList(convs)
            self.bns = nn.ModuleList(bns)
        elif not self.with_dcn or fallback_on_stride:
            for i in range(scales - 1):
                convs.append(
                    build_conv_layer(
                        self.conv_cfg,
                        width,
                        width,
                        kernel_size=3,
                        stride=self.conv2_stride,
                        padding=self.dilation,
                        dilation=self.dilation,
                        bias=False))
                bns.append(
                    build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1])
            self.convs = nn.ModuleList(convs)
            self.bns = nn.ModuleList(bns)
        else:
            assert self.conv_cfg is None, 'conv_cfg must be None for DCN'
            for i in range(scales - 1):
                convs.append(
                    build_conv_layer(
                        self.dcn,
                        width,
                        width,
                        kernel_size=3,
                        stride=self.conv2_stride,
                        padding=self.dilation,
                        dilation=self.dilation,
                        bias=False))
                bns.append(
                    build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1])
            self.convs = nn.ModuleList(convs)
            self.bns = nn.ModuleList(bns)

        self.conv3 = build_conv_layer(
            self.conv_cfg,
            width * scales,
            self.planes * self.expansion,
            kernel_size=1,
            bias=False)
        self.add_module(self.norm3_name, norm3)

        self.stage_type = stage_type
        self.scales = scales
        self.width = width
        delattr(self, 'conv2')
        delattr(self, self.norm2_name)

    def forward(self, x):
        """Forward function."""

        def _inner_forward(x):
            identity = x

            out = self.conv1(x)
            out = self.norm1(out)
            out = self.relu(out)

            if self.with_plugins:
                out = self.forward_plugin(out, self.after_conv1_plugin_names)

            spx = torch.split(out, self.width, 1)
            sp = self.convs[0](spx[0].contiguous())
            sp = self.relu(self.bns[0](sp))
            out = sp
            for i in range(1, self.scales - 1):
                if self.stage_type == 'stage':
                    sp = spx[i]
                else:
                    sp = sp + spx[i]
                sp = self.convs[i](sp.contiguous())
                sp = self.relu(self.bns[i](sp))
                out = torch.cat((out, sp), 1)

            if self.stage_type == 'normal' or self.conv2_stride == 1:
                out = torch.cat((out, spx[self.scales - 1]), 1)
            elif self.stage_type == 'stage':
                out = torch.cat((out, self.pool(spx[self.scales - 1])), 1)

            if self.with_plugins:
                out = self.forward_plugin(out, self.after_conv2_plugin_names)

            out = self.conv3(out)
            out = self.norm3(out)

            if self.with_plugins:
                out = self.forward_plugin(out, self.after_conv3_plugin_names)

            if self.downsample is not None:
                identity = self.downsample(x)

            out += identity

            return out

        if self.with_cp and x.requires_grad:
            out = cp.checkpoint(_inner_forward, x)
        else:
            out = _inner_forward(x)

        out = self.relu(out)

        return out

class Res2Layer(nn.Sequential):
    """Res2Layer to build Res2Net style backbone.

    Args:
        block (nn.Module): block used to build ResLayer.
        inplanes (int): inplanes of block.
        planes (int): planes of block.
        num_blocks (int): number of blocks.
        stride (int): stride of the first block. Default: 1
        avg_down (bool): Use AvgPool instead of stride conv when
            downsampling in the bottle2neck. Default: False
        conv_cfg (dict): dictionary to construct and config conv layer.
            Default: None
        norm_cfg (dict): dictionary to construct and config norm layer.
            Default: dict(type='BN')
        scales (int): Scales used in Res2Net. Default: 4
        base_width (int): Basic width of each scale. Default: 26
    """

    def __init__(self,
                 block,
                 inplanes,
                 planes,
                 num_blocks,
                 stride=1,
                 avg_down=True,
                 conv_cfg=None,
                 norm_cfg=dict(type='BN'),
                 scales=4,
                 base_width=26,
                 **kwargs):
        self.block = block

        downsample = None
        if stride != 1 or inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.AvgPool2d(
                    kernel_size=stride,
                    stride=stride,
                    ceil_mode=True,
                    count_include_pad=False),
                build_conv_layer(
                    conv_cfg,
                    inplanes,
                    planes * block.expansion,
                    kernel_size=1,
                    stride=1,
                    bias=False),
                build_norm_layer(norm_cfg, planes * block.expansion)[1],
            )

        layers = []
        layers.append(
            block(
                inplanes=inplanes,
                planes=planes,
                stride=stride,
                downsample=downsample,
                conv_cfg=conv_cfg,
                norm_cfg=norm_cfg,
                scales=scales,
                base_width=base_width,
                stage_type='stage',
                **kwargs))
        inplanes = planes * block.expansion
        for i in range(1, num_blocks):
            layers.append(
                block(
                    inplanes=inplanes,
                    planes=planes,
                    stride=1,
                    conv_cfg=conv_cfg,
                    norm_cfg=norm_cfg,
                    scales=scales,
                    base_width=base_width,
                    **kwargs))
        super(Res2Layer, self).__init__(*layers)

@BACKBONES.register_module()
class Res2Net(ResNet):
    arch_settings = {
        50: (Bottle2neck, (3, 4, 6, 3)),
        101: (Bottle2neck, (3, 4, 23, 3)),
        152: (Bottle2neck, (3, 8, 36, 3))
    }

    def __init__(self,
                 scales=4,
                 base_width=26,
                 style='pytorch',
                 deep_stem=True,
                 avg_down=True,
                 sac=None,
                 stage_with_sac=(False, False, False, False),
                 pretrained=None,
                 **kwargs):
        self.scales = scales
        self.base_width = base_width
        self.sac = sac
        self.stage_with_sac = stage_with_sac
        self.pretrained = pretrained
        super(Res2Net, self).__init__(
            style='pytorch', deep_stem=deep_stem, avg_down=avg_down, **kwargs)
        self.inplanes = self.stem_channels
        self.res_layers = []

        for i, num_blocks in enumerate(self.stage_blocks):
            stride = self.strides[i]
            dilation = self.dilations[i]
            dcn = self.dcn if self.stage_with_dcn[i] else None
            sac = self.sac if self.stage_with_sac[i] else None
            if self.plugins is not None:
                stage_plugins = self.make_stage_plugins(self.plugins, i)
            else:
                stage_plugins = None
            planes = self.base_channels * 2 ** i
            res_layer = self.make_res_layer(
                block=self.block,
                inplanes=self.inplanes,
                planes=planes,
                num_blocks=num_blocks,
                stride=stride,
                dilation=dilation,
                style=self.style,
                avg_down=self.avg_down,
                with_cp=self.with_cp,
                conv_cfg=self.conv_cfg,
                norm_cfg=self.norm_cfg,
                dcn=dcn,
                sac=sac,
                plugins=stage_plugins)
            self.inplanes = planes * self.block.expansion
            layer_name = f'layer{i + 1}'
            self.add_module(layer_name, res_layer)
            self.res_layers.append(layer_name)

        self._freeze_stages()

    def make_res_layer(self, **kwargs):
        return Res2Layer(
            scales=self.scales,
            base_width=self.base_width,
            base_channels=self.base_channels,
            **kwargs)

    def _freeze_stages(self):
        if self.frozen_stages >= 0:
            if self.deep_stem:
                self.conv1.eval()
                for param in self.conv1.parameters():
                    param.requires_grad = False
            else:
                self.norm1.eval()
                for m in [self.conv1, self.norm1]:
                    for param in m.parameters():
                        param.requires_grad = False

        for i in range(1, self.frozen_stages + 1):
            m = getattr(self, f'layer{i}')
            m.eval()
            for param in m.parameters():
                param.requires_grad = False

    def _make_stem_layer(self, in_channels, stem_channels):
        if self.deep_stem:
            self.conv1 = nn.Sequential(
                build_conv_layer(
                    self.conv_cfg,
                    in_channels,
                    stem_channels // 2,
                    kernel_size=3,
                    stride=2,
                    padding=1,
                    bias=False),
                build_norm_layer(self.norm_cfg, stem_channels // 2)[1],
                nn.ReLU(inplace=True),
                build_conv_layer(
                    self.conv_cfg,
                    stem_channels // 2,
                    stem_channels // 2,
                    kernel_size=3,
                    stride=1,
                    padding=1,
                    bias=False),
                build_norm_layer(self.norm_cfg, stem_channels // 2)[1],
                nn.ReLU(inplace=True),
                build_conv_layer(
                    self.conv_cfg,
                    stem_channels // 2,
                    stem_channels,
                    kernel_size=3,
                    stride=1,
                    padding=1,
                    bias=False))
        else:
            self.conv1 = build_conv_layer(
                self.conv_cfg,
                in_channels,
                stem_channels,
                kernel_size=7,
                stride=2,
                padding=3,
                bias=False)
        self.norm1_name, norm1 = build_norm_layer(
            self.norm_cfg, stem_channels, postfix=1)
        self.add_module(self.norm1_name, norm1)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

    def init_weights(self, pretrained=None):
        """Initialize the weights in backbone.

        Args:
            pretrained (str, optional): Path to pre-trained weights.
                Defaults to None.
        """
        if isinstance(pretrained, str):
            #logger = get_root_logger()
            #load_checkpoint(self, filename=pretrained, map_location=lambda storage, loc: storage, strict=False, logger=logger)
            checkpoint = model_zoo.load_url(pretrained, map_location=lambda storage, loc: storage)
            self.load_state_dict(checkpoint, strict=False)

        elif pretrained is None:
            for m in self.modules():
                if isinstance(m, nn.Conv2d):
                    kaiming_init(m)
                elif isinstance(m, (_BatchNorm, nn.GroupNorm)):
                    constant_init(m, 1)

            # if self.dcn is not None:
            #     for m in self.modules():
            #         if isinstance(m, Bottle2neck) and hasattr(
            #                 m.conv2, 'conv_offset'):
            #             constant_init(m.conv2.conv_offset, 0)

            if self.zero_init_residual:
                for m in self.modules():
                    if isinstance(m, Bottle2neck):
                        constant_init(m.norm3, 0)

        else:
            raise TypeError('pretrained must be a str or None')

    def forward(self, x):
        """Forward function."""
        outs = []
        x = self.conv1(x)
        x = self.norm1(x)
        x = self.relu(x)
        x = self.maxpool(x)
        for i, layer_name in enumerate(self.res_layers):
            res_layer = getattr(self, layer_name)
            x = res_layer(x)
            if i in self.out_indices:
                outs.append(x)
        return tuple(outs)


import os
import time
import re
from inspect import signature
import boto3

from retrying import retry
from fabric import Connection
from botocore.config import Config
from botocore.exceptions import ClientError

from test.test_utils import is_pr_context
from . import DEFAULT_REGION, UL_AMI_LIST, LOGGER, BENCHMARK_RESULTS_S3_BUCKET

EC2_INSTANCE_ROLE_NAME = "ec2TestInstanceRole"

def get_ec2_instance_type(default, processor, disable_p3dn=False):
    """
    Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default
    for contexts where the variable is not present (i.e. PR, Nightly, local testing)

    :param default: Default instance type to use - Should never be p3dn
    :param processor: "cpu" or "gpu"
    :param disable_p3dn: Boolean to determine whether or not to run tests on p3dn. If set to true, default
    gpu instance type will be used.

    :return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be
    a list.
    """
    allowed_processors = ("cpu", "gpu")
    p3dn = "p3dn.24xlarge"
    if processor not in allowed_processors:
        raise RuntimeError(
            f"Aborting EC2 test run. Unrecognized processor type {processor}. "
            f"Please choose from {allowed_processors}"
        )
    if default == p3dn:
        raise RuntimeError("Default instance type should never be p3dn.24xlarge")
    instance_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE", default)
    if instance_type == p3dn and disable_p3dn:
        instance_type = default
    return [instance_type]

def get_ec2_accelerator_type(default, processor):
    """
    Get EC2 instance type from associated EC2_[CPU|GPU]_INSTANCE_TYPE env variable, or set it to a default
    for contexts where the variable is not present (i.e. PR, Nightly, local testing)

    :param default: Default instance type to use - Should never be p3dn
    :param processor: "eia"

    :return: one item list of instance type -- this is used to parametrize tests, and parameter is required to be
    a list.
    """
    allowed_processors = "eia"
    if processor not in allowed_processors:
        raise RuntimeError(
            f"Aborting EC2 test run. Unrecognized processor type {processor}. "
            f"Please choose from {allowed_processors}"
        )
    accelerator_type = os.getenv(f"EC2_{processor.upper()}_INSTANCE_TYPE", default)
    return [accelerator_type]

def launch_instance(
    ami_id,
    instance_type,
    ei_accelerator_type,
    ec2_key_name=None,
    region=DEFAULT_REGION,
    user_data=None,
    iam_instance_profile_name=None,
    instance_name="",
):
    """
    Launch an instance
    :param ami_id: AMI ID to be used for launched instance
    :param instance_type: Instance type of launched instance
    :param region: Region where instance will be launched
    :param user_data: Script to run when instance is launched as a str
    :param iam_instance_profile_arn: EC2 Role to be attached
    :param instance_name: Tag to display as Name on EC2 Console
    :return: <dict> Information about the instance that was launched
    """
    if not ami_id:
        raise Exception("No ami_id provided")
    if not ec2_key_name:
        raise Exception("Ec2 Key name must be provided")
    client = boto3.Session(region_name=region).client("ec2")

    # Construct the dictionary with the arguments for API call
    arguments_dict = {
        "KeyName": ec2_key_name,
        "ImageId": ami_id,
        "InstanceType": instance_type,
        "MaxCount": 1,
        "MinCount": 1,
        "TagSpecifications": [
            {"ResourceType": "instance", "Tags": [{"Key": "Name", "Value": f"CI-CD {instance_name}"}],},
        ],
    }
    if user_data:
        arguments_dict["UserData"] = user_data
    if iam_instance_profile_name:
        arguments_dict["IamInstanceProfile"] = {"Name": iam_instance_profile_name}
    if ei_accelerator_type:
        arguments_dict["ElasticInferenceAccelerators"] = ei_accelerator_type
        availability_zones = {
            "us-west": ["us-west-2a", "us-west-2b", "us-west-2c"],
            "us-east": ["us-east-1a", "us-east-1b", "us-east-1c"],
        }
        for a_zone in availability_zones[region]:
            arguments_dict["Placement"] = {"AvailabilityZone": a_zone}
            try:
                response = client.run_instances(**arguments_dict)
                if response and len(response["Instances"]) >= 1:
                    break
            except ClientError as e:
                print(f"Failed to launch in {a_zone} with Error: {e}")
                continue
    else:
        response = client.run_instances(**arguments_dict)

    if not response or len(response["Instances"]) < 1:
        raise Exception(
            "Unable to launch the instance. \
                         Did not return any response"
        )

    return response["Instances"][0]

def get_ec2_client(region):
    return boto3.client("ec2", region_name=region, config=Config(retries={"max_attempts": 10}))

def get_instance_from_id(instance_id, region=DEFAULT_REGION):
    """
    Get instance information using instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <dict> Information about instance with matching instance ID
    """
    if not instance_id:
        raise Exception("No instance id provided")
    client = boto3.Session(region_name=region).client("ec2")
    instance = client.describe_instances(InstanceIds=[instance_id])
    if not instance:
        raise Exception(
            "Unable to launch the instance. \
                         Did not return any reservations object"
        )
    return instance["Reservations"][0]["Instances"][0]

@retry(stop_max_attempt_number=16, wait_fixed=60000)
def get_public_ip(instance_id, region=DEFAULT_REGION):
    """
    Get Public IP of instance using instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <str> IP Address of instance with matching instance ID
    """
    instance = get_instance_from_id(instance_id, region)
    if not instance["PublicIpAddress"]:
        raise Exception("IP address not yet available")
    return instance["PublicIpAddress"]

@retry(stop_max_attempt_number=16, wait_fixed=60000)
def get_public_ip_from_private_dns(private_dns, region=DEFAULT_REGION):
    """
    Get Public IP of instance using private DNS
    :param private_dns:
    :param region:
    :return: <str> IP Address of instance with matching private DNS
    """
    client = boto3.Session(region_name=region).client("ec2")
    response = client.describe_instances(Filters={"Name": "private-dns-name", "Value": [private_dns]})
    return response.get("Reservations")[0].get("Instances")[0].get("PublicIpAddress")

@retry(stop_max_attempt_number=16, wait_fixed=60000)
def get_instance_user(instance_id, region=DEFAULT_REGION):
    """
    Get "ubuntu" or "ec2-user" based on AMI used to launch instance
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <str> user name
    """
    instance = get_instance_from_id(instance_id, region)
    user = "ubuntu" if instance["ImageId"] in UL_AMI_LIST else "ec2-user"
    return user

def get_instance_state(instance_id, region=DEFAULT_REGION):
    """
    Get state of instance using instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <str> State of instance with matching instance ID
    """
    instance = get_instance_from_id(instance_id, region)
    return instance["State"]["Name"]

@retry(stop_max_attempt_number=16, wait_fixed=60000)
def check_instance_state(instance_id, state="running", region=DEFAULT_REGION):
    """
    Compares the instance state with the state argument.
    Retries 8 times with 120 seconds gap between retries.
    :param instance_id: Instance ID to be queried
    :param state: Expected instance state
    :param region: Region where query will be performed
    :return: <str> State of instance with matching instance ID
    """
    instance_state = get_instance_state(instance_id, region)
    if state != instance_state:
        raise Exception(f"Instance {instance_id} not in {state} state")
    return instance_state

def get_system_state(instance_id, region=DEFAULT_REGION):
    """
    Returns health checks state for instances
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <tuple> System state and Instance state of instance with matching instance ID
    """
    if not instance_id:
        raise Exception("No instance id provided")
    client = boto3.Session(region_name=region).client("ec2")
    response = client.describe_instance_status(InstanceIds=[instance_id])
    if not response:
        raise Exception(
            "Unable to launch the instance. \
                         Did not return any reservations object"
        )
    instance_status_list = response["InstanceStatuses"]
    if not instance_status_list:
        raise Exception(
            "Unable to launch the instance. \
                         Did not return any reservations object"
        )
    if len(instance_status_list) < 1:
        raise Exception(
            "The instance id seems to be incorrect {}. \
                         reservations seems to be empty".format(
                instance_id
            )
        )

    instance_status = instance_status_list[0]
    return (
        instance_status["SystemStatus"]["Status"],
        instance_status["InstanceStatus"]["Status"],
    )

@retry(stop_max_attempt_number=96, wait_fixed=10000)
def check_system_state(instance_id, system_status="ok", instance_status="ok", region=DEFAULT_REGION):
    """
    Compares the system state (Health Checks).
    Retries 96 times with 10 seconds gap between retries
    :param instance_id: Instance ID to be queried
    :param system_status: Expected system state
    :param instance_status: Expected instance state
    :param region: Region where query will be performed
    :return: <tuple> System state and Instance state of instance with matching instance ID
    """
    instance_state = get_system_state(instance_id, region=region)
    if system_status != instance_state[0] or instance_status != instance_state[1]:
        raise Exception(
            "Instance {} not in \
                         required state".format(
                instance_id
            )
        )
    return instance_state

def terminate_instance(instance_id, region=DEFAULT_REGION):
    """
    Terminate EC2 instances with matching instance ID
    :param instance_id: Instance ID to be terminated
    :param region: Region where instance is located
    """
    if not instance_id:
        raise Exception("No instance id provided")
    client = boto3.Session(region_name=region).client("ec2")
    response = client.terminate_instances(InstanceIds=[instance_id])
    if not response:
        raise Exception("Unable to terminate instance. No response received.")
    instances_terminated = response["TerminatingInstances"]
    if not instances_terminated:
        raise Exception("Failed to terminate instance.")
    if instances_terminated[0]["InstanceId"] != instance_id:
        raise Exception("Failed to terminate instance. Unknown error.")

def get_instance_type_details(instance_type, region=DEFAULT_REGION):
    """
    Get instance type details for a given instance type
    :param instance_type: Instance type to be queried
    :param region: Region where query will be performed
    :return: <dict> Information about instance type
    """
    client = boto3.client("ec2", region_name=region)
    response = client.describe_instance_types(InstanceTypes=[instance_type])
    if not response or not response["InstanceTypes"]:
        raise Exception("Unable to get instance details. No response received.")
    if response["InstanceTypes"][0]["InstanceType"] != instance_type:
        raise Exception(
            f"Bad response received. Requested {instance_type} "
            f"but got {response['InstanceTypes'][0]['InstanceType']}"
        )
    return response["InstanceTypes"][0]

def get_instance_details(instance_id, region=DEFAULT_REGION):
    """
    Get instance details for instance with given instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <dict> Information about instance with matching instance ID
    """
    if not instance_id:
        raise Exception("No instance id provided")
    instance = get_instance_from_id(instance_id, region=region)
    if not instance:
        raise Exception("Could not find instance")

    return get_instance_type_details(instance["InstanceType"], region=region)

@retry(stop_max_attempt_number=30, wait_fixed=10000)
def get_instance_num_cpus(instance_id, region=DEFAULT_REGION):
    """
    Get number of VCPUs on instance with given instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <int> Number of VCPUs on instance with matching instance ID
    """
    instance_info = get_instance_details(instance_id, region=region)
    return instance_info["VCpuInfo"]["DefaultVCpus"]

@retry(stop_max_attempt_number=30, wait_fixed=10000)
def get_instance_memory(instance_id, region=DEFAULT_REGION):
    """
    Get total RAM available on instance with given instance ID
    :param instance_id: Instance ID to be queried
    :param region: Region where query will be performed
    :return: <int> Total RAM available on instance with matching instance ID
    """
    instance_info = get_instance_details(instance_id, region=region)
    return instance_info["MemoryInfo"]["SizeInMiB"]

@retry(stop_max_attempt_number=30, wait_fixed=10000)
def get_instance_num_gpus(instance_id=None, instance_type=None, region=DEFAULT_REGION):
    """
    Get total number of GPUs on instance with given instance ID
    :param instance_id: Instance ID to be queried
    :param instance_type: Instance Type to be queried
    :param region: Region where query will be performed
    :return: <int> Number of GPUs on instance with matching instance ID
    """
    assert instance_id or instance_type, "Input must be either instance_id or instance_type"
    instance_info = (
        get_instance_type_details(instance_type, region=region)
        if instance_type
        else get_instance_details(instance_id, region=region)
    )
    return sum(gpu_type["Count"] for gpu_type in instance_info["GpuInfo"]["Gpus"])

def get_ec2_fabric_connection(instance_id, instance_pem_file, region):
    """
    establish connection with EC2 instance if necessary
    :param instance_id: ec2_instance id
    :param instance_pem_file: instance key name
    :param region: Region where ec2 instance is launched
    :return: Fabric connection object
    """
    user = get_instance_user(instance_id, region=region)
    conn = Connection(
        user=user, host=get_public_ip(instance_id, region), connect_kwargs={"key_filename": [instance_pem_file]},
    )
    return conn

def execute_ec2_training_test(
    connection,
    ecr_uri,
    test_cmd,
    region=DEFAULT_REGION,
    executable="bash",
    large_shm=False,
    host_network=False,
    container_name="ec2_training_container",
):
    if executable not in ("bash", "python"):
        raise RuntimeError(f"This function only supports executing bash or python commands on containers")
    if executable == "bash":
        executable = os.path.join(os.sep, "bin", "bash")
    docker_cmd = "nvidia-docker" if "gpu" in ecr_uri else "docker"
    container_test_local_dir = os.path.join("$HOME", "container_tests")

    # Make sure we are logged into ECR so we can pull the image
    connection.run(f"$(aws ecr get-login --no-include-email --region {region})", hide=True)

    # Run training command
    shm_setting = '--shm-size="1g"' if large_shm else ""
    network = '--network="host" ' if host_network else ""
    connection.run(
        f"{docker_cmd} run --name {container_name} {network}-v {container_test_local_dir}:{os.path.join(os.sep, 'test')}"
        f" {shm_setting} -itd {ecr_uri}",
        hide=True,
    )
    return connection.run(
        f"{docker_cmd} exec --user root {container_name} {executable} -c '{test_cmd}'", hide=True, timeout=3000,
    )

def execute_ec2_inference_test(connection, ecr_uri, test_cmd, region=DEFAULT_REGION):
    docker_cmd = "nvidia-docker" if "gpu" in ecr_uri else "docker"
    container_test_local_dir = os.path.join("$HOME", "container_tests")

    # Make sure we are logged into ECR so we can pull the image
    connection.run(f"$(aws ecr get-login --no-include-email --region {region})", hide=True)

    # Run training command
    connection.run(
        f"{docker_cmd} run --name ec2_inference_container -v {container_test_local_dir}:{os.path.join(os.sep, 'test')}"
        f" -itd {ecr_uri} bash",
        hide=True,
    )
    connection.run(
        f"{docker_cmd} exec --user root ec2_inference_container {os.path.join(os.sep, 'bin', 'bash')} -c '{test_cmd}'",
        hide=True,
        timeout=3000,
    )

def execute_ec2_training_performance_test(
    connection, ecr_uri, test_cmd, region=DEFAULT_REGION, post_process=None, data_source="", threshold=None,
):
    docker_cmd = "nvidia-docker" if "gpu" in ecr_uri else "docker"
    container_test_local_dir = os.path.join("$HOME", "container_tests")

    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
    log_name = f"{data_source}_results_{os.getenv('CODEBUILD_RESOLVED_SOURCE_VERSION')}_{timestamp}.txt"
    log_location = os.path.join(container_test_local_dir, "benchmark", "logs", log_name)

    # Make sure we are logged into ECR so we can pull the image
    connection.run(f"$(aws ecr get-login --no-include-email --region {region})", hide=True)

    connection.run(f"{docker_cmd} pull -q {ecr_uri}")

    # Run training command, display benchmark results to console
    connection.run(
        f"{docker_cmd} run --user root "
        f"-e LOG_FILE={os.path.join(os.sep, 'test', 'benchmark', 'logs', log_name)} "
        f"-e PR_CONTEXT={1 if is_pr_context() else 0} "
        f"-v {container_test_local_dir}:{os.path.join(os.sep, 'test')} {ecr_uri} "
        f"{os.path.join(os.sep, 'bin', 'bash')} -c {test_cmd}"
    )
    ec2_performance_upload_result_to_s3_and_validate(
        connection, ecr_uri, log_location, data_source, threshold, post_process, log_name,
    )

def execute_ec2_inference_performance_test(
    connection, ecr_uri, test_cmd, region=DEFAULT_REGION, post_process=None, data_source="", threshold=None,
):
    docker_cmd = "nvidia-docker" if "gpu" in ecr_uri else "docker"
    container_test_local_dir = os.path.join("$HOME", "container_tests")
    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
    log_name = f"{data_source}_results_{os.getenv('CODEBUILD_RESOLVED_SOURCE_VERSION')}_{timestamp}.txt"
    # Make sure we are logged into ECR so we can pull the image
    connection.run(f"$(aws ecr get-login --no-include-email --region {region})", hide=True)
    connection.run(f"{docker_cmd} pull -q {ecr_uri}")

    # Run training command, display benchmark results to console
    repo_name, image_tag = ecr_uri.split("/")[-1].split(":")
    container_name = f"{repo_name}-performance-{image_tag}-ec2"
    connection.run(
        f"{docker_cmd} run -d --name {container_name} "
        f"-e LOG_FILE={os.path.join(os.sep, 'test', 'benchmark', 'logs', log_name)} "
        f"-v {container_test_local_dir}:{os.path.join(os.sep, 'test')} {ecr_uri}"
    )
    try:
        connection.run(
            f"{docker_cmd} exec --user root {container_name} " f"{os.path.join(os.sep, 'bin', 'bash')} -c {test_cmd}"
        )
    except Exception as e:
        raise Exception("Failed to exec benchmark command.\n", e)
    finally:
        connection.run(f"docker rm -f {container_name}")
    log_location = os.path.join(container_test_local_dir, "benchmark", "logs", log_name)
    ec2_performance_upload_result_to_s3_and_validate(
        connection, ecr_uri, log_location, data_source, threshold, post_process, log_name,
    )

def ec2_performance_upload_result_to_s3_and_validate(
    connection, ecr_uri, log_location, data_source, threshold, post_process, log_name
):
    framework = "tensorflow" if "tensorflow" in ecr_uri else "mxnet" if "mxnet" in ecr_uri else "pytorch"
    framework_version = re.search(r"\d+(\.\d+){2}", ecr_uri).group()
    py_version = "py2" if "py2" in ecr_uri else "py37" if "py37" in ecr_uri else "py3"
    processor = "gpu" if "gpu" in ecr_uri else "cpu"
    work_type = "training" if "training" in ecr_uri else "inference"
    s3_location = os.path.join(
        BENCHMARK_RESULTS_S3_BUCKET, framework, framework_version, "ec2", work_type, processor, py_version, log_name,
    )
    params = {"connection": connection, "log_location": log_location}
    if "threshold" in signature(post_process).parameters:
        params["threshold"] = threshold
    performance_number = post_process(**params)
    unit = (
        "s"
        if work_type == "inference" and framework == "tensorflow"
        else "ms"
        if work_type == "inference" and framework == "pytorch"
        else "s/epoch"
        if work_type == "training" and framework == "pytorch" and data_source == "imagenet"
        else "images/sec"
    )
    description = "p99 latency " if unit == "s" or unit == "ms" else ""
    for k, v in performance_number.items():
        performance_statement = (
            f"{framework} {framework_version} ec2 {work_type} {processor} {py_version} "
            f"{data_source} {k} {description}: {v} {unit}, threshold: {threshold[k]} {unit}"
        )
        connection.run(f"echo {performance_statement} | sudo tee -a {log_location}")
        LOGGER.info(f"{performance_statement}")
    connection.run(f"aws s3 cp {log_location} {s3_location}")
    LOGGER.info(f"To retrieve complete benchmark log, check {s3_location}")

    def _assertion_results():
        if "Cost" in performance_number:
            return performance_number["Cost"] < threshold["Cost"]
        if "Throughput" in performance_number:
            return performance_number["Throughput"] > threshold["Throughput"]
        if len(performance_number) == 0:
            return False
        failure_count = 0
        for k, v in performance_number.items():
            if v > threshold[k]:
                failure_count += 1
        return failure_count <= 2

    for _ in performance_number:
        assert _assertion_results(), (
            f"{framework} {framework_version} ec2 {work_type} {processor} {py_version} {data_source} "
            f"Benchmark Result {performance_number} does not reach the threshold {threshold}"
        )

def post_process_inference(connection, log_location, threshold):
    log_content = connection.run(f"cat {log_location}").stdout.split("\n")
    performance_number = {}
    for line in log_content:
        if "p99" in line:
            for key in threshold.keys():
                if key in line:
                    performance_number[key] = float(
                        re.search(r"(p99[ ]*(Latency)?[ ]*:[ ]*)(?P<result>[0-9]+\.?[0-9]+)", line,).group("result")
                    )
                    break
    return performance_number

def post_process_mxnet_ec2_performance(connection, log_location):
    log_content = connection.run(f"cat {log_location}").stdout.split("\n")
    total = 0.0
    n = 0
    for line in log_content:
        if "samples/sec" in line and "warmup" not in line:
            throughput = re.search(r"((?P<throughput>[0-9]+\.?[0-9]+)[ ]+samples/sec)", line).group("throughput")
            total += float(throughput)
            n += 1
    if total and n:
        return {"Throughput": total / n}
    else:
        raise ValueError("total: {}; n: {} -- something went wrong".format(total, n))


# -*- coding: utf-8 -*-

import time
import unittest
from threading import Lock, Thread
from freezegun import freeze_time

import cached_property

def CheckFactory(cached_property_decorator, threadsafe=False):
    """
    Create dynamically a Check class whose add_cached method is decorated by
    the cached_property_decorator.
    """

    class Check(object):
        def __init__(self):
            self.control_total = 0
            self.cached_total = 0
            self.lock = Lock()

        @property
        def add_control(self):
            self.control_total += 1
            return self.control_total

        @cached_property_decorator
        def add_cached(self):
            if threadsafe:
                time.sleep(1)
                # Need to guard this since += isn't atomic.
                with self.lock:
                    self.cached_total += 1
            else:
                self.cached_total += 1

            return self.cached_total

        def run_threads(self, num_threads):
            threads = []
            for _ in range(num_threads):
                thread = Thread(target=lambda: self.add_cached)
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()

    return Check

class TestCachedProperty(unittest.TestCase):
    """Tests for cached_property"""

    cached_property_factory = cached_property.cached_property

    def assert_control(self, check, expected):
        """
        Assert that both `add_control` and 'control_total` equal `expected`
        """
        self.assertEqual(check.add_control, expected)
        self.assertEqual(check.control_total, expected)

    def assert_cached(self, check, expected):
        """
        Assert that both `add_cached` and 'cached_total` equal `expected`
        """
        self.assertEqual(check.add_cached, expected)
        self.assertEqual(check.cached_total, expected)

    def test_cached_property(self):
        Check = CheckFactory(self.cached_property_factory)
        check = Check()

        # The control shows that we can continue to add 1
        self.assert_control(check, 1)
        self.assert_control(check, 2)

        # The cached version demonstrates how nothing is added after the first
        self.assert_cached(check, 1)
        self.assert_cached(check, 1)

        # The cache does not expire
        with freeze_time("9999-01-01"):
            self.assert_cached(check, 1)

        # Typically descriptors return themselves if accessed though the class
        # rather than through an instance.
        self.assertTrue(isinstance(Check.add_cached, self.cached_property_factory))

    def test_reset_cached_property(self):
        Check = CheckFactory(self.cached_property_factory)
        check = Check()

        # Run standard cache assertion
        self.assert_cached(check, 1)
        self.assert_cached(check, 1)

        # Clear the cache
        del check.add_cached

        # Value is cached again after the next access
        self.assert_cached(check, 2)
        self.assert_cached(check, 2)

    def test_none_cached_property(self):
        class Check(object):
            def __init__(self):
                self.cached_total = None

            @self.cached_property_factory
            def add_cached(self):
                return self.cached_total

        self.assert_cached(Check(), None)

    def test_set_cached_property(self):
        Check = CheckFactory(self.cached_property_factory)
        check = Check()
        check.add_cached = "foo"
        self.assertEqual(check.add_cached, "foo")
        self.assertEqual(check.cached_total, 0)

    def test_threads(self):
        Check = CheckFactory(self.cached_property_factory, threadsafe=True)
        check = Check()
        num_threads = 5

        # cached_property_with_ttl is *not* thread-safe!
        check.run_threads(num_threads)
        # This assertion hinges on the fact the system executing the test can
        # spawn and start running num_threads threads within the sleep period
        # (defined in the Check class as 1 second). If num_threads were to be
        # massively increased (try 10000), the actual value returned would be
        # between 1 and num_threads, depending on thread scheduling and
        # preemption.
        self.assert_cached(check, num_threads)
        self.assert_cached(check, num_threads)

        # The cache does not expire
        with freeze_time("9999-01-01"):
            check.run_threads(num_threads)
            self.assert_cached(check, num_threads)
            self.assert_cached(check, num_threads)

class TestThreadedCachedProperty(TestCachedProperty):
    """Tests for threaded_cached_property"""

    cached_property_factory = cached_property.threaded_cached_property

    def test_threads(self):
        Check = CheckFactory(self.cached_property_factory, threadsafe=True)
        check = Check()
        num_threads = 5

        # threaded_cached_property_with_ttl is thread-safe
        check.run_threads(num_threads)
        self.assert_cached(check, 1)
        self.assert_cached(check, 1)

        # The cache does not expire
        with freeze_time("9999-01-01"):
            check.run_threads(num_threads)
            self.assert_cached(check, 1)
            self.assert_cached(check, 1)

class TestCachedPropertyWithTTL(TestCachedProperty):
    """Tests for cached_property_with_ttl"""

    cached_property_factory = cached_property.cached_property_with_ttl

    def test_ttl_expiry(self):
        Check = CheckFactory(self.cached_property_factory(ttl=100000))
        check = Check()

        # Run standard cache assertion
        self.assert_cached(check, 1)
        self.assert_cached(check, 1)

        # The cache expires in the future
        with freeze_time("9999-01-01"):
            self.assert_cached(check, 2)
            self.assert_cached(check, 2)

        # Things are not reverted when we are back to the present
        self.assert_cached(check, 2)
        self.assert_cached(check, 2)

    def test_threads_ttl_expiry(self):
        Check = CheckFactory(self.cached_property_factory(ttl=100000), threadsafe=True)
        check = Check()
        num_threads = 5

        # Same as in test_threads
        check.run_threads(num_threads)
        self.assert_cached(check, num_threads)
        self.assert_cached(check, num_threads)

        # The cache expires in the future
        with freeze_time("9999-01-01"):
            check.run_threads(num_threads)
            self.assert_cached(check, 2 * num_threads)
            self.assert_cached(check, 2 * num_threads)

        # Things are not reverted when we are back to the present
        self.assert_cached(check, 2 * num_threads)
        self.assert_cached(check, 2 * num_threads)

class TestThreadedCachedPropertyWithTTL(
    TestThreadedCachedProperty, TestCachedPropertyWithTTL
):
    """Tests for threaded_cached_property_with_ttl"""

    cached_property_factory = cached_property.threaded_cached_property_with_ttl

    def test_threads_ttl_expiry(self):
        Check = CheckFactory(self.cached_property_factory(ttl=100000), threadsafe=True)
        check = Check()
        num_threads = 5

        # Same as in test_threads
        check.run_threads(num_threads)
        self.assert_cached(check, 1)
        self.assert_cached(check, 1)

        # The cache expires in the future
        with freeze_time("9999-01-01"):
            check.run_threads(num_threads)
            self.assert_cached(check, 2)
            self.assert_cached(check, 2)

        # Things are not reverted when we are back to the present
        self.assert_cached(check, 2)
        self.assert_cached(check, 2)


import os
import pandas as pd
import plotly.graph_objects as go
import streamlit as st

st.set_page_config(page_title="Stock Dashboard", page_icon=":chart_with_upwards_trend:", layout="wide")

# Load stock data from CSV files
@st.cache(allow_output_mutation=True)
def load_stock_data():
    folder_path = "equity_data"
    csv_files = [file for file in os.listdir(folder_path) if file.endswith(".csv")]
    data = {}

    for file in csv_files:
        stock_name = os.path.splitext(file)[0]  # Extract stock name from file name
        file_path = os.path.join(folder_path, file)
        df = pd.read_csv(file_path, parse_dates=["date"],index_col=[0])
        # df = df.drop("Unnamed: 0", axis=1)  # Remove the "Unnamed: 0" column
        data[stock_name] = df

    return data

stock_data = load_stock_data()

# Sidebar filters
st.sidebar.header("Filters")
stock_name = st.sidebar.selectbox("Select Stock", options=list(stock_data.keys()))

# Apply filters
filtered_df = stock_data[stock_name]

# Main page content
st.title(f"Financial Dashboard - {stock_name}")
st.markdown("---")

# Display filtered data
st.subheader(f"Stock Data - {stock_name}")
st.dataframe(filtered_df)

# Line Chart: Stock Closing Price over Time
fig = go.Figure()
fig.add_trace(go.Scatter(x=filtered_df["date"], y=filtered_df["close"], name="Closing Price"))
fig.update_layout(title="Stock Closing Price over Time", xaxis_title="Date", yaxis_title="Price")
st.plotly_chart(fig)


import logging
import colorlog
import sys

def init_log(*args,lname="Log",loglevel=logging.DEBUG,filename="initlog"):
  stdout = colorlog.StreamHandler(stream=sys.stdout)
  fmt = colorlog.ColoredFormatter(
    "%(name)s: %(white)s%(asctime)s%(reset)s | %(log_color)s%(levelname)s%(reset)s | %(blue)s%(filename)s:%(lineno)s - %(funcName)20s()%(reset)s | %(process)d >>> %(log_color)s%(message)s%(reset)s"
  )
  stdout.setFormatter(fmt)
  logger=logging.getLogger(lname)
  logger.addHandler(stdout)
  logger.setLevel(loglevel)
  if (filename!=""):
    fileHandler = logging.FileHandler(f"{filename}.log")
    fileHandler.setFormatter(fmt)
    logger.addHandler(fileHandler)
  logger.debug("Fin logger")
  return logger


from Crypto import Random
from Crypto.Cipher import AES

KEY = '2018081620180816'

"""

将字符串加密

:param raw 原始字符串

"""

def encrypt_str(raw):
    iv = Random.new().read(AES.block_size)  # block_size = 16
    cipher = AES.new(KEY, AES.MODE_CFB, iv)
    return iv + cipher.encrypt(raw)

"""

将字符串解密

:param raw 加密后的字符串

"""

def decrypt_str(data):
    iv = data[:16]
    cipher = AES.new(KEY, AES.MODE_CFB, iv)
    return str(cipher.decrypt(data[16:]), 'utf-8')


#scipu积分
import numpy as np
import scipy.integrate as ig
import math
#一重积分
f1=lambda x:np.exp(-x**2)#标准正态分布的lambda表示
#以函数形式定义标准正态分布
def f2(x):
    y=np.exp(-x**2)
    return y
i=ig.quad(f1,0,5)
print(i)#quad函数返回两个值，第一个值是积分的值，第二个值是对积分值的绝对误差估计。
print('------------------------------------------------------')
#含参数的的积分
def f3(x,a,b):
    y=a*(x**2)+b
    return y
i1=ig.quad(f3,0,1,args=(3,1))
print(i1)

#二重积分
f=lambda x,y:19*x*y                     #被积函数
g=lambda x:0                            #y的下限函数
h=lambda y:math.sqrt(1-4*y**2)          #y的上限函数
i2=ig.dblquad(f,0,0.5,g,h)
print(i2)


import config_store as config
import sqlite3
import glob
import csv

def initDB():
  config.dbconn = sqlite3.connect(config.dir_path + "/db/data.db", check_same_thread=False)
  config.db = config.dbconn.cursor()
  scanFails()

def scanFails():
  for file in glob.glob("assets/f2d/*.txt"):
    result = config.db.execute("SELECT * FROM files WHERE filename = ?", (file,))
    if result.fetchone() is None:
      importFail(file)

def importFail(filename):
  csv.register_dialect('piper', delimiter='|', quoting=csv.QUOTE_NONE)
  rows = []
  with open(filename, "r") as file:
    for row in csv.DictReader(file, dialect="piper"):
      rows.append(
        (
          row["CUSIP"],
          row["SYMBOL"],
          row["DESCRIPTION"]
        )
      )
  config.db.executemany(
    "INSERT INTO fails_to_deliver (cusip, ticker, description) VALUES (?, ?, ?)",
    rows
  )
  config.db.execute(
    "INSERT INTO files (filename, scanned) VALUES (?, ?)",
    (filename, 1)
  )

  config.dbconn.commit()


import csv

import datasets

_CITATION = """\
@inproceedings{hateoffensive,
title = {Automated Hate Speech Detection and the Problem of Offensive Language},
author = {Davidson, Thomas and Warmsley, Dana and Macy, Michael and Weber, Ingmar},
booktitle = {Proceedings of the 11th International AAAI Conference on Web and Social Media},
series = {ICWSM '17},
year = {2017},
location = {Montreal, Canada},
pages = {512-515}
}
"""

_DESCRIPTION = """\
An annotated dataset for hate speech and offensive language detection on tweets.
"""

_HOMEPAGE = "https://github.com/t-davidson/hate-speech-and-offensive-language"

_LICENSE = "MIT"

_URL = "https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.csv"

_CLASS_MAP = {
    "0": "hate speech",
    "1": "offensive language",
    "2": "neither",
}

class HateSpeechOffensive(datasets.GeneratorBasedBuilder):
    """An annotated dataset for hate speech and offensive language detection on tweets."""

    VERSION = datasets.Version("1.0.0")

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                {
                    "count": datasets.Value("int64"),
                    "hate_speech_count": datasets.Value("int64"),
                    "offensive_language_count": datasets.Value("int64"),
                    "neither_count": datasets.Value("int64"),
                    "class": datasets.ClassLabel(names=["hate speech", "offensive language", "neither"]),
                    "tweet": datasets.Value("string"),
                }
            ),
            supervised_keys=("tweet", "class"),
            homepage=_HOMEPAGE,
            license=_LICENSE,
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
        """Returns SplitGenerators."""

        data_file = dl_manager.download_and_extract(_URL)
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={
                    "filepath": data_file,
                },
            ),
        ]

    def _generate_examples(self, filepath):
        """Yields examples."""

        with open(filepath, encoding="utf-8") as f:
            reader = csv.reader(f)
            for id_, row in enumerate(reader):
                if id_ == 0:
                    continue

                yield id_, {
                    "count": row[1],
                    "hate_speech_count": row[2],
                    "offensive_language_count": row[3],
                    "neither_count": row[4],
                    "class": _CLASS_MAP[row[5]],
                    "tweet": row[6],
                }

def get_hsol(split='train'):
    '''
    >> print(data[0])
    {'document': "japan 's nec corp. and UNK computer corp. of the united states said wednesday they had agreed to join forces in supercomputer sales .",
    'summary': 'nec UNK in computer sales tie-up'}
    '''
    raw_data = HateSpeechOffensive()
    raw_data.download_and_prepare()
    data = raw_data.as_dataset(split=split)
    return data