#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Edit_OEM_ROM_Project
Copyright (C) 2022 Abdalrohman Alnasier

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import argparse
import os
import re
import sys
from pathlib import Path
from typing import List, Tuple

from loguru import logger

from .config import Configs
from .img2sdat.img2sdat import img2sdat
from .interface import Colors, Fx, Mv
from .utils import (
    cat,
    list_folder,
    remove,
    run_command,
)
from .variables import CONFIG_FILE

# Check if the config file exists and write default values if not
config_file_path = Path(CONFIG_FILE)
config = Configs(config_file_path)

class CreateExt4:
    """
    Class for creating ext4 images.

    Attributes:
        configs (Configs): The Configs class instance.
        mke2fs (str): The path to the mke2fs executable.
        mke2fs_conf (str): The path to the mke2fs configuration file.
        e2fsdroid (str): The path to the e2fsdroid executable.
        img2simg (str): The path to the img2simg executable.
        brotli_tool (str): The path to the brotli executable.
        main_project (str): The main project path loaded from the configuration file.
        config_dir (Path): The path to the configuration directory.
        build_dir (Path): The path to the build directory.
        out_dir (Path): The path to the output directory.

    Methods:
        dump_data(file: str) -> Tuple[str, str, str, str, str, str]: Extracts specific data from the output of the tune2fs command.
        make_ext4(raw: bool = False, sparse: bool = False, brotli: bool = False, active: bool = True, list_build: list = "") -> None: Make ext4 filesystem.
        correct_contexts(self, fs_context_file: str) -> None: Correct the file contexts and add additional ones.
        define_file_contexts(self, fs_context_file: str) -> None: Define the file contexts.
    """

    def __init__(self):
        """
        Construct all the necessary attributes.
        """
        # Load configuration values from the configuration file
        self.configs = config
        self.mke2fs = self.configs.load_config("LINUX", "mke2fs")
        self.mke2fs_conf = self.configs.load_config("LINUX", "mke2fs_conf")
        self.e2fsdroid = self.configs.load_config("LINUX", "e2fsdroid")
        self.img2simg = self.configs.load_config("LINUX", "img2simg")
        self.brotli_tool = self.configs.load_config("LINUX", "brotli")

        # Load main project path from the configuration file
        self.main_project = self.configs.load_config("MAIN", "main_project")

        # Construct paths to the config, build, and output directories
        self.config_dir = Path(self.main_project, "Config")
        self.build_dir = Path(self.main_project, "Build")
        self.out_dir = Path(self.main_project, "Output")

    def dump_data(self, file: str) -> Tuple[str, str, str, str, str, str]:
        """Extracts specific data from the output of the tune2fs command.

        Parameters:
            file: The path to the file containing the output of the tune2fs command.

        Returns:
            A tuple containing the extracted data in the following order:
            - UUID
            - Inode size
            - Inode count
            - Block size
            - Reserved percent
            - Partition size
        """
        # Define the search patterns to extract specific data from the file
        search_patterns = {
            "uuid": r"Filesystem UUID:\s*(\S+)",
            "inode_size": r"Inode size:\s*(\d+)",
            "reserved_percent": r"Reserved block count:\s*(\d+)",
            "block_size": r"Block size:\s*(\d+)",
            "inode_count": r"Inode count:\s*(\d+)",
            "part_size": r"Partition Size:\s*(\d+)",
        }
        # Create an empty dictionary to store the extracted data
        data = {}
        # Read the entire file
        with open(file, "r", encoding="UTF-8") as f:
            file_content = f.read()
        # Extract data using regular expressions
        for key, pattern in search_patterns.items():
            match = re.search(pattern, file_content)
            if match:
                data[key] = match.group(1)
        # Return the extracted data as a tuple
        return (
            data.get("uuid", ""),
            data.get("inode_size", ""),
            data.get("inode_count", ""),
            data.get("block_size", ""),
            data.get("reserved_percent", ""),
            data.get("part_size", ""),
        )

    def __print_images(self) -> List[str]:
        """Return a list of images to build.

        Returns:
            list_build (List[str]): A list of images to build.
        """
        dict_build = list_folder(self.out_dir)
        logger.info(f"Images list in {self.main_project}: ")
        logger.info(f"{Fx.bold}{Colors.yellow}Type [all/a] to choose all images")
        logger.info(f"{Fx.bold}{Colors.yellow}Type [exit/e] to exit{Mv.d(1)}")
        try:
            logger.info("Choose partition(s) to build: ")
            ans = input("Enter the partition number(s) [1, 2, ...]: ").split(",")
            logger.info(f"Your choice is {ans}.")
            if ans == ["all"] or ans == ["a"]:
                return list(dict_build.values())
            elif ans == ["exit"] or ans == ["e"]:
                logger.info("Exiting...")
                raise SystemExit(0)
            else:
                return [dict_build[int(num_part)] for num_part in ans]
        except KeyboardInterrupt:
            logger.info("Terminating...")
            sys.exit(1)
        except (KeyError, ValueError) as e:
            logger.exception(f"Terminating. Invalid input {e}.")
            sys.exit(1)
        except Exception as e:
            logger.exception(f"An error occurred {e}.")
            sys.exit(1)

    def make_ext4(
            self,
            raw: bool = False,
            sparse: bool = False,
            brotli: bool = False,
            active: bool = True,
            list_build: list = "",
    ) -> None:
        """Make ext4 filesystem.

        Parameters:
            raw (bool, optional): If True, create a raw ext4 image. Defaults to False.
            sparse (bool, optional): If True, create a sparse ext4 image. Defaults to False.
            brotli (bool, optional): If True, compress the ext4 image using brotli. Defaults to False.
            active (bool, optional): If True, use the images returned by the __print_images() method. Defaults to True.
            list_build (str, optional): A list of images to build. Defaults to "".
        """
        images_build = self.__print_images() if active else list_build
        for partition in images_build:
            if raw:
                part_path = os.path.join(self.out_dir, partition)
                filesystem_config = os.path.join(
                    self.config_dir, partition + "_filesystem_config.txt"
                )
                file_contexts = os.path.join(
                    self.config_dir, partition + "_file_contexts.txt"
                )
                if not os.path.exists(part_path):
                    continue
                file_features = os.path.join(
                    self.config_dir, partition + "_filesystem_features.txt"
                )
                try:
                    (
                        uuid,
                        inode_size,
                        inode_count,
                        block_size,
                        reserved_percent,
                        part_size,
                    ) = self.dump_data(file_features)
                except UnboundLocalError:
                    logger.exception(f"Check {file_features}")
                    sys.exit(1)
                # Truncate output file since mke2fs will keep verity section in existing file
                with open(
                        os.path.join(self.build_dir, partition + ".img"), "w"
                ) as output_file:
                    output_file.truncate()
                label = partition
                last_mounted_directory = "/" + partition
                # run mke2fs
                if partition in ("system", "system_a"):
                    last_mounted_directory = "/"
                # FIXME delete this after fix extract_erofs to write inode_size to info file
                inode_size = 256
                reserved_percent = 0
                block_size = 4096
                mke2fs_cmd = [
                    self.mke2fs,
                    "-O",
                    "^has_journal",
                    "-L",
                    label,
                    "-N",
                    str(inode_count),
                    "-I",
                    str(inode_size),
                    "-M",
                    last_mounted_directory,
                    "-m",
                    str(reserved_percent),
                    "-U",
                    uuid,
                    "-t",
                    "ext4",
                    "-b",
                    str(block_size),
                    os.path.join(self.build_dir, partition + ".img"),
                    str(int(part_size) // int(block_size)),
                ]
                mke2fs_env = {
                    "MKE2FS_CONFIG": self.mke2fs_conf,
                    "E2FSPROGS_FAKE_TIME": "1230768000",
                }
                output, ret = run_command(mke2fs_cmd, mke2fs_env)
                if ret != 0:
                    logger.error(f"Failed to run mke2fs: {output}")
                    sys.exit(4)
                # Run e2fsdroid
                e2fsdroid_env = {"E2FSPROGS_FAKE_TIME": "1230768000"}
                file_con_cmd = []
                if os.path.exists(filesystem_config) and os.path.exists(file_contexts):
                    file_con_cmd = ["-C", filesystem_config] + ["-S", file_contexts]
                else:
                    logger.warning(
                        f"Try without ({filesystem_config}) and {file_contexts}"
                    )
                    filesystem_config = ""
                    file_contexts = ""
                # The options must be ordered
                e2fsdroid_cmd = (
                        [
                            self.e2fsdroid,
                            "-e",
                            "-T",
                            "1230768000",
                        ]
                        + file_con_cmd
                        + ["-S", os.path.join(self.config_dir, "file_contexts.txt")]
                        + ["-f", os.path.join(self.out_dir, partition)]
                        + ["-a", last_mounted_directory]
                        + [os.path.join(self.build_dir, partition + ".img")]
                )
                output, ret = run_command(e2fsdroid_cmd, e2fsdroid_env)
                if ret != 0:
                    logger.error(f"Failed to run e2fsdroid_cmd: {output}")
                    remove(os.path.join(self.build_dir, partition + ".img"))
                    sys.exit(4)
                print("")
            if sparse:
                raw_img = os.path.join(self.build_dir, partition + ".img")
                sparse_img = os.path.join(self.build_dir, partition + ".sparse")
                if not os.path.exists(raw_img):
                    continue
                logger.info("Convert raw image to sparse...")
                cmd = [self.img2simg, raw_img, sparse_img]
                run_command(cmd, verbose=True)
                if os.path.isfile(sparse_img):
                    remove(raw_img)
            if brotli:
                sparse_img = os.path.join(self.build_dir, partition + ".sparse")
                sdat_img = os.path.join(self.build_dir, partition + ".new.dat")
                if not os.path.exists(sparse_img):
                    continue
                logger.info("Convert sparse image to sdat...")

                img2sdat(
                    input_image=sparse_img,
                    prefix=partition,
                    cache_size=402653184,
                    out_dir=self.build_dir,
                    version=4,
                )

                if os.path.isfile(sdat_img):
                    remove(sparse_img)
                logger.info("Compress with brotli...")
                cmd = [self.brotli_tool, "-q", "6", "-v", "-f", sdat_img]
                run_command(cmd, verbose=True)
                if os.path.isfile(sdat_img + ".br"):
                    remove(sdat_img)

    def correct_contexts(self, fs_context_file: str) -> None:
        """Correct the file contexts and add additional ones.

        Parameters:
            fs_context_file (str): The path to the file containing the file contexts.
        """
        logger.info(f"Correcting {fs_context_file}")
        fs_context = []
        # Define a regular expression to match custom prefixes
        my_str = "(my_(engineering|version|product|company|preload|bigball|carrier|region|stock|heytap|manifest|custom)|special_preload)"
        # Define another regular expression to match specific prefixes
        my_str2 = "(my_version|odm)"
        # Add the corrected file contexts to the list
        fs_context.extend(
            [
                f"/{my_str}(/.*)?           u:object_r:system_file:s0",
                f"/{my_str}/overlay(/.*)?   u:object_r:vendor_overlay_file:s0",
                f"/{my_str}/non_overlay/overlay(/.*)?   u:object_r:vendor_overlay_file:s0",
                f"/{my_str}/vendor_overlay/[0-9]+/.*   u:object_r:vendor_file:s0",
                f"/{my_str}/vendor/etc(/.*)?    u:object_r:vendor_configs_file:s0",
                f"/{my_str2}/build.prop                                             u:object_r:vendor_file:s0",
                f"/{my_str2}/vendor_overlay/[0-9]+/lib(64)?(/.*)?    u:object_r:same_process_hal_file:s0",
                f"/{my_str2}/vendor_overlay/[0-9]+/etc/camera(/.*)?  u:object_r:same_process_hal_file:s0",
                f"/{my_str2}/vendor_overlay/[0-9]+/camera(/.*)?      u:object_r:vendor_file:s0",
                f"/{my_str2}/lib64/camera(/.*)?                      u:object_r:vendor_file:s0",
                f"/{my_str2}/vendor_overlay/lib?(/.*)?         u:object_r:same_process_hal_file:s0",
                f"/{my_str2}/vendor_overlay/lib(64)?(/.*)?     u:object_r:same_process_hal_file:s0",
                "/my_manifest/build.prop u:object_r:vendor_file:s0",
                "/my_company/theme_bak(/.*)? u:object_r:oem_theme_data_file:s0",
                "/my_product/etc/project_info.txt                                   u:object_r:vendor_file:s0",
                "/my_product/product_overlay/framework(/.*)?          u:object_r:system_file:s0",
                "/my_product/product_overlay/etc/permissions(/.*)?    u:object_r:system_file:s0",
                "/(vendor|my_engineering|system/vendor)/bin/factory	u:object_r:factory_exec:s0",
                "/(vendor|my_engineering|system/vendor)/bin/pcba_diag	u:object_r:pcba_diag_exec:s0",
                "/my_product/lib(64)?/libcolorx-loader\\.so                         u:object_r:same_process_hal_file:s0",
                "/my_product/vendor/firmware(/.*)      u:object_r:vendor_file:s0",
                "/my_version/vendor/firmware(/.*)?      u:object_r:vendor_file:s0",
                '/my_bigball(/.*)?                    u:object_r:rootfs:s0"',
                "/my_carrier(/.*)?                    u:object_r:rootfs:s0",
                "/my_company(/.*)?                    u:object_r:rootfs:s0",
                "/my_engineering(/.*)?                u:object_r:rootfs:s0",
                "/my_heytap(/.*)?                     u:object_r:rootfs:s0",
                "/my_manifest(/.*)?                   u:object_r:rootfs:s0",
                "/my_preload(/.*)?                    u:object_r:rootfs:s0",
                "/my_product(/.*)?                    u:object_r:rootfs:s0",
                "/my_region(/.*)?                     u:object_r:rootfs:s0",
                "/my_stock(/.*)?                      u:object_r=rootfs=s0",
                "/special_preload(/.*)?               u=object_r=rootfs=s0",
                "/firmware(/.*)?         u=object_r=firmware_file=s0",
                "/bt_firmware(/.*)?      u=object_r=bt_firmware_file=s0",
                "/persist(/.*)?          u=object_r=mnt_vendor_file=s0",
                "/dsp                    u=object_r=system_file=s0",
                "/oem                    u=object_r=system_file=s0",
                "/op1                    u=object_r=system_file=s0",
                "/op2                    u=object_r=system_file=s0",
                "/charger_log            u=object_r=system_file=s0",
                "/audit_filter_table     u=object_r=system_file=s0",
                "/keydata                u=object_r=system_file=s0",
                "/keyrefuge              u=object_r=system_file=s0",
                "/omr                    u=object_r=system_file=s0",
                "/publiccert.pem         u=object_r=system_file=s0",
                "/sepolicy_version       u=object_r=system_file=s0",
                "/cust                   u=object_r=system_file=s0",
                "/donuts_key             u=object_r=system_file=s0",
                "/v_key                  u=object_r=system_file=s0",
                "/carrier                u=object_r=system_file=s0",
                "/dqmdbg                 u=object_r=system_file=s0",
                "/ADF                    u=object_r=system_file=s0",
                "/APD                    u=object_r=system_file=s0",
                "/asdf                   u:object_r:system_file:s0",
                "/batinfo                u:object_r:system_file:s0",
                "/voucher                u:object_r:system_file:s0",
                "/xrom                   u:object_r:system_file:s0",
                "/custom                 u:object_r:system_file:s0",
                "/cpefs                  u:object_r:system_file:s0",
                "/modem                  u:object_r:system_file:s0",
                "/module_hashes          u:object_r:system_file:s0",
                "/pds                    u:object_r:system_file:s0",
                "/tombstones             u:object_r:system_file:s0",
                "/factory                u:object_r:system_file:s0",
                "/oneplus(/.*)?          u:object_r:system_file:s0",
                "/addon.d                u:object_r:system_file:s0",
                "/op_odm                 u:object_r:system_file:s0",
                "/avb                    u:object_r:system_file:s0",
                "/opconfig               u:object_r:system_file:s0",
                "/opcust                 u:object_r:system_file:s0",
                "/mi_ext(/.*)?           u:object_r:system_file:s0",
                "/(odm|vendor/odm)(/.*)?                       u:object_r:vendor_file:s0",
            ]
        )

        # Sort the list alphabetically
        fs_context.sort()
        # Write the list to the file
        cat(msg="\n".join(fs_context), out_file=fs_context_file)

    def define_file_contexts(self) -> None:
        """
        Defines the file contexts by checking the existence of specific file paths and writing them to a file.
        """

        # Define paths to the vendor, system, product, and system_ext file contexts
        vendor_context = os.path.join(
            self.out_dir, "vendor/etc/selinux/vendor_file_contexts"
        )
        system_context = os.path.join(
            self.out_dir, "system/system/etc/selinux/plat_file_contexts"
        )
        product_context = os.path.join(
            self.out_dir, "product/etc/selinux/product_file_contexts"
        )
        system_ext_context = os.path.join(
            self.out_dir, "system_ext/etc/selinux/system_ext_file_contexts"
        )
        file_context = os.path.join(self.config_dir, "file_contexts.txt")

        # Check if the file contexts file exists
        file_context_exists = os.path.exists(file_context)

        if not file_context_exists:
            # If the file contexts file does not exist, check if the vendor, system, product, and system_ext file contexts exist and write them to the file
            if os.path.exists(system_context):
                logger.info(f"Writing from {system_context}")
                cat(in_file=system_context, out_file=file_context)
            if os.path.exists(vendor_context):
                logger.info(f"Writing from {vendor_context}")
                cat(in_file=vendor_context, out_file=file_context)
            if os.path.exists(system_ext_context):
                logger.info(f"Writing from {system_ext_context}")
                cat(in_file=system_ext_context, out_file=file_context)
            if os.path.exists(product_context):
                logger.info(f"Writing from {product_context}")
                cat(in_file=product_context, out_file=file_context)

        file_context_exists = os.path.exists(file_context)

        if file_context_exists:
            # If the file contexts file exists, correct the file contexts
            self.correct_contexts(file_context)
        else:
            # If the file contexts file does not exist, log an error and exit
            logger.error(f"{file_context} not found!!")
            sys.exit(1)

    def main(
            self, raw: bool = False, sparse: bool = False, brotli: bool = False
    ) -> None:
        """Main method for creating ext4 images.

        Parameters:
            raw (bool, optional): If True, create a raw ext4 image. Defaults to False.
            sparse (bool, optional): If True, create a sparse ext4 image. Defaults to False.
            brotli (bool, optional): If True, compress the ext4 image using brotli. Defaults to False.
        """
        # ensure file_context.txt exist
        self.define_file_contexts()

        # Make the ext4 images
        self.make_ext4(raw, sparse, brotli)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Create ext4 images")
    parser.add_argument(
        "-b",
        "--build-image",
        dest="build_image",
        metavar="TYPE",
        type=lambda s: [t.strip() for t in s.split(",")],
        help="""Build image of type [raw:r, sparse:s, brotli:b] (Linux only).
            Multiple values can be passed separated by commas.
            raw:     This will build an ext4 filesystem.
            sparse:  This will build a sparse filesystem.
            brotli:  This will compress the sparse image with brotli.
            These options must be used in the following order: first raw, then sparse, then brotli.
        """,
    )

    args = parser.parse_args()

    if args.build_image:
        build_image_set = set(args.build_image)
        raw = "raw" in build_image_set or "r" in build_image_set
        sparse = "sparse" in build_image_set or "s" in build_image_set
        brotli = "brotli" in build_image_set or "b" in build_image_set

        if raw or sparse or brotli:
            CreateExt4().main(raw, sparse, brotli)


import json
from unittest.mock import patch

import pytest

from src.main import app as flask_app
from src.main import get_port, get_secret_key
from src.file import FileOpenMock

@pytest.fixture
def app():
    yield flask_app

@pytest.fixture
def client(app):
    return app.test_client()

@patch("src.main.os.listdir")
@patch('src.main.os.chdir')
def test_get_all_files_exception(mock_chdir, mock_listdir, client):
    mock_listdir.side_effect = Exception()
    res = client.get('/files')
    """res.get_data(), res.status_code, res """
    assert res.status_code == 500

@patch("src.main.f")
@patch('src.main.open')
@patch('src.main.os.chdir')
@patch('src.main.os.makedirs')
@patch('src.main.os.umask')
@patch('src.main.os.path')
def test_create_file(mock_path, mock_umask, mock_dir, mock_chdir, mock_open, mock_write, client):
    res = client.post('/files', json={
        "name": "my4u",
        "data": "New file created"
    })
    data = json.loads(res.get_data(as_text=True))
    """res.get_data(), res.status_code, res """
    assert res.status_code == 409

    res = client.post('/files', json={
        "data": "New file created"
    })
    data = json.loads(res.get_data(as_text=True))
    """res.get_data(), res.status_code, res """
    assert res.status_code == 400

    mock_path.join.return_value = False
    mock_path.exists.return_value = True
    mock_umask.return_value = True
    mock_dir.return_value = False

    mock_path.isfile.return_value = False
    res = client.post('/files', json={
        "name": "my4u",
        "data": "New file created"
    })
    data = json.loads(res.get_data(as_text=True))
    mock_write.writelines.return_value = True


from flask import *
from configparser import ConfigParser
from tkinter import messagebox
import requests

api_url = 'http://api.openweathermap.org/data/2.5/weather?q={}&appid={}'

app = Flask(__name__)

config_file = 'config.ini'
config = ConfigParser()
config.read(config_file)
api_key = config['api_key']['key']

def get_weather(city):
    result = requests.get(api_url.format(city, api_key))
    if result:
        json = result.json()
        city = json['name']
        country = json['sys']['country']
        temp_kelvin = json['main']['temp']
        temp_celsius = round(temp_kelvin - 273.15, 2)
        temp_ferenheit = round(temp_celsius*9/5+32, 2)
        icon = "icons/"+(json['weather'][0]['icon'])+".png"
        weather = json['weather'][0]['main']
        final = (city, country, temp_celsius, temp_ferenheit, icon, weather)
        return final
    else:
        return None

@app.route('/', methods=["POST", "GET"])
def Weather_app():
    if request.method == "POST":
        city_name = request.form["city"]
        return redirect(url_for("city_weather", city=city_name))
    else:
        return render_template('index.html')

@app.route('/<city>')
def city_weather(city):
    city_info = get_weather(city)
    if city_info:
        return render_template('weatherOfCity.html', city_data = city_info)
    else:
        messagebox.showerror('[ERROR]',"Cannot Find City {}".format(city))

if __name__ == '__main__':
    app.run()


import boto3
import json
import logging
import time
import botocore
import os
import sys
from datetime import datetime

logger = logging.getLogger()
logger.setLevel(logging.INFO)
ec2_client = boto3.client('ec2')
asg_client = boto3.client('autoscaling')
ec2 = boto3.resource('ec2')

def lambda_handler(event, context):
    instance_id = event['detail']['EC2InstanceId']
    LifecycleHookName = event['detail']['LifecycleHookName']
    AutoScalingGroupName = event['detail']['AutoScalingGroupName']
    secgroup_id = os.environ['SecGroupId']
    vpc_id = os.environ['VPCId']
    route_table_id = os.environ['WANRouteTable']
    cidr = str(os.environ['VIPCIDRBlock'])
    vip = str(os.environ['VIPAddress']).split('/')[0]
    eipaddress = str(os.environ['EIPAddress']).split('/')[0]
    eipallocation = str(os.environ['EIPAllocationId']).split('/')[0]
    LambdaInfoTracing = str(os.environ['LambdaInfoTracing'])
    InstanceRequiresReboot = str(os.environ['InstanceRequiresReboot'])
    SubnetCreationAttempts = int(os.environ['SubnetCreationAttempts'])

    # printing event received:
    infolog("lambda_handler -- Event keys: {}".format(list(event['detail'].keys())),LambdaInfoTracing)
    infolog("lambda_handler -- Complete Event: {}".format(str(event['detail'])),LambdaInfoTracing)

    # Find out AZ from the instance
    AZ_list = []

    try:
        event_instanceid = event['detail']['EC2InstanceId']
        infolog("lambda_handler -- EC2 instance triggering event: {}".format(event_instanceid),LambdaInfoTracing)
    except botocore.exceptions.ClientError as e:
        errorlog("Error extracting EC2 instance from event details: {}".format(e.response['Error']))

    try:
        instance_list = ec2_client.describe_instances()
        infolog("lambda_handler -- EC2 instances description response: {}".format(instance_list),LambdaInfoTracing)
        for reservation in instance_list["Reservations"]:
            for instance in reservation.get("Instances", []):
                if instance['InstanceId'] == event_instanceid:
                    AZ_list.append(instance["Placement"]["AvailabilityZone"])
        infolog("lambda_handler -- AZs out of EC2 instances description response: {}".format(AZ_list),LambdaInfoTracing)
    except botocore.exceptions.ClientError as e:
            errorlog("Error extracting AZ from EC2 instances description response: {}".format(e.response['Error']))

    if AZ_list:
        AZ = AZ_list[0]
    else:
        errorlog("No AZs could be extracted")
        return

    # Tracing EC2 details
    infolog("lambda_handler -- Event: {}".format(event["detail-type"]),LambdaInfoTracing)
    infolog("lambda_handler -- Instance Id: {}".format(instance_id),LambdaInfoTracing)
    infolog("lambda_handler -- LifecycleHookName: {}".format(LifecycleHookName),LambdaInfoTracing)
    infolog("lambda_handler -- AutoScalingGroupName: {}".format(AutoScalingGroupName),LambdaInfoTracing)
    infolog("lambda_handler -- secgroup_id: {}".format(secgroup_id),LambdaInfoTracing)
    infolog("lambda_handler -- vpc_id: {}".format(vpc_id),LambdaInfoTracing)
    infolog("lambda_handler -- route_table_id: {}".format(route_table_id),LambdaInfoTracing)
    infolog("lambda_handler -- cidr: {}".format(cidr),LambdaInfoTracing)
    infolog("lambda_handler -- vip: {}".format(vip),LambdaInfoTracing)
    infolog("lambda_handler -- eipaddress: {}".format(eipaddress),LambdaInfoTracing)
    infolog("lambda_handler -- eipallocation: {}".format(eipallocation),LambdaInfoTracing)
    infolog("lambda_handler -- AZ: {}".format(AZ),LambdaInfoTracing)
    infolog("lambda_handler -- InstanceRequiresReboot: {}".format(InstanceRequiresReboot),LambdaInfoTracing)
    infolog("lambda_handler -- SubnetCreationAttempts: {}".format(SubnetCreationAttempts),LambdaInfoTracing)

    if event["detail-type"] == "EC2 Instance-launch Lifecycle Action":

        subnet_id = None
        attempts = 0
        # Attempts to create secondary subnet in same AZ and associate it to Route Table
        while (not subnet_id) and attempts < SubnetCreationAttempts :
            infolog("lambda_handler -- Attempt nr. {} to create and associate subnet".format(attempts),LambdaInfoTracing)
            subnet_id = create_and_associate_subnet(vpc_id,cidr,AZ,route_table_id,LambdaInfoTracing)
            attempts += 1
            time.sleep(10)

        if not subnet_id:
            # No subnet could be created after SubnetCreationAttempts attempts, abandon lifecycle hook
            # Lifecycle Hook event failed
            complete_lifecycle_action_failure(LifecycleHookName,AutoScalingGroupName,instance_id,LambdaInfoTracing)
            return

        # Create ENI within secondary subnet in same AZ
        interface_id = create_interface(subnet_id,secgroup_id,vip,eipaddress,eipallocation,LambdaInfoTracing)

        if not interface_id:
            # No ENI could be created
            # Lifecycle Hook event failed
            complete_lifecycle_action_failure(LifecycleHookName,AutoScalingGroupName,instance_id,LambdaInfoTracing)
            disassociate_delete_subnet(subnet_id,route_table_id,LambdaInfoTracing)
            return

        # Index is 1 because it is secondary interface to the instance
        attachment = attach_interface(interface_id,instance_id,1,LambdaInfoTracing)

        if not attachment:
            # ENI could not be attached
            # Lifecycle Hook event failed
            complete_lifecycle_action_failure(LifecycleHookName,AutoScalingGroupName,instance_id,LambdaInfoTracing)
            delete_interface(interface_id,eipaddress,eipallocation,LambdaInfoTracing)
            return

        if str(InstanceRequiresReboot) == "true":
            # ENI attachment requires instance reboot
            time.sleep(30)
            restart_instance(instance_id,LambdaInfoTracing)
            time.sleep(120)

        # Lifecycle Hook event successfully completed otherwise
        complete_lifecycle_action_success(LifecycleHookName,AutoScalingGroupName,instance_id,LambdaInfoTracing)
        return

    if event["detail-type"] == "EC2 Instance-terminate Lifecycle Action":

        # Obtained Subnet ID from same VPC and CIDR range
        subnet_id = get_subnet(vpc_id,cidr,LambdaInfoTracing)

        # Obtained Interface ID from same subnet
        interface_id = get_interface(subnet_id,vip,LambdaInfoTracing)

        # Interface ID could be extracted from Subnet ID
        if interface_id is not None:
            try:
                # Detach the ENI from the instance
                detach_interface(interface_id,LambdaInfoTracing)
            except botocore.exceptions.ClientError as e:
                errorlog("Error detaching interface: {}".format(e.response['Error']))

            try:
                # After detaching, delete the interface
                delete_interface(interface_id,eipaddress,eipallocation,LambdaInfoTracing)
            except botocore.exceptions.ClientError as e:
                errorlog("Error deleting interface: {}".format(e.response['Error']))

        if subnet_id is not None:
            try:
                # After having detached and deleted the ENI, subnet can be deleted
                disassociate_delete_subnet(subnet_id,route_table_id,LambdaInfoTracing)
            except botocore.exceptions.ClientError as e:
                errorlog("Error deleting subnet: {}".format(e.response['Error']))

        # After detaching ENI, deleting it and deleting the subnet, this is a successful lifecycle hook
        complete_lifecycle_action_success(LifecycleHookName,AutoScalingGroupName,instance_id,LambdaInfoTracing)
        return

def create_and_associate_subnet(vpc_id,cidr,az,route_table_id,LambdaInfoTracing):
    """
    create subnet id from VPC in a specific AZ with a private IPv4 CIDR range

    :param vpc_id: VPC id
    :param cidr: CIDR IPv4 range for subnet
    :param az: Availability Zone
    :param route_table_id: Route Table id

    """
    subnet_id = None
    if vpc_id and cidr:
        try:
            infolog("create_and_associate_subnet -- VPC ID parameter: {}".format(vpc_id),LambdaInfoTracing)
            infolog("create_and_associate_subnet -- CIDR parameter: {}".format(cidr),LambdaInfoTracing)
            infolog("create_and_associate_subnet -- AZ parameter: {}".format(az),LambdaInfoTracing)
            subnet = ec2_client.create_subnet(TagSpecifications=[{'ResourceType': 'subnet', 'Tags': [ {'Key': 'Name', 'Value': 'VIP Subnet'}]}],AvailabilityZone=az,CidrBlock=cidr,VpcId= vpc_id)
            infolog("create_and_associate_subnet -- EC2 create subnet response: {}".format(subnet),LambdaInfoTracing)
            subnet_id = subnet['Subnet']['SubnetId']
            infolog("create_and_associate_subnet -- EC2 created subnet ID: {}".format(subnet_id),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error creating subnet: {}".format(e.response['Error']))
            if e.response['Error']['Code'] == 'InvalidSubnet.Conflict':
                errorlog("create_and_associate_subnet -- Previous subnet {} has not been deleted yet".format(cidr))

    if subnet_id:
        try:
            infolog("create_and_associate_subnet -- Route Table parameter: {}".format(route_table_id),LambdaInfoTracing)
            infolog("create_and_associate_subnet -- created Subnet ID: {}".format(subnet_id),LambdaInfoTracing)
            response = ec2_client.associate_route_table(RouteTableId=route_table_id,SubnetId=subnet_id)
            infolog("create_and_associate_subnet -- found Route Table: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error associating subnet: {}".format(e.response['Error']))

    return subnet_id

def create_interface(subnet_id,sg_id,vip,eipaddress,eipallocation,LambdaInfoTracing):
    """
    create interface id with subnet, Security Group and a specific private IPv4 address

    :param subnet_id: subnet id within VPC
    :param sg_id: Security Group ID
    :param vip: (virtual) private IPv4 address that is mapped to interface

    """

    network_interface_id = None
    if subnet_id:
        try:
            infolog("create_interface -- subnet ID parameter: {}".format(subnet_id),LambdaInfoTracing)
            infolog("create_interface -- Security Group ID parameter: {}".format(sg_id),LambdaInfoTracing)
            infolog("create_interface -- Virtual IP address parameter:: {}".format(vip),LambdaInfoTracing)
            network_interface = ec2_client.create_network_interface(Description='VIP ENI',Groups=[sg_id],SubnetId=subnet_id,PrivateIpAddress=vip)
            infolog("create_interface -- EC2 create ENI response: {}".format(network_interface),LambdaInfoTracing)
            network_interface_id = network_interface['NetworkInterface']['NetworkInterfaceId']
            infolog("create_interface -- EC2 created ENI ID: {}".format(network_interface_id),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error creating network interface: {}".format(e.response['Error']))

    # Associate existing EIP allocation
    if eipallocation:
        try:
            infolog("create_interface -- eipaddress parameter: {}".format(eipaddress),LambdaInfoTracing)
            infolog("create_interface -- eipallocation parameter: {}".format(eipallocation),LambdaInfoTracing)
            response = ec2_client.associate_address(AllocationId=eipallocation,NetworkInterfaceId=network_interface_id)
            infolog("create_interface -- EC2 associate EIP response: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error associating EIP to network interface: {}".format(e.response['Error']))

    return network_interface_id

def get_subnet(vpc_id,cidr,LambdaInfoTracing):
    """
    obtain subnet id from VPC based on IPv4 CIDR range

    :param vpc_id: VPC id
    :param cidr: CIDR IPv4 range from subnet

    """

    subnet_id = None
    if vpc_id and cidr:
        try:
            infolog("get_subnet -- VPC ID parameter: {}".format(vpc_id),LambdaInfoTracing)
            infolog("get_subnet -- CIDR parameter: {}".format(cidr),LambdaInfoTracing)
            response = ec2_client.describe_subnets(
                Filters=[
                    {
                        'Name': 'cidr-block',
                        'Values': [cidr]
                    }
                ]
            )
            infolog("get_subnet -- EC2 describe subnet reponse: {}".format(response),LambdaInfoTracing)
            if response['Subnets']:
                subnet_id = response['Subnets'][0]['SubnetId']
                infolog("get_subnet -- EC2 obtained subnet ID: {}".format(subnet_id),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining subnet: {}".format(e.response['Error']))
    return subnet_id

def get_interface(subnet_id,vip,LambdaInfoTracing):
    """
    obtain interface id from subnet based on specific IPv4 address

    :param subnet_id: subnet id within VPC
    :param vip: (virtual) IPv4 address that is mapped to interface

    """

    interface_id = None
    if subnet_id:
        try:
            infolog("get_interface -- Subnet ID parameter: {}".format(subnet_id),LambdaInfoTracing)
            infolog("get_interface -- Virtual IP address parameter: {}".format(vip),LambdaInfoTracing)
            response = ec2_client.describe_network_interfaces(
                Filters=[{"Name": "private-ip-address", "Values": [vip]}]
            )
            infolog("get_interface -- EC2 describe ENI response: {}".format(response),LambdaInfoTracing)
            if response['NetworkInterfaces']:
                interface_id = response['NetworkInterfaces'][0]['NetworkInterfaceId']
                infolog("get_interface -- EC2 obtained interface ID: {}".format(interface_id),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining interface: {}".format(e.response['Error']))
    return interface_id

def detach_interface(network_interface_id,LambdaInfoTracing):
    """
    detach  interface if it is attached to instance

    :param network_interface_id: network interface id that
                               we previously obtain

    """

    attachment = None
    response = None
    if network_interface_id:
        try:
            infolog("detach_interface -- Network Interface ID parameter: {}".format(network_interface_id),LambdaInfoTracing)
            response = ec2_client.describe_network_interfaces(
                Filters=[{"Name": "network-interface-id", "Values": [network_interface_id]}]
            )
            infolog("detach_interface -- EC2 describe ENI response: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining interface description: {}".format(e.response['Error']))

    if response:
        try:
            if "Attachment" in response['NetworkInterfaces'][0]:
                attachment = response['NetworkInterfaces'][0]['Attachment']['AttachmentId']
                infolog("detach_interface -- EC2 obtained attachmend id: {}".format(attachment),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining attachment: {}".format(e.response['Error']))

    if attachment:
        try:
            response = ec2_client.detach_network_interface(AttachmentId=attachment,Force=True)
            infolog("detach_interface -- EC2 obtained response from interface detachment: {}".format(response),LambdaInfoTracing)
            # Wait time to accomplish detachment
            time.sleep(60)
        except botocore.exceptions.ClientError as e:
            errorlog("Error trying detachment: {}".format(e.response['Error']))

    return attachment

def attach_interface(network_interface_id,instance_id,index,LambdaInfoTracing):
    """
    attach  interface to instance

    :param network_interface_id: network interface id that
                               we previously obtain
    :param instance_id: instance ID to attach interface to
    :param index: index for interface attachment (starting from '0')

    """

    attachment = None
    if network_interface_id and instance_id:
        try:
            infolog("attach_interface -- Network Interface ID parameter: {}".format(network_interface_id),LambdaInfoTracing)
            infolog("attach_interface -- Instance ID parameter: {}".format(instance_id),LambdaInfoTracing)
            attach_interface = ec2_client.attach_network_interface(
                NetworkInterfaceId=network_interface_id,
                InstanceId=instance_id,
                DeviceIndex=index
            )
            infolog("attach_interface -- EC2 attach ENI response: {}".format(attach_interface),LambdaInfoTracing)
            attachment = attach_interface['AttachmentId']
            infolog("attach_interface -- created network attachment ID: {}".format(attachment),LambdaInfoTracing)

            network_interface = ec2.NetworkInterface(network_interface_id)

            #modify_attribute doesn't allow multiple parameter change at once..
            network_interface.modify_attribute(
                SourceDestCheck={
                    'Value': False
                }
            )
            network_interface.modify_attribute(
                Attachment={
                    'AttachmentId': attachment,
                    'DeleteOnTermination': True
                },
            )
            infolog("attach_interface -- created network interface: {}".format(network_interface),LambdaInfoTracing)

        except botocore.exceptions.ClientError as e:
            errorlog("Error attaching network interface: {}".format(e.response['Error']))

    return attachment

def delete_interface(network_interface_id,eipaddress,eipallocation,LambdaInfoTracing):
    """
    delete interface

    :param network_interface_id: network interface id to be deleted

    """
    association = None

    if network_interface_id:
        try:
            infolog("delete_interface -- Network Interface ID parameter: {}".format(network_interface_id),LambdaInfoTracing)
            response = ec2_client.describe_network_interfaces(
                Filters=[{"Name": "network-interface-id", "Values": [network_interface_id]}]
            )
            infolog("delete_interface -- EC2 describe ENI response: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining interface description: {}".format(e.response['Error']))

    # Disassociate existing EIP allocation
    if ( "NetworkInterfaces" in response ) and eipallocation:
        try:
            if response['NetworkInterfaces'][0]:
                if "Association" in response['NetworkInterfaces'][0]:
                    association = response['NetworkInterfaces'][0]['Association']['AssociationId']
                    infolog("delete_interface -- EC2 obtained association id: {}".format(association),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error obtaining association: {}".format(e.response['Error']))

    if association:
        try:
            infolog("delete_interface -- eipaddress parameter: {}".format(eipaddress),LambdaInfoTracing)
            infolog("delete_interface -- eipallocation parameter: {}".format(eipallocation),LambdaInfoTracing)
            response = ec2_client.disassociate_address(AssociationId=association)
            infolog("delete_interface -- EC2 disassociate EIP response: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error disassociating EIP from network interface: {}".format(e.response['Error']))

    # Then delete the interface
    try:
        infolog("delete_interface -- Network Interface ID parameter: {}".format(network_interface_id),LambdaInfoTracing)
        ec2_client.delete_network_interface(
            NetworkInterfaceId=network_interface_id
        )
        infolog("delete_interface -- EC2 deleted network interface: {}".format(network_interface_id),LambdaInfoTracing)
        return True

    except botocore.exceptions.ClientError as e:
        errorlog("Error deleting interface {}: {}".format(network_interface_id,e.response['Error']))

def disassociate_delete_subnet(subnet_id,route_table_id,LambdaInfoTracing):
    """
    disassociate_delete subnet

    :param subnet_id: subnet id to be deleted
    :param route_table_id: route table id to disassociate subnet from

    """
    try:
        infolog("disassociate_delete_subnet -- Subnet ID parameter: {}".format(subnet_id),LambdaInfoTracing)
        infolog("disassociate_delete_subnet -- Route Table ID parameter: {}".format(route_table_id),LambdaInfoTracing)
        response = ec2_client.describe_route_tables(RouteTableIds=[route_table_id])
        infolog("disassociate_delete_subnet -- EC2 obtained route table description: {}".format(response),LambdaInfoTracing)
        RouteTableAssociationId = response['RouteTables'][0]['Associations'][0]['RouteTableAssociationId']
        infolog("disassociate_delete_subnet -- EC2 obtained route table id subnet: {}".format(RouteTableAssociationId),LambdaInfoTracing)
    except botocore.exceptions.ClientError as e:
        errorlog("Error obtaining Route Table Association {}: {}".format(subnet_id,e.response['Error']))

    if RouteTableAssociationId:
        try:
            infolog("disassociate_delete_subnet -- Subnet ID parameter: {}".format(subnet_id),LambdaInfoTracing)
            infolog("disassociate_delete_subnet -- Route Table ID parameter: {}".format(route_table_id),LambdaInfoTracing)
            response = ec2_client.disassociate_route_table(AssociationId=RouteTableAssociationId)
            infolog("disassociate_delete_subnet -- EC2 disassociating subnet: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error disassociating subnet {}: {}".format(subnet_id,e.response['Error']))

    try:
        infolog("disassociate_delete_subnet -- Subnet ID parameter: {}".format(subnet_id),LambdaInfoTracing)
        ec2_client.delete_subnet(
            SubnetId=subnet_id
        )
        infolog("disassociate_delete_subnet -- EC2 deleted subnet: {}".format(subnet_id),LambdaInfoTracing)
        return True
    except botocore.exceptions.ClientError as e:
        errorlog("Error deleting subnet {}: {}".format(subnet_id,e.response['Error']))

def complete_lifecycle_action_success(hookname,groupname,instance_id,LambdaInfoTracing):
    """
    Complete Lifecycle Action Success

    Complete the lifecycle with success if no exception occurs

    :param hookname: Life cycle hook name
    :param groupname: Autoscaling group name
    :param instanceid: Instance id for newly launched instance

    """

    try:
        infolog("complete_lifecycle_action_success -- hookname parameter: {}".format(hookname),LambdaInfoTracing)
        infolog("complete_lifecycle_action_success -- ASG parameter: {}".format(groupname),LambdaInfoTracing)
        infolog("complete_lifecycle_action_success -- Instance ID parameter: {}".format(instance_id),LambdaInfoTracing)
        asg_client.complete_lifecycle_action(
            LifecycleHookName=hookname,
            AutoScalingGroupName=groupname,
            InstanceId=instance_id,
            LifecycleActionResult='CONTINUE'
        )
        infolog("complete_lifecycle_action_success -- Lifecycle hook CONTINUEd for: {}".format(instance_id),LambdaInfoTracing)
    except botocore.exceptions.ClientError as e:
            errorlog("Error completing life cycle hook for instance {}: {}".format(instance_id, e.response['Error']))
            errorlog('{"Error": "1"}')

def complete_lifecycle_action_failure(hookname,groupname,instance_id,LambdaInfoTracing):
    """
    Complete Lifecycle Action Failure

    Complete the lifecycle with failure if exception occurs

    :param hookname: Life cycle hook name
    :param groupname: Autoscaling group name
    :param instanceid: Instance id for newly launched instance

    """

    try:
        infolog("complete_lifecycle_action_failure -- hookname parameter: {}".format(hookname),LambdaInfoTracing)
        infolog("complete_lifecycle_action_failure -- ASG parameter: {}".format(groupname),LambdaInfoTracing)
        infolog("complete_lifecycle_action_failure -- Instance ID parameter: {}".format(instance_id),LambdaInfoTracing)
        asg_client.complete_lifecycle_action(
            LifecycleHookName=hookname,
            AutoScalingGroupName=groupname,
            InstanceId=instance_id,
            LifecycleActionResult='ABANDON'
        )
        infolog("complete_lifecycle_action_failure -- Lifecycle hook ABANDONed for: {}".format(instance_id),LambdaInfoTracing)
    except botocore.exceptions.ClientError as e:
            errorlog("Error completing life cycle hook for instance {}: {}".format(instance_id, e.response['Error']))
            errorlog('{"Error": "1"}')

def restart_instance(instance_id,LambdaInfoTracing):
    """
    restart instance for proper ENI attachment

    :param instance_id: instance ID to restart and discover with newly attached interface

    """
    if instance_id:
        try:
            infolog("restart_instance -- Instance ID: {}".format(instance_id),LambdaInfoTracing)
            response = ec2_client.reboot_instances(
                InstanceIds=[instance_id]
            )
            infolog("restart_instance -- EC2 restart EC2 response: {}".format(response),LambdaInfoTracing)
        except botocore.exceptions.ClientError as e:
            errorlog("Error restarting EC2 instance: {}".format(e.response['Error']))

def errorlog(error):
    """
    Log

    takes message as an input and print it with time in iso format
    """
    logger.error('%s', error)

def infolog(string,LambdaInfoTracing):
    """
    Log

    takes message as an input and print it with time in iso format
    """
    if str(LambdaInfoTracing) == "true":
        logger.info('%s', string)


"""
This module is used in order to display the data
in a complete and good-looking way
"""

import pandas as pd
from tabulate import tabulate

def display_table(data):
    """
    Get the weather information and display it in a nice way through a table
    :param data: weather information
    :return: table to be displayed to the user in which there are the
    weather, the temperature, the description, the icon of the requested city
    or its history
    """
    # create a list of dictionaries
    data = [data] if type(data) is dict else data
    # create a dataframe
    df = pd.DataFrame(data)
    if 'id' in df.columns:
        df.drop(['id'], axis=1, inplace=True)
    # use capital letters for the headers
    df.columns = df.columns.str.upper()
    # create the table using tabulate
    table = tabulate(df, headers='keys', showindex=False)
    print(table)


dict = {}

user_count = 6040

gender = {}
gender['M'] = 1
gender['F'] = 2

dict[1] = "Gender-male"
dict[2] = "Gender-female"

age = {}
age['1'] = 3
age['18'] = 4
age['25'] = 5
age['35'] = 6
age['45'] = 7
age['50'] = 8
age['56'] = 9

dict[3] = "Age-under 18"
dict[4] = "Age-18-24"
dict[5] = "Age-25-34"
dict[6] = "Age-35-44"
dict[7] = "Age-45-49"
dict[8] = "Age-50-55"
dict[9] = "Age-56+"

feature_size = 9
occ = {}
for i in range(21):
	feature_size += 1
	occ[str(i)] = feature_size

dict[10] = "Occupation-other"
dict[11] = "Occupation-academic/educator"
dict[12] = "Occupation-artist"
dict[13] = "Occupation-clerical/admin"
dict[14] = "Occupation-college/grad student"
dict[15] = "Occupation-customer service"
dict[16] = "Occupation-doctor/health care"
dict[17] = "Occupation-executive/managerial"
dict[18] = "Occupation-farmer"
dict[19] = "Occupation-homemaker"
dict[20] = "Occupation-K-12 student"
dict[21] = "Occupation-lawyer"
dict[22] = "Occupation-programmer"
dict[23] = "Occupation-retired"
dict[24] = "Occupation-sales/marketing"
dict[25] = "Occupation-scientist"
dict[26] = "Occupation-self-employed"
dict[27] = "Occupation-technician/engineer"
dict[28] = "Occupation-tradesman/craftsman"
dict[29] = "Occupation-unemployed"
dict[30] = "Occupation-writer"

f = open('users.dat', 'r')
zipcode = {}
for i in range(1, user_count + 1):
	line = f.readline()
	line = line[:-1]
	l = line.split('::')
	if zipcode.get(l[-1]) == None:
		feature_size += 1
		zipcode[l[-1]] = feature_size
		dict[feature_size] = "Zipcode-" + str(l[-1])
f.close()

f = open('users.dat', 'r')
user_i = [[]]
user_v = [[]]
for i in range(1, user_count + 1):
	line = f.readline()
	line = line[:-1]
	l = line.split('::')
	user_i.append([gender[l[1]], age[l[2]], occ[l[3]], zipcode[l[4]]])
	user_v.append([1, 1, 1, 1])
f.close()
print("The number of user's feature is:", len(user_i))

movie_count = 3883

max_gen = 0
min_gen = 10

year_dict = {}
for i in range(1919, 1930):
	year_dict[i] = 1
for i in range(1930, 1940):
	year_dict[i] = 2
for i in range(1940, 1950):
	year_dict[i] = 3
for i in range(1950, 1960):
	year_dict[i] = 4
for i in range(1960, 1970):
	year_dict[i] = 5
for i in range(1970, 2001):
	year_dict[i] = 6 + i - 1970

f = open('movies.dat', 'r', encoding="ISO-8859-1")
genres = {}
for i in range(1, movie_count + 1):
	line = f.readline()
	line = line[:-1]
	l = line.split('::')
	s = l[-1]
	l = s.split('|')
	if len(l) > max_gen:
		max_gen = len(l)
	if len(l) < min_gen:
		min_gen = len(l)
	if len(l) == 0:
		print('error')
	for _ in l:
		if genres.get(_) == None:
			feature_size += 1
			genres[_] = feature_size
			dict[feature_size] = "Genre-" + _
f.close()
print("2222", feature_size)
print(len(dict))

print('The max number is :', max_gen)

#feature_size += 1 # for year of release

f = open('movies.dat', 'r', encoding="ISO-8859-1")
movie_i = {}
movie_v = {}
for i in range(1, movie_count + 1):
	line = f.readline()
	line = line[:-1]
	l = line.split('::')
	MovieID = int(l[0])
	Year = int(l[1][-5:-1])
	l = l[-1].split('|')
	new_i = []
	new_v = []
	for _ in l:
		new_i.append(genres[_])
		new_v.append(1)
	t = 6 - len(l) # 0 ~ 5 remain
	for _ in range(feature_size + 1, feature_size + t + 1):
		new_i.append(_)
		new_v.append(0)
	#new_i.append(feature_size + 6)
	#new_v.append(Year)
	new_i.append(feature_size + 5 + year_dict[Year])
	new_v.append(1)
	movie_i[MovieID] = new_i
	movie_v[MovieID] = new_v
f.close()

print(feature_size + 1, feature_size + 5)
#feature_size += 6
dict[feature_size + 1] = "Genre-NULL"
dict[feature_size + 2] = "Genre-NULL"
dict[feature_size + 3] = "Genre-NULL"
dict[feature_size + 4] = "Genre-NULL"
dict[feature_size + 5] = "Genre-NULL"
feature_size += 5

feature_size += 1
dict[feature_size] = "Release-1919-1929"
feature_size += 1
dict[feature_size] = "Release-1930-1939"
feature_size += 1
dict[feature_size] = "Release-1940-1949"
feature_size += 1
dict[feature_size] = "Release-1950-1959"
feature_size += 1
dict[feature_size] = "Release-1960-1969"
for y in range(1970, 2001):
	feature_size += 1
	dict[feature_size] = "Release-" + str(y)

print("####: ", feature_size)
print(len(dict))

print("The number of movie's feature is:", len(movie_i))

feature_size += 1 # for timestamp
dict[feature_size] = "Timestamp"

f = open('ratings.dat', 'r')

data_i = []
data_v = []
Y = []
#U = []
#I = []
all_count = 1000209
ratings_count = 0
for i in range(1, all_count + 1):
	line = f.readline()
	line = line[:-1]
	l = line.split('::')
	y = int(l[2])
	new_i = user_i[int(l[0])].copy()
	new_v = user_v[int(l[0])].copy()
	new_i.extend(movie_i[int(l[1])])
	new_v.extend(movie_v[int(l[1])])
	new_i.append(feature_size)
	new_v.append(int(l[3]))
	if y > 3:
		y = 1
	elif y < 3:
		y = 0
	else:
		y = -1

	if y != -1:
		data_i.append(new_i)
		data_v.append(new_v)
	#	U.append(int(l[0]))
	#	I.append(int(l[1]))
		Y.append(y)
		ratings_count += 1
f.close()
print('valid number of ratings:', len(data_v))

print('Positive number =', sum(Y))
print(feature_size)
print("Dict: ", len(dict))
print('All =', len(data_i))

import numpy as np
import random
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
R = []
for i in range(ratings_count):
	R.append([data_v[i][-1]])
#print(R)
#print(np.max(R))
#print(np.min(R))
R = scaler.fit_transform(R)
#print(R)

for i in range(ratings_count):
	data_v[i].pop()
	data_v[i].append(R[i][0])
#	data_v[i].append(U[i])
#	data_v[i].append(I[i])
print(data_v[0])
perm = []
for i in range(ratings_count):
	perm.append(i)

random.seed(2019)
random.shuffle(perm)

train_count = int(ratings_count * 0.8)
valid_count = int(ratings_count * 0.9)
X_i_tr = []
X_v_tr = []
Y_tr = []
for i in range(train_count):
	X_i_tr.append(data_i[perm[i]])
	X_v_tr.append(data_v[perm[i]])
	Y_tr.append(Y[perm[i]])

X_i_tr = np.array(X_i_tr)
X_v_tr = np.array(X_v_tr)
Y_tr = np.array(Y_tr)

i1 = X_i_tr[:, 0:4]
i2 = X_i_tr[:, 4:10]
i3 = X_i_tr[:, 10:]
x1 = X_v_tr[:, 0:4]
x2 = X_v_tr[:, 4:10]
x3 = X_v_tr[:, 10:]
i4 = np.concatenate((i1,i3), axis=1)
x4 = np.concatenate((x1,x3), axis=1)

np.save("train_i_genre.npy", i2)
np.save("train_i_other.npy", i4)
np.save("train_x_genre.npy", x2)
np.save("train_x_other.npy", x4)
np.save("train_y.npy", np.array(Y_tr))
#np.save("train_ui.npy", np.array(ui_tr))
X_i_va = []
X_v_va = []
Y_va = []
for i in range(train_count, valid_count):
	X_i_va.append(data_i[perm[i]])
	X_v_va.append(data_v[perm[i]])
	Y_va.append(Y[perm[i]])
#	ui_va.append([U[perm[i]], I[perm[i]])
X_i_va = np.array(X_i_va)
X_v_va = np.array(X_v_va)
Y_va = np.array(Y_va)

i1 = X_i_va[:, 0:4]
i2 = X_i_va[:, 4:10]
i3 = X_i_va[:, 10:]
x1 = X_v_va[:, 0:4]
x2 = X_v_va[:, 4:10]
x3 = X_v_va[:, 10:]
i4 = np.concatenate((i1,i3), axis=1)
x4 = np.concatenate((x1,x3), axis=1)

np.save("valid_i_genre.npy", i2)
np.save("valid_i_other.npy", i4)
np.save("valid_x_genre.npy", x2)
np.save("valid_x_other.npy", x4)
np.save("valid_y.npy", np.array(Y_va))

X_i_te = []
X_v_te = []
Y_te = []
for i in range(valid_count, ratings_count):
	X_i_te.append(data_i[perm[i]])
	X_v_te.append(data_v[perm[i]])
	Y_te.append(Y[perm[i]])
#	ui_te.append(U[perm[i]]], I[perm[i]])

X_i_te = np.array(X_i_te)
X_v_te = np.array(X_v_te)
Y_te = np.array(Y_te)

i1 = X_i_te[:, 0:4]
i2 = X_i_te[:, 4:10]
i3 = X_i_te[:, 10:]
x1 = X_v_te[:, 0:4]
x2 = X_v_te[:, 4:10]
x3 = X_v_te[:, 10:]
i4 = np.concatenate((i1,i3), axis=1)
x4 = np.concatenate((x1,x3), axis=1)

np.save("test_i_genre.npy", i2)
np.save("test_i_other.npy", i4)
np.save("test_x_genre.npy", x2)
np.save("test_x_other.npy", x4)
np.save("test_y.npy", np.array(Y_te))

print(len(X_i_tr))
print(len(X_i_va))
print(len(X_i_te))
print(len(Y))

f = open("feature.txt", 'w')
f.write(str(dict))


import sys

import pygame
from pygame import *
from pygame.mixer import Sound

import Config
from Config import *
from classes.menus.main_menu import main_menu

Config.current_menu = main_menu

def main():
    running = True

    while running:
        clock.tick(FPS)
        for p in main_menu.player_sprite:
            if Config.x > 0:
                Config.x = p.rect.x - WIN.get_width()/2
                if Config.x + Config.WIDTH < Config.map.get_width():
                    Config.x = p.rect.x - WIN.get_width() / 2
                else:
                    if p.rect.x + WIDTH/2 < Config.map.get_width():
                        Config.x = p.rect.x - WIN.get_width() / 2
                    Config.x = p.rect.x - (WIDTH - (Config.map.get_width() - p.rect.x))
            else:
                if p.rect.x - WIN.get_width()/2 > 0:
                    Config.x = p.rect.x - WIN.get_width()/2

            if p.rect.y - HEIGHT/2 > 0:
                if p.rect.y + Config.HEIGHT/2 < Config.map.get_height():
                    Config.y = p.rect.y - Config.HEIGHT/2
                elif p.rect.y + Config.HEIGHT / 2 > Config.map.get_height():
                    Config.y = Config.map.get_height() - HEIGHT
                else:
                    Config.y = 0

        Config.map.convert_alpha()
        WIN.blit(Config.map, (-1 * Config.x, -1 * Config.y))
        Config.current_menu.run()

        events = pygame.event.get()
        for event in events:
            if event.type == pygame.QUIT:
                running = False

    pygame.quit()
    sys.exit()

if __name__ == "__main__":
    main()


import scipy.io as io
from PIL import Image
from glob import glob
import os
from einops import rearrange
import numpy as np
import torch

class GTA_dataset():
    def __init__(self,
                path,
                split="train",
                img_size = (1914, 1052),
                mean=[0.5, 0.5, 0.5],
                std=[1, 1, 1],
                ignore_index=250):

        if not (split == "train" or split == "test" or split == "val"):
            assert("split only train/test/val")
        splits = io.loadmat(os.path.join(path, 'split.mat'))
        ids = None
        if (split == "train"):
            ids = splits["trainIds"][:,0]
        elif (split == "test"):
            ids = splits["testIds"][:,0]
        elif (split == "val"):
            ids = splits["valIds"][:,0]
        ids = [str(i).zfill(5)+".png" for i in ids]
        self.img_paths = [os.path.join(path, "images", item) for item in ids]
        self.lbl_paths = [os.path.join(path, "labels", item) for item in ids]
        self.img_size = img_size
        self.mean = mean
        self.std = std
        self.ignore_index = ignore_index
        self._init()
    def __len__(self):
        return len(self.lbl_paths)

    def __getitem__(self, index):
        img = Image.open(self.img_paths[index])
        lbl = Image.open(self.lbl_paths[index])
        img = img.resize(self.img_size, Image.BILINEAR)
        lbl = lbl.resize(self.img_size, Image.NEAREST)
        img = np.array(img, dtype=np.uint8)
        lbl = np.array(lbl, dtype=np.uint8)
        lbl = self.encode_segmap(lbl) # 将官方label的index变为我们关心的
        # 归一化
        img = img / 255.0
        img = img - self.mean
        img = img / self.std
        img = rearrange(img, "h w c -> c h w")
        # 转为tensor
        img = torch.from_numpy(img).float()
        lbl = torch.from_numpy(lbl).long()
        return img, lbl
    def _init(self):
        self.n_classes = 19
        self.valid_classes = [7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33,]
        self.class_names = ["road","sidewalk","building","wall","fence","pole","traffic_light",
            "traffic_sign","vegetation","terrain","sky","person","rider","car","truck","bus","train",
            "motorcycle","bicycle",]
        self.class_map = dict(zip(self.valid_classes, range(self.n_classes)))
    def encode_segmap(self, mask):
        """将label官方index转为想要的

        Args:
            mask (_type_): _description_

        Returns:
            _type_: _description_
        """
        label_copy = self.ignore_index * np.ones_like(mask, dtype=np.uint8)
        for k, v in list(self.class_map.items()):
            label_copy[mask == k] = v
        return label_copy


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 19 16:32:27 2021

@author: pierre
"""

import glob
from astropy.io import fits
import numpy as np
import os
import scipy.signal
from sinfobj import sinfobj
from get_mag import get_coord_sinfoni, get_2MASS_mags
from scipy.signal import medfilt
import matplotlib.pyplot as plt
import numpy.ma as ma
rep_save = '/media/pierre/Disque_2/SNR/CALIBRATED_1/'

def get_obj_header(filename):
    hdu = fits.open(filename)
    obj = hdu[0].data
    header = hdu[0].header
    hdu.close()
    return np.nan_to_num(obj), header

def get_fov(filename):
    hdu = fits.open(filename)
    header = hdu[0].header
    pscale = header['CDELT2']*3600
    fov = pscale*np.shape(hdu[0].data)[1]
    hdu.close()
    return fov

def get_coord(filename):
    hdu = fits.open(filename)
    header = hdu[0].header
    RA = header['RA']
    dec = header['DEC']
    hdu.close()
    return RA, dec

def get_band(filename):
    hdu = fits.open(filename)
    header = hdu[0].header
    wl = header['CRVAL3']
    if wl < 1.3:
        band ='j'
    elif wl < 1.8:
        band='h'
    else:
        band='k'
    return band

def get_zmin_zmax(filter_interp):
    zmin = 0
    zmax = len(filter_interp)
    first_half = True
    done = False
    for x in range(len(filter_interp)):
        # print(x)
        y = filter_interp[x]
        if y == 0 and first_half:
            zmin = x
        elif first_half:
            first_half = False
        elif  y == 0 and (not done):
            zmax = x
            done = True
    return zmin, zmax

# for rep1 in glob.glob(rep_save+'/*'):
#     for rep2 in glob.glob(rep1+'/*'):
#         for rep3 in glob.glob(rep2+'/*'):
#             for file in glob.glob(rep3+'/calibrated.fits'):
#                 if 'NGC' in file:
#                     print(file)
#                     obj, header = get_obj_header(file)
#                     raobj, decobj = get_coord(file)
#                     fov = get_fov(file)
#                     band = get_band(file)
#                     mags_obj = np.array(get_2MASS_mags(raobj, decobj, fov=fov, band=band, verbose=False))
#                     mag_obj = np.mean(mags_obj[~np.isnan(mags_obj)])

#                     xmax, ymax = np.unravel_index(np.argmax(scipy.signal.medfilt(np.median(np.nan_to_num(obj),0),(5,5))), np.shape(obj[0]))

#                     limits = [np.max([0,xmax-28]), np.min([xmax+28, np.shape(obj[0])[0]]),np.max([0,ymax-28]), np.min([ymax+28, np.shape(obj[0])[1]])]

#                     if band == 'j':
#                         F_0 = 3.129e-13*1e4# *1e4 pour passer de W/cm2/um à W/m2/um
#                         bwidth = 0.162
#                         filter_name = '/home/pierre/Documents/2021/SINFONI_analysis/2MASS_filters/J.txt'
#                         filt = np.loadtxt(filter_name).T

#                     if band == 'h':
#                         F_0 = 3.129e-13*1e4# *1e4 pour passer de W/cm2/um à W/m2/um
#                         bwidth = 0.162
#                         filter_name = '/home/pierre/Documents/2021/SINFONI_analysis/2MASS_filters/J.txt'
#                         filt = np.loadtxt(filter_name).T

#                     if band =='k':
#                         F_0 = 4.283e-14*1e4# *1e4 pour passer de W/cm2/um à W/m2/um
#                         bwidth = 0.261
#                         filter_name = '/home/pierre/Documents/2021/SINFONI_analysis/2MASS_filters/K.txt'
#                         filt = np.loadtxt(filter_name).T

#                     wl = (np.arange(len(obj))-header['CRPIX3'])*header['CDELT3']+header['CRVAL3']

#                     filter_interp = np.interp(wl, filt[0], filt[1])

#                     zmin, zmax = get_zmin_zmax(filter_interp)

#                     obj_2MASSed = obj*filter_interp[:, np.newaxis, np.newaxis]
#                     tot_adu = np.sum(np.nan_to_num(obj_2MASSed[:,limits[0]:limits[1],limits[2]:limits[3]]))
#                     med_adu = np.median(np.nan_to_num(obj_2MASSed[zmin:zmax,limits[0]:limits[1],limits[2]:limits[3]]))*len(obj_2MASSed[zmin:zmax, limits[0]:limits[1], limits[2]:limits[3]].flatten())
#                     if abs(2.5*np.log10(tot_adu/bwidth/F_0)-2.5*np.log10(med_adu/bwidth/F_0))>1.2:
#                         print(band, fov, mag_obj, -2.5*np.log10(tot_adu/bwidth/F_0), -2.5*np.log10(med_adu/bwidth/F_0), '(', med_adu, ' and not ', tot_adu, ')')
#                     else:
#                         print('OK')

filenames = glob.glob('/media/pierre/Disque_2/SNR/CALIBRATED_1/*Circi*/*50/N_*/*.fits')

filenames = glob.glob('/media/pierre/Disque_2/SNR/CALIBRATED_1/*1808*/H*/N_*/*.fits')
filename = '/media/pierre/Disque_2/SNR/archive/076B0098A/reflex_end_products/2021-04-23T11:22:26/SINFO.2005-03-24T00:07:57.876_tpl/calibrated/calibrated.fits'
specs = []
for filename in filenames:
    hdu = fits.open(filename)
    obj = sinfobj(hdu)
    obj.plot_spec(-0.5,0.5,-0.5,0.5, newfig=False)
    spec = obj.get_spec(-0.5,0.5,-0.5,0.5)[1]
    im = obj.get_im(1.6,1.7,-1,1,-1,1)
    xmax, ymax = np.unravel_index(np.argmax(medfilt(im,(5,5))), np.shape(im))
    if xmax > 5:
        specs.append(ma.masked_invalid(spec))
        print(xmax, ymax)
        print(obj.header['OBJECT'])

plt.figure()
plt.plot(np.mean(specs, 0))


from dashboard_api.models import SentimentModel
from dashboard_api.models import Companies
import pandas as pd
from django.db.models import Q
import os
import json

BASE="dashboard_api/cik_data/sentiment/"
failed = []
for file in os.listdir(BASE):
    print(file)
    if(1):
        derived_met_list=pd.read_csv(BASE+file, index_col=0)
        list = []
        for _, row in derived_met_list.iterrows():
            dict_ = {}
            dict_['item'] = row.name
            dict_['company'] = Companies.objects.get(cik=file.split('_')[1])
            if row['confidence'] != "empty" :
                dict_['confidence'] = row['confidence']
            else:
                continue
            dict_['label'] = row['label']
            # dict_['accession_no'] = file.split('_')[3]
            date = file.split('_')[2]
            dict_['filing_date'] = date[:4]+"-"+date[4:6]+"-"+date[6:]

            pos_neg = row['sentences_highlight'].replace('(', '[').replace(')', ']')
            dict_['positive'] = json.loads(pos_neg.split("'negative': ")[0][13:-2])
            dict_['negative'] = json.loads(pos_neg.split("'negative': ")[1][:-1])
            list.append(SentimentModel(**dict_))
    SentimentModel.objects.bulk_create(list)


import requests
import random_url
import sys
import json
import time
import pickle
import os
import datetime

def connect(url):
    r = requests.get(url)
    # print("Status Code:", r.status_code)

    if r.status_code != requests.codes.ok:
        print("Received a bad status code from the server -- aborting")
        print(r.text)
        # sys.exit()
    return r

def getPageResult(res):
    return json.loads(res.text)['results']

def getArrivalList(key, source):
    return [entry[key] for entry in source]

def getStops(predictions):
    stops = [(entry['sequence'], entry['stop_code_lbsl'])
             for entry in predictions]
    sorted(stops, key=lambda x: x[0])
    stops = [x[1] for x in stops]
    return stops

def getTime(l):
    return datetime.datetime.fromtimestamp(int(l / 1000))

def getPredictions(pre_res, tfl_res):
    predictions = getPageResult(pre_res)
    tfl_timetable = getPageResult(tfl_res)

    stops = getStops(predictions)

    historical = getArrivalList('cumulative_travel_time', predictions)
    print(historical)
    current = getArrivalList('curr_cumulative_travel_time', predictions)
    print(current)
    reference = getArrivalList('cumulative_travel_time', tfl_timetable)
    return (historical, current, reference, stops)

def connect_to_countdown(route, run, naptan_atco):
    url = ("http://countdown.api.tfl.gov.uk/interfaces/ura/instant_V1?"
           "LineName={}&DirectionID={}&StopCode2={}&"
           "ReturnList=StopID,LineName,DirectionID,VehicleID,TripID,"
           "EstimatedTime,ExpireTime")
    url = url.format(route, run, naptan_atco)
    # print(url)
    r = requests.get(url, auth=('LiveBus95085', 'rU9HUx4ZEm'))
    # print("Status Code:", r.status_code)

    if r.status_code != requests.codes.ok:
        print("Received a bad status code from the server -- aborting")
        print(r.text)
        # sys.exit()
    return r

def getCountdowns(r):
    buses = []
    for line in r.iter_lines():
        if not line:
            continue
        line = json.loads(line.decode('utf-8-sig'))
        if line[0] != 1:
            continue
        line = line[1:]
        buses.append(line)
    return buses

def getNextVehicle(countdown_res):
    # next_vehicles = []
    # while next_vehicles == []:
        # print("Getting countdowns...")
        # print(countdown_res)
        # print(len(countdown_res.lines()))
    next_vehicles = getCountdowns(countdown_res)
    sorted(next_vehicles, key=lambda x: x[5])
    if not next_vehicles:
        return []
    nxt = next_vehicles[0]
    vehicle = {}
    vehicle['stop_code_lbsl'] = nxt[0]
    vehicle['route'] = nxt[1]
    vehicle['run'] = nxt[2]
    vehicle['vehicle_id'] = nxt[3]
    vehicle['trip_id'] = nxt[4]
    vehicle['arrival_time'] = nxt[5]
    return vehicle

def load_current_predictions():
    if os.path.isfile("evaluate_api.p"):
        return pickle.load(open("evaluate_api.p", "rb"))
    else:
        return {}

def updateRecord(record, pre_res, tfl_res, countdown_res,
                 params, next_vehicle, record_time):
    print("Getting predictions")
    historical, current, reference, stops = getPredictions(pre_res, tfl_res)
    record_key = (params['day'], params['hour'], params['route'],
                  params['run'], params['naptan_atco'])
    record[record_key] = {'historical': historical,
                          'current': current,
                          'reference': reference,
                          'pre_url': pre_url,
                          'tfl_url': tfl_url,
                          'next_vehicle': next_vehicle,
                          'stops': stops,
                          'record_time': record_time}
    return record

for i in range(100):
    record = load_current_predictions()
    next_vehicle = []
    try:
        while not next_vehicle:
            pre_url, tfl_url, params = random_url.getCurrentURLs()

            print("Connecting to TfL countdown")
            print(params)
            countdown_res = connect_to_countdown(params['route'],
                                                 params['run'],
                                                 params['naptan_atco'])
            next_vehicle = getNextVehicle(countdown_res)
            record_time = time.time()
            print('next_vehicle')
            print(next_vehicle)
            if countdown_res.status_code != requests.codes.ok:
                continue
            if not next_vehicle:
                continue

            if getTime(next_vehicle['arrival_time']) - datetime.datetime.now() > datetime.timedelta(minutes=5):
                continue
                print('next loop')

            print("Connecting to endpoints")
            print(pre_url)
            pre_res, tfl_res = connect(pre_url), connect(tfl_url)
            if pre_res.status_code != requests.codes.ok:
                continue
            if tfl_res.status_code != requests.codes.ok:
                continue

            print("Updating record")
            print(countdown_res.text)
            record = updateRecord(record, pre_res, tfl_res, countdown_res,
                                  params, next_vehicle, record_time)
            print(len(record))
            print("Saving to pickle")
            pickle.dump(record, open("evaluate_api.p", "wb"))
            print("Finish")
    except ValueError:
        print("Error,")


from torch import nn
import torch
from torch.nn.init import normal, constant
import numpy as np
from torch.nn import functional as F
import math
from einops import repeat
from torch.autograd import Variable

class FocalLoss(nn.Module):
    def __init__(self, class_num, alpha=None, gamma=2, size_average=True):
        super(FocalLoss, self).__init__()
        if alpha is None:
            self.alpha = Variable(torch.ones(class_num, 1))
        else:
            if isinstance(alpha, Variable):
                self.alpha = alpha
            else:
                self.alpha = Variable(alpha)
        self.gamma = gamma
        self.class_num = class_num
        self.size_average = size_average

    def forward(self, inputs, targets):
        N = inputs.size(0)
        C = inputs.size(1)
        P = F.softmax(inputs)
        class_mask = inputs.data.new(N, C).fill_(0)
        class_mask = Variable(class_mask)
        ids = targets.view(-1, 1)
        class_mask.scatter_(1, ids.data, 1.)
        if inputs.is_cuda and not self.alpha.is_cuda:
            self.alpha = self.alpha.cuda()
        alpha = self.alpha[ids.data.view(-1)]
        probs = (P*class_mask).sum(1).view(-1,1)
        log_p = probs.log()
        batch_loss = -alpha*(torch.pow((1-probs), self.gamma))*log_p
        if self.size_average:
            loss = batch_loss.mean()
        else:
            loss = batch_loss.sum()
        return loss

class PositionalEncoding(nn.Module):

    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):
        super().__init__()
        self.dropout = nn.Dropout(p=dropout)

        position = torch.arange(max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))
        pe = torch.zeros(max_len, 1, d_model)
        pe[:, 0, 0::2] = torch.sin(position * div_term)
        pe[:, 0, 1::2] = torch.cos(position * div_term)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:x.size(0)]
        return self.dropout(x)

class SingleTransformer(nn.Module):
    def __init__(self, feat_in, hidden=768, nheads=8, nlayers=6, dropout=0.1):
        super().__init__()
        # from transformer import TransformerEncoderLayer
        self.feat_in = feat_in
        self.cls_token = nn.Parameter(torch.randn(1, 1, feat_in))#用这个token来聚合特征
        self.pos_encoder = PositionalEncoding(feat_in, dropout) #根据feat_in生成位置编码
        encoder_layer = nn.TransformerEncoderLayer(d_model=feat_in, nhead=nheads, dropout=dropout)
        # encoder_layer = TransformerEncoderLayer(d_model=feat_in, nhead=nheads, dropout=dropout)
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, nlayers, norm=nn.LayerNorm(feat_in))
        self.mlp = nn.Sequential(nn.Dropout(dropout), nn.Linear(feat_in, hidden))

    def forward(self, feats, output_all=False, key_padding_mask=None):
        # input: bs, seq, feat_in; output: bs, hidden
        b, n, _ = feats.shape
        cls_tokens = repeat(self.cls_token, '() n d -> b n d', b=b)
        feats = torch.cat((cls_tokens, feats), dim=1).permute(1, 0, 2)
        feats = self.pos_encoder(feats)
        if key_padding_mask is not None:
            cls_mask = key_padding_mask.new_zeros(b, 1)
            key_padding_mask = torch.cat([cls_mask, key_padding_mask], dim=1)
            feats = self.transformer_encoder(feats, src_key_padding_mask=key_padding_mask)
        else:
            feats = self.transformer_encoder(feats)
        # feats = self.transformer_encoder(feats)
        if output_all:
            return self.mlp(feats.permute(1, 0, 2))
        return self.mlp(feats[0])

class SingleTransformerClassifier(nn.Module):
    def __init__(self, feat_in, hidden=768, nheads=8, nlayers=6, dropout=0.1, num_classes=2513):
        super().__init__()
        self.single_transformer = SingleTransformer(feat_in, hidden, nheads, nlayers, dropout)
        self.linear = nn.Linear(hidden, num_classes)

    def forward(self, feats, output_feats=False):
        if output_feats:
            return self.single_transformer(feats)
        return self.linear(self.single_transformer(feats))

class SingleTransformerPretrain(nn.Module):
    def __init__(self, feat_in, hidden=768, nheads=8, nlayers=6, dropout=0.1):
        super().__init__()
        # self.single_transformer = SingleTransformer(feat_in, hidden, nheads, nlayers, dropout)
        self.transformer_encoder = SingleTransformer(feat_in, hidden, nheads, nlayers, dropout)
        self.linear = nn.Linear(hidden, feat_in)

    def forward(self, feats, output_all=False):
        # input: bs, seq, feat_in; output: bs, feat_in
        # return self.linear(self.single_transformer(feats, output_all=output_all))
        return self.linear(self.transformer_encoder(feats, output_all=output_all))

class FusionTransformer(nn.Module):
    def __init__(self, modality_in, hidden=768, feat_out=1024, num_class=2513, nheads=8, nlayers=6, dropout=0.1):
        super(FusionTransformer, self).__init__()
        self.pre_cls_token = nn.Parameter(torch.randn(1, 1, hidden))
        self.fut_cls_token = nn.Parameter(torch.randn(1, 1, hidden))
        self.modality_embeddings = nn.Parameter(torch.randn(1, modality_in, hidden))
        encoder_layer = nn.TransformerEncoderLayer(d_model=hidden, nhead=nheads, dropout=dropout)
        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, nlayers, norm=nn.LayerNorm(hidden))
        self.to_cls_token = nn.Identity()
        self.mlp_head = nn.Sequential(nn.Dropout(dropout), nn.Linear(hidden, feat_out))
        self.fc = nn.Sequential(nn.Dropout(dropout), nn.Linear(feat_out, num_class))

    def forward(self, modality_feats, task):
        b, n, _ = modality_feats.shape
        modality_feats += self.modality_embeddings[:, :(n + 1)]
        pre_cls_tokens = repeat(self.pre_cls_token, '() n d -> b n d', b=b)
        fut_cls_tokens = repeat(self.fut_cls_token, '() n d -> b n d', b=b)
        modality_feats = torch.cat((fut_cls_tokens, modality_feats), dim=1)
        modality_feats = torch.cat((pre_cls_tokens, modality_feats), dim=1).permute(1, 0, 2)
        modality_feats = self.transformer_encoder(modality_feats)
        hpre = self.mlp_head(modality_feats[0])
        hfut = self.mlp_head(modality_feats[1])
        if task == 'pre':
            action = self.fc(hpre)
        else:
            action = self.fc(hfut)
        return action, hpre, hfut

class MultiHeadAttention(nn.Module):

    def __init__(self, dim_self, dim_ref, num_heads, bias=True, dropout=0.2):
        super().__init__()
        self.num_heads = num_heads
        head_dim = dim_self // num_heads
        self.scale = head_dim ** -0.5
        self.to_queries = nn.Linear(dim_self, dim_self, bias=bias)
        self.to_keys_values = nn.Linear(dim_ref, dim_self * 2, bias=bias)
        self.project = nn.Linear(dim_self, dim_self)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, y=None, mask=None):
        device = x.device
        y = y if y is not None else x
        b, n, c = x.shape
        _, m, d = y.shape
        queries = self.to_queries(x).reshape(b, n, self.num_heads, c // self.num_heads)
        keys_values = self.to_keys_values(y).reshape(b, m, 2, self.num_heads, c // self.num_heads)
        keys, values = keys_values[:, :, 0], keys_values[:, :, 1]
        attention = torch.einsum('bnhd,bmhd->bnmh', queries, keys) * self.scale
        if mask is not None:
            mask = ~mask
            if mask.dim() == 2:
                mask = mask.unsqueeze(1)
            mask = mask.unsqueeze(3)
            attention = attention.masked_fill(mask, -float("inf"))  # batch_size, L, L, head
        zeros = torch.zeros(attention.shape[2], attention.shape[2]).to(device) # 256,256
        mask_templates = torch.eye(4).to(device)
        zeros[:4,:4] = mask_templates
        mask_templates = zeros.bool()
        attention = attention.softmax(dim=2)
        out = torch.einsum('bnmh,bmhd->bnhd', attention, values).reshape(b, n, c)
        out = self.project(out)
        return out, attention

class FusionLinear(nn.Module):
    def __init__(self, modality_in, hidden=768, feat_out=1024, num_class=2513, nheads=8, nlayers=6, dropout=0.5):
        super(FusionLinear, self).__init__()
        self.num_class = num_class
        self.feat_out = feat_out
        self.mlp_head = nn.Sequential(
            nn.Dropout(dropout),
            nn.Linear(hidden * modality_in, self.feat_out * 2),
        )

        self.fc = nn.Sequential(
            nn.Dropout(dropout),
            nn.Linear(feat_out, self.num_class)
        )
    def forward(self, modality_feats, task):
        b, n, d = modality_feats.shape
        modality_feats = self.mlp_head(modality_feats.view(b, n * d))
        hpre = modality_feats[:, :self.feat_out]
        hfut = modality_feats[:, self.feat_out:]
        if task == 'pre':
            action = self.fc(hpre)
        else:
            action = self.fc(hfut)
        return action, hpre, hfut


import telebot
from collections import defaultdict as def_dict
from telebot import types
import db_script
import time
import pass_key

Token = pass_key.token

add, name, picture, res_location, confirmation = range(5)

user_state = def_dict(lambda: add)

def get_state(message):  # узнает состояние
    return user_state[message.chat.id]

def update_state(message, state):  # обновляет состояние
    user_state[message.chat.id] = state

restarans = def_dict(lambda: {})  # словарь для ресторанов

def update_restarans(user_id, key, value):
    restarans[user_id][key] = value

def get_resarans(user_id):
    return restarans[user_id]

bot = telebot.TeleBot(Token, parse_mode=None)  # You can set parse_mode by default. HTML or MARKDOWN

@bot.message_handler(commands=['start'])
def handler_message(message):
    markup = types.InlineKeyboardMarkup(row_width=3)
    item1 = types.InlineKeyboardButton("add", callback_data='add')
    item2 = types.InlineKeyboardButton("reset", callback_data='reset')
    item3 = types.InlineKeyboardButton("list", callback_data='list')
    markup.add(item1, item2, item3)

    bot.send_message(message.chat.id,
                     "Добро пожаловать, {0.first_name}!\nЯ - <b>{1.first_name}</b>, бот созданный чтобы быть подопытным кроликом.\n Я могу запоминать название ресторанов, их фотографии и локацию".format(
                         message.from_user, bot.get_me()),
                     parse_mode='html', reply_markup=markup)

def reset_(message):
    markup = types.InlineKeyboardMarkup(row_width=1)
    item1 = types.InlineKeyboardButton("add", callback_data='add')
    markup.add(item1)
    db_script.reset(message.chat.id)
    bot.send_message(message.chat.id, text='База чиста', parse_mode='html', reply_markup=markup)

@bot.callback_query_handler(func=lambda message: message.data == 'reset')  # срабатывает на кнопку
def handler_message(message):
    reset_(message.message)

@bot.message_handler(commands=['reset'])  # срабатывает на команду
def handler_message(message):
    reset_(message)

def lits_(message):
    data = db_script.list(message.chat.id)
    counter = 0
    if data:
        for i in data:
            counter += 1
            bot.send_message(message.chat.id, text=f'---------- location {counter} ------------ ')
            try:
                bot.send_photo(message.chat.id, open(i[3], 'rb'), caption=f'Название ресторана: {i[2]}')
            except FileNotFoundError as e:
                bot.send_message(message.chat.id, text=f'фото не найдено!')

            bot.send_location(message.chat.id, i[5], i[4])  # latitude(широта), longitude(долгота)
    else:
        bot.send_message(message.chat.id, text='База пуста')

@bot.callback_query_handler(func=lambda message: message.data == 'list')  # срабатывает на кнопку
def handler_message(message):
    lits_(message.message)

@bot.message_handler(commands=['list'])  # срабатывает на команду
def handler_message(message):
    lits_(message)

@bot.callback_query_handler(func=lambda message: message.data == 'add')
def handler_message(message):
    bot.send_message(message.message.chat.id, text='Напиши название ресторана')
    update_state(message.message, name)

@bot.message_handler(commands=['add'])  # срабатывает на команду
def handler_message(message):
    bot.send_message(message.chat.id, text='Напиши название ресторана')
    update_state(message, name)

@bot.message_handler(func=lambda message: get_state(message) == name)
def handler_message(message):
    update_restarans(message.chat.id, 'name', message.text)
    bot.send_message(message.chat.id, text='Добавь фотографию ресторана')
    update_state(message, picture)

@bot.message_handler(content_types=['photo'], func=lambda message: get_state(message) == picture)
def handler_message(message):
    if message.photo:
        try:
            file_info = bot.get_file(message.photo[0].file_id)
            downloaded_file = bot.download_file(file_info.file_path)
            filepath = r'C:/Users/GoodMan/Desktop/telegram_bot/media/'

            src = filepath + message.photo[0].file_id
            with open(src, 'wb') as new_file:
                new_file.write(downloaded_file)
            bot.reply_to(message, "Фото добавлено")
            # print(src)

        except Exception as e:
            bot.reply_to(message, e)

        update_restarans(message.chat.id, 'picture', src)
        bot.send_message(message.chat.id, text='Добавь геопозицию ресторана')
        update_state(message, res_location)
    else:
        bot.send_message(message.chat.id, text=f'Вы прислали не фото: {message.text}.\n Попробуйте еще раз')
        update_state(message, picture)

@bot.message_handler(func=lambda message: get_state(message) == res_location, content_types='location')
def handler_message(message):
    if message.location:

        latitude = message.location.latitude
        longitude = message.location.longitude

        update_restarans(message.chat.id, 'res_location', [longitude, latitude])
        restaran = get_resarans(message.chat.id)
        update_state(message, confirmation)

        markup = types.InlineKeyboardMarkup(row_width=2)
        item1 = types.InlineKeyboardButton("Да", callback_data='Yes')
        item2 = types.InlineKeyboardButton("Нет", callback_data='No')
        markup.add(item1, item2)

        bot.send_message(message.chat.id, \
                         text=f'Напиши Да, если все правильно: \nНазвание: {restaran["name"]} \nГеопозиция: {restaran["res_location"]}',
                         parse_mode='html', reply_markup=markup)

    else:
        bot.send_message(message.chat.id, text=f'Вы прислали не локацию: {message.text}.\n Попробуйте еще раз')
        update_state(message, res_location)

@bot.callback_query_handler(func=lambda message: True)
# @bot.message_handler(func=lambda  message: get_state(message) == confirmation )
def handler_message(message):

    restaran = get_resarans(message.message.chat.id)

    markup = types.InlineKeyboardMarkup(row_width=3)
    item1 = types.InlineKeyboardButton("add", callback_data='add')
    item2 = types.InlineKeyboardButton("reset", callback_data='reset')
    item3 = types.InlineKeyboardButton("list", callback_data='list')
    markup.add(item1, item2, item3)

    if message.data == 'Yes':
        bot.send_message(message.message.chat.id, text='Сохранино в базу', parse_mode='html', reply_markup=markup)
        db_script.add_notice(message.message.chat.id, restaran)
        update_state(message.message, add)

    else:
        bot.send_message(message.message.chat.id, text='Попробуем еще раз)', parse_mode='html', reply_markup=markup)
        update_state(message.message, add)

bot.polling(none_stop=True, timeout=123)


import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt

dat = pd.read_csv('tput.csv')
dat = dat.groupby(by=["Buffer"]).mean().reset_index()
bufcap_arr = np.multiply(1000,np.array([10,100,1000,5000,10000]))
val_arr_1 = np.divide(np.array(dat.BBR_goodput),1000)
plt.xscale("log")
plt.plot(bufcap_arr,val_arr_1, label="BBR")
val_arr_2 = np.divide(np.array(dat.CUBIC_goodput),1000)
plt.xscale("log")
plt.plot(bufcap_arr,val_arr_2, label="CUBIC")
val_arr_3 = np.add(val_arr_1,val_arr_2)
plt.xscale("log")
plt.plot(bufcap_arr,val_arr_3, label="Total")
plt.title("Goodput comparison against buffer capacity")
plt.xlabel("Buffer capacity (in bits)")
plt.ylabel("Goodput (in Mbps)")
plt.legend(["BBR", "CUBIC","Total"])
plt.show()


#!/usr/bin/env python
# coding: utf-8
import click

from ..utils.logger import get_logger

@click.command(context_settings={"ignore_unknown_options": True})
@click.option(
    "--config", type=str, help="config json file",
)
@click.option("--max_itr", type=int, default=3, help="maximum iteration")
@click.option("--opt_path", type=click.Path(), default="opt/", help="path")
@click.option("--domain", type=str, default=None, help="domain file")
@click.option("--gpu", type=str, default=None, help="[kgcn arg]")
@click.option("--cpu", is_flag=True, help="[kgcn arg]")
def main(**kwargs):
    """ kGCN opt entry point
    """
    logger = get_logger("tkgcn-opt")
    logger.debug(f"arguments {kwargs}")


"""These two are used via decorators to validate the user is allowed to call the endpoint."""

from functools import wraps
from flask import abort, request
from datetime import datetime

from apps.users.models import UsersAccessTokens, UsersAccessMapping, Users
from settings.config import CONFIG

def invalid_token(user_id, access_token):
    """Check that user and user's token is valid."""
    token = UsersAccessTokens.query.filter_by(UserID=user_id).first()
    if token is None:
        return True
    if token.AccessToken != access_token:
        return True
    if token.ExpirationDate <= datetime.now():
        return True
    return False

def user_is_admin(user_id):
    """Check if the user is admin or not."""
    user = Users.query.filter_by(UserID=user_id).first()
    user_map = UsersAccessMapping.query.filter_by(UserID=user_id).first()
    if user is None:
        return False
    if user_map is None:
        return False
    if user_map.UsersAccessLevelID == CONFIG.ADMIN_LEVEL:
        return True
    return False

def user_is_registered_or_more(user_id):
    """Check that user is registered, moderator, or admin."""
    user = Users.query.filter_by(UserID=user_id).first()
    user_map = UsersAccessMapping.query.filter_by(UserID=user_id).first()
    if user is None:
        return False
    if user_map is None:
        return False
    if user_map.UsersAccessLevelID >= CONFIG.REGISTERED_LEVEL:
        return True
    return False

def admin_only(f):
    """When a route has this @admin_only decorator, that endpoint can only be accessed with a valid
    user_id and access_token provided in the request. Anything else will be a 401 Unauthorized."""
    @wraps(f)
    def check_user_level(*args, **kwargs):
        user_id = request.headers.get("User", "")
        access_token = request.headers.get("Authorization", "")

        if user_id and access_token:
            # Check that user is admin
            if user_is_admin(user_id) is False:
                abort(401)
            # And make sure the access token the user provided is still valid for that user
            if invalid_token(user_id, access_token):
                abort(401)
        else:
            abort(401)
        return f(*args, **kwargs)  # pragma: no cover
    return check_user_level

def registered_only(f):
    """When a route decorated with @registered_only is accessed, the user must be at least a valid
    Registered User level, or higher. Anything else will be a 401 Unauthorized."""
    @wraps(f)
    def check_user_level(*args, **kwargs):
        user_id = request.headers.get("User", "")
        access_token = request.headers.get("Authorization", "")

        if user_id and access_token:
            # Check that user is at least registered level. If less than that, respond with 401
            if user_is_registered_or_more(user_id) is False:
                abort(401)
            # And make sure the access token the user provided is still valid for that user
            if invalid_token(user_id, access_token):
                abort(401)
        else:
            abort(401)
        return f(*args, **kwargs)  # pragma: no cover
    return check_user_level

def user_id_or_guest(user_id):
    """Validate the input and return the userID, or the guest userID (= 1)"""
    try:
        valid_id = int(user_id)
    except (ValueError, TypeError):
        valid_id = 1
    if valid_id <= 0:
        valid_id = 1

    return valid_id

def validate_user(headers):
    """Validate the user and return the results."""
    user_id = headers.get("User", "")
    token = headers.get("Authorization", "")
    registered = False

    if user_id:
        valid_user_id = user_id_or_guest(user_id)
        registered = valid_user_id > 1
    else:
        valid_user_id = 1

    is_token_invalid = invalid_token(user_id, token)

    return valid_user_id, registered, is_token_invalid


import json
import tempfile
from pathlib import Path

from poif.tests import write_image_in_file
from poif.utils import hash_object, hash_string
from poif.versioning.directory import VersionedDirectory

temp_dir = Path(tempfile.mkdtemp())
img_folder = temp_dir / "image"
img_folder.mkdir(exist_ok=True)

img1_path = img_folder / "01.jpg"
img2_path = img_folder / "02.jpg"

write_image_in_file(img1_path)
write_image_in_file(img2_path)

img1_hash = hash_object(img1_path)
img2_hash = hash_object(img2_path)

dir_hash = hash_string(img1_hash + img2_hash)

def test_writing():
    directory = VersionedDirectory(base_dir=temp_dir, data_dir=img_folder)
    vdir_file = directory.write_vdir_to_folder(temp_dir)

    with open(vdir_file, "r") as f:
        vdir_contents = json.load(f)

    assert vdir_contents["data_folder"] == "image"
    assert vdir_contents["tag"] == dir_hash

    mapping_contents = directory.get_parsed()

    assert mapping_contents[img1_hash] == "image/01.jpg"
    assert mapping_contents[img2_hash] == "image/02.jpg"