"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.

When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B

"""
import copy
import datetime
import sys
from datetime import timezone

from celery import Celery
from flask import current_app

from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service

flask_app = create_app()

def make_celery(app):
    celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
                    broker=app.config['CELERY_BROKER_URL'])
    celery.conf.update(app.config)
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)

    celery.Task = ContextTask
    return celery

celery = make_celery(flask_app)

@celery.task()
def fetch_acme_cert(id):
    """
    Attempt to get the full certificate for the pending certificate listed.

    Args:
        id: an id of a PendingCertificate
    """
    log_data = {
        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
    }
    pending_certs = pending_certificate_service.get_pending_certs([id])
    user = user_service.get_by_username('lemur')
    new = 0
    failed = 0
    wrong_issuer = 0
    acme_certs = []

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            acme_certs.append(cert)
        else:
            wrong_issuer += 1

    authority = plugins.get("acme-issuer")
    resolved_certs = authority.get_ordered_certificates(acme_certs)

    for cert in resolved_certs:
        real_cert = cert.get("cert")
        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)

        if real_cert:
            # If a real certificate was returned from issuer, then create it in Lemur and delete
            # the pending certificate
            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
            pending_certificate_service.delete_by_id(pending_cert.id)
            # add metrics to metrics extension
            new += 1
        else:
            failed += 1
            error_log = copy.deepcopy(log_data)
            error_log["message"] = "Pending certificate creation failure"
            error_log["pending_cert_id"] = pending_cert.id
            error_log["last_error"] = cert.get("last_error")
            error_log["cn"] = pending_cert.cn

            if pending_cert.number_attempts > 4:
                error_log["message"] = "Deleting pending certificate"
                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
            else:
                pending_certificate_service.increment_attempt(pending_cert)
                pending_certificate_service.update(
                    cert.get("pending_cert").id,
                    status=str(cert.get("last_error"))
                )
                # Add failed pending cert task back to queue
                fetch_acme_cert.delay(id)
            current_app.logger.error(error_log)
    log_data["message"] = "Complete"
    log_data["new"] = new
    log_data["failed"] = failed
    log_data["wrong_issuer"] = wrong_issuer
    current_app.logger.debug(log_data)
    print(
        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
            new=new,
            failed=failed,
            wrong_issuer=wrong_issuer
        )
    )

@celery.task()
def fetch_all_pending_acme_certs():
    """Instantiate celery workers to resolve all pending Acme certificates"""
    pending_certs = pending_certificate_service.get_pending_certs('all')

    # We only care about certs using the acme-issuer plugin
    for cert in pending_certs:
        cert_authority = get_authority(cert.authority_id)
        if cert_authority.plugin_name == 'acme-issuer':
            if cert.last_updated == cert.date_created or datetime.datetime.now(
                    timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
                fetch_acme_cert.delay(cert.id)


"""Module with helper function to find libraries, load behaviours, ..."""
from __future__ import nested_scopes
from jython_compat import *

import os
import sys
import time

# get main configuration
import config

# to identify behaviours
from behaviour import Behaviour

# private constants
_behaviour_classes = (Behaviour, )

# counters
_agent_id = 0

class World:
    """The World class that is used to communicate with the world
    initialisation script.

    Upon running the world initialisation script, using the L{run_world_script},
    an instance of this class, named 'world' is given to the script. The
    script can use this instance to gather information on how the world is to
    be initialised, and can return the world object and other information to
    the instance that calls this script.
    """
    def __init__(self, library, world_args = None, agents_init = None):
        """Initialises an instance with the neccessary information.

        @param library: name of the behaviour library that is to be used.
        @type library: string
        @param world_args: arguments to be given to the world initialisation
            script.
        @type world_args: string
        @param agents_init: structure containing information to initialise
            the agents.
        @type agents_init: as returned by
            L{POSH.agentinitparser.parse_agent_init_file}
        """
        self._library = library
        self._args = world_args
        if agents_init == None:
            self._agents_init = []
        else:
            self._agents_init = agents_init
        self._creates_agents = False
        self._world = None

    def args(self):
        """Returns the arguments for customised world initialisation.

        If no arguments are given, None is returned.

        @return: arguments for world initialisation.
        @rtype: string or C{None}
        """
        return self._args

    def set(self, x):
        """Sets the world object for use when initialising the agents.

        The world object given to this method is given to the agents upon
        initialisation.

        @param x: The world object
        @type x: any
        """
        self._world = x

    def library(self):
        """Returns the behaviour library name that the agents are to use.

        @return: name of the library to use
        @rtype: string
        """
        return self._library

    def agentsInit(self):
        """Returns the agents initialisation structure.

        @return: agents initialisation structure.
        @rtype: as returned by L{POSH.agentinitparser.parse_agent_init_file}
        """
        return self._agents_init

    def createsAgents(self):
        """Specifies that the agents are created an run by the world
        initialisation script.

        By default, the world initialisation script is only responsible for
        setting up the world, and eventually returning the world object to
        initialise the agents with. Calling this method from the world
        initialisation script indicates that both creation and running the
        agents is performed by the world initialisation script. For this
        purpose, the script can use L{library()} and L{agentsInit()}.
        """
        self._creates_agents = True

# to provide current_time()
if sys.platform == "win32":
    # On Windows, the best timer is time.clock()
    current_time = time.clock
else:
    # On most other platforms the best timer is time.time()
    current_time = time.time

def get_root_path():
    """Returns the root path that POSH is installed in. Assumes that this
    module resides in the root path.

    @return: Root path
    @rtype: string
    """
    return os.path.split(config.__file__)[0]

def _get_library_path(library = None):
    """Returns the path to the behaviour library. If no library is given,
    the the base path of the library is returned.

    @param library: The library to return the path for
    @type library: string
    @return: The base library path or the path to the given library
    @rtype: string
    """
    if library:
        return os.path.join(get_root_path(), config.LIBRARYPATH, library)
    else:
        return os.path.join(get_root_path(), config.LIBRARYPATH)

def _get_plans_path(library):
    """Returns the path to the plans of the given behaviour library.

    @param library: The library to return the path for
    @type library: string
    @return: The path to the plans
    @rtype: string
    """
    return os.path.join(_get_library_path(library), config.PLANSPATH)

def _is_behaviour_class(c):
    """Returns if the given class object is a subclass of a behaviour class.

    @param c: A class object
    @type c: class
    @return: If the given class is a subclass of L{POSH.Behaviour}
    @rtype: bool
    """
    for behaviour_class in _behaviour_classes:
        try:
            if issubclass(c, behaviour_class) and c != behaviour_class:
                return True
        except TypeError:
            # will be raise whenever c is not an object of type classobj
            pass
    return False

def get_plans(library):
    """Returns a list of available plans for the given behaviour library.

    The plans need to be located at library/PLANS and end in PLANENDING.

    @param library: The library to return the list of plans for
    @type library: string
    @return: A list of plans without file ending
    @rtype: sequence of strings
    """
    planpath = _get_plans_path(library)
    try:
        dirlist = os.listdir(planpath)
    except OSError:
        return []
    planfilter = lambda f: \
        (not f[:1] == '.') and \
        f[-len(config.PLANENDING):].lower() == config.PLANENDING.lower() and \
        os.path.isfile(os.path.join(planpath, f))
    planfiles = filter(planfilter, dirlist)
    # remove the endings of the files
    return [filename[:-len(config.PLANENDING)] for filename in planfiles]

def get_plan_file(library, plan):
    """Returns the plan file name for the given library and plan

    @param library: The library that the plan is from
    @type library: string
    @param plan: The name of the plan (without the .lap ending)
    @type plan: string
    @return: The filename with full path of the plan
    @rtype: string
    """
    return os.path.join(_get_plans_path(library),
                        "%s%s" % (plan, config.PLANENDING))

def get_library_file(library, file):
    """Returns the file name for the given library (not for plans, see above)

    @param library: The library that the file is in
    @type library: string
    @param plan: The name of the file (including any needed ending)
    @type plan: string
    @return: The filename with full path
    @rtype: string
    """
    return os.path.join(_get_library_path(library),
                        "%s" % (file))

def is_plan(library, plan):
    """Returns if the given plan of the given library exists.

    This method only checks if the plan file exists, not if its syntax is
    correct.

    @param library: The library that the plan is from
    @type library: string
    @param plan: The name of the plan (without the .lap ending)
    @type plan: string
    @return: If the plan exists (i.e. is a file)
    @rtype: bool
    """
    return os.path.isfile(get_plan_file(library, plan))

def get_libraries():
    """Returns a list of available behaviour libraries.

    The function assumes that any directory in the libraries path that
    does not start with an '.' is a behaviour library.

    @return: List of libraries
    @rtype: sequence of strings
    """
    library_path = _get_library_path()
    dirlist = os.listdir(library_path)
    libraryfilter = lambda d: \
        (not d[:1] == '.') and \
        os.path.isdir(os.path.join(library_path, d))
    return filter(libraryfilter, dirlist)

def is_library(library):
    """Returns if the given library is a valid behaviour library.

    This is the case if the following requirements are met:
        - there is a directory with the given name in LIBRARYPATH
        - the directory can be loaded as a module, which requires it to at least
          contain a __init__.py file

    @param library: Name of the library to check
    @type library: string
    @return: If the library is a valid library
    @rtype: bool
    """
    try:
        lib_module = __import__("%s.%s" % (config.LIBRARYPATH, library))
    except ImportError:
        return False
    return True

def get_behaviours(library, log = None):
    """Returns a sequence of classes, containing all behaviour classes that
    are available in a particular library.

    The method searches the behvaiour subclasses by attempting to import
    all file in the library ending in .py, except for the WORLDSCRIPT, and
    search through all classes they contain to see if they are derived from
    any behaviour class.

    If a log object is given, then logging output at the debug level is
    produced.

    @param library: Name of the library to find the classes for
    @type library: string
    @param log: A log object
    @type log: as from the Python logging module
    @return: The list of classes
    """
    # get list of python files is behaviour library
    if log != None:
        log.debug("Scanning library '%s' for behaviour classes" % library)
    librarypath = _get_library_path(library)
    try:
        dirlist = os.listdir(librarypath)
    except OSError:
        return []
    pyfilter = lambda f: \
        (not f[:1] == '.') and \
        f[-3:].lower() == '.py' and \
        os.path.isfile(os.path.join(librarypath, f))
    pylist = filter(pyfilter, dirlist)
    # try to load the library modules to collect the behaviour classes
    behaviour_classes = []
    for pyfile in pylist:
        if pyfile == config.WORLDSCRIPT:
            continue
        modname = pyfile[:-3]
        # try to load the module
        try:
            behaviour_mod = __import__('%s.%s.%s' % \
                                       (config.LIBRARYPATH, library, modname))
        except Exception:
            # ignore any non-importable python file
            if log != None:
                log.debug("Loading the file '%s' failed" % pyfile)
            continue
        # workaround to get the real module rather than its top-level parent
        # (see __import__ documentation)
        behaviour_mod = getattr(getattr(behaviour_mod, library), modname)
        # add all behaviour class objects in module to list
        mod_objs = [getattr(behaviour_mod, objname) \
                    for objname in dir(behaviour_mod)]
        found_classes = filter(_is_behaviour_class, mod_objs)
        if log != None:
            class_names = ', '.join([c.__name__ for c in found_classes])
            log.debug("Found %d behaviour class(es) in '%s': %s" % \
                      (len(found_classes), pyfile, class_names))
        behaviour_classes += found_classes
    # remove double entries
    behaviour_set = {}
    for behaviour_class in behaviour_classes:
        behaviour_set[behaviour_class] = None
    return behaviour_set.keys()

def default_agent_init(library):
    """Returns the default agent initialisation file filename for the given
    library.

    If no such file exists, and emtpy string is returned.

    @param library: Name of the library
    @type library: string
    @return: Agent initialisation file filename, or '' if not found
    @rtype: string
    """
    agent_init = os.path.join(_get_library_path(library), config.AGENTINIT)
    if os.path.isfile(agent_init):
        return agent_init
    else:
        return ''

def unique_agent_id():
    """Returns a unique agent id string of the form 'Axx', where xx is
    an increasing number, starting from 00.

    If more than 99 agents are created, the string length adjusts to the
    length of the number.

    @return: Unique agent id
    @rtype: string
    """
    global _agent_id
    _agent_id += 1
    return 'A%02d' % (_agent_id - 1)

def default_world_script(library):
    """Returns the default world initialisation script for the given library.

    If no script was found, then an empty string is returned.

    @param library: Name of the library
    @type library: string
    @return: World initialisatoin script filename, of '' if not found
    @rtype: string
    """
    world_script = os.path.join(_get_library_path(library), config.WORLDSCRIPT)
    if os.path.isfile(world_script):
        return world_script
    else:
        return ''

def run_world_script(script_file, library,
                     world_args = None, agents_init = None):
    """Runs the given file to initialise the world and returns the world object
    and if the world initialisation script creates and runs the agents
    itself.

    This method creates an instance of the class L{World} and makes it
    accessible to the world initialisation script under the instance name
    'world'. Using this instance, the world intialisation script can
    cimmunicate with this function.

    @param script_file: The filename of the script to run
    @type script_file: string
    @param library: name of the behaviour library to use by agents
    @type library: string
    @param world_args: arguments given to the world initialisation script
    @type world_args: string
    @param agents_init: agent initialisation information structure
    @type agents_init: as returned by
        L{POSH.agentinitparser.parse_agent_init_file}
    @return: tuple (world object, if script created and ran agents)
    @rtype: (any, bool)
    @raise IOError: If it cannot file the script
    @raise Exception: If the script causes an exception
    """
    w = World(library, world_args, agents_init)
    # world instance is given as local variable
    # note that if the global and local variables (when calling execfile)
    # refer to a different object, then some strange things happen as soon as
    # the script starts importing other modules (don't know why). So both
    # arguments have to refer to the same object!
    variables = {'world' : w}
    execfile(script_file, variables, variables)
    return (w._world, w._creates_agents)

#There must be a better way (& place) to do this...
#Note, because jython (or maybe mason.jar) is currently 1.4, we need to be sure to compile our classes in 1.4 too
def compile_mason_java():
      import os
      ext1='.java'
      ext2='.class'
      dir = os.path.join(get_root_path(), config.MASONPATH)
      dir=get_root_path()+'/platform_files/MASON/'
      java_src=filter((lambda str:str!="__init__"),map((lambda str:str[:len(str)-len(ext1)]),filter((lambda str: str.endswith(ext1)),os.listdir(dir))))
      classes=filter((lambda str:str!="__init__$py"),map((lambda str:str[:len(str)-len(ext2)]),filter((lambda str:str.endswith(ext2)),os.listdir(dir))))

      if java_src!=classes:
            cmd = 'javac -target 1.4 -source 1.4 -cp %smason.jar %s*.java' % (dir,dir)
            os.system(cmd)


import csv
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.linear_model import Lasso
import plotly.graph_objects as go

# since the feature of yellow is similiar to green taxi, using the green as representive
# use the preprocessed dataset to run this python file
newdf = pd.DataFrame()
count = 4
while count < 7:
    # thru three monthes data, break huge data into chunks for reserving memory
    for chunk in pd.read_csv("dataset/g0"+str(count)+".csv", chunksize=10000):

        # keep relevant featrues for pairwise analysis
        pool = chunk[['RateCodeID', 'Pickup_longitude', 'Pickup_latitude',
       'Dropoff_longitude', 'Dropoff_latitude', 'Passenger_count',
       'Trip_distance', 'Extra', 'MTA_tax', 'Tip_amount',
       'Tolls_amount', 'improvement_surcharge', 'Total_amount', 'Payment_type',
       'Trip_type ']].copy()

        # sampling 20% from the data each chunk to save memory
        sampling = pool.sample()
        newdf = newdf.append(sampling)

    count += 1

# since looking for ppl willing to pay high tips, tip is the goal we need to predict
X = newdf.drop('Tip_amount', axis=1)
y= newdf['Tip_amount']

# use lasso to help the regression more precise
lasso = Lasso()
lasso.fit(X, y)

# perform feature selection visualization
kept_cols = [feature for feature, weight in zip(X.columns.values, lasso.coef_) if weight != 0]
figt = go.Figure(
         go.Waterfall(name= "Lasso Coefficients",
                      orientation= "h",
                      y = X.columns.values,
                      x = lasso.coef_))

figt.update_layout(title = "Coefficients of Lasso Regression Model")

figt.show()


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash_html_components.P import P
from dash_html_components.Span import Span
import plotly.express as px
import pandas as pd
from tqdm import tqdm
from datetime import datetime
import augusto
from dash.dependencies import Input, Output
import locale
import os
import time

locale.setlocale(locale.LC_TIME, 'pt_BR.UTF-8')

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

# para deploy na url exemplo.com/painel, adicionar
# requests_pathname_prefix='/painel/'
app = dash.Dash(__name__, external_stylesheets=external_stylesheets, meta_tags=[
        {"name": "viewport", "content": "width=device-width, initial-scale=1"}
    ])
# titulo da pagina
app.title = "Vacinômetro"
server = app.server

CURRENT_DIR = os.path.dirname(__file__)

## Lê o timestamp dos dados, converte para float e traduz para datetime e mostra no dashboard
timestamp = float(open(os.path.join(CURRENT_DIR, 'datasets', 'data_timestamp.txt'), 'r').read())
DATA_DATETIME = datetime.fromtimestamp(timestamp).strftime("%d de %B de %Y, às %H:%M")

## Lê a string com a data coletada na hora de baixar os dados com baixar_csv.py
DATA_DATE_STR = open(os.path.join(CURRENT_DIR, 'datasets', 'data_date_str.txt'), 'r').read()

lista_estados = sorted(os.listdir(os.path.join(CURRENT_DIR, 'datasets')))

exclude_list = ['data_timestamp.txt', 'data_date_str.txt', 'velocidades_doses.csv', 'cod_cidades.csv', 'faixas_niveis_2020.zip', 'df_tratado.csv', 'faixas_niveis_2020_ex.csv', 'datasus', 'ibge', 'raw']

all_options = {
    item: ['TODOS']+sorted([mun for mun in os.listdir(os.path.join(CURRENT_DIR, 'datasets', item, 'abandono-atraso-vacinal')) if mun!='TODOS']) for item in lista_estados if item not in exclude_list
}

app.layout = html.Div(children=[
    html.H1(children='Painel Vacinação COVID-19'),

    html.Div(children=[
        html.Span('Iniciativa do '),
        html.A('Laboratório de Estatística e Ciência de Dados da UFAL', href='http://led-ufal.xyz/'),
        html.Span(' e dentro do projeto '),
        html.A('ModCovid19', href='http://www.cemeai.icmc.usp.br/ModCovid19/'),
    ], style=dict(fontSize='25px')),
    html.Div(children="""
        Os gráficos abaixo são interativos. Você pode clicar nas legendas para selecionar as opções, dar zoom
        e selecionar partes dos gráficos, fazer download, fazer comparações interativas ao longo dos eixos, etc.
        """, style=dict(fontSize='18px', marginTop='8px')),
    html.Div(children='Dados coletados em: {}'.format(DATA_DATE_STR),
        style=dict(fontSize='18px', marginTop='8px')),
    html.Div([
        html.Div([
            html.H4('Estado'),
            dcc.Dropdown(
		        id='estado-dropdown',
		        options=[{'label': k, 'value': k} for k in all_options.keys()],
		        value='BRASIL'
		    ),
        ], className="one columns"),

        html.Div([
            html.H4('Município'),
            dcc.Dropdown(
		        id='municipio-dropdown',
		    )
        ], className="two columns"),
    ], className="row", style=dict(display='flex', justifyContent='center')),

    html.H1('Doses aplicadas e atraso vacinal',
        style=dict(display='flex', justifyContent='center', marginTop='50px', marginBottom='50px')),

    html.Div(id='indicadores'),

    html.H1('Cobertura Vacinal',
            style=dict(display='flex', justifyContent='center', marginTop='50px', marginBottom='50px')),

    html.Div([
        html.Div([
            html.Div(id='mostrar-cobertura'),
        ], className="six columns"),

    html.Div([
        html.Div([
            html.Div(id='mostrar-cobertura-total'),
        ], className="six columns"),

    ], className="row"),]),

    html.Div([
        html.Div([
            html.H3('Doses aplicadas por dia'),
            html.H5('Número total de doses aplicadas nos últimos dias por tipo de vacina. As cores representam os diferentes tipos de doses.'),
            html.H6('Tipo de vacina'),
                dcc.Dropdown(
                    id='tipo-vacina-dropdown',
                    options=[
                        {'label': 'TODAS', 'value': 'todas'},
                        {'label': 'AstraZeneca', 'value': 'astrazeneca'},
                        {'label': 'CoronaVac', 'value': 'coronavac'},
                        {'label': 'Pfizer', 'value': 'pfizer'}],
                    value='todas'
                ),
            html.Div(id='mostrar-doses-aplicadas'),
        ], className="six columns"),

        html.Div([
            html.Div(id='mostrar-atraso'),
        ], className="six columns"),
    ], className="row"),

    html.H1('Demandas por vacina',
        style=dict(display='flex', justifyContent='center', marginTop='50px', marginBottom='50px')),

    html.Div([
        html.Div([
            html.Div(id='mostrar-demanda'),
        ], className="six columns"),

        html.Div([
            html.Div(id='mostrar-demanda-vacina'),
        ], className="six columns"),
    ], className="row"),

    html.H1('Vacinação Precoce e Abandono Vacinal',
        style=dict(display='flex', justifyContent='center',  marginTop='50px', marginBottom='50px')),

    html.Div([
        html.Div([
            html.Div(id='mostrar-adiantamento'),
        ], className="six columns"),

        html.Div([
            html.Div(id='mostrar-abandono'),
        ], className="six columns"),
    ], className="row"),
        dcc.Markdown(
        """
        ### Intervalos para a segunda dose recomendados pelo fabricante

        |  Tempo | Astrazeneca | CoronaVac |  Pfizer |
        |:------:|:-----------:|:---------:|:-------:|
        | Mínimo |   56 dias   |  14 dias  | 21 dias |
        | Máximo |   84 dias   |  28 dias  | 84 dias |

        ### * Observações

        Foram tratadas as seguintes anomalias para a realização das análises:

        * Foram removidos os registros de vacinas com nome de "Pendente Identificação";
        * Foram removidas as pessoas cuja 2ª dose foi registrada antes da 1ª dose;
        * Foram removidas as pessoas cuja 1ª dose foi aplicada antes de 2021;
        * Foram removidas as doses informadas mais de uma vez para a mesma pessoa, com mesma data de aplicação;
        * Foram transformadas em 1ª dose as aplicações de 2ª dose correspondentes às pessoas que têm apenas uma dose informada;
        * Foram removidas as pessoas que tomaram doses de vacinas diferentes;
        * Foi utilizado 80% das pessoas com mais de 20 anos de acordo com estimativa de 2020 do Datasus para o cálculo do Fim da Vacinação.

        """
    ),

])

@app.callback(
    Output('municipio-dropdown', 'options'),
    Input('estado-dropdown', 'value'))
def set_opcoes_municipios(estado_selecionado):
    return [{'label': i, 'value': i} for i in all_options[estado_selecionado]]

@app.callback(
    Output('municipio-dropdown', 'value'),
    Input('municipio-dropdown', 'options'))
def set_valor_municipio(opcoes_disponiveis):
    return opcoes_disponiveis[0]['value']

@app.callback(
    Output('mostrar-doses-aplicadas', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'),
    Input('tipo-vacina-dropdown', 'value'))
def set_display_children(estado_selecionado, municipio_selecionada, vacina_selecionada):
    return [
            dcc.Graph(
                id='doses-graph',
                figure=augusto.plotar_doses_por_dia(
                    estado_selecionado, municipio_selecionada, vacina_selecionada)
            )]

@app.callback(
    Output('mostrar-demanda', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'))
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Demanda por dia'),
        html.H5('Número total acumulado de doses necessárias para aplicar a segunda dose somente nas pessoas cujo o prazo máximo '
        'de aplicação vence nos próximos dias.'),
            dcc.Graph(
                id='demanda-graph',
                figure=augusto.plotar_demanda_por_dia(estado_selecionado, municipio_selecionada)
            )
    ]

@app.callback(
    Output('mostrar-demanda-vacina', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'))
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Demanda por vacina'),
        html.H5('Número total acumulado de doses necessárias para aplicar a segunda dose somente nas pessoas cujo o prazo máximo '
        'de aplicação vence nos últimos dias, por tipo de vacina.'),
            dcc.Graph(
                id='demanda-vacina-graph',
                figure=augusto.plotar_demanda_por_vacina(estado_selecionado, municipio_selecionada)
            )
    ]

@app.callback(
    Output('mostrar-atraso', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value')
)
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Atraso de vacinacão'),
        html.H5('Número total de pessoas que receberam a segunda dose e que estavam com o '
                'calendário de vacinação  atrasado em cada dia, de acordo com os períodos mínimos e máximos '
                'recomendados pelos fabricantes de cada vacina listados no fim da página.'),
        dcc.Graph(
            id='delay-graph',
            figure=augusto.plot_delay(estado_selecionado, municipio_selecionada, "Fora do prazo (atrasou)")
        )
    ]

@app.callback(
    Output('mostrar-adiantamento', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value')
)
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Adiantamento de vacinacão'),
        html.H5('Número total de pessoas que receberam a segunda dose antes do dia '
                'recomendado pelo fabricante da vacina  em cada dia.'),
        dcc.Graph(
            id='sooner-graph',
            figure=augusto.plot_delay(estado_selecionado, municipio_selecionada, "Fora do prazo (antecipou)")
        )
    ]

@app.callback(
    Output('mostrar-abandono', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value')
)
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Evolução do atraso de vacinação'),
        html.H5('Número total de pessoas que iniciaram o esquema vacinal (tomaram a '
                'primeira dose) e não concluíram (já deveriam ter tomado a segunda dose).'),
        dcc.Graph(
            id='abandon-graph',
            figure=augusto.plot_delay(estado_selecionado, municipio_selecionada, "Abandono")
        )
    ]

@app.callback(
    Output('mostrar-cobertura', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'))
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Cobertura vacinal por faixa'),
        html.H5('Número de pessoas por sexo e idade vacinadas no município. '),
            dcc.Graph(
                id='cobertura-graph',
                figure=augusto.plotar_cobertura(estado_selecionado, municipio_selecionada)
            )
    ]

@app.callback(
    Output('mostrar-cobertura-total', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'))
def set_display_children(estado_selecionado, municipio_selecionada):
    return [
        html.H3('Cobertura percentual'),
        html.H5('Percentual de pessoas por status de vacinação. '),
            dcc.Graph(
                id='cobertura-total-graph',
                figure=augusto.plotar_cobertura_total(estado_selecionado, municipio_selecionada)
            )
    ]

@app.callback(
    Output('indicadores', 'children'),
    Input('estado-dropdown', 'value'),
    Input('municipio-dropdown', 'value'),
    Input('tipo-vacina-dropdown', 'value'))
def indicadores(estado_selecionado, municipio_selecionada, vacina_selecionada):
    total, qnt_1as_doses, qnt_2as_doses, media_1as, media_2as, previsao = augusto.indicadores(
        estado_selecionado, municipio_selecionada, vacina_selecionada
    )

    r = html.Div([
        html.Div([
            html.H2('Total Aplicado'),
            html.H4(total)
        ], className='two columns'),
        html.Div([
            html.H2('1ª Doses'),
            html.H4(qnt_1as_doses)
        ], className='two columns'),
        html.Div([
            html.H2('2ª Doses'),
            html.H4(qnt_2as_doses)
        ], className='two columns'),
        html.Div([
            html.H2('Ritmo de Vacinação'),
            html.H4("{} doses/dia".format(media_1as)),
            html.P('para as primeiras doses')
        ], className='two columns'),
        html.Div([
            html.H2('Ritmo de Vacinação'),
            html.H4("{} doses/dia".format(media_2as)),
            html.P('para as segundas doses')
        ], className='two columns'),
        html.Div([
            html.H2('Fim da Vacinação*'),
            html.H4('{}'.format(previsao)),
        ], className='two columns'),
    ], className="row")

    return r

if __name__ == '__main__':
    app.run_server(host='0.0.0.0',port=5000, debug=True)


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data

class MTLCNN_single(nn.Module):
    def __init__(self, texture_labels):
        super(MTLCNN_single, self).__init__()

        # shared conv layers
        self.conv1 = nn.Conv2d(3, 96, kernel_size=11, stride=4)
        # torch.nn.init.xavier_uniform(self.conv1.weight)
        self.bn1 = nn.BatchNorm2d(96)

        self.conv2 = nn.Conv2d(96, 256, kernel_size=5, stride=1, padding=2)
        # torch.nn.init.xavier_uniform(self.conv2.weight)
        self.bn2 = nn.BatchNorm2d(256)

        self.conv3 = nn.Conv2d(256, 384, kernel_size=3, stride=1, padding=1)
        # torch.nn.init.xavier_uniform(self.conv3.weight)

        self.max_pool = nn.MaxPool2d(3, 2)
        self.drop_out = nn.Dropout(p=0.5)

        self.avg_pool = nn.AvgPool2d(13, 1)

        # Texture classification task
        self.fc_texture_1 = nn.Linear(in_features=384, out_features=4096)
        # torch.nn.init.xavier_uniform(self.fc_texture_1.weight)
        self.fc_texture_2 = nn.Linear(in_features=4096, out_features=4096)
        # torch.nn.init.xavier_uniform(self.fc_texture_2.weight)
        self.texture_out = nn.Linear(in_features=4096, out_features=len(texture_labels))

    def forward(self, x):
        if torch.cuda.is_available():
            x = x.float().cuda()
        else:
            x = x.float()

        # shared conv
        x = F.relu(self.bn1(self.conv1(x)))
        x = self.max_pool(x)
        x = F.relu(self.bn2(self.conv2(x)))
        x = self.max_pool(x)
        x = F.relu(self.conv3(x))

        # texture classification

        texture_head = self.avg_pool(x)
        texture_head = texture_head.reshape(-1, 384 * 1)

        texture_head = F.relu(self.fc_texture_1(texture_head))
        texture_head = F.relu(self.fc_texture_2(texture_head))
        texture_head = self.texture_out(texture_head)

        return texture_head


from ..adapters.mapping import MapAdapter
from ..client import from_tree, record_history

tree = MapAdapter({})
client = from_tree(tree)

def test_history():
    "Very basic exercise of history"
    with record_history() as history:
        repr(client)  # trigger a request
    assert history.requests
    assert history.responses


from ftw.dictstorage.interfaces import IDictStorage
from ftw.solr.interfaces import ISolrSearch
from ftw.solr.query import escape
from ftw.tabbedview.interfaces import IGridStateStorageKeyGenerator
from opengever.base import _
from opengever.base.behaviors.utils import set_attachment_content_disposition
from opengever.base.reporter import DATE_NUMBER_FORMAT
from opengever.base.solr import batched_solr_results
from opengever.base.solr import OGSolrDocument
from opengever.base.solr.fields import DateListingField
from plone import api
from plone.app.contentlisting.interfaces import IContentListingObject
from Products.Five.browser import BrowserView
from zExceptions import BadRequest
from zope.component import getUtility
from zope.component import queryMultiAdapter
import json

class BaseReporterView(BrowserView):
    """Base class for XLS Reporting views, which provide the functionality
    to use the users grid settings (visible columns, column order).
    """

    @property
    def _columns(self):
        """Returns a list of dicts of all available excel columns.
        """
        raise NotImplementedError

    def columns(self):
        return self.filter_and_order_by_tabbedview_settings(self._columns)

    def filter_and_order_by_tabbedview_settings(self, excel_columns):
        active_columns = []
        tabbedview_columns = self.get_active_tabbedview_columns()
        if not tabbedview_columns:
            return excel_columns

        for col in tabbedview_columns:
            column = self.excel_column_by_tabbedview_name(excel_columns, col.get('id'))
            if column:
                active_columns.append(column)

        return active_columns

    def excel_column_by_tabbedview_name(self, excel_columns, tv_col_name):
        """Get the Excel column matching the given TabbedView column name.

        For Catalog based Excel exports, the Excel column's sort_index is
        identical to the TabbedView column name.
        """
        excel_columns_by_name = {
            col.get('sort_index', col.get('id')): col
            for col in excel_columns
        }
        return excel_columns_by_name.get(tv_col_name)

    def get_grid_state(self, view_name):
        """Load tabbedview gridstate of the logged in users for the given view.
        """
        tabbedview = self.context.restrictedTraverse(
            "@@tabbedview_view-{}".format(view_name), default=None)
        if not tabbedview:
            return None

        generator = queryMultiAdapter((self.context, tabbedview, self.request),
                                      IGridStateStorageKeyGenerator)
        key = generator.get_key()
        storage = IDictStorage(tabbedview)
        state = storage.get(key)

        if not state:
            return {}

        return json.loads(state)

    def get_active_tabbedview_columns(self):
        """Loads corresponding tabbedview grid-state and returns an orderd
        list of the current visible columns.
        """
        view_name = self.request.form.get('view_name', None)
        grid_state = self.get_grid_state(view_name)
        if grid_state:
            return [col for col in
                    grid_state.get('columns') if not col.get('hidden')]

    def return_excel(self, reporter):
        data = reporter()
        if not data:
            msg = _(u'Could not generate the report.')
            api.portal.show_message(msg, request=self.request, type='error')
            return self.request.RESPONSE.redirect(self.context.absolute_url())

        response = self.request.RESPONSE
        response.setHeader(
            'Content-Type',
            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
        set_attachment_content_disposition(self.request, self.filename)

        return data

class SolrReporterView(BaseReporterView):

    batch_size = 1000
    field_mapper = None

    def __init__(self, *args, **kwargs):
        super(SolrReporterView, self).__init__(*args, **kwargs)
        self.solr = getUtility(ISolrSearch)
        self.fields = self.field_mapper(self.solr)

    def get_selected_items(self):
        paths = self.request.get('paths')
        listing_name = self.request.get('listing_name')
        if not paths and not listing_name:
            return

        if paths and listing_name:
            raise BadRequest('You can query either by paths or by a listing.')

        fields = [col['id'] for col in self.columns()]

        solr_query = {}
        solr_query['rows'] = self.batch_size
        solr_query['fl'] = self.fields.get_query_fields(fields) + ['path']

        if paths:
            self._extend_selected_items_query_by_paths(solr_query, paths)
        elif listing_name:
            self._extend_selected_items_query_by_listing(solr_query, listing_name)

        for batch in batched_solr_results(**solr_query):
            for doc in batch:
                doc = OGSolrDocument(doc, fields=self.solr.manager.schema.fields)
                yield IContentListingObject(doc)

    def _extend_selected_items_query_by_paths(self, query, paths):
        # Sort results according to paths passed in request. Those should be
        # sorted exactly as the user saw them in the UI (classic and geverui).
        query['sort'] = 'score asc'
        query['query'] = 'path:({})'.format(
            'OR '.join(['{}^{}'.format(escape(path), score_value)
                       for score_value, path in enumerate(paths)])
        )

    def _extend_selected_items_query_by_listing(self, solr_query, listing_name):
        listing = queryMultiAdapter((self.context, self.request), name="GET_application_json_@listing")
        listing.listing_name = listing_name
        query, filters, start, rows, sort, field_list, params = listing.prepare_solr_query(self.request.form)

        solr_query['sort'] = sort
        solr_query['query'] = query
        solr_query['filters'] = filters

    @property
    def is_frontend_request(self):
        return 'columns' in self.request

    def columns(self):
        if self.is_frontend_request:
            return self.filter_and_order_by_request_columns(self._columns)
        else:
            return self.filter_and_order_by_tabbedview_settings(self._columns)

    def filter_and_order_by_request_columns(self, excel_columns):
        requested_columns = self.get_requested_column_names()
        return sorted(
            excel_columns,
            key=lambda col: requested_columns.index(col['id'])
        )

    @property
    def _columns(self):
        requested_cols = self.get_requested_column_names()

        columns = []
        for requested_col in requested_cols:
            # Dynamically create a column definition based on field mapper
            field = self.fields.get(requested_col, only_allowed=True)
            if not field:
                continue

            column = {
                'id': field.field_name,
                'accessor': field.accessor,
                'sort_index': field.sort_index,
                'title': field.get_title(),
            }

            if isinstance(field, DateListingField):
                column['number_format'] = DATE_NUMBER_FORMAT

            # Update it with overrides from explicit column settings
            column_settings = self.get_column_settings(requested_col)
            column.update(column_settings)

            columns.append(column)

        return columns

    def get_requested_column_names(self):
        column_names = self.request.get('columns', self.get_default_column_names())
        return map(self.canonical_name, column_names)

    def get_default_column_names(self):
        return [c['id'] for c in self.column_settings if c.get('is_default')]

    @property
    def column_settings_by_id(self):
        return {c['id']: c for c in self.column_settings}

    @property
    def column_aliases(self):
        return dict(
            zip(
                [c.get('alias', c['id']) for c in self.column_settings],
                [c['id'] for c in self.column_settings]
            )
        )

    def canonical_name(self, column_name):
        return self.column_aliases.get(column_name, column_name)

    def get_column_settings(self, column_name):
        return self.column_settings_by_id.get(column_name, {})

    def excel_column_by_tabbedview_name(self, excel_columns, tv_col_name):
        """Get the Excel column matching the given TabbedView column name.

        For Solr based Excel exports, the corresponding sort_index in Solr
        usually matches the TabbedView column name. Exceptions from this rule
        can be controlled using 'tabbedview_column' in the column settings.
        """
        excel_columns_by_id = {col['id']: col for col in excel_columns}
        tabbedview_columns = {
            c.get('tabbedview_column', c['sort_index']): c['id']
            for c in self._columns
        }
        col_id = tabbedview_columns.get(tv_col_name, tv_col_name)
        return excel_columns_by_id.get(col_id)


import time
import os

from django.test import TestCase

from ..lockfile import FileLock, FileLocked

class LockTest(TestCase):

    def test_process_killed_force_unlock(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write('9999999')
        assert os.path.exists(lockfile)
        with FileLock('test.txt'):
            assert True

    def test_force_unlock_in_same_process(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        with FileLock('test.txt', force=True):
            assert True

    def test_exception_after_timeout(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        try:
            with FileLock('test.txt', timeout=1):
                assert False
        except FileLocked:
            assert True

    def test_force_after_timeout(self):
        lockfile = 'test.txt.lock'
        with open(lockfile, 'w+') as f:
            f.write(str(os.getpid()))

        timeout = 1
        start = time.time()
        with FileLock('test.txt', timeout=timeout, force=True):
            assert True
        end = time.time()
        assert end - start > timeout


import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import utils
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.utils import shuffle

def build_train_data(data):
    # 处理数据
    train_data=data.groupby(by=['buyer_admin_id','item_id']).size()
    train_data.name='item_counts'
    train_data=train_data.reset_index()

    temp=data.groupby(by=['buyer_admin_id']).size()
    temp.name='counts'
    temp=temp.reset_index()

    train_data=train_data.join(temp.set_index('buyer_admin_id'),on='buyer_admin_id')
    train_data['item_ratio']=train_data['item_counts']/train_data['counts']

    # 对id编码
    le_buyer = preprocessing.LabelEncoder()
    le_buyer.fit(data.buyer_admin_id.unique())
    le_item = preprocessing.LabelEncoder()
    le_item.fit(data.item_id.unique())
    train_data['buyerid_le']=le_buyer.transform(train_data.buyer_admin_id.values)
    train_data['itemid_le']=le_item.transform(train_data.item_id.values)
    return train_data, le_buyer, le_item

def batch_generator(data,batch_size,is_shuffle=True):
    '''批数据生成器'''
    if is_shuffle:
        data = shuffle(data)
    batch_count=0
    check=1
    while check:
        start = batch_count * batch_size
        end = start + batch_size
        if end > len(data):
            yield data.iloc[start:]
            check=0
        else:
            yield data.iloc[start:end]
            batch_count += 1

def mse(y, y_pred):
    """Returns the mean squared error between
    ground truths and predictions.
    """
    return np.mean((y - y_pred) ** 2)

def rmse(y, y_pred):
    """Returns the root mean squared error between
    ground truths and predictions.
    """
    return np.sqrt(mse(y, y_pred))

def mae(y, y_pred):
    """Returns mean absolute error between
    ground truths and predictions.
    """
    return np.mean(np.fabs(y - y_pred))

# SVD模型
class SVD():
    """Collaborative filtering model based on SVD algorithm.
    """

    def __init__(self, num_users, num_items, num_factors, sess):
        self.num_users = num_users
        self.num_items = num_items
        self.num_factors = num_factors
        self._sess = sess

        self.reg_b_u = 0.0001
        self.reg_b_i = 0.0001
        self.reg_p_u = 0.005
        self.reg_q_i = 0.005
        self.reg_y_u = 0.005
        self.reg_g_i = 0.005
        self._built=False

    def _create_placeholders(self):
        """Returns the placeholders.
        """
        with tf.variable_scope('placeholder'):
            users = tf.placeholder(tf.int32, shape=[None, ], name='users')
            items = tf.placeholder(tf.int32, shape=[None, ], name='items')
            ratings = tf.placeholder(tf.float32, shape=[None, ], name='ratings')
        return users, items, ratings

    def _create_constants(self, mu):
        """Returns the constants.
        """
        with tf.variable_scope('constant'):
            _mu = tf.constant(mu, shape=[], dtype=tf.float32)
        return _mu

    def _create_user_terms(self, users):
        """Returns the tensors related to users.
        """
        num_users = self.num_users
        num_factors = self.num_factors

        with tf.variable_scope('user', reuse=tf.AUTO_REUSE):
            self.user_embeddings = tf.get_variable(
                name='embedding',
                shape=[num_users, num_factors],
                initializer=tf.contrib.layers.xavier_initializer(),
                regularizer=tf.contrib.layers.l2_regularizer(self.reg_p_u))

            self.user_bias = tf.get_variable(
                name='bias',
                shape=[num_users, ],
                initializer=tf.contrib.layers.xavier_initializer(),
                regularizer=tf.contrib.layers.l2_regularizer(self.reg_b_u))

            p_u = tf.nn.embedding_lookup(self.user_embeddings, users, name='p_u')
            b_u = tf.nn.embedding_lookup(self.user_bias, users, name='b_u')
        return p_u, b_u

    def _create_item_terms(self, items):
        """Returns the tensors related to items.
        """
        num_items = self.num_items
        num_factors = self.num_factors

        with tf.variable_scope('item', reuse=tf.AUTO_REUSE):
            self.item_embeddings = tf.get_variable(
                name='embedding',
                shape=[num_items, num_factors],
                initializer=tf.contrib.layers.xavier_initializer(),
                regularizer=tf.contrib.layers.l2_regularizer(self.reg_q_i))

            self.item_bias = tf.get_variable(
                name='bias',
                shape=[num_items, ],
                initializer=tf.contrib.layers.xavier_initializer(),
                regularizer=tf.contrib.layers.l2_regularizer(self.reg_b_i))

            q_i = tf.nn.embedding_lookup(self.item_embeddings, items, name='q_i')
            b_i = tf.nn.embedding_lookup(self.item_bias, items, name='b_i')
        return q_i, b_i

    def _get_item_terms(self):
        """Returns the tensors related to items.
        """
        import pickle
        item_embeddings = self.item_embeddings.eval()
        with open('./model/item_embeddings.pkl', 'wb') as fw:
            pickle.dump({'embeddings': item_embeddings}, fw)

    def _create_prediction(self, mu, b_u, b_i, p_u, q_i):
        """Returns the tensor of prediction.

           Note that the prediction
            r_hat = \mu + b_u + b_i + p_u * q_i
        """
        with tf.variable_scope('prediction'):
            pred = tf.reduce_sum(tf.multiply(p_u, q_i),axis=1)
            pred = tf.add_n([b_u, b_i, pred])
            pred = tf.add(pred, mu, name='pred')

        return pred

    def _create_loss(self, pred, ratings):
        """Returns the L2 loss of the difference between
            ground truths and predictions.

           The formula is here:
            L2 = sum((r - r_hat) ** 2) / 2
        """
        with tf.variable_scope('loss'):
            loss = tf.nn.l2_loss(tf.subtract(ratings, pred), name='loss')

        return loss

    def _create_optimizer(self, loss):
        """Returns the optimizer.

           The objective function is defined as the sum of
            loss and regularizers' losses.
        """
        with tf.variable_scope('optimizer', reuse=tf.AUTO_REUSE):
            objective = tf.add(
                loss,
                tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)),
                name='objective'
            )

            try:
                optimizer = tf.contrib.keras.optimizers.Nadam().minimize(objective, name='optimizer')
            except:
                optimizer = tf.train.AdamOptimizer().minimize(objective, name='optimizer')

        return optimizer

    def _build_graph(self, mu):
        _mu = self._create_constants(mu)
        self._users, self._items, self._ratings = self._create_placeholders()
        p_u, b_u = self._create_user_terms(self._users)
        q_i, b_i = self._create_item_terms(self._items)
        self._pred = self._create_prediction(_mu, b_u, b_i, p_u, q_i)
        loss = self._create_loss(self._ratings, self._pred)
        self._optimizer = self._create_optimizer(loss)
        self._built = True

    def _run_train(self, data, epochs, batch_size, validation_data):

        steps_per_epoch = np.ceil(len(data) / batch_size).astype(int)

        self._sess.run(tf.global_variables_initializer())
        for e in range(1, epochs + 1):
            print('Epoch {}/{}'.format(e, epochs))
            # 批数据生成器，默认打乱
            data_gen = batch_generator(data, batch_size)
            pbar = utils.Progbar(steps_per_epoch,stateful_metrics=['rmse','mae','val_rmse','val_mae'])
            step = 0
            for batch_data in data_gen:
                step += 1
                users = batch_data['buyerid_le'].values
                items = batch_data['itemid_le'].values
                ratings = batch_data['item_ratio'].values

                self._sess.run(
                    self._optimizer,
                    feed_dict={
                        self._users: users,
                        self._items: items,
                        self._ratings: ratings
                    })

                pred = self.predict(batch_data[['buyerid_le','itemid_le']])

                update_values = [
                    ('rmse', rmse(ratings, pred)),
                    ('mae', mae(ratings, pred))
                ]
                # 进度条更新
                pbar.update(step, values=update_values)

            if validation_data is not None:
                valid_x = validation_data[['buyerid_le','itemid_le']]
                valid_y = validation_data['item_ratio']
                valid_pred = self.predict(valid_x)

                update_values += [
                    ('val_rmse', rmse(valid_y, valid_pred)),
                    ('val_mae', mae(valid_y, valid_pred))
                ]
                # 进度条更新
                pbar.update(step, values=update_values)

    def train(self, data, epochs=10, batch_size=1024, validation_data=None):
        if not self._built:
            self._build_graph(np.mean(data.item_ratio.values))
        self._run_train(data, epochs, batch_size, validation_data)
        # self._get_item_terms()

    def predict(self, x):
        if not self._built:
            return 'The model must be trained before prediction.'

        if x.shape[1] != 2:
            raise ValueError('The shape of x should be (samples, 2)')

        pred = self._sess.run(
            self._pred,
            feed_dict={
                self._users: x['buyerid_le'].values,
                self._items: x['itemid_le'].values
            })

        return pred

    def save_model(self,model_dir):
        self._saver = tf.train.Saver()
        model_name = type(self).__name__
        if not os.path.exists(model_dir):
            os.makedirs(model_dir)
        model_path = os.path.join(model_dir, model_name)
        print(model_dir)
        print(model_name)
        self._saver.save(self._sess, model_path)

    def load_model(self,model_dir):
        ckpt = tf.train.get_checkpoint_state(model_dir)
        self._saver = tf.train.import_meta_graph(ckpt.model_checkpoint_path +'.meta')  # 加载图结构
        gragh = tf.get_default_graph()
        tensor_name_list = [tensor.name for tensor in gragh.as_graph_def().node]
        self._users=gragh.get_tensor_by_name('placeholder/users:0')
        self._items=gragh.get_tensor_by_name('placeholder/items:0')
        self._pred=gragh.get_tensor_by_name('prediction/pred:0')
        self._saver.restore(self._sess, tf.train.latest_checkpoint(model_dir))
        self._built=True

def knn_recommend(pkl_path, le_item):
    import pickle
    from sklearn.neighbors import NearestNeighbors

    with open(pkl_path,'rb') as f:
        item_emb = pickle.load(f)
    item_emb=item_emb['embeddings']
    nbrs = NearestNeighbors(n_neighbors=31, algorithm='ball_tree').fit(item_emb)
    print('KNN fit')

    # 读取测试集
    test=pd.read_csv('./Antai_AE_round1_test_20190626.csv')
    test_irank_min = test.groupby(['buyer_admin_id']).apply(lambda t: t[t.irank == t.irank.min()]).reset_index(
        drop=True)
    test_irank_max = test.groupby(['buyer_admin_id']).apply(lambda t: t[t.irank == t.irank.max()]).reset_index(
        drop=True)
    buyer_item = test_irank_min.append(test_irank_max)
    buyer_item = buyer_item[['buyer_admin_id','item_id']].drop_duplicates().reset_index(drop=True)
    buyer_item['item_le'] = le_item.transform(buyer_item.item_id.values)
    # embedding
    test_item_emb = []
    for idx in range(len(buyer_item)):
        item_le = buyer_item.iloc[idx]['item_le']
        test_item_emb.append(item_emb[item_le])
    print('embedding finish')

    distances, indices = nbrs.kneighbors(test_item_emb)
    with open('./model/kneighbors_distances.pkl', 'wb') as f:
        pickle.dump(distances, f)
    with open('./model/kneighbors_indices.pkl', 'wb') as f:
        pickle.dump(indices, f)
    print('kneighbors finish')

    indices_inversed = le_item.inverse_transform(indices)
    for i in range(indices_inversed.shape[1]):
        buyer_item['recomend_' + str(i)] = indices_inversed[:, i]
    buyer_item.to_csv('./Data/knn_rec.csv', index=False)
    print('finish')

if __name__=='__main__':
    datayy = pd.read_csv('./Data/round1_data.csv')
    datayy = datayy.loc[datayy.buyer_country_id == 'yy']

    train_data, le_buyer, le_item = build_train_data(datayy)
    num_users = train_data.buyer_admin_id.nunique()
    num_items = train_data.item_id.nunique()
    num_factors = 100

    # ---------------------------KNN----推荐-------------------------------------------
    # del(datayy)
    # del(train_data)
    # knn_recommend('./model/item_embeddings_100.pkl', le_item)
    #----------------------------训-------练-------------------------------------------
    # 随机取样 90%训练 10%测试
    train_data = train_data[['buyerid_le', 'itemid_le', 'item_ratio']]
    traindf,validdf=train_test_split(train_data, test_size = 0.1)

    with tf.Session() as sess:
        model=SVD(num_users,num_items,num_factors,sess)
        model.train(traindf,epochs=5,batch_size=1024,validation_data=validdf)
        # model.load_model('./model/')

        y_pred = model.predict(validdf[['buyerid_le', 'itemid_le']])
        print('rmse: {}, mae: {}'.format(rmse(validdf['item_ratio'], y_pred), mae(validdf['item_ratio'], y_pred)))

        # Save model
        # model.save_model('./model/')


import os
import pickle
import shutil
import time
from collections import deque
from os import path

import numpy as np
import DDPG_baseline_v2.baselines.common.tf_util as U
import tensorflow as tf
from mpi4py import MPI
from DDPG_baseline_v2.baselines import logger
from DDPG_baseline_v2.baselines.ddpg.ddpg import DDPG
from DDPG_baseline_v2.baselines.ddpg.memory import load_from_cedric, load_from_geppg

from DDPG_baseline_v2.baselines.ddpg.util import mpi_mean, mpi_std, mpi_sum

def run_agent(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
    normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise,
    popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory, study,
    buffer_location, trial_id, data_path, nb_eval_episodes,  tau=0.01, eval_env=None, param_noise_adaption_interval=50,
    gep_memory=None, **kwargs):
    rank = MPI.COMM_WORLD.Get_rank()

    assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    max_action = env.action_space.high
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.shape,
                 gamma=gamma, tau=tau, normalize_returns=normalize_returns, normalize_observations=normalize_observations,
                 batch_size=batch_size, action_noise=action_noise, param_noise=param_noise, critic_l2_reg=critic_l2_reg,
                 actor_lr=actor_lr, critic_lr=critic_lr, enable_popart=popart, clip_norm=clip_norm,
                 reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))

    # Set up logging stuff only for a single worker.
    if rank == 0:
        saver = tf.train.Saver()
    else:
        saver = None

    episode = 0
    best_score = -1000 # initialize to negative values to keep track of best score
    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    saver_best = tf.train.Saver(max_to_keep=1)

    with U.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        size_buffer = 0
        # load buffer if necessary
        if study == 'GEP_PG' or study == 'GEP_FPG':
            if study ==  'GEP_FPG':
                logger.info('This is a frozen buffer study')
            if gep_memory is not None:
                buffer_gep = load_from_geppg(gep_memory)
                del gep_memory
                logger.info('Load buffer from gep memory')
            else:
                assert buffer_location != '', 'study is ' + study + ', a buffer location should be provided.'
                # fill replay buffer
                buffer_gep = load_from_cedric(filename=buffer_location)
                logger.info('Load buffer from file')

            size_buffer = len(buffer_gep[0])
            for i in range(size_buffer):
                agent.store_transition(np.array(buffer_gep[0][i]), np.array(buffer_gep[1][i]),
                                       buffer_gep[2][i][0], np.array(buffer_gep[3][i]), buffer_gep[4][i])
            logger.info('Buffer of size ' + str(size_buffer) + ' has been loaded')

        agent.reset()
        obs = env.reset()
        if eval_env is not None:
            eval_obs = eval_env.reset()
        done = False
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        t = size_buffer # t statt at size buffer if a buffer is provided, 0 otherwise

        epoch = 0
        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        epoch_episode_eval_rewards = []
        epoch_episode_eval_steps = []
        epoch_start_time = time.time()
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0

        max_steps = nb_rollout_steps*nb_epoch_cycles*nb_epochs

        for epoch in range(nb_epochs):
            if t>=max_steps:
                break
            for cycle in range(nb_epoch_cycles):
                if t>=max_steps:
                    break
                # Perform rollouts.
                if study != 'GEP_FPG':
                    agent, env, episode_reward, episode_step, epoch_actions, epoch_qs, epoch_episode_rewards, episode_rewards_history, \
                    epoch_episode_steps, epoch_episodes, episodes, t = rollout(nb_rollout_steps, agent, obs, rank, env, max_action,
                                                                               episode_reward, episode_step, epoch_actions,
                                                                               epoch_qs,epoch_episode_rewards, episode_rewards_history,
                                                                               epoch_episode_steps, epoch_episodes, episodes, render, t, epoch
                                                                               )
                # Train.
                agent, epoch_actor_losses, epoch_critic_losses, epoch_adaptive_distances = train(nb_train_steps, memory,
                                                                                batch_size, t,
                                                                                param_noise_adaption_interval, agent
                                                                                )
            # Evaluate.
            agent, eval_episode_rewards, eval_qs, best_score, eval_obs, eval_env, \
            eval_episode_rewards_history = evaluate(agent, eval_obs, max_action, eval_env, render_eval, sess, t, eval_episode_rewards_history,
                                                     best_score, data_path, saver_best, nb_eval_episodes, study, max_steps, size_buffer
                                                    )

            # update decreasing_ou if necessary
            try:
                action_noise.adapt()
                print(action_noise.sigma)
            except:
                pass

            # Log stats.
            epoch_train_duration = time.time() - epoch_start_time
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = {}
            for key in sorted(stats.keys()):
                combined_stats[key] = mpi_mean(stats[key])

            # Added statistics

            # Rollout statistics.
            combined_stats['rollout/return'] = mpi_mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = mpi_mean(np.mean(episode_rewards_history))
            combined_stats['rollout/episode_steps'] = mpi_mean(epoch_episode_steps)
            combined_stats['rollout/episodes'] = mpi_sum(epoch_episodes)
            combined_stats['rollout/actions_mean'] = mpi_mean(epoch_actions)
            combined_stats['rollout/actions_std'] = mpi_std(epoch_actions)
            combined_stats['rollout/Q_mean'] = mpi_mean(epoch_qs)

            # Train statistics.
            try:
                combined_stats['train/loss_actor'] = mpi_mean(epoch_actor_losses)
                combined_stats['train/loss_critic'] = mpi_mean(epoch_critic_losses)
                combined_stats['train/param_noise_distance'] = mpi_mean(epoch_adaptive_distances)
            except:
                pass

            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = mpi_mean(eval_episode_rewards)
                combined_stats['eval/return_history'] = mpi_mean(np.mean(eval_episode_rewards_history))
                combined_stats['eval/Q'] = mpi_mean(eval_qs)
                combined_stats['eval/episodes'] = mpi_mean(len(eval_episode_rewards))

            # Total statistics.
            combined_stats['total/duration'] = mpi_mean(duration)
            combined_stats['total/steps_per_second'] = mpi_mean(float(t) / float(duration))
            combined_stats['total/episodes'] = mpi_mean(episodes)
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = t

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as f:
                        pickle.dump(env.get_state(), f)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as f:
                        pickle.dump(eval_env.get_state(), f)

def rollout(nb_rollout_steps, agent, obs, rank, env, max_action, episode_reward,
            episode_step, epoch_actions, epoch_qs,epoch_episode_rewards, episode_rewards_history,
            epoch_episode_steps, epoch_episodes, episodes, render, t, epoch):

    for t_rollout in range(nb_rollout_steps):
        # Predict next action.
        action, q = agent.pi(obs, apply_noise=True, compute_Q=True)
        assert action.shape == env.action_space.shape

        # Execute next action.
        if rank == 0 and render:
            env.render()
        assert max_action.shape == action.shape
        new_obs, r, done, info = env.step(max_action * action)  # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
        t += 1
        if rank == 0 and render:
            env.render()
        episode_reward += r
        episode_step += 1
        #print ("rollout episode_step : ", episode_step, " t:", t)

        # Book-keeping.
        epoch_actions.append(action)
        epoch_qs.append(q)
        agent.store_transition(obs, action, r, new_obs, done)
        obs = new_obs

        if done:
            # Episode done.
            epoch_episode_rewards.append(episode_reward)
            episode_rewards_history.append(episode_reward)
            epoch_episode_steps.append(episode_step)
            episode_reward = 0.
            episode_step = 0
            epoch_episodes += 1
            episodes += 1

            agent.reset()
            obs = env.reset()

    return agent, env, episode_reward, episode_step, epoch_actions,epoch_qs,epoch_episode_rewards, episode_rewards_history, \
            epoch_episode_steps, epoch_episodes, episodes, t

def train(nb_train_steps, memory, batch_size, t, param_noise_adaption_interval, agent):

    epoch_actor_losses = []
    epoch_critic_losses = []
    epoch_adaptive_distances = []
    for t_train in range(nb_train_steps):
        # Adapt param noise, if necessary.
        if memory.nb_entries >= batch_size and t % param_noise_adaption_interval == 0:
            distance = agent.adapt_param_noise()
            epoch_adaptive_distances.append(distance)

        cl, al = agent.train()
        epoch_critic_losses.append(cl)
        epoch_actor_losses.append(al)
        agent.update_target_net()
    return agent, epoch_actor_losses, epoch_critic_losses, epoch_adaptive_distances

def evaluate(agent, eval_obs, max_action, eval_env, render_eval, sess, t, eval_episode_rewards_history,
              best_score, data_path, saver_best, nb_eval_episodes, study, max_steps, size_buffer):

    eval_episode_rewards = []
    eval_qs = []
    if eval_env is not None:
        eval_episode_reward = 0.
        for i in range(nb_eval_episodes): # 5 rollouts
            while True:
                eval_action, eval_q = agent.pi(eval_obs, apply_noise=False, compute_Q=True)
                eval_obs, eval_r, eval_done, eval_info = eval_env.step(max_action * eval_action)  # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                if render_eval:
                    eval_env.render()
                eval_episode_reward += eval_r
                eval_qs.append(eval_q)
               # print("evaluate eval_done : ", eval_done, " eval_episode_reward:", eval_episode_reward)
                if eval_done:
                    eval_obs = eval_env.reset()
                    eval_episode_rewards.append(eval_episode_reward)
                    eval_episode_rewards_history.append(eval_episode_reward)
                    eval_episode_reward = 0.
                    break
        if mpi_mean(eval_episode_rewards) > best_score:
            best_score = mpi_mean(eval_episode_rewards)
            if not os.path.exists(data_path+'tf_save/'):
                os.mkdir(data_path+'tf_save/')
                os.mkdir(data_path+'tf_save/'+'best1_5M/') # optional, to save GEP data at 1.5M
            for file in os.listdir(data_path+'tf_save/'):
                if path.isfile(data_path+'tf_save/'+file): # optional, to save GEP data at 1.5M
                    os.remove(data_path+'tf_save/'+file)
            saver_best.save(sess, data_path+'tf_save/'+ 'best_actor_step'+str(t)+'_score'+str(int(best_score)))

    return agent, eval_episode_rewards, eval_qs, best_score, eval_obs, eval_env, \
            eval_episode_rewards_history


#!/usr/bin/env python3

import argparse
import logging
import json
from google.cloud import translate
logger = logging.getLogger(__name__)

def translate_text(source="en-US", target="fr", text="Example Application", project_id="rosy-griffin-312113"):
    """Translating Text."""

    client = translate.TranslationServiceClient()

    location = "global"

    parent = f"projects/{project_id}/locations/{location}"

    # Detail on supported types can be found here:
    # https://cloud.google.com/translate/docs/supported-formats
    response = client.translate_text(
        request={
            "parent": parent,
            "contents": [text],
            "mime_type": "text/plain",  # mime types: text/plain, text/html
            "source_language_code": f"{source}",
            "target_language_code": f"{target}",
        }
    )

    return response.translations

def find(dict_var, key):
    for k, v in dict_var.items():
        if k == key:
            if len(v) > 0:
                dict_var[k] = "XXX"
            yield v
        elif isinstance(v, dict):
            for id_val in find(v, key):
                yield id_val

def replace_item(obj, key, replace_value):
    for k, v in obj.items():
        if isinstance(v, dict):
            obj[k] = replace_item(v, key, replace_value)
    if key in obj:
        if len(obj[key]) >0:
            try:
                translation = translate_text(text=obj[key], target=ARGS.target)
                obj[key] = translation[0].translated_text

            except Exception as e:
                logger.warning(e)

    return obj

def main():
    schema = json.load(ARGS.file)
    replace_item(schema,"description", "XXX")
    replace_item(schema,"title", "XXX")
    print(json.dumps(schema,indent=4))

if __name__ == "__main__":

    logging.basicConfig(level=logging.WARNING)

    parser = argparse.ArgumentParser(
        description="Translate the titles and/or descriptions in a JSON-SCHEMA file"
    )
    parser.add_argument(
        "-v", "--verbose", help="print verbose output to console", action="store_true"
    )
    parser.add_argument(
        "-d", "--debug", help="turn on debug for asyncio", action="store_true"
    )

    parser.add_argument(
        "-s", "--source", help="source language", default="en-US"
    )
    parser.add_argument(
        "-t", "--target", help="target language", default="fr"
    )

    parser.add_argument(
        "-f", "--file", help="JSON-SCHEMA file to translate", type=argparse.FileType("r"),
        required=True,
    )

    ARGS = parser.parse_args()

    if ARGS.verbose:
        logging.getLogger().setLevel(logging.INFO)

    logger.info("Start")
    # translations = translate_text(source=ARGS.source, target=ARGS.target)
    # # Display the translation for each input text provided
    # for translation in translations:
    #     print("Translated text: {}".format(translation.translated_text))
    main()
    logger.info("Done")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('songs', '0009_remove_song_published'),
    ]

    operations = [
        migrations.CreateModel(
            name='Annotation',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
                ('reviewed', models.BooleanField(editable=False, default=False)),
                ('pub_date', models.DateTimeField(editable=False)),
                ('title', models.CharField(help_text='Tytuł adnotacji, np. "Aspazja", "Francuska choroba", "Rok 1788". Adnotacje\npoświęcone dziełom artystycznym tytułuj wg schematu "Nazwa - Autor", np.\n"Fortepian Chopina - Cyprian Kamil Norwid" lub "Wojna postu z karnawałem - Piotr\nBreugel.', max_length=100)),
                ('text_trevor', models.TextField()),
                ('image', models.ImageField(help_text='Ilustracja adnotacji. Pamiętaj o wskazaniu źródła tak samo jak w przypadku\ntreści zawartych w tekście adnotacji.', blank=True, upload_to='song_annotations', null=True)),
                ('source_url1', models.URLField(help_text='Link do źródła informacji, jeśli źródłem jest strona internetowa.', blank=True, null=True)),
                ('source_url2', models.URLField(help_text='Link do źródła informacji, jeśli źródłem jest strona internetowa.', blank=True, null=True)),
                ('source_ref1', models.TextField(help_text='Nazwa publikacji źródłowej, jeśli adnotacja jest oparta na publikacjii.', blank=True, null=True)),
                ('source_ref2', models.TextField(help_text='Nazwa publikacji źródłowej, jeśli adnotacja jest oparta na publikacjii.', blank=True, null=True)),
                ('slug', models.SlugField(editable=False, unique=True, max_length=200, help_text='Used in urls, has to be unique.')),
                ('text_html', models.TextField(editable=False)),
                ('author', models.ForeignKey(editable=False, to=settings.AUTH_USER_MODEL)),
                ('song', models.ForeignKey(editable=False, to='songs.Song')),
            ],
            options={
                'abstract': False,
            },
        ),
    ]


import sys
sys.path.insert(0, 'loading/')
from load_movies import get_all_actors
import pickle
import json

ACTOR_COLOUR = "yellow"
DIRECTOR_COLOUR = "green"
GENRE_COLOUR = "blue"
TITLE_COLOUR = "red"

ids = ['tt0289992','tt0338013','tt0308644','tt0162222','tt0362227','tt0203009','tt0373926','tt0264464','tt0315733','tt0327056']
paper_ids = ['tt0264464', 'tt0203009']

def paper_minimal():
	data = pickle.load(open("db/movie_metadata.pkl","r"))

	films = dict()
	for uid in ids:
		films[uid] = data[uid]

	afile = open('paper_films.pkl', 'wb')
	pickle.dump(films, afile)
	afile.close()

def paper_minimal_json():
	data = pickle.load(open("db/paper_films.pkl", "r"))
	ids = []
	graph = dict()
	graph["nodes"] = []
	graph["links"] = []

	for k, v in data.iteritems():
		if k in paper_ids:
			movie_root = v["title"] + "-" + k
			root_node = dict()
			root_node["id"] = movie_root
			root_node["colour"] = TITLE_COLOUR
			graph["nodes"].append(root_node)
			ids.append(movie_root)

			if v["director"] not in ids:
				ids.append(v["director"])
				director_node = dict()
				director_node["id"] = v["director"]
				director_node["colour"] = DIRECTOR_COLOUR
				graph["nodes"].append(director_node)
			director_link = dict()
			director_link["source"] = movie_root
			director_link["target"] = v["director"]
			graph["links"].append(director_link)

			for a in v["actors"]:
				if a not in ids:
					ids.append(a)
					actor_node = dict()
					actor_node["id"] = a
					actor_node["colour"] = ACTOR_COLOUR
					graph["nodes"].append(actor_node)
				actor_link = dict()
				actor_link["source"] = movie_root
				actor_link["target"] = a
				graph["links"].append(actor_link)

			for g in v["genre"]:
				if g not in ids:
					ids.append(g)
					genre_node = dict()
					genre_node["id"] = g
					genre_node["colour"] = GENRE_COLOUR
					graph["nodes"].append(genre_node)
				genre_link = dict()
				genre_link["source"] = movie_root
				genre_link["target"] = g
				graph["links"].append(genre_link)

	with open("paper_graph.json", "w") as outfile:
		json.dump(graph, outfile)

def get_popular_actors(actor_data, actors):
	popular_actors = actors
	for x in actor_data:
		for actor in x["stars"]:
			if actor not in popular_actors:
				popular_actors.append(actor)
	return popular_actors

def create_paper_metadata(movie_data, actor_data, popular_actors):
	movie_ratings = dict()

	for movie in movie_data:
		assert len(movie["ratings"]) <= 500
		movie_ratings[movie["imdbID"]] = movie["ratings"]

	afile = open("movie_ratings_500_id.pkl", "wb")
	pickle.dump(movie_ratings, afile)
	afile.close()

	movie_metadata = dict()
	for ad in actor_data:
		movie_metadata[ad["imdbID"]] = dict()
		movie_metadata[ad["imdbID"]]["title"] = ad["title"]
		movie_metadata[ad["imdbID"]]["director"] = ad["director"]
		movie_metadata[ad["imdbID"]]["genre"] = ad["genre"]

		actors = []
		for actor in ad["stars"]:
			actors.append(actor)

		for actor in ad["cast"]:
			if actor not in actors and actor in popular_actors:
				actors.append(actor)
		movie_metadata[ad["imdbID"]]["actors"] = actors

	afile = open("movie_metadata.pkl", "wb")
	pickle.dump(movie_metadata, afile)
	afile.close()

def paper_db():
	actors = get_all_actors()
	movie_data = pickle.load(open("db/movies_db_500.pkl","r"))
	actor_data = pickle.load(open("db/imdb_movies_actors.pkl","r"))
	assert len(movie_data) == len(actor_data)

	actors = get_popular_actors(actor_data, actors)
	create_paper_metadata(movie_data, actor_data, actors)

if __name__ == "__main__":
	# paper_db()
	paper_minimal_json()


import  requests
from bs4 import BeautifulSoup
import pandas as pd
from datetime import date

def parseData():
    ## Initial data
    url = 'https://www.curs.md/ro/curs_valutar_banci'
    response = requests.get(url)
    parse_html = BeautifulSoup(response.text, 'html.parser')
    ##

    ## --- > Bank name
    clas_bank_name = parse_html.find(id="tabelBankValute").select('a')

    bank_list = []
    for elem in clas_bank_name:
        s2 = elem.get_text()
        bank_list.append(s2)

    bank_list_new = [x.replace('\n', '') for x in bank_list]
    bank_list_new = [x.replace(' ', '') for x in bank_list_new]
    bank_list_name =[]
    count = 0
    for number in bank_list_new:
        count+=1
        bank_list_name.append(number)
        #print(str(count)+ str(' ') +number)

    #USD Vinzare / Cumparare

    data_USD = parse_html.body.findChildren(class_='column-USD column-ind-0')
    set_of_data_USD = []
    for resul in data_USD:

        usd_text = resul.get_text()
        set_of_data_USD.append(usd_text)
    del set_of_data_USD[0]

    usd_cumparare = set_of_data_USD[::2]
    usd_vinzare = []

    for i in range(len(usd_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_USD[i] == usd_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_USD[i] #добавляем в пустой список элемент из первого, если он меньше
            usd_vinzare.append(set_of_data_USD[i])

    del usd_cumparare[0]
    del usd_vinzare[0]

    ### end

    # Eur vinzare / cumparare

    data_EUR = parse_html.body.findChildren(class_='column-EUR column-ind-1')
    set_of_data_EUR = []
    for resul in data_EUR:

        eur_text = resul.get_text()
        set_of_data_EUR.append(eur_text)
    del set_of_data_EUR[0]

    eur_cumparare = set_of_data_EUR[::2]
    eur_vinzare = []

    for i in range(len(eur_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_EUR[i] == eur_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_EUR[i] #добавляем в пустой список элемент из первого, если он меньше
            eur_vinzare.append(set_of_data_EUR[i])

    del eur_cumparare[0]
    del eur_vinzare[0]

    ## End

    #Rub vinzare/cumparare

    data_RUB = parse_html.body.findChildren(class_='column-RUB column-ind-2')
    set_of_data_RUB = []
    for resul in data_RUB:

        rub_text = resul.get_text()
        set_of_data_RUB.append(rub_text)
    del set_of_data_RUB[0]

    rub_cumparare = set_of_data_RUB[::2]
    rub_vinzare = []

    for i in range(len(rub_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_RUB[i] == rub_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_RUB[i] #добавляем в пустой список элемент из первого, если он меньше
            rub_vinzare.append(set_of_data_RUB[i])

    del rub_cumparare[0]
    del rub_vinzare[0]

    ## end

    ## RON vinzare / cumparare

    #Rub vinzare/cumparare

    data_RON = parse_html.body.findChildren(class_='column-RON column-ind-3')
    set_of_data_RON = []
    for resul in data_RON:

        ron_text = resul.get_text()
        set_of_data_RON.append(ron_text)
    del set_of_data_RON[0]

    ron_cumparare = set_of_data_RON[::2]
    ron_vinzare = []
    date_now = []
    for i in range(len(ron_cumparare)): #перебираем элементы списка (в пределах длины первого списка)
        if set_of_data_RON[i] == ron_cumparare[i]: #сравниваем элементы первого списка с элементами второго списка
            del set_of_data_RON[i] #добавляем в пустой список элемент из первого, если он меньше
            ron_vinzare.append(set_of_data_RON[i])
            date_now.append(date.today())

    del ron_cumparare[0]
    del ron_vinzare[0]

    ## end
    desired_width=320

    pd.set_option('display.width', desired_width)

    pd.set_option('display.max_columns',10)

    table1 = (list(zip(bank_list_name,usd_cumparare,usd_vinzare,eur_cumparare,eur_vinzare, rub_cumparare, rub_vinzare,ron_cumparare,ron_vinzare,date_now)))

    df = pd.DataFrame(table1,columns=['Denumirea Bancii',
                                      'Cumparare USD',  'Vinzare USD',
                                      'Cumparare EUR',  'Vinzare EUR',
                                      'Cumparare RUB',  'Vinzare RUB',
                                      'Cumparare RON',  'Vinzare RON',
                                      'Date'])

    return df

vallet_data = parseData()


# This is a very simple fashion classifier
# It uses a data set called "fashion mnist", a set of small b&w images of apparel
#
# This training script classifies with around 87% accuracy currently.
#
# Can you get the validation accuracy (val_acc) above 90% with a CNN?
#
# The most common error looks something like:
# ValueError: Error when checking input: expected conv2d_input to have 4 dimensions,
#    but got array with shape (60000, 28, 28)
#
# This means you need to reshape your b&w image input from (28, 28) to (28, 28, 1)
# You can use numpy.reshape or the keras Reshape layer.
# There is example code in the examples/keras-cnn directory.
#
# Getting the CNN to work better than a multi-layer perceptron on this dataset
# may take some experimentation.

from tensorflow.keras.utils import to_categorical
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Reshape, Dropout,LeakyReLU,ELU
import tensorflow
import keras
import wandb
from wandb.keras import WandbCallback

def swish(x):
   beta = 1
   return beta * x * keras.backend.sigmoid(x)
# logging code
run = wandb.init(project="fashion")
config = run.config
config.first_layer_convs = 32
config.first_layer_conv_width = 3
config.first_layer_conv_height = 3
config.dropout = 0.2
config.dense_layer_size = 128
config.img_width = 28
config.img_height = 28
config.epochs = 10
config.activation = swish
config.activation_layers="swish"
#config.activation_layers = keras.layers.ELU(alpha=1.0)
#config.activation_layers = PReLU(alpha_initializer='zeros', weights=None)
#
# load data
(X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()
img_width = X_train.shape[1]
img_height = X_train.shape[2]
X_train = X_train.reshape(
    X_train.shape[0], 28,28, 1)
X_test = X_test.reshape(
    X_test.shape[0], 28, 28, 1)
# one hot encode outputs
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
labels = ["T-shirt/top", "Trouser", "Pullover", "Dress",
          "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]

X_train = X_train / 255.
X_test = X_test / 255.

num_classes = y_train.shape[1]

# create model
model = Sequential()
model.add(Conv2D(32,
                 (config.first_layer_conv_width, config.first_layer_conv_height),
                 input_shape=(28, 28, 1),
                 activation=config.activation))
#model.add(ELU(alpha=1.0))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64,
                 (3, 3),
                 activation=config.activation))
#model.add(ELU(alpha=1.0))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.3))
model.add(Flatten())
model.add(Dense(config.dense_layer_size, activation=config.activation))
#model.add(ELU(alpha=1.0))
model.add(Dropout(0.4))
model.add(Dense(num_classes, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam',
              metrics=['accuracy'])
model.summary()

# Fit the model
model.fit(X_train, y_train, epochs=50, validation_data=(X_test, y_test),
          callbacks=[WandbCallback(data_type="image", labels=labels)])