from statistics import mean, stdev
from sklearn import preprocessing
from sklearn.model_selection import StratifiedKFold
from sklearn import ensemble
from sklearn import tree
from sklearn.metrics import confusion_matrix
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.inspection import permutation_importance
import Source.Objects.Database as db
import pandas as pd
import Source.Objects.Config as Config

__SCRIPT_NAME="classification"

def cm_analysis(y_true, y_pred, labels, ymap=None):
    if ymap is not None:
        y_pred = [ymap[yi] for yi in y_pred]
        y_true = [ymap[yi] for yi in y_true]
        labels = [ymap[yi] for yi in labels]
    cm = confusion_matrix(y_true, y_pred, labels=labels)
    cm_sum = np.sum(cm, axis=1, keepdims=True)
    cm_perc = cm / cm_sum.astype(float) * 100
    annot = np.empty_like(cm).astype(str)
    nrows, ncols = cm.shape
    for i in range(nrows):
        for j in range(ncols):
            c = cm[i, j]
            p = cm_perc[i, j]
            if i == j:
                s = cm_sum[i]
                annot[i, j] = '%.1f%%\n%d/%d' % (p, c, s)
            elif c == 0:
                annot[i, j] = ''
            else:
                annot[i, j] = '%.1f%%\n%d' % (p, c)
    return cm

def inspect_data(df,target,features,outpath,appender):
    # Input_x_Features.
    x = df[features].values

    # Input_ y_Target_Variable.
    y = df[target].values
    unique_values = len(list(np.unique(y)))

    # Feature Scaling for input features.
    scaler = preprocessing.MinMaxScaler()
    x_scaled = scaler.fit_transform(x)

    # Create  classifier object.
    config = Config.load()
    if config.get("classification_classificator")=="Randomforest":
        lr = ensemble.RandomForestClassifier()
    elif config.get("classification_classificator")=="DecisionTree":
        lr = tree.DecisionTreeClassifier()
    elif config.get("classification_classificator")=="BaggedDecisionTree":
        lr = ensemble.BaggingClassifier()
    splits = 10
    for tar in df[target].unique():
        cur = df.loc[df[target]== tar]
        splits = min(splits,len(cur.index))
    # Create StratifiedKFold object.
    skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=1)
    lst_accu_stratified = []
    cm = np.array([])
    forest_importances = pd.Series()
    for train_index, test_index in skf.split(x, y):
        x_train_fold, x_test_fold = x_scaled[train_index], x_scaled[test_index]
        y_train_fold, y_test_fold = y[train_index], y[test_index]
        lr.fit(x_train_fold, y_train_fold)
        y_pred=lr.predict(x_test_fold)
        result_new = permutation_importance(
            lr, x_test_fold, y_test_fold, n_repeats=splits, random_state=42, n_jobs=2
        )
        forest_importances_new = pd.Series(result_new.importances_mean, index=features)
        forest_importances = forest_importances.add(forest_importances_new, fill_value=0)

        lst_accu_stratified.append(lr.score(x_test_fold, y_test_fold))
        cur = cm_analysis(y_test_fold, y_pred, lr.classes_, ymap=None)
        if cm.size == 0:
            cm = cur
        elif not cur.shape == (unique_values,unique_values):
            pass
        else:
            cm = cm + cur
    forest_importances = forest_importances.rename("Importances")
    a = pd.Series.to_frame(forest_importances)
    a['ID'] = list(a.index)
    a.to_excel(outpath+appender+"_Feature_importances.xlsx")
    labels = lr.classes_
    cm_sum = np.sum(cm, axis=1, keepdims=True)
    cm_perc = cm / cm_sum.astype(float) * 100
    annot = np.empty_like(cm).astype(str)
    nrows, ncols = cm.shape
    for i in range(nrows):
        for j in range(ncols):
            c = cm[i, j]
            p = cm_perc[i, j]
            if i == j:
                s = cm_sum[i]
                annot[i, j] = '%.1f%%' % (p)
            elif c == 0:
                annot[i, j] = ''
            else:
                annot[i, j] = '%.1f%%' % (p)
    annot2 = np.empty_like(cm)
    nrows, ncols = cm.shape
    for i in range(nrows):
        for j in range(ncols):
            p = cm_perc[i, j]
            if i == j:
                annot2[i, j] = float(p)
            elif c == 0:
                annot2[i, j] = float(0)
            else:
                annot2[i, j] = float(p)

    cm = pd.DataFrame(annot2, index=labels, columns=labels)
    cm.index.name = 'Actual'
    cm.columns.name = 'Predicted'

    fig, ax = plt.subplots(figsize=(10,10))
    size = 20
    res = sns.heatmap(cm, annot=annot, fmt='', ax=ax, cbar=False,cmap=matplotlib.colors.LinearSegmentedColormap.from_list("", ["black","green"]),annot_kws={"size":size})
    res.set_xticklabels(res.get_xmajorticklabels(), fontsize = size)
    res.set_yticklabels(res.get_ymajorticklabels(), fontsize = size)

    plt.xlabel('Predicted', fontsize=size)
    plt.ylabel('Actual', fontsize=size)
    plt.title('Splits for StratifiedKFold: '+str(splits)+'\nMaximum Accuracy That can be obtained from this model is: '+str(max(lst_accu_stratified)*100)+ '%\nMinimum Accuracy: '+str(min(lst_accu_stratified)*100)+ '%\nOverall Accuracy: '+str( mean(lst_accu_stratified)*100)+ '%\nStandard Deviation is: '+str( stdev(lst_accu_stratified)))
    plt.savefig(outpath+appender+"_rf_stratkfold_validation.png")
    return a

def main():
    config = Config.load()
    df = db.load().get_database_dataframe()
    counter = 0
    features = df.select_dtypes('number').columns
    if not config.get("classification_features").lower() == "all":
        features = config.get("classification_features").split(",")
    forest_importances = inspect_data(df,config.get("classification_group"),features,config.get("output"),str(counter))
    if config.get("classification_featuresselection") == "True":
        while (forest_importances["Importances"].values < 0).any():
            counter += 1
            good_features = forest_importances.loc[forest_importances["Importances"]>0]
            pathomic_profile = list(good_features["ID"])
            pathomic_profile.append(config.get("classification_group"))
            df_filtered = df[pathomic_profile]
            pathomic_profile.remove(config.get("classification_group"))
            forest_importances = inspect_data(df_filtered,config.get("classification_group"),pathomic_profile,config.get("output"),str(counter))


import logging as log
import os
import sys
import warnings
from functools import wraps

from loguru import logger
from pytest import deprecated_call
from requests.exceptions import BaseHTTPError

from conf import BASE_DIR
from utils.suport.exception import RequestError

def log_path():
    """
    日志文件路径
    """
    path = os.path.join(BASE_DIR, 'logs')
    log_file = os.path.join(path, '{}.log'.format('access_log'))
    if not os.path.exists(path):
        os.makedirs(path)
        if not os.path.exists(log_file):
            with open(log_file, 'w', encoding="utf-8") as fp:
                pass

    return log_file

# 重写emit使日志信息可以输出到allure
class PropagateHandler(log.Handler):
    def emit(self, record):
        log.getLogger(record.name).handle(record)

def get_logging():
    config = {
        "handlers": [
            # 配置info日志
            {"sink": log_path(), "format": "{time:YYYY-MM-DD HH:mm:ss} | {level} | {message}", "level": "INFO",
             "colorize": False, "enqueue": True},
            # loguru报告重定logging输出
            {"sink": PropagateHandler(), "format": "| {time:YYYY-MM-DD HH:mm:ss} | {message}", "level": "INFO",
             "colorize": False, "enqueue": True, "backtrace": True, "diagnose": True},
            # 输出到控制台
            {"sink": sys.stdout, "level": "INFO", "colorize": True, "enqueue": False}
        ],
        "extra": {"user": "someone"}
    }

    logger.configure(**config)

    return logger

logging = get_logging()

def api_logger(func):
    """
    装饰器
    采集API接口调用时的日志信息
    """

    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            response = func(*args, **kwargs)
        except BaseHTTPError as e:
            logging.error(f'接口调用异常: {str(e)}')
            raise RequestError
        else:
            logging.info(f'请求地址:{response.request.url}')
            logging.info(f'请求方式:{response.request.method}')
            logging.info(f"请求体:{response.request.body if response.request.body else None}")

            # 根据响应状态码确实使用什么日志方法
            log_func = getattr(logging, 'info' if response.status_code in (200, 201) else 'error')
            log_func('状态码:{}'.format(response.status_code))
            try:
                # deprecated_call() 将被标记了 DeprecationWarning 或者 PendingDeprecationWarning 的函数能够正常被调用。
                # 不会在执行时出现 相应的 warning 信息。
                with deprecated_call():
                    # DeprecationWarning 告警不再建议此操作。此处和deprecated_call配合使用，使其无效。
                    # 如果实际需要提示时，去掉deprecated_call()上下文即可
                    warnings.warn('主动抛出警告', DeprecationWarning)
                    log_func('响应体:{}'.format(response.text.encode('latin-1').decode('unicode_escape')))
            except UnicodeError:
                log_func('响应体:{}'.format(response.text))
        return response

    return wrapper


from django.urls import path, include
from rest_framework.routers import DefaultRouter

from api.views import *

router = DefaultRouter()
router.register('bread', BreadViewSet, basename="bread")
router.register('topping', ToppingViewSet, basename="topping")
router.register('cheese', CheeseViewSet, basename="cheese")
router.register('sauce', SauceViewSet, basename="sauce")
router.register('sandwich', SandwichViewSet, basename="sandwich")

app_name = "api"
urlpatterns = [
    # api
    path("", include(router.urls)),
]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created on Tue Dec 10
@author: karlvandesman
"""

import pandas as pd
from matplotlib import pyplot as plt

from scipy.stats import kurtosis
from scipy.stats import skew
from statsmodels.graphics.gofplots import qqplot

# Statistical tests
from scipy.stats import shapiro
from scipy.stats import kstest
from scipy.stats import normaltest

# Non-parametric tests
from scipy.stats import kruskal
from scikit_posthocs import posthoc_nemenyi

path = 'dataset/recife_etlsinasc_2016.csv'

df = pd.read_csv(path, index_col=0)

print('df dimensões (original): ', df.head())

# Criar classe binária para peso saudável
#df['PESO_BAIXO'] = 0 + ((df_drop['PESO']>2500).values)

#%%
# =============================================================================
# Estatística Descritiva
# =============================================================================

peso = df.PESO.values

qqplot(peso, fit=True, line='45')
plt.xlabel('Quantis da Normal')
plt.ylabel('Quantis observados da amostra')
plt.title('Gráfico Q-Q dos pesos de nascidos em Recife - 2016')
plt.figure(figsize=(20,20))
plt.show()

curtose_peso = kurtosis(peso)
assimetria_peso = skew(peso)

print('Características da distribuição de peso:')
print('Curtose = ', curtose_peso)
print('Assimetria = ', assimetria_peso)
print()

#%%
# =============================================================================
# Testes de aderência: Normalidade
# =============================================================================

# Kolmogorov-Smirnov
ks_stat, ks_p = kstest(df.PESO.values, 'norm')

# Shapiro-Wilk
shapiro_stat, shapiro_p = shapiro(df.PESO.values)   #UserWarning: p-value may
                                                    #not be accurate for N>5000

# D’Agostino e Pearson
DP_stat, DP_p = normaltest(df.PESO.values)

print('*** Testes de aderência: Normalidade ***')
print('Kolmogorov-Smirnov: estatística=%.4f, p-valor=%.8f'%(ks_stat, ks_p))
print('Shapiro-Wilk: estatística=%.4f, p-valor=%.8f'%(shapiro_stat, shapiro_p))
print('D’Agostino e Pearson: estatística=%.4f, p-valor=%.8f'%(DP_stat, DP_p))
print()

#%%
############################################
# Teste de Hipótese para peso segundo raça #
############################################

# Separar por raça os valores dos pesos
peso_branca = df.PESO[df.RACACOR==1].values
peso_preta = df.PESO[df.RACACOR==2].values
peso_amarela = df.PESO[df.RACACOR==3].values
peso_parda = df.PESO[df.RACACOR==4].values
peso_indigena = df.PESO[df.RACACOR==5].values

pesos_racas = [peso_branca, peso_preta, peso_amarela, peso_parda,
               peso_indigena]

# Teste de Kruskal-Wallis, que compara o conjunto
stat_peso_raca, p_peso_raca = kruskal(peso_branca, peso_preta, peso_amarela,
                                      peso_parda, peso_indigena)

print('Teste de Kruskal-Wallis para variação de raça/cor:')
print('Estatística do teste = %.2f, p-valor = %.7f'%(stat_peso_raca,
                                                     p_peso_raca))

# Se o teste de Kruskal rejeitar a hipótese nula (mesmas distribuições), então
# é feito o pós-teste de Nemenyi, comparando par a par os testes

alpha = 0.05
if p_peso_raca > alpha:
    print('Mesmas distribuições de peso para as raças/cores (falha em rejeitar H0)')
    print()
else:
    print('Diferentes distribuições segundo a raça/cor (rejeita H0)')
    print()
    print('Pós-teste de Nemenyi:')
    print(posthoc_nemenyi(pesos_racas))


# -*- coding: utf-8 -*-
# from selenium.common.exceptions import NoSuchElementException
import os, time, re
from bs4 import BeautifulSoup
from xhtml2pdf import pisa
from selenium import webdriver
from selenium.webdriver.common.by import By
from PIL import Image

from sys import path
path.append(r'..\..\_comum')
from chrome_comum import _initialize_chrome, _find_by_id
from comum_comum import _time_execution, _escreve_relatorio_csv, _open_lista_dados, _where_to_start, _indice
from captcha_comum import _solve_text_captcha

os.makedirs('execução/documentos', exist_ok=True)

def reset_table(table):
    table.attrs = None
    for tag in table.findAll(True):
        tag.attrs = None
        try:
            tag.string = tag.string.strip()

        except:
            pass

    return table

def format_data(content):
    # tabelas = []
    soup = BeautifulSoup(content, 'html.parser')
    tabela_header = soup.find('table', attrs={'id': 'table6'})
    tabela_debitos = soup.find('div', attrs={'id': 'grid'}).find('table')
    tabela_totais = soup.find('table', attrs={'id': 'tableTotais'})

    linhas = tabela_header.findAll('td', attrs={'class': 'linhaInformacao'})
    paragrafo = ', '.join(linha.text for linha in linhas)

    if len(tabela_debitos.findAll('tr')) <= 1:
        return False
    # tabela_debitos.attrs = None
    for tag in tabela_debitos.findAll(True):
        # tag.attrs = None
        try:
            tag.string = tag.string.strip()
        except:
            pass
        if 'input' == tag.name:
            tag.parent.extract()

    tabela_totais.find('tr', attrs={'id': 'tr4'}).extract()
    tabela_totais.find('td', attrs={'id': 'td2'}).extract()
    tabela_totais.find('td', attrs={'id': 'td4'}).extract()
    # tabela_totais = reset_table(tabela_totais)

    with open('style.css', 'r') as f:
        css = f.read()

    style = ""
    new_soup = BeautifulSoup(f'<meta charset="UTF-8">{style}

', 'html.parser')
    new_soup.body.p.append(paragrafo)
    new_soup.body.append(tabela_debitos)
    new_soup.body.append(tabela_totais)

    return str(new_soup)

def login(driver, cnpj, insc_muni, nome):
    base = 'http://179.108.81.10:9081/tbw'
    url_inicio = f'{base}/loginWeb.jsp?execobj=ServicoPesquisaISSQN'

    driver.get(url_inicio)
    while not _find_by_id('span7Menu', driver):
        time.sleep(1)
    button = driver.find_element(by=By.ID, value='span7Menu')
    button.click()

    while not _find_by_id('captchaimg', driver):
        time.sleep(1)
    element = driver.find_element(by=By.ID, value='captchaimg')
    location = element.location
    size = element.size
    driver.save_screenshot('ignore\captcha\pagina.png')
    x = location['x']
    y = location['y']
    w = size['width']
    h = size['height']
    width = x + w
    height = y + h
    time.sleep(2)
    im = Image.open(r'ignore\captcha\pagina.png')
    im = im.crop((int(x), int(y), int(width), int(height)))
    im.save(r'ignore\captcha\captcha.png')
    time.sleep(1)
    captcha = _solve_text_captcha(driver, 'captchaimg')

    if not captcha:
        print('Erro Login - não encontrou captcha')
        return driver, 'erro captcha'

    while not _find_by_id('input1', driver):
        time.sleep(1)
    element = driver.find_element(by=By.ID, value='input1')
    element.send_keys(insc_muni)

    while not _find_by_id('input4', driver):
        time.sleep(1)
    element = driver.find_element(by=By.ID, value='input4')
    element.send_keys(cnpj)

    while not _find_by_id('captchafield', driver):
        time.sleep(1)
    element = driver.find_element(by=By.ID, value='captchafield')
    element.send_keys(captcha)

    while not _find_by_id('imagebutton1', driver):
        time.sleep(1)
    button = driver.find_element(by=By.ID, value='imagebutton1')
    button.click()

    time.sleep(3)

    timer = 0
    while not _find_by_id('td30', driver):
        print('>>> Aguardando site')
        driver.save_screenshot(r'ignore\debug_screen.png')
        time.sleep(1)

        if _find_by_id('tdMsg', driver):
            try:
                erro = re.compile(r'id=\"tdMsg\".+tipo=\"td\">(.+)\.').search(driver.page_source).group(1)
                print(f'❌ {erro}')
                return driver, erro
            except:
                pass
        timer += 1
        if timer >= 10:
            print(f'❌ Erro no login')
            return driver, 'Erro no login'

    return driver, 'ok'

def consulta(driver, cnpj, insc_muni, nome):
    print('>>> Consultando empresa')
    html = driver.page_source.encode('utf-8')
    try:
        driver.save_screenshot(r'ignore\debug_screen.png')
        str_html = format_data(html)

        if str_html:
            _escreve_relatorio_csv(f'{cnpj};{insc_muni};{nome};Com débitos')
            print('>>> Gerando arquivo')
            nome_arq = ';'.join([cnpj, 'INF_FISC_REAL', 'Debitos Municipais'])
            with open('execução/documentos/' + nome_arq + r'.pdf', 'w+b') as pdf:
                pisa.showLogging()
                pisa.CreatePDF(str_html, pdf)
                print('❗ Arquivo gerado')
        else:
            _escreve_relatorio_csv(f'{cnpj};{insc_muni};{nome};Sem débitos')
            print('✔ Não há débitos')
    except:
        _escreve_relatorio_csv(f'{cnpj};{insc_muni};{nome};Erro na geração do PDF')
        driver.save_screenshot(r'ignore\debug_screen.png')
        print('❌ Erro na geração do PDF')

    return driver, True

@_time_execution
def run():
    empresas = _open_lista_dados()

    index = _where_to_start(tuple(i[0] for i in empresas))
    if index is None:
        return False

    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--window-size=1920,1080')

    total_empresas = empresas[index:]
    for count, empresa in enumerate(empresas[index:], start=1):
        cnpj, insc_muni, nome = empresa

        _indice(count, total_empresas, empresa, index)

        resultado = 'Texto da imagem incorreto'
        while resultado == 'Texto da imagem incorreto':
            status, driver = _initialize_chrome(options)
            driver, resultado = login(driver, cnpj, insc_muni, nome)

            if resultado == 'ok':
                driver, resultado = consulta(driver, cnpj, insc_muni, nome)
            elif resultado == 'Texto da imagem incorreto':
                pass
            else:
                _escreve_relatorio_csv(f'{cnpj};{insc_muni};{nome};{resultado}')
            driver.close()

if __name__ == '__main__':
    run()

Serviços Prefeitura/Consulta Débitos Municipais Valinhos/consulta_debitos_municipais_valinhos.py


'''
Sample input:
1d10 2d20 3d4 6d12

Sample output:

1d10
5

2d20
4
6

3d4
1
3
2

6d12
10
3
2
4
6
7
'''

import sys
import secrets

def main():
    try:
        if len(sys.argv) == 1:
            print(f'\nError: no dice specified to roll')
            raise ValueError

        for x in range(1, len(sys.argv)):
            Roll(sys.argv[x].split('d'))
    except:
        print('\nUsage: specify any number of dice with roll count, e.g.:')
        print('python main.py 1d10 2d4')
        print('\nValid dice are d4, d6, d8, d10, d12, d20, d100')

def Roll(roll_count_die):
    roll_count = int(roll_count_die[0])
    die = int(roll_count_die[1])

    if die not in [4, 6, 8, 10, 12, 20, 100]:
        print(f'\nError: d{die} is not a valid D&D die')
        raise ValueError

    print(f'\n{roll_count}d{die}')

    for x in range(0, roll_count):
        roll = secrets.randbelow(die) + 1

        print(roll)

if __name__ == "__main__":
    main()


import hashlib
import json
import logging
import re

from pyhocon import ConfigFactory, HOCONConverter

from .dict_tool import merge_dict

logger = logging.getLogger(__name__)

NEW_LINE = '\n'
RE_HOCON_INCLUDE = [
    r'include\s+(?:required|url|file|classpath)\(.*\)',
    r'include\s+".*\.(?:conf|hocon)"',
]
RE_HOCONSTRING_INCLUDE = r'HOCONSTRING_INCLUDE_(?:.*)\s*=\s*"(?:.*)"'
RE_HOCONSTRING_INCLUDE_VALUE = r'HOCONSTRING_INCLUDE_(?:.*)\s*=\s*"(.*)"'
HOCONSTRING_INCLUDE_KEY = 'HOCONSTRING_INCLUDE_{id}'

def escape_double_quotes(double_quotes):
    return double_quotes.replace('"', '\\"')

def unescape_double_quotes(escaped_double_quotes):
    return escaped_double_quotes.replace('\\"', '"')

def is_valid_include(include):
    is_valid_format = False
    for regex in RE_HOCON_INCLUDE:
        if re.findall(regex, include):
            is_valid_format = True
            break

    return is_valid_format

def get_include_key(include_str):
    """Use md5sum hash of the whole include statement string for a key."""
    return hashlib.md5(include_str.encode()).hexdigest()

def wrap_includes(hocon_str):
    """Convert `include` statement string into key = val format.
    Returns '{key} = "{double_quote_escaped_val}"'.
    """
    for regex in RE_HOCON_INCLUDE:
        for include in re.findall(regex, hocon_str):
            if '\\"' in include:
                continue

            logger.debug('Found include in HOCON: {include}'.format(include=include))

            hocon_str = hocon_str.replace(
                include,
                '{key} = "{val}"'.format(
                    key=HOCONSTRING_INCLUDE_KEY.format(id=get_include_key(include)),
                    val=escape_double_quotes(include),
                ),
            )
    return hocon_str

def unwrap_includes(key_val_str):
    """Convert '{key} = "{val}"" formatted string to the original `include` statement string.
    Args:
        key:
            HOCONSTRING_INCLUDE_KEY with `id` as md5sum hash of the original
            `include` statement string.
        val:
            Double-quote-escaped `include` statement string.
    """
    val = re.findall(RE_HOCONSTRING_INCLUDE_VALUE, key_val_str)
    if val:
        if len(val) > 1:
            raise ValueError(
                'Found multiple matches. Wrong include key=val format? {val}'.format(
                    val=val
                )
            )
        return unescape_double_quotes(val[0])

class HOCONString:
    def __init__(self, hocon_str):
        """Find an `include` statement (VALUE) in HOCON string and then convert it
        into a HOCONSTRING_INCLUDE_KEY="VALUE" pair in HOCON.

        Double-quotes will be escaped with double slashes.
        Then the VALUE is kept as it is as a value and can be recovered later when
        it is converted back to HOCON string.

        This workaround is to skip parsing `include` statements since there is no
        information about `classpath` at the parsing time and pyhocon will error out and
        will stop parsing.

        e.g. we don't know what's in `classpath` before the backend conf file is
        passed to Cromwell.
        """
        if not isinstance(hocon_str, str):
            raise ValueError('HOCONString() takes str type only.')

        self._hocon_str = wrap_includes(hocon_str)

    def __str__(self):
        return self.get_contents()

    @classmethod
    def from_dict(cls, d, include=''):
        """Create HOCONString from dict.

        Args:
            include:
                `include` statement to be added to the top of the HOCONString.
        """
        hocon = ConfigFactory.from_dict(d)
        hocon_str = HOCONConverter.to_hocon(hocon)

        if include:
            if not is_valid_include(include):
                raise ValueError(
                    'Wrong HOCON include format. {include}'.format(include=include)
                )
            hocon_str = NEW_LINE.join([include, hocon_str])

        return cls(hocon_str=hocon_str)

    def to_dict(self, with_include=True):
        """Convert HOCON string into dict.

        Args:
            with_include:
                If True then double-quote-escaped `include` statements will be kept as a plain string
                under key HOCONSTRING_INCLUDE_KEY.
                Otherwise, `include` statements will be excluded.
        """
        if with_include:
            hocon_str = self._hocon_str
        else:
            hocon_str = self.get_contents(with_include=False)

        c = ConfigFactory.parse_string(hocon_str)
        j = HOCONConverter.to_json(c)

        return json.loads(j)

    def merge(self, b, update=False):
        """Merge self with b and then returns a plain string of merged.
        Args:
            b:
                HOCONString, dict, str to be merged.
                b's `include` statement will always be ignored.
            update:
                If True then replace self with a merged one.
        Returns:
            String of merged HOCONs.
        """
        if isinstance(b, HOCONString):
            d = b.to_dict()
        elif isinstance(b, str):
            d = HOCONString(b).to_dict()
        elif isinstance(b, dict):
            d = b
        else:
            raise TypeError('Unsupported type {t}'.format(t=type(b)))

        self_d = self.to_dict()
        merge_dict(self_d, d)

        hocon = ConfigFactory.from_dict(self_d)

        hocon_str = HOCONConverter.to_hocon(hocon)
        if update:
            self._hocon_str = hocon_str

        return HOCONString(hocon_str).get_contents()

    def get_contents(self, with_include=True):
        """Check if `include` statement is stored as a plain string.
        If exists, converts it back to HOCON `include` statement.

        Args:
            with_include: (renamed/changed from without_include)
                If True then recover all includes statements from include key=val form
                (RE_HOCONSTRING_INCLUDE).
                Otherwise, excludes all `include` statements.
        """
        hocon_str = self._hocon_str

        for include_key_val in re.findall(RE_HOCONSTRING_INCLUDE, self._hocon_str):
            logger.debug(
                'Found include key in HOCONString: {include_key_val}'.format(
                    include_key_val=include_key_val
                )
            )
            if with_include:
                original_include_str = unwrap_includes(include_key_val)
                if original_include_str:
                    hocon_str = hocon_str.replace(include_key_val, original_include_str)
            else:
                hocon_str = hocon_str.replace(include_key_val, '')

        return hocon_str


import time
import math
import numpy as np
import pandas as pd
import pickle as pkl
from collections import Counter

import tensorflow as tf

# Custom functions. #
def compute_bleu_score(
    reference_corpus, translated_corpus, max_order=4, smooth=False):

    def _get_n_grams(segment, max_order):
        n_gram_counts = Counter()
        for order in range(1, max_order+1):
            for i in range(0, len(segment)-order+1):
                ngram = tuple(segment[i:(i+order)])
                n_gram_counts[ngram] += 1
        return n_gram_counts

    matches_by_order = [0]*max_order
    possible_matches_by_order = [0]*max_order

    reference_length   = 0
    translation_length = 0

    for (references, translation) in \
        zip(reference_corpus, translated_corpus):
        reference_length += min(len(r) for r in references)
        translation_length += len(translation)

        merged_ref_ngram_counts = Counter()
        for reference in references:
            merged_ref_ngram_counts |= _get_n_grams(reference, max_order)
        translated_ngram_counts = _get_n_grams(translation, max_order)

        overlap = translated_ngram_counts & merged_ref_ngram_counts
        for ngram in overlap:
            matches_by_order[len(ngram)-1] += overlap[ngram]
        for order in range(1, max_order+1):
            possible_matches = len(translation) - order + 1
            if possible_matches > 0:
                possible_matches_by_order[order-1] += possible_matches

    precisions = [0]*max_order
    for i in range(0, max_order):
        if smooth:
            precisions[i] = \
                (matches_by_order[i]+1.0) / possible_matches_by_order[i]
        else:
            if possible_matches_by_order[i] > 0:
                precisions[i] = \
                    float(matches_by_order[i]) / possible_matches_by_order[i]
            else:
                precisions[i] = 0.0

    if min(precisions) > 0:
        p_log_sum = \
            sum((1.0/max_order) * math.log(p) for p in precisions)
        geo_mean = math.exp(p_log_sum)
    else:
        geo_mean = 0.0

    tmp_ratio = float(translation_length) / reference_length
    if tmp_ratio > 1.0:
        bp = 1.0
    else:
        bp = math.exp(1.0 - (1.0/tmp_ratio))

    bleu = geo_mean*bp
    return bleu

# Model Parameters. #
batch_size = 256
sub_batch  = 64
attn_flag  = True
seq_length = 21
kernel_sz  = 3
num_stacks = 3
num_layers = 4

prob_keep = 0.9
hidden_size = 256

tmp_path = "C:/Users/admin/Desktop/Codes/"
train_loss_file = "C:/Users/admin/Desktop/Codes/"
if attn_flag:
    import tf_ver2_seq_cnn_attn_v0 as tf_model
    model_ckpt_dir  = \
        "C:/Users/admin/Desktop/TF_Models/dialogue_seq_cnn_attn"
    train_loss_file += "train_loss_dialogue_seq_cnn_attn.csv"
else:
    import tf_ver2_seq_cnn as tf_model
    model_ckpt_dir  = \
        "C:/Users/admin/Desktop/TF_Models/dialogue_seq_cnn"
    train_loss_file += "train_loss_dialogue_seq_cnn.csv"

# Load the data. #
tmp_pkl_file = \
    "C:/Users/admin/Desktop/Codes/movie_dialogues.pkl"
with open(tmp_pkl_file, "rb") as tmp_load_file:
    data_tuple = pkl.load(tmp_load_file)
    idx2word = pkl.load(tmp_load_file)
    word2idx = pkl.load(tmp_load_file)

vocab_size = len(word2idx)
print("Vocabulary Size:", str(vocab_size)+".")

SOS_token = word2idx["SOS"]
EOS_token = word2idx["EOS"]
PAD_token = word2idx["PAD"]
UNK_token = word2idx["UNK"]

# Set the number of threads to use. #
tf.config.threading.set_intra_op_parallelism_threads(1)
tf.config.threading.set_inter_op_parallelism_threads(1)

# Build the Transformer. #
print("Building the Sequence CNN Model.")
start_time = time.time()

gpt_model = tf_model.build_model(
    hidden_size, vocab_size,
    num_layers, p_drop=1.0-prob_keep,
    n_stacks=num_stacks, kernel_size=kernel_sz)
gpt_optimizer = tf.keras.optimizers.Adam(
    beta_1=0.9, beta_2=0.98, epsilon=1.0e-9)

elapsed_time = (time.time()-start_time) / 60
print("Sequence CNN Model Built", "("+str(elapsed_time)+" mins).")

# Create the model checkpoint. #
ckpt = tf.train.Checkpoint(
    step=tf.Variable(0),
    gpt_model=gpt_model,
    gpt_optimizer=gpt_optimizer)

manager = tf.train.CheckpointManager(
    ckpt, model_ckpt_dir, max_to_keep=1)

ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
    print("Model restored from {}".format(manager.latest_checkpoint))
else:
    print("Error: No latest checkpoint found.")

# Train the Transformer model. #
tmp_test_in = np.zeros([1, seq_length], dtype=np.int32)

# Warmup learning schedule. #
n_iter = ckpt.step.numpy().astype(np.int32)

print("-" * 50)
print("Testing the Sequence CNN Network",
      "(" + str(n_iter) + " iterations).")
print("-" * 50)

# Update the neural network's weights. #
while True:
    tmp_in_phrase = input("Enter Input Phrase: ")
    if tmp_in_phrase == "":
        break
    else:
        tmp_test_in[:, :] = PAD_token

        tmp_in_phrase = tmp_in_phrase.lower()
        tmp_p_tokens  = tmp_in_phrase.split(" ")

        tmp_p_idx = [word2idx.get(
            x, UNK_token) for x in tmp_p_tokens]
        n_tokens  = len(tmp_p_idx) + 1
        tmp_test_in[0, :n_tokens] = tmp_p_idx + [SOS_token]

        tmp_infer  = tf_model.infer(
            gpt_model, tmp_test_in[:, :n_tokens], seq_length)
        gen_phrase = [idx2word[x] for x in tmp_infer[0].numpy()]
        gen_output = " ".join(gen_phrase[n_tokens:])
        del n_tokens

        print("")
        print("Input Phrase:")
        print(tmp_in_phrase)
        print("Generated Response:")
        print(gen_output)
        print("-" * 50)


import sublime, sublime_plugin

import subprocess
import json
import os
import re
import threading
import queue
import atexit

#
# TODO:
# - define gofmt binary.
# - user configuration.
# - Oracle scope from the keyboard (not edit configuration file).
# - syntax to top level?
#

from .errormarkers import clear_error_marks, clear_error_marks_view, add_error_mark, show_error_marks, \
                         update_statusbar, erase_error_marks, has_error_marks

from .common import *

from .thread_progress import *

# Add flymake*.go so we avoid spamming the file view when flymaking.
def update_file_exclude_patterns():
    s = sublime.load_settings("Preferences.sublime-settings")
    file_exclude_patterns = s.get('file_exclude_patterns', [])
    if file_exclude_patterns is None:
        return
    found = False
    for f in file_exclude_patterns:
        if f == "flymake*.go":
            found = True
    if not found:
        file_exclude_patterns.append("flymake*.go")
        s.set('file_exclude_patterns', file_exclude_patterns)
        sublime.save_settings("Preferences.sublime-settings")
update_file_exclude_patterns()

def get_output_view(window):
    view = None
    buff_name = 'Go Mode'

    if get_setting("output", "buffer") == "output_panel":
        view = window.create_output_panel(buff_name)
    else:
        # If the output file is already open, use that.
        for v in window.views():
            if v.name() == buff_name:
                view = v
                break
        # Otherwise, create a new one.
        if view is None:
            view = window.new_file()

    view.set_name(buff_name)
    view.set_scratch(True)
    view_settings = view.settings()
    view_settings.set('line_numbers', False)
    # view.set_syntax_file('Packages/GoMode/GoOracleResults.tmLanguage')

    return view

def sel(view, i=0):
    try:
        s = view.sel()
        if s is not None and i < len(s):
            return s[i]
    except Exception:
        pass

    return sublime.Region(0, 0)

def is_go_source_view(view=None, strict=True):
    if view is None:
        return False

    selector_match = view.score_selector(sel(view).begin(), 'source.go') > 0
    if selector_match:
        return True

    if strict:
        return False

    fn = view.file_name() or ''
    return fn.lower().endswith('.go')

packages = {
     "gocode": "github.com/nsf/gocode",
     "goimports": "golang.org/x/tools/cmd/goimports",
     "godef": "github.com/rogpeppe/godef",
     "oracle": "golang.org/x/tools/cmd/oracle",
     "gorename": "golang.org/x/tools/cmd/gorename",
     "goflymake": "github.com/dougm/goflymake",
}
# "github.com/golang/lint/golint",
#             "github.com/kisielk/errcheck",
#             "github.com/jstemmer/gotags",

import time

def install_packages(view):
    for k in packages:
        view.run_command('go_mode_output_insert', {'text': "Installing %s\n" % (k)})
        args = ["go", "get", "-u", "-v", "-f", packages[k] ]
        view.run_command('go_mode_output_insert', {'text': " ".join(args)})
        view.run_command('go_mode_output_insert', {'text': "\n"})
        try:
            env = getenv()
            child = openProcess(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = child.communicate()
            if child.returncode != 0:
                err = stderr.decode('utf-8')
                view.run_command('go_mode_output_insert', {'text': "Failed %s\n" % (err)})
            view.run_command('go_mode_output_insert', {'text': stdout.decode('utf-8')})
            view.run_command('go_mode_output_insert', {'text': "\n"})
        except Exception as e:
            view.run_command('go_mode_output_insert', {'text': "Failed %s\n" % (e)})

class GoModeInstallBinaries(sublime_plugin.WindowCommand):
    def run(self):
            t = threading.Thread(target=install_packages, args=(get_output_view(self.window),))
            t.start()
            ThreadProgress(t, "installing binaries", "installing GoMode binaries complete")

class GoModeGoFmtCommand(sublime_plugin.TextCommand):
    def is_enabled(self):
        return is_go_source_view(self.view)

    def run(self, edit, saving=False):
        # Get the content of the current window from the text editor.
        selection = sublime.Region(0, self.view.size())
        content = self.view.substr(selection)

        try:
            env = getenv()
            child = openProcess(["goimports"], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = child.communicate(input=content.encode('utf-8'))
            if child.returncode != 0:
                err = stderr.decode('utf-8')
                print("goimports failed: %s" % (err))
                return

            # Put the result back.
            self.view.replace(edit, selection, stdout.decode('utf8'))
        except Exception as e:
            print(e)

class GoModeOutputInsertCommand(sublime_plugin.TextCommand):
    def run(self, edit, text):
        self.view.insert(edit, self.view.size(), text)

class GoModeGoRenameCommand(sublime_plugin.WindowCommand):
    def run(self):
        view = self.window.active_view()
        filename = view.file_name()

        region = view.sel()[0]
        if region.empty():
            self.window.run_command('find_under_expand') # super+d
            region = view.sel()[0]

        if region.empty(): # the cursor is on whitespace not bordering any word
            sublime.message_dialog('Select an identifier you would like to rename and try again.')
            return

        current_selection = view.substr(region)
        # TODO: should we try to detect if the selected region could not be an actual renamable identifier?

        def on_done(new_name):
            if new_name == current_selection:
                return
            try:
                offset = filename + ':#{0}'.format(region.begin())
                env = getenv()
                args = ['gorename', '-offset', offset, '-to', new_name]
                p = openProcess(args, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdout, stderr = p.communicate()
                if p.returncode != 0:
                    view = get_output_view(self.window)
                    self.window.focus_view(view)
                    view.run_command('go_mode_output_insert', {'text': "go rename failed\n%s\n" % (str(e))})
            except Exception as e:
                view = get_output_view(self.window)
                self.window.focus_view(view)
                view.run_command('go_mode_output_insert', {'text': "go rename not found: %s\n" % (e)})

        self.window.show_input_panel("New name:", current_selection, on_done, None, None)

navigation_stack = []

class GoModeBack(sublime_plugin.TextCommand):
    def run(self, edit):
        if len(navigation_stack) > 0:
            self.view.window().open_file(
                navigation_stack.pop()[0], sublime.ENCODED_POSITION)

    def is_enabled(self):
        return is_go_source_view(sublime.active_window().active_view()) and len(navigation_stack) > 0

    def is_visible(self):
        return is_go_source_view(sublime.active_window().active_view())

def format_current_file(view):
    row, col = view.rowcol(view.sel()[0].a)
    return "%s:%d:%d" % (view.file_name(), row + 1, col + 1)

def navigation_stack_open(view, target):
    navigation_stack.append((format_current_file(view), target))
    view.window().open_file(target, sublime.ENCODED_POSITION)

class GoModeGoDefCommand(sublime_plugin.WindowCommand):
    def run(self):
        try:
            view = self.window.active_view()
            select = view.sel()[0]
            string_before = view.substr(sublime.Region(0, select.begin()))
            string_before.encode("utf-8")
            offset = len(bytearray(string_before, encoding = "utf8"))

            filename = view.file_name()

            args = [ "godef", "-f", filename, "-o", str(offset) ]
            env = getenv()
            p = openProcess(args, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = p.communicate()
            if p.returncode != 0:
                err = stderr.decode('utf-8')
                print("godef failed: %s" % (err))
                return

            location = stdout.decode("utf-8").rstrip().split(":")
            if len(location) == 3:
                navigation_stack_open(view, (location[0] + ":" + location[1] + ":" + location[2]))
        except Exception as e:
            print(e)

def log_output(out, view):
    for line in iter(out.readline, b''):
        view.run_command('go_mode_output_insert', {'text': line.decode('utf-8') })

class GoModeGoCodeDaemon:
    def __init__(self):
        self.p = None

    def fork_gocode(self):
        print("fork_gocode")
        kill_gocode()

        outputView = get_output_view(sublime.active_window())
        env = getenv()
        try:
            addr = get_setting("gocode_address", "-addr=localhost:37777")
            debug = get_setting("gocode_debug", "false")
            # XXX: Parameterize gocode addr.
            self.p = openProcess(["gocode", "-sock=tcp", addr, "-s=true", "-debug=%s" % (debug)], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            t = threading.Thread(target=log_output, args=(self.p.stdout, outputView))
            t.daemon = True # thread dies with the program
            t.start()
            t = threading.Thread(target=log_output, args=(self.p.stderr, outputView))
            t.daemon = True # thread dies with the program
            t.start()

        except Exception as e:
            outputView.run_command('go_mode_output_insert', {'text': "cannot fork gocode\n%s" % (e) })

    def kill_gocode(self):
        env = getenv()
        addr = get_setting("gocode_address", "-addr=localhost:37777")
        p = openProcess(["gocode", "-sock=tcp", addr, "close"], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
            if not self.p is None:
                self.p.terminate()
        if not self.p is None:
            self.p.wait()
            self.p = None

daemon = GoModeGoCodeDaemon()
sublime.set_timeout(lambda: daemon.fork_gocode(), 0)

def kill_gocode():
    daemon.kill_gocode()
atexit.register(kill_gocode)

class GoModeRestartGoCode(sublime_plugin.WindowCommand):
    def run(self):
        daemon.fork_gocode()

class GoModeAutocomplete(sublime_plugin.EventListener):
    def on_query_completions(self, view, prefix, locations):
        results = []
        try:
            pos = locations[0]
            env = getenv()
            p = openProcess(["gocode", "-sock=tcp", "-addr=localhost:37777", "-f=json", "autocomplete", view.file_name().encode('utf-8'), str(pos)], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            src = view.substr(sublime.Region(0, view.size()))
            stdout, stderr = p.communicate(input=src.encode())
            if p.returncode != 0:
                err = stderr.decode('utf-8')
                print("gocode failed: %s" % (err))
                return []

            try:
                j = json.loads(stdout.decode("utf8"))
            except ValueError:
                print("Not valid JSON")
                return []

            if len(j) == 2:
                for r in j[1]:
                    #results.append(['trigger': r['name'], 'contents': r['name']])
                    results.append([ r['name'], r['name']])
        except Exception as e:
            print(e)
        return results

    def on_pre_save(self, view):
        view.run_command("go_mode_go_fmt")

class GoModeCompiler:
    def __init__(self):
        self.lock = threading.Lock()
        self.targets = {}
        self.queue = queue.Queue(maxsize=0)
        self.worker = threading.Thread(target=self.do_compile)
        self.worker.setDaemon(True)
        self.worker.start()
        self.recompile_timer = None
        self.recompile_delay = 0.1

    # Run in main thread.
    def show_results(self, view, returncode, stdout, stderr):
        # clear_error_marks()
        file_name = view.file_name().encode('utf-8')
        outputView = get_output_view(view.window())
        outputView.run_command('go_mode_output_insert', {'text': "%s\n%s" % (view.file_name(), stdout.decode("utf-8"))})

        # print("show results: view=%s file=%s" % (view, file_name))
        # print("stdout\n%s" %(stdout.decode("utf-8")))
        clear_error_marks_view(file_name)
        print(stdout.decode("utf-8"))
        lines = stdout.decode("utf-8").split('\n')
        for l in lines:
            # XXX: Compile.
            #
            # Building tests with flymake results in file:line:col:error
            # Building a regular file with flymake results in file:line:error
            m = re.search('^([^:]+):(\d+):(?:\d+:){0,1} (.*)', l)
            if m is None:
                continue
            f = m.group(1)
            base = os.path.basename(f)
            if len(base) > len("flymake_") and base[:len("flymake_")] == "flymake_":
                base = base[len("flymake_"):]
            # All files must be in the same directory as the view.file_name()
            f = os.path.join(os.path.dirname(view.file_name()), base)
            add_error_mark(f.encode('utf-8'), int(m.group(2))-1, m.group(3))
        show_error_marks(view)

    # Run in worker thread.
    def do_compile(self):
        while True:
            (filename, view, data) = self.queue.get()
            try:
                print("-> compiling %s" %(filename))
                dirname = os.path.dirname(filename)
                flyname = "flymake_" + os.path.basename(filename)
                target_name = os.path.join(dirname, flyname)

                print("flymake `%s'" % (target_name))

                target = open(target_name, "w")
                target.truncate()
                target.write(data)
                target.close()

                args = ["goflymake", flyname]
                env = getenv()
                p = openProcess(args, cwd=dirname, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
                stdout, stderr = p.communicate()
                os.unlink(target_name)

                sublime.set_timeout(lambda view=view, code=p.returncode: self.show_results(view, code, stdout, stderr), 0)

            except Exception as e:
                print("compilation aborted: %s" % (filename))

            self.lock.acquire(True)
            del self.targets[filename]
            print("<- compiling %s - targets=%s" %(filename, self.targets))
            self.lock.release()

            self.queue.task_done()

    # Content is filename, view, string
    def compile(self, content):
        (filename, view, data) = content
        try:
            self.lock.acquire(True)
            if content[0] in self.targets:
                print("COMPILE")
                print(self.targets)
                return False

            self.targets[content[0]] = True
            self.queue.put(content)
            return True
        finally:
            self.lock.release()

c = GoModeCompiler()

# XXX: Missing some cases.
class GoModeGoFlymake(sublime_plugin.EventListener):
    def __init__(self):
        # view isn't hashable, so this is filename->view.
        self.views = {}
        self.recompile_timer = None
        # Seconds
        self.recompile_delay = 1.0

    def restart_recompile_timer(self, timeout):
        # Do a recompile now, but not another for 5s.
        if self.recompile_timer == None:
            self.recompile()
        else:
            self.recompile_timer.cancel()
        self.recompile_timer = threading.Timer(timeout, sublime.set_timeout,
                                               [self.recompile, 0])
        self.recompile_timer.start()

    # Block recompiles while a compilation is in progress.
    def recompile(self):
        erase = []
        for k in self.views:
            view = self.views[k]
            if c.compile((view.file_name(), view, view.substr(sublime.Region(0, view.size())))):
                erase.append(k)
            else:
                self.restart_recompile_timer(1.0)
        for k in erase:
            del self.views[k]
        print(self.views)

    def on_modified(self, view):
        if not is_go_source_view(view):
            return

        self.views[view.file_name()] = view
        self.restart_recompile_timer(self.recompile_delay)

    def show_errors(self, view):
        if not is_go_source_view(view):
            return

        if has_error_marks(view):
            show_error_marks(view)

    #def on_activated(self, view):
        #self.show_errors(view)

    #def on_load(self, view):
        #self.show_errors(view)


import json
import os

import tensorflow as tf

from models.TransformerLayers import EncoderLayer
from models.UtilLayers import PositionalEmbedding

class EncoderModel(tf.keras.Model):
    def __init__(
        self,
        embedding_size: int,
        d_model: int,
        num_heads: int,
        num_layers: int,
        vocab_size: int,
        num_classes: int,
        seq_len: int,
        pe: int = 1024,
        rate: float = 0.1,
    ):
        super(EncoderModel, self).__init__()

        self.embedding_size = embedding_size
        self.d_model = d_model
        self.num_heads = num_heads
        self.num_layers = num_layers
        self.vocab_size = vocab_size
        self.num_classes = num_classes
        self.seq_len = seq_len
        self.pe = pe
        self.rate = rate

        self.embedding = PositionalEmbedding(vocab_size, embedding_size, pe)
        if embedding_size != d_model:
            self.embedding_intermediate = tf.keras.layers.Dense(d_model)

        self.encoders = [
            EncoderLayer(d_model, num_heads, rate) for _ in range(self.num_layers)
        ]
        self.intermediate_layers = [
            tf.keras.layers.Dense(self.d_model, activation="gelu")
            for _ in range(self.num_layers)
        ]
        self.dropout = [
            tf.keras.layers.Dropout(rate=self.rate) for _ in range(self.num_layers)
        ]
        self.pooling_embedding = tf.keras.layers.Embedding(1, self.d_model)
        self.output_layer = tf.keras.layers.Dense(
            self.num_classes, activation="sigmoid"
        )

    def dot_attention(self, q, k, v, mask=None):
        logits = tf.matmul(q, k, transpose_b=True)

        if mask is not None:
            logits += tf.cast((1 - mask[:, tf.newaxis, :]), tf.float32) * -1e9

        attention_weights = tf.nn.softmax(logits, axis=-1)
        output = tf.matmul(attention_weights, v)

        return output

    def call(
        self,
        inputs,
        # input_ids,
        # attention_mask=None,
        training=False,
        **kwargs,
    ):
        input_ids = inputs["input_ids"]
        attention_mask = inputs.get("attention_mask")

        # forward
        encoder_output = self.embedding(
            input_ids
        )  # shape [batch_size, sequance_length]
        if self.embedding_size != self.d_model:
            encoder_output = self.embedding_intermediate(encoder_output)

        for i in range(self.num_layers):

            encoder_output = self.encoders[i](
                encoder_output, attention_mask, training=training
            )
            encoder_output = self.intermediate_layers[i](encoder_output)
            encoder_output = self.dropout[i](encoder_output, training=training)
        # shape [batch_size, sequance_length, d_model]

        # add pooling layer
        pooling_code = self.pooling_embedding(  # code_embedding
            tf.range(1, dtype=tf.int32)
        )  # shape: (batch_size, 1, d_model)
        encoder_output = self.dot_attention(
            pooling_code, encoder_output, encoder_output, mask=attention_mask
        )  # shape: (batch_size, 1, d_model). query의 shape를 따라감.
        encoder_output = tf.squeeze(
            encoder_output, [1]
        )  # == tf.keras.Layers.Flatten()(encoder_output)

        output = self.output_layer(
            encoder_output
        )  # shape [batch_size, sequance_length, num_classes]
        return output

    def get_config(self):
        return {
            "embedding_size": self.embedding_size,
            "d_model": self.d_model,
            "num_heads": self.num_heads,
            "num_layers": self.num_layers,
            "vocab_size": self.vocab_size,
            "num_classes": self.num_classes,
            "seq_len": self.seq_len,
            "pe": self.pe,
            "rate": self.rate,
        }

    def _get_sample_data(self):
        sample_data = {
            "input_ids": tf.random.uniform((1, 8), 0, self.vocab_size, dtype=tf.int64),
        }
        return sample_data

    def save(self, save_dir):
        if not os.path.isdir(save_dir):
            os.mkdir(save_dir)

        with open(os.path.join(save_dir, "config.json"), "w") as f:
            json.dump(self.get_config(), f)

        self(self._get_sample_data())
        self.save_weights(os.path.join(save_dir, "model_weights.h5"))

        return os.listdir(save_dir)

    @classmethod
    def load(cls, save_dir):
        with open(os.path.join(save_dir, "config.json"), "r") as f:
            config = json.load(f)

        model = cls(**config)
        model(model._get_sample_data())
        model.load_weights(os.path.join(save_dir, "model_weights.h5"))

        return model


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from typing import List, Optional
from pydantic import BaseModel
import ruspy as rp

app = FastAPI()

# Configure CORS
# This allows all origins (you can specify specific origins as well)
origins = ["*"]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

class ComplexArray(BaseModel):
    re: List[str]
    im: Optional[List[str]] = None

@app.get("/")
async def welcome():
    return "Welcome to FFT API"

@app.post("/post")
async def fft(values: ComplexArray):
    re = values.re
    im = values.im if values.im is not None else [0] * len(re)
    values = [complex(float(re[i]), float(im[i])) for i in range(len(re))]
    output = rp.fft(rp.array(values))
    return {
        "re": [str(output[i].real) for i in range(len(output))],
        "im": [str(output[i].imag) for i in range(len(output))],
    }

@app.get("/calc")
async def fft(re: List[str], im: Optional[List[str]] = None):
    re = re
    im = im if im is not None else [0] * len(re)
    values = [complex(float(re[i]), float(im[i])) for i in range(len(re))]
    output = rp.fft(rp.array(values))
    return {
        "re": [str(output[i].real) for i in range(len(output))],
        "im": [str(output[i].imag) for i in range(len(output))],
    }


from multiprocessing import Process, Queue, Lock, current_process, Pool
import time
import queue

def task_implementation(task_to_complete, task_which_are_done):
    while True:
        try:
            received_number = task_to_complete.get_nowait() #throw exception queue.Empty if Queue is empty
        except queue.Empty:
            break
        else:
            square_of_number = received_number * received_number
            print('Task no : ',received_number)
            task_which_are_done.put('Square of '+ str(received_number) + ' is ' + str(square_of_number) + ' done by : '+current_process().name)
            time.sleep(0.5)

def task_distribution_queue_process(number_of_task, total_process):
     #Queue class is already synchronized. no need of Lock class to block multiple process to access the same queue object
    task_to_complete = Queue()
    task_which_are_done = Queue()

    for i in range(number_of_task):
        task_to_complete.put(i)

    process_objects = []

    #1st method without using Pool
    for _ in range(total_process):
        process_obj = Process(target=task_implementation,args=(task_to_complete, task_which_are_done))
        process_objects.append(process_obj)
        process_obj.start()
    for process_obj in process_objects:
        process_obj.join()

    while not task_which_are_done.empty():
         print(task_which_are_done.get())
    return True

def main():
    total_task = 10
    total_process = 4

    task_distribution_queue_process(total_task, total_process)

if __name__ == "__main__":
    main()


from optparse import OptionParser, OptionGroup
from collections import defaultdict
import os
import sys
from tqdm import tqdm
import itertools
sys.path.append('src')
import utils.file_utils as utils
import fungcat_settings as f_settings
import algorithms.gain_scipy.alg_utils as alg_utils
# import plotting libraries
import matplotlib
matplotlib.use('Agg') # To save files remotely.  Must be before importing matplotlib.pyplot or pylab!
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import numpy as np
import time
import json

# tried to be fancy :P
# colors: https://coolors.co/ef6e4a-0ec9aa-7c9299-5d88d3-96bd33
my_palette = ["#EF6E4A", "#0EC9AA", "#7C9299", "#5D88D3", "#96BD33", "#937860", "#efd2b8"]
# for comparing sinksource with local
#my_palette = ["#EF6E4A", sns.xkcd_rgb["deep sky blue"], "#96BD33", "#937860", "#efd2b8"]
#my_palette = [sns.xkcd_rgb["orange"], sns.xkcd_rgb["azure"], "#96BD33", "#937860", "#efd2b8"]

# these are the main options used to determine which parameters to use for the algorithm
# (unless the plot is specifically comparing parameters, then all will be used)
# for now, just check if the entire string is in the results file. If it is, then use it
# TODO use the JSON file
MAIN_WEIGHT_TYPE = {
    "birgrank": "swsn",
    "sinksource": "swsn",
    "genemania": "swsn",
    "localplus": "swsn",
    }
# TODO make an option for this
MAIN_ALG_OPTS = {
    'sinksource': '-l0-a0_95-eps0_0-maxi10.txt',  # TODO maxi10 at the end also matches maxi1000
    #'sinksource': '-l0-a0_95-eps0_0-maxi20.txt',  # TODO maxi10 at the end also matches maxi1000
    'genemania': "-l0-a0_95-eps0_0-maxi20-tol1e-05",  # TODO remove a, eps and maxi from the output file name
    'birgrank': "-l0-a0_9-eps0_0001-maxi1000-t0_5-m0_5-l0_01",
    'localplus': "-l0-a0_95-eps0_0-maxi10.txt",  # TODO remove a, eps and maxi from the output file name
    }
ALG_NAMES = {
    'localplus': 'Local', 'local': 'Local-',
    'sinksource': 'SinkSource', 'sinksourceplus': 'SinkSource+',
    'genemania': 'GeneMANIA',
    'birgrank': 'BirgRank', 'aptrank': 'AptRank OneWay',
    'sinksource-squeeze': 'SinkSource Squeeze',
    }

def main(versions, **kwargs):
    kwargs['main_weight_type'] = MAIN_WEIGHT_TYPE
    kwargs['main_alg_opts'] = MAIN_ALG_OPTS
    kwargs['alg_names'] = ALG_NAMES

    ev_code_results = get_results(versions, **kwargs)
    results_overview(ev_code_results, **kwargs)

def get_results(versions, **kwargs):
    ev_code_results = {}
    for version in versions:
        for ev_codes, eval_ev_codes, string, h in itertools.product(
                *[kwargs['ev_codes'], kwargs['eval_ev_codes'], kwargs['string_nets']+[''], kwargs['hierarchy']]):
            exp_name = "%s-%s%s%s-%s%s" % (ev_codes, kwargs['size_range'], eval_ev_codes, '-%s'%string if string != '' else '', h.lower(), kwargs['pos_neg_str'])

            if 'string' not in version and \
               ('core' in exp_name or 'all' in exp_name or 'nontransferred' in exp_name):
                continue
            elif 'string' in version and \
                 ('core' not in exp_name and 'all' not in exp_name and 'nontransferred' not in exp_name):
                continue

            df_all = get_results_alg_params(version, exp_name, **kwargs)
            #print(df_all.head())
            ev_code_results[(version, exp_name)] = df_all
    return ev_code_results

#def get_results(version, weight_type, **kwargs):
#main_alg_opts=None, main_weight_type=None, alg_names=None,
def get_results_alg_params(version, exp_name, **kwargs):
    """ Get the fmax results for a given set of algorithm parameters
    """
    # see if these are set
    main_alg_opts = kwargs.pop('main_alg_opts', None)
    main_weight_type = kwargs.pop('main_weight_type', None)
    alg_names = kwargs.pop('alg_names', None)

    results_files = set()
    df_all = pd.DataFrame()
    for alg in kwargs['algorithm']:
        for weight_type, alpha, theta, mu, br_lambda, eps, maxi, tol in itertools.product(
                *[kwargs['weight_type'], kwargs['alpha'], kwargs['theta'], kwargs['mu'], kwargs['br_lambda'], kwargs['eps'], kwargs['max_iters'], kwargs['tol']]):

            # setup the weight_type
            if 'string' in version:
                if alg == 'birgrank' and weight_type in ['', 'gm2008']:
                    weight_type = 'swsn'
            # if the version doesn't have STRING in it, then don't use the weight str
            else:
                weight_type = ""
            # add the dash to the weight str
            weight_str = "-%s"%weight_type if weight_type != "" else ""

            results_file = alg_utils.get_filepath(
                version, alg, exp_name, weight_str=weight_str, exp_type='loso',
                alpha=alpha, theta=theta, mu=mu, br_lambda=br_lambda, eps=eps, max_iters=maxi, tol=tol,
                sinksourceplus_lambda=kwargs['sinksourceplus_lambda'])
            # make sure to not read files multiple times
            # for example: if alg is sinksource, but there are multiple 'tol' parameters,
            # they will be ignored when getting the results file
            if results_file in results_files:
                continue
            results_files.add(results_file)

            if kwargs['verbose']:
                print("Reading results from %s" % (results_file))
            if not os.path.isfile(results_file):
                if kwargs['verbose']:
                    print("\tdoesn't exist. Skipping")
                continue
            df = pd.read_csv(results_file, sep='\t')

            # weight str for non-main weight types
            if weight_type != "":
                if main_weight_type and weight_type == main_weight_type[alg]:
                    weight_str = ""
                else:
                    weight_str = "%s-"%weight_type
            if main_alg_opts and main_weight_type and alg_names and \
               alg in main_alg_opts and main_alg_opts[alg] in results_file and \
               (weight_type == "" or main_weight_type[alg] == weight_type):
                df['Algorithm'] = alg_names[alg]
            # include the alg-specific params in the name
            elif alg == "birgrank":
                df['Algorithm'] = 'birgrank %sa%s-t%s-m%s-l%s' % (weight_str, alpha, theta, mu, br_lambda)
            elif alg == "sinksource":
                df['Algorithm'] = 'sinksource %sa%s-maxi%s' % (weight_str, alpha, maxi)
            elif alg == "genemania":
                df['Algorithm'] = "genemania %stol%s" % (weight_str, tol)
            else:
                df['Algorithm'] = alg

            if kwargs['verbose']:
                print("%d sp-goterm pairs" % (len(df['Algorithm'])))

            df_all = pd.concat((df_all, df))

    for alg in kwargs['algorithm']:
        alg_results = False
        for alg2 in df_all['Algorithm'].unique():
            if alg in alg2 or (alg_names and alg2 == alg_names[alg]):
                alg_results = True
                break
        if alg_results is False:
            "WARNING: no results found for algorithm %s" % (alg)
            time.sleep(2)

    return df_all

def results_overview(ev_code_results, measures=['fmax'], **kwargs):
    if 'main_alg_opts' in kwargs:
        print("main_alg_opts:")
        for alg in kwargs['algorithm']:
            print("\t%s: %s" % (alg, kwargs['main_alg_opts'][alg]))
    for version, exp_name in sorted(ev_code_results, key=lambda x: (x[1], x[0])):
        df_curr = ev_code_results[(version, exp_name)]
        print(version, exp_name)
        # limit the goterms to those that are also present for SinkSource(?)
        for measure in measures:
            for alg in sorted(df_curr['Algorithm'].unique()):
                print("\t%s: %0.3f \t\t(%d sp-goterm pairs)" % (alg, df_curr[df_curr['Algorithm'] == alg][measure].median(), len(df_curr[df_curr['Algorithm'] == alg][measure])))

def setup_optparse():
    ## Parse command line args.
    usage = '%s [options]\n' % (sys.argv[0])
    parser = OptionParser(usage=usage)

    # general parameters
    group = OptionGroup(parser, 'Main Options (all can be comma-delimited)')
    group.add_option('','--version',type='string', default="2018_06-seq-sim-e0_1",
                     help="Version of the PPI to run. Options are: %s." % (', '.join(f_settings.ALLOWEDVERSIONS)))
    #group.add_option('-N','--net-file',type='string',
    #                 help="Network file to use. Can specify one per version. Default is the version's default network")
    group.add_option('-A', '--algorithm', default='sinksource',
                     help="Algorithm for which to get predictions. Default is 'sinksource' of them. Options: '%s'" % ("', '".join(alg_utils.ALGORITHMS)))
    #group.add_option('', '--exp-name', type='string', default='expc-50-1000-bp-use-neg',
    #                 help="Experiment name to used when running algorithm")
    # the experiment name is built with the following:
    # <ev_codes>-<goterm_size_range><eval_ev_codes>-<string_nets>-<h>-<use_neg>-<keep_ann>
    # example1: expc-50-1000-bp-use-neg
    # example2: expc-comp-50-1000iea-core-bp-use-neg-keep-ann
    group.add_option('', '--ev-codes', default='expc',
                     help="Evidence codes which define the pos/neg examples. Default='expc'")
    group.add_option('', '--size-range', default='50-1000',
                     help="GO term size range. Default='50-1000'")
    group.add_option('', '--eval-ev-codes', default='',
                     help="Evidence codes used to evaluate (if different than <ev-codes>. Default=''")
    group.add_option('', '--string-nets', default='core',
                     help="String networks used ('all', 'nontransferred', or 'core'). This will be left blank if string isn't in the version. Default='core'")
    group.add_option('-H', '--hierarchy', default='bp',
                     help="GO hierarchy. Either 'bp' or 'mf' or 'bp,mf'. Default='bp'")
    group.add_option('', '--pos-neg-str', default='-use-neg',
                     help="Either '', '-use-neg', '-non-pos-neg', '-use-neg-keep-ann', or '-non-pos-neg-keep-ann'. Default='-use-neg'")
    parser.add_option_group(group)

    add_alg_opts(parser)
    #run_algs.add_string_opts(parser)

    return parser

def add_alg_opts(parser):
    # parameters for running algorithms
    group = OptionGroup(parser, 'Algorithm options (all can be comma-delimited)')
    # should've put the string-nets and weight-type together...
    group.add_option('', '--weight-type', default="swsn",
                     help="Method used to integrate networks, or setup weights. Can be either '', 'unw', 'gm2008', or 'swsn'. Default='swsn' (for versions that include STRING)")
    group.add_option('-a', '--alpha', default="0.95,0.9",
                     help="Alpha insulation parameter, or restart parameter for BirgRank. Default=0.95")
    group.add_option('', '--eps', default="0.0,0.0001",
                     help="Stopping criteria for SinkSource or BirgRank. Default='0.0,0.0001'")
    group.add_option('', '--max-iters', default="10,20,1000",
                     help="Maximum # of iterations for SinkSource or BirgRank. Default=10,20,1000")
    group.add_option('', '--tol', default="1e-5",
                     help="Tolerance for convergance for the Scipy Sparse Conjugate Gradient solver (for GeneMANIA). Default=1e-05")
    group.add_option('', '--theta', default="0.5",
                     help="BirgRank: (1-theta) percent of Rtrain used in seeding vectors. Default=0.5")
    group.add_option('', '--mu', default="0.5",
                     help="BirgRank: (1-mu) percent of random walkers diffuse from G via Rtrain to H. Default=0.5")
    group.add_option('', '--br-lambda', default="0.01",
                     help="BirgRank: (1-lambda) percent of random walkers which diffuse downwards within H. Default=0.01")
    parser.add_option_group(group)

    # TODO
#    # aptrank options
#    group = OptionGroup(parser, 'AptRank options')
#    group.add_option('', '--apt-k',
#                     help="Markov Chain iterations. Default=8")
#    group.add_option('', '--apt-s',
#                     help="Number of shuffles. Default=5")
#    group.add_option('', '--apt-t',
#                     help="Split percentage. Default=.5")
#    group.add_option('', '--diff-type', default="twoway",
#                     help="Diffusion type: oneway (G to H) or twoway (G to H and H to G). Default=twoway")
#    parser.add_option_group(group)

def validate_opts(opts):
    kwargs = vars(opts)

    #try:
    for opt in ['version', 'algorithm', 'ev_codes', 'eval_ev_codes',
            'string_nets', 'hierarchy', 'weight_type']:
        kwargs[opt] = kwargs[opt].split(',')
    for opt in ['alpha', 'eps', 'tol', 'theta', 'mu', 'br_lambda']:
        kwargs[opt] = [float(param) for param in kwargs[opt].split(',')]
    for opt in ['max_iters']:
        kwargs[opt] = [int(param) for param in kwargs[opt].split(',')]
    #except:
    #    print("")

    for version in kwargs['version']:
        if version not in f_settings.ALLOWEDVERSIONS:
            print("ERROR: '%s' not an allowed version. Options are: %s." % (version, ', '.join(f_settings.ALLOWEDVERSIONS)))
            sys.exit(1)

    #if kwargs.algorithm not in f_settings.ALGORITHM_OPTIONS:
    #    print "--algorithm %s is not a valid algorithm name" % (kwargs.algorithm)
    #    sys.exit(1)

    # TODO
    for alg in kwargs['algorithm']:
        if alg not in alg_utils.ALGORITHMS:
            print("ERROR: '%s' not a valid algorithm name. Algorithms are: '%s'." % (alg, ', '.join(alg_utils.ALGORITHMS)))
            sys.exit(1)

    if kwargs['alg_plot_opts'] is not None:
        # set the gobal parameters
        global MAIN_WEIGHT_TYPE, MAIN_ALG_OPTS, ALG_NAMES
        alg_opts = json.load(open(kwargs['alg_plot_opts']))
        MAIN_WEIGHT_TYPE, MAIN_ALG_OPTS, ALG_NAMES = alg_opts['main_weight_type'], alg_opts['main_alg_opts'], alg_opts['alg_names']

    return kwargs

def add_plot_opts(parser):
    group = OptionGroup(parser, 'Plotting options')
    group.add_option('', '--forceplot', action="store_true", default=False,
                     help="Option to overwrite previous plots")
    group.add_option('', '--for-pub', action="store_true", default=False,
                     help="Option to remove headers and overall make plot nicer for publication")
    group.add_option('', '--postfix', default='',
                     help="str to add to end of filename")
    group.add_option('', '--alg-plot-opts',
                     help="JSON file containing the parameters to use for each algorithm ('main_alg_opts'), weighting type ('main_weight_type'), and name ('alg_names').")
    parser.add_option_group(group)

    group = OptionGroup(parser, 'Additional options')
    group.add_option('', '--unweighted', action="store_true", default=False,
                     help="Option to ignore edge weights when running algorithms. Default=False (weighted)")
    group.add_option('-l', '--sinksourceplus-lambda', type=float,
                     help="lambda parameter to specify the weight connecting the unknowns to the negative 'ground' node. Default=None")
    group.add_option('', '--verbose', action="store_true", default=False,
                     help="Print additional info about running times and such")
    parser.add_option_group(group)

    # TODO make this an option
    #group.add_option('', '--keep-ann', action='store_true', default=False,
    #                 help="Don't leave out annotations when running the algorithms" +
    #                 "TODO allow for the option to run and evaluate everything together")

def parse_args(args):
    parser = setup_optparse()
    add_plot_opts(parser)

    (opts, args) = parser.parse_args(args)
    kwargs = validate_opts(opts)

    return kwargs

if __name__ == "__main__":
    kwargs = parse_args(sys.argv)
    versions = kwargs.pop('version')

    main(versions, **kwargs)


import csv
import concurrent.futures
import nmap
import socket

def resolve_host(domain):
    try:
        return socket.gethostbyname(domain)
    except socket.gaierror:
        return None

def scan_ports(domain):
    nm = nmap.PortScanner()

    host = resolve_host(domain)
    if host is None:
        return None

    # Print a message indicating the domain that is being scanned
    print(f"Scanning domain: {domain}")

    # Perform the port scan, scanning only the top 1000 most common ports
    nm.scan(host, arguments='-sS -T4 --max-retries 1')  # Stealth SYN scan with retries

    scan_results = []
    if host in nm.all_hosts():  # Check if the host is in the scan result
        for protocol in nm[host].all_protocols():
            for port in nm[host][protocol].keys():
                r = nm[host][protocol][port]
                scan_results.append(f"{port}/{protocol}/{r['state']}/{r['name']}")

    return domain, "; ".join(scan_results)

def main(input_file, output_file):
    # Read the input file
    with open(input_file, 'r') as file:
        domains = [line.strip() for line in file]

    # Process each domain
    results = []
    with concurrent.futures.ProcessPoolExecutor() as executor:
        scan_results = list(executor.map(scan_ports, domains))
        for result in scan_results:
            if result is not None:
                results.append(result)

    # Write the output file
    with open(output_file, 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(['Domain', 'Scan Results'])
        writer.writerows(results)

# Run the main function
if __name__ == "__main__":
    main('tester.csv', 'scan_results.csv')


import asyncio
import click
import os
import time
import utils
import uuid
import uvloop
from aiokafka import AIOKafkaProducer, AIOKafkaConsumer

@click.command()
@click.option('--client_type', help='Test: producer or consumer', required=True)
@click.option('--brokers', help='List of brokers.', required=True)
@click.option('--topic', help='Topic to send message to.', required=True)
@click.option('--num_messages', type=click.INT, help='Number of messages to send to broker.', required=True)
@click.option('--msg_size', type=click.INT, help='Size of each message.', required=True)
@click.option('--num_runs', type=click.INT, help='Number of times to run the test.', required=True)
def benchmark(client_type, brokers, topic, num_messages, msg_size, num_runs):
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

    loop = asyncio.get_event_loop()
    if client_type == "producer":
        loop.run_until_complete(_producer_benchmark(brokers, topic, num_messages, msg_size, num_runs))
    else:
        loop.run_until_complete(_consumer_benchmark(brokers, topic, num_messages, msg_size, num_runs))
    loop.close()

async def _producer_benchmark(brokers, topic, num_messages, msg_size, num_runs):
    payload = bytearray(b"m" * msg_size)
    producer_config = dict(
        bootstrap_servers=brokers,
    )

    loop = asyncio.get_event_loop()
    producer = AIOKafkaProducer(loop=loop, **producer_config)
    await producer.start()

    print("Starting benchmark for AIOKafka Producer.")
    run_times = []

    try:
        for _ in range(num_runs):
            run_start_time = time.time()
            await _produce(producer, topic, payload, num_messages)
            run_time_taken = time.time() - run_start_time
            run_times.append(run_time_taken)
    except asyncio.CancelledError:
        pass
    finally:
        await producer.stop()

    utils.print_results(
        "AIOKafka Producer", run_times, num_messages, msg_size
    )

async def _produce(producer, topic, payload, num_messages):
    for _ in range(num_messages):
        await producer.send(topic, payload)

async def _consumer_benchmark(brokers, topic, num_messages, msg_size, num_runs):
    loop = asyncio.get_event_loop()
    consumer = AIOKafkaConsumer(
        topic, group_id=str(uuid.uuid1()),
        auto_offset_reset="earliest",
        enable_auto_commit=False,
        loop=loop
    )

    await consumer.start()

    print("Starting benchmark for AIOKafka Consumer.")
    run_times = []

    try:
        for _ in range(num_runs):
            run_start_time = time.time()
            await _consume(consumer, num_messages)
            run_time_taken = time.time() - run_start_time
            run_times.append(run_time_taken)
    except asyncio.CancelledError:
        pass
    finally:
        await consumer.stop()

    utils.print_results(
        "AIOKafka Consumer", run_times, num_messages, msg_size
    )

async def _consume(consumer, num_messages):
    num_messages_consumed = 0
    async for msg in consumer:
        num_messages_consumed += 1
        if num_messages_consumed > num_messages:
            return

    # msg_set = await consumer.getmany(timeout_ms=1000)
    # if not msg_set:
    #     return

    # for msgs in msg_set.values():
    #     len_msgs = len(msgs)
    #     num_messages_consumed += len_msgs
    # if num_messages_consumed > num_messages:
    #     return

if __name__ == '__main__':
    benchmark()


import paho.mqtt.client as paho
from faker import Faker
import json
import datetime
import time

def on_publish(client,userdata,result):
    print("data published \n")
    print("userdata: " + str(userdata))
    print("result: " + str(result))
    pass

fake = Faker()

requests_per_minute = 30

broker="test.mosquitto.org"
port=1883

deviceId = fake.pyint(min_value=0, max_value=9999, step=1)

# the coordinates are generated in the area of gothenburg

gbLat = 57.710	# latitude of the center of the city of gothenburg
gbLng = 11.973	# longitude of the center of the city of gothenburg

# RUSH HOURS 7 to 9, 15 to 18
client = paho.Client()
client.on_publish = on_publish
client.connect(broker,port,60)

currentTime = datetime.datetime.now()

startTime = datetime.datetime.now()
prevTime = 0

i=0

while True:
	if int((datetime.datetime.now() - startTime).total_seconds()) > prevTime:
		prevTime = int((datetime.datetime.now() - startTime).total_seconds())
		currentTime += datetime.timedelta(minutes=1)

	requestId = i
	i+=1

	epoch = time.time()

	latA = str(fake.coordinate(center=gbLat, radius=0.10))
	lngA = str(fake.coordinate(center=gbLng, radius=0.15))
	latB = str(fake.coordinate(center=gbLat, radius=0.10))
	lngB = str(fake.coordinate(center=gbLng, radius=0.15))

	ts = currentTime.strftime("%Y-%m-%d %H:%M")

	if fake.pyint(min_value=0, max_value=1, step=1) == 0:
		purpose = "work"
	else:
		purpose = "leisure"

	json_data = {
		'deviceId': deviceId,
		'requestId': requestId,
		'origin':{
			'latitude': latA,
			'longitude': lngA
		},
		'destination':{
			'latitude': latB,
			'longitude': lngB
		},
		'timeOfDeparture': ts,
		'purpose': purpose,
		'issuance' : epoch
	}

	print(json.dumps(json_data, indent=4, sort_keys=True))

	time.sleep(60/requests_per_minute)
	client.publish("external", json.dumps(json_data))

	#hour = currentTime.hour
	#if (hour >= 7 and hour < 9) or (hour >= 15 and hour < 18):
	#	sleepTime = 0.1
	#else:
	#	sleepTime = 1

	#time.sleep(sleepTime)

client.disconnect()


# -*- coding: utf-8 -*-
from selenium import webdriver

from warnings import filterwarnings
from bs4 import BeautifulSoup
from pathlib import Path
from time import sleep
from PIL import Image
import os, shutil, uuid

# variáveis globais
_base = Path('.')

# mensagens de erro curtas, relevantes e genéricas
# para serem reutilizáveis
_errors = {
    "ndownload": "Não baixou .exe do driver",
    "nfound": "Chrome não estava em C:\\Program Files",
    "ntype": "Tipo de driver informado não foi encontrado",
    "nclose": f"Existe um .exe do driver na pasta {os.getcwd()} ?",
}

# Obtém a url de download do pahntomjs e o nome do arquivo
def get_url_phantomjs(content):
    tipo = 'linux-64-bit' if os.name == 'posix' else 'windows'
    soup = BeautifulSoup(content, 'html.parser')

    url = soup.find('h2', attrs={'id': tipo})
    url = url.find_next("a").get('href', '')

    return url, url.split('/')[-1]

# download versão mais recente do phantomjs
def download_phantomjs():
    import requests, wget, tarfile, zipfile
    print(">>> download phantom js")

    res = requests.get('https://phantomjs.org/download.html')
    url, file = get_url_phatomjs(res.content)
    if not all((url, file)): return False

    phantomzip = _base / file
    if phantomzip.is_file(): os.remove(phantomzip)

    wget.download(url, str(phantomzip))

    path_driver = _base / 'ignore' / 'driver'
    os.makedirs(path_driver, exist_ok=True)

    if os.name == 'posix':
        arq = tarfile.open(phantomzip, 'r:bz2')
        driver, aux = 'phantomjs', 'linux64'
    else:
        arq = zipfile.ZipFile(phantomzip, 'r')
        driver, aux = 'phantomjs.exe', 'win32'

    root_file = file.rstrip('.zip').rstrip('.tar.bz2')
    path_file = str(Path(root_file, 'bin', driver))

    # zip files sempre usam slash / como separador
    #
    arq.extract(f'{root_file}/bin/{driver}')
    arq.close()

    shutil.move(path_file, (path_driver / f'{aux}-{driver}'))
    shutil.rmtree(root_file)

    os.remove(phantomzip)
    return True
_download_phantomjs = download_phantomjs

# Cria uma instância de webdriver utilizando o phantom js com
# um certificado p12 'cert' caso haja, cabeçalho 'headers' caso haja
# e opções 'args' caso haja mais a opções default '--ignore-ssl-errors=true'
# Retorna True e a instância webdriver criada caso sucesso
# Retorna False e uma mensagem caso falha
def exec_phantomjs(cert=None, user_args=None, headers=None, size=(1600, 900)):
    print(">>> inicializando phantom js")
    filterwarnings('ignore')

    if headers:
        for key, value in enumerate(headers):
            capability_key = 'phantomjs.page.customHeaders.{}'.format(key)
            webdriver.DesiredCapabilities.PHANTOMJS[capability_key] = value

    args = ['--ignore-ssl-errors=true']
    if user_args: args.extend(user_args)

    if cert: args.append(f'--ssl-client-certificate-file={cert}')

    exe = check_driver_exe('phantom')
    if exe:
        driver = webdriver.PhantomJS(executable_path=exe, service_args=args)
        driver.set_window_size(*size)
        driver.delete_all_cookies()
        return driver

    try:
        download_phantomjs()
    except PermissionError:
        return _errors["nclose"]
    except Exception as e:
        print(str(e))
        return _errors["ndownload"]

    exe = check_driver_exe('phantom')
    if not exe: return _errors['ntype']

    driver = webdriver.PhantomJS(executable_path=exe, service_args=args)
    driver.set_window_size(*size)
    driver.delete_all_cookies()
    return driver
_exec_phantomjs = exec_phantomjs


import pathlib
import unittest as ut

# Must import this for logging.
import configuration as cfg
import fixtures

class MinerFTSBasicTest(fixtures.TrackerMinerFTSTest):
    """
    Tests different contents in a single file
    """

    def test_01_single_word(self):
        TEXT = "automobile"
        self.basic_test(TEXT, TEXT)

    def test_02_multiple_words(self):
        TEXT = "automobile with unlimited power"
        self.set_text(TEXT)

        results = self.search_word("automobile")
        self.assertEqual(len(results), 1)
        self.assertIn(self.uri(self.testfile), results)

        results = self.search_word("unlimited")
        self.assertEqual(len(results), 1)
        self.assertIn(self.uri(self.testfile), results)

    def test_03_long_word(self):
        # TEXT is longer than the 200 characters specified in the fts configuration
        TEXT = "ai" * 200
        self.set_text(TEXT)

        results = self.search_word(TEXT)
        self.assertEqual(len(results), 0)

    def test_04_non_existent_word(self):
        TEXT = "This a trick"
        self.set_text(TEXT)
        results = self.search_word("trikc")
        self.assertEqual(len(results), 0)

    def test_05_word_multiple_times_in_file(self):
        TEXT = "automobile is red. automobile is big. automobile is great!"
        self.basic_test(TEXT, "automobile")

    def test_06_sentence(self):
        TEXT = "plastic is fantastic"
        self.basic_test(TEXT, TEXT)

    def test_07_partial_sentence(self):
        TEXT = "plastic is fantastic"
        self.basic_test(TEXT, "is fantastic")

    @ut.skip("Currently fails with: fts5: syntax error near '.'")
    def test_08_strange_word(self):
        # FIXME Not sure what are we testing here
        TEXT = "'summer.time'"
        self.basic_test(TEXT, "summer.time")

    # Skip the test 'search for .'

    def test_09_mixed_letters_and_numbers(self):
        TEXT = "abc123"
        self.basic_test(TEXT, "abc123")

    @ut.skip(
        "We don't ignore numbers by default since https://gitlab.gnome.org/GNOME/tracker/merge_requests/172."
    )
    def test_10_ignore_numbers(self):
        TEXT = "palabra 123123"
        self.set_text(TEXT)
        results = self.search_word("123123")
        self.assertEqual(len(results), 0)

class MinerFTSWAllowlistTest(fixtures.TrackerMinerFTSTest):
    """
    Test that only an allowlisted set of plaintext files get FTS indexed.

    We can create performance problems by indexing big trees of sourcecode,
    video game data, etc., so we use an extension-based allowlist to limit
    what gets FTS indexed.
    """

    def test_no_extension_file(self):
        self.testfile = "test-monitored/miner-fts-test-no-extension"

        path = pathlib.Path(self.path(self.testfile))
        url = self.uri(self.testfile)
        expected = f"a nfo:Document; nie:isStoredAs <{url}>"
        with self.tracker.await_insert(
            fixtures.DOCUMENTS_GRAPH, expected, timeout=cfg.AWAIT_TIMEOUT
        ):
            path.write_text("Definitely do not index this file.")

        # No results for full text search.
        results = self.search_word("Definitely")
        self.assertEqual(len(results), 0)

        # However, there should be an nfo:PlainTextDocument resource as normal.
        self.assertEqual(1, self.tracker.count_instances("nfo:PlainTextDocument"))

if __name__ == "__main__":
    fixtures.tracker_test_main()


from rest_framework import serializers
from .models import User

class TinyUserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = (
            "name",
            "avatar",
            "username",
        )

class PrivateUserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        exclude = ("password", "is_superuser", "id", "is_staff", "is_active",
                   "first_name", "last_name", "groups", "user_permissions",)

class PublucUserSerializer(serializers.ModelSerializer):
    total_reviews = serializers.SerializerMethodField()
    total_rooms = serializers.SerializerMethodField()

    class Meta:
        model = User
        include = ("name",
                   "gender",
                   "avatar",
                   "username",
                   "language",
                   "total_reviews",
                   "total_rooms",)
        # review몇개만 보여주기 가능한지.

    def get_total_reviews(self, user):
        return user.total_reviews()

    def get_total_rooms(self, user):
        return user.total_rooms()


# -*- coding: utf-8 -*-
# 读写2003 excel
import openpyxl
import xlwt
import xlrd
from xlutils.copy import copy
# 读写2007 excel

def write03Excel(path):
    book = xlwt.Workbook()
    sheet1 = book.add_sheet("2003测试表")
    value = [["1,1", "1,2", "1,3"],
             ["2,1", "2,2", "2,3"],
             ["3,1","3,2","3,3"]]
    for i in range(0, len(value)):
        for j in range(0, len(value[i])):
            sheet1.write(i, j, value[i][j])
    book.save(path)
    print("写入数据成功！")

def read03Excel(path):
    book = xlrd.open_workbook(path)
    sheet1 = book.sheet_names()
    worksheet = book.sheet_by_name(sheet1[0])
    for i in range(0, worksheet.nrows):
        row = worksheet.row(i)
        for j in range(0, worksheet.ncols):
            print(worksheet.cell_value(i, j), "\t", end="")
        print()

import openpyxl

def write07Excel(path):
    book = openpyxl.Workbook()
    sheet1 = book.active
    sheet1.title = 'Sheet1'

    value = [["1,1", "1,2", "1,3"],
             ["2,1", "2,2", "2,3"],
             ["3,1","3,2","3,3"]]
    for i in range(0, len(value)):
        for j in range(0, len(value[i])):
            sheet1.cell(row=i+1, column=j+1, value=str(value[i][j]))

    book.save(path)
    print("写入数据成功！")

def read07Excel(path):
    book = openpyxl.load_workbook(path)
    sheet1 = book.get_sheet_by_name('Sheet1')

    for row in sheet1.rows:
        for cell in row:
            print(cell.value, "\t", end="")
        print()

def excelwrite(filename,value):
    workbook = xlrd.open_workbook(filename)
    sheet = workbook.sheet_by_index(0)
    rowNum = sheet.nrows
    colNum = sheet.ncols
    newbook = copy(workbook)
    newsheet = newbook.get_sheet(0) # 在末尾增加新行
    for i in range(0,len(value)):
        newsheet.write(rowNum, i, value[i]) # 覆盖保存
    newbook.save(filename)

def add07Excel(path,value):
    wb = openpyxl.load_workbook(path)
    ws = wb['Sheet1']
    #for x in data:
    ws.append(value)
    wb.save(path)
    print("写入成功")

file_2003 = 'test/2003.xls'
file_2007 = 'test/2007.xlsx'

write03Excel(file_2003)
read03Excel(file_2003)

write07Excel(file_2007)
read07Excel(file_2007)


# General imports
import sys

# Local imports
from common.JSONWriter import JSONWriter

# Global values for error handling
out_writer = JSONWriter()  # Serves to hold writer object which can be set by set_error_JSONWriter

# Serves to set json writer object to use when writing output
def set_error_json_writer(json_writer: JSONWriter = None):
    global out_writer
    out_writer = json_writer

# Serves to write an "error" JSON object to an output file with a variable error message. Exits -1 on completion.
def handle_error(error_message: str):
    print(error_message)  # Print error message prior to writing it to output file
    if out_writer.output_file is None:
        print('No output file specified for JSONWriter in handle_errors.py. '
              'Use set_error_json_writer to provide valid JSONWriter.')
    error_json = {"error": error_message}  # Create error json object
    out_writer.write_json_to_output_file(error_json)  # Send object to be written to output file
    sys.exit(-1)  # Exit with error