import json
import os.path
import pickle
import random
import sys
from datetime import datetime

import numpy
from gensim.models import Word2Vec
from keras.layers import Dense, LSTM, GRU, BatchNormalization
from keras.models import Sequential
from keras.preprocessing import sequence
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.utils import class_weight

import helper

class OSVModelBuilder:
    def __init__(self):
        self.mode = "sql"

        self.restriction = [20000, 5, 6, 10]
        self.step = 5
        self.full_length = 200
        self.mode2 = f'{str(self.step)}_{str(self.full_length)}'

        self.min_count = 10
        self.iteration = 300
        self.s = 200
        self.w = "withString"

        self.w2v = f"word2vec_{self.w}{str(self.min_count)}-{str(self.iteration)}-{str(self.s)}"
        self.w2vmodel = f"w2v/{self.w2v}.model"
        self.w2v_model = None
        self.word_vectors = None

        self.all_blocks = []

        self.train_X = []
        self.train_Y = []
        self.validate_X = []
        self.validate_Y = []
        self.final_test_X = []
        self.final_test_Y = []

    def load_data(self):
        if len(sys.argv) > 1:
            self.mode = sys.argv[1]

        if not (os.path.isfile(self.w2vmodel)):
            print("word2vec model is still being created...")
            sys.exit()

        self.w2v_model = Word2Vec.load(self.w2vmodel)
        self.word_vectors = self.w2v_model.wv

        # with open(self.w2vmodel, 'rb') as model_file:
        #     self.w2v_model = pickle.load(model_file)
        # self.w2v_model = numpy.load(self.w2vmodel, allow_pickle=True)

        # w = KeyedVectors.load_word2vec_format(self.w2vmodel, binary=True)
        # self.w2v_model = KeyedVectors.load_word2vec_format(self.w2vmodel, binary=True)

        with open(f'data/plain_{self.mode}', 'r') as infile:
            data = json.load(infile)

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("finished loading. ", now_format)

        progress = 0
        count = 0
        for r in data:
            progress += 1

            for c in data[r]:

                if "files" in data[r][c]:
                    #  if len(data[r][c]["files"]) > self.restriction[3]:
                    #    continue

                    for f in data[r][c]["files"]:

                        #      if len(data[r][c]["files"][f]["changes"]) >= self.restriction[2]:
                        #         continue

                        if "source" not in data[r][c]["files"][f]:
                            continue

                        if "source" in data[r][c]["files"][f]:
                            sourcecode = data[r][c]["files"][f]["source"]
                            #     if len(sourcecode) > self.restriction[0]:
                            #       continue

                            all_bad_parts = []

                            for change in data[r][c]["files"][f]["changes"]:
                                bad_parts = change["badparts"]
                                count += len(bad_parts)

                                #     if len(badparts) > self.restriction[1]:
                                #       break

                                for bad in bad_parts:
                                    pos = helper.find_position(bad, sourcecode)
                                    if not -1 in pos:
                                        all_bad_parts.append(bad)

                            #   if (len(allbadparts) > self.restriction[2]):
                            #     break

                            if len(all_bad_parts) > 0:
                                #   if len(allbadparts) < self.restriction[2]:
                                positions = helper.find_positions(all_bad_parts, sourcecode)
                                blocks = helper.get_blocks(sourcecode, positions, self.step, self.full_length)

                                for b in blocks:
                                    self.all_blocks.append(b)

    def prepare_learning_dataset(self):
        keys = []

        for i in range(len(self.all_blocks)):
            keys.append(i)
        random.shuffle(keys)

        cutoff = round(0.7 * len(keys))
        cutoff2 = round(0.85 * len(keys))

        keys_train = keys[:cutoff]
        keys_test = keys[cutoff:cutoff2]
        keys_final_test = keys[cutoff2:]

        print("cutoff " + str(cutoff))
        print("cutoff2 " + str(cutoff2))

        with open(f'data/{self.mode}_dataset_keystrain', 'wb') as fp:
            pickle.dump(keys_train, fp)
        with open(f'data/{self.mode}_dataset_keystest', 'wb') as fp:
            pickle.dump(keys_test, fp)
        with open(f'data/{self.mode}_dataset_keysfinaltest', 'wb') as fp:
            pickle.dump(keys_final_test, fp)

        print(f"Creating training dataset... ({self.mode})")
        for k in keys_train:
            block = self.all_blocks[k]
            code = block[0]
            token = helper.get_tokens(code)
            vectorlist = []
            for t in token:
                if t in self.word_vectors.vocab and t != " ":
                    vector = self.w2v_model[t]
                    vectorlist.append(vector.tolist())
            self.train_X.append(vectorlist)
            self.train_Y.append(block[1])

        print("Creating validation dataset...")
        for k in keys_test:
            block = self.all_blocks[k]
            code = block[0]
            token = helper.get_tokens(code)
            vectorlist = []
            for t in token:
                if t in self.word_vectors.vocab and t != " ":
                    vector = self.w2v_model[t]
                    vectorlist.append(vector.tolist())
            self.validate_X.append(vectorlist)
            self.validate_Y.append(block[1])

        print("Creating finaltest dataset...")
        for k in keys_final_test:
            block = self.all_blocks[k]
            code = block[0]
            token = helper.get_tokens(code)
            vectorlist = []
            for t in token:
                if t in self.word_vectors.vocab and t != " ":
                    vector = self.w2v_model[t]
                    vectorlist.append(vector.tolist())
            self.final_test_X.append(vectorlist)
            self.final_test_Y.append(block[1])

        print("Train length: " + str(len(self.train_X)))
        print("Test length: " + str(len(self.validate_X)))
        print("Finaltesting length: " + str(len(self.final_test_X)))
        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("time: ", now_format)

        with open(f'data/plain_{self.mode}_dataset-train-X_{self.w2v}__{self.mode2}', 'wb') as fp:
            pickle.dump(self.train_X, fp)
        with open(f'data/plain_{self.mode}_dataset-train-Y_{self.w2v}__{self.mode2}', 'wb') as fp:
            pickle.dump(self.train_Y, fp)
        with open(f'data/plain_{self.mode}_dataset-validate-X_{self.w2v}__{self.mode2}', 'wb') as fp:
            pickle.dump(self.validate_X, fp)
        with open(f'data/plain_{self.mode}_dataset-validate-Y_{self.w2v}__{self.mode2}', 'wb') as fp:
            pickle.dump(self.validate_Y, fp)
        with open(f'data/{self.mode}_dataset_finaltest_X', 'wb') as fp:
            pickle.dump(self.final_test_X, fp)
        with open(f'data/{self.mode}_dataset_finaltest_Y', 'wb') as fp:
            pickle.dump(self.final_test_Y, fp)
        print("saved finaltest.")

    def processing_LSTM_and_GRU_model(self):
        X_train = numpy.array(self.train_X)
        y_train = numpy.array(self.train_Y)
        X_test = numpy.array(self.validate_X)
        y_test = numpy.array(self.validate_Y)
        X_finaltest = numpy.array(self.final_test_X)
        y_finaltest = numpy.array(self.final_test_Y)

        for i in range(len(y_train)):
            y_train[i] = 1 if y_train[i] == 0 else 0

        for i in range(len(y_test)):
            y_test[i] = 1 if y_test[i] == 0 else 0

        for i in range(len(y_finaltest)):
            y_finaltest[i] = 1 if y_finaltest[i] == 0 else 0

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("numpy array done. ", now_format)

        print(str(len(X_train)) + " samples in the training set.")
        print(str(len(X_test)) + " samples in the validation set.")
        print(str(len(X_finaltest)) + " samples in the final test set.")

        csum = sum(y_train)
        print("percentage of vulnerable samples: " + str(int((csum / len(X_train)) * 10000) / 100) + "%")

        test_vul = sum(1 for y in y_test if y == 1)
        print("absolute amount of vulnerable samples in test set: " + str(test_vul))

        max_length = self.full_length

        # parameters
        dropout = 0.2
        neurons = 100
        optimizer = "adam"
        epochs = 100
        batch_size = 128

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("Starting LSTM: ", now_format)

        print("Dropout: " + str(dropout))
        print("Neurons: " + str(neurons))
        print("Optimizer: " + optimizer)
        print("Epochs: " + str(epochs))
        print("Batch Size: " + str(batch_size))
        print("max length: " + str(max_length))

        X_train = sequence.pad_sequences(X_train, maxlen=max_length)
        X_test = sequence.pad_sequences(X_test, maxlen=max_length)
        X_finaltest = sequence.pad_sequences(X_finaltest, maxlen=max_length)

        # creating LSTM model
        model_LSTM = Sequential()
        model_LSTM.add(LSTM(neurons, dropout=dropout, recurrent_dropout=dropout))
        model_LSTM.add(Dense(1, activation='sigmoid'))
        model_LSTM.compile(loss=helper.f1_loss, optimizer='adam', metrics=[helper.f1])

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("Compiled LSTM: ", now_format)

        class_weights_LSTM = class_weight.compute_class_weight('balanced', numpy.unique(y_train), y_train)

        history_LSTM = model_LSTM.fit(
            X_train,
            y_train,
            epochs=epochs,
            batch_size=batch_size,
            class_weight=class_weights_LSTM
        )

        # creating GRU model
        model_GRU = Sequential()
        model_GRU.add(GRU(neurons, dropout=dropout, recurrent_dropout=dropout))
        model_GRU.add(BatchNormalization())
        model_GRU.add(Dense(1, activation='sigmoid'))
        print(model_GRU.summary())

        model_GRU.compile(
            loss=helper.f1_loss,
            optimizer="sgd",
            metrics=[helper.f1],
        )

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print("Compiled GRU: ", now_format)

        class_weights_GRU = class_weight.compute_class_weight('balanced', numpy.unique(y_train), y_train)

        history_GRU = model_GRU.fit(
            X_train,
            y_train,
            epochs=epochs,
            batch_size=batch_size,
            class_weight=class_weights_GRU
        )

        # evaluation data
        for dataset in ["train", "test", "finaltest"]:
            print(f"Now predicting on {dataset} set ({str(dropout)} dropout)")

            if dataset == "train":
                yhat_classes_LSTM = model_LSTM.predict_classes(X_train, verbose=0)
                yhat_classes_GRU = model_GRU.predict_classes(X_train, verbose=0)
                accuracy_LSTM = accuracy_score(y_train, yhat_classes_LSTM)
                precision_LSTM = precision_score(y_train, yhat_classes_LSTM)
                recall_LSTM = recall_score(y_train, yhat_classes_LSTM)
                F1Score_LSTM = f1_score(y_train, yhat_classes_LSTM)
                accuracy_GRU = accuracy_score(y_train, yhat_classes_GRU)
                precision_GRU = precision_score(y_train, yhat_classes_GRU)
                recall_GRU = recall_score(y_train, yhat_classes_GRU)
                F1Score_GRU = f1_score(y_train, yhat_classes_GRU)

            if dataset == "test":
                yhat_classes_LSTM = model_LSTM.predict_classes(X_test, verbose=0)
                yhat_classes_GRU = model_GRU.predict_classes(X_test, verbose=0)
                accuracy_LSTM = accuracy_score(y_test, yhat_classes_LSTM)
                precision_LSTM = precision_score(y_test, yhat_classes_LSTM)
                recall_LSTM = recall_score(y_test, yhat_classes_LSTM)
                F1Score_LSTM = f1_score(y_test, yhat_classes_LSTM)
                accuracy_GRU = accuracy_score(y_test, yhat_classes_GRU)
                precision_GRU = precision_score(y_test, yhat_classes_GRU)
                recall_GRU = recall_score(y_test, yhat_classes_GRU)
                F1Score_GRU = f1_score(y_test, yhat_classes_GRU)

            if dataset == "finaltest":
                yhat_classes_LSTM = model_LSTM.predict_classes(X_finaltest, verbose=0)
                yhat_classes_GRU = model_GRU.predict_classes(X_finaltest, verbose=0)
                accuracy_LSTM = accuracy_score(y_finaltest, yhat_classes_LSTM)
                precision_LSTM = precision_score(y_finaltest, yhat_classes_LSTM)
                recall_LSTM = recall_score(y_finaltest, yhat_classes_LSTM)
                F1Score_LSTM = f1_score(y_finaltest, yhat_classes_LSTM)
                accuracy_GRU = accuracy_score(y_finaltest, yhat_classes_GRU)
                precision_GRU = precision_score(y_finaltest, yhat_classes_GRU)
                recall_GRU = recall_score(y_finaltest, yhat_classes_GRU)
                F1Score_GRU = f1_score(y_finaltest, yhat_classes_GRU)

            print("Accuracy LSTM: " + str(accuracy_LSTM))
            print("Precision LSTM: " + str(precision_LSTM))
            print("Recall LSTM: " + str(recall_LSTM))
            print('F1 score LSTM: %f' % F1Score_LSTM)
            print("Accuracy GRU: " + str(accuracy_GRU))
            print("Precision GRU: " + str(precision_GRU))
            print("Recall GRU: " + str(recall_GRU))
            print('F1 score GRU: %f' % F1Score_GRU)
            print("\n")

        now = datetime.now()
        now_format = now.strftime("%H:%M")
        print(f"saving LSTM model {self.mode}. {now_format}")
        model_LSTM.save(f'model/LSTM_model_{self.mode}.h5')
        model_GRU.save(f'model/GRU_model_{self.mode}.h5')
        print("\n\n")

builder = OSVModelBuilder()
builder.load_data()
builder.prepare_learning_dataset()
builder.processing_LSTM_and_GRU_model()


from app01 import views
from app02 import views as app02_views
from app02.models import Department, Employee, PrettyNum
from django.contrib import admin
from django.urls import path, include
from django.contrib.auth.models import User
from rest_framework import routers, serializers, viewsets

class UserSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = User
        fields = ['url', 'username', 'email', 'is_staff']

# ViewSets define the view behavior.

class UserViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UserSerializer

class EmployeeSerializer(serializers.ModelSerializer):
    class Meta:
        model = Employee
        fields = ['url', 'name', 'age', 'account',
                  'password', 'hiredate', 'gender', 'depart']

class EmployeeViewSet(viewsets.ModelViewSet):
    queryset = Employee.objects.all()
    serializer_class = EmployeeSerializer

class DepartmentSerializer(serializers.ModelSerializer):
    class Meta:
        model = Department
        fields = ['url', 'title', 'add_date', 'mod_date']

class DepartmentViewSet(viewsets.ModelViewSet):
    queryset = Department.objects.all()
    serializer_class = DepartmentSerializer

class PrettyNumSerializer(serializers.ModelSerializer):
    class Meta:
        model = PrettyNum
        fields = ['url', 'mobile', 'price', 'level', 'status']

class PrettyNumViewSet(viewsets.ModelViewSet):
    queryset = PrettyNum.objects.all()
    serializer_class = PrettyNumSerializer

# Routers provide an easy way of automatically determining the URL conf.
router = routers.DefaultRouter()
router.register(r'users', UserViewSet)
router.register(r'employee', EmployeeViewSet)
router.register(r'department', DepartmentViewSet)
router.register(r'pretty_num', PrettyNumViewSet)

urlpatterns = [
    path('admin/', admin.site.urls),
    # DRF
    path('', include(router.urls)),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),

    # path('', RedirectView.as_view(url='employee/list')),
    path('home/', views.home_index),
    path('home/templates', views.home_templates),
    path('grammar/summary', views.grammar_summary),

    # 登录案例
    path('user/add', views.user_add),
    path('user/list', views.user_list),
    path('user/login', views.user_login),
    path('user/delete', views.user_delete),

    # FBV 模式
    path('dept/list', app02_views.dept_index),
    path('employee/list', app02_views.employee_index),
    path('employee/add', app02_views.employee_add_user_form),
    path('employee/<int:nid>/edit', app02_views.employee_edit_user),
    path('employee/<int:nid>/delete', app02_views.employee_delete_user),

    # CBV 模式 - mobile
    path('mobile/list', app02_views.Mobile.as_view()),
    path('mobile/<int:nid>', app02_views.mobile_list),
    # path('mobile/<int:nid>', app02_views.mobile_form_prompt),

    # API
    path('api/dept', app02_views.dept),
    path('api/employee/add_user', app02_views.employee_add_user)
]


import sqlite3
from demo_data import school, students

database_path = 'db/database.db'

def tableupdate(name, lat, lng, isdriver, exptdif, drvrcap, drvrcrd, pssngrdrvr, d_time):
    table = sqlite3.connect(database_path)
    t = table.cursor()

    def findname(name):
        find = False
        cursor = t.execute("SELECT NAME FROM PERSONAL_INFO")
        for row in cursor:
            if row[0] == name:
                find = True
        return find

    def createnew():
        if findname(name) == False:
            t.execute(f"INSERT INTO PERSONAL_INFO(NAME, LAT, LNG, ISDRVR, EXPTDIF, DRVRCAP, DRVRCRD, PSSNGRDRVR, DPRTRT)\
                VALUES (? , ?, ?, ?, ?, ?, ?,?,?); ", (name, lat, lng, isdriver, exptdif, drvrcap, drvrcrd, pssngrdrvr, d_time))
            table.commit()

    def infoupdate():
        if findname(name) == True:
            print('updated')
            print(lat)
            sql = f"UPDATE PERSONAL_INFO SET \
                NAME = \'{name}\', \
                LAT = {lat}, \
                LNG = {lng}, \
                ISDRVR = {isdriver}, \
                EXPTDIF = {exptdif}, \
                DRVRCAP = {drvrcap}, \
                DRVRCRD = {drvrcrd}, \
                PSSNGRDRVR = \'{pssngrdrvr}\', \
                DPRTRT = \'{d_time}\'\
                WHERE NAME = \'{name}\';"
            print(sql)
            t.execute(sql)
            table.commit()

    createnew()
    infoupdate()
    table.close()

def all_passengers_of_a_driver(drivername):
    table = sqlite3.connect(database_path)
    t = table.cursor()

    def find_passenger_driver(drivername):
        namelist = ''
        cursor = t.execute("SELECT NAME,PSSNGRDRVR FROM PERSONAL_INFO ")
        for rows in cursor:
            if rows[1] == drivername:
                namelist += rows[0]
                namelist += ', '
        return namelist

    return find_passenger_driver(drivername)

def name():
    table = sqlite3.connect(database_path)
    t = table.cursor()

def tablereturn():
    table = sqlite3.connect(database_path)
    t = table.cursor()
    csr = t.execute("SELECT * FROM PERSONAL_INFO")
    res = []
    for rows in csr:
        res.append(dict(zip(["NAME", "LAT", "LNG", "ISDRVR", "EXPTDIF",
                             "DRVRCAP", "DRVRCRD", "PSSNGRDRVR", "DPRTRT"], rows)))

    return res

if __name__ == '__main__':
    rows = [list(student.values()) for student in students]

    for row in rows:
        row.pop(0)
        tableupdate(*tuple(row))
    print(tablereturn())


# -----------------------------------------------------------------------------------------------------------------------
# - In this script we will get all the reimbursements data and then we will enrich
#   them with more information about the supplier for that reimbursement in order to
#   know the owners of the establishment.
# - We will use an API called "Receita WS", where we can use the of the establishment
#   to get this extra information that we need.
# -----------------------------------------------------------------------------------------------------------------------
import os
import json
import time
import requests
import numpy as np
import pandas as pd
from pymongo import MongoClient
from serenata_toolbox.chamber_of_deputies.reimbursements import Reimbursements

# ---------------------------------------------------------
# Configure a database to store the receita_ws responses
"""
As there are a lot of different CNPJs to be checked and the
Receita WS API has a limit of requests per month, we are
going to store the response for every CNPJ we check in a
MongoDB database so we can freely review that data later.
"""
# ---------------------------------------------------------

MONGO_CONNECTION_URL = os.environ['MONGO_DB__CONNECTION_URL']
MONGO_CLUSTER_NAME = os.environ['MONGO_DB__CLUSTER_NAME']
MONGO_COLLECTION = os.environ['MONGO_DB__COLLECTION_NAME']

client = MongoClient(MONGO_CONNECTION_URL)
db = client[MONGO_CLUSTER_NAME]
receita_colection = db[MONGO_COLLECTION]

print(f'-> MongoDB connected to cluster "{MONGO_CLUSTER_NAME}" - collection "{MONGO_COLLECTION}"')

# --------------------------------------------------------------------
# Get the reimbursements from 2018 from Serenata de Amor Operation
# --------------------------------------------------------------------
fetch_chamber_reimbursements = Reimbursements('2018', 'data/')
fetch_chamber_reimbursements()
reimbursements_raw_df = pd.read_csv(
    'data/reimbursements-2018.csv',
    dtype={
        'applicant_id': np.str,
        'cnpj_cpf': np.str,
        'congressperson_id': np.str,
        'leg_of_the_trip': np.str,
        'subquota_group_description': np.str,
        'subquota_number': np.str,
    }
)
print('-> Reimbursements dataframe imported from Serenata Toolbox: Shape {}'.format(reimbursements_raw_df.shape))

# --------------------------------------------------
# Get a list with the CNPJs that we need to process
# --------------------------------------------------

# First let's get all the different CNPJs in the dataset
cnpjs_list = reimbursements_raw_df.cnpj_cpf.unique()

def remove_already_processed_cnpjs(cnpjs_series):
    # for all the cnpj's already processed
    for doc in receita_colection.find():
        # if there was no problem with the cnpj
        if not doc.get("retry"):
            # remove that cnpj from the list of those that we will process
            cnpjs_series = cnpjs_series[cnpjs_series != doc.get("raw_cnpj")]
    return cnpjs_series

# And them let's keep only the ones that we still have no information stored in the database
cnpjs_list = remove_already_processed_cnpjs(cnpjs_list)

print()
print("- Amount of different CNPJs in the reimbursements data frame: {}".format(len(cnpjs_list)))
print("- Amount of different CNPJs that still need to be processed: {}".format(len(cnpjs_list)))
print()

# --------------------------------------------------
# Using the Receita WS API
# --------------------------------------------------

RECEITA_API_AUTH_TOKEN = f'Bearer {os.environ["RECEITA_WS_API__AUTH_TOKEN"]}'

processed_cnpjs = 0
total_cnpjs = len(cnpjs_list)

print('STARTING TO FETCH')
print('--------------------')
for cnpj in cnpjs_list:
    processed_cnpjs += 1
    print(f'[{processed_cnpjs}/{total_cnpjs}] CNPJ: {cnpj}')

    time.sleep(30)  # Delay 30 seconds so the API don't block our request
    try:
        # perform API request
        response = requests.get(
            f'https://www.receitaws.com.br/v1/cnpj/{cnpj}',
            headers={'Authorization': RECEITA_API_AUTH_TOKEN}
        )
        # insert data into database
        if response.status_code == 200:
            response_as_json = json.loads(response.text)
            response_as_json["raw_cnpj"] = cnpj
            receita_colection.insert_one(response_as_json)

            # Report status
            if len(response_as_json.get('qsa', [])) != 0:
                print(f'\t- [SUCCESS] CNPJ: {cnpj}')
            elif response_as_json.get('message'):
                print(f'\t- [INVALID] CNPJ: {cnpj} | Message: {response_as_json.get("message")}')
            else:
                print(f'\t- [MISSING OWNERS] CNPJ: {cnpj} | Company name: "{response_as_json.get("nome")}"')
        else:
            response_as_json = {
                'retry': response.status_code,
                'raw_cnpj': cnpj
            }
            print(f'\t- [REQUEST ERROR] CNPJ: {cnpj} | Status code: {response.status_code}')
            receita_colection.insert_one(response_as_json)
            time.sleep(10)  # Delays for 10 additional seconds.
    except Exception as e:
        print("\t- [EXCEPTION] {}".format(e))
        time.sleep(5)  # Delays for 5 additional seconds.

# Report
print()
print('-' * 30)
print('FINISHED DATA FETCHING')
print()
print("- TOTAL DOCS IN DATABASE: {}\n".format(receita_colection.count_documents({})))
print("-> Amount of success........................: {}".format(receita_colection.count_documents({"$and": [{"qsa": {"$exists": True}}, {"qsa": {"$not": {"$size": 0}}}]})))
print("-> Amount of invalid cnpj's.................: {}".format(receita_colection.count_documents({"message": {"$exists": True}})))
print("-> Amount of cnpj's missing owners info.....: {}".format(receita_colection.count_documents({"qsa": {"$size": 0}})))
print("-> Amount of errors with the cnpj request...: {}".format(receita_colection.count_documents({"retry": {"$exists": True}})))

# ----------------------------------------------------------------------
# Finally enrich the reimbursements data using the data in the database
# ----------------------------------------------------------------------

print()
print('REIMBURSEMENTS DATA ENRICHMENT...')

# let's create 5 more columns in the Serenata's reimbursements dataframe to store
# the metadata we found about the supplier company
reimbursements_raw_df['company_owners_list'] = ''         # list with the company owners info
reimbursements_raw_df['company_qtd_owners'] = None        # amount of owners that the company have
reimbursements_raw_df['receita_api_company_name'] = ''    # company name registered in the receitaWS API
reimbursements_raw_df['company_metadata'] = None          # json with all the info we got about the company
reimbursements_raw_df['receita_api_erro'] = None          # flag indicating if we got an error trying to get the company in the API or no

# correct cnpjs
print(f'[1/2] Adding info of the companies that we properly got information about...')
for document in receita_colection.find({"qsa": {"$exists": True}}, no_cursor_timeout=True):
    df_filter = reimbursements_raw_df.cnpj_cpf == document.get('raw_cnpj')

    reimbursements_raw_df.loc[df_filter, ['company_owners_list']] = str(document.get('qsa'))
    reimbursements_raw_df.loc[df_filter, ['company_qtd_owners']] = len(document.get('qsa'))
    reimbursements_raw_df.loc[df_filter, ['receita_api_company_name']] = document.get('nome')
    reimbursements_raw_df.loc[df_filter, ['company_metadata']] = str(document)
    reimbursements_raw_df.loc[df_filter, ['receita_api_erro']] = False
    break
print("\t- Done")

# cnpjs and cpfs with error
print(f'[2/2] Adding info of the companies that did not got information about...')
for document in receita_colection.find({"$or": [{"retry": {"$exists": True}}, {"message": {"$exists": True}}]},
                                       no_cursor_timeout=True):
    df_filter = reimbursements_raw_df.cnpj_cpf == document.get('raw_cnpj')

    reimbursements_raw_df.loc[df_filter, ['receita_api_company_name']] = document.get('name')
    reimbursements_raw_df.loc[df_filter, ['company_metadata']] = str(document)
    reimbursements_raw_df.loc[df_filter, ['receita_api_erro']] = True
    break
print("\t- Done")

# ----------------------------
# SAVE DATASET FOR BACKUP
# ----------------------------
file_name = 'reimbursements_2018_complete_df.csv'
reimbursements_raw_df.to_csv(f"datasets/{file_name}")
print()
print(f'-> Reimbursement dataset saved in "{file_name}"')
print('DONE')


"""
Куйвалайнен Д.А.
kuyvalaynen@gmail.com
ЗЕБЗ-01-16
"""

import math
import json

class Complex:
    """Класс Complex реализует работу с комплексными числами."""

    def __init__(self, real=0, imaq=0):
        """
        Инициализация класса.
        Параметры:
            - self.real (int, float) - действительная часть
            - self.imaq (int, float) - мнимая часть
        """

        if not self.check_complex(real, imaq):
            raise TypeError("Тип переданных параметров не 'int' или 'float'!")

        self.real = real
        self.imaq = imaq

    def __str__(self):
        """Вернуть строковое представление класса."""

        zn = ""
        if self.imaq >= 0:
            zn = "+"
        return "({}{}{}j)".format(self.real, zn, self.imaq)

    def __add__(self, other):
        """Создать новый объект как сумму self и other."""

        if isinstance(other, Complex):
            return Complex(self.real + other.real, self.imaq + other.imaq)
        else:
            raise TypeError("Число - {} не является Complex".
                            format(type(other)))

    def __sub__(self, other):
        """Создать новый объект как разность self и other."""

        if isinstance(other, Complex):
            return Complex(self.real - other.real, self.imaq - other.imaq)
        else:
            raise TypeError("Число - {} не является Complex".
                            format(type(other)))

    def __mul__(self, other):
        """Создать новый объект как произведение self и other."""

        if isinstance(other, Complex):
            real = self.real * other.real - self.imaq * other.imaq
            imaq = self.real * other.imaq + other.real * self.imaq
            return Complex(real, imaq)
        else:
            raise TypeError("Число - {} не является Complex".
                            format(type(other)))

    def __truediv__(self, other):
        """Создать новый объект как частное self и other."""
        if isinstance(other, Complex):
            dl = other.real**2 + other.imaq**2
            real = (self.real * other.real + self.imaq * other.imaq) / dl
            imaq = (other.real * self.imaq - self.real * other.imaq) / dl
            return Complex(real, imaq)
        else:
            raise TypeError("Число - {} не является Complex".
                            format(type(other)))

    def __abs__(self):
        """Возвращает модуль комплексного числа"""
        return self._abs()

    def __eq__(self, other):
        """Сравнивает два комплексных числа"""
        if isinstance(other, Complex):
            return self.real == other.real and self.imaq == other.imaq
        else:
            raise TypeError("Число - {} не является Complex".
                            format(type(other)))

    def _abs(self):
        """Считает модуль комплексного числа"""

        return (self.real**2 + self.imaq**2)**(1/2)

    def conjugate(self):
        """Создать новый объект как сопряжение self"""

        return Complex(self.real, -self.imaq)

    def __pow__(self, other):
        """Создать новый объект как self в степени other"""

        if isinstance(other, (int, float)):
            mod = self._abs()
            arg = math.atan(self.imaq / self.real)
            real = mod**other * math.cos(other * arg)
            imaq = mod**other * math.sin(other * arg)
            if isinstance(other, int) and other >= 0:
                real = math.floor(real)
                imaq = math.floor(imaq)
            return Complex(real, imaq)
        else:
            raise TypeError("Не могу возвести Complex число - {0}".
                            format(type(other)))

    @classmethod
    def from_string(cls, str_value):
        """Создать новый объект из строки str_value"""

        if not isinstance(str_value, str):
            raise TypeError("Переданное значение не 'str'!")

        try:
            char = str_value[0]
            if '-' in str_value[2:-1]:
                lst = str_value[1:].split('-')
                fl = 1
            elif '+' in str_value:
                lst = str_value[1:].split('+')
                fl = 0
            else:
                raise
            lst[0] = char + lst[0]
            if "." in lst[0]:
                real = float(lst[0])
            else:
                real = int(lst[0])
            if "." in lst[1]:
                imaq = float(lst[1][0:-1])
            else:
                imaq = int(lst[1][0:-1])

            if fl:
                imaq = -imaq
        except Exception:
            raise ValueError("Не удалось получить комплексное"
                             "число из строки - {}".format(str_value))
        return cls(real, imaq)

    @staticmethod
    def check_complex(real, imaq):
        """Проверяет типы переданных данных"""

        fl = 1
        if not isinstance(real, (int, float)):
            fl = 0
        if not isinstance(imaq, (int, float)):
            fl = 0
        return fl

    def save(self, filename="data.json"):
        """Сохраняет комплексное число в json файл"""

        lst = filename.split(".")
        assert lst[-1] == "json", "Формат файла не json"

        data = {"real": self.real,
                "imaq": self.imaq}
        with open(filename, "w", encoding="utf-8") as fh:
            fh.write(json.dumps(data, ensure_ascii=False, indent=4))
        print("Файл успешно записан")

    def load(self, filename="data.json"):
        """Загружает комплексное число из json файла"""

        lst = filename.split(".")
        assert lst[-1] == "json", "Формат файла не json"
        with open(filename, "r", encoding="utf-8") as fh:
            data = json.loads(fh.read())
        assert isinstance(data, dict), "Загруженные данные не типа dict!"
        assert "real" in data.keys() and "imaq" in data.keys(),\
            "Загруженный словарь не имеет нужных ключей!"
        assert self.check_complex(data["real"], data["imaq"]),\
            "Тип переданных параметров не 'int' или 'float'!"
        self.real = data["real"]
        self.imaq = data["imaq"]
        print("Файл успешно прочитан")


#!python3.9
# -*- coding: utf-8 -*-
# ---
# @Software: PyCharm
# @Author: zhaoyj
# @Site:
# @Time: 9月 07, 2022
# ---
from time import sleep
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select

from config import setting
from public.models.GetYaml import getyaml
from public.models.log import Log
from public.page_obj.base import Page

testData = getyaml(setting.OPS_TEST_Element_YAML + '/' + 'applyMaterialsALL.yaml')
log = Log()
class applyMaterialsALL(Page):
    """
    备用物料申请页面
    """
    url = ''
    # 申请物料
    applyMaterials_click0=(By.XPATH, testData.get_elementinfo(0))
    applyMaterials_click1=(By.XPATH, testData.get_elementinfo(1))
    applyMaterials_click2=(By.XPATH, testData.get_elementinfo(2))
    applyMaterials_click3=(By.XPATH, testData.get_elementinfo(3))
    applyMaterials_click4=(By.XPATH, testData.get_elementinfo(4))
    applyMaterials_click5=(By.XPATH, testData.get_elementinfo(5))

    def applyMaterialsTestALL(self,materialstype):
        """
        提交物料申请
        :param materialstype:
        :return:
        """
        self.driver.refresh()
        above = self.find_element(*self.applyMaterials_click0)
        ActionChains(self.driver).move_to_element(above).perform()
        sleep(2)
        self.find_element(*self.applyMaterials_click1).click()
        sleep(2)
        # self.find_element(*self.applyMaterials_click2).click()
        Select(self.find_element(*self.applyMaterials_click2)).select_by_index(materialstype)
        sleep(2)
        self.find_element(*self.applyMaterials_click3).click()
        sleep(2)
        self.find_element(*self.applyMaterials_click4).click()
        sleep(2)
        self.find_element(*self.applyMaterials_click5).click()
        sleep(3)


import click
import bz2
import csv
import gzip
import io
import os
import warnings

from onecodex.auth import login_required
from onecodex.exceptions import OneCodexException, ValidationError
from onecodex.utils import download_file_helper, get_download_dest, pretty_errors

def fastfastq(file_path):
    filename, ext = os.path.splitext(file_path)

    if ext in {".gz", ".gzip"}:
        open_func = gzip.open
    elif ext in {".bz", ".bz2", ".bzip", ".bzip2"}:
        open_func = bz2.open
    else:
        open_func = io.open

    with open_func(file_path, "rb") as fp:
        idx = 0
        buf = []

        for line in fp:
            idx += 1
            buf.append(line)

            if len(buf) == 4:
                buf = b"".join(buf)

                if not buf.startswith(b"@"):
                    raise ValidationError("FASTQ record line {} does not start with @".format(idx))

                yield buf

                buf = []

def validating_parser(file_path, **io_kwargs):
    import skbio

    for rec in skbio.io.read(file_path, **io_kwargs):
        buf = io.BytesIO()
        rec.write(buf, **io_kwargs)
        buf.seek(0)
        yield buf.read()

def get_filtered_filename(file_path):
    filename = os.path.basename(file_path)
    filename, ext = os.path.splitext(filename)

    if ext in {".gz", ".gzip", ".bz", ".bzip", ".bz2", ".bzip2"}:
        filename, ext = os.path.splitext(filename)

    return "{}.filtered{}".format(filename, ext), ext

def make_taxonomy_dict(classification, parent=False):
    """Parse a Classification object into a `dict` mapping a tax_id to its children or parent.

    Notes
    -----
    Restricted to taxonomic IDs that are represented in the classification results.
    """

    tax_id_map = {}

    if parent:
        for row in classification.results()["table"]:
            if row["parent_tax_id"] is not None and row["tax_id"] is not None:
                tax_id_map[row["tax_id"]] = row["parent_tax_id"]
    else:
        for row in classification.results()["table"]:
            if row["parent_tax_id"] is not None and row["tax_id"] is not None:
                try:
                    tax_id_map[row["parent_tax_id"]].add(row["tax_id"])
                except KeyError:
                    tax_id_map[row["parent_tax_id"]] = set([row["tax_id"]])

    return tax_id_map

def recurse_taxonomy_map(tax_id_map, tax_id, parent=False):
    """Traverse up or down through a taxonomic tree.

    Finds all children (or parents) of the given tax_id.

    Parameters
    ----------
    tax_id_map : `dict`
        The output of make_taxonomy_map().
    tax_id : `int`
        The taxonomic ID of the node to begin the traversal from.
    parent : `bool`
        Return parents of the taxonomic node rather than children.
    """

    if parent:
        # TODO: allow filtering on tax_id and its parents, too
        raise NotImplementedError
    else:

        def _child_recurse(tax_id, visited):
            try:
                children = [tax_id] + list(tax_id_map[tax_id])
            except KeyError:
                children = [tax_id]

            for child in children:
                if child not in visited:
                    visited.append(child)
                    children.extend(_child_recurse(child, visited))

            return children

        return list(set(_child_recurse(tax_id, [])))

def too_many_fastx_records():
    raise ValidationError(
        "FASTX file(s) provided have more records than the classification results"
    )

@click.command(
    "subset_reads",
    help="Subset a FASTX file based on the taxonomic results from a CLASSIFICATION_ID. "
    "By default, reads (or pairs or reads) matching the given taxonomic ID are "
    "written to the path provided, ignoring low confidence taxonomic assignments. ",
)
@click.argument("classification_id")
@click.argument("fastx", type=click.Path())
@click.option(
    "-t",
    "--tax-id",
    "tax_ids",
    required=True,
    multiple=True,
    help="Subset reads mapping to tax IDs. May be passed multiple times.",
)
@click.option("-r", "--reverse", type=click.Path(), help="The reverse (R2) read file, optionally.")
@click.option(
    "--validate/--do-not-validate",
    is_flag=True,
    default=False,
    help="By default, skip validation of FASTQ records (which is slow). Enable if you are uncertain you are working with a valid FASTQ file",
)
@click.option(
    "--with-children",
    default=False,
    is_flag=True,
    help="Match child taxa of those given with -t (e.g., all strains of E. coli)",
)
@click.option(
    "--subset-pairs-independently",
    default=False,
    is_flag=True,
    help="By default, if either read in a pair matches, both will be retained in the subset "
    "file. With this option, R1 and R2 files will be evaluated independently. Note "
    "that the subset output FASTQs are *not* guaranteed to have the same number of "
    "reads!",
)
@click.option(
    "--exclude-reads",
    default=False,
    is_flag=True,
    help="By default, matching reads are kept. Choose this option to instead output "
    "reads that do *not* match.",
)
@click.option(
    "--include-lowconf",
    default=False,
    is_flag=True,
    help="By default, reads with low confidence taxonomic assignments are ignored. "
    "Choose this option to include them.",
)
@click.option(
    "-o", "--out", default=".", type=click.Path(), help="Where to save the filtered outputs"
)
@click.pass_context
@pretty_errors
@login_required
def cli(
    ctx,
    classification_id,
    fastx,
    reverse,
    tax_ids,
    with_children,
    subset_pairs_independently,
    exclude_reads,
    include_lowconf,
    out,
    validate,
):
    if ctx.info_name == "filter_reads":
        warnings.warn(
            "filter_reads will be removed in a future version. Please use subset_reads instead!"
        )

    if not len(tax_ids):
        raise OneCodexException("You must supply at least one tax ID")

    # fetch classification result object from API
    classification = ctx.obj["API"].Classifications.get(classification_id)
    if classification is None:
        raise ValidationError("Classification {} not found.".format(classification_id))

    # if with children, expand tax_ids by referring to the taxonomic tree
    if with_children:
        tax_id_map = make_taxonomy_dict(classification)

        new_tax_ids = []

        for t_id in tax_ids:
            new_tax_ids.extend(recurse_taxonomy_map(tax_id_map, t_id))

        tax_ids = new_tax_ids

    tax_ids = set(tax_ids)

    # pull the classification result TSV
    tsv_url = classification._readlevel()["url"]
    readlevel_path = get_download_dest("./", tsv_url)
    if not os.path.exists(readlevel_path):
        download_file_helper(tsv_url, "./")
    else:
        click.echo("Using cached read-level results: {}".format(readlevel_path), err=True)

    # count the number of rows in the TSV file
    with gzip.open(readlevel_path, "rt") as tsv:
        try:
            tsv_row_count = 0
            for _ in tsv:
                tsv_row_count += 1
            tsv_row_count -= 1  # discount header line
        except EOFError:
            click.echo(
                "\nWe encountered an error while processing the read "
                "level results. Please delete {} and try again.".format(readlevel_path),
                err=True,
            )
            raise

    if reverse:
        if tsv_row_count % 2 != 0:
            raise ValidationError(
                "Classification results cannot have odd number of records if using --reverse/-r"
            )

        tsv_row_count = int(tsv_row_count / 2.0)

    # determine the name of the output file(s)
    filtered_filename, ext = get_filtered_filename(fastx)
    filtered_filename = os.path.join(out, filtered_filename)
    if reverse:
        rev_filtered_filename = get_filtered_filename(reverse)[0]
        rev_filtered_filename = os.path.join(out, rev_filtered_filename)

    if ext in {".fa", ".fna", ".fasta"}:
        io_kwargs = {"format": "fasta"}
    elif ext in {".fq", ".fastq"}:
        io_kwargs = {"format": "fastq", "variant": "illumina1.8"}
    else:
        raise OneCodexException(
            "{}: extension must be one of .fa, .fna, .fasta, .fq, .fastq".format(fastx)
        )

    # do the actual filtering
    save_msg = "Saving subsetted reads: {}".format(filtered_filename)
    if reverse:
        save_msg += " and {}".format(rev_filtered_filename)
    click.echo(save_msg, err=True)

    # see mainline/#3513. we must set idx=0 here for cases where the fastx file is empty
    idx = 0

    with click.progressbar(length=tsv_row_count) as bar, gzip.open(readlevel_path, "rt") as tsv:
        reader = csv.DictReader(tsv, delimiter="\t")

        if reverse:
            if not validate and io_kwargs["format"] == "fastq":
                fwd_iter = fastfastq(fastx)
                rev_iter = fastfastq(reverse)
            else:
                fwd_iter = validating_parser(fastx, **io_kwargs)
                rev_iter = validating_parser(reverse, **io_kwargs)

            with io.open(filtered_filename, "wb") as out_file, io.open(
                rev_filtered_filename, "wb"
            ) as rev_out_file:  # noqa

                for idx, (fwd, rev) in enumerate(zip(fwd_iter, rev_iter)):
                    if idx == tsv_row_count:
                        too_many_fastx_records()
                    if idx % 1000 == 0:
                        bar.update(1000)
                    row = next(reader)  # necessary to do it this way for py2 compat
                    row2 = next(reader)

                    if subset_pairs_independently:
                        if include_lowconf:
                            if exclude_reads:
                                if row["Tax ID"] not in tax_ids:
                                    out_file.write(fwd)
                                if row2["Tax ID"] not in tax_ids:
                                    rev_out_file.write(rev)
                            else:
                                if row["Tax ID"] in tax_ids:
                                    out_file.write(fwd)
                                if row2["Tax ID"] in tax_ids:
                                    rev_out_file.write(rev)
                        else:
                            if exclude_reads:
                                if (
                                    row.get("Passed Filter", "T") == "T"
                                    and row["Tax ID"] not in tax_ids
                                ):
                                    out_file.write(fwd)
                                if (
                                    row2.get("Passed Filter", "T") == "T"
                                    and row2["Tax ID"] not in tax_ids
                                ):
                                    rev_out_file.write(rev)
                            else:
                                if (
                                    row.get("Passed Filter", "T") == "T"
                                    and row["Tax ID"] in tax_ids
                                ):
                                    out_file.write(fwd)
                                if (
                                    row2.get("Passed Filter", "T") == "T"
                                    and row2["Tax ID"] in tax_ids
                                ):
                                    rev_out_file.write(rev)
                    else:
                        if include_lowconf:
                            if exclude_reads:
                                if row["Tax ID"] not in tax_ids or row2["Tax ID"] not in tax_ids:
                                    out_file.write(fwd)
                                    rev_out_file.write(rev)
                            else:
                                if row["Tax ID"] in tax_ids or row2["Tax ID"] in tax_ids:
                                    out_file.write(fwd)
                                    rev_out_file.write(rev)
                        else:
                            if exclude_reads:
                                if (
                                    row.get("Passed Filter", "T") == "T"
                                    and row["Tax ID"] not in tax_ids
                                ) or (
                                    row2.get("Passed Filter", "T") == "T"
                                    and row2["Tax ID"] not in tax_ids
                                ):
                                    out_file.write(fwd)
                                    rev_out_file.write(rev)
                            else:
                                if (
                                    row.get("Passed Filter", "T") == "T"
                                    and row["Tax ID"] in tax_ids
                                ) or (
                                    row2.get("Passed Filter", "T") == "T"
                                    and row2["Tax ID"] in tax_ids
                                ):
                                    out_file.write(fwd)
                                    rev_out_file.write(rev)
        else:
            if not validate and io_kwargs["format"] == "fastq":
                fwd_iter = fastfastq(fastx)
            else:
                fwd_iter = validating_parser(fastx, **io_kwargs)

            with io.open(filtered_filename, "wb") as out_file:
                for idx, (fwd, row) in enumerate(zip(fwd_iter, reader)):
                    if idx == tsv_row_count:
                        too_many_fastx_records()
                    if idx % 1000 == 0:
                        bar.update(1000)
                    if include_lowconf:
                        if exclude_reads:
                            if row["Tax ID"] not in tax_ids:
                                out_file.write(fwd)
                        else:
                            if row["Tax ID"] in tax_ids:
                                out_file.write(fwd)
                    else:
                        if exclude_reads:
                            if (
                                row.get("Passed Filter", "T") == "T"
                                and row["Tax ID"] not in tax_ids
                            ):
                                out_file.write(fwd)
                        else:
                            if row.get("Passed Filter", "T") == "T" and row["Tax ID"] in tax_ids:
                                out_file.write(fwd)

        if idx < tsv_row_count - 1:  # 0-based idx, 1-based tsv_row_count
            raise ValidationError(
                "FASTX file(s) provided have fewer records than the classification results"
            )

        bar.finish()


from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher
from aiogram.utils import executor
from aiogram.types import ReplyKeyboardRemove,\
    ReplyKeyboardMarkup, KeyboardButton,\
    InlineKeyboardButton, InlineKeyboardMarkup
from config import Token

TOKEN = '1270237768:AAFFEyzePLDVh7-HSDMtiFcCDSJ-iVNyosI'
bot = Bot(token=Token)
dp = Dispatcher(bot)

button_hi = KeyboardButton('Hi!')
greet = ReplyKeyboardMarkup(resize_keyboard=True).add(button_hi)

b1 = KeyboardButton('1')
b2 = KeyboardButton('2')
b3 = KeyboardButton('3')

markup = ReplyKeyboardMarkup().add(
    b1).add(b2).add(b3)

markup_2 = ReplyKeyboardMarkup().row(
    b1, b2, b3
)
markup_1 = ReplyKeyboardMarkup().row(
    b1, b2, b3
).add(KeyboardButton('Middle row'))

b4=KeyboardButton('4')
b5=KeyboardButton('5')
b6=KeyboardButton('6')
markup_1.row(b4, b5)
markup_1.insert(b6)

markup_request = ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Send own contact', request_contact=True)
).add(
    KeyboardButton('Send your location', request_location=True)
)

'''choice = InlineKeyboardMarkup(row_width=2)
buy_thing = InlineKeyboardButton(text="Buy apple", callback_data='ok')
choice.insert(buy_thing)
buy_thing_1 = InlineKeyboardButton(text="Buy raspberri", callback_data='OK')
choice.insert(buy_thing_1)

cancel_button = InlineKeyboardButton(text="Cancel", callback_data='cancel')
choice.insert(cancel_button)'''

#@dp.callback_query_handler(func=lambda c: c.data == 'ok')

inline_b = InlineKeyboardMarkup(row_width=2)
inline1 = InlineKeyboardButton('First button', callback_data = 'b1')
inline2 = InlineKeyboardButton('Second button', callback_data = 'b2')
cancell_button = InlineKeyboardButton('cancel', callback_data='cancel')
inline_b.insert(inline1)
inline_b.insert(inline2)
inline_b.insert(cancell_button)

@dp.message_handler(commands=['inline'])
async  def show_info(message: types.Message):
    await message.answer(text="For sell we have apple and raspberri", reply_markup=inline_b)

@dp.callback_query_handler(text = 'b1')
@dp.callback_query_handler(text='b2')
@dp.callback_query_handler(text = 'cancel')
async def process(call: types.CallbackQuery):
    answer_data = call.data
    #call.answer(f'you pressed one of the button here!')

    if answer_data == 'b1':
       # await bot.send_message(call.from_user.id, 'first button pressed!')
       text = 'first button pressed👍🏻'
    elif answer_data == 'b2':
        #await bot.send_message(call.from_user.id, 'second buttin was pressed')
        text = 'second button pressed👍🏻'
    elif answer_data == 'cancel':
        text = 'you was press button of cancel 🖕🏻'
    else:
        text = 'please press one of the buttons'
    await bot.send_message(call.from_user.id, text)
   # await bot.answer_callback_query(call.id)
    #await bot.send_message(call.from_user.id, 'first button pressed!')

@dp.message_handler(commands=['start'])
async def start(message: types.Message):
    await bot.send_message(message.from_user.id, "please choose button: /button or /but_1", reply_markup=markup_1 )

@dp.message_handler(commands=['button'])
async def send_button(message: types.Message):
    await message.reply("Hi write smth", reply_markup=greet)

@dp.message_handler(commands=['but_1'])
async def send(message: types.Message):
    await bot.send_message(message.from_user.id, "Okk", reply_markup=markup_2)
@dp.message_handler(commands=['but2'])
async def contact_and_geo(message: types.Message):
    await bot.send_message(message.from_user.id, "Please give allow for get information in your phone!", reply_markup=markup_request)
    await bot.send_message(chat_id='@yerekerun', text='You are athlet!')
dp.message_handler(commands=['photo'])
async def send_photo(message: types.Message):
    #caption = 'телеграм'
    await bot.send_photo(message.fr, photo="https://play.google.com/store/apps/details?id=org.telegram.messenger&hl=ru", caption="Telegram")

@dp.message_handler(commands=['remove'])
async def remove(message: types.Message):
    await message.reply("delete all button", reply_markup=ReplyKeyboardRemove())

@dp.message_handler()
async def echo(message: types.Message):
    await bot.send_message(message.from_user.id, message.text)

if __name__ == '__main__':
    executor.start_polling(dp)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import django.contrib.auth.models

class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0006_require_contenttypes_0002'),
        ('digispaceapp', '0006_auto_20160902_0623'),
    ]

    operations = [
        migrations.CreateModel(
            name='CallerDetails',
            fields=[
                ('CallerID', models.AutoField(serialize=False, editable=False, primary_key=True)),
                ('first_name', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('last_name', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('IncomingTelNo', models.CharField(default=None, max_length=200, null=True, blank=True)),
                ('email', models.CharField(default=None, max_length=200, null=True, blank=True)),
                ('CallerArea', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('caller_created_date', models.DateTimeField(null=True, blank=True)),
                ('caller_created_by', models.CharField(max_length=100, null=True, blank=True)),
                ('caller_updated_by', models.CharField(max_length=100, null=True, blank=True)),
                ('caller_updated_date', models.DateTimeField(null=True, blank=True)),
                ('CallerCity', models.ForeignKey(blank=True, to='digispaceapp.City', null=True)),
                ('CallerPincode', models.ForeignKey(blank=True, to='digispaceapp.Pincode', null=True)),
            ],
        ),
        migrations.CreateModel(
            name='CallInfo',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('UCID', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CallerID', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CalledNo', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CallStartTime', models.DateTimeField(null=True, blank=True)),
                ('DialStartTime', models.DateTimeField(null=True, blank=True)),
                ('DialEndTime', models.DateTimeField(null=True, blank=True)),
                ('DisconnectType', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CallStatus', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CallDuration', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('CallType', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('AudioRecordingURL', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('DialedNumber', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('Department', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('Extn', models.CharField(default=None, max_length=100, null=True, blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='EnquiryDetails',
            fields=[
                ('EnquiryID', models.AutoField(serialize=False, editable=False, primary_key=True)),
                ('enquiryFor', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('SelectedArea', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('created_date', models.DateTimeField(null=True, blank=True)),
                ('created_by', models.CharField(max_length=100, null=True, blank=True)),
                ('updated_by', models.CharField(max_length=100, null=True, blank=True)),
                ('updated_date', models.DateTimeField(null=True, blank=True)),
                ('CallerID', models.ForeignKey(blank=True, to='digispaceapp.CallerDetails', null=True)),
                ('SelectedCity', models.ForeignKey(blank=True, to='digispaceapp.City', null=True)),
                ('SelectedPincode', models.ForeignKey(blank=True, to='digispaceapp.Pincode', null=True)),
                ('category_id', models.ForeignKey(blank=True, to='digispaceapp.Category', null=True)),
                ('subcategory_id1', models.ForeignKey(blank=True, to='digispaceapp.CategoryLevel1', null=True)),
                ('subcategory_id2', models.ForeignKey(blank=True, to='digispaceapp.CategoryLevel2', null=True)),
            ],
        ),
        migrations.CreateModel(
            name='Operator',
            fields=[
                ('user_ptr', models.OneToOneField(parent_link=True, auto_created=True, to=settings.AUTH_USER_MODEL)),
                ('operator_id', models.AutoField(serialize=False, editable=False, primary_key=True)),
                ('operator_name', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('operator_email_id', models.CharField(default=None, max_length=100, null=True, blank=True)),
                ('operator_status', models.CharField(default=b'1', max_length=100, null=True, choices=[(b'1', b'1'), (b'0', b'0')])),
                ('user_created_date', models.DateTimeField(null=True, blank=True)),
                ('user_created_by', models.CharField(max_length=100, null=True, blank=True)),
                ('user_updated_by', models.CharField(max_length=100, null=True, blank=True)),
                ('user_updated_date', models.DateTimeField(null=True, blank=True)),
            ],
            options={
                'abstract': False,
                'verbose_name': 'user',
                'verbose_name_plural': 'users',
            },
            bases=('auth.user',),
            managers=[
                ('objects', django.contrib.auth.models.UserManager()),
            ],
        ),
        migrations.RenameField(
            model_name='supplier',
            old_name='country',
            new_name='country_id',
        ),
        migrations.RemoveField(
            model_name='supplier',
            name='city',
        ),
        migrations.AddField(
            model_name='consumerprofile',
            name='consumer_otp',
            field=models.CharField(max_length=100, null=True, blank=True),
        ),
        migrations.AddField(
            model_name='supplier',
            name='area',
            field=models.CharField(default=None, max_length=100, null=True, blank=True),
        ),
        migrations.AddField(
            model_name='supplier',
            name='city_place_id',
            field=models.ForeignKey(blank=True, to='digispaceapp.City_Place', null=True),
        ),
    ]


import matplotlib.pyplot as plt
import numpy as np
from Kidera import kidera

w = 'McPAS-TCR_with_V'
t = 'TCRGP_with_V'
nt = 'TCRGP_negs_with_V'

def tcr_length_dist_comp(data1, data2, title, normalize=False):
    with open(data1, 'r') as data:
        lens1 = {}
        for line in data:
            t = line.strip().split()[0]
            try:
                lens1[len(t)] += 1
            except KeyError:
                lens1[len(t)] = 1
    with open(data2, 'r') as data:
        lens2 = {}
        for line in data:
            t = line.strip().split()[0]
            try:
                lens2[len(t)] += 1
            except KeyError:
                lens2[len(t)] = 1
    x1 = [k for k in sorted(lens1)]
    y1 = [lens1[k] for k in sorted(lens1)]
    x2 = [k for k in sorted(lens2)]
    y2 = [lens2[k] for k in sorted(lens2)]
    if normalize:
        y1 = [y/sum(y1) for y in y1]
        y2 = [y/sum(y2) for y in y2]
    fig, ax = plt.subplots()
    width = 0.35
    plot1 = ax.bar([x + width for x in x1], y1, width,
                    color='SkyBlue', label='McPAS (Weizmann) data')
    plot2 = ax.bar(x2, y2, width,
                    color='IndianRed', label='TCRGP paper data')
    ax.set_ylabel('Number of TCRs')
    ax.set_title(title)
    ax.set_xticks(x1)
    ax.legend()
    plt.show()

# tcr_length_dist_comp(w, t, 'TCR Length Distribution')
# tcr_length_dist_comp(w, t, 'Normalized TCR Length Distribution', normalize=True)
# tcr_length_dist_comp(w, nt, 'Normalized TCR Length Distribution' ,normalize=True)

def amino_corr_map(datafile, c, title):
    with open(datafile, 'r') as data:
        amino_acids = ['start'] + [letter for letter in 'ARNDCEQGHILKMFPSTWYV'] + ['end']
        amino_to_ix = {aa: index for index, aa in enumerate(amino_acids)}
        # P[i, j] = p(j | i) stochastic transition matrix as in markov models
        P = np.zeros([22, 22])
        for line in data:
            if c == 'tcr':
                t = line.strip().split()[0]
            if c == 'pep':
                t = line.strip().split()[1]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            # REMOVE INITIAL 'CAS' AND 'F' SUFFIX
            t = t[3:-1]
            P[amino_to_ix['start'], amino_to_ix[t[0]]] += 1
            for i in range(len(t) - 1):
                P[amino_to_ix[t[i]], amino_to_ix[t[i+1]]] += 1
            P[amino_to_ix[t[-1]], amino_to_ix['end']] += 1
        P = P.astype('float') / P.sum(axis=1)[:, np.newaxis]
        # for i in range(22)
        print(P)
        plt.matshow(P)
        plt.xticks(range(22), amino_acids)
        plt.yticks(range(22), amino_acids)
        plt.title(title)
        plt.show()
    pass

def amino_corr_map_comp(data1, data2, c, title):
    amino_acids = ['start'] + [letter for letter in 'ARNDCEQGHILKMFPSTWYV'] + ['end']
    amino_to_ix = {aa: index for index, aa in enumerate(amino_acids)}
    # P[i, j] = p(j | i) stochastic transition matrix as in markov models
    with open(data1, 'r') as data:
        P1 = np.zeros([22, 22])
        for line in data:
            if c == 'tcr':
                t = line.strip().split()[0]
            if c == 'pep':
                t = line.strip().split()[1]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            # REMOVE INITIAL 'CAS' AND 'F' SUFFIX
            t = t[3:-1]
            P1[amino_to_ix['start'], amino_to_ix[t[0]]] += 1
            for i in range(len(t) - 1):
                P1[amino_to_ix[t[i]], amino_to_ix[t[i+1]]] += 1
            P1[amino_to_ix[t[-1]], amino_to_ix['end']] += 1
        P1 = P1.astype('float') / P1.sum(axis=1)[:, np.newaxis]
        print(P1)
    with open(data2, 'r') as data:
        P2 = np.zeros([22, 22])
        for line in data:
            if c == 'tcr':
                t = line.strip().split()[0]
            if c == 'pep':
                t = line.strip().split()[1]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            # REMOVE INITIAL 'CAS' AND 'F' SUFFIX
            t = t[3:-1]
            P2[amino_to_ix['start'], amino_to_ix[t[0]]] += 1
            for i in range(len(t) - 1):
                P2[amino_to_ix[t[i]], amino_to_ix[t[i+1]]] += 1
            P2[amino_to_ix[t[-1]], amino_to_ix['end']] += 1
        P2 = P2.astype('float') / P2.sum(axis=1)[:, np.newaxis]
        print(P2)
    # P = np.log(P1 / P2)
    P = P1 - P2
    plt.matshow(P)
    plt.xticks(range(22), amino_acids)
    plt.yticks(range(22), amino_acids)
    plt.colorbar()
    plt.title(title)
    plt.show()

# amino_corr_map(w, 'tcr', 'Amino acids correlation map in McPAS data')
# amino_corr_map(t, 'tcr', 'Amino acids correlation map in TCRGP data')
# amino_corr_map_comp(w, t, 'tcr', 'Amino acids correlation maps, P1 - P2')

# amino_corr_map(w, 'tcr', 'Amino acids correlation map in McPAS data')
# amino_corr_map(nt, 'tcr', 'Amino acids correlation map in TCRGP data')
# amino_corr_map_comp(w, nt, 'tcr', 'Amino acids correlation maps difference')

def amino_acids_distribution(data1, data2, title, normalize=False):
    amino_acids = [letter for letter in 'ARNDCEQGHILKMFPSTWYV']
    with open(data2, 'r') as data:
        amino_count2 = {aa: 0 for aa in amino_acids}
        for line in data:
            t = line.strip().split()[0]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            for aa in t:
                amino_count2[aa] += 1
    with open(data1, 'r') as data:
        amino_count1 = {aa: 0 for aa in amino_acids}
        for line in data:
            t = line.strip().split()[0]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            for aa in t:
                amino_count1[aa] += 1
    x = np.arange(len(amino_acids))
    y1 = [amino_count1[k] for k in amino_acids]
    y2 = [amino_count2[k] for k in amino_acids]
    if normalize:
        y1 = [y/sum(y1) for y in y1]
        y2 = [y/sum(y2) for y in y2]
    fig, ax = plt.subplots()
    width = 0.35
    plot1 = ax.bar([key + width for key in x], y1, width,
                   color='SkyBlue', label='McPAS (Weizmann) data')
    plot2 = ax.bar(x, y2, width,
                   color='IndianRed', label='TCRGP paper data')
    ax.set_ylabel('Number of amino acids')
    ax.set_title(title)
    plt.xticks(x, amino_acids)
    ax.legend()
    plt.show()

# amino_acids_distribution(w, t, 'Animo Acids Distribution')
# amino_acids_distribution(w, t, 'Normalized Animo Acids Distribution', normalize=True)
# amino_acids_distribution(w, nt, 'Normalized Animo Acids Distribution', normalize=True)

def v_gene_dist(data1, data2, title, normalize=False):
    with open(data2, 'r') as data:
        v_count2 = {}
        for line in data:
            v = line.strip().split()[1]
            #if '*' in t or '*' in t:
            #    continue
            #if '/' in t:
            #    continue
            try:
                v_count2[v] += 1
            except KeyError:
                v_count2[v] = 1
    '''
    with open(data1, 'r') as data:
        amino_count1 = {aa: 0 for aa in amino_acids}
        for line in data:
            t = line.strip().split()[0]
            if '*' in t or '*' in t:
                continue
            if '/' in t:
                continue
            for aa in t:
                amino_count1[aa] += 1
    '''
    print(v_count2)
    x = np.arange(len(v_count2))
    y1 = [amino_count1[k] for k in amino_acids]
    y2 = [amino_count2[k] for k in amino_acids]
    if normalize:
        y1 = [y/sum(y1) for y in y1]
        y2 = [y/sum(y2) for y in y2]
    fig, ax = plt.subplots()
    width = 0.35
    plot1 = ax.bar([key + width for key in x], y1, width,
                   color='SkyBlue', label='McPAS (Weizmann) data')
    plot2 = ax.bar(x, y2, width,
                   color='IndianRed', label='TCRGP paper data')
    ax.set_ylabel('Number of amino acids')
    ax.set_title(title)
    plt.xticks(x, amino_acids)
    ax.legend()
    plt.show()
    pass

def avg_kidera_score(datafile):
    with open(datafile, 'r') as data:
        index = 0
        avg = np.zeros((10))
        for line in data:
            line = line.split('\t')
            tcr = line[0]
            tcr = tcr[3:-1]
            v = kidera.score_sequence(tcr)
            v = v.values
            avg += v
            index += 1
    avg /= index
    return avg

def plot_kidera():
    w_avg = avg_kidera_score(w)
    nt_avg = avg_kidera_score(nt)
    print('weizmann avg:', w_avg)
    print('TCRGP avg:', nt_avg)
    x = np.arange(10)
    fig, ax = plt.subplots()
    width = 0.35
    plot1 = ax.bar([t + width for t in x], w_avg, width,
                   color='SkyBlue', label='McPAS (Weizmann) data')
    plot2 = ax.bar(x, nt_avg, width,
                   color='IndianRed', label='TCRGP paper data')
    ax.set_ylabel('Average Kidera score')
    ax.set_title('Average Kidera Score')
    ax.set_xticks(x)
    ax.set_xlabel('Kidera factor')
    ax.legend()
    plt.show()

# plot_kidera()

def kidera_hist(data1, data2):
    factor_observations1 = [[] for i in range(10)]
    with open(data1, 'r') as data:
        for line in data:
            line = line.split('\t')
            tcr = line[0]
            tcr = tcr[3:-1]
            v = kidera.score_sequence(tcr)
            v = v.values
            for i in range(len(v)):
                factor_observations1[i].append(v[i])
    factor_observations2 = [[] for i in range(10)]
    with open(data2, 'r') as data:
        for line in data:
            line = line.split('\t')
            tcr = line[0]
            tcr = tcr[3:-1]
            v = kidera.score_sequence(tcr)
            v = v.values
            for i in range(len(v)):
                factor_observations2[i].append(v[i])
    for i in range(len(factor_observations1)):
        fig, ax = plt.subplots()
        a = factor_observations1[i]
        b = factor_observations2[i]
        weights1 = np.ones_like(a) / float(len(a))
        weights2 = np.ones_like(b) / float(len(b))
        bins = np.linspace(-1.0, 1.0, 10)
        plot1 = ax.hist(a, weights=weights1, bins=bins,
                       color='SkyBlue', alpha=0.5, label='McPAS (Weizmann) data')
        plot2 = ax.hist(b, weights=weights2, bins=bins,
                       color='IndianRed', alpha=0.5, label='TCRGP paper data')
        #ax.set_ylabel('Number of TCRs')
        ax.set_title('Kidera ' + str(i+1) + ' factor normalized histogram')
        #ax.set_xticks(x1)
        ax.legend()
        # plt.show()
        plt.savefig('stats_compare_plots/kidera_factors/kidera_bins10_' + str(i+1))

# kidera_hist(w, nt)


from PyQt5.QtWidgets import QMainWindow, QApplication, QLabel, QLineEdit, QPushButton
from PyQt5.QtWidgets import QTextEdit

from PyQt5.QtGui import QIcon

import sys

class MainWindow( QMainWindow ):
    _counter = 0
    _raise = True
    _clicksCount = 0;
    def __init__(self):
        super().__init__()

        self.setFixedSize(400,400) # Fix o'lcham
        # self.setMinimumSize(400,400)
        # self.setMaximumSize(700, 700)
        # self.setFixedHeight(600)
        # self.setFixedWidth(500)
        # self.setWindowTitle("Salom")
        # self.setBaseSize(800, 800)
        # self.setStyleSheet('background-color: black;')
        # self.setWindowIcon(QIcon("IMG_1258.jpg"))

# "================================================================="

        # self.text1 = QLabel("Salom", self)
        # # self.text1.setGeometry(30, 70, 200, 70)
        # self.text1.setStyleSheet("font-size: 40px;");
        # self.text1.adjustSize()

        # self.text2 = QLabel("Hello", self)
        # self.text2.move(30, 70)
        # self.text2.setStyleSheet("font-size: 40px;")
        # self.text2.adjustSize()

# "================================================================="
        # self.lineEdit1 = QLineEdit(self)
        # self.setStyleSheet("""
        #                    font-size: 30px;
        #                    font-family: Times new Roman
        #                    """)

# "================================================================="

        # self.bigText = QTextEdit(self)
        # self.bigText.setFixedSize(200, 200)

# "================================================================="

        self.btn1 = QPushButton(self)
        self.btn1.setText("Button")
        self.btn1.clicked.connect(self.buttonClicked)

        self.labelNumber = QLabel("0",self)
        self.labelNumber.move(40, 40)
        self.labelNumber.setStyleSheet("font-size: 30px")

        self.labelCount = QLabel("0",self)
        self.labelCount.move(70, 40)
        self.labelCount.setStyleSheet("font-size: 30px; color: red;")

    def buttonClicked(self):

        self._clicksCount += 1
        self.labelCount.setText(f"{self._clicksCount}")

        if( self._raise ):
            self._counter += 1
        else:
            self._counter -= 1

        if( self._counter == 10 ):
            self._raise = False
        elif( self._counter == 0 ):
            self._raise = True

        self.labelNumber.setText(f"{self._counter}")

app = QApplication([])
print(sys.argv)
window = MainWindow()
window.show()
app.exec()


#first program for submission for google analytics competition.

import os
import json
import numpy as np
import pandas as pd
from pandas.io.json import json_normalize
import matplotlib.pyplot as plt
import seaborn as sns
color = sns.color_palette()

def load_df(csv_path='train.csv', nrows=None):
    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']

    df = pd.read_csv(csv_path,
                     converters={column: json.loads for column in JSON_COLUMNS},
                     dtype={'fullVisitorId': 'str'}, # Important!!
                     nrows=nrows)

    for column in JSON_COLUMNS:
        column_as_df = json_normalize(df[column])
        column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)
    print(f"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}")
    return df

train_df = load_df(nrows=10000)
test_df = load_df(csv_path="test.csv",nrows=10000)

train_df["totals.transactionRevenue"] = train_df["totals.transactionRevenue"].astype('float')
gdf = train_df.groupby("fullVisitorId")["totals.transactionRevenue"].sum().reset_index()

plt.figure(figsize=(8,6))
plt.scatter(range(gdf.shape[0]), np.sort(np.log1p(gdf["totals.transactionRevenue"].values)))
plt.xlabel('index', fontsize=12)
plt.ylabel('TransactionRevenue', fontsize=12)
plt.show()

Google Analytics Customer Revenue Prediction/GoogleAnalyticsRegression.py


import argparse
import subprocess
from ...core.command import Command

class BranchCreate(Command):
    def __init__(self, parserMgr: argparse._SubParsersAction):
        self._parser = parserMgr.add_parser('create',
                                            help='Create a new branch and switch to it')
        self._parser.set_defaults(func=self.run)
        self._parser.add_argument('branch_name',
                                  type=str,
                                  help='name for the new branch',
                                )

    def run(self, args: argparse.Namespace):
        commandArray = ["git", "checkout", "-b", args.branch_name]
        subprocess.run(commandArray)


# Run on python 3 for CSEProj 145
import numpy as np
import cv2

top = 461
for i in range(1,top):
# Load an color image in grayscalei
	try:
		name = 'img_' + str(i) + '.png'
		img = cv2.imread(name)
	except:
		continue
	if img is not None:
		#print(np.shape(img))
		a = img[:,0:255,:]
		b = img[:,255:510,:]
		c = np.concatenate((b,a),axis=1)
		cv2.imwrite(name,c)
	else:
		print(i)


import argparse
import logging
import time

import numpy as np
import torch

from .. import headmeta, visualizer
from ..annotation import Annotation
from . import utils
from .cifcaf import CifCaf
from .track_annotation import TrackAnnotation
from .track_base import TrackBase

LOG = logging.getLogger(__name__)

class TrackingPose(TrackBase):
    cache_group = [0, -1]
    forward_tracking_pose = True
    track_recovery = False
    single_seed = False

    def __init__(self,
                 cif_meta: headmeta.TSingleImageCif,
                 caf_meta: headmeta.TSingleImageCaf,
                 tcaf_meta: headmeta.Tcaf,
                 *,
                 pose_generator=None):
        super().__init__()
        self.cif_meta = cif_meta
        self.caf_meta = caf_meta
        self.tcaf_meta = tcaf_meta

        # prefer decoders with more keypoints and associations
        self.priority = 1.0
        self.priority += cif_meta.n_fields / 1000.0
        self.priority += caf_meta.n_fields / 1000.0
        self.priority += tcaf_meta.n_fields / 1000.0

        # a similar selector exists for pose similarity
        self.invalid_keypoints = [
            i
            for i, kp in enumerate(cif_meta.keypoints)
            if kp in ('left_ear', 'right_ear')
        ] if cif_meta.dataset == 'posetrack2018' else []

        self.n_keypoints = len(cif_meta.keypoints)
        tracking_keypoints = cif_meta.keypoints * len(self.cache_group)
        tracking_sigmas = cif_meta.sigmas * len(self.cache_group)
        tracking_skeleton = (
            self.caf_meta.skeleton
            + [
                (keypoint_i + 1, keypoint_i + 1 + frame_i * self.n_keypoints)
                for frame_i in range(1, len(self.cache_group))
                for keypoint_i in range(self.n_keypoints)
            ]
        )

        self.tracking_cif_meta = headmeta.Cif(
            'tracking_cif', cif_meta.dataset,
            keypoints=tracking_keypoints,
            sigmas=tracking_sigmas,
            pose=None)
        self.tracking_cif_meta.head_index = 0
        self.tracking_cif_meta.base_stride = cif_meta.base_stride
        self.tracking_cif_meta.upsample_stride = cif_meta.upsample_stride

        self.tracking_caf_meta = headmeta.Caf(
            'tracking_caf', caf_meta.dataset,
            keypoints=tracking_keypoints,
            sigmas=tracking_sigmas,
            skeleton=tracking_skeleton,
            pose=None)
        self.tracking_caf_meta.head_index = 1
        self.tracking_caf_meta.base_stride = caf_meta.base_stride
        self.tracking_caf_meta.upsample_stride = caf_meta.upsample_stride

        self.pose_generator = pose_generator or CifCaf(
            [self.tracking_cif_meta], [self.tracking_caf_meta])
        self.nms_occupancy = utils.Occupancy(2, 4)

        self.vis_multitracking = visualizer.MultiTracking(self.tracking_caf_meta)

    @classmethod
    def cli(cls, parser: argparse.ArgumentParser):
        group = parser.add_argument_group('trackingpose decoder')
        assert not cls.track_recovery
        group.add_argument('--trackingpose-track-recovery', default=False, action='store_true')
        assert not cls.single_seed
        group.add_argument('--trackingpose-single-seed', default=False, action='store_true')

    @classmethod
    def configure(cls, args: argparse.Namespace):
        cls.track_recovery = args.trackingpose_track_recovery
        cls.single_seed = args.trackingpose_single_seed

    @classmethod
    def factory(cls, head_metas):
        if len(head_metas) < 3:
            return []
        return [
            cls(cif_meta, caf_meta, tcaf_meta)
            for cif_meta, caf_meta, tcaf_meta
            in zip(head_metas, head_metas[1:], head_metas[2:])
            if (isinstance(cif_meta, headmeta.TSingleImageCif)
                and isinstance(caf_meta, headmeta.TSingleImageCaf)
                and isinstance(tcaf_meta, headmeta.Tcaf))
        ] + [
            cls(cif_meta, caf_meta, tcaf_meta)
            for cif_meta, caf_meta, tcaf_meta
            in zip(head_metas, head_metas[1:], head_metas[3:])
            if (isinstance(cif_meta, headmeta.TSingleImageCif)
                and isinstance(caf_meta, headmeta.TSingleImageCaf)
                and isinstance(tcaf_meta, headmeta.Tcaf))
        ]

    def soft_nms(self, tracks, frame_number):
        if not tracks:
            return

        # keypoint threshold
        for t in tracks:
            frame_ann = t.pose(self.frame_number)
            if frame_ann is None:
                continue
            kps = frame_ann.data
            kps[kps[:, 2] < utils.nms.Keypoints.get_keypoint_threshold()] = 0.0
            kps[self.invalid_keypoints] = 0.0

        self.nms_occupancy.reset((
            self.n_keypoints,
            int(max(1, max(np.max(t.frame_pose[-1][1].data[:, 1]) for t in tracks) + 1)),
            int(max(1, max(np.max(t.frame_pose[-1][1].data[:, 0]) for t in tracks) + 1)),
        ))

        tracks = sorted(tracks, key=lambda tr: -tr.score(frame_number, current_importance=0.01))
        for track in tracks:
            ann = track.pose(frame_number)
            if ann is None:
                continue

            assert ann.joint_scales is not None
            joint_is = np.flatnonzero(ann.data[:, 2])
            for joint_i in joint_is:
                xyv = ann.data[joint_i]
                if self.nms_occupancy.get(joint_i, xyv[0], xyv[1]):
                    xyv[2] = 0.0
                else:
                    self.nms_occupancy.set(joint_i, xyv[0], xyv[1], ann.joint_scales[joint_i])

        # keypoint threshold
        for t in tracks:
            frame_ann = t.pose(self.frame_number)
            if frame_ann is None:
                continue
            kps = frame_ann.data
            kps[kps[:, 2] < utils.nms.Keypoints.get_keypoint_threshold()] = 0.0

        if self.pose_generator.occupancy_visualizer is not None:
            LOG.debug('Occupied fields after NMS')
            self.pose_generator.occupancy_visualizer.predicted(self.nms_occupancy)

    # pylint: disable=too-many-statements
    def __call__(self, fields, *, initial_annotations=None):
        self.frame_number += 1

        # self.prune_active(frame_number) TODO?

        start = time.perf_counter()

        # initialize tracking poses from self.active tracks
        initial_annotations = []
        for track in self.active:
            tracking_ann = Annotation(
                self.tracking_cif_meta.keypoints,
                self.tracking_caf_meta.skeleton,
            )
            tracking_ann.id_ = track.id_
            for position_i, frame_i in enumerate(self.cache_group[1:], start=1):
                prev_pose = track.pose(self.frame_number + frame_i)
                if prev_pose is not None:
                    tracking_ann.data[
                        self.n_keypoints * position_i:
                        self.n_keypoints * position_i + self.n_keypoints
                    ] = prev_pose.data
                    tracking_ann.joint_scales[
                        self.n_keypoints * position_i:
                        self.n_keypoints * position_i + self.n_keypoints
                    ] = prev_pose.joint_scales

            if self.single_seed:
                inverse_mask = tracking_ann.data[:, 2] < np.amax(tracking_ann.data[:, 2])
                tracking_ann.data[inverse_mask] = 0.0
                tracking_ann.joint_scales[inverse_mask] = 0.0
            tracking_ann.data[tracking_ann.data[:, 2] < 0.05] = 0.0
            if not np.any(tracking_ann.data[:, 2] > 0.0):
                continue
            initial_annotations.append(tracking_ann)
        initial_annotations = list(sorted(
            initial_annotations, key=lambda ann: ann.bbox()[3], reverse=True))
        LOG.debug('initial annotation heights: %s', [ann.bbox()[3] for ann in initial_annotations])

        LOG.debug('using %d initial annotations', len(initial_annotations))

        # use standard pose processor to connect to current frame
        LOG.debug('overwriting CifCaf parameters')
        CifCaf.nms = None
        # CifCaf.keypoint_threshold = 0.001
        tracking_fields = [
            fields[self.cif_meta.head_index],
            torch.cat([
                fields[self.caf_meta.head_index],
                fields[self.tcaf_meta.head_index],
            ], dim=0)
        ]
        tracking_annotations = self.pose_generator(
            tracking_fields, initial_annotations=initial_annotations)

        # extract new pose annotations from tracking pose
        active_by_id = {t.id_: t for t in self.active}
        lost_trackids = {t.id_: t.frame_pose[-1][0] for t in self.active
                         if t.frame_pose[-1][0] < self.frame_number - 1}
        for tracking_ann in tracking_annotations:
            single_frame_ann = Annotation(
                self.cif_meta.keypoints, self.caf_meta.skeleton)
            single_frame_ann.data[:] = tracking_ann.data[:self.n_keypoints]
            single_frame_ann.joint_scales = tracking_ann.joint_scales[:self.n_keypoints]
            single_frame_ann.decoding_order = [
                (jsi, jti, xyv_s, xyv_t)
                for (jsi, jti, xyv_s, xyv_t) in tracking_ann.decoding_order
                if jsi < self.n_keypoints and jti < self.n_keypoints
            ]

            track_id = getattr(tracking_ann, 'id_', -1)
            if track_id == -1:
                new_track = TrackAnnotation().add(self.frame_number, single_frame_ann)
                self.active.append(new_track)
                # assign new track id also to tracking pose for visualization
                tracking_ann.id_ = new_track.id_
                continue
            active_by_id[track_id].add(self.frame_number, single_frame_ann)

        # nms tracks
        self.soft_nms(self.active, self.frame_number)

        # track recovery
        if self.track_recovery:
            removed = set()
            for track in self.active:
                if not lost_trackids:
                    break
                if len(track) > 1:
                    continue
                if track.pose(self.frame_number) is None:
                    continue

                track_id = max(lost_trackids.items(), key=lambda d: d[1])[0]
                del lost_trackids[track_id]
                # tracking_ann.id_ = track_id
                active_by_id[track_id].add(self.frame_number, track.pose(self.frame_number))
                removed.add(track)
                LOG.info('recovered track %d', track_id)
            self.active = [t for t in self.active if t not in removed]

        # tag ignore regions
        # if self.gt_anns:  TODO
        #     self.tag_ignore_region(self.frame_number, self.gt_anns)

        # pruning lost tracks
        self.active = [t for t in self.active if self.track_is_viable(t, self.frame_number)]

        good_track_ids = {t.id_ for t in self.active if self.track_is_good(t, self.frame_number)}
        LOG.info('active tracks = %d, good = %d, track ids = %s',
                 len(self.active), len(good_track_ids),
                 [self.simplified_track_id_map.get(t.id_, t.id_)
                  for t in self.active])

        # visualize good tracking poses with assigned track id
        good_track_annotations = [t for t in tracking_annotations if t.id_ in good_track_ids]
        self.vis_multitracking.predicted(good_track_annotations)
        # if self.track_visualizer or self.track_ann_visualizer:
        #     good_ids = set(t.id_ for t in self.active if self.track_is_good(t, self.frame_number))
        #     good_tracking_anns = [t for t in tracking_annotations
        #                           if getattr(t, 'id_', None) in good_ids]
        #     if self.track_ann_visualizer:
        #         self.track_ann_visualizer.predicted(good_tracking_anns)
        #     if self.track_visualizer:
        #         self.track_visualizer.predicted(
        #             self.frame_number,
        #             [t for t in self.active if t.id_ in good_ids],
        #             self.gt_anns,
        #         )

        LOG.debug('track time: %.3fs', time.perf_counter() - start)
        return self.annotations(self.frame_number)


from fastapi import Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.orm import Session

from .. import crud, models
from ..database import get_session
from .menu_repository import MenuRepository

class SubmenuRepository(MenuRepository):
    def __init__(self, db: Session = Depends(get_session)) -> None:
        self.db = db
        self.orm_model = models.Submenu
        self.detail_404 = 'submenu not found'
        self.detail_400 = 'Submenu with this title already exists'

    async def get_all(self, skip: int = 0, limit: int = 100, **kwargs) -> list[models.Menu | models.Submenu | models.Dish | None]:
        # menus = self.db.query(self.orm_model).filter_by(**kwargs).offset(
        # skip).limit(limit).all()

        lookup_query = select(self.orm_model).filter_by(**kwargs).offset(skip).limit(limit)
        result = await self.db.execute(lookup_query)
        menus = result.scalars().all()
        # count submenus and dishes
        for db_menu in menus:
            db_menu.dishes_count = await crud.get_sumbenus_dishes_count(
                self.db, db_menu.id)
        return menus

    async def get(self, **kwargs):
        # menu = self.db.query(
        # self.orm_model).filter_by(**kwargs).first()
        lookup_query = select(self.orm_model).filter_by(**kwargs)
        result = await self.db.execute(lookup_query)
        menu = result.scalars().first()
        if not menu:
            raise HTTPException(
                status_code=404, detail=f'{self.detail_404}')
        # count submenus and dishes
        menu.dishes_count = await crud.get_sumbenus_dishes_count(self.db, menu.id)
        return menu


import asyncio
import os
import time

from multiprocessing import Process, Queue, cpu_count, Manager
from typing import Any, Callable, Dict, List, Tuple, Union

import pandas as pd

class InvalidMethodTree(Exception):
    pass

class Method(object):
    def __init__(self, key: str, value: Any, arguments: List[Dict[str, Any]]):
        self.key = key
        self.value = value
        self.args = arguments
        self.upstream: Dict[str, bool] = {}
        self.downstream: List[Method] = []
        self.executed: bool = False
        self.executing: bool = False
        self.output: Any = None

    def is_executable(self):
        return all(self.upstream.values())

    def unique_downstream(self):
        self.downstream = list(set(self.downstream))

    def execute(self, method_map: Dict[str, Callable],
                node_map: Dict[str, Any], **kwargs) -> Any:
        if not self.executed:
            foo = method_map[self.value["variable"]]
            args = []
            kw = {}
            for arg in self.args:
                if arg["object"] == "direct":
                    val = arg["variable"]
                elif arg["object"] == "kwargs_obj":
                    val = kwargs[arg["variable"]]
                elif arg["object"] == "method":
                    val = node_map[arg["variable"]]
                else:
                    raise ValueError(
                        "Object type %s not recognized." % arg["object"])
                if "name" in arg:
                    kw[arg["name"]] = val
                else:
                    args.append(val)
            self.output = foo(*args, **kw)
            self.executing = False
            self.executed = True
        return self.output

class MethodDAG(object):
    def __init__(self, tasks: List[Dict[str, Any]]):
        self.node_map: Dict[str, Any] = {}
        self.nodes: Dict[str, Method] = {}
        self.tasks: Dict[str, bool] = {}
        for task in tasks:
            key = task.pop("key")
            args = task.pop("arguments")
            self.tasks[key] = False
            self.node_map[key] = None
            self.nodes[key] = Method(key, task, args)
        self.queue = []

    def set_dependencies(self, streams: List[Dict[str, Any]]):
        for stream in streams:
            method = self.nodes[stream["task"]]
            for task in stream["downstreams"]:
                method.upstream[task] = False
                self.nodes[task].downstream.append(method)

    def set_queue(self):
        for method in self.nodes.values():
            method.unique_downstream()
            if method.is_executable():
                method.executing = True
                self.queue.append(method)

    def compute(self, method_map: Dict[str, Callable], **kwargs) -> Any:
        self.set_queue()
        while len(self.queue) > 0:
            method = self.queue.pop()
            self.node_map[method.key] = method.execute(
                method_map, self.node_map, **kwargs)
            for dependent in method.downstream:
                dependent.upstream[method.key] = True
                if dependent.is_executable():
                    self.queue.append(dependent)

    @classmethod
    def parse(cls, config: Dict[str, Any]) -> "MethodDAG":
        dag = cls(config["tasks"])
        dag.set_dependencies(config["streams"])
        return dag

class MethodDAGConcurrent(MethodDAG):
    def __init__(self, tasks: List[Dict[str, Any]], n: int = -1):
        super(MethodDAGConcurrent, self).__init__(tasks)
        self.queue = asyncio.Queue()
        if n == -1:
            self.num_processes = cpu_count()
        else:
            self.num_processes = n

    async def produce(self, queue: asyncio.Queue):
        for method in self.nodes.values():
            method.unique_downstream()
            if method.is_executable():
                method.executing = True
                await self.queue.put(method)
        while True:
            complete = all([
                method.executed or method.executing
                for method in self.nodes.values()
            ])
            if complete:
                break
            for method in self.nodes.values():
                execution = method.executed or method.executing
                if self.tasks[method.key]:
                    for dependent in method.downstream:
                        dependent.upstream[method.key] = True
                if method.is_executable() and not execution:
                    method.executing = True
                    await self.queue.put(method)
            await asyncio.sleep(0.001)

    async def consume(self, queue: asyncio.Queue(),
                      method_map: Dict[str, Callable], **kwargs):
        while True:
            method = await self.queue.get()
            self.node_map[method.key] = method.execute(
                method_map, self.node_map, **kwargs)
            self.tasks[method.key] = True
            queue.task_done()

    async def _compute(self, method_map: Dict[str, Callable], **kwargs) -> Any:
        producer = self.produce(self.queue)
        consumers = []
        for i in range(self.num_processes):
            p = asyncio.ensure_future(self.consume(
                self.queue, method_map, **kwargs))
            consumers.append(p)
        await asyncio.gather(producer, return_exceptions=True)
        await self.queue.join()
        for consumer in consumers:
            consumer.cancel()

    def compute(self, method_map: Dict[str, Callable], **kwargs) -> Any:
        loop = asyncio.get_event_loop()
        loop.run_until_complete(self._compute(method_map, **kwargs))
        loop.close()


import tkinter as tk
from tkinter import filedialog

from kivy.app import App
from kivy.uix.screenmanager import Screen

import client.utils as utils
from client.screens.common import *
from client.utils import ApiException
from client.utils import background
from definitions import ROOT_DIR

# Load corresponding kivy file
Builder.load_file(
    os.path.join(
        ClientConfig.DATA_DIR,
        'project_tool_screen.kv'))

class ProjectToolScreen(Screen):
    def __init__(self, **kw):
        super().__init__(**kw)
        self.app = App.get_running_app()

    def upload_images(self, *args):
        root = tk.Tk()
        root.withdraw()
        filepath = filedialog.askdirectory(initialdir=ROOT_DIR)
        image_paths = []
        if isinstance(filepath, str):
            for (root, _, filename) in os.walk(filepath):
                for f in filename:
                    # tkinter does not return windows style filepath
                    image_paths.append(root + '/' + f)
        self._upload_images(self.app.current_project_id, image_paths)

        self._upload_images(self.app.current_project_id, image_paths)

    @background
    def _upload_images(self, pid, image_paths):
        resp = utils.add_project_images(pid, image_paths)
        if resp.status_code == 200:
            result = resp.json()
            msg = []
            for row in result["results"]:
                if "error" in row:
                    msg.append(row["error"]["message"])
            msg = '\n'.join(msg)
            raise ApiException(
                message="The following errors occurred while trying to upload images:\n %s" %
                (msg,), code=resp.status_code)
        elif resp.status_code != 201:
            raise ApiException(
                "Failed to upload images to project.",
                resp.status_code)
        else:
            pop_up = Alert()
            pop_up.title = "Success!"
            pop_up.alert_message = "Successfully added new images to the '%s' project" % self.app.current_project_name
            pop_up.open()