""" rearch...
NEXT NEED more TESTS!  + split to modules and... and next function for new field  ! MAP_ALL
lines 91, 1307,

-help for each command (from docstrings?)
-split to modules
-add e-mail commands
-add block in phone and mail commands
-add detail and related... commands
-Cweb2 + video
-описати абстрактний базовий клас для представлень користувача і конкретні реалізації, які наслідують базовий клас і реалізують консольний інтерфейс.

"""
from collections import UserDict
from datetime import datetime, timedelta
import os
import pickle
import re
import sys
from typing import List, NoReturn, Union

AMBUSH = 'AMBUSH!'
TO_NEXT_FILE_NAME = 'new_one_'
DEFAULT_FILE_ADDRESS_BOOK = 'ABook.data'
LIMIT_RECORDS_TO_DISPLAY = 10

PREFORMATING_PHONE = r'^\+[0-9)(-]{12,16}$'
PREFORMATING_EMAIL1 = r'\b[a-zA-z][\w_.]+@[a-zA-z]+\.[a-zA-z]{2,}[ ]'
PREFORMATING_EMAIL2 = r'\b[a-zA-z][\w_.]+@[a-zA-z]+\.[a-zA-z]+\.[a-zA-z]{2,}'

ERROR_MESSAGE = {
    'UnpicklingError':['The File',' is corrupted, my apologies.',],
    'UnpicklingOthers':['Is the File',' corrupted? ',],
    'OpenFile':['No access to File',', system error: ',],
    'WriteFile':['No access to write File',', error: ',],
    'PicklingError':['Can\'t save object in file',', my apologies.',],
    'PicklingOthers':['Something was wrong. File',' is not updated. ',],
    'UnknownCommand':['Unknown command ...',],
    'UnpredictableError':['Unpredictable error: No contact record available?...',],
    'UnexpectedError':['Unexpected error!: ',],
    '':['',],
    '':['',],
    '':['',],}

WARNING_MESSAGE = {
    'name':'At the beginning there can be only a Latin letter!',
    'birthday':'Incorrect or nonexistent date, entry must be in year-month-day format (YYYY-MM-DD).',
    'phone':'The number is obviously incorrect, the value should start with \"+\" and have 12 digits.',
    'email':'The e-mail is obviously incorrect.',
    'main':'Something happened. Will you try again?',
    'name is omitted':'Give me name OR name and phone please.\n',
    'the contact exists':'Such an entry is already in the book. Add or change a number.',
    'invalid name':'A name cannot begin with a number and can only begin with Latin characters!\n',
    'invalid phone':'There are no valid phone numbers.\nThe number must be in the following format with 12 digits(d): +dd(ddd)ddd-dddd .\n',
    'empty record to add':'There were no new entries to add.\n',
    'unsuccessful save':'Failed to save file.',
    'name and phone omitted':'Give me name and new phone(s) please.\n',
    'unknown name':'The user is unknown. There are no records for this name yet. Create it first.',
    'no address book':'No contact records available.\n',
    'name and birthday omitted':'Give me a name and birthday, please.\n',
    'invalid birthday entry':'The year of birth is not correct! A person too old or too young.\n',
    'invalid birthday':'The calendar date is not possible!\n',
    'name and 2 phones omitted':'Give me name and 2 phones please (current and new)\n',
    'no search query':'There is no search query\n',
    'name is missing':'Give me a name too, please.\n',
    '':'',
    '':'',}

OTHER_MESSAGE = {
    'Record': ['\n\nRecord(Name: ', '; Phones: ', '; Birthday: ', ';\n\te-mail: ', ';\n\t details: ', ':\n\t related information: ',],
    'RBirthday':['Birthday already recorded for ','. You can change it.','Birthday not specified for ','. You can add it.',],
    'RPhone':[' already recorded for ','No phone(s) entry in record ',' not specified in the contact ',],
    'ABook':['AddressBook(Records:',],
    'next_page':['Press Enter for next Volume... ',],
    'Bye':['Good bye!',],
    'START':['Can I help you?\n',],
    'Unknown':['It is unclear. Unknown command...',],
    'Hello':['Hello! So...\n',],
    'successful addition':['A record(s) have been added.\n',],
    'update successful':['A record have been added. Address book file has been saved.',],
    'no changes':['No changes have been made.\n',],
    'deleting successful':['Record successfully deleted. Results saved.',],
    'deleting field':['Field record deleted successfully. Results saved.',],
    'no new entries':['There were no entries to add.\n',],
    'found':['Entries found in your contact book:',', birthday: ','(days to next birthday: ','. Will be ',' yrs. old)\n-> phone(s): ',', birthday: unknown\n-> phone(s): ','','',],
    'all list':['Entries in your contact book:',],
    'all commands list':['All commands in current version Personal Virtual Assistant:\n', 'Detailed for each command: \"-h\" after them.\n'],  #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    '':['',],
    '':['',],}

class AddressBook(UserDict):
    """A class of Address book."""

    def __str__(self) -> str:
        ABOOK0 = OTHER_MESSAGE.get('ABook', [AMBUSH])[0]
        return f'{ABOOK0}{self.data})'

    def add_record(self, record) -> None:  # record: Record
        """Adds a new record to the address book dictionary."""
        self.data[record.name.value] = record

    def iterator(self, n_count: int) -> list:
        """Return(yield) of n_count records of all AddressBook."""
        current_value = 0
        dictionary_iterator = iter(self.data)

        while current_value < len(self.data):
            volume = []

            for i in range(n_count):

                try:
                    volume.append(self.data[next(dictionary_iterator)])

                except StopIteration:
                    current_value = len(self.data)

            yield volume

            current_value += n_count

    def remove_record(self, name: str) -> None:
        """Remove a record from the address book dictionary."""
        self.data.pop(name)

class Field:  # superclass for all base fields
    """A base class with a simple field."""

    def __init__(self):
        self.__value = None
        self.__bloke = None

    def __str__(self):
        return f'{self.value}'

    @property
    def value(self):
        return self.__value

    @value.setter
    def value(self, new_value: str):
        self.__value = new_value

    @property
    def bloke(self):
        return self.__bloke

    @bloke.setter
    def bloke(self, new_bloke: str):
        self.__bloke = new_bloke

class Name(Field):
    """Class of user name."""
    @Field.value.setter
    def value(self, new_value: str):

        if new_value[0].isalpha():  # not in 'ьъыЫЬЪ\'"[]_0123456789!@$%^&*()-+?<>~`|\\/'
            self._Field__value = new_value

        else:
            print(WARNING_MESSAGE.get('name', AMBUSH))

class Birthday(Field):
    """Class of Birthday data."""
    @Field.value.setter
    def value(self, new_value: str):
        birthday_data = datetime.strptime(new_value, '%Y-%m-%d')

        if birthday_data:
            self._Field__value = birthday_data

        else:
            print(WARNING_MESSAGE.get('birthday', AMBUSH))

    def __str__(self) -> str:
        return f'{self.value.date()}'

class Phone(Field):
    """Class of phone number."""
    @Field.value.setter
    def value(self, new_value: str):

        if re.search(PREFORMATING_PHONE, new_value):
            self._Field__value = self.__preformating(new_value)

        else:
            print(WARNING_MESSAGE.get('phone', AMBUSH))

    @Field.bloke.setter
    def bloke(self, new_bloke: str):
        self._Field__bloke = new_bloke

    @staticmethod
    def __preformating(value: str) -> str:
        """Preformating of phone string into the form +dd(ddd)ddddddd."""
        value = value.replace('-', '').replace('(', '').replace(')', '')

        value = '('.join((value[: 3], value[3:]))
        value = ')'.join((value[: 7], value[7:]))

        return value

class Details(Field):
    """Details information of user."""
    @Field.value.setter
    def value(self, new_value: str):
        self._Field__value = new_value

    @Field.bloke.setter
    def bloke(self, new_bloke: str):
        self._Field__bloke = new_bloke

class Email(Field):
    """Class of user Email."""
    @Field.value.setter
    def value(self, new_value: str):

        if re.search(PREFORMATING_EMAIL1, new_value) or\
             re.search(PREFORMATING_EMAIL2, new_value):
            self._Field__value = self.new_value.strip()

        else:
            print(WARNING_MESSAGE.get('email', AMBUSH))

    @Field.bloke.setter
    def bloke(self, new_bloke: str):
        self._Field__bloke = new_bloke

class RelatedInformation(Field):
    """Related information of user."""
    @Field.value.setter
    def value(self, new_value: str):
        self._Field__value = new_value

    @Field.bloke.setter
    def bloke(self, new_bloke: str):
        self._Field__bloke = new_bloke

class Record:
    """Record class of users information."""

    def __init__(self, name: str, *phones: str):
        self.name = Name()
        self.name.value = name
        self.phones = []
        self.birthday = None
        self.emails = []
        self.details = None
        self.related_info = None

        if phones:

            for phone in phones:
                self.add_phone(phone)

    def __str__(self) -> str:
        NAME = OTHER_MESSAGE.get('Record', [AMBUSH])[0]
        PHONES = OTHER_MESSAGE.get('Record', [AMBUSH]*2)[1]
        BIRTHDAY = OTHER_MESSAGE.get('Record', [AMBUSH]*3)[2]
        EMAIL = OTHER_MESSAGE.get('Record', [AMBUSH]*4)[3]
        DETAILS = OTHER_MESSAGE.get('Record', [AMBUSH]*5)[4]
        RELATED = OTHER_MESSAGE.get('Record', [AMBUSH]*6)[5]

        return f'{NAME}{self.name}{PHONES}{self.phones}{BIRTHDAY}'\
            f'{self.birthday}{EMAIL}{self.emails}{DETAILS}{self.details}'\
            f'{RELATED}{self.related_info})'

    def add_birthday(self, birthday: str) -> tuple:
        """Adds a new entry for the user's birthday to the address book."""
        if not self.birthday:

            self.birthday = Birthday()
            self.birthday.value = birthday

            return True,

        else:
            BIRTHDAY0 = OTHER_MESSAGE.get('RBirthday', [AMBUSH])[0]
            BIRTHDAY1 = OTHER_MESSAGE.get('RBirthday', [AMBUSH]*2)[1]
            return False, f'{BIRTHDAY0}\"{self.name.value}\"{BIRTHDAY1}'

    def add_phone(self, phone_new: str) -> bool:
        """Adds a new entry for the user's phone to the address book."""
        phone_new1 = Phone()
        phone_new1.value = phone_new

        for phone in self.phones:

            if phone_new1 == phone.value:

                PHONE0 = OTHER_MESSAGE.get('RPhone', [AMBUSH])[0]
                print(f'\"{phone_new1}\"{PHONE0}\"{self.name.value}\"')

                return False

        self.phones.append(phone_new1)

        return True

    def change_birthday(self, birthday: str) -> tuple:
        """Modify an existing user's birthday entry in the address book."""
        if not self.birthday:

            BIRTHDAY2 = OTHER_MESSAGE.get('RBirthday', [AMBUSH]*3)[2]
            BIRTHDAY3 = OTHER_MESSAGE.get('RBirthday', [AMBUSH]*4)[3]
            return False, f'{BIRTHDAY2}\"{self.name.value}\"{BIRTHDAY3}'

        else:

            self.birthday = Birthday()
            self.birthday.value = birthday

            return True,

    def change_phone(self, phone_to_change: str, phone_new: str) -> tuple:
        """Modify an existing user's phone entry in the address book."""
        phone_to_change = Phone._Phone__preformating(phone_to_change)
        phone_new = Phone._Phone__preformating(phone_new)
        verdict = False

        for phone in self.phones:

            if phone.value == phone_new:  # new number already in record
                PHONE0 = OTHER_MESSAGE.get('RPhone', [AMBUSH])[0]
                return False, f'\"{phone_new}\"{PHONE0}\"{self.name.value}\"'

            if phone.value == phone_to_change:  # old number not exist in record
                verdict = True

        if not verdict:
            PHONE2 = OTHER_MESSAGE.get('RPhone', [AMBUSH]*3)[2]
            return verdict, f'\"{phone_to_change}\"{PHONE2}\"{self.name.value}\"'

        for index, phone in enumerate(self.phones):

            if phone.value == phone_to_change:
                phone_new_to = Phone()
                phone_new_to.value = phone_new
                self.phones.remove(phone)
                self.phones.insert(index, phone_new_to)

                return True,

    def days_to_birthday(self) -> int:
        """Count the number of days until the next birthday of the user."""
        if self.birthday:

            user_day = datetime(year=datetime.now().date().year, \
                month=self.birthday.value.month, day=self.birthday.value.day)

            days_left = user_day.date() - datetime.now().date()

            if days_left.days <= 0:

                user_day = datetime(year=datetime.now().date().year + 1, \
                    month=self.birthday.value.month, day=self.birthday.value.day)

                return (user_day.date() - datetime.now().date()).days

            return days_left.days

    def remove_birthday(self) -> Union[bool, None]:
        """Deleting a birthday entry from a user entry in the address book."""
        if self.birthday:
            self.birthday = None
            return True

    def remove_phone(self, phone_to_remove: str) -> Union[bool, None]:
        """Deleting a phone entry from a user entry in the address book."""
        phone_to_remove = Phone._Phone__preformating(phone_to_remove)

        for phone in self.phones:

            if phone.value == phone_to_remove:
                self.phones.remove(phone)

                return True

        PHONE2 = OTHER_MESSAGE.get('RPhone', [AMBUSH]*3)[2]
        print(f'\"{phone_to_remove}\"{PHONE2}\"{self.name.value}\"')

    def years_old(self) -> int:
        """Calculate the number of full years of the user on the next birthday."""
        if self.birthday:

            user_day = datetime(year=datetime.now().date().year, \
                month=self.birthday.value.month, day=self.birthday.value.day)

            return datetime.now().year - self.birthday.value.year \
                if (user_day.date() - datetime.now().date()).days > 0 \
                else datetime.now().year + 1 - self.birthday.value.year

def helper_try_open_file(path_file: str) -> str:
    """Checks if the database file exists and checks if the filename is free if not.
    If exist folder with path_file file name, then return new free file name.
    Return unoccupied name of file (string).

        Parameters:
            path_file (str): Is proposed name of file.

        Returns:
            path_file (str): Unoccupied name of file.
    """
    if os.path.isdir(path_file):

        while os.path.isdir(path_file):

            path_file = os.path.join(path_file.parent, TO_NEXT_FILE_NAME + path_file.name)

    return path_file

def helper_try_load_file(path_file: str) -> Union[AddressBook, str]:
    """Create empty address book (AddressBook) if no file on path_file.
    Or, try to load address book database (read) from file.
    Return loaded address book (AddressBook) and path for address book file.

        Parameters:
            path_file (str): Is existed file (name or name with path).

        Returns:
            contact_dictionary (AddressBook): Loaded from file or new empty class.
            path_file (str): The file name to save the address book in the following steps..
                Unoccupied name of file, if an exception occurred.
    """
    if not os.path.exists(path_file):
        contact_dictionary = AddressBook()
        address_book_saver(contact_dictionary, path_file) #  False if can't save file!
        return contact_dictionary, path_file

    try:
        with open(path_file, 'rb') as fh:
            try:
                contact_dictionary = pickle.load(fh)
                return contact_dictionary, path_file
            except pickle.UnpicklingError:
                ERROR_LOAD0 = ERROR_MESSAGE.get('UnpicklingError', [AMBUSH])[0]
                ERROR_LOAD1 = ERROR_MESSAGE.get('UnpicklingError', [AMBUSH]*2)[1]
                print(f'{ERROR_LOAD0}({path_file}){ERROR_LOAD1}')
                contact_dictionary = AddressBook()

            except Exception as error_: # pickle.PickleError inherits Exception.
                ERROR_LOAD0 = ERROR_MESSAGE.get('UnpicklingOthers', [AMBUSH])[0]
                ERROR_LOAD1 = ERROR_MESSAGE.get('UnpicklingOthers', [AMBUSH]*2)[1]
                print(f'{ERROR_LOAD0}({path_file}){ERROR_LOAD1}\n{repr(error_)}')
                contact_dictionary = AddressBook()

    except OSError as error_:
        ERROR_LOAD0 = ERROR_MESSAGE.get('OpenFile', [AMBUSH])[0]
        ERROR_LOAD1 = ERROR_MESSAGE.get('OpenFile', [AMBUSH]*2)[1]
        print(f'{ERROR_LOAD0}({path_file}){ERROR_LOAD1}\n{type(error_)}: {error_}')
        contact_dictionary = AddressBook()

    except Exception as error_:
        ERROR_LOAD0 = ERROR_MESSAGE.get('OpenFile', [AMBUSH])[0]
        ERROR_LOAD1 = ERROR_MESSAGE.get('OpenFile', [AMBUSH]*2)[1]
        print(f'{ERROR_LOAD0}({path_file}){ERROR_LOAD1}\n{repr(error_)}')
        contact_dictionary = AddressBook()

    while os.path.exists(path_file):
        path_file = os.path.join(path_file.parent, TO_NEXT_FILE_NAME + path_file.name)

    return contact_dictionary, path_file

def address_book_saver(contact_dictionary: AddressBook, path_file: str) -> bool:
    """Save a class AddressBook to a file (path_file).

        Parameters:
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            File save success marker (bool).
    """
    try:

        with open(path_file, 'wb') as db_file:

            try:

                pickle.dump(contact_dictionary, db_file)

            except pickle.PicklingError:
                ERROR_SAVE0 = ERROR_MESSAGE.get('PicklingError', [AMBUSH])[0]
                ERROR_SAVE1 = ERROR_MESSAGE.get('PicklingError', [AMBUSH]*2)[1]
                print(f'{ERROR_SAVE0}({path_file}){ERROR_SAVE1}')
                return False
                # contact_dictionary = AddressBook()

            except Exception as error_: # pickle.PickleError inherits Exception.
                ERROR_SAVE0 = ERROR_MESSAGE.get('PicklingOthers', [AMBUSH])[0]
                ERROR_SAVE1 = ERROR_MESSAGE.get('PicklingOthers', [AMBUSH]*2)[1]
                print(f'{ERROR_SAVE0}({path_file}){ERROR_SAVE1}\n{repr(error_)}')
                return False
                # contact_dictionary = AddressBook()

    except OSError as error_:
        ERROR_WRITE0 = ERROR_MESSAGE.get('WriteFile', [AMBUSH])[0]
        ERROR_WRITE1 = ERROR_MESSAGE.get('WriteFile', [AMBUSH]*2)[1]
        print(f'{ERROR_WRITE0}({path_file}){ERROR_WRITE1}\n{type(error_)}: {error_}')
        return False

    except Exception as error_:
        ERROR_WRITE0 = ERROR_MESSAGE.get('WriteFile', [AMBUSH])[0]
        ERROR_WRITE1 = ERROR_MESSAGE.get('WriteFile', [AMBUSH]*2)[1]
        print(f'{ERROR_WRITE0}({path_file}){ERROR_WRITE1}\n{repr(error_)}')
        return False

    return True

def find_users(search_strings: List[str], record: Record) -> bool:
    """Check a record for matching the search strings.

        Parameters:
            search_strings (List[str]): The data in the list rows in the user's search query.
            record (Record): Record - User account card.

        Returns:
            True or False (bool): Search result (Search success rate).
    """
    name = f'{record.name}'

    for search_string in search_strings:

        if name.find(search_string) >= 0:

            return True

        if record.birthday:

            birthday = f'{record.birthday}'

            if birthday.find(search_string) >= 0:

                return True

        for phone in record.phones:
            candidate = f"{phone.value}"
            candidate = candidate.replace(
                '-', '').replace('+', '').replace('(', '').replace(')', '')

            if candidate.find(search_string.replace('-', '').replace('+', '').replace('(', '').replace(')', '')) >= 0:

                return True

    return False

class TheNameIsOmitted(Exception):
    pass

class TheContactIsExist(Exception):
    pass

class TheNameIsIncorrect(Exception):
    pass

class ThePhoneIsIncorrect(Exception):
    pass

class TheNameAndPhoneAreMissing(Exception):
    pass

class TheContactIsNotExist(Exception):
    pass

class NoAddressBook(Exception):
    pass

class TheNameAndBirthdayAreMissing(Exception):
    pass

class InvalidBirthdayEntry(Exception):
    pass

class InvalidBirthday(Exception):
    pass

class TheNameAnd2PhonesAreMissing(Exception):
    pass

class NoSearchQuery(Exception):
    pass

class TheNameIsMissing(Exception):
    pass

def input_error(handler):
    """User error handler (decorator).

        Parameters:
            handler (function): Incoming function.

        Returns:
            exception_function(function): Exception function for handler functions.

    """
    def exception_function(user_command: list, contact_dictionary: AddressBook, path_file: str) -> Union[str, list]:
        """Exception function for handler functions."""
        try:
            VALIDATION_FUNCTIONS[handler.__name__](user_command,\
                 contact_dictionary)
        except KeyError:
            return ERROR_MESSAGE.get('UnknownCommand', AMBUSH)
        except TheNameIsOmitted:
            return WARNING_MESSAGE.get('name is omitted', AMBUSH)
        except TheContactIsExist:
            return WARNING_MESSAGE.get('the contact exists', AMBUSH)
        except TheNameIsIncorrect:
            return WARNING_MESSAGE.get('invalid name', AMBUSH)
        except ThePhoneIsIncorrect:
            return WARNING_MESSAGE.get('invalid phone', AMBUSH)
        except TheNameAndPhoneAreMissing:
            return WARNING_MESSAGE.get('name and phone omitted', AMBUSH)
        except TheContactIsNotExist:
            return WARNING_MESSAGE.get('unknown name', AMBUSH)
        except NoAddressBook:
            return WARNING_MESSAGE.get('no address book', AMBUSH)
        except TheNameAndBirthdayAreMissing:
            return WARNING_MESSAGE.get('name and birthday omitted', AMBUSH)
        except InvalidBirthdayEntry:
            return WARNING_MESSAGE.get('invalid birthday entry', AMBUSH)
        except InvalidBirthday:
            return WARNING_MESSAGE.get('invalid birthday', AMBUSH)
        except TheNameAnd2PhonesAreMissing:
            return WARNING_MESSAGE.get('name and 2 phones omitted', AMBUSH)
        except NoSearchQuery:
            return WARNING_MESSAGE.get('no search query', AMBUSH)
        except TheNameIsMissing:
            return WARNING_MESSAGE.get('name is missing', AMBUSH)

        try:
            result = handler(user_command, contact_dictionary, path_file)
            ERROR_ = ERROR_MESSAGE.get('UnexpectedError', [AMBUSH])[0]
        except KeyError as error:
            return f'{ERROR_}\n{error}\n'
        except ValueError as error:
            return f'{ERROR_}\n{error}\n'
        except IndexError as error:
            return f'{ERROR_}\n{error}\n'
        except Exception as error:
            return f'{ERROR_}\n{error}\n'

        if result is None:
            return ERROR_MESSAGE.get('UnpredictableError', [AMBUSH])[0]

        return result

    return exception_function

'''
def record_update_controller():
    """......."""
    if verdict[0]:
        if address_book_saver(contact_dictionary, path_file):
            return OTHER_MESSAGE.get('update successful', [AMBUSH])[0]
        else:
            return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)
    else:
        no_changes = OTHER_MESSAGE.get('no changes', [AMBUSH])[0]
        return f'{no_changes}{verdict[1]}'
'''

@ input_error
def handler_add(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"add ...": The bot saves a new contact in contact dictionary
    and save it in file(path_file). Instead of ... the user enters
    the name and phone number(s), necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user [and phone(s)]).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    new_record = Record(name)
    contact_dictionary.add_record(new_record)

    if len(user_command) > 2:

        phones = user_command[2:]
        verdict = False

        for new_phone in phones:
            verdict = contact_dictionary[name].add_phone(new_phone) or verdict

        if not verdict:
            return WARNING_MESSAGE.get('empty record to add', AMBUSH)

    if address_book_saver(contact_dictionary, path_file):
        return OTHER_MESSAGE.get('successful addition', [AMBUSH])[0]
    else:
        return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

@ input_error
def handler_add_birthday(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"add birthday...": The bot saves new information about user
    in contact dictionary and save it in file(path_file).
    Instead of ... the user enters the name and birthday (in format YYYY-MM-DD),
    necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user and birthday).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    verdict = contact_dictionary[name].add_birthday(user_command[2])

    if verdict[0]:

        if address_book_saver(contact_dictionary, path_file):
            return OTHER_MESSAGE.get('update successful', [AMBUSH])[0]
        else:
            return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

    else:
        no_changes = OTHER_MESSAGE.get('no changes', [AMBUSH])[0]
        return f'{no_changes}{verdict[1]}'

@ input_error
def handler_add_phone(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"add ...": The bot saves a new phones to contact in contact dictionary
    and save it in file(path_file). Instead of ... the user enters the name
    and phone number(s), necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user and phone(s)).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    phones = user_command[2:]
    verdict = False
    write_count = 0

    for new_phone in phones:
        verdict = contact_dictionary[name].add_phone(new_phone) or verdict
        write_count += 1 if verdict else 0

    if not write_count:
        return OTHER_MESSAGE.get('no new entries', [AMBUSH])[0]

    if address_book_saver(contact_dictionary, path_file):
        return OTHER_MESSAGE.get('update successful', [AMBUSH])[0]
    else:
        return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

@ input_error
def handler_change(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"change ...": The bot stores the new phone number of the existing
    ontact in contact dictionary and save it in file(path_file).
    Instead of ... the user enters the name and phone numbers (current and new),
    necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user and phones).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    current_phone = user_command[2]
    new_phone = user_command[3]
    verdict = contact_dictionary[name].change_phone(current_phone, new_phone)

    if verdict[0]:

        if address_book_saver(contact_dictionary, path_file):
            return OTHER_MESSAGE.get('update successful', [AMBUSH])[0]
        else:
            return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

    else:
        no_changes = OTHER_MESSAGE.get('no changes', [AMBUSH])[0]
        return f'{no_changes}{verdict[1]}'

@ input_error
def handler_change_birthday(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"change birthday ...": The bot stores the
    "new birthday" (if the previous one was wrong)
    of the existing contact in contact dictionary and save it in file(path_file).
    Instead of ... the user enters the name and birthday (in format YYYY-MM-DD),
    necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user, and birthday).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    user_birthday = user_command[2]
    verdict = contact_dictionary[name].change_birthday(user_birthday)

    if verdict[0]:

        if address_book_saver(contact_dictionary, path_file):
            return OTHER_MESSAGE.get('update successful', [AMBUSH])[0]
        else:
            return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

    else:
        no_changes = OTHER_MESSAGE.get('no changes', [AMBUSH])[0]
        return f'{no_changes}{verdict[1]}'

def forming_user_information(record: Record) -> str:
    """Forming user information from a record and returning it as a string.

        Parameters:
            record (Record): User record from the address book.

        Returns:
            volume (str): Formed information about the user from record.
    """
    volume = ''
    FOUND_P1 = OTHER_MESSAGE.get('found', [AMBUSH])[1]
    FOUND_P2 = OTHER_MESSAGE.get('found', [AMBUSH])[2]
    FOUND_P3 = OTHER_MESSAGE.get('found', [AMBUSH])[3]
    FOUND_P4 = OTHER_MESSAGE.get('found', [AMBUSH])[4]
    FOUND_P5 = OTHER_MESSAGE.get('found', [AMBUSH])[5]

    if record.birthday:
        volume += f'\n\n{record.name}{FOUND_P1}{record.birthday}' \
        f'{FOUND_P2}{record.days_to_birthday()}'\
        f'{FOUND_P3}{record.years_old()}{FOUND_P4}'

    else:
        volume += f'\n\n{record.name}{FOUND_P5}'

    for phone in record.phones:
        volume += f'{phone.value}; '

    return volume

@ input_error
def handler_find(user_command: List[str], contact_dictionary: AddressBook, _=None) -> list:
    """"Find ...": The bot outputs a list of users whose name or phone number
    matches the entered one or more(with an OR setting) string without space(' ').

        Parameters:
            user_command (List[str]): List of user command (strimg(s) for searching).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            found_list (list): Answer for the user - list of string of found users.
    """
    found_list = [OTHER_MESSAGE.get('found', [AMBUSH])[0]]

    for records in contact_dictionary.iterator(LIMIT_RECORDS_TO_DISPLAY):
        volume = ''

        for record in records:

            if find_users(user_command[1:], record):

                volume += forming_user_information(record)

        found_list.append(volume)

    return found_list

def handler_hello(*_) -> str:
    """Reply to the greeting."""
    return OTHER_MESSAGE.get('Hello', [AMBUSH])[0]

@ input_error
def handler_phone(user_command: List[str], contact_dictionary: AddressBook, _=None) -> str:
    """"phone ...": The bot outputs the phone number for the specified
    contact. Instead of ... the user enters the name of the contact
    whose number should be displayed.

        Parameters:
            user_command (List[str]): List of user command (name of user).
            contact_dictionary (AddressBook): Instance of AddressBook.
            _: not matter (path_file (str): Is there path and filename of address book).

        Returns:
            string(str): Answer for the user (phone number(s) of user).
    """
    phones = ''
    name = user_command[1]

    for phone in (contact_dictionary[name]).phones:
        phones += f'{phone.value}; '

    return phones

@ input_error
def handler_remove(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"remove ...": The bot remove a record contact in contact dictionary
    and save it in file(path_file). Instead of ... the user enters the name.

        Parameters:
            user_command (List[str]): List of user command (name of user).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]

    if contact_dictionary.get(name, None):

        contact_dictionary.remove_record(name)

        if address_book_saver(contact_dictionary, path_file):
            return OTHER_MESSAGE.get('deleting successful', [AMBUSH])[0]
        else:
            return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

    else:
        return WARNING_MESSAGE.get('unknown name', AMBUSH)

@ input_error
def handler_remove_birthday(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"remove birthday ...": The bot remove a birthday record from contact in contact dictionary
    and save it in file(path_file). Instead of ... the user enters the name.

        Parameters:
            user_command (List[str]): List of user command (name of user).
            contact_dictionary (AddressBook): Instance of AddressBook .
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    if contact_dictionary.get(name, None):

        if contact_dictionary[name].birthday:

            contact_dictionary[name].remove_birthday()

            if address_book_saver(contact_dictionary, path_file):
                return OTHER_MESSAGE.get('deleting field', [AMBUSH])[0]
            else:
                return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

        else:
            BIRTHDAY2 = OTHER_MESSAGE.get('RBirthday', [AMBUSH]*3)[2]
            BIRTHDAY3 = OTHER_MESSAGE.get('RBirthday', [AMBUSH]*4)[3]
            return f'{BIRTHDAY2}\"{name}\"{BIRTHDAY3}'

    else:  # dublicat 'of except TheContactIsNotExist'
        return WARNING_MESSAGE.get('unknown name', AMBUSH)

@ input_error
def handler_remove_phone(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> str:
    """"remove phone ...": The bot remove a phone record from contact in contact dictionary
    and save it in file(path_file). Instead of ... the user enters the name and phone
    number(s), necessarily with a space.

        Parameters:
            user_command (List[str]): List of user command (name of user).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book.

        Returns:
            string(str): Answer for the user.
    """
    name = user_command[1]
    if contact_dictionary.get(name, None):

        if contact_dictionary[name].phones:

            phone = user_command[2]
            verdict = contact_dictionary[name].remove_phone(phone)

            if verdict:

                if address_book_saver(contact_dictionary, path_file):
                    return OTHER_MESSAGE.get('deleting field', [AMBUSH])[0]
                else:
                    return WARNING_MESSAGE.get('unsuccessful save', AMBUSH)

            else:
                PHONE2 = OTHER_MESSAGE.get('RPhone', [AMBUSH]*3)[2]
                return f'\"{phone}\"{PHONE2}\"{name}\".'

        else:
            PHONE1 = OTHER_MESSAGE.get('RPhone', [AMBUSH]*2)[1]
            return f'{PHONE1}\"{name}\".\n'

    else:
        return WARNING_MESSAGE.get('unknown name', AMBUSH)

@ input_error
def handler_show(user_command: List[str], contact_dictionary: AddressBook, _=None) -> str:
    """"show information about a specific user". With this command, the bot outputs
    birthday, number of days until next birthday and phone numbers to the console.

        Parameters:
            user_command (List[str]): List of user command (name of user [and phone(s)]).
            contact_dictionary (AddressBook): Instance of AddressBook.
            _: not matter (path_file (str): Is there path and filename of address book).

        Returns:
            string(str): Answer for the user (string of information about user).
    """
    name = user_command[1]

    return forming_user_information(contact_dictionary[name])

@ input_error
def handler_show_all(_, contact_dictionary: AddressBook, _a) -> list:
    """"show all": The bot outputs all saved contacts.

        Parameters:
            _: not matter (user_command (List[str]): List of user command).
            contact_dictionary (AddressBook): Instance of AddressBook.
            _a: not matter (path_file (str): Is there path and filename of address book).

        Returns:
            string(str): Answer for the user (list of string of all users).
    """
    all_list = [OTHER_MESSAGE.get('all list', [AMBUSH])[0]]

    for records in contact_dictionary.iterator(10):  # n_count from?
        volume = ''

        for record in records:

            volume += forming_user_information(record)

        all_list.append(volume)

    return all_list

def validation_add(user_command: list, contact_dictionary: AddressBook) ->\
        None:
    """Check the input parameters. Raise a mismatch exception if found."""
    name = user_command[1] if len(user_command) > 1 else None

    if not name:  # len(user_command) < 2:
        raise TheNameIsOmitted

    if name in contact_dictionary:
        raise TheContactIsExist

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if len(user_command) >= 2:
        phone_count = 0

        for phone_candidate in user_command[2:]:
            phone_matches = re.search(PREFORMATING_PHONE, phone_candidate)

            if phone_matches:
                phone_count += 1

            else:
                print(f'{phone_candidate}', WARNING_MESSAGE.get('phone', AMBUSH))

        if phone_count < len(user_command[2:]):  # not phone_count:
            raise ThePhoneIsIncorrect

def validation_add_phone(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if len(user_command) < 3:  # or not name:
        raise TheNameAndPhoneAreMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if name not in contact_dictionary:
        raise TheContactIsNotExist

    phone_count = 0

    for phone_candidate in user_command[2:]:
        phone_matches = re.search(PREFORMATING_PHONE, phone_candidate)

        if phone_matches:
            phone_count += 1

        else:
            print(f'{phone_candidate}', WARNING_MESSAGE.get('phone', AMBUSH))

    if phone_count < len(user_command[2:]):  # not phone_count:
        raise ThePhoneIsIncorrect

def validation_birthday(user_command: list, contact_dictionary: AddressBook) -> Union[str, None]:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if len(user_command) < 3:  # or not name:
        raise TheNameAndBirthdayAreMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if datetime.now().year - 122 > int(user_command[2].split('-')[0]) > datetime.now().year - 8:  # 0, 8, 14, 16, 18 ?
        raise InvalidBirthdayEntry

    else:
        try:
            datetime.strptime(user_command[2], '%Y-%m-%d')

        except ValueError:
            raise InvalidBirthday

def validation_change(user_command: list, contact_dictionary: AddressBook) -> \
        None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if len(user_command) < 4:  # or not name:
        raise TheNameAnd2PhonesAreMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    phone_count = 0

    for phone_candidate in user_command[2:]:
        phone_matches = re.search(PREFORMATING_PHONE, phone_candidate)

        if phone_matches:
            phone_count += 1

        else:
            print(f'{phone_candidate}', WARNING_MESSAGE.get('phone', AMBUSH))

    if phone_count < len(user_command[2:]):  # not phone_count:
        raise ThePhoneIsIncorrect

def validation_find(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    query = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not query:
        raise NoSearchQuery

def validation_phone(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not name:  # len(user_command) < 2 or not name:
        raise TheNameIsMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

def validation_remove(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not name:
        raise TheNameIsMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if name not in contact_dictionary:
        raise TheContactIsNotExist

def validation_remove_birthday(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not name:  # len(user_command) < 2:
        raise TheNameIsMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if name not in contact_dictionary:
        raise TheContactIsNotExist

def validation_remove_phone(user_command: list, contact_dictionary: AddressBook) -> \
        None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not name:
        raise TheNameIsMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

    if name not in contact_dictionary:
        raise TheContactIsNotExist

    phone_count = 0

    for phone_candidate in user_command[2:]:
        phone_matches = re.search(PREFORMATING_PHONE, phone_candidate)

        if phone_matches:
            phone_count += 1

        else:
            print(f'{phone_candidate}', WARNING_MESSAGE.get('phone', AMBUSH))

    if phone_count < 2:  # len(user_command[2:])  # not phone_count:
        raise ThePhoneIsIncorrect

def validation_show(user_command: list, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    name = user_command[1] if len(user_command) > 1 else None

    if not contact_dictionary:
        raise NoAddressBook

    if not name:
        raise TheNameIsMissing

    if name[0].isdigit() or not name[0].isalpha():
        raise TheNameIsIncorrect

def validation_showall(_, contact_dictionary: AddressBook) -> None:
    """Check the input parameters. Return a message (str) about a discrepancy if it is detected."""
    if not contact_dictionary:
        raise NoAddressBook

def handler_exit(*_) -> str:
    """Reply to the exit command."""
    return OTHER_MESSAGE.get('Bye', [AMBUSH])[0]

def handler_help(*_):
    """
    Print all commands and examples.  # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    """
    all_commands_list = [OTHER_MESSAGE.get('all commands list', [AMBUSH])[0]]
    all_commands_list += [f'''{key.replace('_',' ')}\n''' for key in ALL_COMMAND]
    return ''.join(all_commands_list)

ALL_COMMAND = {'hello': handler_hello,
                'add': handler_add,
                'add_phone': handler_add_phone,
                'change': handler_change,
                'phone': handler_phone,
                'show_all': handler_show_all,
                'good_bye': handler_exit,
                'close': handler_exit,
                'exit': handler_exit,
                'show': handler_show,
                'add_birthday': handler_add_birthday,
                'change_birthday': handler_change_birthday,
                'find': handler_find,
                'remove': handler_remove,
                'remove_phone': handler_remove_phone,
                'remove_birthday': handler_remove_birthday,
                'help': handler_help,
                '?': handler_help,
                }

VALIDATION_FUNCTIONS = {
            'handler_add': validation_add,
            'handler_add_birthday': validation_birthday,
            'handler_add_phone': validation_add_phone,
            'handler_change': validation_change,
            'handler_change_birthday': validation_birthday,
            'handler_find': validation_find,
            'handler_phone': validation_phone,
            'handler_remove': validation_remove,
            'handler_remove_birthday': validation_remove_birthday,
            'handler_remove_phone': validation_remove_phone,
            'handler_show': validation_show,
            'handler_show_all': validation_showall,
            # 'unknown': lambda *_: raise UnknownCommand,  # 'Unknown command...'
        }

def main_handler(user_command: List[str], contact_dictionary: AddressBook, path_file: str) -> Union[str, list]:
    """All possible bot commands. Get a list of command and options,
    a dictionary of contacts, and the path to an address book file.
    Call a certain function and return a response to a command request.

        Parameters:
            user_command (List[str]): list of user command (list of command and options).
            contact_dictionary (AddressBook): Instance of AddressBook.
            path_file (str): Is there path and filename of address book (in str).

        Returns:
            The result of the corresponding function (list): The result of the\
                 certain function is a string or a list of strings.
    """
    return ALL_COMMAND.get(user_command[0], lambda *args: None)(user_command, \
        contact_dictionary, path_file) \
        or OTHER_MESSAGE.get('Unknown', [AMBUSH])[0]

def parser(user_input: str) -> List[str]:
    """Command parser. Get a user string - separate the command and parameters -
    return it to the list, where the first element is the command,
    the others are parameters.

        Parameters:
            user_input (str): String line of user input.

        Returns:
            list command of user input (list): list of comands (list of strings).
    """
    command_line = user_input.strip().replace(' ','~').lower()
    all_commands = sorted([el.replace('_','~') for el in ALL_COMMAND], key=len)[::-1]
    for command in all_commands:
        if command_line.startswith(command):
            command.replace('~','_')
            return [command.replace('~','_')] + [word for word in user_input[len(command):].split(' ') if word]

    return user_input.strip().split(' ')  #  OTHER_MESSAGE.get('Unknown', [AMBUSH])[0]

def main() -> NoReturn:
    """The main function of launching a helper console bot that
    recognize the commands entered from the keyboard and respond
    according to the command entered.
    Enter a command - get an answer.
    """
    try:
        path_file = sys.argv[1]

    except IndexError:
        path_file = DEFAULT_FILE_ADDRESS_BOOK

    new_path_file = helper_try_open_file(path_file)

    contact_dictionary, new_path_file = helper_try_load_file(new_path_file)

    while True:
        user_command = input(OTHER_MESSAGE.get('START', [AMBUSH])[0])
        user_request = parser(user_command)  # ['remove_phone', '+38000001', '+5555578']
        bot_answer = main_handler(
            user_request, contact_dictionary, new_path_file)

        if isinstance(bot_answer, str):
            print(bot_answer)

        elif isinstance(bot_answer, list):

            for volume in bot_answer:

                if volume:

                    print(volume)
                    input(OTHER_MESSAGE.get('next_page', [AMBUSH])[0])

        else:
            print(WARNING_MESSAGE.get('main', AMBUSH))

        if bot_answer == OTHER_MESSAGE.get('Bye', [AMBUSH])[0]:
            break

if __name__ == '__main__':
    main()


import os
import logging
from functools import wraps
from urllib.parse import urlparse
import tempfile
from typing import List, Optional

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig, ZincFlavorSpec
from zinc.defaults import defaults
from zinc.formats import Formats
import zinc.helpers as helpers
import zinc.utils as utils

log = logging.getLogger(__name__)

################################################################################

# TODO: rename to CatalogPathHelper?
class ZincCatalogPathHelper(object):

    def __init__(self, format: str = '1'):
        if format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (format))
        self._format = format

    @property
    def format(self):
        return self._format

    @property
    def manifests_dir(self):
        return "manifests"

    @property
    def archives_dir(self):
        return "archives"

    @property
    def objects_dir(self):
        return "objects"

    @property
    def config_dir(self):
        return "config"

    @property
    def config_flavorspec_dir(self) -> str:
        return os.path.join(self.config_dir, "flavorspecs")

    def path_for_index(self) -> str:
        return defaults['catalog_index_name']

    def manifest_name(self, bundle_name: str, version: int) -> str:
        return "%s-%d.json" % (bundle_name, version)

    def path_for_manifest_for_bundle_version(self, bundle_name: str, version: int) -> str:
        manifest_filename = self.manifest_name(bundle_name, version)
        manifest_path = os.path.join(self.manifests_dir, manifest_filename)
        return manifest_path

    def path_for_manifest(self, manifest: ZincManifest) -> str:
        return self.path_for_manifest_for_bundle_version(
            manifest.bundle_name, manifest.version)

    def path_for_file_with_sha(self, sha: str, ext: Optional[str] = None, format=None) -> str:

        if ext is not None and format is not None:
            raise Exception("Should specify either `ext` or `format`, not both.")

        if format is not None:
            ext = helpers.file_extension_for_format(format)
        subdir = os.path.join(self.objects_dir, sha[0:2], sha[2:4])
        file = sha
        if ext is not None:
            file = file + '.' + ext
        return os.path.join(subdir, file)

    def archive_name(self, bundle_name: str, version: int, flavor: Optional[str] = None):
        if flavor is None:
            return "%s-%d.tar" % (bundle_name, version)
        else:
            return "%s-%d~%s.tar" % (bundle_name, version, flavor)

    def path_for_archive_for_bundle_version(
            self, bundle_name: str, version: int, flavor: Optional[str] = None):
        archive_filename = self.archive_name(bundle_name, version, flavor=flavor)
        archive_path = os.path.join(self.archives_dir, archive_filename)
        return archive_path

    def path_for_flavorspec_name(self, flavorspec_name: str):
        filename = '%s.json' % flavorspec_name
        return os.path.join(self.config_flavorspec_dir, filename)

################################################################################

def _ensure_index_lock(func):
    @wraps(func)
    def with_ensure_index_lock(*args, **kwargs):

        slf = args[0]

        assert slf._coordinator

        if not slf.lock().is_locked():
            with slf.lock():
                output = func(*args, **kwargs)
        else:
            output = func(*args, **kwargs)
        return output
    return with_ensure_index_lock

class ZincAbstractCatalog(object):

    def get_index(self):
        """
        Returns an *immutable* copy of the catalog index.
        """
        raise NotImplementedError()

    def get_manifest(self, bundle_name: str, version: int):
        """
        Returns an *immutable* copy of the manifest for the specified
        `bundle_name` and version`.
        """
        raise NotImplementedError()

    def update_bundle(self, new_manifest: ZincManifest):
        raise NotImplementedError()

    # special
    def import_path(self, src_path: str):
        raise NotImplementedError()

    def delete_bundle_version(self, bundle_name: str, version: int):
        raise NotImplementedError()

    def update_distribution(self, distribution_name: str, bundle_name: str, bundle_version: int, save_previous: bool = True):
        raise NotImplementedError()

    def delete_distribution(self, distribution_name: str, bundle_name: str):
        raise NotImplementedError()

    def verify(self):
        raise NotImplementedError()

    def clean(self, **kwargs):
        raise NotImplementedError()

    # Non-abstract methods

    def manifest_for_bundle(self, bundle_name: str, version: Optional[int] = None) -> Optional[ZincManifest]:
        """
        Get a manifest for bundle. If version is not specified, it gets the
        manifest with the highest version number.
        """
        index = self.get_index()
        all_versions = index.versions_for_bundle(bundle_name)

        if version is not None:
            resolved_version = version
        elif version is None and len(all_versions) > 0:
            resolved_version = all_versions[-1]
        else:
            return None  # throw exception?

        return self.get_manifest(bundle_name, resolved_version)

    def manifest_for_bundle_descriptor(self, bundle_descriptor: str) -> Optional[ZincManifest]:
        """
        Convenience method to get a manifest by bundle_descriptor.
        """
        return self.manifest_for_bundle(
            helpers.bundle_id_from_bundle_descriptor(bundle_descriptor),
            helpers.bundle_version_from_bundle_descriptor(bundle_descriptor))

    def bundle_descriptors(self) -> List[str]:
        bundle_descriptors = []
        index = self.get_index()
        for bundle_name in index.bundle_names():
            for version in index.versions_for_bundle(bundle_name):
                bundle_descriptors.append("%s-%d" % (bundle_name, version))
                manifest = self.manifest_for_bundle(bundle_name, version)
                if manifest is None:
                    log.warn('Could not load manifest for %s-%d' % (bundle_name, version))
                    continue
                for flavor in manifest.flavors:
                    bundle_descriptors.append("%s-%d~%s" % (bundle_name, version, flavor))
        return bundle_descriptors

################################################################################

class ZincCatalogLock(object):

    def __init__(self, catalog, lock):
        self._catalog = catalog
        self._lock = lock

    def __enter__(self):
        self._lock.acquire()
        self._catalog._reload()

    def __exit__(self, exc_type, exc_value, traceback):
        self._catalog.save()
        self._lock.release()

    def is_locked(self):
        return self._lock.is_locked()

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, storage=None, coordinator=None, path_helper=None,
                 lock_timeout=None, **kwargs):
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self.lock_timeout = lock_timeout or defaults['catalog_lock_timeout']

        self._reload()

        if self._coordinator is not None:
            self._lock = ZincCatalogLock(self,
                                         self._coordinator.get_index_lock(domain=self.id,
                                                                          timeout=lock_timeout))

    def lock(self):
        assert self._lock
        return self._lock

    # Properties

    @property
    def url(self):
        return self._storage.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    # General Internal Methods

    def _reload(self):

        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        # log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        # config_path = pjoin(self.path, defaults['catalog_config_name'])
        # self.config = load_config(config_path)

    # I/O Helpers

    def _read(self, rel_path):
        with self._storage.get(rel_path) as f:
            return f.read()
        return None

    def _write(self, subpath: str, bytes: bytes, raw: bool = True, gzip: bool = True, max_age: Optional[int] = None):
        log.debug(f'ZincCatalog: _write() called. (subpath: {subpath}, raw: {raw}, gzip: {gzip})')
        if raw:
            self._storage.puts(subpath, bytes, max_age=max_age)
        if gzip:
            self._storage.puts(subpath + '.gz', utils.gzip_bytes(bytes), max_age=max_age)

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath).decode('utf-8')
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        max_age = defaults['catalog_index_max_age_seconds']
        self._write(subpath, bytes, raw=raw, gzip=gzip, max_age=max_age)
        if defaults['catalog_write_legacy_index']:
            self._write('index.json', bytes, raw=raw, gzip=gzip, max_age=max_age)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(bundle_name,
                                                                version)
        bytes = self._read(subpath).decode('utf-8')
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                    'sha': sha,
                    'size': meta['size'],
                    'format': format
                }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        log.debug(f'ZincCatalog: _write_file() called. (sha: {sha}, src_path: {src_path}, format: {format})')
        format = format or Formats.RAW  # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'rb') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                                                               version,
                                                               flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        log.debug(f'ZincCatalog: _write_archive() called. (bundle_name: {bundle_name}, version: {version},'
                  f' src_path: {src_path}, flavor: {flavor})')
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                                                               version,
                                                               flavor=flavor)
        with open(src_path, 'rb') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                                                               version,
                                                               flavor=flavor)
        return self._storage.get(subpath)

    @_ensure_index_lock
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    # "Public" Methods

    def save(self):
        self._write_index(self.index)

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name: str, version: int) -> ZincManifest:
        return self._read_manifest(bundle_name, version)

    @_ensure_index_lock
    def update_bundle(self, new_manifest: ZincManifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        # verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        # TODO: verify archives?

        # write manifest

        self._write_manifest(new_manifest)

        # update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path: str):

        sha = utils.sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path, 'rb') as src_file:
                tmp_file.write(utils.gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(sha, final_src_path, format=format)

        file_info = {
            'sha': sha,
            'size': final_src_size,
            'format': format
        }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return file_info

    @_ensure_index_lock
    def delete_bundle_version(self, bundle_name: str, version: int):
        self.index.delete_bundle_version(bundle_name, version)

    @_ensure_index_lock
    def update_distribution(self, distribution_name: str, bundle_name: str, bundle_version: int, save_previous: bool = True):

        if save_previous:
            cur_version = self.index.version_for_bundle(bundle_name, distribution_name)
            if cur_version is not None and cur_version != bundle_version:
                prev_distro = helpers.distro_previous_name(distribution_name)
                self.index.update_distribution(prev_distro, bundle_name, cur_version)

        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_ensure_index_lock
    def delete_distribution(self, distribution_name: str, bundle_name: str, delete_previous: bool = True):
        self.index.delete_distribution(distribution_name, bundle_name)
        if delete_previous:
            prev_distro = helpers.distro_previous_name(distribution_name)
            self.index.delete_distribution(prev_distro, bundle_name)

    def get_flavorspec_names(self) -> List[str]:
        subpath = self.path_helper.config_flavorspec_dir
        return [os.path.splitext(p)[0] for p in self._storage.list(prefix=subpath)]

    def get_flavorspec(self, flavorspec_name: str) -> ZincFlavorSpec:
        subpath = self._ph.path_for_flavorspec_name(flavorspec_name)
        bytes = self._read(subpath)
        return ZincFlavorSpec.from_bytes(bytes)

    def update_flavorspec_from_json_string(self, name: str, json_string: str):
        subpath = self._ph.path_for_flavorspec_name(name)
        self._write(subpath, json_string.encode('utf8'), raw=True, gzip=False)

    def update_flavorspec_from_path(self, src_path: str, name: Optional[str] = None):
        with open(src_path, 'r') as src_file:
            json_string = src_file.read()
        if name is None:
            name = os.path.splitext(os.path.basename(src_path))[0]
        self.update_flavorspec_from_json_string(name, json_string)

    def delete_flavorspec(self, name: str):
        subpath = self._ph.path_for_flavorspec_name(name)
        self._storage.delete(subpath)

    @_ensure_index_lock
    def clean(self, dry_run: bool = False):
        verb = 'Would remove' if dry_run else 'Removing'

        bundle_descriptors = self.bundle_descriptors()

        # 1. scan manifests for ones that aren't in index

        dir = self._ph.manifests_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".json") or f.endswith(".json.gz")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        # 2. scan archives for ones that aren't in index

        dir = self._ph.archives_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".tar")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        # 3. clean objects

        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            if manifest is None:
                continue
            for f, meta in manifest.files.items():
                all_objects.add(meta['sha'])

        dir = self._ph.objects_dir
        for path in self._storage.list(dir):

            basename = os.path.basename(path)
            obj = os.path.splitext(basename)[0]

            if obj not in all_objects:
                subpath = os.path.join(dir, path)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)


import pytest
import pandas as pd
from polyfuzz.models import TFIDF
from tests.utils import get_test_strings

from_list, to_list = get_test_strings()

@pytest.mark.parametrize("method", ["sparse", "knn", "sklearn"])
def test_distance(method):
    model = TFIDF(cosine_method=method)
    matches = model.match(from_list, to_list)

    assert isinstance(matches, pd.DataFrame)
    assert matches.Similarity.mean() > 0.0
    assert len(matches) == 6
    assert list(matches.columns) == ['From', 'To', 'Similarity']

@pytest.mark.parametrize("n_gram_low, n_gram_high", [(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)])
def test_ngrams(n_gram_low, n_gram_high):
    model = TFIDF(n_gram_range=(n_gram_low, n_gram_high))
    matches = model.match(from_list, to_list)

    assert isinstance(matches, pd.DataFrame)
    assert len(matches) == 6
    assert list(matches.columns) == ['From', 'To', 'Similarity']


from deeplake.core.query.autocomplete import autocomplete
import deeplake

def _test_ds():
    ds = deeplake.dataset("mem://x")
    ds.create_tensor("a")
    ds.create_tensor("b")
    ds.create_tensor("c")
    ds.create_tensor("def")
    ds.create_tensor("g/h/i")
    ds.create_tensor("j/k/l")
    return ds

def test_empty_query():
    ds = _test_ds()
    q = ""
    resp = autocomplete(q, ds)
    assert resp["tokens"] == []
    suggestions = [s["string"] for s in resp["suggestions"]]
    assert suggestions == ["a", "b", "c", "def", "g", "j"]
    assert resp["replace"] == ""

def test_tensor_name():
    ds = _test_ds()
    for q in ["a", "b", "c", "def"]:
        resp = autocomplete(q, ds)
        tokens = resp["tokens"]
        assert len(tokens) == 1
        token = tokens[0]
        assert token["string"] == q
        assert token["start"] == 0
        assert token["end"] == len(q)
        assert token["type"] == "TENSOR"
        suggestions = resp["suggestions"]
        assert resp["replace"] == ""
        assert suggestions == [
            {"string": ".contains", "type": "METHOD"},
            {"string": ".max", "type": "PROPERTY"},
            {"string": ".mean", "type": "PROPERTY"},
            {"string": ".min", "type": "PROPERTY"},
            {"string": ".shape", "type": "PROPERTY"},
            {"string": ".size", "type": "PROPERTY"},
            {"string": " ==", "type": "OP"},
            {"string": " >", "type": "OP"},
            {"string": " <", "type": "OP"},
            {"string": " >=", "type": "OP"},
            {"string": " <=", "type": "OP"},
            {"string": " !=", "type": "OP"},
        ]

def test_tensor_name_partial():
    ds = _test_ds()
    for q in ["d", "de"]:
        resp = autocomplete(q, ds)
        tokens = resp["tokens"]
        assert len(tokens) == 1
        token = tokens[0]
        assert token["string"] == q, (q, token)
        assert token["start"] == 0
        assert token["end"] == len(q)
        assert token["type"] == "UNKNOWN"
        assert resp["replace"] == q
        suggestions = resp["suggestions"]
        assert suggestions == [{"string": "def", "type": "TENSOR"}]

def test_group():
    ds = _test_ds()
    q = "g"
    resp = autocomplete(q, ds)
    suggestions = resp["suggestions"]
    assert suggestions == [
        {
            "string": ".h",
            "type": "GROUP",
        }
    ]
    q = "g.h"
    resp = autocomplete(q, ds)
    suggestions = resp["suggestions"]
    assert suggestions == [
        {
            "string": ".i",
            "type": "TENSOR",
        }
    ]

    q = "g.h.i"
    resp = autocomplete(q, ds)
    suggestions = resp["suggestions"]
    assert suggestions == [
        {"string": ".contains", "type": "METHOD"},
        {"string": ".max", "type": "PROPERTY"},
        {"string": ".mean", "type": "PROPERTY"},
        {"string": ".min", "type": "PROPERTY"},
        {"string": ".shape", "type": "PROPERTY"},
        {"string": ".size", "type": "PROPERTY"},
        {"string": " ==", "type": "OP"},
        {"string": " >", "type": "OP"},
        {"string": " <", "type": "OP"},
        {"string": " >=", "type": "OP"},
        {"string": " <=", "type": "OP"},
        {"string": " !=", "type": "OP"},
    ]

    q = "g.h.k"
    resp = autocomplete(q, ds)
    assert resp["suggestions"] == []

def test_keyword():
    ds = _test_ds()
    q = "'x' in "
    resp = autocomplete(q, ds)
    suggestions = [s["string"] for s in resp["suggestions"]]
    assert "a" in suggestions
    assert "b" in suggestions
    assert "c" in suggestions
    assert "def" in suggestions
    assert "g" in suggestions
    assert "j" in suggestions

def test_const():
    ds = _test_ds()
    for q in ["True", "'abcd'", "123"]:
        resp = autocomplete(q, ds)
        assert resp["suggestions"] == []

def test_property():
    ds = _test_ds()
    q = "g.h.i.mean"
    resp = autocomplete(q, ds)
    suggestions = [s["string"] for s in resp["suggestions"]]
    assert " ==" in suggestions

def test_method():
    ds = _test_ds()
    q = "g.h.i.contains"
    resp = autocomplete(q, ds)
    suggestions = [s["string"] for s in resp["suggestions"]]
    assert "(" in suggestions


from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
from random import shuffle
import pickle as pkl
import glob
import ujson
from sklearn import svm
"""
Inspired by https://github.com/karpathy/arxiv-sanity-lite/blob/master/serve.py
"""

class Analyzer:

    def __init__(self, max_train: int = 5000) -> None:
        self.max_train = max_train
        # get the stopwords for PL
        with open("data/stopwords.txt", "r") as f:
            self.stopwords = f.read().splitlines()

    def make_corpus(self, training: bool) -> list:
        """Create a corpus of all the articles."""
        all_sources = [
            "data/tvp_articles",
            "data/wp_articles",
        ]
        all_article_paths = []
        for src in all_sources:
            all_article_paths.extend(glob.glob(f"{src}/*.json"))
        print(f"Found {len(all_article_paths)} articles.")
        if training:
            shuffle(all_article_paths)
            all_article_paths = all_article_paths[:self.max_train]

        for article in all_article_paths:
            article = ujson.load(open(article, "r"))
            # yield ' '.join([article['source'], article['content']])
            yield article["content"]

    def create_global_corpus(self,
                             max_df: float = 0.1,
                             min_df: float = 3,
                             num: int = 20000) -> None:
        """Create a global corpus of all the articles."""
        v = TfidfVectorizer(
            input="content",
            encoding="utf-8",
            decode_error="replace",
            strip_accents="unicode",
            lowercase=True,
            analyzer="word",
            stop_words=self.stopwords,
            token_pattern=r"(?u)\b[a-zA-Z_][a-zA-Z0-9_]+\b",
            ngram_range=(1, 2),
            max_features=num,
            norm="l2",
            use_idf=True,
            smooth_idf=True,
            sublinear_tf=True,
            max_df=max_df,
            min_df=min_df,
        )
        v.fit(self.make_corpus(True))
        X = v.transform(self.make_corpus(False)).astype(np.float32)
        pkl.dump(
            {
                "X": X,
                "vocab": v.vocabulary_,
                "idf": v.idf_
            },
            open("data/global_corpus.pkl", "wb"),
        )

    def load_features(self):
        """Load the features from the global corpus."""
        feats = pkl.load(open("data/global_corpus.pkl", "rb"))
        X = feats["X"]
        y = np.zeros_like(X.shape[0])
        return X, y, feats["vocab"], feats["idf"]

    def rank_news(self, C: float = 0.01):
        """Rank the news."""
        # classify
        X, y, vocab, idf = self.load_features()

        clf = svm.LinearSVC(class_weight="balanced",
                            verbose=False,
                            max_iter=10000,
                            tol=1e-6,
                            C=C)
        clf.fit(X, y)
        s = clf.decision_function(X)
        sortix = np.argsort(-s)
        scores = [100 * float(s[ix]) for ix in sortix]

        ivocab = {v: k for k, v in vocab.items()}  # index to word mapping
        weights = clf.coef_[0]  # (n_features,) weights of the trained svm
        sortix = np.argsort(-weights)
        words = []
        for ix in list(sortix[:40]) + list(sortix[-20:]):
            words.append({
                "word": ivocab[ix],
                "weight": weights[ix],
            })
        return scores, words

if __name__ == "__main__":
    a = Analyzer()
    a.create_global_corpus()


from evolucao import Evolucao
from matplotlib import pyplot as plt

geracoes = 70
volume_maximo = 3
taxa_mutacao = 0.05
numero_de_cromossomos = 112
volumes_e_precos = [
    [0.635, 849],
    [0.527, 3999],
    [0.751, 999.9],
    [0.00350, 2499],
    [0.496, 199.90],
    [0.0319, 299.29],
    [0.400, 4346.99],
    [0.290, 3999.90],
    [0.200, 2999.90],
    [0.0424, 308.66],
    [0.0544, 429.90],
    [0.870, 1199.89],
    [0.498, 1999.90],
    [0.0000899, 2199.12],
]

evolucao = Evolucao(
    geracoes=geracoes,
    volume_maximo=volume_maximo,
    taxa_mutacao=taxa_mutacao,
    volumes_e_precos=volumes_e_precos,
    numero_de_cromossomos=numero_de_cromossomos,
)

populacao = evolucao.criar_populacao()
historico_de_fitness = [evolucao.media_fitness(populacao)]

for _ in range(geracoes):
    populacao = evolucao.evoluir(
        populacao,
    )
    historico_de_fitness.append(evolucao.media_fitness(populacao))

print("\nSuguestões de boas soluções:")
for index in range(5):
    print(populacao[index])

plt.plot(range(len(historico_de_fitness)), historico_de_fitness)
plt.grid(True, zorder=0)
plt.xlabel("Gerações")
plt.ylabel("Preços médios do caminhão")
plt.show()


import nltk
nltk.data.path.append('./nltk_data/')
from collections import Counter
from nltk import word_tokenize
from nltk.tree import MultiParentedTree
from awesome_print import ap
import copy
import re
from itertools import groupby
from parse_forest_lib import *
import itertools

class Fsm:

	def __init__(self ):
		self.start = None
		self.end = None
		self.last_unique_node_id = 0
		self.token_start_dictionary = {}
		self.token_end_dictionary = {}
		self.all_nodes = []

	# isntance method
	def get_next_unique_node_id(self):
		self.last_unique_node_id += 1
		return self.last_unique_node_id

	# instance method
	def get_node(self, id):
		nodes = self.list_nodes()
		return filter(lambda node: node.id == id, nodes)[0]

	# merge b in a's place
	def is_connected(self, node_a, node_b):
		return node_b in node_a.nexts.values() or node_a in node_b.nexts.values()

	def is_remotely_connected(self, node_a, node_b):
		return self.is_in_forward_path(node_a, node_b) or self.is_in_forward_path(node_b, node_a) or self.is_in_backward_path(node_a, node_b) or self.is_in_backward_path(node_b, node_a)

	def is_in_forward_path(self, node_x, node_y):
		if node_x == node_y:
			return True
		if self.end == node_x:
			return False
		accumalation = False
		for next_node in node_x.nexts.values():
			accumalation = accumalation or (self.is_in_forward_path(next_node, node_y))
		return accumalation

	def is_in_backward_path(self, node_x, node_y):
		if node_y == node_x:
			return True
		if self.start == node_x:
			return False
		accumalation = False
		for prev_node in node_x.previouses.values():
			accumalation = accumalation or (self.is_in_backward_path(prev_node, node_y))
		return accumalation

	def merge_fsm_nodes(self, fsm_node_a, fsm_node_b ):
		if fsm_node_a.id != fsm_node_b.id:
			for key, prev_node in fsm_node_b.previouses.items():
				prev_node.nexts[key] = fsm_node_a
				fsm_node_a.previouses[key] = prev_node
				self.token_start_dictionary[key] = prev_node
				self.token_end_dictionary[key] = fsm_node_a
			for key, next_node in fsm_node_b.nexts.items():
				fsm_node_a.nexts[key] = next_node
				next_node.previouses[key] = fsm_node_a
				self.token_start_dictionary[key] = fsm_node_a
				self.token_end_dictionary[key] = next_node
			self.all_nodes.remove(fsm_node_b)

	def print_fsm( self ):
		start = self.start
		ap("Node Id: {}".format( start.id ))
		for key, next_node in start.nexts.items():
			print("key: {}  => Node Id: {}".format(key, next_node.id ))
		print("\n")
		for key, next_node in start.nexts.items():
			ap( "From Node {} taking key {}".format(start.id, key))
			self.print_sub_fsm(next_node)

	def print_sub_fsm(self, start):
		ap("Node Id: {}".format( start.id ))
		if self.end == start:
			ap("===== Reached End ======")

		for key, next_node in start.nexts.items():
			print("key: {}  => Node Id: {}".format(key, next_node.id ))
		print("\n")
		for key, next_node in start.nexts.items():
			ap( "From Node {} taking key {}".format(start.id, key))
			self.print_sub_fsm(next_node)

	def get_graphvis_commands(self):
		start = self.start
		commands = []
		for key, next_node in start.nexts.items():
			command = "	{} -- {} [ label = \"{}\" ];".format(start.id, next_node.id, key)
			commands.append( command )
			commands.extend( self.get_graphvis_subcommands(next_node) )
		return list(set(commands))

	def get_graphvis_subcommands(self, start):
		commands = []
		for key, next_node in start.nexts.items():
			command = "	{} -- {} [ label = \"{}\" ];".format(start.id, next_node.id, key)
			commands.append( command )
			commands.extend( self.get_graphvis_subcommands(next_node) )
		return commands

	# instance method
	def load_tokens(self, tokens):

		if self.start is None:
			start_fsm_node = FsmNode(self.get_next_unique_node_id())
			self.start = start_fsm_node
			self.all_nodes.append(start_fsm_node)
		start_fsm_node = self.start

		if self.end is None:
			end_fsm_node = FsmNode(self.get_next_unique_node_id())
			self.end = end_fsm_node
			self.all_nodes.append(end_fsm_node)
		end_fsm_node = self.end

		for token in tokens[0:(len(tokens)-1)]:
			next_fsm_node = FsmNode(self.get_next_unique_node_id())
			start_fsm_node.set_next_fsm_node(next_fsm_node, token)
			self.token_start_dictionary[token] = start_fsm_node
			self.token_end_dictionary[token] = next_fsm_node
			start_fsm_node = next_fsm_node
			self.all_nodes.append(next_fsm_node)

		start_fsm_node.set_next_fsm_node(end_fsm_node, tokens[-1])
		self.token_start_dictionary[tokens[-1]] = start_fsm_node
		self.token_end_dictionary[tokens[-1]] = end_fsm_node

	# instance method
	def merge_parallel_tokens(self, tokens_1, tokens_2):
		node_x = self.token_start_dictionary[tokens_1[0]]
		node_y = self.token_start_dictionary[tokens_2[0]]
		self.merge_fsm_nodes( node_x, node_y )
		node_x = self.token_end_dictionary[tokens_1[-1]]
		node_y = self.token_end_dictionary[tokens_2[-1]]
		self.merge_fsm_nodes( node_x, node_y )

	def list_nodes(self):
		start = self.start
		nodes = [start]
		nodes.extend(start.nexts.values())
		for next_node in start.nexts.values():
			nodes.extend(self.sublist_nodes(next_node))
		all_nodes = list(set(nodes))
		self.all_nodes = all_nodes
		return list(set(nodes))

	def sublist_nodes(self, start):
		nodes = []
		nodes.extend(start.nexts.values())
		for next_node in start.nexts.values():
			nodes.extend(self.sublist_nodes(next_node))
		return nodes

	def convert_to_word_edges(self):
		self.all_nodes
		for node_x in self.all_nodes:
			for key, node_y in node_x.nexts.items():
				node_x.nexts.__delitem__(key)
				node_x.nexts[ParseForest.id_to_word_dictionary[int(key)]] = node_y
			for key, node_y in node_x.previouses.items():
				node_x.previouses[ParseForest.id_to_word_dictionary[int(key)]] = node_y

	def sqeeze(self):
		all_merges_made = set()
		merges_made = 100
		while merges_made > 0:
			merges_made = 0
			for node_x in self.all_nodes:
				word_keys = map( lambda key: ParseForest.id_to_word_dictionary[int(key)] , node_x.nexts.keys() )
				for word_key, word_key_count in Counter(word_keys).items():
					if word_key_count > 1:
						keys = filter( lambda key: ParseForest.id_to_word_dictionary[int(key)] == word_key , node_x.nexts.keys())
						nodes = map( lambda key: node_x.nexts[key], keys)
						for node_a, node_b in itertools.combinations( nodes, 2):
							if node_a.id != node_b.id:
								intersection = [x for x in [node_a.id, node_b.id] if x in [self.start.id, self.end.id]]
								if not all_merges_made.__contains__( "-".join([str(node_a.id), str(node_b.id)])) and len(intersection) == 0:
									if not self.is_remotely_connected(node_a, node_b):
										if node_b in self.all_nodes:
											self.merge_fsm_nodes(node_a, node_b)
											all_merges_made.add("-".join([str(node_a.id), str(node_b.id)]))
											all_merges_made.add("-".join([str(node_b.id), str(node_a.id)]))
											merges_made += 1
									else:
										if node_b in node_a.nexts.values():
											word_id = ParseForest.next_unique_word_id("*e*")
											node_a.nexts[word_id] = node_b
											node_b.previouses[word_id] = node_a
											redundant_edge = filter( lambda key: node_x.nexts.get(key, None) == node_b, keys)
											if (len(redundant_edge) > 0):
												redundant_edge = redundant_edge[0]
												target_node = node_x.nexts.get(redundant_edge, None)

												if target_node:
													node_x.nexts.__delitem__(redundant_edge)
													target_node.previouses.__delitem__(redundant_edge)
										else:
											word_id = ParseForest.next_unique_word_id("*e*")
											node_b.nexts[word_id] = node_a
											node_a.previouses[word_id] = node_b
											redundant_edge = filter( lambda key: node_x.nexts.get(key, None) == node_a, keys)
											if (len(redundant_edge) > 0):
												redundant_edge = redundant_edge[0]
												target_node = node_x.nexts.get(redundant_edge, None)
												if target_node:
													node_x.nexts.__delitem__(redundant_edge)
													target_node.previouses.__delitem__(redundant_edge)
										all_merges_made.add("-".join([str(node_a.id), str(node_b.id)]))

		merges_made = 100
		while merges_made > 0:
			merges_made = 0
			for node_x in self.all_nodes:
				word_keys = map( lambda key: ParseForest.id_to_word_dictionary[int(key)] , node_x.previouses.keys() )
				for word_key, word_key_count in Counter(word_keys).items():
					if word_key_count > 1:
						keys = filter( lambda key: ParseForest.id_to_word_dictionary[int(key)] == word_key , node_x.previouses.keys())
						nodes = map( lambda key: node_x.previouses[key], keys)
						for node_a, node_b in itertools.combinations( nodes, 2):
							if node_a.id != node_b.id:
								intersection = [x for x in [node_a.id, node_b.id] if x in [self.start.id, self.end.id]]
								if not all_merges_made.__contains__( "-".join([str(node_a.id), str(node_b.id)])) and len(intersection) == 0:
									if not self.is_remotely_connected(node_a, node_b):
										if node_b in self.all_nodes:
											self.merge_fsm_nodes(node_a, node_b)
											all_merges_made.add("-".join([str(node_a.id), str(node_b.id)]))
											all_merges_made.add("-".join([str(node_b.id), str(node_a.id)]))
											merges_made += 1
									else:
										if node_b in node_a.nexts.values():
											word_id = ParseForest.next_unique_word_id("*e*")
											node_a.nexts[word_id] = node_b
											node_b.previouses[word_id] = node_a
											redundant_edge = filter( lambda key: node_x.previouses.get(key, None) == node_a, keys)
											if (len(redundant_edge) > 0):
												redundant_edge = redundant_edge[0]
												target_node = node_x.previouses.get(redundant_edge, None)
												if target_node:
													node_x.previouses.__delitem__(redundant_edge)
													target_node.nexts.__delitem__(redundant_edge)
										else:
											word_id = ParseForest.next_unique_word_id("*e*")
											node_b.nexts[word_id] = node_a
											node_a.previouses[word_id] = node_b
											redundant_edge = filter( lambda key: node_x.previouses.get(key, None) == node_b, keys)
											if (len(redundant_edge) > 0):
												redundant_edge = redundant_edge[0]
												target_node = node_x.previouses.get(redundant_edge, None)
												if target_node:
													node_x.previouses.__delitem__(redundant_edge)
													target_node.nexts.__delitem__(redundant_edge)

										all_merges_made.add("-".join([str(node_a.id), str(node_b.id)]))

	def get_graphviz_code(self):
		pre = "graph finite_state_machine {\n	rankdir=LR; \n        size=\"9,9\";\n"
		# start_end_definitions = "	node [shape = doublecircle ]; 1 2;\n".format(self.start.id, self.end.id)
		# node_definition = "	node [shape = circle ];\n"
		start_end_definitions = "	node [shape = doublecircle, width=.2, fixedsize=true, label=\"\"]; 1 2;\n".format(self.start.id, self.end.id)
		node_definition = "	node [shape = circle, width=.2, fixedsize=true, label=\"\" ];\n"
		fsm_commands = (self.get_graphvis_commands())
		fsm_commands = list( set(fsm_commands) )
		transition_definitions = "\n".join( fsm_commands )
		post = "\n}"
		fsm_gv_code_snipped = pre + start_end_definitions + node_definition + transition_definitions + post
		return fsm_gv_code_snipped

class FsmNode:

	def __init__(self, id ):
		self.id = id
		self.nexts = {}
		self.previouses = {}

	# instance method
	def set_next_fsm_node(self, node, key ):
		self.nexts[key] = node
		node.previouses[key] = self


import gym
import time
import numpy as np
import sklearn.neural_network as nn

class Agent(object):
    '''
    '''
    def __init__(self, env=None, hidden_sizes=[20, 20]):
        '''
        '''
        self.env = env
        self.hidden_sizes = hidden_sizes
        self.n_actions = env.action_space.n
        self.actions = range(env.action_space.n)
        self.policy = self.init_policy()

    def init_policy(self, ):
        '''
        '''
        policy = nn.MLPClassifier(hidden_layer_sizes=self.hidden_sizes, activation='tanh')
        policy.partial_fit([self.env.reset()] * self.n_actions, range(self.n_actions), range(self.n_actions))
        print('---init MLP done---')
        return policy

    def action(self, state):
        '''
        '''
        probs = self.policy.predict_proba(state)
        a = np.random.choice(self.actions, p=probs[0])

        return probs

    def action_probs(self, state):
        '''
        '''
        probs = self.policy.predict_proba(state)
        return probs

class Solver(object):
    '''
    '''
    def __init__(self, env, agent, batches=100, epoches=10, percentile=50):
        '''
        '''
        self.env = env
        self.agent = agent
        self.batches = batches
        self.epoches = epoches
        self.percentile = percentile
        self.actions = range(self.env.action_space.n)

    def train(self, ):
        '''
        '''
        tic = time.time()
        for _ in range(self.epoches):
            states, actions, _rewards = self.generate_train_data(batches=self.batches)
            self.agent.policy.partial_fit(states, actions, self.actions)

            print(np.mean(_rewards))

            if np.mean(_rewards) > 200:
                print('---Done--Using {} s-'.format(time.time() - tic))
                break

        return self.agent.policy

    def generate_session(self, t_max=1000):
        '''
        '''
        states = []
        actions = []
        rewards = 0.

        s = env.reset()

        for _ in range(t_max):

            probs = self.agent.action_probs([s, ])[0]
            a = np.random.choice(self.actions, p=probs)
            s_next, r, done, _ = self.env.step(a)

            states.append(s)
            actions.append(a)
            rewards += r

            s = s_next

            if done:
                break

        return states, actions, rewards

    def select_elites(self, states, actions, rewards):
        '''
        '''
        # states, actions, rewards = map(np.array, [states, actions, rewards])
        # print(states.shape)
        # print(actions.shape)
        r_threshold = np.percentile(rewards, q=self.percentile)
        idx = (np.array(rewards) >= r_threshold).nonzero()[0]
        elite_states = [s for i in idx for s in states[i]]
        elite_actions = [a for i in idx for a in actions[i]]

        return np.array(elite_states), np.array(elite_actions), np.array(rewards)

    def generate_train_data(self, batches=1):
        '''
        '''
        states, actions, rewards = zip(*[self.generate_session() for _ in range(batches)])
        states, actions, _rewards = self.select_elites(states, actions, rewards)

        return states, actions, _rewards

if __name__ == '__main__':

    env = gym.make('CartPole-v0').env
    # obs0 = env.reset()
    # print(env.action_space.n)
    # print(env.observation_space.shape)
    # print(type(obs0), obs0.shape)

    agent = Agent(env=env)
    # probs = agent.action([obs0, env.reset()])
    # print(probs.shape)
    # print(probs.dtype)
    # print(probs[0])

    solver = Solver(env, agent, percentile=50, epoches=100)
    # sss, ass, rss = solver.generate_train_data(batches=10)
    # print(sss.shape, sss.dtype)
    # print(ass.shape, ass.dtype)
    # print(rss.shape, rss.dtype)

    solver.train()


import numpy as np
from sklearn.cluster import MeanShift, estimate_bandwidth
from sklearn.cluster import AffinityPropagation
from sklearn import preprocessing
import pandas as pd
import matplotlib.pyplot as plt
import pickle

#df = getdata
class Clustering():

	def train_affinity_propogation_model(self, X, savePickle = True):
		clf = AffinityPropagation(preference=-50).fit(X)
		cluster_centers_indices = clf.cluster_centers_indices_

		if savePickle:
			with open('affinity_propagation.pickle','wb') as f:
				pickle.dump(clf, f)

		return clf

	def load_affinity_propogation_model_pickle(self):
		pickle_in = open('affinity_propagation.pickle', 'rb')
		clf = pickle.load(pickle_in)

		return clf

	def train_mean_shift_model(self, X, savePickle = True):
		X = preprocessing.scale(X)

		bandwidth = estimate_bandwidth(X, quantile=0.1, n_samples=500)

		clf = MeanShift(bandwidth=bandwidth, bin_seeding=True)
		clf.fit(X)
		if savePickle:
			with open('meanshift.pickle','wb') as f:
				pickle.dump(clf, f)

		return clf

	def load_mean_shift_model_pickle(self):
		pickle_in = open('meanshift.pickle', 'rb')
		clf = pickle.load(pickle_in)

		return clf


from django.urls import path
from .import views

app_name = 'cart'

urlpatterns = [
    path('', views.cartView, name='cart'),
    path("checkout/", views.checkout, name='checkout'),
    path('add/', views.addToCart , name='add'),
    path('remove/', views.removeCartItems , name='removeAllItems'),
    path('<int:product_id>/<int:qty>/delete', views.deleteCartItem , name='delete'),
    ]