import os
import requests
import json
import getpass
from cryptography.fernet import Fernet
import base64

course_id = '164'
get_item1 = requests.get('https://gitlab.com/ga-cyberworkforceacademy/labtainer/-/raw/master/item1')
item1 = base64.b64decode(get_item1.content)
item2 = 'MkFuYk4wYjJCX25nbUdULWxCZHBCNktCZmptRklsREVobk5OQVQ4SjV5Zz0='
item2 = item2.encode('utf8')
item2 = base64.b64decode(item2)
f = Fernet(item2)
access_token = f.decrypt(item1).decode('utf8')
working_dir = '/home/{}/labtainer_xfer'.format(getpass.getuser())
auth_header = {'Authorization': 'Bearer {}'.format(access_token)}

def parseUserName():
    # Parse json activity file for username information
    with open(activity_file, 'r') as read_json:
        user_name = json.load(read_json)
    user_name = list(user_name.keys())[0]
    # Clean up output for user email
    user_name = user_name.replace('_at_','@')
    # user_name = user_name.split('.', 2)[:1]
    user_email= "".join(user_name)
    user_email = user_email.split('@')[0]
    return user_email

def getUserID():
    #Query Canvas API endpoint for user ID using user_email
    response = requests.get('https://gacybercenter.instructure.com/api/v1/accounts/self/users?search_term={}'.format(user_email), headers=auth_header)
    user_data = json.loads(response.text)[0]
    user_id = user_data['id']
    return str(user_id)

def getAssignments():
    # Pull assignment data and create dictionary for reference
    get_assignments = requests.get('https://gacybercenter.instructure.com/api/v1/courses/{}/assignments'.format(course_id), headers=auth_header)
    assignment_data = json.loads(get_assignments.text)
    assignment_dict = [d.get('name') for d in assignment_data]
    assignment_dict2 = [d.get('id') for d in assignment_data]
    global assignment_data_dict
    assignment_data_dict = dict(zip(assignment_dict, assignment_dict2))
    return assignment_data_dict

def getAssignmentID():
    # Associate ID with assignment name
    # Parse json activity file for assignment name information
    with open(activity_file, 'r') as read_json:
        activity_name = json.load(read_json)
    activity_name = list(activity_name.keys())[0]
    # Clean up output for user email
    activity_name = activity_name.replace('_at_','@')
    activity_name = activity_name.rsplit('.', 1)[1]
    assignment_id = ""
    if activity_name in assignment_data_dict:
        assignment_id = assignment_data_dict[activity_name]
    return assignment_id, activity_name

def gradeAssignment():
    # Parse activity for grade information
    with open(activity_file, 'r') as read_json:
        activity_data = json.load(read_json)
    dict_name = list(activity_data.keys())[0]
    grade_dict = activity_data['{}'.format(dict_name)]['grades']
    if True in grade_dict.values():
        grade_pts = 100
        return grade_pts
    else:
        grade_pts = 0
        return grade_pts

def uploadGrade():
    # Upload grade to Canvas API endpoint
    # Check if grade already exists
        grade_data = 'grade_data[{}][posted_grade]={}'.format(user_id, grade_pts)
        requests.post('https://gacybercenter.instructure.com/api/v1/courses/{}/assignments/{}/submissions/update_grades'.format(course_id, assignment_id), data=grade_data, headers=auth_header)

def checkGradeExists():
    #Check Canvas API endpoint for existing grade data
    get_assignments = requests.get('https://gacybercenter.instructure.com/api/v1/courses/{}/assignments/{}/submissions/{}'.format(course_id, assignment_id, user_id), headers=auth_header)
    graded_data = json.loads(get_assignments.text)
    grade_exists = ''
    grade = graded_data.get('grade')
    if grade == '100':
        grade_exists = True
    elif grade == '0':
        grade_exists = True
    else:
        grade_exists = False
    return grade, grade_exists

# Pull current assignments from Canvas and create a dictionary for the values
getAssignments()

# Loop through json files in the assignment directory

for root, dirs, files in os.walk(working_dir):
    for json_file in files:
        activity_file = os.path.join(root, json_file)
        if activity_file.endswith('.json'):
            # Determine user Email
            try:
                user_email = parseUserName()
            # Determine UserID from Canvas API Call
                user_id = getUserID()
            # Determine assignment ID
                assignment_data = getAssignmentID()
                assignment_id = assignment_data[0]
                assignment_name = assignment_data[1]
            # Grade Assignment
                grade_pts = gradeAssignment()
            except:
                pass
            # Check for existing grade and push grade to assignment data if equal to 100
            try:
                grade_query = checkGradeExists()
                grade_exists = grade_query[1]
                grade = grade_query[0]
                if (grade_exists != True) and (grade_pts == 100):
                    uploadGrade()
                    print('{} assignment with a grade of {} submitted!'.format(assignment_name, grade_pts))
                elif (grade_exists == True) and (grade_pts != 100):
                    uploadGrade()
                    print(f'{assignment_name} with a grade of {grade_pts} currently exists, try the lab again for a better grade!')
                elif (grade_exists == True) and (grade == 0) and (grade_pts == 100):
                    uploadGrade()
                    print(f'{assignment_name} with a grade of {grade_pts} has been updated!')
                else:
                    print('{} assignment has already been submitted with a grade of {}!'.format(assignment_name, grade))
            except:
                print(f"{assignment_name} doesn't exist on Canvas as a graded item")
                pass


""" Katie Lunceford and Jon Li

This is a plotter class that was used in an old
checkpoint but not implemented in the final project.

The initialization takes in two lists and puts these
lists in the attributes self.y and self.x.

It has two methods: scatterPlot and barGraph which
make the appropriate plots out of self.y and self.x
"""

import matplotlib.pyplot as plt
import numpy as np

class Plotter:
    def __init__(self, list1, list2 = []):
        if list2 == []:
            self.y = list1
            self.x = [i + 1 for i in range(len(list1))]
        else:
            self.x = list1
            self.y = list2

    def scatterPlot(self):
        area = np.pi*15
        plt.scatter(self.x, self.y, s = area)
        plt.show()

    def barGraph(self):
        plt.bar(self.x, self.y, width = 0.5, align = 'center')
        plt.show()


import urllib.request
from bs4 import BeautifulSoup
from time import sleep
from send_server import get_time, send_data, play_sound
import webbrowser

def run_scraper(drug, site):
    webpage = urllib.request.urlopen(site)
    xml = BeautifulSoup(webpage, 'xml')
    for entry in xml.findAll('li'):
        for item in entry.findAll('a'):
            title = item.get('title')
            link = item.get('href')
            if title != None and drug.lower() in title.lower() and title not in old_titles:
                title = title.replace('Click to view ', '')
                new_link = 'https://www.accessdata.fda.gov/' + link
                new_link = new_link.replace('.process=', '.process&ApplNo=')
                send_data('Drug Approval', title, get_time(), new_link)
                webbrowser.open(new_link)
                print('Drug Approval', title, get_time(), new_link)
                play_sound('approval.mp3')
                old_titles.append(title)

old_titles = []
drugs = ['Qbrexza']

while True:
    try:
        for drug in drugs:
            first_letter = drug[0]
            site = 'https://www.accessdata.fda.gov/scripts/cder/daf/index.cfm?event=browseByLetter.page&productLetter='
            site = site + first_letter
            run_scraper(drug, site)
            sleep(.8)
    except:
        play_sound('error.mp3')


import pandas as pd
import matplotlib.pyplot as plt

def analysis():

    #df['Timeseries'] = pd.date_range(start='09:30:00',end=None,periods=20)

    #Load CSV Data into a DF
    df = pd.read_csv("Data/Quandl_Intraday_Prices.csv")

    #Get a timeseries
    #range = df[(df['Timestamp'] >= '09:30:00') & (df['Timestamp'] <= '09:45:00')]
    range = df[(df.Timestamp >= '09:30:00') & (df.Timestamp <= '09:45:00')]

    #Get data for one security from the DF
    df2 = range.loc[range.Ticker == 'TSS']

    #Filter certain columns
    usecols = df2[['Timestamp','HighPrice','LowPrice']]

    #Plot data
    ax = usecols.plot(title='Tick Data', fontsize=12)
    ax.set_xlabel("Minutes")
    ax.set_ylabel("Price")

    plt.show()

if __name__ == "__main__":
    analysis()


import re
import itertools
from functools import reduce
from operator import mul
import pandas as pd

# DATA
other_tickets = []
rules = {}
my_ticket = []
with open("Data - Day16.txt") as file:
    oth_marker = False
    rule_marker = True
    my_maker = False
    for line in file:
        if oth_marker:
            other_tickets.append(line.strip().split(","))
        elif "nearby tickets" in line:
            oth_marker = True
        elif line == "\n":
            rule_marker = False
            my_maker = False
        elif rule_marker:
            r = re.search(r"(^[ a-z]+): ([a-z-0-9 ]+)", line)
            k = r.group(1)
            v = r.group(2)
            rules[k] = v
        elif my_maker:
            my_ticket.extend(line.strip().split(","))
        elif "your ticket" in line:
            my_maker = True

# GOAL 1
"""
The rules for ticket fields specify a list of fields that exist somewhere on the ticket and the valid ranges of values for each field.
Adding together all of the invalid values produces your ticket scanning error rate.
"""
valid_ranges = {"valid_from": [], "valid_to": [], "category": []}

for i in rules:
    rule = re.findall(r"[0-9]+", rules[i])
    rule = [int(j) for j in rule]
    valid_ranges["valid_from"].extend([rule[0], rule[2]])
    valid_ranges["valid_to"].extend([rule[1], rule[3]])
    valid_ranges["category"].extend([i, i])

valid_ranges = pd.DataFrame(valid_ranges)

def check_valid(valid, tickets):
    invalid = []

    for ticket in tickets:
        for num in ticket:
            num = int(num)
            between = False
            for idx in valid.index:
                if valid["valid_from"][idx] <= num <= valid["valid_to"][idx]:
                    between = True
                    break
            if not between:
                invalid.append(num)

    return sum(invalid)

print(f"The ticket scanning error rate equals: {check_valid(valid_ranges, other_tickets)} for part 1")

# GOAL 2
"""
Now that you've identified which tickets contain invalid values, discard those tickets entirely.
Use the remaining valid tickets to determine which field is which.

Once you work out which field is which,
look for the six fields on your ticket that start with the word departure.
What do you get if you multiply those six values together?
"""
def only_valid(valid, tickets):
    invalid = []
    for ticket in tickets:
        for num in ticket:
            num = int(num)
            between = False
            for idx in valid.index:
                if valid["valid_from"][idx] <= num <= valid["valid_to"][idx]:
                    between = True
                    break
            if not between:
                invalid.append(ticket)

    valid_tickets = [i for i in tickets if i not in invalid]

    return valid_tickets

def create_from_to_except(valid_ranges):
    new_df = {"cat": [], "fro": [], "to": [], "ex_fro": [], "ex_to": []}

    uni_cats = valid_ranges["category"].unique()
    for i in uni_cats:
        temp = valid_ranges[valid_ranges["category"] == i].reset_index(drop=True)

        new_df["cat"].append(i)
        new_df["fro"].append(temp["valid_from"][0])
        new_df["to"].append(temp["valid_to"][1])
        new_df["ex_fro"].append(temp["valid_to"][0])
        new_df["ex_to"].append(temp["valid_from"][1])

    return pd.DataFrame(new_df)

def check_category(rule_row, tickets, ticket_values):
    for tic in tickets:
        for idx in ticket_values:
            if (
                rule_row["fro"] <= int(tic[idx]) <= rule_row["to"] and not
                rule_row["ex_fro"] < int(tic[idx]) < rule_row["ex_to"]
                ):
                pass
            else:
                ticket_values[idx].remove(rule_row["cat"])

    return ticket_values

def find_idx_cat(ticket_values):
    final = []
    rem = "placeholder"
    while len(ticket_values) > 0:
        for i in ticket_values:
            if len(ticket_values[i]) == 1:
                final.append((i, ticket_values[i][0]))
                rem = ticket_values[i][0]
                del ticket_values[i]
                break

        for i in ticket_values:
            if rem in ticket_values[i]:
                ticket_values[i].remove(rem)

    return final

def prod(iterable): # default function in the math module in python 3.9
    return reduce(mul, iterable, 1)

df_ranges = create_from_to_except(valid_ranges)
tickets = only_valid(valid_ranges, other_tickets)
tickets.append(my_ticket)

ticket_values = {}
for i in range(len(my_ticket)):
    ticket_values[i] = list(rules.keys())

for row in df_ranges.index:
    ticket_values = check_category(df_ranges.iloc[row], tickets, ticket_values)

comb = find_idx_cat(ticket_values)

indices = [i for i,j in comb if "departure" in j]

print(f"The multiplication answer is {prod([int(j) for i,j in enumerate(my_ticket) if i in indices])}")


import gym
import numpy as np
from operator import itemgetter
env = gym.make('CartPole-v0')

ninputs = env.observation_space.shape[0]
if (isinstance(env.action_space, gym.spaces.box.Box)):
 noutputs = env.action_space.shape[0]
else:
 noutputs = env.action_space.n
nhiddens=10
pvariance = 0.1 # variance of initial parameters
ppvariance = 0.02 # variance of perturbations
# initialize the training parameters randomly by using a gaussian distribution with average 0.0 and variance 0.1
# biases (thresholds) are initialized to 0.0
W1 = np.random.randn(nhiddens,ninputs) * pvariance # first layer
W2 = np.random.randn(noutputs, nhiddens) * pvariance # second layer
b1 = np.zeros(shape=(nhiddens, 1)) # bias first layer
b2 = np.zeros(shape=(noutputs, 1)) # bias second layer
def initialize_weights(n,ninputs,nhiddens,noutputs):
    par_dic=[]
    for i in range(n):
        par_dic.append( {"W1": np.random.randn(nhiddens,ninputs),
                    "W2": np.random.randn(noutputs, nhiddens),
                    "b1": np.zeros((nhiddens, 1)),
                    "b2": np.zeros((noutputs, 1))})

    return par_dic

def predict(observation,W1,W2,b1,b2):
    Z1 = np.dot(W1, observation) + b1
    A1 = np.tanh(Z1)
    Z2 = np.dot(W2, A1) + b2
    A2 = np.tanh(Z2)
    if (A2[0] < 0):
        action = 0
    else:
        action = 1
    return action

def add_noise(dic,n):
    for i in range(int(n/2)):
        dic[i]['W1']=dic[int(n/2)+i]['W1']+np.random.randn(nhiddens,ninputs)*ppvariance
        dic[i]['W2']=dic[int(n/2)+i]['W2']+np.random.randn(noutputs, nhiddens)*ppvariance
        dic[i]['b1']=dic[int(n/2)+i]['b1']+np.random.randn(nhiddens, 1)*ppvariance
        dic[i]['b2']=dic[int(n/2)+i]['b2']+np.random.randn(noutputs, 1)*ppvariance
    return dic

reward1=0
popul = 10
par_dic = initialize_weights(popul,ninputs,nhiddens,noutputs)
t1 = 0
flag = 1
while(flag):
    for i in range(popul):
        observation = env.reset()
        reward1 = 0
        for t in range(200):
            observation.resize(ninputs, 1)
            W1 = par_dic[i]['W1']
            W = par_dic[i]['W2']
            b1 = par_dic[i]['b1']
            b2 = par_dic[i]['b2']
            action = predict(observation,W1,W2,b1,b2)
            observation, reward, done, info = env.step(action)
            reward1=reward1+reward
            if done == True:
                    t1 = t
                    par_dic[i]['reward']=reward1
                    break
    par_dic = sorted(par_dic,key=itemgetter('reward'))
    print(par_dic[len(par_dic)-1]['reward'])
    par_dic = add_noise(par_dic,popul)
    if par_dic[len(par_dic)-1]['reward']>199.0:
        print('we are done now!!!')
        flag = 0
        break


"""
Это парсер данных с авито. Он проходится по каждому объявлению и собирает нужные
данные. Во время парсинга авито может указывать количество объявлений указанные
по параметрам с рекламой, так что иногда счётчик может показывать неправильное количество.
Парсер принимает параметры из xlsx файла. Его стандарт должен выглядить примерно так:
     Ключи     |     Регион    |     Категория     |     Подкатегория
---------------------------------------------------------------------------------------
 Дезинфекция  | Москва       | Услуги             | Уборка
 ...
"""
from urllib import parse
from sys import exit
from typing import List, NewType
from random import choice, random, randint
import json
import datetime
import time

import requests
from requests import Response
from bs4 import BeautifulSoup
import transliterate
import openpyxl

# CONSTANTS
HOST: str = 'https://www.avito.ru'
URL: str = ''
USER_AGENTS: List['user_agent'] = [user_agent.strip() for user_agent in open('user-agents.txt').readlines()]
MONTH_IN_NUMBER = {
    'января': '01',
    'февраля': '02',
    'марта': '03',
    'апреля': '04',
    'мая': '05',
    'июня': '06',
    'июля': '07',
    'августа': '08',
    'сентября': '09',
    'октября': '10',
    'ноября': '11',
    'декабря': '12'
}
# ANNOTATIONS
url_type = NewType('url_type', str)
int_price = NewType('int_price', int)
# To read the first fifty ads
counter_first_fifty_ads: int = 50
max_parsed_ad_on_one_session = 100
list_statistic_about_ad: List[dict] = []
start_time_script = time.time()
output_xlsx_file: dict = {}
parsed_ads: List[url_type] = []
names_columns_in_statistic_by_ad = [
    'Позиция (сортировка по умолчанию)',
    'Название объявления',
    'Описание',
    'Адрес',
    'Цена(в рублях)',
    'Имя продовца',
    'Количество объявлений у продовца',
    'Количество просмотров всего',
    'Количество просмотров за сегодня',
    'Ссылка на объявление',
    'Дата публикации',
    'Время публикации'
]

def get_response(url: url_type, params: dict = {}) -> Response or False:
    """Tries to get an answer by url from Avito 10 times. If it failed, it returns False"""
    number_of_request_try: int = 10
    for try_ in range(number_of_request_try):
        try:
            response: Response = requests.get(url, headers={'User-Agent': choice(USER_AGENTS)},
                                              timeout=60, params=params)
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
            print('Повторяем попытку...')
        except requests.exceptions.MissingSchema:
            print('Ссылка невалидна.')
        else:
            return response
    else:
        return False

def process_url_for_validity(ad_name: str, region: str, category: str,
                             subcategory: str, url: url_type) -> url_type or False:
    """
    Process url for parser. Avito generates a lot of noise when parsing data from its data.
    Sometimes the parameters by which the parser will search for data are invalid for the link,
    and this function does everything possible to create the correct link.

    ad_name: str -> ad name parameter.
    region: str -> region by which will search ads.
    category: str -> category by which will search ads.
    subcategory: str -> subcategory by which will search ads.

    @return: url or False -> If url success handled, return handled url else parameters incorrect, return False and
    stop parsing ads by this parameters.
    """
    for try_number in range(1, 4 + 1):
        try:
            test_request = get_response(url)
            bs_html: BeautifulSoup = BeautifulSoup(test_request.content, 'html.parser')
            test_get_text = bs_html.find('span', class_='page-title-count-1oJOc').get_text

        except AttributeError:
            if bs_html.find('h2', class_='no-results-title-3kn6E') is not None:

                print('Ничего не найдено в выбранной области поиска')
                return False

            elif bs_html.find('h2', class_='firewall-title') is None:

                if try_number == 1:

                    past_region = region
                    region = region.split()[-1].strip()
                    region_for_url = transliterate.translit(
                        region, reversed=True).replace(' ', '_').strip().replace("'", '').replace('j', 'y')
                    url = f"{HOST}/{region_for_url}{url[21 + url[21:].find('/'):]}"

                elif try_number == 2:
                    print(f'Введите регион ({region}), что бы он отвечал на вопрос (область чья?) '
                          'или образовывал словосочитание по типу (Московская область).\n'
                          'Примеры:\n'
                          ' Киров -> Кировская\n'
                          ' Москва -> Московская\n')
                    region = input('Вводите: ').strip() + ' область ' + region
                    region_for_url = transliterate.translit(
                        region, reversed=True).replace(' ', '_').strip().replace("'", '').replace('j', 'y')

                    url = f"{HOST}/{region_for_url}{URL[21 + URL[21:].find('/'):]}"

                elif try_number == 3:

                    print('Введите ссылку вручную. Её можно получить на Авито '
                          'https://www.avito.ru/ в адресной строке, указав в поисковик параметры:\n'
                          f'Ключ: {ad_name}; Регион: {past_region}; Категория: {category}; Подкатегория: {subcategory};\n')
                    url = input('Вводите: ')
                    region = transliterate.translit(URL[21:URL.find('/', 21)], 'ru')

                else:
                    print('По вашим параметрам ничего не найдено.')
                    return False
            elif bs_html.find('h2', class_='firewall-title') is not None:
                print('Ваш IP адрес заблокировал Avito на время. Следуйте указаниям файла help.txt.')
                exit()

        else:
            return url

def generate_valid_url_for_parsing(ad_name: str, region: str, category: str, subcategory: str) -> url_type:
    """Generate url for parsing ads from avito."""
    global URL

    # Parameters may be with comma. Them need delete.
    if region.count(',') == 1:
        region = region.replace(',', '')
    elif region.count(',') > 1:
        region = region.split(',')[-1].strip()

    ad_name_for_url = parse.urlencode({'q': ad_name})
    region_for_url = transliterate.translit(
        region, reversed=True).replace(' ', '_').strip().replace("'", '').replace('j', 'y')

    # Avito for this two category make unique name in link
    if category == 'услуги':
        category_for_url = 'predlozheniya_uslug'
    elif category == 'готовый бизнес и оборудование':
        category_for_url = 'dlya_biznesa'
    else:
        category_for_url = transliterate.translit(
            category, reversed=True).replace(' ', '_').strip().replace("'", '').replace('j', 'y')

    try:
        if subcategory == 'уборка':
            subcategory_for_url = 'uborka_klining'
        else:
            subcategory_for_url = transliterate.translit(
                subcategory, reversed=True).strip().replace(' ', '_').replace("'", '').replace('j', 'y')
    except transliterate.exceptions.LanguageDetectionError:
        subcategory_for_url = ''
    handled_url = process_url_for_validity(
        ad_name, region, category, subcategory,
        f'{HOST}/{region_for_url}/{category_for_url}/{subcategory_for_url}?{ad_name_for_url}')
    URL = handled_url

def set_common_amount_of_ad() -> None:
    """Set common amount of ad."""

    page_with_ads = get_response(URL)
    bs_html: BeautifulSoup = BeautifulSoup(page_with_ads.content, 'html.parser')
    amount_ad = bs_html.find('span', class_='page-title-count-1oJOc')
    output_xlsx_file['Общее количество объявлений'] = int(amount_ad.string.replace(' ', ''))

def set_total_amount_views(views_on_ad_page: list) -> None:
    """
    Set total amount views on today and in generally first 50 ads and all.

    views_on_ad_page: list -> list consist of views all time and today. ['{views all time}', '(+{views today})']
    """
    global counter_first_fifty_ads

    if counter_first_fifty_ads != 0:
        output_xlsx_file['Общее количество просмотров первых 50 объявлений (всего)'] += int(
            views_on_ad_page[0])
        output_xlsx_file['Общее количество просмотров первых 50 объявлений (сегодня)'] += int(
            views_on_ad_page[1][2:-1])
        counter_first_fifty_ads -= 1
    output_xlsx_file['Общее количество просмотров всего'] += int(views_on_ad_page[0])
    output_xlsx_file['Общее количество просмотров за сегодня'] += int(views_on_ad_page[1][2:-1])

def add_price_to_price_from_all_ads(price_of_product: int_price) -> None:
    """
    Adds the price to the price from all ads.

    price_of_product: int_price -> price specified in the ad.
    """
    output_xlsx_file['Средняя цена всех со всех объявлений'] += price_of_product

def set_date_of_publication_of_ad() -> None:
    """Sets the date of publication of the ad at 10, 20 and 50 places."""
    list_of_ad_publication_dates = []
    max_number_of_attempts = 10

    while not list_of_ad_publication_dates:
        time.sleep(3)

        ads_sorted_by_date = get_response(URL, {'s': 104})
        bs_page_ads = BeautifulSoup(ads_sorted_by_date.content, 'html.parser')
        ads_publication_dates = bs_page_ads.find_all('div', class_='snippet-date-info')

        if max_number_of_attempts == 0:
            print('Повторите попытку.')
            exit()

        max_number_of_attempts -= 1

        for date_tag in ads_publication_dates[:output_xlsx_file['Общее количество объявлений']]:
            try:
                day_publication, month_publication = date_tag['data-tooltip'].split()[:2]
            except ValueError:
                continue
            month_publication = MONTH_IN_NUMBER[month_publication]
            list_of_ad_publication_dates.append(f'{day_publication}.{month_publication}.2020')

    # If there are fewer ads, the script tries to install what it can.
    try:
        output_xlsx_file['Дата публикации 10-ого объявления (сортировка по дате)'] = list_of_ad_publication_dates[10]
    except IndexError:
        return

    try:
        output_xlsx_file['20-ого объявления (сортировка по дате)'] = list_of_ad_publication_dates[20]
    except IndexError:
        return

    try:
        output_xlsx_file['50-ого объявления (сортировка по дате)'] = list_of_ad_publication_dates[-1]
    except IndexError:
        return

def set_average_price_of_all_ads() -> None:
    """Sets and calculates the arithmetic average of prices from all ads."""
    output_xlsx_file['Средняя цена всех со всех объявлений'] = \
        (output_xlsx_file['Средняя цена всех со всех объявлений']
         / output_xlsx_file['Общее количество объявлений'])

def date_handler(date_publication: str) -> str:
    """Handled received date from ad. Return data in format day.month.year."""
    datetime_today = datetime.date.today()
    datetime_for_format = datetime.date(datetime_today.year, datetime_today.month, datetime_today.day)

    if date_publication.lower() == 'сегодня':
        date_publication = datetime_for_format.strftime('%d.%m.%Y')
    elif date_publication.lower() == 'вчера':
        date_publication = str(datetime_today.day - 1) + datetime_for_format.strftime('.%m.%Y')
    else:
        day_publication, month_publication = date_publication.split()
        month_publication = MONTH_IN_NUMBER[month_publication]
        date_publication = f'{day_publication}.{month_publication}.2020'
    return date_publication

def set_data_about_ad(ad_page: Response, views_on_ad: list, price_of_product: int_price, link: url_type) -> None:
    """Fill dictionary need data and add filled dictionary in list."""
    bs_ad_page: BeautifulSoup = BeautifulSoup(ad_page.content, 'html.parser')

    try:
        total_views: int = int(views_on_ad[0])
        views_today: int = int(views_on_ad[1][2:-1])
    except IndexError:
        total_views = 0
        views_today = 0

    statistic_about_ad = {
        'Название объявления': '',
        'Описание': '',
        'Адрес': '',
        'Цена': price_of_product,
        'Имя продовца': '',
        'Количество объявлений у продовца': 0,
        'Количество просмотров всего': total_views,
        'Количество просмотров за сегодня': views_today,
        'Ссылка на объявление': link,
        'Дата публикации': '',
        'Время публикации': ''
    }

    try:
        ad_description: str = bs_ad_page.find('div', class_='item-description').get_text().strip()
    except AttributeError:
        ad_description: str = 'Нету'
    try:
        seller_address: str = bs_ad_page.find('div', class_='item-address').get_text().strip()
    except AttributeError:
        seller_address: str = 'Нету'
    try:
        button_with_number_active_ads = bs_ad_page.find(class_='seller-info-favorite-seller-buttons').get('data-props')
        number_active_ads = int(json.loads(button_with_number_active_ads)['summary'].split()[0])
    except (AttributeError, IndexError):
        number_active_ads = 0

    ad_title: str = bs_ad_page.find('span', class_='title-info-title-text').string.strip()
    date_and_time_publication = bs_ad_page.find('div', class_='title-info-metadata-item-redesign').string.strip()
    seller_name: str = bs_ad_page.find('div', class_='seller-info-name').get_text()
    publication_date = date_handler(date_and_time_publication[:-5].strip().replace(' в', ''))
    publication_time = date_and_time_publication[-5:]  # 16:24

    statistic_about_ad['Название объявления'] = ad_title
    statistic_about_ad['Описание'] = ad_description
    statistic_about_ad['Адрес'] = seller_address
    statistic_about_ad['Имя продовца'] = seller_name
    statistic_about_ad['Количество объявлений у продовца'] = number_active_ads
    statistic_about_ad['Дата публикации'] = publication_date
    statistic_about_ad['Время публикации'] = publication_time

    list_statistic_about_ad.append(statistic_about_ad)

def bypass_traps_avito(bs_ad_html: BeautifulSoup, ad_page: Response, link: url_type) -> None:
    """
    Bypasses the traps that Avito makes. For example, it can send that
     the page was not found, although it exists. If the price of the ad
     does not indicate the product price is 0. If Avito sends an error 429,
     the script stops working because it can no longer send requests due to IP blocking.

    bs_ad_html: BeautifulSoup -> object from BeautifulSoup from ad page.
    ad_page: Response -> response from ad page Avito.
    link: str -> link on ad page.
    """
    try:
        views_on_ad_page: str = bs_ad_html.find(class_='title-info-metadata-item').get_text()[1:].split()
        try:
            price_of_product: int_price = int(
                str(bs_ad_html.find('span', class_='js-item-price').text).replace(' ', ''))
        except AttributeError:
            price_of_product: int_price = 0
    except AttributeError:
        if ad_page.status_code == 404:
            try:
                ad_page: Response = get_response(link)
                bs_ad_html: BeautifulSoup = BeautifulSoup(ad_page.content, 'html.parser')
                views_on_ad_page: list = bs_ad_html.find(class_='title-info-metadata-item').get_text()[1:].split()
                try:
                    price_of_product: int_price = int(
                        str(bs_ad_html.find('span', class_='js-item-price').text).replace(' ', ''))
                except AttributeError:
                    price_of_product: int_price = 0
            except AttributeError:
                print('''
                      Это исключение возможно лишь при том, если Avito отвечает не тем сайтом, который пар-
                      сер ожидал увидеть. За остальной информацией обращайтесь к файлу help.txt.

                      Если программа продолжает парсить, необращайте внимания.
                      ''')
            else:
                try:
                    set_total_amount_views(views_on_ad_page)
                except IndexError:
                    pass
                add_price_to_price_from_all_ads(price_of_product)
                set_data_about_ad(ad_page, views_on_ad_page, price_of_product, link)
        elif ad_page.status_code == 429:
            print('''
                  Ваш IP был заблокирован на время. Нужно подождать некоторое время, либо зайти на
                  на сайт через браузер и ввести капчу. Если ничто из этого не помогло, следуйте указа-
                  ниям в файле help.txt.
                  ''')
            exit()
    else:
        try:
            set_total_amount_views(views_on_ad_page)
        except IndexError:
            pass
        add_price_to_price_from_all_ads(price_of_product)
        set_data_about_ad(ad_page, views_on_ad_page, price_of_product, link)

def send_ad_data_to_functions(max_pages: int) -> None:
    """
    Send data to functions, received from ad page. In this function implementation
     simulation of the human factor (makes random pauses before the request
     (about 8-10 seconds) and requests a random number of times (from 1 to 5)).
     This is done so that Avito does not consider the parser a bot. Also protects
     against advertisements from another city that spoil statistics. He function
     goes through the pages (if the number of ads allows), opens the ads and sends
     the desired data to the functions. If Avito give sponsored links, function skip this.

     max_pages: int -> maximum number of pages with ads.
    """
    maximum_amount_of_open_links_without_pause: int = randint(1, 5)
    next_page: int = 1
    counter_parsed_link: int = 1

    # Sometimes ads are not enough for one page and Avito fills it with similar ads.
    if output_xlsx_file['Общее количество объявлений'] <= 50:
        ad_limit = output_xlsx_file['Общее количество объявлений']
    else:
        ad_limit = 'More than 50'

    while next_page != max_pages + 1:

        link_on_page_with_ads: url_type = get_response(URL).url

        # Avito does not allow you to navigate through the pages if a link with random letters is not specified
        # (this is how it looks like avito.ru/chita/krasota_i_zdorove/kupit-meditsinskie_izdeliya-ASgBAgICAUSEAqgJ),
        # this link can be obtained by request. This is the only way to go through the pages
        if '?' in link_on_page_with_ads:
            link_to_navigate_through_pages: url_type = link_on_page_with_ads + f'&p={next_page}'
        else:
            link_to_navigate_through_pages: url_type = link_on_page_with_ads + f'?p={next_page}'
        time.sleep(2.2)

        page_with_ad: Response = get_response(link_to_navigate_through_pages)
        content_of_ad_page: BeautifulSoup = BeautifulSoup(page_with_ad.content, 'html.parser')
        links_on_ads: List[url_type] = []

        for element in content_of_ad_page.find_all(class_='item_table-description'):
            link_on_ad = HOST + element.find('a').get('href')
            if link_on_ad[21:].find('/') == -1:
                continue
            else:
                links_on_ads.append(link_on_ad)

        if not links_on_ads:
            # Avito sometimes gives out the wrong data that it usually gives out (we are talking about classes).
            for element in content_of_ad_page.find_all(class_='iva-item-body-NPl6W'):
                link_on_ad = HOST + element.find('a').get('href')
                if link_on_ad[21:].find('/') == -1:
                    continue
                else:
                    links_on_ads.append(link_on_ad)
        if not links_on_ads:
            open('new_tags_неудалять.html', 'w').write(page_with_ad.text)
            print('Авито прислал невалидный сайт, перезапустите программу с теми параметрами, '
                  'на которых остановился парсер.')
            exit()

        time.sleep(3)

        for link in links_on_ads:
            if link not in parsed_ads:
                # It is done to imitate a person, so that Avito does not consider the parser a bot.
                # If delete this code, Avito can give block by IP for a while.
                if maximum_amount_of_open_links_without_pause == 0:
                    time.sleep(round(randint(8, 10) + random(), 2))
                    maximum_amount_of_open_links_without_pause = randint(1, 5)
                maximum_amount_of_open_links_without_pause -= 1

                ad_page: Response = get_response(link)
                bs_ad_content: BeautifulSoup = BeautifulSoup(ad_page.content, 'html.parser')
                bypass_traps_avito(bs_ad_content, ad_page, link)
                parsed_ads.append(link)

                print(f'{link[12:]: <115} спарсено удачно.',
                      f'Осталось {counter_parsed_link}/{output_xlsx_file["Общее количество объявлений"]}')
                counter_parsed_link += 1
                if isinstance(ad_limit, int):
                    ad_limit -= 1
                if ad_limit == 0:
                    break

        if isinstance(ad_limit, int): break
        print(f'{next_page} из {max_pages} спарсено.')
        next_page += 1

    set_average_price_of_all_ads()
    print('Парсинг завершен успешно!')

# Write to xlsx
def write_first_list(workbook_list) -> None:
    """Writes in first leaf (Общая статистика) received data in the process of parsing."""
    row: int = 2
    column_letters = 'ABCDEFGHIJK'
    row_is_free: bool = False

    while True:

        for col_letter in column_letters:
            if workbook_list[col_letter + str(row)].value is None:
                row_is_free = True
            else:
                row_is_free = False
                break

        if row_is_free:
            for column in range(1, 11 + 1):
                workbook_list.cell(row=row, column=column).value = list(output_xlsx_file.values())[column - 1]
            break
        else:
            row += 1

def write_second_list(workbook_list) -> None:
    """Writes in second leaf (Статистика по объявлениям) received data in the process of parsing."""
    row: int = 2
    column_letters = 'ABCDEFGHIJKL'
    row_is_free: bool = False
    counter_rows: int = 0

    while True:

        for col_letter in column_letters:
            if workbook_list[col_letter + str(row)].value is None:
                row_is_free = True
            else:
                row_is_free = False
                break
        if row_is_free:
            for statistic_ad in list_statistic_about_ad:
                counter_rows += 1
                workbook_list.cell(row=row, column=1).value = counter_rows
                for column in range(2, len(names_columns_in_statistic_by_ad) + 1):
                    workbook_list.cell(row=row, column=column).value = list(statistic_ad.values())[column - 2]
                row += 1
            break
        else:
            row += 1

def send_workbook_lists() -> None:
    """
    Send functions workbook list in which need write data. If the excel file
    table does not exist, it creates a new one with the required parameters and sheets.
    """
    xlsx_file_name: str = 'avito_statistic.xlsx'

    try:
        workbook = openpyxl.load_workbook(xlsx_file_name)
    except FileNotFoundError:
        new_workbook = openpyxl.Workbook()

        for sheet_name in new_workbook.sheetnames:
            sheet = new_workbook[sheet_name]
            new_workbook.remove(sheet)

        new_workbook_list_1 = new_workbook.create_sheet('Общая статистика')

        for column in range(1, 11 + 1):
            new_workbook_list_1.cell(row=1, column=column).value = list(output_xlsx_file.keys())[column - 1]

        new_workbook_list_2 = new_workbook.create_sheet('Статистика по объявлениям')

        for column in range(1, len(names_columns_in_statistic_by_ad) + 1):
            new_workbook_list_2.cell(row=1, column=column).value = names_columns_in_statistic_by_ad[column - 1]

        new_workbook.save(xlsx_file_name)
        new_workbook.close()
        workbook = openpyxl.load_workbook(xlsx_file_name)

    workbook_list_1 = workbook['Общая статистика']
    workbook_list_2 = workbook['Статистика по объявлениям']

    write_first_list(workbook_list_1)
    write_second_list(workbook_list_2)

    workbook.save(xlsx_file_name)
    workbook.close()

def run():
    global list_statistic_about_ad, counter_first_fifty_ads, output_xlsx_file, parsed_ads, URL

    row: int = 2
    column_letters = 'ABCD'
    row_is_fill: bool = False

    xlsx_file_path = input('Введите название файла с параметрами или его путь: ')
    workbook = openpyxl.load_workbook(xlsx_file_path)
    workbook_list = workbook['Лист1']

    while True:

        for col_letter in column_letters:
            if workbook_list[col_letter + str(row)].value is None:
                row_is_fill = False
            else:
                row_is_fill = True
                break

        if row_is_fill:
            print('Подождите 40-65 секунд...')
            time.sleep(round(randint(40, 65) + random(), 2))

            # Parameters for link which need parsed
            ad_name = workbook_list[f'A{row}'].value
            region = workbook_list[f'B{row}'].value.lower().strip()
            category = workbook_list[f'C{row}'].value.lower().strip()
            subcategory = workbook_list[f'D{row}'].value.lower().strip()

            generate_valid_url_for_parsing(ad_name, region, category, subcategory)
            if not URL:
                continue

            row += 1

            # Reset past data
            list_statistic_about_ad = []
            counter_first_fifty_ads = 50
            parsed_ads = []
            output_xlsx_file = {
                'Ключ': ad_name,
                'Регион': region,
                'Общее количество объявлений': 0,
                'Общее количество просмотров всего': 0,
                'Общее количество просмотров за сегодня': 0,
                'Дата публикации 10-ого объявления (сортировка по дате)': 'Нету',
                '20-ого объявления (сортировка по дате)': 'Нету',
                '50-ого объявления (сортировка по дате)': 'Нету',
                'Средняя цена всех со всех объявлений': 0,
                'Общее количество просмотров первых 50 объявлений (сегодня)': 0,
                'Общее количество просмотров первых 50 объявлений (всего)': 0
            }

            print(f'Парсится ссылка {URL}')
            print(f'{"Ключ": ^25} | {"Регион": ^26} | {"Категория": ^29} | {"Подкатегория": ^32}')
            print('_' * 121)
            print(f'{ad_name: ^23} | {region: ^26} | {category: ^28} | {subcategory: ^32}')

            set_common_amount_of_ad()
            set_date_of_publication_of_ad()
            time.sleep(3)

            avito_response: Response = get_response(URL)
            avito_page_content: BeautifulSoup = BeautifulSoup(avito_response.content, 'html.parser')
            try:
                max_pages = int(avito_page_content.find_all('span', class_='pagination-item-1WyVp')[-2].text)
            except IndexError:
                max_pages = 1

            send_ad_data_to_functions(max_pages)

            print('Записываем данные...')
            send_workbook_lists()
            print('Данные успешно записаны!')
        else:
            break

    print('Время работы парсера', round(time.time() - start_time_script), 'секунды')

if __name__ == '__main__':
    run()


# -*- coding: utf-8 -*-
from google.appengine.api import users
from google.appengine.ext import db
import logging

from utils import slugify, versionate

from datetime import datetime

# Generic Functions
def get_exposed_managed_tables():
    values = []
    values.append({'model_table': 'Noticia', 'icon_file': '/static/images/Noticia.png', 'class_name': 'NOTÍCIAS'})
    values.append({'model_table': 'Evento', 'icon_file': '/static/images/Evento.png', 'class_name': 'EVENTOS'})
    values.append({'model_table': 'Revista', 'icon_file': '/static/images/Revista.png', 'class_name': 'REVISTAS'})
    values.append({'model_table': 'Grupo', 'icon_file': '/static/images/Grupo.png', 'class_name': 'GRUPOS'})
    values.append({'model_table': 'Aviso', 'icon_file': '/static/images/Aviso.png', 'class_name': 'AVISOS'})
    values.append({'model_table': 'Banner', 'icon_file': '/static/images/Banner.png', 'class_name': 'BANNERS'})
    values.append({'model_table': 'Foto', 'icon_file': '/static/images/Foto.png', 'class_name': 'FOTOS'})
    values.append({'model_table': 'Menu', 'icon_file': '/static/images/Menu.png', 'class_name': 'MENUS'})
    values.append({'model_table': 'Pagina', 'icon_file': '/static/images/Pagina.png', 'class_name': 'PÁGINAS'})
    return values

def latest_records(entity, q=None, start=1, PAGESIZE=10, order=None):
    limit=int(PAGESIZE)
    has_query=q != '' and q != 'NULL' and q != None and q != 'None' and q != 'all'
    if has_query:
        if order is not None:
            sql="SELECT * FROM %(entity)s WHERE %(query)s ORDER BY %(order)s" % {'entity': str(entity.kind()), 'query': q, 'order': order}
        else:
            sql="SELECT * FROM %(entity)s WHERE %(query)s" % {'entity': str(entity.kind()), 'query': q}
        query=db.GqlQuery(sql)
        return query.fetch(limit, start)
    else:
        if order is not None:
            try:
                query = db.Query(entity)
                query.order(order)
                return query.fetch(limit,int(start))
            except:
                query = db.Query(entity)
                query.order(order)
                return query.fetch(limit)
        else:
            try:
                return db.Query(entity).fetch(limit,int(start))
            except:
                return db.Query(entity).fetch(limit)

def count_records(entity):
    return entity.all().count()

def entity_by_id(entity, id):
    return entity.get_by_id(int(id))

def delete_entity(entity, id):
    entity=entity_by_id(entity, id)
    db.delete(entity)

def exists_entity(entity, field, value):
    q=db.Query(entity).filter('%s =' % field, value).get()
    return q

def find_slug(Entity, slug):
    q = db.Query(Entity).filter('slug =', slug).get()
    return q is not None

def get_model_by_name(name):
    if name == 'Noticia':
        return Noticia
    elif name == 'Evento':
        return Evento
    elif name == 'Revista':
        return Revista
    elif name == 'Grupo':
        return Grupo
    elif name == 'Aviso':
        return Aviso
    elif name == 'Banner':
        return Banner
    elif name == 'Foto':
        return Foto
    elif name == 'Parametro':
        return Parametro
    elif name == 'Pagina':
        return Pagina
    elif name == 'Menu':
        return Menu
    else:
        return None

def save_entity(entity, form):
    if entity == 'Noticia':
        save_noticia(form)
    elif entity == 'Evento':
        save_evento(form)
    elif entity == 'Revista':
        save_revista(form)
    elif entity == 'Grupo':
        save_grupo(form)
    elif entity == 'Aviso':
        save_aviso(form)
    elif entity == 'Banner':
        save_banner(form)
    elif entity == 'Foto':
        save_foto(form)
    elif entity == 'Parametro':
        save_parametro(form)
    elif entity == 'Pagina':
        save_pagina(form)
    elif entity == 'Menu':
        save_menu(form)
    else:
        pass

def get_entity_index_list(entity, filtro, valor, order, limit):
    query = db.Query(entity)
    if filtro is not None:
        q = db.Query(entity).filter('%s = ' % filtro, valor)
        q.order(order)
        return q.fetch(limit)
    else:
        query = db.Query(entity)
        query.order(order)
        return query.fetch(limit)

# App Models.
class Noticia(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_hora = db.DateTimeProperty()
    author = db.UserProperty()
    palavras_chaves = db.StringProperty(required=True)
    intro_noticia = db.TextProperty(required=True)
    noticia_completa = db.TextProperty(required=True)
    thumb_noticia = db.StringProperty(required=False)
    foto_01 = db.StringProperty(required=False)
    thumb_01 = db.StringProperty(required=False)
    foto_02 = db.StringProperty(required=False)
    thumb_02 = db.StringProperty(required=False)
    foto_03 = db.StringProperty(required=False)
    thumb_03 = db.StringProperty(required=False)
    destaque = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Noticia"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Notícias"

    @classmethod
    def get_default_field_order(cls):
        return "-data_hora"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Data/Hora', 'Destaque']
        fields = ['titulo', 'data_hora', 'destaque']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field):
            if field == 'data_hora':
                data_hora = getattr(cls, field)
                if data_hora:
                    return data_hora.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            else:
                return getattr(cls, field)

def save_noticia(form):
    entity=exists_entity(Noticia, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_hora=datetime.strptime(form.data_hora,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.palavras_chaves=form.palavras_chaves
        entity.intro_noticia=form.intro_noticia
        entity.noticia_completa=form.noticia_completa
        entity.thumb_noticia=form.thumb_noticia
        entity.foto_01=form.foto_01
        entity.thumb_01=form.thumb_01
        entity.foto_02=form.foto_02
        entity.thumb_02=form.thumb_02
        entity.foto_03=form.foto_03
        entity.thumb_03=form.thumb_03
        entity.destaque=form.destaque
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_hora=datetime.strptime(form.data_hora,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.palavras_chaves=form.palavras_chaves
        entity.intro_noticia=form.intro_noticia
        entity.noticia_completa=form.noticia_completa
        entity.thumb_noticia=form.thumb_noticia
        entity.foto_01=form.foto_01
        entity.thumb_01=form.thumb_01
        entity.foto_02=form.foto_02
        entity.thumb_02=form.thumb_02
        entity.foto_03=form.foto_03
        entity.thumb_03=form.thumb_03
        entity.destaque=form.destaque
        db.put(entity)
    else:
        while find_slug(Noticia, slug):
            slug = versionate(slug)
        db.put(Noticia(
            titulo=form.titulo,
            slug=slug,
            data_hora = datetime.strptime(form.data_hora,'%Y-%m-%d %H:%M:%S'),
            author = users.get_current_user(),
            palavras_chaves=form.palavras_chaves,
            intro_noticia = form.intro_noticia,
            noticia_completa = form.noticia_completa,
            thumb_noticia = form.thumb_noticia,
            foto_01 = form.foto_01,
            thumb_01 = form.thumb_01,
            foto_02 = form.foto_02,
            thumb_02 = form.thumb_02,
            foto_03 = form.foto_03,
            thumb_03 = form.thumb_03,
            destaque=form.destaque))

def all_news_rss():
    query = db.Query(Noticia)
    query.order('-data_hora')
    result = query.fetch(1000)
    return result

class Evento(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_publicacao = db.DateTimeProperty()
    data_hora_evento = db.DateTimeProperty()
    author = db.UserProperty()
    palavras_chaves = db.StringProperty(required=True)
    thumb_evento = db.StringProperty(required=False)
    foto_evento = db.StringProperty(required=False)
    texto = db.TextProperty(required=True)
    url_album_evento = db.StringProperty(required=False)
    url_canal_evento = db.StringProperty(required=False)
    destaque = db.StringProperty(required=False)

    @classmethod
    def get_class_name(cls):
        return "Evento"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Eventos"

    @classmethod
    def get_default_field_order(cls):
        return "-data_hora_evento"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo'), ('destaque', 'Destaque')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Data/Hora Evento', 'Publicado Em', 'Destaque']
        fields = ['titulo', 'data_hora_evento', 'data_publicacao', 'destaque']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field):
            if field == 'data_hora_evento':
                data_hora_evento = getattr(cls, field)
                if data_hora_evento:
                    return data_hora_evento.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            elif field == 'data_publicacao':
                data_publicacao = getattr(cls, field)
                if data_publicacao:
                    return data_publicacao.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            else:
                return getattr(cls, field)

def save_evento(form):
    entity=exists_entity(Evento, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.data_hora_evento=datetime.strptime(form.data_hora_evento,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.palavras_chaves=form.palavras_chaves
        entity.thumb_evento=form.thumb_evento
        entity.foto_evento=form.foto_evento
        entity.texto=form.texto
        entity.url_album_evento=form.url_album_evento
        entity.url_canal_evento=form.url_canal_evento
        entity.destaque=form.destaque
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.data_hora_evento=datetime.strptime(form.data_hora_evento,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.palavras_chaves=form.palavras_chaves
        entity.thumb_evento=form.thumb_evento
        entity.foto_evento=form.foto_evento
        entity.texto=form.texto
        entity.url_album_evento=form.url_album_evento
        entity.url_canal_evento=form.url_canal_evento
        entity.destaque=form.destaque
        db.put(entity)
    else:
        while find_slug(Evento, slug):
            slug = versionate(slug)
        db.put(Evento(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            data_hora_evento = datetime.strptime(form.data_hora_evento,'%Y-%m-%d %H:%M:%S'),
            author = users.get_current_user(),
            palavras_chaves=form.palavras_chaves,
            thumb_evento = form.thumb_evento,
            foto_evento = form.foto_evento,
            texto=form.texto,
            url_album_evento=form.url_album_evento,
            url_canal_evento=form.url_canal_evento,
            destaque=form.destaque))

def all_events():
    query = db.Query(Evento)
    result = query.fetch(1000)
    return result

def all_events_rss():
    query = db.Query(Evento)
    query.order('-data_publicacao')
    result = query.fetch(1000)
    return result

def nexts_eventos():
    hoje = datetime.now().date()
    query = db.Query(Evento)
    query.filter('data_hora_evento >=', hoje)
    query.order('-data_hora_evento')
    result = query.fetch(1000)
    return result

def next_eventos():
    from datetime import date, timedelta
    hoje = date.today()
    dia = timedelta(days=1)
    query = db.Query(Evento)
    query.filter('data_hora_evento >=', hoje-dia)
    query.filter('destaque = ', 'Sim')
    query.order('-data_hora_evento')
    result = query.fetch(1)
    return result

def ultimo_evento_publicado():
    import datetime
    last = db.Query(Evento).order('-data_publicacao').get()
    return last.data_publicacao if last else datetime.now()

class Revista(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_publicacao = db.DateTimeProperty()
    thumb_revista = db.StringProperty(required=True)
    link_arquivo = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Revista"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Revistas"

    @classmethod
    def get_default_field_order(cls):
        return "-data_publicacao"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Publicado Em']
        fields = ['titulo', 'data_publicacao']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field):
            if field == 'data_publicacao':
                data_publicacao = getattr(cls, field)
                if data_publicacao:
                    return data_publicacao.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            else:
                return getattr(cls, field)

def save_revista(form):
    entity=exists_entity(Revista, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.thumb_revista=form.thumb_revista
        entity.link_arquivo=form.link_arquivo
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.thumb_revista=form.thumb_revista
        entity.link_arquivo=form.link_arquivo
        db.put(entity)
    else:
        while find_slug(Revista, slug):
            slug = versionate(slug)
        db.put(Revista(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            author = users.get_current_user(),
            thumb_revista = form.thumb_revista,
            link_arquivo = form.link_arquivo))

def latest_revista():
    query = db.Query(Revista)
    query.order('-data_publicacao')
    result = query.fetch(1)
    if result:
        return result[0]
    else:
        return None

class Grupo(db.Model):
    titulo = db.StringProperty(required=True)
    ordem = db.IntegerProperty(required=True)
    slug = db.StringProperty(required=True)
    thumb_banner = db.StringProperty(required=False)
    endereco_link = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Grupo"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Grupos"

    @classmethod
    def get_default_field_order(cls):
        return "ordem"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Ordem']
        fields = ['titulo', 'ordem']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field): return getattr(cls, field)

def save_grupo(form):
    entity=exists_entity(Grupo, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.ordem=int(form.ordem)
        entity.slug=slug
        entity.thumb_banner=form.thumb_banner
        entity.endereco_link=form.endereco_link
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.ordem=int(form.ordem)
        entity.slug=slug
        entity.thumb_banner=form.thumb_banner
        entity.endereco_link=form.endereco_link
        db.put(entity)
    else:
        while find_slug(Grupo, slug):
            slug = versionate(slug)
        db.put(Grupo(
            titulo=form.titulo,
            ordem=int(form.ordem),
            slug=slug,
            thumb_banner = form.thumb_banner,
            endereco_link = form.endereco_link))

class Aviso(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_publicacao = db.DateTimeProperty()
    author = db.UserProperty()
    texto = db.TextProperty(required=True)
    ativo = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Aviso"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Avisos"

    @classmethod
    def get_default_field_order(cls):
        return "-data_publicacao"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Ativo', 'Publicado Em']
        fields = ['titulo', 'ativo', 'data_publicacao']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field):
            if field == 'data_publicacao':
                data_publicacao = getattr(cls, field)
                if data_publicacao:
                    return data_publicacao.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            else:
                return getattr(cls, field)

def save_aviso(form):
    entity=exists_entity(Aviso, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.texto=form.texto
        entity.ativo=form.ativo
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.author=users.get_current_user()
        entity.texto=form.texto
        entity.ativo=form.ativo
        db.put(entity)
    else:
        while find_slug(Aviso, slug):
            slug = versionate(slug)
        db.put(Aviso(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            author = users.get_current_user(),
            texto=form.texto,
            ativo=form.ativo))

class Banner(db.Model):
    titulo = db.StringProperty(required=True)
    ordem = db.IntegerProperty(required=True)
    slug = db.StringProperty(required=True)
    author = db.UserProperty()
    thumb_banner = db.StringProperty(required=True)
    foto_banner = db.StringProperty(required=True)
    endereco_site_patrocinador = db.StringProperty(required=False)
    exibir_pagina_inicial = db.StringProperty(required=True)
    status = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Banner"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Banners"

    @classmethod
    def get_default_field_order(cls):
        return "titulo"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo'), ('exibir_pagina_inicial', 'Pág. Inicial'), ('status', 'Status')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Ordem', 'Exibir na Página Inicial', 'Status']
        fields = ['titulo', 'ordem', 'exibir_pagina_inicial', 'status']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field): return getattr(cls, field)

def save_banner(form):
    entity=exists_entity(Banner, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.ordem=int(form.ordem)
        entity.slug=slug
        entity.author=users.get_current_user()
        entity.thumb_banner=form.thumb_banner
        entity.foto_banner=form.foto_banner
        entity.endereco_site_patrocinador=form.endereco_site_patrocinador
        entity.exibir_pagina_inicial=form.exibir_pagina_inicial
        entity.status=form.status
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.ordem=int(form.ordem)
        entity.slug=slug
        entity.author=users.get_current_user()
        entity.thumb_banner=form.thumb_banner
        entity.foto_banner=form.foto_banner
        entity.endereco_site_patrocinador=form.endereco_site_patrocinador
        entity.exibir_pagina_inicial=form.exibir_pagina_inicial
        entity.status=form.status
        db.put(entity)
    else:
        while find_slug(Banner, slug):
            slug = versionate(slug)
        db.put(Banner(
            titulo=form.titulo,
            ordem=int(form.ordem),
            slug=slug,
            author = users.get_current_user(),
            thumb_banner = form.thumb_banner,
            foto_banner = form.foto_banner,
            endereco_site_patrocinador=form.endereco_site_patrocinador,
            exibir_pagina_inicial=form.exibir_pagina_inicial,
            status=form.status))

class Foto(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_publicacao = db.DateTimeProperty()
    thumb = db.StringProperty(required=False)
    foto = db.StringProperty(required=False)
    status = db.StringProperty(required=True) #(sim, não)
    link_acao_foto = db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Foto"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Fotos"

    @classmethod
    def get_default_field_order(cls):
        return "titulo"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo'), ('status', 'Status')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo', 'Publicado Em', 'Status']
        fields = ['titulo', 'data_publicacao', 'status']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field):
            if field == 'data_publicacao':
                data_publicacao = getattr(cls, field)
                if data_publicacao:
                    return data_publicacao.strftime('%d/%m/%Y %H:%M')
                else:
                    return None
            else:
                return getattr(cls, field)

def save_foto(form):
    entity=exists_entity(Foto, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.thumb=form.thumb
        entity.foto=form.foto
        entity.status=form.status
        entity.link_acao_foto=form.link_acao_foto
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.data_publicacao=datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S')
        entity.thumb=form.thumb
        entity.foto=form.foto
        entity.status=form.status
        entity.link_acao_foto=form.link_acao_foto
        db.put(entity)
    else:
        while find_slug(Foto, slug):
            slug = versionate(slug)
        db.put(Foto(
            titulo=form.titulo,
            slug=slug,
            data_publicacao = datetime.strptime(form.data_publicacao,'%Y-%m-%d %H:%M:%S'),
            thumb = form.thumb,
            foto = form.foto,
            status = form.status,
            link_acao_foto=form.link_acao_foto))

class Pagina(db.Model):
    titulo = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    descricao_completa = db.StringProperty(required=True)
    palavras_chaves = db.StringProperty(required=True)
    data_criacao = db.DateTimeProperty()
    ultima_alteracao = db.DateTimeProperty()
    usuario_alteracao = db.UserProperty()
    conteudo=db.TextProperty(required=False)

    @classmethod
    def get_class_name(cls):
        return "Pagina"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Páginas"

    @classmethod
    def get_default_field_order(cls):
        return "titulo"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Titulo']
        fields = ['titulo']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field): return getattr(cls, field)

def save_pagina(form):
    entity=exists_entity(Pagina, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    if (entity is not None):
        entity.titulo=form.titulo
        entity.slug=slug
        entity.descricao_completa=form.descricao_completa
        entity.palavras_chaves=form.palavras_chaves
        entity.data_criacao=datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S')
        entity.ultima_alteracao=datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S')
        entity.usuario_alteracao=users.get_current_user()
        entity.conteudo=form.conteudo
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.titulo=form.titulo
        entity.slug=slug
        entity.descricao_completa=form.descricao_completa
        entity.palavras_chaves=form.palavras_chaves
        entity.data_criacao=datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S')
        entity.ultima_alteracao=datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S')
        entity.usuario_alteracao=users.get_current_user()
        entity.conteudo=form.conteudo
        db.put(entity)
    else:
        while find_slug(Pagina, slug):
            slug = versionate(slug)
        db.put(Pagina(
            titulo=form.titulo,
            slug=slug,
            descricao_completa=form.descricao_completa,
            palavras_chaves=form.palavras_chaves,
            data_criacao = datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S'),
            ultima_alteracao = datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S'),
            usuario_alteracao = users.get_current_user(),
            conteudo=form.conteudo))

class Menu(db.Model):
    ordem = db.IntegerProperty(required=True)
    titulo = db.StringProperty(required=True)
    descricao = db.StringProperty(required=True)
    slug = db.StringProperty(required=True)
    data_criacao = db.DateTimeProperty()
    ultima_alteracao = db.DateTimeProperty()
    usuario_alteracao = db.UserProperty()
    pagina=db.ReferenceProperty(Pagina, required=False, collection_name='pagina_set')
    endereco=db.StringProperty(required=False)
    menu_pai = db.SelfReferenceProperty(required=False)
    nivel =db.StringProperty(required=True)

    @classmethod
    def get_class_name(cls):
        return "Menu"

    @classmethod
    def get_list_title(cls):
        return "Listagem de Menus"

    @classmethod
    def get_default_field_order(cls):
        return "ordem"

    @classmethod
    def exposed_search_properties(cls):
        return [('all', 'Todos'), ('ordem', 'Ordem'), ('nivel', 'Nível'), ('titulo', 'Titulo')]

    @classmethod
    def exposed_list_properties(cls):
        colunms = ['Ordem', 'Nível', 'Titulo', 'Endereço']
        fields = ['ordem', 'nivel', 'titulo', 'endereco']
        attributes = {'colunms': colunms, 'fields':fields}
        return attributes

    def get_field_value(cls, field):
        if hasattr(cls, field): return getattr(cls, field)

def save_menu(form):
    entity=exists_entity(Menu, 'titulo', form.titulo)
    slug = slugify(form.titulo)
    parent = None
    if form.menu_pai is not None:
        parent = Menu.get_by_id(int(form.menu_pai))
    pagina = None
    if form.pagina is not None:
        pagina = Pagina.get_by_id(int(form.pagina))

    if (entity is not None):
        entity.ordem=int(form.ordem)
        entity.titulo=form.titulo
        entity.descricao=form.descricao
        entity.slug=slug
        entity.data_criacao=datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S')
        entity.ultima_alteracao=datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S')
        entity.usuario_alteracao=users.get_current_user()
        entity.pagina=pagina
        entity.endereco=form.endereco
        entity.menu_pai=parent
        entity.nivel=form.nivel
        db.put(entity)
    elif (str(form.key) != ''):
        entity=db.get(form.key)
        entity.ordem=int(form.ordem)
        entity.titulo=form.titulo
        entity.descricao=form.descricao
        entity.slug=slug
        entity.data_criacao=datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S')
        entity.ultima_alteracao=datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S')
        entity.usuario_alteracao=users.get_current_user()
        entity.pagina=pagina
        entity.endereco=form.endereco
        entity.menu_pai=parent
        entity.nivel=form.nivel
        db.put(entity)
    else:
        while find_slug(Menu, slug):
            slug = versionate(slug)
        db.put(Menu(
            ordem=int(form.ordem),
            titulo=form.titulo,
            descricao=form.descricao,
            slug=slug,
            data_criacao = datetime.strptime(form.data_criacao,'%Y-%m-%d %H:%M:%S'),
            ultima_alteracao = datetime.strptime(form.ultima_alteracao,'%Y-%m-%d %H:%M:%S'),
            usuario_alteracao = users.get_current_user(),
            pagina=pagina,
            endereco=form.endereco,
            menu_pai=parent,
            nivel=form.nivel))

def get_pagina(endereco):
    menu = db.Query(Menu).filter('endereco =', endereco).get()
    if menu != None:
        pagina = Pagina.get_by_id(int(menu.pagina.key().id()))
    else:
        slug = endereco.replace('/pagina/','')
        pagina = exists_entity(Pagina, 'slug', slug)
    return pagina or None

def get_avaliables_menus():
    sql="SELECT * FROM Menu WHERE ordem > 0 ORDER BY ordem, menu_pai, nivel"
    query=db.GqlQuery(sql)
    return query.fetch(10000)

def get_menu_itens():
    menus = get_avaliables_menus()
    menu_sup = 0
    tem_submenu = False
    menu_html = ''
    for menu in menus:
        nivel = int(menu.nivel)
        if nivel == 1:
            if tem_submenu:
                menu_html = menu_html + '</ul>'
                menu_html = menu_html + '</li>'
                tem_submenu = False
            menu_sup = str(menu.key().id())
            menu_html = menu_html + '<li class="first_level_menu"><a href="' + menu.endereco + '" class="menu" title="' + menu.descricao + '">' + menu.titulo + '</a>'
        elif nivel == 2:
            if menu.menu_pai is not None:
                if str(menu_sup) == str(menu.menu_pai.key().id()):
                    if tem_submenu == False:
                        tem_submenu = True
                        menu_html = menu_html + '<ul>'
                        menu_html = menu_html + '<li><a href="' + menu.endereco + '" title="' + menu.descricao + '">' + menu.titulo + '</a></li>'
                    else:
                        menu_html = menu_html + '<li><a href="' + menu.endereco + '" title="' + menu.descricao + '">' + menu.titulo + '</a></li>'
    if tem_submenu:
        menu_html = menu_html + '</ul>'
    menu_html = menu_html + '</li>'
    #logging.debug(' ======== HTML MENU ========= ' + menu_html)
    return menu_html


# Correct the below code so that it displays all the rows from the Cars table with their column names

import sqlite3 as lite

con = lite.connect('test.db')
with con:
    cur = con.cursor()
    cur.execute('SELECT * FROM Cars')
    data = cur.fetchall()

    for d in data:
        print(d[0], d[1], d[2])


from matplotlib import pyplot as plt
from fitness import *
import sys

import random
from math import *
import numpy as np
import sympy as sy
from math import sin, cos, sqrt, floor
random.seed()

CONDITION = True
GENERATIONS = 20
POPULATION_SIZE = 20
MUTATION_CHANCE = 1.0  # mutation between 0 and 100 percent
if MUTATION_CHANCE > 100:
        MUTATION_CHANCE = 100.0
if MUTATION_CHANCE < 0:
        MUTATION_CHANCE = 0.0
NOTE="GA - random recomb."

print(NOTE)
print("PARAMETERS\n")
print("Generations: ", GENERATIONS)
print("Population size: ", POPULATION_SIZE)
print("Mutation chance: ", MUTATION_CHANCE, "%")
print("\n\n")

def genPopulation(geneList):
    for i in range(POPULATION_SIZE):
        g = Gene()
        geneList.append(g)

genes = []
best_fit = []

#generate the beginning population
genPopulation(genes)

for i in range(GENERATIONS):

    print(int(((i+1)/GENERATIONS)*100), "%", "DONE", end="\r")
    #genes[:] = []
    #sort

    #for x in genes:
    #    print(x.fitness)
    #store best

    #delete worse half of survivors
    del genes[-floor(len(genes)/2):]

    #refill the population
    for i in range(POPULATION_SIZE-len(genes)):
        (geneA, geneB) = random.sample(genes, 2)
        #print("A", geneA.a1, geneA.a2)
        #print("B",geneB.a1, geneB.a2)
        g = Gene()
        g.setParameters(a1=random.choice([geneA.a1, geneB.a1]), a2=random.choice([geneA.a1, geneB.a1]))
        #random mutation
        if random.random() < MUTATION_CHANCE/100:
                #print("RANDOM MUTATION")
                g.setParameters(a1=random.choice(populationValues),
                                a2=random.choice(populationValues))
        genes.append(g)

    genes.sort(key=lambda x: x.fitness, reverse=False)
    best_fit.append(genes[0].fitness)
DELTA = best_fit[0]-best_fit[GENERATIONS-1]

#PLOTTING

plt.figure()
plt.title("Change of q1 and q2, "+NOTE)

plt.plot(genes[0].q1)
plt.plot(genes[0].q2)
plt.scatter(np.arange(0, len(genes[0].q1)), genes[0].q1, label="q1")
plt.scatter(np.arange(0, len(genes[0].q2)), genes[0].q2, label="q2")
plt.xlabel("Time [s]")
plt.ylabel("Joint angle [rad]")
plt.legend(loc="upper left")
plt.show()

plt.figure()
plt.title("Change of torques, "+NOTE)
plt.plot(genes[0].tau1)
plt.plot(genes[0].tau2)
plt.plot(genes[0].tau3)
plt.plot(genes[0].tau4)
plt.scatter(np.arange(0, len(genes[0].q1)), genes[0].tau1, label="$\\tau_1$")
plt.scatter(np.arange(0, len(genes[0].q1)), genes[0].tau2, label="$\\tau_2$")
plt.scatter(np.arange(0, len(genes[0].q1)), genes[0].tau3, label="$\\tau_3$")
plt.scatter(np.arange(0, len(genes[0].q1)), genes[0].tau4, label="$\\tau_4$")
plt.xlabel("Time [s]")
plt.ylabel("Joint Torque [Nm]")
plt.legend(loc="upper left")
plt.show()

plt.figure()
plt.title(NOTE+"\nPopulation = "+str(POPULATION_SIZE) +
          ", $\\Delta = $" + str(DELTA)[0:4])

plt.xlabel("Generation")
plt.ylabel("Fitness")
plt.plot(
    best_fit, label="$\sum \sqrt{\\tau_1^2+\\tau_2^2+\\tau_3^2+\\tau_4^2}$")
plt.legend(loc='upper right')

plt.show()


#!/usr/bin/python3
#
# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#
import sqlite3
import jetson.inference
import jetson.utils
import os
import argparse
import sys
import time
import datetime
import schedule
import csv
#import schelude_folder_file_delete as file_cleanup
import numpy as np

video_output = ' '
image_output = ' '
image_folder = ' '
image_name = ' '
get_image_time = ' '
image_folder_label = 'imageNet_'
image_date = ' '
image_file_label = 'imageNet_'
record_video_ext = '.jpg'
image_capture_ext = '.jpg'
image_capture = ' '
v_out = ' '
img_out = ' '
storege_location = ' '

#camera_1_folder = 'test_camera'
camera_1_folder = 'camera_1'
camera_2_folder = 'camera_2'
camera_3_folder = 'camera_3'
camera_4_folder = 'camera_4'
camera_sub_dir = ' '
make_directory = ' '

time_stamp = 0
location = 0
description = 0
description_id = 0
confidence = 0
color = 0
axel_count = 0
license_plate = 0
rfid = 0
speed = 0
image_path = 0
lane_id = 0
take_action = ' '
conn = 0

detection_out = 0
FILTER1 = 'vehicle'
FILTER2 = 'bike'
FILTER3 = 'pickup truck'
START_MESS1 = 0
START_MESS2 = 400
EXIT_MESS1 = 900
EXIT_MESS2 = 1000
EXIT_COUNT = 0
distance = 0
center_l = 0
center_r = 0
count = 0
counter = 0

width = 1280 ## rtsp stream camera
height = 720 ## rtsp stream camera

MODEL = 'vehicleModel'
STORAGE_DIRECTORY = '/media/bob/ssd128/' # Storage database, images
if os.path.exists(STORAGE_DIRECTORY):
    pass
else:
    os.mkdir(STORAGE_DIRECTORY)

os.chdir(STORAGE_DIRECTORY)

#CREATE TABLE
#Following Python program will be used to create a table in previously created database:
def createTable():
    conn.execute('''CREATE TABLE DATA_AQUISITION2
    (TIMESTAMP TEXT  PRIMARY KEY NOT NULL,
    LOCATION TEXT NOT NULL,
    DESCRIPTION TEXT NOT NULL,
    DESCRIPTION_ID TEXT NOT NULL,
    CONFIDENCE TEXT NOT NULL,
    COLOR TEXT NOT NULL,
    AXEL_COUNT TEXT NOT NULL,
    LICENSE_PLATE TEXT NOT NULL,
    RFID TEXT NOT NULL,
    SPEED TEXT NOT NULL,
    LANE_ID TEXT NOT NULL,
    IMAGE_PATH TEXT NOT NULL);''')
    print ("Table created successfully")

if os.path.exists('Dbox_3.db'):
    conn = sqlite3.connect('Dbox_3.db')
else:
    conn = sqlite3.connect('Dbox_3.db')
    createTable()

print ("Opened database successfully")
print ('data acqusition started....')

def image_folder_name():
    get_image_time = datetime.datetime.now()
    camera_sub_dir = get_image_time.strftime("%Y_%m_%d")
    return camera_sub_dir

class cameraDirectory: #camera_root_dir, camera_sub_dir
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def getCameraDirectory(self):
        os.chdir(self.x)
        path = (os.getcwd())
        if os.path.exists(self.y):
            pass
        else:
            os.mkdir(os.path.join(path, self.y))
            return

class imageDirectory: #STORAGE_DIRECTORY, camera_sub_dir
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def getImageDirectory(self):
        os.chdir(STORAGE_DIRECTORY)
        os.chdir(self.x)
        path = (os.getcwd())
        if os.path.exists(self.y):
            pass
        else:
            os.mkdir(os.path.join(path, self.y))
            return

#test image root directory
image_directory = cameraDirectory(STORAGE_DIRECTORY, camera_1_folder)
image_directory.getCameraDirectory()
image_directory = cameraDirectory(STORAGE_DIRECTORY, camera_2_folder)
image_directory.getCameraDirectory()
image_directory = cameraDirectory(STORAGE_DIRECTORY, camera_3_folder)
image_directory.getCameraDirectory()
image_directory = cameraDirectory(STORAGE_DIRECTORY, camera_4_folder)
image_directory.getCameraDirectory()

#test image sub-directory
image_sub_directory = imageDirectory(camera_1_folder, image_folder_name())
image_sub_directory.getImageDirectory()
#os.chdir(STORAGE_DIRECTORY)
image_sub_directory = imageDirectory(camera_2_folder, image_folder_name())
image_sub_directory.getImageDirectory()
#os.chdir(STORAGE_DIRECTORY)
image_sub_directory = imageDirectory(camera_3_folder, image_folder_name())
image_sub_directory.getImageDirectory()
#os.chdir(STORAGE_DIRECTORY)
print('STORAGE_DIRECTORY IS: ', STORAGE_DIRECTORY)
image_sub_directory = imageDirectory(camera_4_folder, image_folder_name())
image_sub_directory.getImageDirectory()

boxes = []
confidence = []
class_id = []
classes = []

def getVideo():
    get_image_time = datetime.datetime.now()
    get_image_time = get_image_time.strftime("%Y-%m-%d-%H%M%S%f")
    #image_date, image_time = get_image_time.split('_')
    #image_name = '{}{}{}'.format(image_file_label, get_image_time, record_video_ext)
    video_out_1 = '{}{}'.format(get_image_time, record_video_ext)
    video_out_2 = '{}{}{}'.format(camera_2_folder, get_image_time, image_capture_ext)
    video_out_3 = '{}{}{}'.format(camera_3_folder, get_image_time, image_capture_ext)
    video_out_4 = '{}{}{}'.format(camera_4_folder, get_image_time, image_capture_ext)
    #image_folder = str(image_folder)
    #image_name = str(image_name)
    os.chdir(STORAGE_DIRECTORY)
    path = (os.getcwd())
    #time.sleep(.02)
    video_output = (os.path.join(path, camera_1_folder, image_folder_name(), video_out_1))
    #video_output = (os.path.join(path, image_folder, image_name))
    return video_output, get_image_time

#net = jetson.inference.imageNet('alexnet')
net = jetson.inference.imageNet('googlenet-12')
#net_detect = jetson.inference.detectNet("ssd-mobilenet-v2", threshold=0.4) ## default is 0.5 higher number less sensitive
net_detect = jetson.inference.detectNet('ssd-mobilenet-v2',['--model=/jetson-inference/python/training/detection/ssd/models/{}/ssd-mobilenet.onnx'.format(MODEL),'--labels=/jetson-inference/python/training/detection/ssd/models/{}/labels.txt'.format(MODEL),'--input-blob=input_0','--output-cvg=scores','--output-bbox=boxes'])
LABELS_FILE = '/jetson-inference/python/training/detection/ssd/models/{}/labels.txt'.format(MODEL)
#LABELS_FILE = '/ssd500/jetson-inference/data/networks/SSD-Mobilenet-v2/ssd_coco_labels.txt'
#net_detect = jetson.inference.detectNet("ssd-inception-v2", threshold=0.5)
#net = jetson.inference.imageNet('resnet-152',['--model=/home/rdl-xavier/jetson-inference/python/training/classification/MyTrainedCars/resnet18.onnx','--input_blob=input_0','--output_blob=output_0','--labels=/home/rdl-xavier/jetson-inference/my_models/MyTrainedCars/labels.txt'])
#net = jetson.inference.imageNet('googlenet',['--model=~/jetson-inference/python/training/classification/people/resnet18.onnx','--input_blob=input_0','--output_blob=output_0','--lables=~/jetson-inference/people/labels.txt'])
#net = jetson.inference.imageNet('googlenet')
#net = jetson.inference.imageNet('resnet-18') ## 18, 50, 101, and 152
#net = jetson.inference.imageNet('vgg-19')  ## 16, 19
#net = jetson.inference.imageNet('inception-v4')

VIDEO_DIRECTORY = '/media/bob/my_ext_ssd' #'/'

DIR_LIST = os.listdir(VIDEO_DIRECTORY)
os.system('clear')
print('AI AUDITOR')

#    print('DATA-SET LIST: ',DIR_LIST)
print()
print('File List')
for line in range(len(DIR_LIST)):
        print('{} {} {}'.format(line, '-'*50, DIR_LIST[line]))

print()
print('Select video from the file list above: ')
#print('1 ---------------------------- Enter search date ')
#print('2 ---------------------------------- Select file ')
print()
VIDEO_FILE = int(input('Select number for VIDEO from the File List above: '))
VIDEO_FILE = '{}/{}'.format(VIDEO_DIRECTORY, DIR_LIST[VIDEO_FILE])
print('video file: ', VIDEO_FILE)

#v_out = getVideo()
#input = jetson.utils.videoSource('csi://0')
#input = jetson.utils.videoSource('rtsp://root:TTItest1@10.4.0.190:554/axis-media/media.amp?videocodec=h264')
#input = jetson.utils.videoSource('rtsp://root:AVCaudit1@10.4.0.187:554/axis-media/media.amp?videocodec=h264')
#input = jetson.utils.videoSource('/dev/video0') ## USB camera
#input = jetson.utils.videoSource('/media/robert/ssd128/axis_2021-03-10_08_59_44_325.mp4') #/ssd500/cars17.mp4') #'/dev/video1')
input = jetson.utils.videoSource(VIDEO_FILE)

def speedTime():
    speed_time = datetime.datetime.now()
    speed_time = speed_time.strftime("%f")
    #print('speed time: ', speed_time)
    return float(speed_time)

def newVideo():
    v_out = getVideo()
    #output = jetson.utils.videoOutput(v_out) ##is_headless
    return v_out

font = jetson.utils.cudaFont()
def collectData():
    location = ' '
    count = 0
    #capture the next image
    img = input.Capture()
    #detections = net.Detect(img, overlay=opt.overlay) overlay options (--overlay=box,labels,conf, and overlay=none)
    #detections  = net_detect.Detect(img, overlay='none')
    detections  = net_detect.Detect(img)
    #print("detected {:d} objects in image".format(len(detections)))
    objects = (len(detections))
    objects = int(objects)
    detection = detections
    #perameter = jetson.inference.detectNet.Detection()
    #print (perameter)
    #print (objects)
    for detection in detections:
        #print('detections: ',detection)
        #net_detect.PrintProfilerTimes()
        class_id_detectNet = int(detection.ClassID)
        Confidence = detection.Confidence
        Left = int(detection.Left)
        Top = int(detection.Top)
        Right = int(detection.Right)
        Bottom = int(detection.Bottom)
        Width = int(detection.Width)
        Height = int(detection.Height)
        Area = int(detection.Area)
        center_x, center_y = detection.Center
        center_x = int(center_x)
        center_y = int(center_y)
        #if objects > 0:
        if class_id_detectNet > 0:
            class_id, confidence = net.Classify(img)
            # find the object description
            class_desc = net.GetClassDesc(class_id)

            #overlay the result on the image
            confidence = int(confidence * 100)
            Confidence = int(Confidence * 100)
            class_id = int(class_id)
            w  = Width
            h = Height
            # In odrer to get the positions of the upper left corner
            x = int(center_x - w/2)
            y = int(center_y - h/2)
            #print ('x_position:{} y_position:{}'.format(x, y))

            #Set trigger x or y trigger position here
            if 100 < x < 300:
                MESSAGE = 'START_ENTRY'
                print(MESSAGE)
            if 400 < x < 600:
                MESSAGE = 'STOP_ENTRY'
                print(MESSAGE)
            if 600 < x < 800:
                MESSAGE = 'START_EXIT'
                print(MESSAGE)
            if 800 < x < 1000:
                MESSAGE = 'STOP_EXIT'
                print(MESSAGE)

                if class_desc == FILTER1:
                    if confidence >= 1:
                        t1 = time.time()
                        #t1 = speedTime()
                        v_out, time_stamp = getVideo()
                        font.OverlayText(img, img.width, img.height, "Time:{} |{:05.2f}% {:s}".format(time_stamp, confidence, class_desc, net.GetNetworkFPS()), 5, 5, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Top:{}".format(Top), 5, 45, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Left:{}".format(Left), 5, 85, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Right:{}".format(Right), 5, 120, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Bottom:{}".format(Bottom), 5, 160, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Height:{}".format(Height), 5, 195, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Width:{}".format(Width), 5, 235, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Area:{}".format(Area), 5, 270, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Center_X:{}".format(x), 5, 305, font.White, font.Gray40)
                        font.OverlayText(img, img.width, img.height, "Center_Y:{}".format(y), 5, 345, font.White, font.Gray40)
                        jetson.utils.saveImageRGBA(v_out, img, width, height)
                        #time.sleep(0.0050)

                    if 700 < x < 900:
                        t2 = time.time()
                        #t2 = speedTime()
                        t1 = float(t1)
                        t2 = float(t2)
                        t_delta = t2 - t1
                        #print('time delta: ', t_delta)
                        #CALCULATE SPEED
                        #SPEED = D/T
                        distance = float(300)
                        #speed = int(distance/t_delta) #meters per second
                        speed = float((distance/t_delta)*2.236936) ##mph
                        #print('speed-time delta: ', t_delta)
                    else:
                        speed = 'UNKOWN'
                    os.chdir(STORAGE_DIRECTORY)
                    path = (os.getcwd())

                    object_names = open(LABELS_FILE)
                    #for line in enumerate(pvr):
                    object_name = object_names.readlines()
                    object_name = object_name[class_id_detectNet]
                    #print('Object name: ',object_name)

                    location = 'TP2' #GPS
                    description =  class_desc
                    description_id = object_name
                    #confidence = int(confidence * 100)
                    color = 0          #vehicle color
                    axle_count = MESSAGE #objects ##
                    license_plate = 0 #######
                    rfid = 0 ################
                    image_path = v_out
                    lane_id = 0
                    speed = 0
                    #INSERT INTO DATABASE
                    data =  (time_stamp, location, description, description_id, Confidence, color, axle_count, license_plate, rfid, speed, lane_id, image_path)
                    conn.execute('INSERT INTO DATA_AQUISITION2 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', data)
                    conn.commit()
                    #class_id_detectNet = 0
                    #return description  #testing only

#SELECT DATA
def selectData():
    #SELECT Operation
    #Following Python program shows how we can fetch and display records from our DATA_AQUISITION table
    #created in above example:
    #!/usr/bin/python
    #import sqlite3
    conn = sqlite3.connect('Dbox_3.db')
    #print ("Opened database successfully")
    cursor = conn.execute('SELECT TIMESTAMP, DESCRIPTION, DESCRIPTION_ID, CONFIDENCE from DATA_AQUISITION2')
    for row in cursor:
        print ("Time stamp: ", row[0])
        print ("Description: ", row[1])
        take_action = row[1]
        print ("Description_ID: ", row[2])
        print ("Confidence: ", row[3], "\n")

def runInference():
    collectData()

while True:
    runInference()
    #time.sleep(2)