"""
This module contains methods that analyse the recorded
habits and their trackings.
It also contains methods that have a connection to
the database.
"""
import sqlite3
from datetime import datetime
from itertools import groupby
from functools import reduce
from collections import Counter
from operator import itemgetter

from .habit import Habit

class Analytics:
    """Initialize and manipulate SQLite3 database"""

    def __init__(self):
        """
        Connect to the database
        or create it if does not exist.
        """
        try:
            # Connect to database
            self.connection = sqlite3.connect('DB_Habitsbox_app.db')
            # Create a cursor
            self.cursor = self.connection.cursor()
            # Create table habits with six columns if does not exist
            self.cursor.execute("""CREATE TABLE IF NOT EXISTS habits(
                             HabitID INTEGER PRIMARY KEY,
                             Name TEXT UNIQUE,
                             Periodicity TEXT,
                             Motivation TEXT,
                             Description TEXT,
                             Creation_date TEXT
                          )""")
            # Create table trackings with four columns if does not exist
            self.cursor.execute("""CREATE TABLE IF NOT EXISTS trackings(
                             TrackingID INTEGER PRIMARY KEY,
                             Date TEXT,
                             Time TEXT,
                             HabitID INTEGER NOT NULL,
                             FOREIGN KEY (HabitID) REFERENCES habits (HabitID)
                          )""")
        # print error if any error occurs
        except sqlite3.Error as error:
            print('Error while connectiong to SQLite', error)

    def insert_habit(self, name, periodicity, motivation, description):
        """
        Initialize the Habit class in the habit module and
        insert a habit to the DB.

        Parameters
        ----------
        name : str
            Name of the habit
        periodicity : str
            Periodicity of the habit, daily or weekly
        motivation :
            Motivation of the user to perform the habit
        description :
            Description of the habit
        """
        self.habit = Habit(name, periodicity, motivation, description)

        with self.connection:
            self.cursor.execute("""
                            INSERT INTO habits (Name,
                                                Periodicity,
                                                Motivation,
                                                Description,
                                                Creation_date)
                            VALUES (:name,
                                    :periodicity,
                                    :motivation,
                                    :description,
                                    :creation_date)""",
                            {'name': self.habit.name,
                            'periodicity':self.habit.periodicity,
                            'motivation':self.habit.motivation,
                            'description':self.habit.description,
                            'creation_date':self.habit.creation_date})

    def habits_table(self):
        """
        Return all the habits with its information contained in
        the table habits in the DB. ID and name of the habit,
        periodicity, motivation, description, and the day it was recorded.

        Returns
        -------
        list
            a list of tuples with the information of the registered habits

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Before lunch', '2021-04-26'),
         (2, 'Run', 'weekly', 'Being healthier', 'At weekends', '2021-04-26')]
        """
        self.cursor.execute("SELECT * FROM habits")
        return self.cursor.fetchall()

    def trackings_table(self):
        """
        Return the date and time when the trackings were
        registered, as well as to which habit-id they correspond.

        Returns
        -------
        list
            a list of tuples with the habit id, date and time

        Example
        -------
        [(1, '2021-02-01', '19:52'), (2, '2021-02-03', '02:42')]
        """
        self.cursor.execute("""SELECT HabitID, Date, Time
                            FROM trackings""")
        return self.cursor.fetchall()

    def habits_trackings_table(self):
        """
        Join the habits table and the trackings table using the
        id of the habit.

        Returns
        -------
        list
            a list of tuples of the join of the habits and trackings table

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-06-21', '09:06'),
         (3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-06-21', '15:26')]
        """
        self.cursor.execute("""SELECT h.HabitID, h.Name, h.Periodicity,
                            h.Motivation, h.Description, t.Date, t.Time
                            FROM habits h
                            JOIN trackings t
                            USING(HabitID)""")
        return self.cursor.fetchall()

    def select_column(self, table, number_column):
        """
        Select a column of the indicated table.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        number_column : int
            Index number, i.e. column

        Returns
        -------
        list
            a list with the selected column
        """
        return list(map(lambda x: x[number_column], table))

    def select_columns(self, table, start=0, stop=1):
        """
        Select several columns of the indicated table.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        start : int, optinal
            Number of the first index (inclusive), i.e. column
            (default is 0)
        stop : int, optional
            Number of the last index (exclusive), i.e. column
            (default is 1)

        Returns
        -------
        list
            a list of tuples with selected columns
        """
        return list(map(lambda x: x[start:stop], table))

    def get_all_ids(self, table):
        """
        First column of a table in a list.

        Parameters
        ----------
        table : list of tuples
            Name of the table

        Returns
        -------
        list
            a list of numbers representing habit ids
        """
        return list(self.select_column(table, 0))

    def unique_ids(self, table):
        """
        Unique ids of a table

        Parameters
        ----------
        table: list of tuples
            Name of the table

        Returns
        -------
        list
            a list of numbers representing unique ids
        """
        return self.unique_data(
            self.get_all_ids(
                table))

    def ids_without_trackings(self, habits_table, habits_trackings_table):
        """
        All habit ids that have no trackings.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list of numbers representing habit ids without
            trackings
        """
        return list(
                set(self.unique_ids(habits_table))-set(self.unique_ids(habits_trackings_table))
                )

    def ids_with_trackings(self, habits_table, ids_without_trackings):
        """
        All habit ids that have trackings.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        ids_without_trackings : list
            a list of habit ids without trackings

        Returns
        -------
        list
            a list of numbers representing habit ids trackings
        """
        return list(set(self.unique_ids(habits_table))-set(ids_without_trackings))

    def select_rows(self, table, number_column, feature):
        """
        Select all rows with the same feature.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        number_column : int
            Index number, column where the criterion
            will be searched
        feature : int, str
            the criteria used for selection

        Returns
        -------
        list
            a list of tuples with all rows that
            satisfy the condition
        """
        return list(filter(lambda x: x[number_column]==feature, table))

    def display_elements(self, my_list, separator='\n'):
        """
        Each item in the list is separated by a separator.

        Parameters
        ----------
        my_list : list
            List of elements to be separated
        separator : str, optional
            (default is '\n')

        Returns
        -------
        str
            a string with elements separated by the separator
        """
        return separator.join(map(str, my_list))

    def unique_data(self, my_list):
        """
        Give unique values from a list of data.

        Parameters
        ----------
        my_list : list
            A list that contains duplicate elements

        Returns
        -------
        list
            a list of unique elements
        """
        return reduce(lambda x, y: x + [y] if y not in x else x, my_list, [])

    def display_unique_elements(self, table, column):
        """
        Unique elements of a column are
        separated by commas in a string.

        Parameters
        ----------
        table : list of tuples
            Name of the table
        column : int
            Index number, i.e. column

        Returns
        -------
        str
            Elements separated by comma
        """
        return self.display_elements(
                self.unique_data(
                    self.select_column(
                        table,
                        column)), ', ')

    def format_to_date(self, column):
        """
        Items are converted to datetime.date.

        Parameters
        ----------
        column : list of tuples
            a list of strings containing the dates to
            be converted

        Returns
        -------
        iterator
            dates in datetime.date
        """
        return map(lambda x: datetime.strptime(x, "%Y-%m-%d").date(), column)

    def format_to_time(self, column):
        """
        Items are converted to datetime.time.

        Parameters
        ----------
        column : list of tuples
            a list of strings containing the times to
            be converted

        Returns
        -------
        iterator
            times in datetime.time
        """
        return map(lambda x: datetime.strptime(x, "%H:%M").time(), column)

    def to_calender_week(self, dates):
        """
        Give the calendar week number for each date.

        Parameters
        ----------
        dates : list
            a list of dates

        Returns
        -------
        iterator
            calendar week numbers
        """
        return map(lambda x: x.isocalendar()[1], dates)

    def zipping_unique_data(self, data):
        """
        A list is sliced into one list without the first element
        and into another list without the last element.
        The first elements of each list are matched in a tuple,
        then the second and so on.

        Parameters
        ----------
        data : list
            a list of dates or numbers

        Returns
        -------
        iterator
            pairs of numbers or dates

        Example
        -------
        [52, 1, 2, 3, 5] has the following result

        [(1, 52), (2, 1), (3, 2), (5, 3)]
        """
        return zip(data[1:], data[:-1])

    def differences(self, pairs):
        """
        Differences between numbers or dates.

        Parameters
        ----------
        pairs : list of tuples
            pairs of numbers or dates

        Returns
        -------
        iterator
            substract of the second element from the first

        Example
        -------
        [(1, 52), (2, 1), (5, 3)]

        results in [-51, 1, 2]
        """
        return map(lambda x: (x[0]-x[1]), pairs)

    def difference_in_days(self, differences):
        """
        Number of days from a datetime.timedelta object.

        Parameters
        ----------
        differences : list
            a list of datetime.timedelta elements

        Returns
        -------
        iterator
            number of days

        Example
        -------
        [timedelta(days=2), timedelta(days=1)]

        results in [2, 1]
        """
        return map(lambda x: x.days, differences)

    def dif_5152_to_1(self, differences):
        """
        Change the number to 1 if the number in the list
        is -51 o -52.

        Parameters
        ----------
        differences : list
            a list of numbers

        Returns
        -------
        iterator
            numbers representing days
        """
        return map(lambda x: 1
                   if x in (-51, -52)
                   else x, differences)

    def grouping_differences(self, differences):
        """
        Same numbers are grouped and designed to a key.
        A new group is formed each time the number(key)
        changes.

        Parameters
        ----------
        differences : list
            a list of numbers

        Returns
        -------
        sub-iterators
            keys and groups from the list

        Example
        -------
        [1, 1, 1, 2, 1, 1, 2, 2] is grouped into

        [[1, [1, 1, 1]], [2, [2]], [1, [1, 1]], [2, [2, 2]]]
        """
        return groupby(differences)

    def streaks(self, grouping_differences):
        """
        Count the number of items contained in the group
        if the key is the number one.

        Parameters
        ----------
        grouping_differences : list
            a list of keys and groups

        Returns
        -------
        list
            a list of numbers

        Example
        -------
        [[1, [1, 1, 1]], [2, [2]], [1, [1, 1]], [2, [2, 2]]]

        results in [3, 2]
        """
        return [sum(group) for key, group in grouping_differences if key == 1]

    def longest_streak(self, streaks):
        """
        Choose the maximum number and add 1 to it.
        If the list is empty, returns the number one.

        Parameters
        ----------
        streaks : list
            a list of numbers

        Returns
        -------
        int
            a number representing the longest streak
        """
        if len(streaks) != 0:
            return max(streaks) + 1
        return 1

    def longest_streak_periodicity(self, habits_trackings_table, periodicity, col_date=5):
        """
        The longest streak of a habit depending on
        the habit's periodicity.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        col_date : int, optional
            index number i.e. column containing the dates
            (default is number 5)

        Returns
        -------
        int
            a number representing the longest streak of a
            habit
        """
        if len(habits_trackings_table) == 0:
            return []
        if periodicity == 'daily':
            return self.longest_streak(
                self.streaks(
                    self.grouping_differences(
                        self.difference_in_days(
                            self.differences(
                                self.zipping_unique_data(
                                    self.unique_data(
                                        self.format_to_date(
                                            self.select_column(
                                                habits_trackings_table,
                                                col_date
                                                )))))))))
        return self.longest_streak(
            self.streaks(
                self.grouping_differences(
                    self.dif_5152_to_1(
                        self.differences(
                            self.zipping_unique_data(
                                self.unique_data(
                                    self.to_calender_week(
                                        self.format_to_date(
                                            self.select_column(
                                                habits_trackings_table,
                                                col_date
                                                ))))))))))

    def start_habit(self, habits_trackings_table, col_date = 5):
        """
        The earliest date from a column of dates.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        datetime.date
            The earliest date
        """
        if len(habits_trackings_table) == 0:
            return []
        return min(
            self.format_to_date(
                self.select_column(
                    habits_trackings_table,
                    col_date
                    )))

    def last_day(self, habits_trackings_table, col_date = 5):
        """
        The most recent date in a column of dates.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        datetime.date
            The most recent date
        """
        if len(habits_trackings_table) == 0:
            return []
        return max(
            self.format_to_date(
                self.select_column(
                    habits_trackings_table,
                    col_date
                    )))

    def activity(self, periodicity, trackings, col_date=5):
        """
        Return the number of days or weeks in which the habit
        has been checked off, depending on whether the habit
        is weekly or daily.

        Parameters
        ----------
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        trackings : list of tuples
            a list with the information of the trackings
            registered for a certain habit
        col_date : int, optional
            index number i.e. column containing the dates
            (default is number 5)

        Returns
        -------
        int
            number of days or weeks in which the habit
            has been checked off
        """
        date_column = self.format_to_date(
            self.select_column(
                trackings, col_date
                )
            )

        if periodicity == 'daily':
            return len(self.unique_data(date_column))
        return len(self.unique_data(self.to_calender_week(date_column)))

    def only_hours(self, time):
        """
        Get hours without minutes

        Parameters
        ----------
        time : list
            a list of times

        Returns
        -------
        iterator
            numbers representing the hours
        """
        return map(lambda x: x.hour, time)

    def sort_hours(self, hours):
        """
        Each hour is renamed according to the part of the
        day it corresponds to (Morning, Afternoon, Evening
        or Overnight).

        Parameters
        ----------
        hours : list
            a list of numbers representing the hours

        Returns
        -------
        iterator
            parts of the day in strings

        Example
        -------
        [3, 18, 19, 2] results in

        ['Overnight', 'Evening', 'Evening', 'Overnight']
        """
        return map(lambda x: 'Morning'
                   if 5 <= x < 12
                   else ('Afternoon'
                         if 12 <= x < 18
                         else ('Evening'
                               if 18 <= x < 24
                               else 'Overnight')), hours)

    def count_sorted_hours(self, sort_hours):
        """
        Count the number of times the words of the parts
        of the day appear.

        Parameters
        ----------
        sort_hours : list
            a list of strings representing different parts
            of the day

        Returns
        -------
        counter
             a dictionary whose keys are the parts of the day and
             values are the frequencies with which they appear in
             the list

        Example
        -------
        ['Overnight', 'Evening', 'Evening', 'Overnight', 'Evening']

        results in {'Evening': 3, 'Overnight': 2}
        """
        return Counter(sort_hours)

    def active_time_dict(self, trackings, col_time=6):
        """
        A dictionary whose keys are the parts of the day
        (Morning, Afternoon, Evening and Overnight) in which a
        habit was checked and whose values indicate how often
        the activity is performed in these parts of the day.

        Parameters
        ----------
        trackings : list of tuples
            a list with the information of the trackings
            registered for a certain habit
        col_time : int, optional
            index number i.e. column containing the times
            (default is number 6)

        Returns
        -------
        counter
            a dictionary whose keys are the parts of the day and
            values are the frequencies with which the habit is
            performed
        """
        return self.count_sorted_hours(
            self.sort_hours(
                self.only_hours(
                    self.format_to_time(
                        self.select_column(
                            trackings, col_time)))))

    def max_value(self, active_time_dict):
        """
        The highest value from a dictionary.

        Parameters
        ----------
        active_time_dict : dict
            a dictionary whose keys are the parts of the day
            and values are the frequencies with which the
            habit is performed

        Returns
        -------
        int
            a number which is the highest value
        """
        if len(active_time_dict) == 0:
            return {}
        return max(active_time_dict.values())

    def most_active_time(self, active_time_dict, max_value):
        """
        Get the dictionary key(s) with the maximum value.

        Parameters
        ----------
        active_time_dict : dict
            a dictionary whose keys are the parts of the day
            and values are the frequencies with which the
            habit is performed
        max_value : int
            a number which is the highest value of the
            dictionary

        Returns
        -------
        list
            a list of string(s) representing the part(s) of
            the day when a habit is most frequently checked off

        Example
        -------
        {'Morning':4, 'Evening': 3, 'Overnight': 2}

        results in ['Morning']
        """
        if len(active_time_dict) == 0:
            return {}
        return [k for k, v in active_time_dict.items() if v == max_value]

    def unique_ids_periodicity(self, table, col_periodicity, periodicity):
        """
        Get the ids of the habits with the same periodicity.

        Parameters
        ----------
        table : list of tuples
            name of the table
        col_periodicity : int
            index number i.e. column containing the periodicity
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'

        Returns
        -------
        list
            a list of numbers representing ids
        """
        return self.unique_ids(
            self.select_rows(table, col_periodicity, periodicity))

    def grouped_habits(self,table, unique_ids):
        """
        Habits grouped by id.

        Parameters
        ----------
        table : list of tuples
            name of the table
        unique_ids : list
            a list of ids of the habits with the same periodicity

        Returns
        -------
        list
            Habits are grouped by id
        """
        return [self.select_rows(table, 0, id_n) for id_n in unique_ids]

    def lists_periodicity(self, table, col_periodicity, periodicity):
        """
        Information and trackings of the habits with the same
        periodicity are grouped together by id.

        Parameters
        ----------
        table : list of tuples
            name of the table
        col_periodicity : int
            index number i.e. column containing the periodicity
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'

        Returns
        -------
        list
            a list of lists. Each list has information and
            trackings for each habit with the same periodicity

        Example
        -------
        For the daily periodicity in the join of the habits and trackings
        table gives the following
        [[(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-07-31', '16:41'),
          (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-02', '18:44'),
          (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-03', '13:43')],
         [(3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-08-03', '15:44')]]
        """
        return self.grouped_habits(table,
                                   self.unique_ids_periodicity(table,
                                                               col_periodicity,
                                                               periodicity))

    def periodicity_info(self, lists_periodicity, periodicity, col_date=5):
        """
        A list containing information on each habit according
        to its periodicity. ID and name of the habit, date of
        the first and last tracking, part of the day when the
        user is most active to perform this habit, days or weeks
        of activity, and the longest streak.

        Parameters
        ----------
        lists_periodicity : list
            a list of lists of habits with the same periodicity.
            Each list has information of the habit and its trackings
        periodicity : str
            periodicity of the habit, 'weekly' or 'daily'
        col_date : int, optional
            index number i.e. column containing dates
            (default is number 5)

        Returns
        -------
        list
            a list of tuples containing information on each habit

        Example
        -------
        [(1, 'Yoga', '2021-03-18', '2021-04-17', 'Evening', 5, 2),
         (2, 'Reading', '2021-03-19', '2021-04-25', 'Afternoon', 8, 2)]
        """
        habits_info = []

        for habit_info in lists_periodicity:

            active_time_dictionary = self.active_time_dict(
                                habit_info,
                                6)
            max_value_active_time = self.max_value(
                                active_time_dictionary)
            most_active_time = self.most_active_time(
                                active_time_dictionary,
                                max_value_active_time)

            habits_info.append(
                (
                    self.display_unique_elements(habit_info, 0),
                    self.display_unique_elements(habit_info, 1),
                    ''.join(min(self.select_column(habit_info, col_date))),
                    ''.join(max(self.select_column(habit_info, col_date))),
                    self.display_elements(most_active_time, ', '),
                    self.activity(
                        periodicity,
                        habit_info,
                        col_date),
                    self.longest_streak_periodicity(
                        habit_info,
                        periodicity,
                        col_date))
                )

        return habits_info

    def lists_both_periodicities(self, habits_trackings_table):
        """
        Habits are separated according to periodicity.
        Information and trackings of each habit are grouped together.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            list of lists. Information and trackings for each habit
            are grouped together, and habits with the same periodicity
            are grouped are separated.

        Example
        -------
        [[[(1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-07-31', '16:41'),
           (1, 'Yoga', 'daily', 'Be more flexible', 'Mornings', '2021-08-02', '18:44')],
          [(3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-08-03', '15:44')]],
         [[(2, 'Run', 'weekly', 'Be faster', 'At weekends', '2021-07-31', '16:46'),
           (2, 'Run', 'weekly', 'Be faster', 'At weekends', '2021-08-03', '14:36')]]]
        """
        return list(map(self.lists_periodicity, *zip((habits_trackings_table, 2, 'daily'),
                                             (habits_trackings_table, 2, 'weekly'))))

    def both_periodicities_info(self, lists_both_periodicities):
        """
        Habits separated by periodicity with general information
        about each habit. ID and Name of the habit, date of the
        first and last tracking, part of the day when the user
        is most active to perform this habit, days or weeks of
        activity, and the longest streak.

        Parameters
        ----------
        lists_both_periodicities : list
            Habits are separated according to periodicity. Information
            and trackings of each habit are grouped together.

        Returns
        -------
        list
            list of lists with general information about each habit

        Example
        -------
        [[('1', 'Yoga', '2021-07-31', '2021-08-03', 'Afternoon', 3, 2),
          ('3', 'Read', '2021-08-03', '2021-08-03', 'Afternoon', 1, 1)],
         [('2', 'Run', '2021-07-31', '2021-08-03', 'Afternoon', 2, 2)]]
        """
        return list(map(self.periodicity_info, *zip((lists_both_periodicities[0], 'daily'),
                                              (lists_both_periodicities[1], 'weekly'))))

    def info_all_habits(self, both_periodicities_info):
        """
        Information on the habits separated by periodicity
        is merged into a single list.

        Parameters
        ----------
        both_periodicities_info : list
            list of lists with general information about each habit

        Returns
        -------
        list
            a list with general information about each habit

        Example
        -------
        [('1', 'Yoga', '2021-07-31', '2021-08-03', 'Afternoon', 3, 2),
         ('3', 'Read', '2021-08-03', '2021-08-03', 'Afternoon', 1, 1)]
        """
        return [habit for l in both_periodicities_info for habit in l]

    def tracked_habits(self, habits_table, habits_trackings_table):
        """
        Habits with trackings and their information from
        the habits table of the DB.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list of tuples of tracked habit information
        """
        return self.info_all_habits(
            self.grouped_habits(habits_table,
                                self.unique_ids(habits_trackings_table))
            )

    def habits_without_trackings(self, habits_table, ids_without_trackings):
        """
        Untracked habits and their information from the habits table
        of the DB.

        Parameters
        ----------
        habits_table : list of tuples
            a list with the information of the registered habits
        ids_without_trackings : list
            habit ids that have no trackings

        Returns
        -------
        list
            a list of tuples of untracked habits and their information
        """
        return [self.select_rows(habits_table, 0, id_n)[0]
                for id_n in ids_without_trackings]

    def habit_info_longest_streak(self, habits_trackings_table):
        """
        Information of the habit(s) that has(have) the
        maximum streak.

        Parameters
        ----------
        habits_trackings_table : list of tuples
            a list with the information of the habits and
            its trackings

        Returns
        -------
        list
            a list with general information of the habit(s)
            that has(have) the maximum streak.

        Example
        -------
        [('1', 'Yoga', '2021-06-26', '2021-07-23', 'Morning', 17, 9)]
        """
        all_periodicity_habits = self.info_all_habits(
            self.both_periodicities_info(
                self.lists_both_periodicities(
                    habits_trackings_table)))

        if len(all_periodicity_habits) > 1:
            # get the number of the maximum streak
            max_n = max(all_periodicity_habits, key=itemgetter(-1))[-1]
            # get all habits which have the same number of the maximum streak
            return self.select_rows(all_periodicity_habits, -1, max_n)
        return all_periodicity_habits

    def name_habit_longest_streak(self, habit_info_longest_streak):
        """
        Give the name(s) of the habit(s) with the longest streak
        and the streak.

        Parameters
        ----------
        habit_info_longest_streak : list
             a list with general information of the habit(s)
             that has(have) the maximum streak.

        Returns
        -------
        list
            a list with the name and longest streak of
            habit(s) with the longest streak

        Example
        -------
        [('Yoga', 2), ('Read', 2), ('Run', 2)]
        """
        return list(zip(self.select_column(habit_info_longest_streak, 1),
                        self.select_column(habit_info_longest_streak, -1)))

    def lengths(self, habits_info):
        """
        Length of each element

        Parameters
        ----------
        habits_info : list
             a list of tuples with general information
             about each habit

        Returns
        -------
        list
            a list of lists of numbers representing lenghts

        Example
        -------
        [(1, 'Yoga', 'daily', 'Be more flexible', 'Preferably in the morning', '2021-07-20'),
         (3, 'Read', 'daily', 'Read 12 books a year', 'Afternoons', '2021-07-21')]

        Results in
        [[1, 4, 5, 16, 25, 10],
         [1, 4, 5, 20, 10, 10]]
        """
        return [[len(str(x)) for x in row] for row in habits_info]

    def max_lengths(self, lengths, table):
        """
        The longest lengths

        Parameters
        ----------
        lengths : list
             a list of lists of numbers representing lenghts
        table : list of tuples
             name of the table

        Returns
        -------
        list
            a list with the longest lengths

        Example
        -------
        [[1, 4, 5, 16, 25, 10], [1, 4, 5, 20, 10, 10], [1, 3, 6, 9, 11, 10]]

        Results in [1, 4, 6, 20, 25, 10].
        """
        return list(max(map(itemgetter(x), lengths)) for x in range(0, len(table[0])))

    def distance_format(self, max_lengths):
        """
        Create a string formating that represents
        the distance to be maintained between each
        element.

        Parameters
        ----------
        max_lengths : list
             a list with the longest lengths

        Returns
        -------
        str
            a string formating

        Example
        -------
        if max_lengths = [1, 4, 6, 20, 25, 10]

        "%-1s    %-4s    %-6s    %-20s    %-25s    %-10s    "
        """
        return ''.join(map(lambda x: '%%-%ss    ' % x, max_lengths))

    def aligned_columns(self, format_distance, table):
        """
        Elements of a table are separated and
        aligned.

        Parameters
        ----------
        format_distance : str
             a string formating that represents the distance
             to be maintained between each element
        table : list of tuples
             name of the table

        Returns
        -------
        iterator
            strings whose elements are aligned with
            the given distance
        """
        return map(lambda x: format_distance % x, table)

    def line(self, max_lengths):
        """
        Create a line by adding the elements of a list
        of numbers (maximum lengths) and the spaces between
        the elements.

        Parameters
        ----------
        max_lengths : list
             a list with the longest lengths

        Returns
        -------
        str:
            a line with the given length
        """
        return '_' * (sum(max_lengths) + len(max_lengths) * 4)

    def add_colnames(self, col_names, table):
        """
        Add the names of each column.

        Parameters
        ----------
        col_names : str
            contains the names of each column
        table : list of tuples
             name of the table

        Returns
        -------
        list
            a list of tuples
        """
        return [col_names]+table

    def table_line(self, table):
        """
        Adjust the length of the line in relation to
        the data in the table.

        Parameters
        ----------
        table : list of tuples
             name of the table

        Returns
        -------
        str
            a line with the right length
        """
        return self.line(
            self.max_lengths(
                self.lengths(table), table))

    def display_table(self, col_names, table, header):
        """
        Display the data in a table.

        Parameters
        ----------
        col_names : str
            contains the names of each column
        table : list of tuples
            name of the table
        header : list of tuples
            title of the table
        """
        data_colnames = self.add_colnames(col_names, table)

        print(self.table_line(data_colnames))
        print("{}".format(header))
        print(self.table_line(data_colnames))
        print(self.display_elements(
            self.aligned_columns(
                self.distance_format(
                    self.max_lengths(
                        self.lengths(data_colnames),
                        data_colnames)),
                data_colnames)))
        print(self.table_line(data_colnames))

    def remove_habit(self, name):
        """
        The habit is removed from the DB in the habit table along
        with all trackings related to its id.

        Parameters
        ----------
        name : str
            name of the habit
        """
        with self.connection:
            self.cursor.execute("SELECT HabitID FROM habits WHERE Name=:name",
                       {'name': name})
            id_habit = self.cursor.fetchone()[0]
            self.cursor.execute("DELETE from trackings WHERE HabitID = :habitID",
                             {'habitID': id_habit})
            self.cursor.execute("DELETE from habits WHERE HabitID = :habitID",
                             {'habitID': id_habit})

    def insert_day(self, id_habit):
        """
        Register in the trackings table the day and time
        when the user checks a habit off, by choosing
        the id of the habit.

        Parameters
        ----------
        id_habit : int
            id of the habit
        """
        date = datetime.now().strftime("%Y-%m-%d")
        time = datetime.now().strftime("%H:%M")

        with self.connection:
            self.cursor.execute("""INSERT INTO trackings (Date, Time, HabitID)
                                VALUES (:date, :time, :habitID)""",
                            {'date': date, 'time': time, 'habitID': id_habit})

    def close(self):
        """
        Close SQLite3 connection
        """
        self.connection.close()


from django.shortcuts import render
from phantom.models import Advertisement, ContactData
from django.views.generic import ListView, DetailView, FormView
from django.http import HttpResponse
from django.urls import reverse_lazy

# def get_form_data(request):
#     if request.method == 'POST':
#         print("ok")
#     return render(request, 'phantom/success.html', {})

class IndexPageView(FormView, ListView):
    template_name = 'phantom/index.html'
    model = Advertisement
    form_class = ContactDataForm
    context_object_name = 'advertisements'

    def get_queryset(self, *args, **kwargs):
        return Advertisement.objects.filter(active=True)

class IndexDetailView(DetailView):
    template_name = 'phantom/generic.html'
    model = Advertisement
    context_object_name = 'advertisement'
    form_class = ContactDataForm


# -*- coding: utf-8 -*-
"""
Created on Mon Dec  7 12:16:20 2015

@author: mittelberger
"""

import os
import numpy as np
from scipy.interpolate import interp1d
from ScanMap.find_positions import Positionfinder

class Positioncollector(object):

    positionsfilepath = '/home/mittelberger2/git/ScanMap/positioncollection.npz'

    def __init__(self, **kwargs):
        self.paths = kwargs.get('paths')
        if os.path.isfile(Positioncollector.positionsfilepath):
            self.positionsfile = np.load(Positioncollector.positionsfilepath)
        else:
            self.positionsfile = None

        if self.positionsfile is None:
            # the structure is always: (y,x) on the first axis, datasets on the second axis (line or column, depending
            # on if it is x- or y-data) and the actual deviations from the ideal positions on the third axis.
            self.firstlines = np.empty((2, 0, 101))
            self.evenlines = np.empty((2, 0, 101))
            self.oddlines = np.empty((2, 0, 101))
            # mapnames contain name and pixelsize of the maps
            self.mapnames = (np.empty(0)).astype(np.dtype([('mapname', 'U256'), ('pixelsize', 'f32'),
                                                           ('size_overview', 'f32'), ('size_frames', 'f32'),
                                                           ('number_frames', '2f32')]))
        else:
            self.firstlines = self.positionsfile['firstlines']
            self.evenlines = self.positionsfile['evenlines']
            self.oddlines = self.positionsfile['oddlines']
            self.mapnames = self.positionsfile['mapnames']

    def get_data(self, folder):
        Finder = Positionfinder(framepath=folder)
        Finder.load_data()
        bad_data = False

        if Finder.optimized_positions is None or Finder.positions is None:
            print('No position data found in ' + folder + '.')
            bad_data = True
        if not Finder.number_frames:
            print('Number of frames not found in ' + folder + '.')
            bad_data = True
        if not Finder.size_overview or not Finder.size_frames:
            print('Size of the overview image and/or the single frames not found in ' + folder + '.')
            bad_data = True
        if Finder.overview is None:
            print('No overview image was found in ' + folder + '.')
            bad_data = True

        if not bad_data:
            return (Finder.optimized_positions.copy(), Finder.positions.copy(),
                    Finder.number_frames, Finder.size_overview, Finder.overview.shape, Finder.size_frames)

    def append_data(self, folder):
        data = self.get_data(folder)
        if data is None:
            print('No data received.')
            return
        else:
            optimized_positions, positions, number_frames, size_overview, shape_overview, size_frames = data
            pixelsize = size_overview/np.amax(shape_overview)

        if folder in self.mapnames['mapname']:
            print('\nReplacing ' + folder + ' because it is already in the saved data.')
            i = np.where(self.mapnames['mapname'] == folder)[0].item()
        else:
            print('\nAppending ' + folder + '.')
            shape = self.evenlines.shape
            i = shape[1]

            resized_evenlines = np.empty((shape[0], i+1, shape[2]), dtype=self.evenlines.dtype)
            resized_evenlines[:, :i, :] = self.evenlines
            self.evenlines = resized_evenlines

            resized_oddlines = np.empty((shape[0], i+1, shape[2]), dtype=self.oddlines.dtype)
            resized_oddlines[:, :i, :] = self.oddlines
            self.oddlines = resized_oddlines

            resized_firstlines = np.empty((shape[0], i+1, shape[2]), dtype=self.firstlines.dtype)
            resized_firstlines[:, :i, :] = self.firstlines
            self.firstlines = resized_firstlines

            resized_mapnames = np.empty(i+1, dtype=self.mapnames.dtype)
            resized_mapnames[:i] = self.mapnames
            self.mapnames = resized_mapnames

#        for i in range(self.evenlines.shape[1]):
#            if np.isnan(self.evenlines[0, i]).all():
        self.mapnames['mapname'][i] = folder
        self.mapnames['pixelsize'][i] = pixelsize
        self.mapnames['number_frames'][i] = np.array(number_frames)
        self.mapnames['size_frames'][i] = size_frames
        self.mapnames['size_overview'][i] = size_overview
        differences = positions - optimized_positions
        # Even lines have odd indices
        evenlines = differences[1::2]
        yevenlines = np.median(evenlines[:,:,0], axis=1)
        xevenlines = np.median(evenlines[:,:,1], axis=0)
        inter_yevenlines = interp1d(np.mgrid[0:100:len(yevenlines)*1j], yevenlines)
        inter_xevenlines = interp1d(np.mgrid[0:100:len(xevenlines)*1j], xevenlines)
        self.evenlines[0, i] = inter_yevenlines(np.arange(101))
        self.evenlines[1, i] = inter_xevenlines(np.arange(101))
        # odd lines have even indices
        oddlines = differences[2::2]
        yoddlines = np.median(oddlines[:,:,0], axis=1)
        xoddlines = np.median(oddlines[:,:,1], axis=0)
        inter_yoddlines = interp1d(np.mgrid[0:100:len(yoddlines)*1j], yoddlines)
        inter_xoddlines = interp1d(np.mgrid[0:100:len(xoddlines)*1j], xoddlines)
        self.oddlines[0, i] = inter_yoddlines(np.arange(101))
        self.oddlines[1, i] = inter_xoddlines(np.arange(101))
        # special treatment for first line
        firstlines = differences[0]
        yfirstlines = np.median(firstlines[:,0])
        xfirstlines = firstlines[:,1]
        inter_xfirstlines = interp1d(np.mgrid[0:100:len(xfirstlines)*1j], xfirstlines)
        self.firstlines[0, i] = np.resize(yfirstlines, (101))
        self.firstlines[1, i] = inter_xfirstlines(np.arange(101))

#        break

    def save(self):
        savedict = {}
        savedict['firstlines'] = self.firstlines
        savedict['evenlines'] = self.evenlines
        savedict['oddlines'] = self.oddlines
        savedict['mapnames'] = self.mapnames

        np.savez(Positioncollector.positionsfilepath, **savedict)

    def main(self, prefix='/3tb/maps_data'):
        for path in self.paths:
            print('\nAppending data of ' + path + '...')
            if prefix:
                path = os.path.join(prefix, path)

            self.append_data(path)

        print('Done Appending.\n\nSaving data...')
        self.save()
        print('Done')

if __name__ == '__main__':
#    paths = ['map_2015_04_15_13_13', 'map_2015_04_16_00_25', 'map_2015_06_30_14_44/all', 'map_2015_08_18_17_07',
#             'map_2015_10_19_17_49', 'map_2015_10_19_20_44', 'map_2015_10_19_22_26', 'map_2015_12_10_12_30',
#             'map_2015_12_10_17_27']
    paths = ['map_2017_05_24_00_07']

    Collector = Positioncollector(paths=paths)
    Collector.main()


import streamlit as st
import pandas as pd

# Exibir o logotipo centralizado com tamanho 200x200
st.image("farma.png", use_column_width=False, caption="", output_format="PNG", width=200)

# Carregar os dados do Excel
url = "https://docs.google.com/spreadsheets/d/e/2PACX-1vSulTerCVzXwOlraQucdzZsvxg-XGDZPA9xAXiMpFkQJ7GlfisoPoWzh3MrJEKCQPZYnDer7Cd0u5qE/pub?output=xlsx"
df = pd.read_excel(url, usecols=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])  # Lê todas as colunas

# Renomear as colunas
df.columns = ["Razão Social", "Período Inicial", "Período Final", "Loja", "CNPJ", "Faturamento ST", "Ressarcimento", "Complemento", "% Ressarcimento", "Status", "Prioridade"]

# Função para formatar o valor em "R$ 75.809.091,57"
def formatar_valor(valor):
    return f"R$ {valor:,.2f}".replace(".", ",")

# Widget de seleção de Razão Social
razao_social_opcoes = ["Selecionar todos"] + list(df["Razão Social"].unique())
razao_social_selecionada = st.selectbox("Selecione a Razão Social:", razao_social_opcoes)

# Filtrar dados pela Razão Social selecionada
if razao_social_selecionada == "Selecionar todos":
    dados_razao_social_selecionada = df
else:
    dados_razao_social_selecionada = df[df["Razão Social"] == razao_social_selecionada]

# Widget de seleção de status
status_selecionado = st.selectbox("Selecione o Status:", dados_razao_social_selecionada["Status"].unique())

# Filtrar dados pelo status selecionado
dados_status_selecionado = dados_razao_social_selecionada[dados_razao_social_selecionada["Status"] == status_selecionado]

# Adicionar menu lateral para escolher entre "Prioridade" e "Geral"
menu_selecionado = st.sidebar.radio("Selecione o Menu:", ["Prioridade", "Geral"])

# Filtro de Lojas
if menu_selecionado == "Prioridade":
    lojas = dados_status_selecionado[dados_status_selecionado["Prioridade"] == "Sim"]["Loja"].unique()
else:
    lojas = dados_status_selecionado["Loja"].unique()

lojas_selecionadas = st.multiselect("Selecione as lojas:", ["Selecionar todos"] + lojas.tolist(), default="Selecionar todos", key="lojas", help="Escolha uma ou mais lojas")

# Filtrar dados das lojas selecionadas
if "Selecionar todos" not in lojas_selecionadas:
    if menu_selecionado == "Prioridade":
        dados_lojas_selecionadas = dados_status_selecionado[(dados_status_selecionado["Loja"].isin(lojas_selecionadas)) & (dados_status_selecionado["Prioridade"] == "Sim")]
    else:
        dados_lojas_selecionadas = dados_status_selecionado[dados_status_selecionado["Loja"].isin(lojas_selecionadas)]
else:
    dados_lojas_selecionadas = dados_status_selecionado

# Organizar os blocos de total em uma grade
total_container = st.container()
total_block = st.columns(5)

# Estilo para centralizar e formatar os valores
value_style = "display: flex; justify-content: center; align-items: center; text-align: center; border: 2px solid #FF6400; padding: 10px; font-size: 20px;"

# Bloco de Faturamento ST
with total_block[0]:
    st.subheader("Faturamento ST")
    total_faturamento_st = dados_lojas_selecionadas["Faturamento ST"].sum()
    st.markdown(f'
{formatar_valor(total_faturamento_st)}
', unsafe_allow_html=True)

# Bloco de Ressarcimento
with total_block[1]:
    st.subheader("Ressarcimento")
    total_ressarcimento = dados_lojas_selecionadas["Ressarcimento"].sum()
    st.markdown(f'
{formatar_valor(total_ressarcimento)}
', unsafe_allow_html=True)

# Bloco de Complemento
with total_block[2]:
    st.subheader("Complemento")
    total_complemento = dados_lojas_selecionadas["Complemento"].sum()
    st.markdown(f'
{formatar_valor(total_complemento)}
', unsafe_allow_html=True)

# Bloco de Diferença Ressarcimento - Complemento
with total_block[3]:
    st.subheader("Ressar - Compl")
    diferenca_ressarcimento_complemento = total_ressarcimento - total_complemento
    st.markdown(f'
{formatar_valor(diferenca_ressarcimento_complemento)}
', unsafe_allow_html=True)

# Calculadora de Média % Ressarcimento apenas para "Em Desenvolvimento"
if status_selecionado == "Em Desenvolvimento":
    with total_block[4]:
        st.subheader("% Ressarcimento")

        if not dados_lojas_selecionadas.empty:
            media_percentual_ressarcimento = dados_lojas_selecionadas["% Ressarcimento"].mean() * 100
            st.markdown(f'
{media_percentual_ressarcimento:.1f}%
', unsafe_allow_html=True)

            # Widget de entrada para a porcentagem
            nova_porcentagem = st.number_input("Nova Porcentagem (%)", min_value=0.0, max_value=100.0, value=media_percentual_ressarcimento)

            # Calcular o novo valor de ressarcimento com base na nova porcentagem
            novo_ressarcimento = total_faturamento_st * (nova_porcentagem / 100)
            st.markdown(f'
Novo Ressarcimento: {formatar_valor(novo_ressarcimento)}
', unsafe_allow_html=True)
        else:
            st.write("Apenas nos Status 'Em Desenvolvimento'")
else:
    with total_block[4]:
        st.subheader("%Ressarcimento")
        st.write("Apenas nos outros Status")

# Gráfico de Barras (Faturamento ST)
st.subheader("Gráfico de Barras (Faturamento ST)")
st.bar_chart(dados_lojas_selecionadas.set_index("Loja")["Faturamento ST"], use_container_width=True)

# Gráfico de Barras (Ressarcimento)
st.subheader("Gráfico de Barras (Ressarcimento)")
st.bar_chart(dados_lojas_selecionadas.set_index("Loja")["Ressarcimento"], use_container_width=True)

# Gráfico de Barras (Complemento)
st.subheader("Gráfico de Barras (Complemento)")
st.bar_chart(dados_lojas_selecionadas.set_index("Loja")["Complemento"], use_container_width=True)


# Figures for papersize

import time
import numpy as np
import matplotlib.pyplot as plt
import cPickle as pickle
import cv2
from data import load_stereo_video, calc_ground_truth
from bp_wrapper import downsample, foveal_bp, coarse_bp
from data import KittiSource, KittiMultiViewSource
from transform import DisparityMemory
from importance import UnusuallyClose, get_average_disparity
from filter import expand_coarse, cost, cost_on_points, Filter
from kitti.stereo import load_pair
from astropy.units import bar

def trim(disp, vmax, edge):
    return disp[edge-1:-edge,vmax:-edge]

def remove_axes():
    plt.gca().set_frame_on(False)
    plt.gca().get_yaxis().set_visible(False)
    plt.gca().get_xaxis().set_visible(False)

def smudge_vs_interp():
    source = KittiSource(51, 34)
    gt3 = source.ground_truth[32]
    pos3 = source.positions[32]
    gt4 = source.ground_truth[33]
    pos4 = source.positions[33]

    fig = plt.figure(1)

    plt.subplot(221)
    plt.imshow(gt4, vmin=0, vmax=128)
    remove_axes()

    plt.subplot(222)
    mem = DisparityMemory(gt3.shape, 0, fill_method='none')
    mem.remember(pos3, gt3)
    mem.move(pos4) #creates transform of gt3 data to pos4
    plt.imshow(mem.transforms[0], vmin=0, vmax=128)
    remove_axes()

    plt.subplot(223)
    mem = DisparityMemory(gt3.shape, 0, fill_method='smudge')
    mem.remember(pos3, gt3)
    mem.move(pos4)
    plt.imshow(mem.transforms[0], vmin=0, vmax=128)
    remove_axes()

    plt.subplot(224)
    mem = DisparityMemory(gt3.shape, 0, fill_method='interp')
    mem.remember(pos3, gt3)
    mem.move(pos4)
    plt.imshow(mem.transforms[0], vmin=0, vmax=128)
    remove_axes()

    plt.tight_layout()
    plt.show(block=True)

def fovea_examples():
    video = load_stereo_video(51, 4)
    frame = video[3]

    params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3}
    down_factor = 0

    seed = np.zeros((0,0), dtype='uint8')

    fig = plt.figure(1)

    values = 128/2**down_factor
    edge = 25

    plt.subplot(211)
    disp = foveal_bp(frame, (910, 200), seed, down_factor=down_factor, iters=5, **params)
    plt.imshow(trim(disp, values, edge), vmin=0, vmax=values)
    plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
    plt.gca().set_frame_on(False)
    plt.gca().get_yaxis().set_visible(False)
    plt.gca().get_xaxis().set_visible(False)

    plt.subplot(212)
    disp = foveal_bp(frame, (590, 50), seed, down_factor=down_factor, iters=5, **params)
    plt.imshow(trim(disp, values, 25), vmin=0, vmax=values)
    plt.scatter(590-values, 50-edge, s=100, c='white', marker='+', linewidths=2)
    plt.gca().set_frame_on(False)
    plt.gca().get_yaxis().set_visible(False)
    plt.gca().get_xaxis().set_visible(False)

    plt.tight_layout()
    plt.show(block=True)

def seed_outside_fovea():
    source = KittiSource(51, 5)

    params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3}

    down_factor = 0
    values = 128/2**down_factor
    edge = 25

    fn = 3 #frame number

    seed = np.zeros((0,0), dtype='uint8')
    disp_no_seed = foveal_bp(source.video[fn], 910, 200, seed, down_factor=down_factor, iters=5, **params)

    plt.subplot(311)
    plt.imshow(trim(disp_no_seed, values, edge), vmin=0, vmax=values)
    plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
    remove_axes()

    disp_previous_frame = foveal_bp(source.video[fn-1], 590, 50, seed, down_factor=down_factor, iters=5, **params)
    mem = DisparityMemory(disp_previous_frame.shape, fovea_shape=(100,300), fill_method='smudge')
    mem.remember(source.positions[fn-1], disp_previous_frame, fovea_centre=(50,590))
    mem.remember_pos(source.positions[fn]) #TODO: change method name to move
    seed = mem.transforms[0].astype('uint8')
    disp_seed = foveal_bp(source.video[fn], 910, 200, seed, down_factor=down_factor, iters=5, **params)

    plt.subplot(312)
    plt.imshow(trim(seed, values, edge), vmin=0, vmax=values)
    plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=0)
    remove_axes()

    plt.subplot(313)
    plt.imshow(trim(disp_seed, values, edge), vmin=0, vmax=values)
    plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
    remove_axes()

    plt.tight_layout()
    plt.show(block=True)

def importance():
    source = KittiSource(51, 100)
    average_disparity = get_average_disparity(source.ground_truth)
    uc = UnusuallyClose(average_disparity)

    frame_num = 5
    importance = uc.get_importance(source.ground_truth[frame_num])

    plt.figure()
    plt.subplot(311)
    plt.imshow(average_disparity, vmin=0, vmax=128)
    remove_axes()

    plt.subplot(312)
    plt.imshow(source.ground_truth[frame_num], vmin=0, vmax=128)
    remove_axes()

    plt.subplot(313)
    plt.imshow(importance, vmin=0, vmax=64)
    remove_axes()

    plt.tight_layout()
    plt.show(block=True)

def rationale():
    """
    Figure that illustrates rationale for fovea approach, i.e. diminishing returns
    with increasing runtime via further iterations at a given downfactor, but
    possibly too-high cost of using lower downfactor.
    """

#    n_frames = 194
    n_frames = 50

    frame_down_factor = 1

    source = KittiMultiViewSource(0)
    full_shape = source.frame_ten[0].shape
    frame_shape = downsample(source.frame_ten[0], frame_down_factor).shape

    params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3}

#     iterations = [1, 2, 3]
    iterations = [1, 2, 3, 4, 5, 7, 10, 15]
    mean_coarse_times = []
    mean_coarse_costs = []
    mean_fine_times = []
    mean_fine_costs = []

    for j in range(len(iterations)):
        print(str(iterations[j]) + ' iterations')
        coarse_times = []
        coarse_costs = []
        fine_times = []
        fine_costs = []

        for i in range(n_frames):
            print(' frame ' + str(i))

            source = KittiMultiViewSource(i)
            true_points = source.get_ground_truth_points(occluded=False)

            frame_down_factor = 1

            coarse_time, coarse_cost = _evaluate_frame(source.frame_ten, true_points, frame_shape,
                    full_shape, frame_down_factor+1, frame_down_factor, iterations[j], params)
            fine_time, fine_cost = _evaluate_frame(source.frame_ten, true_points, frame_shape,
                    full_shape, frame_down_factor+0, frame_down_factor, iterations[j], params)

            coarse_times.append(coarse_time)
            coarse_costs.append(coarse_cost)
            fine_times.append(fine_time)
            fine_costs.append(fine_cost)

        mean_coarse_times.append(np.mean(coarse_times))
        mean_coarse_costs.append(np.mean(coarse_costs))
        mean_fine_times.append(np.mean(fine_times))
        mean_fine_costs.append(np.mean(fine_costs))

    print(mean_coarse_times)
    print(mean_coarse_costs)
    print(mean_fine_times)
    print(mean_fine_costs)

    plt.plot(mean_coarse_times, mean_coarse_costs, color='k', marker='s', markersize=12)
    plt.plot(mean_fine_times, mean_fine_costs, color='k', marker='o', markersize=12)
    plt.xlabel('Runtime (s)', fontsize=18)
    plt.ylabel('Mean absolute disparity error (pixels)', fontsize=18)
    plt.gca().tick_params(labelsize='18')
    plt.show()

def foveation_sequence():
    frame_down_factor = 1
    mem_down_factor = 2     # relative to the frame down factor
    coarse_down_factor = 2  # for the coarse comparison

    fs = 80
    fovea_shape = (fs, fs)
    full_values = 128
    values = full_values / 2**frame_down_factor

    index = 15
    n_frames = 10
    source = KittiMultiViewSource(index, test=False, n_frames=n_frames)
    full_shape = source.frame_ten[0].shape
    frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
                 downsample(source.frame_ten[1], frame_down_factor)]
    frame_shape = frame_ten[0].shape

    average_disp = source.get_average_disparity()
    average_disp = cv2.pyrUp(average_disp)[:frame_shape[0],:frame_shape[1]-values]

    filter = Filter(average_disp, frame_down_factor, mem_down_factor,
                    fovea_shape, frame_shape, values, verbose=False, memory_length=0)

    plt.figure()
    import matplotlib.cm as cm
    for i in range(0, 10, 2):
        frame = [downsample(source.frame_sequence[i][0], frame_down_factor),
                 downsample(source.frame_sequence[i][1], frame_down_factor)]
        filter_disp, fovea_corner = filter.process_frame(None, frame)

        edge = 5

        plt.subplot(5,1,i/2+1)
#        plt.subplot(5,2,i+1)
        plt.imshow(trim(frame[0], values, edge), cmap = cm.Greys_r)
#         remove_axes()

#         plt.subplot(5,2,i+2)
#         plt.imshow(trim(filter_disp, values, edge), vmin=0, vmax=full_values)

        fovea_corner = fovea_corner[0]
#        plot_edges(fovea_ij, (fs, fs))
        fi, fj = fovea_corner
        fm = fs
        fn = fs
        plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')

#        plt.scatter(fovea_corner[1]-values+fs/2, fovea_corner[0]-edge+fs/2, s=100, c='green', marker='+', linewidths=2)
#        plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)
#        plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
#        plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
#        plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)

        remove_axes()

    plt.tight_layout(-1)
    plt.show()

def _evaluate_frame(frame, true_points, frame_shape, full_shape, down_factor, frame_down_factor, iters, params):
    values = 128/2**frame_down_factor
    start_time = time.time()
    coarse_disp = coarse_bp(frame, down_factor=down_factor,
                            iters=iters, values=128, **params)
    disp = expand_coarse(coarse_disp, down_factor - frame_down_factor)
    disp = disp[:frame_shape[0],:frame_shape[1]]
    elapsed_time = time.time() - start_time

#    unweighted_cost.append(cost_on_points(coarse_disp[:, values:], true_points, full_shape=full_shape))

#    true_points = source.true_points[frame_num]
    return elapsed_time, cost_on_points(disp[:,values:], true_points, full_shape=full_shape)

def disparity_minus_mean():
    down_factor = 2
#    index = 1
    test=False
    n_disp = 128

    differences = []
    border = 10
    for index in range(2):
        print(index)
        source = KittiMultiViewSource(index, test=test, n_frames=0)
        average_disparity = source.get_average_disparity()

#        frame = load_pair(index, test, frame=i, multiview=True)
        frame = source.frame_ten
        gt_frame = calc_ground_truth(frame, n_disp, down_factor=down_factor, iters=50)
        d = gt_frame.astype('int') - average_disparity.astype('int')
        d = d[border:-border,border:-border]
        differences.append(d.flatten())

#        plt.subplot(211)
#        plt.imshow(gt_frame)
#        plt.colorbar()
#        plt.subplot(212)
#        plt.imshow(average_disparity)
#        plt.colorbar()
#        plt.show()

    foo = np.array(differences)
    print()
    print(np.array(differences).shape)
#    plt.hist(np.array(differences).flatten(), 50)
#    plt.show()

if __name__ == '__main__':
#     fovea_examples()
#     smudge_vs_interp()
#     seed_outside_fovea()
#     importance()
    rationale()
#    foveation_sequence()
#    disparity_minus_mean()


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import hashlib
import json
import six

from google.appengine.ext import ndb

from dashboard import list_tests
from dashboard.common import descriptor
from dashboard.common import request_handler
from dashboard.common import datastore_hooks
from dashboard.models import page_state

from flask import make_response, request

def ShortUriHandlerGet():
  """Handles getting page states."""
  state_id = request.args.get('sid')

  if not state_id:
    return request_handler.RequestHandlerReportError(
        'Missing required parameters.', status=400)

  state = ndb.Key(page_state.PageState, state_id).get()

  if not state:
    return request_handler.RequestHandlerReportError('Invalid sid.', status=400)

  if request.args.get('v2', None) is None:
    return make_response(state.value)

  if state.value_v2 is None:
    state.value_v2 = _Upgrade(state.value)
    # If the user is not signed in, then they won't be able to see
    # internal_only TestMetadata, so value_v2 will be incomplete.
    # If the user is signed in, then value_v2 is complete, so it's safe to
    # store it.
    if datastore_hooks.IsUnalteredQueryPermitted():
      state.put()
  return make_response(state.value_v2)

def ShortUriHandlerPost():
  """Handles saving page states and getting state id."""

  state = request.values.get('page_state')

  if not state:
    return request_handler.RequestHandlerReportError(
        'Missing required parameters.', status=400)

  state_id = GetOrCreatePageState(state)

  return make_response(json.dumps({'sid': state_id}))

def GetOrCreatePageState(state):
  state = state.encode('utf-8')
  state_id = GenerateHash(state)
  if not ndb.Key(page_state.PageState, state_id).get():
    page_state.PageState(id=state_id, value=state).put()
  return state_id

def GenerateHash(state_string):
  """Generates a hash for a state string."""
  return hashlib.sha256(six.ensure_binary(state_string)).hexdigest()

def _UpgradeChart(chart):
  groups = []
  if isinstance(chart, list):
    groups = chart
  elif isinstance(chart, dict):
    groups = chart['seriesGroups']

  suites = set()
  measurements = set()
  bots = set()
  cases = set()

  for prefix, suffixes in groups:
    if suffixes == ['all']:
      paths = list_tests.GetTestsMatchingPattern(
          prefix + '/*', only_with_rows=True)
    else:
      paths = []
      for suffix in suffixes:
        if suffix == prefix.split('/')[-1]:
          paths.append(prefix)
        else:
          paths.append(prefix + '/' + suffix)

    for path in paths:
      desc = descriptor.Descriptor.FromTestPathSync(path)
      suites.add(desc.test_suite)
      bots.add(desc.bot)
      measurements.add(desc.measurement)
      if desc.test_case:
        cases.add(desc.test_case)

  return {
      'parameters': {
          'testSuites': list(suites),
          'measurements': list(measurements),
          'bots': list(bots),
          'testCases': list(cases),
      },
  }

def _Upgrade(statejson):
  try:
    state = json.loads(statejson)
  except ValueError:
    return statejson
  if 'charts' not in state:
    return statejson
  state = {
      'showingReportSection': False,
      'chartSections': [_UpgradeChart(chart) for chart in state['charts']],
  }
  statejson = json.dumps(state)
  return six.ensure_binary(statejson)


import os
import shutil
from astropy.io import fits
import sys
import numpy as np
import logging
import multiprocessing
from multiprocessing import Pool
import csv
import pandas as pd
import matplotlib.pyplot as plt
import math
from statistics import mean
import ipywidgets as widgets

from ipywidgets import HBox, VBox

from IPython.display import display

from setup import datapathWidget,number,coadd, res_widget,ispecpathWidget, max_widget, min_widget, s1d_widget,unit
ispec_dir = ispecpathWidget.value

if ispec_dir[-1] != '/':
            ispec_dir=ispec_dir+'/'
import ispec
c_str=s1d_widget.value

def degrade_norm(b):
    print('normalising')
    resolution=res_widget.value
    #path=[]
    wavelength_min = min_widget.value
    wavelength_max=max_widget.value

    if number.value=='Multiple' and coadd.value=='No':
        paths = [ f.path for f in os.scandir(datapathWidget.value) if f.is_dir()]
    if number.value=='Single' and coadd.value=='No':
        paths = [datapathWidget.value]
    print(paths)
    with out8:
        for p in paths:
            print([f.name for f in os.scandir(p+'/') if f.is_file()])
            raw_spectra = [f.name for f in os.scandir(p+'/') if f.is_file() and str(c_str) in f.name] #and 'S1D' in f.name]
            print(raw_spectra)
            cor_spectrum = ispec.read_spectrum(p+'/'+raw_spectra[0])
            h=fits.open(p+'/'+raw_spectra[0])
            print(h)

            print(cor_spectrum['flux'])
            #cor_spectrum['flux'] = [i/1e-13 for i in cor_spectrum['flux']]
            print(cor_spectrum['flux'])
            plt.plot(cor_spectrum['waveobs'],cor_spectrum['flux'])
            if str(unit.value) == 'Angstroms':
                cor_spectrum['waveobs']=cor_spectrum['waveobs']/10
            snr = ispec.estimate_snr(cor_spectrum['flux'], num_points=20)
            cor_spectrum['err'] = cor_spectrum['flux']/snr

            logging.info("Radial velocity determination with template...")
            template = ispec.read_spectrum(ispec_dir + "/input/spectra/templates/NARVAL.Sun.370_1048nm/template.txt.gz")
            models, ccf = ispec.cross_correlate_with_template(cor_spectrum, template, \
                                            lower_velocity_limit=-1000, upper_velocity_limit=1000, \
                                            velocity_step=1.0, fourier=False)
            components = len(models)
            print(components)
                    # First component:
            rv = np.round(models[0].mu(), 2) # km/s
            rv_err = np.round(models[0].emu(), 2) # km/s
            cor_spectrum = ispec.correct_velocity(cor_spectrum, rv)
            if resolution > 47000:
                to_resolution = 47000
                deg_spectrum = ispec.convolve_spectrum(cor_spectrum, to_resolution, \
                                                                from_resolution=resolution)
                print('degraded resolution')
            else:
                to_resultion = resolution
                deg_spectrum=cor_spectrum
                print('kept resolution the same')
          #  deg_spectrum=cor_spectrum
                                     #--- ontinuum fit
            model = "Splines" # "Polynomy"
            degree = 2
            nknots = None # Automatic: 1 spline every 5 nm
            from_resolution = to_resolution = 47000

            # Strategy: Filter first median values and secondly MAXIMUMs in order to find the continuum
            order='median+max'
            median_wave_range=0.05
            max_wave_range=1.0
            plt.scatter(deg_spectrum['waveobs'], deg_spectrum['flux'])
            plt.show()
            star_continuum_model = ispec.fit_continuum(deg_spectrum, from_resolution=from_resolution, \
                                                        nknots=nknots, degree=degree, \
                                                        median_wave_range=median_wave_range, \
                                                        max_wave_range=max_wave_range, \
                                                        model=model, order=order, \
                                                        automatic_strong_line_detection=True, \
                                                        strong_line_probability=0.5, \
                                                        use_errors_for_fitting=True)
                            #--- Normalize -------------------------------------------------------------
            normalized_star_spectrum = ispec.normalize_spectrum(deg_spectrum, star_continuum_model,     consider_continuum_errors=False)

            print('done normalising')
            file_name=p +'/prepared_1_.fits'
            print(file_name)
            ispec.write_spectrum(normalized_star_spectrum, file_name)
            #print('writing')
go = widgets.Button(description='Prepare Spectra')
go.style.button_color = '#ffbc3c'
out7 = widgets.Output(layout={'border': '1px solid black'})
box7 = widgets.VBox([widgets.VBox([go]),out7])

nom = widgets.Button(description='Normalise Spectra')
nom.style.button_color = '#ffbc3c'
out8 = widgets.Output(layout={'border': '1px solid black'})
box8 = widgets.VBox([widgets.VBox([nom]),out8])

nom.on_click(degrade_norm)

if coadd.value=='Yes':
    display(box7)
elif coadd.value=='No':
    display(box8)
def coadd_spectra(b):
    print('Coadding...')
    resolution=res_widget.value
    paths=[]
    to_resolution=resolution #only for now
    wavelength_min = min_widget.value
    wavelength_max=max_widget.value
    if number.value=='Multiple' and coadd.value=='Yes':
        paths = [f.path for f in os.scandir(datapathWidget.value) if f.is_dir()]
        #print(paths)
    if number.value=='Single' and coadd.value=='Yes':
        paths = [datapathWidget.value]
    with out7:
        for p in paths:
            raw_spectra = [f.name for f in os.scandir(p) if f.is_file() and str(c_str) in f.name]
            #print(raw_spectra)
            spectra=[]
            snr_list=[]
            for i in range(len(raw_spectra)):
                print(raw_spectra[i])
                spectrum = ispec.read_spectrum(p+'/'+raw_spectra[i])
                if str(unit.value) == 'Angstroms':
                    spectrum['waveobs']=spectrum['waveobs']/10
                print('read spectrum!')
                hdul = fits.open(p+'/'+raw_spectra[i])
                hdr=hdul[0].header
                spectrum['flux'] = [i/1e-13 for i in spectrum['flux']]
                snr = ispec.estimate_snr(spectrum['flux'], num_points=10)
                spectrum['err'] = spectrum['flux']/snr
                snr_list.append(snr)
                if snr> 10:

                    logging.info("Radial velocity determination with template...")
                    template = ispec.read_spectrum(ispec_dir + "/input/spectra/templates/NARVAL.Sun.370_1048nm/template.txt.gz")
                    models, ccf = ispec.cross_correlate_with_template(spectrum, template, \
                                            lower_velocity_limit=-200, upper_velocity_limit=200, \
                                            velocity_step=1.0, fourier=False)
                    components = len(models)
                    print(components)
                    # First component:
                    rv = np.round(models[0].mu(), 2) # km/s
                    rv_err = np.round(models[0].emu(), 2) # km/s
                    cor_spectrum = ispec.correct_velocity(spectrum, rv)

                    if resolution > 47000:
                        to_resolution = 47000
                        deg_spectrum = ispec.convolve_spectrum(cor_spectrum, to_resolution, \
                                                                from_resolution=resolution)
                    else:
                        to_resultion = 40000
                        deg_spectrum=cor_spectrum
                                 #--- Continuum fit -------------------------------------------------------------
                    model = "Splines" # "Polynomy"
                    degree = 2
                    nknots = None # Automatic: 1 spline every 5 nm
                    from_resolution = to_resolution

                    # Strategy: Filter first median values and secondly MAXIMUMs in order to find the continuum
                    order='median+max'
                    median_wave_range=0.05
                    max_wave_range=1.0

                    star_continuum_model = ispec.fit_continuum(deg_spectrum, from_resolution=from_resolution, \
                                                nknots=nknots, degree=degree, \
                                                median_wave_range=median_wave_range, \
                                                max_wave_range=max_wave_range, \
                                                model=model, order=order, \
                                                automatic_strong_line_detection=True, \
                                                strong_line_probability=0.5, \
                                                use_errors_for_fitting=True)
                    #--- Normalize -------------------------------------------------------------
                    normalized_star_spectrum = ispec.normalize_spectrum(deg_spectrum, star_continuum_model,     consider_continuum_errors=False)
                    spectra.append(normalized_star_spectrum)
            wavelengths = np.arange(wavelength_min, wavelength_max, 0.001)
            flux=[]
            err=[]
            for i in spectra:
                if len(spectra)>=1:
                    resampled_star_spectrum = ispec.resample_spectrum(i, wavelengths, method='linear', zero_edges=True)

                    flux.append(resampled_star_spectrum['flux'])
                    err.append(resampled_star_spectrum['err'])
            if len(spectra)>=1:
                coadded_spectrum = ispec.create_spectrum_structure(wavelengths, flux = np.average(flux, axis=0),err=np.average(err, axis=0))
                spec_snr = ispec.estimate_snr(coadded_spectrum['flux'], num_points=10)
                smoothed_star_spectrum = ispec.convolve_spectrum(coadded_spectrum, to_resolution)
                snr_list.append(spec_snr)
                file=open(p +'/SNR_list.txt', 'w')
                for element in snr_list:
                    file.write(str(element)+ '\n')
                file_name=p +'/prepared_1_.fits'
                ispec.write_spectrum(coadded_spectrum, file_name)
            else:
                file=open(p+'/Spectra_too_low_SNR', 'w')
                for element in snr_list:
                    file.write(str(element)+ '\n')
        return
go.on_click(coadd_spectra)