import numpy as np
from glob import glob
from matplotlib import pyplot as plt
import dill as pickle
import os
from tqdm import tqdm

def get_name(filepath):
    # Gets the experiments name from the filepath, removing the .pkl extension and the -vpts if present
    name = filepath.replace('\\','/').split('/')[-1]
    if name.endswith('.pkl'):
        name = name[:-4]
    if name.endswith('-vpts'):
        name = name[:-5]
    return name

def get_system(name):
    return name.split('-')[0]

def gen_plot(ax, data_dict, is_rand, max_vpt, colors, n_bins=20, label_name=True, label_mean=True, key_sort=None, **plot_params):
    data = {key:data_dict[key][is_rand][is_rand] for key in data_dict if is_rand in data_dict[key].keys()}
    bins = np.linspace(0, max_vpt, n_bins)

    if key_sort is not None:
        # Sort the keys using the given lambda function as the sort criterion
        keys = list(data.keys())
        keys.sort(key=key_sort)
    else:
        keys = data.keys()

    for key in keys:
        mean = np.mean(data[key])

        # Generate the label
        label = ''
        if label_name:
            label = '-'.join(key)
        if label_mean:
            if len(label) > 0:
                label += '; '
            label += r"$\mu = {:.2f}$".format(mean)

        ax.hist(data[key], bins=bins, color=colors[key], edgecolor=colors[key], label=label, **plot_params)
        ax.axvline(x=mean, color=colors[key],linestyle="--",alpha=0.8)

def main(verbose=False, extension='pdf', **plot_params):
    result_dir = "histograms"
    os.makedirs(result_dir, exist_ok=True)

    colors = {
        ('augmented','activ_f'):'b',
        ('standard','activ_f'):'g',
        ('standard','random'):'r',
    }

    data_dir = '../vpt_results'
    data = dict()

    if verbose:
        print("Loading data...")
    # Load the data
    for filename in glob(os.path.join(data_dir, '*.pkl')):
        name = get_name(filename)
        parts = name.split('-')

        outer_key = (parts[0], *parts[4:])
        inner_key = (parts[1], parts[3])
        is_rand = parts[2]

        if not outer_key in data.keys():
            data[outer_key] = dict()

        if not inner_key in data[outer_key].keys():
            data[outer_key][inner_key] = dict()

        with open(filename, 'rb') as file:
            # item at index 0 is just the experiment/parameters
            data[outer_key][inner_key][is_rand] = pickle.load(file)[1]

    # Generate plots
    keys = data.keys()
    if verbose:
        print("Generating plots...")
        keys = tqdm(keys)
        progress_bar = keys

    for key in keys:
        if verbose:
            progress_bar.set_description('-'.join(key))

        dest_file = os.path.join(result_dir, '{}-{}-{}.{}'.format(*key, extension))

        fig, ax = plt.subplots(2,1, figsize=(6,8))
        # determine the largest vpt for plotting reasons

        max_vpt = max([np.max(data[key][subkey][is_rand][is_rand]) for subkey in data[key].keys() for is_rand in {'continue','random'} if is_rand in data[key][subkey].keys()])

        ax[0].set_title('Continue predictions')
        gen_plot(ax[0], data[key], 'continue', max_vpt, colors, **plot_params)
        ax[0].legend(loc='upper right')
        ax[1].set_title('Random predictions')
        gen_plot(ax[1], data[key], 'random', max_vpt, colors, label_name=False, **plot_params)
        ax[1].legend(loc='upper right')
        fig.savefig(dest_file)
        plt.close(fig)

def main_traintime(verbose=False, extension='pdf', **plot_params):
    result_dir = "histograms/traintime"
    os.makedirs(result_dir, exist_ok=True)

    _raw_colors = ['r','g','b','orange','m']
    colors = {
        'lorenz':{(f"tr{v}",):c for v,c
                in zip([1.0, 3.0, 6.6, 10.0, 30.0,], _raw_colors)},
        'rossler':{(f"tr{v}",):c for v,c
                in zip([5.0, 15.0, 50.0, 165.0, 300.0,], _raw_colors)},
        'thomas':{(f"tr{v}",):c for v,c
                in zip([10.0, 30.0, 100.0, 660.0, 1000.0,], _raw_colors)},
    }

    data_dir = '../traintimes/vpts'

    ## Nested dictionary that will hold all of the parameters
    # The format:
    # dict[experiment][tr_time][is_rand]
    data = dict()

    if verbose:
        print("Loading data...")
    # Load the data
    for filename in glob(os.path.join(data_dir, '*.pkl')):
        name = get_name(filename)
        parts = name.split('-')

        outer_key = (*parts[:2], *parts[3:6])
        inner_key = (parts[6],)
        is_rand = parts[2]

        if not outer_key in data.keys():
            data[outer_key] = dict()

        if not inner_key in data[outer_key].keys():
            data[outer_key][inner_key] = dict()

        with open(filename, 'rb') as file:
            # first item is just the experiment/parameters
            data[outer_key][inner_key][is_rand] = pickle.load(file)[1]

    # Generate plots
    keys = data.keys()
    if verbose:
        print("Generating plots...")
        keys = tqdm(keys)
        progress_bar = keys

    key_sort_func = lambda s: float(s[0][2:])

    for key in keys:
        if verbose:
            progress_bar.set_description('-'.join(key))

        dest_file = os.path.join(result_dir, '{}.{}'.format('-'.join(map(str,key)), extension))

        fig, ax = plt.subplots(2,1, figsize=(6,8))

        # determine the largest vpt for plotting reasons
        max_vpt = max([np.max(data[key][subkey][is_rand][is_rand]) for subkey in data[key].keys() for is_rand in {'continue','random'} if is_rand in data[key][subkey].keys()])

        ax[0].set_title('Continue predictions')
        gen_plot(ax[0], data[key], 'continue', max_vpt, colors[key[0]], key_sort=key_sort_func, **plot_params)
        ax[0].legend(loc='upper right')
        ax[1].set_title('Random predictions')
        gen_plot(ax[1], data[key], 'random', max_vpt, colors[key[0]], key_sort=key_sort_func, label_name=False, **plot_params)
        ax[1].legend(loc='upper right')
        fig.savefig(dest_file)
        plt.close(fig)

if __name__=="__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Generate histograms from the data')
    parser.add_argument('--traintimes', dest='mode', action='store_const',
                    const='traintimes', default='default')
    args = parser.parse_args()

    if args.mode == 'default':
        main(extension='png', verbose=True, alpha=0.3, n_bins=40)
    elif args.mode == 'traintimes':
        main_traintime(extension='png', verbose=True, alpha=0.3, n_bins=40)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Tanky Woo@2013-06-23

""" Convert Markdown file to html, which is embeded in html template.

Requirement:

    * argparse : python 2.7.x include it by default
    * Python-Markdown : https://github.com/waylan/Python-Markdown

Use this tool, please:

    sudo ln -s /path/to/mdgen.py /usr/bin/mdgen

Usage:

    mdgen -h

"""

from __future__ import print_function

import os
import sys
import re
import argparse
import markdown

from os import path as osp

import configs
import comm

__author__  = "Tanky Woo <me@tankywoo.com>"
__version__ = "2.0"
__license__ = "MIT License"

def _check_path_exists(path):
    """Check if the path(include file and directory) exists."""
    if not osp.exists(path):
        return False
    return True

def _check_suffix(md_file):
    """Check if the md_file's suffix is right.

    Supported markdown suffixes are configs.SUFFIXES. Do not use
    other suffixes.
    """
    if comm.filter_suffix(md_file):
        _, md_name = _get_dir_and_md_name(md_file)
        sys.exit(comm.color_error("[%s] suffix is wrong!" % md_name))

def _get_dir_and_md_name(md_file):
    """Get the subdir's name and markdown file's name."""
    dir_name = md_file.split("/")[-2] # html subdir
    md_name = md_file.split("/")[-1].split(".")[0]
    return (dir_name, md_name)

def _get_title(md_file):
    """Get the wiki's title.

    Established:
        The first line of wiki is the title, written in html comment syntax.
        such as:

    """
    notations = {"left" : ""}
    _, md_name = _get_dir_and_md_name(md_file)
    with open(md_file, "rb") as fd:
        first_line = fd.readline().strip()
        if "title" not in first_line.lower():
            sys.exit(comm.color_error(
                "[%s] first line must have `title` keyword" % md_name))
        if not (first_line.startswith(notations["left"])
                and first_line.endswith(notations["right"])):
            sys.exit(comm.color_error("[%s] title syntax is wrong!" % md_name))

        for notation in notations.values():
            first_line = first_line.replace(notation, "")
        first_line = first_line.strip()

        # strip the space between `title` and `:`
        first_line = first_line.split(":", 1)
        first_line[0] = first_line[0].strip().lower()
        first_line = ":".join(first_line)

        title = first_line.lstrip("title:").strip()

        return title

def _md2html(md_file, title):
    """Generate the html from md file, and embed it in html template."""
    md_file_text = unicode("".join(open(md_file, "rb").readlines()), "utf-8")
    content = markdown.markdown(md_file_text, \
            extensions=["fenced_code", "codehilite(guess_lang=False)"])
    tpl = open(configs.TPL_PATH, "rb")
    tpl_html = "".join(tpl.readlines())
    html = re.sub("{{ content }}", content, unicode(tpl_html, "utf-8"))
    html = re.sub("{{ title }}", unicode(title, "utf-8"), html)
    html = html.encode("utf-8")

    return html

def _update_dir_page(dir_name, md_name, title):
    """The dir_page is directory page to list the wiki(html)'s link.

    A
 label's syntax is:

<a href="../`dir_name`/`md_name`.html">`title`

    Insert the new
..
 in the first line of the
 lists. Make sure
    the top wiki is the newest.

    TODO:
        * if there is blanklines between
, delete it.
        * if there is blanklinks out of
..
, delete it.
    """
    dir_page_file = osp.join(configs.HTML_PATH, dir_name+".html")

    # append mode will ignore seek
    if _check_path_exists(dir_page_file):
        file_mode = "rb+"
    else:
        file_mode = "wb+"

    with open(dir_page_file, file_mode) as fd:
        li_list = fd.readlines()[1:-1]  # ignore the
..
 label
        new_li = "
%s
" % \
                (dir_name, md_name, title)

        # if new_li exists, ignore; otherwise insert it in the first
 list
        for li in li_list:
            if new_li in li:
                return
        fd.seek(0) # set current position to the beginning of the file
        li_list.insert(0, "\t%s\n"%new_li)
        new_content = "
\n" + "".join(li_list) + "
\n"
        fd.write(new_content)

def _update_wiki_page(dir_name, md_name, html):
    """Write the wiki html to file.

    If the parent directory not exists, mkdir it.
    """
    html_dir_path = osp.join(configs.HTML_PATH, dir_name)
    if not _check_path_exists(html_dir_path):
        os.mkdir(html_dir_path)
    html_path = osp.join(html_dir_path, md_name+".html")
    with open(html_path, "wb") as wfd:
        wfd.write(html)

def generator(md_file, debug_mode=False):
    if not _check_path_exists(md_file):
        _, md_name = _get_dir_and_md_name(md_file)
        sys.exit(comm.color_error("[%s] file does not exists" % md_name))
    _check_suffix(md_file)

    dir_name, md_name = _get_dir_and_md_name(md_file)
    title = _get_title(md_file)
    html = _md2html(md_file, title)

    if debug_mode:
        print(html)
    else:
        _update_wiki_page(dir_name, md_name, html)
        _update_dir_page(dir_name, md_name, title)
        comm.print_ok("[%s]" % md_name, " updated ok.")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="")
    parser.add_argument("-f", "--file", dest="md_file", required=True,
            help="The markdown file")
    parser.add_argument("--debug", dest="debug", action="store_true",
            help="Display to stdout")
    parser.add_argument("--version", action="version",
            version="%(prog)s " + __version__)

    parse_args = parser.parse_args()
    md_file = osp.realpath(parse_args.md_file)
    debug_mode = parse_args.debug

    generator(md_file, debug_mode)


import argparse
import os
import json
import collections
import requests
import singer
import singer.bookmarks as bookmarks
import singer.metrics as metrics

from singer import metadata

session = requests.Session()
logger = singer.get_logger()

REQUIRED_CONFIG_KEYS = ['access_token', 'repository']

KEY_PROPERTIES = {
    'commits': ['sha'],
    'comments': ['id'],
    'issues': ['id'],
    'assignees': ['id'],
    'collaborators': ['id'],
    'pull_requests':['id'],
    'stargazers': ['user_id'],
    'releases': ['id'],
    'reviews': ['id'],
    'review_comments': ['id'],
    'pr_commits': ['id'],
    'events': ['id'],
    'issue_events': ['id'],
    'issue_labels': ['id'],
    'issue_milestones': ['id'],
    'pull_request_reviews': ['id'],
    'commit_comments': ['id'],
    'projects': ['id'],
    'project_columns': ['id'],
    'project_cards': ['id'],
    'repos': ['id'],
    'teams': ['id'],
    'team_members': ['id'],
    'team_memberships': ['url']
}

class AuthException(Exception):
    pass

class NotFoundException(Exception):
    pass

def translate_state(state, catalog, repositories):
    '''
    This tap used to only support a single repository, in which case the
    state took the shape of:
    {
      "bookmarks": {
        "commits": {
          "since": "2018-11-14T13:21:20.700360Z"
        }
      }
    }
    The tap now supports multiple repos, so this function should be called
    at the beginning of each run to ensure the state is translate to the
    new format:
    {
      "bookmarks": {
        "singer-io/tap-adwords": {
          "commits": {
            "since": "2018-11-14T13:21:20.700360Z"
          }
        }
        "singer-io/tap-salesforce": {
          "commits": {
            "since": "2018-11-14T13:21:20.700360Z"
          }
        }
      }
    }
    '''
    nested_dict = lambda: collections.defaultdict(nested_dict)
    new_state = nested_dict()

    for stream in catalog['streams']:
        stream_name = stream['tap_stream_id']
        for repo in repositories:
            if bookmarks.get_bookmark(state, repo, stream_name):
                return state
            if bookmarks.get_bookmark(state, stream_name, 'since'):
                new_state['bookmarks'][repo][stream_name]['since'] = bookmarks.get_bookmark(state, stream_name, 'since')

    return new_state

def get_bookmark(state, repo, stream_name, bookmark_key):
    repo_stream_dict = bookmarks.get_bookmark(state, repo, stream_name)
    if repo_stream_dict:
        return repo_stream_dict.get(bookmark_key)
    return None

# pylint: disable=dangerous-default-value
def authed_get(source, url, headers={}):
    with metrics.http_request_timer(source) as timer:
        session.headers.update(headers)
        resp = session.request(method='get', url=url)
        if resp.status_code == 401:
            raise AuthException(resp.text)
        if resp.status_code == 403:
            raise AuthException(resp.text)
        if resp.status_code == 404:
            raise NotFoundException(resp.text)

        timer.tags[metrics.Tag.http_status_code] = resp.status_code
        return resp

def authed_get_all_pages(source, url, headers={}):
    while True:
        r = authed_get(source, url, headers)
        r.raise_for_status()
        yield r
        if 'next' in r.links:
            url = r.links['next']['url']
        else:
            break

def get_abs_path(path):
    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)

def generate_pr_commit_schema(commit_schema):
    pr_commit_schema = commit_schema.copy()
    pr_commit_schema['properties']['pr_number'] = {
        "type":  ["null", "integer"]
    }
    pr_commit_schema['properties']['pr_id'] = {
        "type": ["null", "string"]
    }
    pr_commit_schema['properties']['id'] = {
        "type": ["null", "string"]
    }

    return pr_commit_schema

def load_schemas():
    schemas = {}

    for filename in os.listdir(get_abs_path('schemas')):
        path = get_abs_path('schemas') + '/' + filename
        file_raw = filename.replace('.json', '')
        with open(path) as file:
            schemas[file_raw] = json.load(file)

    schemas['pr_commits'] = generate_pr_commit_schema(schemas['commits'])
    return schemas

class DependencyException(Exception):
    pass

def validate_dependencies(selected_stream_ids):
    errs = []
    msg_tmpl = ("Unable to extract {0} data. "
                "To receive {0} data, you also need to select {1}.")

    if 'reviews' in selected_stream_ids and 'pull_requests' not in selected_stream_ids:
        errs.append(msg_tmpl.format('reviews','pull_requests'))

    if 'review_comments' in selected_stream_ids and 'pull_requests' not in selected_stream_ids:
        errs.append(msg_tmpl.format('review_comments','pull_requests'))

    if errs:
        raise DependencyException(" ".join(errs))

def write_metadata(mdata, values, breadcrumb):
    mdata.append(
        {
            'metadata': values,
            'breadcrumb': breadcrumb
        }
    )

def populate_metadata(schema_name, schema):
    mdata = metadata.new()
    #mdata = metadata.write(mdata, (), 'forced-replication-method', KEY_PROPERTIES[schema_name])
    mdata = metadata.write(mdata, (), 'table-key-properties', KEY_PROPERTIES[schema_name])

    for field_name in schema['properties'].keys():
        if field_name in KEY_PROPERTIES[schema_name]:
            mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'automatic')
        else:
            mdata = metadata.write(mdata, ('properties', field_name), 'inclusion', 'available')

    return mdata

def get_catalog():
    raw_schemas = load_schemas()
    streams = []

    for schema_name, schema in raw_schemas.items():

        # get metadata for each field
        mdata = populate_metadata(schema_name, schema)

        # create and add catalog entry
        catalog_entry = {
            'stream': schema_name,
            'tap_stream_id': schema_name,
            'schema': schema,
            'metadata' : metadata.to_list(mdata),
            'key_properties': KEY_PROPERTIES[schema_name],
        }
        streams.append(catalog_entry)

    return {'streams': streams}

def do_discover():
    catalog = get_catalog()
    # dump catalog
    print(json.dumps(catalog, indent=2))

def get_all_teams(schemas, repo_path, state, mdata):
    org = repo_path.split('/')[0]
    with metrics.record_counter('teams') as counter:
        for response in authed_get_all_pages(
                'teams',
                'https://api.github.com/orgs/{}/teams?sort=created_at&direction=desc'.format(org)
        ):
            teams = response.json()
            extraction_time = singer.utils.now()

            for r in teams:
                r['_sdc_repository'] = repo_path

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('teams', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'teams', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

                if schemas.get('team_members'):
                    team_slug = r['slug']
                    for team_members_rec in get_all_team_members(team_slug, schemas['team_members'], repo_path, state, mdata):
                        singer.write_record('team_members', team_members_rec, time_extracted=extraction_time)
                        singer.write_bookmark(state, repo_path, 'team_members', {'since': singer.utils.strftime(extraction_time)})

                if schemas.get('team_memberships'):
                    team_slug = r['slug']
                    for team_memberships_rec in get_all_team_memberships(team_slug, schemas['team_memberships'], repo_path, state, mdata):
                        singer.write_record('team_memberships', team_memberships_rec, time_extracted=extraction_time)

    return state

def get_all_team_members(team_slug, schemas, repo_path, state, mdata):
    org = repo_path.split('/')[0]
    with metrics.record_counter('team_members') as counter:
        for response in authed_get_all_pages(
                'team_members',
                'https://api.github.com/orgs/{}/teams/{}/members?sort=created_at&direction=desc'.format(org, team_slug)
        ):
            team_members = response.json()
            for r in team_members:
                r['_sdc_repository'] = repo_path

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                counter.increment()

                yield rec

    return state

def get_all_team_memberships(team_slug, schemas, repo_path, state, mdata):
    org = repo_path.split('/')[0]
    for response in authed_get_all_pages(
            'team_members',
            'https://api.github.com/orgs/{}/teams/{}/members?sort=created_at&direction=desc'.format(org, team_slug)
        ):
        team_members = response.json()
        with metrics.record_counter('team_memberships') as counter:
            for r in team_members:
                username = r['login']
                for res in authed_get_all_pages(
                        'memberships',
                        'https://api.github.com/orgs/{}/teams/{}/memberships/{}'.format(org, team_slug, username)
                ):
                    team_membership = res.json()
                    team_membership['_sdc_repository'] = repo_path
                    with singer.Transformer() as transformer:
                        rec = transformer.transform(team_membership, schemas, metadata=metadata.to_map(mdata))
                    counter.increment()
                    yield rec
    return state

def get_all_issue_events(schemas, repo_path, state, mdata):
    bookmark_value = get_bookmark(state, repo_path, "issue_events", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('issue_events') as counter:
        for response in authed_get_all_pages(
                'issue_events',
                'https://api.github.com/repos/{}/issues/events?sort=created_at&direction=desc'.format(repo_path)
        ):
            events = response.json()
            extraction_time = singer.utils.now()
            for event in events:
                event['_sdc_repository'] = repo_path
                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                updated_at = event.get('created_at') if event.get('updated_at') is None else event.get('updated_at')
                if bookmark_time and singer.utils.strptime_to_utc(updated_at) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(event, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('issue_events', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'issue_events', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_events(schemas, repo_path, state, mdata):
    # Incremental sync off `created_at`
    # https://developer.github.com/v3/issues/events/#list-events-for-a-repository
    # 'https://api.github.com/repos/{}/issues/events?sort=created_at&direction=desc'.format(repo_path)

    bookmark_value = get_bookmark(state, repo_path, "events", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('events') as counter:
        for response in authed_get_all_pages(
                'events',
                'https://api.github.com/repos/{}/events?sort=created_at&direction=desc'.format(repo_path)
        ):
            events = response.json()
            extraction_time = singer.utils.now()
            for r in events:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                updated_at = r.get('created_at') if r.get('updated_at') is None else r.get('updated_at')
                if bookmark_time and singer.utils.strptime_to_utc(updated_at) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('events', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'events', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_issue_milestones(schemas, repo_path, state, mdata):
    # Incremental sync off `due on` ??? confirm.
    # https://developer.github.com/v3/issues/milestones/#list-milestones-for-a-repository
    # 'https://api.github.com/repos/{}/milestones?sort=created_at&direction=desc'.format(repo_path)
    bookmark_value = get_bookmark(state, repo_path, "issue_milestones", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('issue_milestones') as counter:
        for response in authed_get_all_pages(
                'milestones',
                'https://api.github.com/repos/{}/milestones?direction=desc'.format(repo_path)
        ):
            milestones = response.json()
            extraction_time = singer.utils.now()
            for r in milestones:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                if bookmark_time and singer.utils.strptime_to_utc(r.get('due_on')) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('issue_milestones', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'issue_milestones', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_issue_labels(schemas, repo_path, state, mdata):
    # https://developer.github.com/v3/issues/labels/
    # not sure if incremental key
    # 'https://api.github.com/repos/{}/labels?sort=created_at&direction=desc'.format(repo_path)

    with metrics.record_counter('issue_labels') as counter:
        for response in authed_get_all_pages(
                'issue_labels',
                'https://api.github.com/repos/{}/labels'.format(repo_path)
        ):
            issue_labels = response.json()
            extraction_time = singer.utils.now()
            for r in issue_labels:
                r['_sdc_repository'] = repo_path

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('issue_labels', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'issue_labels', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_commit_comments(schemas, repo_path, state, mdata):
    # https://developer.github.com/v3/repos/comments/
    # updated_at? incremental
    # 'https://api.github.com/repos/{}/comments?sort=created_at&direction=desc'.format(repo_path)
    bookmark_value = get_bookmark(state, repo_path, "commit_comments", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('commit_comments') as counter:
        for response in authed_get_all_pages(
                'commit_comments',
                'https://api.github.com/repos/{}/comments?sort=created_at&direction=desc'.format(repo_path)
        ):
            commit_comments = response.json()
            extraction_time = singer.utils.now()
            for r in commit_comments:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                if bookmark_time and singer.utils.strptime_to_utc(r.get('updated_at')) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('commit_comments', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'commit_comments', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_projects(schemas, repo_path, state, mdata):
    bookmark_value = get_bookmark(state, repo_path, "projects", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('projects') as counter:
        #pylint: disable=too-many-nested-blocks
        for response in authed_get_all_pages(
                'projects',
                'https://api.github.com/repos/{}/projects?sort=created_at&direction=desc'.format(repo_path),
                { 'Accept': 'application/vnd.github.inertia-preview+json' }
        ):
            projects = response.json()
            extraction_time = singer.utils.now()
            for r in projects:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                if bookmark_time and singer.utils.strptime_to_utc(r.get('updated_at')) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('projects', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'projects', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

                project_id = r.get('id')

                # sync project_columns if that schema is present (only there if selected)
                if schemas.get('project_columns'):
                    for project_column_rec in get_all_project_columns(project_id, schemas['project_columns'], repo_path, state, mdata):
                        singer.write_record('project_columns', project_column_rec, time_extracted=extraction_time)
                        singer.write_bookmark(state, repo_path, 'project_columns', {'since': singer.utils.strftime(extraction_time)})

                        # sync project_cards if that schema is present (only there if selected)
                        if schemas.get('project_cards'):
                            column_id = project_column_rec['id']
                            for project_card_rec in get_all_project_cards(column_id, schemas['project_cards'], repo_path, state, mdata):
                                singer.write_record('project_cards', project_card_rec, time_extracted=extraction_time)
                                singer.write_bookmark(state, repo_path, 'project_cards', {'since': singer.utils.strftime(extraction_time)})
    return state

def get_all_project_cards(column_id, schemas, repo_path, state, mdata):
    bookmark_value = get_bookmark(state, repo_path, "project_cards", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('project_cards') as counter:
        for response in authed_get_all_pages(
                'project_cards',
                'https://api.github.com/projects/columns/{}/cards?sort=created_at&direction=desc'.format(column_id)
        ):
            project_cards = response.json()
            for r in project_cards:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                if bookmark_time and singer.utils.strptime_to_utc(r.get('updated_at')) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                counter.increment()
                yield rec

    return state

def get_all_project_columns(project_id, schemas, repo_path, state, mdata):
    bookmark_value = get_bookmark(state, repo_path, "project_columns", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('project_columns') as counter:
        for response in authed_get_all_pages(
                'project_columns',
                'https://api.github.com/projects/{}/columns?sort=created_at&direction=desc'.format(project_id)
        ):
            project_columns = response.json()
            for r in project_columns:
                r['_sdc_repository'] = repo_path

                # skip records that haven't been updated since the last run
                # the GitHub API doesn't currently allow a ?since param for pulls
                # once we find the first piece of old data we can return, thanks to
                # the sorting
                if bookmark_time and singer.utils.strptime_to_utc(r.get('updated_at')) < bookmark_time:
                    return state

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                counter.increment()
                yield rec

    return state

def get_all_releases(schemas, repo_path, state, mdata):
    # Releases doesn't seem to have an `updated_at` property, yet can be edited.
    # For this reason and since the volume of release can safely be considered low,
    #    bookmarks were ignored for releases.

    with metrics.record_counter('releases') as counter:
        for response in authed_get_all_pages(
                'releases',
                'https://api.github.com/repos/{}/releases?sort=created_at&direction=desc'.format(repo_path)
        ):
            releases = response.json()
            extraction_time = singer.utils.now()
            for r in releases:
                r['_sdc_repository'] = repo_path

                # transform and write release record
                with singer.Transformer() as transformer:
                    rec = transformer.transform(r, schemas, metadata=metadata.to_map(mdata))
                singer.write_record('releases', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'releases', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_pull_requests(schemas, repo_path, state, mdata):
    '''
    https://developer.github.com/v3/pulls/#list-pull-requests
    '''

    bookmark_value = get_bookmark(state, repo_path, "pull_requests", "since")
    if bookmark_value:
        bookmark_time = singer.utils.strptime_to_utc(bookmark_value)
    else:
        bookmark_time = 0

    with metrics.record_counter('pull_requests') as counter:
        with metrics.record_counter('reviews') as reviews_counter:
            for response in authed_get_all_pages(
                    'pull_requests',
                    'https://api.github.com/repos/{}/pulls?state=all&sort=updated&direction=desc'.format(repo_path)
            ):
                pull_requests = response.json()
                extraction_time = singer.utils.now()
                for pr in pull_requests:

                    # skip records that haven't been updated since the last run
                    # the GitHub API doesn't currently allow a ?since param for pulls
                    # once we find the first piece of old data we can return, thanks to
                    # the sorting
                    if bookmark_time and singer.utils.strptime_to_utc(pr.get('updated_at')) < bookmark_time:
                        return state

                    pr_num = pr.get('number')
                    pr_id = pr.get('id')
                    pr['_sdc_repository'] = repo_path

                    # transform and write pull_request record
                    with singer.Transformer() as transformer:
                        rec = transformer.transform(pr, schemas['pull_requests'], metadata=metadata.to_map(mdata))
                    singer.write_record('pull_requests', rec, time_extracted=extraction_time)
                    singer.write_bookmark(state, repo_path, 'pull_requests', {'since': singer.utils.strftime(extraction_time)})
                    counter.increment()

                    # sync reviews if that schema is present (only there if selected)
                    if schemas.get('reviews'):
                        for review_rec in get_reviews_for_pr(pr_num, schemas['reviews'], repo_path, state, mdata):
                            singer.write_record('reviews', review_rec, time_extracted=extraction_time)
                            singer.write_bookmark(state, repo_path, 'reviews', {'since': singer.utils.strftime(extraction_time)})

                            reviews_counter.increment()

                    # sync review comments if that schema is present (only there if selected)
                    if schemas.get('review_comments'):
                        for review_comment_rec in get_review_comments_for_pr(pr_num, schemas['review_comments'], repo_path, state, mdata):
                            singer.write_record('review_comments', review_comment_rec, time_extracted=extraction_time)
                            singer.write_bookmark(state, repo_path, 'review_comments', {'since': singer.utils.strftime(extraction_time)})

                    if schemas.get('pr_commits'):
                        for pr_commit in get_commits_for_pr(
                                pr_num,
                                pr_id,
                                schemas['pr_commits'],
                                repo_path,
                                state,
                                mdata
                        ):
                            singer.write_record('pr_commits', pr_commit, time_extracted=extraction_time)
                            singer.write_bookmark(state, repo_path, 'pr_commits', {'since': singer.utils.strftime(extraction_time)})

    return state

def get_reviews_for_pr(pr_number, schema, repo_path, state, mdata):
    for response in authed_get_all_pages(
            'reviews',
            'https://api.github.com/repos/{}/pulls/{}/reviews'.format(repo_path,pr_number)
    ):
        reviews = response.json()
        for review in reviews:
            review['_sdc_repository'] = repo_path
            with singer.Transformer() as transformer:
                rec = transformer.transform(review, schema, metadata=metadata.to_map(mdata))
            yield rec

        return state

def get_review_comments_for_pr(pr_number, schema, repo_path, state, mdata):
    for response in authed_get_all_pages(
            'comments',
            'https://api.github.com/repos/{}/pulls/{}/comments'.format(repo_path,pr_number)
    ):
        review_comments = response.json()
        for comment in review_comments:
            comment['_sdc_repository'] = repo_path
            with singer.Transformer() as transformer:
                rec = transformer.transform(comment, schema, metadata=metadata.to_map(mdata))
            yield rec

        return state

def get_commits_for_pr(pr_number, pr_id, schema, repo_path, state, mdata):
    for response in authed_get_all_pages(
            'pr_commits',
            'https://api.github.com/repos/{}/pulls/{}/commits'.format(repo_path,pr_number)
    ):

        commit_data = response.json()
        for commit in commit_data:
            commit['_sdc_repository'] = repo_path
            commit['pr_number'] = pr_number
            commit['pr_id'] = pr_id
            commit['id'] = '{}-{}'.format(pr_id, commit['sha'])
            with singer.Transformer() as transformer:
                rec = transformer.transform(commit, schema, metadata=metadata.to_map(mdata))
            yield rec

        return state

def get_all_assignees(schema, repo_path, state, mdata):
    '''
    https://developer.github.com/v3/issues/assignees/#list-assignees
    '''
    with metrics.record_counter('assignees') as counter:
        for response in authed_get_all_pages(
                'assignees',
                'https://api.github.com/repos/{}/assignees'.format(repo_path)
        ):
            assignees = response.json()
            extraction_time = singer.utils.now()
            for assignee in assignees:
                assignee['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(assignee, schema, metadata=metadata.to_map(mdata))
                singer.write_record('assignees', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'assignees', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_collaborators(schema, repo_path, state, mdata):
    '''
    https://developer.github.com/v3/repos/collaborators/#list-collaborators
    '''
    with metrics.record_counter('collaborators') as counter:
        for response in authed_get_all_pages(
                'collaborators',
                'https://api.github.com/repos/{}/collaborators'.format(repo_path)
        ):
            collaborators = response.json()
            extraction_time = singer.utils.now()
            for collaborator in collaborators:
                collaborator['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(collaborator, schema, metadata=metadata.to_map(mdata))
                singer.write_record('collaborators', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'collaborator', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_commits(schema, repo_path,  state, mdata):
    '''
    https://developer.github.com/v3/repos/commits/#list-commits-on-a-repository
    '''
    bookmark = get_bookmark(state, repo_path, "commits", "since")
    if bookmark:
        query_string = '?since={}'.format(bookmark)
    else:
        query_string = ''

    with metrics.record_counter('commits') as counter:
        for response in authed_get_all_pages(
                'commits',
                'https://api.github.com/repos/{}/commits{}'.format(repo_path, query_string)
        ):
            commits = response.json()
            extraction_time = singer.utils.now()
            for commit in commits:
                commit['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(commit, schema, metadata=metadata.to_map(mdata))
                singer.write_record('commits', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'commits', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_all_issues(schema, repo_path,  state, mdata):
    '''
    https://developer.github.com/v3/issues/#list-issues-for-a-repository
    '''

    bookmark = get_bookmark(state, repo_path, "issues", "since")
    if bookmark:
        query_string = '&since={}'.format(bookmark)
    else:
        query_string = ''

    with metrics.record_counter('issues') as counter:
        for response in authed_get_all_pages(
                'issues',
                'https://api.github.com/repos/{}/issues?state=all&sort=updated&direction=asc{}'.format(repo_path, query_string)
        ):
            issues = response.json()
            extraction_time = singer.utils.now()
            for issue in issues:
                issue['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(issue, schema, metadata=metadata.to_map(mdata))
                singer.write_record('issues', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'issues', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()
    return state

def get_all_comments(schema, repo_path, state, mdata):
    '''
    https://developer.github.com/v3/issues/comments/#list-comments-in-a-repository
    '''

    bookmark = get_bookmark(state, repo_path, "comments", "since")
    if bookmark:
        query_string = '&since={}'.format(bookmark)
    else:
        query_string = ''

    with metrics.record_counter('comments') as counter:
        for response in authed_get_all_pages(
                'comments',
                'https://api.github.com/repos/{}/issues/comments?sort=updated&direction=asc{}'.format(repo_path, query_string)
        ):
            comments = response.json()
            extraction_time = singer.utils.now()
            for comment in comments:
                comment['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(comment, schema, metadata=metadata.to_map(mdata))
                singer.write_record('comments', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'comments', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()
    return state

def get_all_stargazers(schema, repo_path, state, mdata):
    '''
    https://developer.github.com/v3/activity/starring/#list-stargazers
    '''
    bookmark = get_bookmark(state, repo_path, "stargazers", "since")
    if bookmark:
        query_string = '&since={}'.format(bookmark)
    else:
        query_string = ''

    stargazers_headers = {'Accept': 'application/vnd.github.v3.star+json'}

    with metrics.record_counter('stargazers') as counter:
        for response in authed_get_all_pages(
                'stargazers',
                'https://api.github.com/repos/{}/stargazers?sort=updated&direction=asc{}'.format(repo_path, query_string), stargazers_headers
        ):
            stargazers = response.json()
            extraction_time = singer.utils.now()
            for stargazer in stargazers:
                stargazer['_sdc_repository'] = repo_path
                with singer.Transformer() as transformer:
                    rec = transformer.transform(stargazer, schema, metadata=metadata.to_map(mdata))
                rec['user_id'] = rec['user']['id']
                singer.write_record('stargazers', rec, time_extracted=extraction_time)
                singer.write_bookmark(state, repo_path, 'stargazers', {'since': singer.utils.strftime(extraction_time)})
                counter.increment()

    return state

def get_selected_streams(catalog):
    '''
    Gets selected streams.  Checks schema's 'selected'
    first -- and then checks metadata, looking for an empty
    breadcrumb and mdata with a 'selected' entry
    '''
    selected_streams = []
    for stream in catalog['streams']:
        stream_metadata = stream['metadata']
        if stream['schema'].get('selected', False):
            selected_streams.append(stream['tap_stream_id'])
        else:
            for entry in stream_metadata:
                # stream metadata will have empty breadcrumb
                if not entry['breadcrumb'] and entry['metadata'].get('selected',None):
                    selected_streams.append(stream['tap_stream_id'])

    return selected_streams

def get_stream_from_catalog(stream_id, catalog):
    for stream in catalog['streams']:
        if stream['tap_stream_id'] == stream_id:
            return stream
    return None

SYNC_FUNCTIONS = {
    'commits': get_all_commits,
    'comments': get_all_comments,
    'issues': get_all_issues,
    'assignees': get_all_assignees,
    'collaborators': get_all_collaborators,
    'pull_requests': get_all_pull_requests,
    'releases': get_all_releases,
    'stargazers': get_all_stargazers,
    'events': get_all_events,
    'issue_events': get_all_issue_events,
    'issue_milestones': get_all_issue_milestones,
    'issue_labels': get_all_issue_labels,
    'projects': get_all_projects,
    'commit_comments': get_all_commit_comments,
    'teams': get_all_teams
}

SUB_STREAMS = {
    'pull_requests': ['reviews', 'review_comments', 'pr_commits'],
    'projects': ['project_cards', 'project_columns'],
    'teams': ['team_members', 'team_memberships']
}

def do_sync(config, state, catalog):
    access_token = config['access_token']
    session.headers.update({'authorization': 'token ' + access_token})

    # get selected streams, make sure stream dependencies are met
    selected_stream_ids = get_selected_streams(catalog)
    validate_dependencies(selected_stream_ids)

    repositories = list(filter(None, config['repository'].split(' ')))

    state = translate_state(state, catalog, repositories)
    singer.write_state(state)

    #pylint: disable=too-many-nested-blocks
    for repo in repositories:
        logger.info("Starting sync of repository: %s", repo)
        for stream in catalog['streams']:
            stream_id = stream['tap_stream_id']
            stream_schema = stream['schema']
            mdata = stream['metadata']

            # if it is a "sub_stream", it will be sync'd by its parent
            if not SYNC_FUNCTIONS.get(stream_id):
                continue

            # if stream is selected, write schema and sync
            if stream_id in selected_stream_ids:
                singer.write_schema(stream_id, stream_schema, stream['key_properties'])

                # get sync function and any sub streams
                sync_func = SYNC_FUNCTIONS[stream_id]
                sub_stream_ids = SUB_STREAMS.get(stream_id, None)

                # sync stream
                if not sub_stream_ids:
                    state = sync_func(stream_schema, repo, state, mdata)

                # handle streams with sub streams
                else:
                    stream_schemas = {stream_id: stream_schema}

                    # get and write selected sub stream schemas
                    for sub_stream_id in sub_stream_ids:
                        if sub_stream_id in selected_stream_ids:
                            sub_stream = get_stream_from_catalog(sub_stream_id, catalog)
                            stream_schemas[sub_stream_id] = sub_stream['schema']
                            singer.write_schema(sub_stream_id, sub_stream['schema'],
                                                sub_stream['key_properties'])

                    # sync stream and it's sub streams
                    state = sync_func(stream_schemas, repo, state, mdata)

                singer.write_state(state)

@singer.utils.handle_top_exception(logger)
def main():
    args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)

    if args.discover:
        do_discover()
    else:
        catalog = args.properties if args.properties else get_catalog()
        do_sync(args.config, args.state, catalog)

if __name__ == '__main__':
    main()


from odoo import api, fields, models, _
from dateutil.relativedelta import relativedelta
from odoo.exceptions import UserError
import pytz
from datetime import datetime, date
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, float_round
from math import log10
from ..utils.data_util import format_report_data, format_header_data, format_body_data, \
    format_quantity_description_data, format_table_data, format_header_cell, format_content_cell, format_datetime, \
    format_float_number
from ..utils import graph_config as graph
from ..utils import color_util as color

class DemandForecastReport(models.AbstractModel):
    _name = 'demand.forecast.report'
    _description = "Forecasted Demand Report"

    open_mos_data = {}
    open_sos_data = {}
    forecasted_demand = {}
    product_id = False
    factor = 0
    float_format = (lambda number: f'{number:,.2f}')
    labels = []

    def init(self):
        create_actual_demand_stmt = """
           CREATE TABLE IF NOT EXISTS actual_sales_demand (
                            week_end_date date,
                            product_id int,
                            partner_id int,
                            sale_qty float,
                            sale_uom_id int
           );
           CREATE INDEX IF NOT EXISTS actual_sales_demand_date_index ON actual_sales_demand (week_end_date, product_id, partner_id);
        """
        create_daily_demand_stmt = """
           CREATE TABLE IF NOT EXISTS actual_daily_sales_demand (
                            date_order date,
                            product_id int,
                            partner_id int,
                            sale_qty float,
                            sale_uom_id int
           );
           CREATE INDEX IF NOT EXISTS actual_daily_sales_demand_date_index ON actual_daily_sales_demand (date_order, product_id);
        """
        create_table_stmt =  create_actual_demand_stmt + create_daily_demand_stmt
        self.env.cr.execute(create_table_stmt)

    @api.model
    def get_last_updated_time(self):
        last_updated = self.env['ir.config_parameter'].sudo().get_param('phd_demand_forecast.last_updated')
        try:
            last_updated = last_updated and fields.datetime.strptime(last_updated, DEFAULT_SERVER_DATETIME_FORMAT) or ''
            return last_updated
        except ValueError:
            return ''

    @api.model
    def compute_sales_demand_data(self):
        params = self.env['ir.config_parameter'].sudo()
        has_intialize_data = params.get_param('phd_demand_forecast.has_initialized_data') == 'True'
        where_stmt = ""
        where_order_stmt = ""
        week_start_offset = 8 - int(self.env['res.lang']._lang_get(self.env.user.lang).week_start)
        daily_where_stmt = ""
        daily_order_stmt = ""
        if has_intialize_data:
            start_date, end_date = self.get_period_dates(demand_type='recompute')
            where_stmt = "WHERE week_end_date >= '{}' AND week_end_date <= '{}'".format(start_date, end_date)
            daily_where_stmt = "WHERE date_order >= (CURRENT_DATE - interval '30 days')::date AND date_order <= CURRENT_DATE"
            where_order_stmt = "AND date_order::date >= '{}' AND date_order::date <= '{}'".format(start_date, end_date)
            daily_order_stmt = "AND date_order::date >= (CURRENT_DATE - interval '30 days')::date AND date_order::date <= CURRENT_DATE"
        query_actual_demand_stmt = """
            DELETE FROM actual_sales_demand {where_stmt};
            INSERT INTO actual_sales_demand (week_end_date, product_id, partner_id, sale_qty, sale_uom_id)
            SELECT (date_trunc('week', date_order + {interval})::date - {interval} + interval '6 days')::date as week_end_date,
                product_id, partner_id, SUM(sale_qty) as sale_qty, sale_uom_id
            FROM actual_daily_sales_demand demand
            GROUP BY date_trunc('week', date_order + {interval})::date - {interval}, product_id, partner_id, sale_uom_id
            ORDER BY week_end_date desc;
        """.format(where_stmt=where_stmt, interval=week_start_offset, where_order_stmt=where_order_stmt)

        query_daily_demand_stmt = """
            DELETE FROM actual_daily_sales_demand {where_stmt};
            INSERT INTO actual_daily_sales_demand (date_order, product_id, partner_id, sale_qty, sale_uom_id)
            SELECT so.date_order::date as date_order,
                sol.product_id, so.partner_id, SUM(sol.product_uom_qty) as sale_qty, suom.id as sale_uom_id
            FROM sale_order_line sol
            JOIN (
                SELECT id, date_order, partner_id
                FROM sale_order
                WHERE state IN ('sale', 'done') {where_order_stmt}
            ) so ON sol.order_id = so.id
            JOIN uom_uom suom ON sol.product_uom = suom.id
            GROUP BY so.date_order::date, sol.product_id, so.partner_id, suom.id
            ORDER BY date_order desc;
        """.format(where_stmt=daily_where_stmt, where_order_stmt=daily_order_stmt)

        query_stmt = query_daily_demand_stmt + query_actual_demand_stmt
        self.env.cr.execute(query_stmt)
        # Update parameter
        if not has_intialize_data:
            params.set_param('phd_demand_forecast.has_initialized_data', 'True')
        params.set_param('phd_demand_forecast.last_updated',
                         datetime.now(pytz.utc).strftime(DEFAULT_SERVER_DATETIME_FORMAT))

    @api.model
    def get_sales_demand_query_stmt(self, product_id, demand_type='actual'):
        if not product_id:
            raise UserError(_("Product ID is invalid."))
        start_date, end_date = self.get_period_dates(demand_type)
        query_stmt = """
            SELECT ts.week_end_date, COALESCE(demand_qty, 0) as demand_qty
            FROM (
                SELECT ts::date as week_end_date
                FROM generate_series('{start_date}', '{end_date}', '7 day'::interval) ts) ts
            LEFT JOIN (
                SELECT week_end_date, ROUND(COALESCE(SUM(demand_qty), 0)::numeric, rounding::int) as demand_qty
                FROM
                    (SELECT week_end_date,
                         (CASE WHEN suom.factor = 0 OR suom.factor IS NULL
                            THEN demand.sale_qty
                            ELSE (demand.sale_qty * COALESCE(puom.factor, 1) / suom.factor) END) as demand_qty,
                        -log(puom.rounding) as rounding
                    FROM (
                        SELECT *
                        FROM actual_sales_demand
                        WHERE product_id = {product_id}
                            AND week_end_date >= '{start_date}' AND week_end_date <= '{end_date}') demand
                        JOIN product_product pp ON demand.product_id = pp.id
                        JOIN product_template pt ON pp.product_tmpl_id = pt.id
                        JOIN uom_uom puom ON pt.uom_id = puom.id
                        JOIN uom_uom suom ON demand.sale_uom_id = suom.id
                    WHERE puom.category_id = suom.category_id) t
                GROUP BY week_end_date, rounding
            ) actual_demand ON ts.week_end_date = actual_demand.week_end_date
            ORDER BY week_end_date ASC;
        """.format(start_date=start_date, end_date=end_date, product_id=product_id)
        return query_stmt

    @api.model
    def get_sales_demand(self, product_id, demand_type='actual'):
        """
        Get sales demand of product in the past
        :param product_id: id of product which will be forecasted
        :param demand_type: `actual` or `historical`
        :return: a dictionary contains the data for sales demand
        """
        query_stmt = self.get_sales_demand_query_stmt(product_id, demand_type)
        self.env.cr.execute(query_stmt)
        result = self.env.cr.dictfetchall()
        week_numbers = [i for i in range(-4, 13) if i != 0]
        week_index = 0
        for period in result:
            period['week'] = week_numbers[week_index]
            week_index += 1
        return result

    @api.model
    def get_forecasted_demand_query_stmt(self, product_id):
        if not product_id:
            raise UserError(_("Product ID is invalid."))
        start_date, end_date = self.get_period_dates(demand_type='forecast')
        # Query data
        query_stmt = """
            SELECT ts.week_end_date, COALESCE(demand_qty, 0) as forecasted_demand_qty
            FROM (
                SELECT ts::date as week_end_date
                FROM generate_series('{start_date}', '{end_date}', '7 day'::interval) ts) ts
            LEFT JOIN (
                SELECT week_end_date, sum(demand_qty) as demand_qty
                FROM demand_forecast_item
                WHERE product_id = {product_id} AND week_end_date >= '{start_date}' AND week_end_date <= '{end_date}'
                GROUP BY week_end_date
            ) demand ON ts.week_end_date = demand.week_end_date
            ORDER BY week_end_date asc;
        """.format(product_id=product_id, start_date=start_date, end_date=end_date)
        return query_stmt

    @api.model
    def get_forecasted_demand(self, product_id):
        """
        Get imported forecasted demand data
        :param product_id: id of product which will be forecasted
        :return: Dictionary contains forecasted demand data for 16 week from week #-4 to week #12
        """
        query_stmt = self.get_forecasted_demand_query_stmt(product_id)
        self.env.cr.execute(query_stmt)
        result = self.env.cr.dictfetchall()
        week_numbers = [i for i in range(-4, 13) if i != 0]
        week_index = 0
        for period in result:
            period['week'] = week_numbers[week_index]
            week_index += 1
        return result

    @api.model
    def get_period_dates(self, demand_type='actual'):
        today = datetime.now(pytz.utc).date()
        week_start_offset = 8 - int(self.env['res.lang']._lang_get(self.env.user.lang).week_start)
        base_date = today + relativedelta(days=week_start_offset)
        # The end date of last week
        last_week_end_date = base_date - relativedelta(days=base_date.weekday() + week_start_offset + 1)
        if demand_type == 'actual':
            # Last 4 weeks
            start_date = last_week_end_date - relativedelta(weeks=3)
            end_date = last_week_end_date
        elif demand_type == 'historical':
            # Retrieve 16 weeks of last year with the same week number of forecasted demand in this year
            first_start_date = last_week_end_date - relativedelta(weeks=3)
            week_number = first_start_date.isocalendar()[1]
            year_number = first_start_date.isocalendar()[0]
            last_start_date = datetime.strptime('{} {} 1'.format(year_number - 1, week_number, 1), '%G %V %u').date()
            last_base_date = last_start_date + relativedelta(days=week_start_offset)
            start_date = last_base_date - relativedelta(days=last_base_date.weekday() + week_start_offset - 6)
            end_date = start_date + relativedelta(weeks=15)
        elif demand_type == 'forecast':
            # Retrieve 16 weeks of this year in the chart
            start_date = last_week_end_date - relativedelta(weeks=3)
            end_date = start_date + relativedelta(weeks=15)
        else:
            end_date = last_week_end_date
            start_date = end_date - relativedelta(weeks=4, days=-1)
        return start_date, end_date

    @api.model
    def get_open_mo_query_stmt(self, product_id):
        query_stmt = """
            SELECT mo.id, product_id, mo.name, mo.date_planned_finished,
                ROUND((CASE WHEN muom.factor = 0 OR muom.factor IS NULL
                          THEN mo.product_qty
                          ELSE (mo.product_qty * COALESCE(puom.factor, 1) / muom.factor) END)::numeric, -log(puom.rounding)::int) as quantity
            FROM (
                SELECT id, name, product_id, product_qty, product_uom_id, date_planned_finished
                FROM mrp_production
                WHERE product_id={} AND state not in ('done', 'cancel')
            ) mo JOIN product_product pp ON mo.product_id = pp.id
            JOIN product_template pt ON pp.product_tmpl_id = pt.id
            JOIN uom_uom puom ON pt.uom_id = puom.id
            JOIN uom_uom muom ON mo.product_uom_id = muom.id
            ORDER BY mo.date_planned_finished asc
        """.format(product_id)
        return query_stmt

    @api.model
    def get_open_manufacturing_orders(self, product_id):
        if not product_id:
            return {}
        query_stmt = self.get_open_mo_query_stmt(product_id)
        self.env.cr.execute(query_stmt)
        result = self.env.cr.dictfetchall()
        return result

    @api.model
    def get_open_so_query_stmt(self, product_id):
        query_stmt = """
            SELECT so.id, so.name, osol.product_id, so.partner_id, rp.name as partner_name, so.commitment_date, osol.quantity
            FROM (
                SELECT sol.order_id, sol.product_id,
                   SUM(ROUND((CASE WHEN suom.factor = 0 OR suom.factor IS NULL
                               THEN move.product_uom_qty
                               ELSE (move.product_uom_qty * COALESCE(puom.factor, 1) / suom.factor) END)::numeric, -log(puom.rounding)::int)) as quantity
                FROM (
                    SELECT sm.product_uom_qty, sm.product_uom, sm.sale_line_id
                    FROM (
                        SELECT product_uom_qty, product_uom, sale_line_id, picking_type_id
                        FROM stock_move
                        WHERE sale_line_id IS NOT NULL
                            AND state not in ('cancel', 'done')
                            AND product_id = {product_id}) sm
                    JOIN (
                        SELECT id
                        FROM stock_picking_type
                        WHERE code='outgoing'
                    ) spt ON sm.picking_type_id = spt.id
                ) move
                    JOIN (
                        SELECT id, order_id, product_id
                        FROM sale_order_line
                        WHERE state NOT IN ('draft', 'sent', 'cancel')
                            AND product_id = {product_id}
                    ) sol ON move.sale_line_id = sol.id
                    JOIN product_product pp ON sol.product_id = pp.id
                    JOIN product_template pt ON pp.product_tmpl_id = pt.id
                    JOIN uom_uom puom ON pt.uom_id = puom.id
                    JOIN uom_uom suom ON move.product_uom = suom.id
                GROUP BY sol.order_id, sol.product_id
            ) osol
            JOIN sale_order so ON osol.order_id = so.id
            JOIN res_partner rp ON rp.id = so.partner_id
            ORDER BY so.commitment_date asc
        """.format(product_id=product_id)
        return query_stmt

    @api.model
    def get_open_sale_orders(self, product_id):
        if not product_id:
            return {}
        query_stmt = self.get_open_so_query_stmt(product_id)
        self.env.cr.execute(query_stmt)
        result = self.env.cr.dictfetchall()
        return result

    def __get_quantity_on_hand(self):
        return self.float_format(self.product_id.qty_available)

    def __get_quantity_available(self):
        return self.float_format(self.product_id.free_qty)

    def __get_quantity_in_open_mos(self):
        open_mos_data = self.open_mos_data or {}
        if not len(open_mos_data):
            open_mos_data = self.get_open_manufacturing_orders(self.product_id.id)
        open_mos = [row.get('quantity', 0) for row in open_mos_data]
        return self.float_format(sum(open_mos))

    def __get_quantity_in_open_sos(self):
        open_sos_data = self.open_sos_data or {}
        if not len(open_sos_data):
            open_sos_data = self.get_open_manufacturing_orders(self.product_id.id)
        open_sos = [row.get('quantity', 0) for row in open_sos_data]
        return self.float_format(sum(open_sos))

    def get_quantity_description(self):
        res_lst = list()
        if self.product_id.type == 'product':
            res_lst.append(format_quantity_description_data("On Hand", self.__get_quantity_on_hand()))
            res_lst.append(format_quantity_description_data("Available", self.__get_quantity_available()))
        res_lst.append(format_quantity_description_data("Under Production", self.__get_quantity_in_open_mos()))
        res_lst.append(format_quantity_description_data("In Sales", self.__get_quantity_in_open_sos()))
        return res_lst

    def _get_header_content(self):
        quantity_description = self.get_quantity_description()
        product_id = self.product_id
        return format_header_data(product_id.id, product_id._name, product_id.display_name, quantity_description,
                                  self.get_last_updated_time(), product_id.uom_id.name)

    def _get_chart_content(self):
        product_id = self.product_id.id
        actual_demand = self.get_sales_demand(product_id, 'actual')
        historical_demand = self.get_sales_demand(product_id, 'historical')
        forecasted_demand = self.get_forecasted_demand(product_id)
        labels, tooltip_extend_labels = [], []
        if len(actual_demand) != 0 and len(historical_demand) != 0 and len(forecasted_demand) != 0:
            max_week = max(actual_demand[-1].get('week'), historical_demand[-1].get('week'),
                           forecasted_demand[-1].get('week'))
            min_week = max(actual_demand[0].get('week'), historical_demand[0].get('week'),
                           forecasted_demand[0].get('week'))
            labels = list(map(lambda x: f'Week {x if x < 0 else x + 1}', range(min_week, max_week)))
            tooltip_extend_labels = [format_datetime(x['week_end_date']) for x in forecasted_demand]

        self.forecasted_demand = forecasted_demand
        self.labels = labels
        chart_type = 'line'
        element_config = [
            graph.get_chart_element_config('Actual',
                                           list(map(lambda x: (x.get('demand_qty')), actual_demand)),
                                           color.ACTUAL_DEMAND_COLOR, fill=False, chart_type=chart_type),
            graph.get_chart_element_config('Forecasted',
                                           list(map(lambda x: (x.get('forecasted_demand_qty')),
                                                    forecasted_demand)),
                                           color.FORECASTED_DEMAND_COLOR, fill=False, chart_type=chart_type),
            graph.get_chart_element_config('Historical', list(
                map(lambda x: (x.get('demand_qty')), historical_demand)),
                                           color.HISTORICAL_DEMAND_COLOR, fill=False, chart_type=chart_type),
        ]
        title_config = graph.get_chart_title_config("Forecasted Demand Chart", fontColor='#FFB800', display=False)
        legend_config = graph.get_chart_legend_config(position='top')
        tooltip_config = graph.get_chart_tooltip_config(display_mode='index')
        x_axis = graph.get_chart_axis_config(stacked=False)
        y_axis = graph.get_chart_axis_config(stacked=False)
        chart_config = graph.get_chart_config_from(chart_type='line', element_configs=element_config,
                                                   title_config=title_config, legend_config=legend_config,
                                                   tooltip_config=tooltip_config, axis_labels=labels,
                                                   tooltip_extend_labels=tooltip_extend_labels,
                                                   x_axis_configs=x_axis, y_axis_configs=y_axis)

        return format_body_data(chart_config)

    def _get_table_contents(self):
        table_lst = list()
        table_lst.append(self._get_forecasted_table_content())
        table_lst.append(self._get_mos_table_content())
        table_lst.append(self._get_sos_table_content())
        return table_lst

    def _get_forecasted_table_content(self, ):
        # Extract Header
        week_key, forecasted_date_key, forecasted_demand_key = 'week_key', 'forecasted_date', 'forecasted_demand'
        headers = {
            **format_header_cell(week_key, 'Week #', sequence=1),
            **format_header_cell(forecasted_date_key, 'Date', sequence=2),
            **format_header_cell(forecasted_demand_key, 'Forecasted Demand', is_number=True, sequence=3)
        }

        forecasted_demand = self.forecasted_demand or {}
        if not len(forecasted_demand):
            forecasted_demand = self.get_sales_demand(self.product_id.id, 'historical')
        # Extract Content
        labels = self.labels
        if not labels:
            labels = ['' for _ in range(len(forecasted_demand))]
        content = [{
            week_key: format_content_cell(label),
            forecasted_date_key: format_content_cell(format_datetime(row.get('week_end_date'))),
            forecasted_demand_key: format_content_cell(self.float_format(row.get('forecasted_demand_qty')))
        } for row, label in zip(forecasted_demand, labels)] or {}
        return format_table_data("Forecasted Demand", "None", headers, content, size=4)

    def _get_mos_table_content(self):
        # Extract Header
        mo_nbr, mo_qty, mo_ready_date = 'mo_number', 'mo_qty', 'mo_ready_date'
        headers = dict()
        headers = {**headers,
                   **format_header_cell(mo_nbr, 'MO#', is_clickable=True, sequence=1),
                   **format_header_cell(mo_qty, 'Quantity', is_number=True, sequence=2),
                   **format_header_cell(mo_ready_date, 'Ready Date', sequence=3)
                   }
        # Extract Content
        open_mos_data = self.get_open_manufacturing_orders(self.product_id.id)
        self.open_mos_data = open_mos_data
        content = [{
            mo_nbr: format_content_cell(row.get('name'), row.get('id')),
            mo_qty: format_content_cell(self.float_format(row.get('quantity'))),
            mo_ready_date: format_content_cell(format_datetime(row.get('date_planned_finished')))
        } for row in open_mos_data] or {}
        return format_table_data("Open MOs", "mrp.production", headers, content, size=4)

    def _get_sos_table_content(self):
        # Extract Header
        customer_key, so_nbr_key, so_qty_key, so_confirmed_key = 'customer', 'so_number', 'so_qty', 'so_confirmed_date'
        headers = dict()
        headers = {**headers,
                   **format_header_cell(customer_key, 'Customer', is_clickable=True, sequence=2),
                   **format_header_cell(so_nbr_key, 'SO#', is_clickable=True, sequence=1),
                   **format_header_cell(so_qty_key, 'Quantity', is_number=True, sequence=3),
                   **format_header_cell(so_confirmed_key, 'Commitment Date', sequence=4)
                   }
        # Extract Content
        open_sos_data = self.get_open_sale_orders(self.product_id.id)
        self.open_sos_data = open_sos_data
        content = [{
            customer_key: format_content_cell(row.get('partner_name'), row.get('partner_id'),
                                              **{'model': 'res.partner'}),
            so_nbr_key: format_content_cell(row.get('name'), row.get('id')),
            so_qty_key: format_content_cell(self.float_format(row.get('quantity'))),
            so_confirmed_key: format_content_cell(format_datetime(row.get('commitment_date')))
        } for row in open_sos_data] or {}
        return format_table_data("Open SOs", "sale.order", headers, content, size=4)

    def get_initial_request_data(self):
        if self._context.get('report_product_id', False):
            self.product_id = self.env['product.product'].browse(self._context.get('report_product_id', -1))
            # self.factor = int(abs(log10(self.product_id.uom_id.rounding)))
            self.float_format = lambda number: f'{number:,.{self.factor}f}'
        else:
            raise UserError(_('Please provide specific product!'))

    @api.model
    def get_forecasted_report_data(self):
        self.get_initial_request_data()
        body = self._get_chart_content()
        footer = self._get_table_contents()
        header = self._get_header_content()

        config = format_report_data(header, body, footer)
        config['formatFactor'] = self.factor
        return config


from bot.ranking import Ranking
from add_users_from_log import DummyBot
from telegram import Update, Message, Chat, ChatMember, User
from bot.commands import parse_dual_update, parse_game_update
import bot.commands
import sqlite3
import json
from os import path as op

RESET_SINGLES = ("""DROP TABLE IF EXISTS singles;""" +
                 """CREATE TABLE IF NOT EXISTS""" + """ singles(submitter_id int, rival_id int, """ +
                      """submitter_uname varchar(128), rival_uname varchar(128), """ +
                      """submitter_score TINYINT, rival_score TINYINT, timestamp INTEGER)""")

RESET_DUALS = ("""DROP TABLE IF EXISTS duals;""" +
               """CREATE TABLE IF NOT EXISTS""" + """ duals(submitter_id int, submitter_teammate int,""" +
                    """rival_1_id int, rival_2_id int, submitter_uname varchar(128), """ +
                    """submitter_teammate_uname varchar(128),""" +
                    """rival_1_uname varchar(128), rival_2_uname varchar(128),""" +
                    """ submitter_score TINYINT, rival_score TINYINT, timestamp INTEGER)""")

def read_updates_from_log(chat_name):
    history = []

    missed_posts = 0

    with open(op.join("message_history", chat_name + ".txt"), "r") as f:
        for line in f:
            line = line.replace("\'", "\"")
            line = line.replace("True", "true")
            line = line.replace("False", "false")
            try:
                d = json.loads(line)
                history.append(Update.de_json(d, DummyBot()))
            except json.decoder.JSONDecodeError:
                print(line)
                missed_posts += 1

    print("Had problem reading {0:.2}% of posts".format(missed_posts/(len(history)+missed_posts)))
    print("--------------------------------------")
    return history

if __name__ == "__main__":

    import sys
    # Script breaking bug below
    sys.exit(0)

    # TODO Update at runtime.
    # In form [a|-]123123123
    chat_name = ""
    chat_id = chat_name if chat_name[0] != "a" else "-" + chat_name[1:]
    chat_id = int(chat_id)

    db_fp = op.join(op.dirname(__file__), "persistent_storage", chat_name+".db")
    conn = sqlite3.connect(db_fp)
    c = conn.cursor()

    c.executescript(RESET_DUALS)
    c.executescript(RESET_SINGLES)
    c.execute("""UPDATE ratings SET mu = 25, sigma = 8.333333333333334""")

    R = Ranking(chat_id, conn=conn)
    R.rate_limit = 250

    accepted = []

    with open("accepted_matches.txt", "r") as f:
        for line in f:
            line = line.replace("\'", "\"")
            line = line.replace("True", "true")
            line = line.replace("False", "false")

            d = json.loads(line)
            accepted.append(Update.de_json(d, DummyBot()))

    bot.commands.RANKINGS[chat_id] = R

    # BUG Games should be imported in the order of the timestamps
    # DO NOT USE BEFORE ABOVE bug IS FIXED.
    # Singles
    for upd in accepted:
        if upd.message.text is None:
            continue

        try:
            sub, sub_score, riv, riv_score = parse_game_update(upd)
            ts = int(upd.message.date.timestamp())
            R.add_singles_match(sub, riv, sub_score, riv_score, ts)
        except ValueError as err:
            if "'/singles'" not in str(err):
                print("Encountered value error")
                print()
                print(err)
                print()
                print(upd)
                print()
        except AttributeError:
             print(upd)

    # Duals
    # TODO Finalize, requires fix for parse_dual_update
    for upd in accepted:
        if upd.message.text is None:
           continue

        try:
            sub, sub_tm, rival, rival_tm, sub_score, riv_score = parse_dual_update(upd)
            ts = int(upd.message.date.timestamp())
            #print(parse_dual_update(upd))
            R.add_dual_match(sub, sub_tm, rival, rival_tm, sub_score, riv_score, ts)
        except ValueError as err:
            if "'/dual'" not in str(err):
                print(err)
                print(upd)
                print()
        except AttributeError as err:
             print(upd)
             print(err)


import math

from sklearn.cluster import DBSCAN

import numpy as np
from typing import List

from resources.bboxes import sort_boxes_top2down_wrt_left2right_order, sort_boxes, fit_bbox
from resources.dtos import Paragraph, Line

# КОД ДЛЯ СТУДЕНТА
class ParagraphFinder:
    def __init__(
            self,
            angle_threshold: int = 10,
            overlapping_threshold: float = 0.3,
            max_distance: int = 1,
            cluster_eps: float = 0.5
    ):
        self.angle_threshold = angle_threshold
        self.overlapping_threshold = overlapping_threshold
        self.max_distance = max_distance
        self.cluster_eps = cluster_eps

    @staticmethod
    def _distance(line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        y1m = (y12 + y11) / 2
        y2m = (y22 + y21) / 2
        return abs(y1m - y2m)

    @staticmethod
    def _length(line: Line) -> float:
        return np.linalg.norm(line.normalized_bbox[0] - line.normalized_bbox[1])

    def _overlapping(self, line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        x_min = max(x11, x21)
        x_max = min(x12, x22)

        if x_max - x_min < 0:
            return 0

        return (x_max - x_min) / self._length(line1)

    @staticmethod
    def _angle(line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        d_x1 = x12 - x11
        d_y1 = y12 - y11
        d_x2 = x22 - x21
        d_y2 = y22 - y21

        theta = math.atan(d_y2 / d_x2) - math.atan(d_y1 / d_x1)
        theta = math.degrees(theta)

        if theta > 90:
            theta -= 180
        if theta < -90:
            theta += 180

        return theta

    def paragraph_distance(self, line1: np.ndarray, line2: np.ndarray) -> float:
        line1 = Line(normalized_bbox=line1.reshape(4, 2))
        line2 = Line(normalized_bbox=line2.reshape(4, 2))

        if abs(self._angle(line1, line2)) > self.angle_threshold:
            return self.max_distance

        if self._overlapping(line1, line2) < self.overlapping_threshold:
            return self.max_distance

        return self._distance(line1, line2)

    @staticmethod
    def prepare_lines(lines: List[Line]) -> np.ndarray:
        return np.array([line.normalized_bbox.reshape(-1) for line in lines])

    def fit_cluster(self, lines: List[Line]) -> DBSCAN:
        prepared_lines = self.prepare_lines(lines)
        cluster = DBSCAN(metric=self.paragraph_distance, eps=self.cluster_eps)
        cluster.fit(prepared_lines)
        return cluster

    @staticmethod
    def sort_paragraphs(paragraphs):
        for par in paragraphs:
            par.items = sort_boxes(par.items, sorting_type='top2down')  # сортировка сверху вниз
        paragraphs = sort_boxes_top2down_wrt_left2right_order(paragraphs)  # сортировка в порядке чтения
        return paragraphs

    def find_paragraphs(self, lines: List[Line]) -> List[Paragraph]:
        cluster = self.fit_cluster(lines)

        paragraphs = []
        for label in set(cluster.labels_):
            line_indexes = np.argwhere(cluster.labels_ == label)
            par_lines = [lines[idx[0]] for idx in line_indexes]
            bbox = fit_bbox(np.array([lines[idx[0]].bbox for idx in line_indexes]))
            par = Paragraph(items=par_lines, bbox=bbox)
            paragraphs.append(par)
        paragraphs = self.sort_paragraphs(paragraphs)
        return paragraphs


import os
import io
import unittest
import tempfile
import shutil
import contextlib
from pathlib import Path
from publish import Application, Git, parse_args, ACTION_FIXUP, ACTION_MISSING, ACTION_BACKUP

SETTINGS="""\
services = ['test1', 'test2']
git = True
"""

known_subdirs = ['test1', 'test2']

def create_settings(directory):
    path = Path(directory) / 'settings.py'
    path.write_text(SETTINGS)

    return path

class TestFixup(unittest.TestCase):
    def test_fixup_creates_directories(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_FIXUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            # when
            app = Application(args, git)
            app.run()

            # then
            for subdir in known_subdirs:
                assert (publish / subdir).is_dir()
                assert (publish / subdir / 'published').is_dir()

    def test_create_missing_small_images(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_FIXUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            photo1 = publish / 'DSC_0001.JPG'
            photo2 = publish / 'DSC_0002.JPG'
            photo3 = publish / 'DSC_0002_processed.JPG'

            source = Path('sample.jpg').absolute()
            assert source.exists()

            photo1.symlink_to(source)
            photo2.symlink_to(source)
            shutil.copy(source, photo3)

            # when
            app = Application(args, git)
            app.run()

            # then
            assert (publish / 'DSC_0001_small.jpg').exists()
            assert (publish / 'DSC_0001_small.jpg').is_file()
            assert (publish / 'DSC_0002_small.jpg').exists()
            assert (publish / 'DSC_0002_small.jpg').is_file()
            for subdir in known_subdirs:
                path = publish / subdir
                assert (path / 'DSC_0001_small.jpg').exists()
                assert (path / 'DSC_0001_small.jpg').is_symlink()
                assert (path / 'DSC_0002_small.jpg').exists()
                assert (path / 'DSC_0002_small.jpg').is_symlink()

    def test_add_missing_small_images(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_FIXUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            app = Application(args, git)
            app.run()

            # when (add small files)
            names = ['DSC_0001.NEF', 'DSC_0002.CR3', 'DSC_0003.JPG', 'DSC_0004.JPG']
            for name in names:
                large = publish / name
                small = publish / (large.stem + '_small.jpg')

                large.symlink_to('picture')
                small.write_text('small')

            app.run()

            # then
            for subdir in known_subdirs:
                for name in names:
                    small = publish / subdir / (large.stem + '_small.jpg')
                    assert small.is_symlink()

    def test_fixup_source_symbolic_links(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_FIXUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            rootdir = Path('/mnt') / 'arch' / '2022' / '01' / '2022-01-01'
            photo1 = publish / 'DSC_0001.NEF'
            photo1.symlink_to(rootdir / photo1.name)
            photo2 = publish / 'DSC_0002.CR3'
            photo2.symlink_to(rootdir / photo2.name)
            photo3 = publish / 'DSC_0002.JPG'
            photo3.symlink_to(rootdir / photo3.name)

            # when
            app = Application(args, git)
            app.run()

            # then
            assert f'../{photo1.name}' == os.readlink(photo1)
            assert f'../{photo2.name}' == os.readlink(photo2)
            assert f'../{photo3.name}' == os.readlink(photo3)

    def test_fixup_published_symbolic_links(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_FIXUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            app = Application(args, git)
            app.run()

            rootdir = Path('/mnt') / 'arch' / '2022' / '01' / '2022-01-01'
            for subdir in known_subdirs:
                published = publish / subdir / 'published'
                photo = published / 'DSC_0001_small.jpg'
                photo.symlink_to(rootdir / photo.name)

            # when
            app.run()

            # then
            for subdir in known_subdirs:
                photo = publish / subdir / 'published' / 'DSC_0001_small.jpg'
                assert f'../../{photo.name}' == os.readlink(photo)

class TestMissing(unittest.TestCase):
    def test_missing_prints_missing_processed_photos(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_MISSING,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            photos = [('link', 'DSC_0001.JPG'), # DSC_0001 is ready -> processed.jpg exists
                      ('file', 'DSC_0001_processed.JPG'),
                      ('link', 'DSC_0002.JPG'), # DSC_0002 is ready -> (no processed.jpg, but it's fine)
                      ('link', 'DSC_0003.NEF'), # DSC_0003 is ready -> there's corresponding JPG
                      ('file', 'DSC_0003.JPG'),
                      ('link', 'DSC_0004.CR3'), # DSC_0004 is ready -> there's corresponding JPG
                      ('file', 'DSC_0004.JPG'),
                      ('link', 'DSC_0005.NEF'), # no jpeg
                      ('link', 'DSC_0006.CR3'), # no jpeg
            ]

            existing = Path('sample.jpg').absolute()
            assert existing.exists()

            for typ, photo in photos:
                path = publish / photo
                if typ == 'link':
                    path.symlink_to(existing)
                elif typ == 'file':
                    shutil.copy(existing, path)

            # when
            out = io.StringIO()
            with contextlib.redirect_stdout(out):
                app = Application(args, git)
                app.run()

            # then
            missing = {Path(line).name for line in out.getvalue().splitlines()}
            assert missing == {'DSC_0005.NEF', 'DSC_0006.CR3'}

class TestNotPublished(unittest.TestCase):
    def test_print_not_published_files(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args(['test1',
                               directory,
                               '--config', str(settings)])

            test1 = args.path / '2022-01-01' / 'publish' / 'test1'
            test1.mkdir(parents=True)

            published = test1 / 'published'
            published.mkdir()

            git = Git(args.path)
            git.execute("init")

            small1 = test1 / 'DSC_0001_small.jpg'
            small2 = published / 'DSC_0002_small.jpg'
            small3 = published / 'DSC_0003_small.jpg'
            small4 = test1 / 'DSC_0004_small.jpg'

            small1.symlink_to('../DSC_0001_small.jpg')
            small2.symlink_to('../DSC_0002_small.jpg')
            small3.symlink_to('../DSC_0003_small.jpg')
            small4.symlink_to('../DSC_0004_small.jpg')

            # when
            out = io.StringIO()
            with contextlib.redirect_stdout(out):
                app = Application(args, git)
                app.run()

            # then
            files = {Path(line).name for line in out.getvalue().splitlines()}
            assert files == {'DSC_0001_small.jpg', 'DSC_0004_small.jpg'}

class TestBackup(unittest.TestCase):
    def test_backup_makes_hard_links_to_source_photos(self):
        with tempfile.TemporaryDirectory() as directory:
            # given
            settings = create_settings(directory)
            args = parse_args([ACTION_BACKUP,
                               directory,
                               '--config', str(settings)])

            subdir = args.path / '2022-01-01'
            subdir.mkdir()

            publish = subdir / 'publish'
            publish.mkdir()

            git = Git(args.path)
            git.execute("init")

            source1 = subdir / 'DSC_0001.NEF'
            source2 = subdir / 'DSC_0002.NEF'
            source3 = subdir / 'DSC_0003.NEF'

            existing = Path('sample.jpg').absolute()
            assert existing.exists()

            shutil.copy(existing, source1)
            shutil.copy(existing, source2)
            shutil.copy(existing, source3)

            photo1 = publish / 'DSC_0001.NEF'
            photo1.symlink_to('../DSC_0001.NEF')
            assert photo1.exists()

            photo2 = publish / 'DSC_0002.NEF'
            photo2.symlink_to('../DSC_0002.NEF')
            assert photo2.exists()

            photo3 = publish / 'DSC_0003.NEF'
            photo3.symlink_to('../DSC_0003.NEF')
            assert photo3.exists()

            # when
            app = Application(args, git)
            app.run()

            # then
            backup1 = publish / '.backup' / 'DSC_0001.NEF'
            assert backup1.exists()
            assert backup1.stat().st_ino == source1.stat().st_ino

            backup2 = publish / '.backup' / 'DSC_0002.NEF'
            assert backup2.exists()
            assert backup2.stat().st_ino == source2.stat().st_ino

            backup3 = publish / '.backup' / 'DSC_0003.NEF'
            assert backup3.exists()
            assert backup3.stat().st_ino == source3.stat().st_ino

if __name__ == '__main__':
    unittest.main()


from django.conf import settings
from django.conf.urls import patterns, include, url
from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns('',
    url(r'^admin/', include(admin.site.urls)),
    url(r'^about\-us/', include('about_us.urls')),
    url(r'^$', 'about_us.views.home', name='home'),
)

if settings.DEBUG:
    urlpatterns += patterns('',
        url(r'^assets/(?P<path>.*)$', 'django.views.static.serve', {
            'document_root': settings.MEDIA_ROOT,
        }),
   )


"""from json2xml import json2xml,readfromjson

data = readfromjson("358448 k.kavya 20180323 15-30-00.json")
xml=json2xml.Json2xml(data).to_xml()
print(xml)"""

import os
from json2xml import json2xml,readfromjson
cdir=os.getcwd()
outdir='annotations'
import pdb

for directory in ['data/train','data/test']:
    if not os.path.exists(os.path.join(cdir,directory,outdir)):
        os.mkdir('{}/{}/{}'.format(cdir,directory,outdir))
        #pdb.set_trace()
    for files in os.listdir(os.path.join(cdir,directory,'json')):
        if files.endswith('.json'):
            data=readfromjson(os.path.join(cdir,directory,'json',files))
            xml=json2xml.Json2xml(data).to_xml()
            name=files.replace('.json','.xml')
            txt_file= open(os.path.join(cdir,directory,outdir,name),'w')
            txt_file.write(xml)
            txt_file.close()


# -*- coding: utf-8 -*-
"""
Created on Sun Dec  1 14:07:08 2019

@author: vamsi
"""
curpath = r'C:\Users\vamsi\OneDrive\Documents\fwdaudiotoaudiocomparision'
import os
os.chdir(curpath)
video = r'dnc-2004-speech.mp4'  # Path where the videos are located
import moviepy.editor as mp
clip = mp.VideoFileClip(video).subclip(0,20)
clip.audio.write_audiofile("theaudio.wav")

import speech_recognition as sr
r = sr.Recognizer()

hellow=sr.AudioFile('theaudio.wav')
with hellow as source:
    audio = r.record(source)
try:
    s = r.recognize_google(audio)
    print("Text: "+s)
    file = open('data.txt','w')
    file.write(s)
    file.close()
except Exception as e:
    print("Exception: "+str(e))

import spacy
nlp = spacy.load('en_core_web_lg')

search_doc = nlp(s)

main_doc = nlp(s)

print(main_doc.similarity(search_doc))