from datetime import datetime
import logging
import json

from django.conf import settings

from requests.auth import HTTPBasicAuth
from email.header import Header
from email.utils import formataddr
from api.models.user_profile import UserProfile
from api.models.notification import Notification
from api.models.credit_transfer_statuses import CreditTransferStatuses
from api.models.sales_submission_statuses import SalesSubmissionStatuses
from api.models.model_year_report_statuses import ModelYearReportStatuses
from api.models.vehicle_statuses import VehicleDefinitionStatuses
from api.models.credit_agreement_statuses import CreditAgreementStatuses
from api.models.notification_subscription import NotificationSubscription
from api.models.organization import Organization
import requests
from django.db.models import Q
from typing import List

LOGGER = logging.getLogger(__name__)

def get_email_service_token() -> dict:
    """
    Retrieves an authentication token from the Common Hosted Email Service (CHES) API.

    This method sends a request to the CHES authentication URL to obtain an access token
    using the client_id and client_secret from the settings. The token is required to send
    emails through the CHES API.

    :return: A dictionary containing the token information if successful, or None if an error occurs.
    """
    if settings.DEBUG:
        print('starting email service')

    client_id = settings.EMAIL['EMAIL_SERVICE_CLIENT_ID']
    client_secret = settings.EMAIL['EMAIL_SERVICE_CLIENT_SECRET']
    url = settings.EMAIL['CHES_AUTH_URL']
    if not client_id:
        LOGGER.error("Email service client id is not configured")
        return {"error": "Email service client id is not configured"}
    if not client_secret:
        LOGGER.error("Email service client secret is not configured")
        return {"error": "Email service client secret is not configured"}
    if not url:
        LOGGER.error("Common hosted email service authentication url is not configured")
        return {"error": "Common hosted email service authentication url is not configured"}
    payload = {"grant_type": "client_credentials"}
    header = {"content-type": "application/x-www-form-urlencoded"}
    try:
        if settings.DEBUG:
            print('fetching email token')

        token_rs = requests.post(
            url,
            data=payload,
            auth=HTTPBasicAuth(client_id, client_secret),
            headers=header,
            verify=True
        )
        if not token_rs.status_code == 200:
            if settings.DEBUG:
                print("Error: Unexpected response", token_rs.text.encode('utf8'))
            LOGGER.error("Error: Unexpected response", token_rs.text.encode('utf8'))
            return {"error": "Unexpected response"}
        json_obj = token_rs.json()
        return json_obj
    except requests.exceptions.RequestException as e:
        if settings.DEBUG:
            print("Error: {}".format(e))
        LOGGER.error("Error: {}".format(e))
        return {"error": str(e)}

def generate_email_body(email_type: str, test_info: dict) -> str:
    """
    Generates an email body based on the provided email_type and test_info parameters.

    Constructs an email body with the given email_type and test_info. The email content may
    differ depending on the environment (prod or non-prod). In non-prod environments, the
    email will include additional action information.

    :param email_type: A string describing the type of email being sent, e.g. "credit transfer update".
    :param test_info: A dictionary containing test information, such as user, actions, action descriptions, and time.
    :return: A string representing the email body in HTML format.
    """

    body = f"""\

    This email was generated by the Government of B.C. Zero-Emission Vehicle Reporting System.

A {email_type} has occurred within the Zero-Emission Vehicle Reporting System that may require your action or be of interest to you, please logon to the system at: https://zeroemissionvehicles.gov.bc.ca/

You received this email because you subscribed at the site above, to stop receiving these email logon to your account here https://zeroemissionvehicles.gov.bc.ca/notifications

    """

    if settings.ENV_NAME != "prod":
        action_info = "".join(
            f"""
Action: {action}

Description: {description}
"""
            for action, description in zip(test_info["actions"], test_info["action_descriptions"])
        )

        body += f"""\

User: {test_info['user']}

        {action_info}

Time: {test_info['time']}

        """
    else:
        body += """\

        """
    if settings.DEBUG:
        print('generated email body', body)

    return body

def send_email(recipient_emails: List[str], email_type: str, test_info: dict):
    """
    Sends an email to the specified recipients with the given email type and test information.

    This method constructs an email body based on the provided email_type and test_info parameters, then
    sends the email using the Common Hosted Email Service (CHES) API. The email will be sent to the
    specified recipient_emails, with the subject set as "BC ZEVA Notification". The email's content may
    differ depending on the environment and the email_type.

    :param recipient_emails: A list of recipient email addresses as strings.
    :param email_type: A string describing the type of email being sent, e.g. "credit transfer update".
    :param test_info: A dictionary containing test information, such as user, actions, action descriptions, and time.
    :return: None if the email is sent successfully, or a dictionary containing an error message if an error occurs.
    """

    if settings.DEBUG:
        print('drafting email')

    sender_email = settings.EMAIL['SENDER_EMAIL']
    sender_name = settings.EMAIL['SENDER_NAME']
    url = settings.EMAIL['CHES_EMAIL_URL']

    if not sender_email:
        LOGGER.error("Sender email address not configured")
        return
    if not url:
        LOGGER.error("CHES email url not configured")
        return
    if not sender_name:
        LOGGER.error("Sender name not configured")
        return
    if not recipient_emails:
        LOGGER.error("No recipient email address provided")
        return
    if not email_type:
        LOGGER.error("No email type provided")
        return

    body = generate_email_body(email_type, test_info)

    subject = "BC ZEVA Notification"
    bodyType = "html"
    attachment = ""

    token = get_email_service_token()
    if not token or 'access_token' not in token:
        if settings.DEBUG:
            print('access_token failed')
        LOGGER.error("No email service token provided", token)
        return
    auth_token = token['access_token']

    sender_info = formataddr((str(Header(sender_name, "utf-8")), sender_email))

    data = {
            "bcc": recipient_emails,
            "bodyType": bodyType,
            "body": body,
            "cc": [],
            "delayTS": 0,
            "encoding": "utf-8",
            "from": sender_info,
            "priority": "normal",
            "subject": subject,
            "to": ["Undisclosed recipients<donotreply@gov.bc.ca>"],
            "tag": "email_1",
            "attachments": attachment
           }

    headers = {"Authorization": 'Bearer ' + auth_token,
               "Content-Type": "application/json"}
    try:
        if settings.DEBUG:
            print('posting email request')

        response = requests.post(
            url,
            data=json.dumps(data),
            headers=headers
        )
        if not response.status_code == 201:
            if settings.DEBUG:
                print("Error: Email failed! %s", response.text.encode('utf8'))
            LOGGER.error("Error: Email failed! %s", response.text.encode('utf8'))
            return

        email_res = response.json()
        if email_res:
            if settings.DEBUG:
                print("Email sent successfully!", email_res['messages'][0]['msgId'])
            LOGGER.debug("Email sent successfully!", email_res['messages'][0]['msgId'])
            return
    except requests.exceptions.RequestException as e:
        LOGGER.error("Error: {}".format(e))
        return

def get_subscribed_user_emails(notifications, obj, request_type):
    """
    Retrieves the subscribed user emails for the given notifications, object, and request type.

    This method filters the user profiles that are subscribed to specific notifications,
    belong to certain organizations, and have valid email addresses. The obj parameter
    can be different types of objects (e.g., CreditTransfer, ModelYearReport) depending
    on the method that is calling this function.

    :param notifications: A list of notification IDs to filter the subscriptions.
    :param obj: An object that can be of different types depending on the calling method.
    :param request_type: A string that indicates the type of request ('credit_transfer', 'model_year_report', etc.).
    :return: A list of subscribed user emails.
    """
    if settings.DEBUG:
        print('starting get_subscribed_user_emails')

    govt_org = Organization.objects.filter(is_government=True).first()
    subscribed_users = NotificationSubscription.objects.values_list('user_profile_id', flat=True).filter(
        notification__id__in=notifications).filter(user_profile__is_active=True)

    if settings.DEBUG:
        print('subscribed users', subscribed_users)

    if request_type == 'credit_transfer':
        user_emails = UserProfile.objects.values_list('email', flat=True).filter(
            Q(organization_id__in=[obj.debit_from_id, obj.credit_to_id, govt_org.id]) &
            Q(id__in=subscribed_users)).exclude(email__isnull=True).exclude(email__exact='').exclude(username=obj.update_user)
    elif request_type == 'model_year_report':
        user_emails = UserProfile.objects.values_list('email', flat=True).filter(
            Q(organization_id__in=[obj.organization.id, govt_org.id]) &
            Q(id__in=subscribed_users)).exclude(email__isnull=True).exclude(email__exact='').exclude(username=obj.update_user)
    else:
        user_emails = UserProfile.objects.values_list('email', flat=True).filter(
            Q(organization_id__in=[obj.organization, govt_org.id]) &
            Q(id__in=subscribed_users)).exclude(email__isnull=True).exclude(email__exact='').exclude(username=obj.update_user)

    if settings.DEBUG:
        print('get_subscribed_user_emails', user_emails)

    return user_emails

def get_notification_objects(notifications):
    """
    Retrieves the Notification objects for the given list of notification IDs.

    :param notifications: A list of notification IDs.
    :return: A QuerySet of Notification objects.
    """
    return Notification.objects.filter(id__in=notifications)

def prepare_test_info(request, notification_objects):
    """
    Prepares a dictionary containing test information based on the given request and notification objects.

    This method extracts user, action names, action descriptions, and the current time from the provided
    parameters and stores them in a dictionary.

    :param request: A request object containing user information.
    :param notification_objects: A QuerySet of Notification objects.
    :return: A dictionary containing test information.
    """
    if settings.DEBUG:
        print('preparing test info')

    test_info = {}
    try:
        test_info['user'] = request.update_user
    except AttributeError as error:
        test_info['user'] = request.user.update_user

    test_info['actions'] = []
    test_info['action_descriptions'] = []
    for object in notification_objects:
        test_info['actions'].append(object.name)
        test_info['action_descriptions'].append(object.description)
    test_info['time'] = datetime.now().strftime("%d/%m/%Y %H:%M:%S")

    if settings.DEBUG:
        print('test info', test_info)

    return test_info

def notifications_credit_transfers(transfer: object, transfer_sent_back_to_analyst: bool, transfer_saved_by_analyst: bool):
    """
    Handles notifications for credit transfer events based on the transfer status.

    :param transfer: The credit transfer object with its current status.
    :param transfer_sent_back_to_analyst: whether the transfer was sent back to the analyst
    :param transfer_saved_by_analyst: whether the transfer was saved by the analyst
    """
    validation_status = transfer.status
    notifications = None
    if validation_status == CreditTransferStatuses.VALIDATED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='CREDIT_TRANSFER_RECORDED_GOVT') |
            Q(notification_code='CREDIT_TRANSFER_RECORDED'))

    elif validation_status == CreditTransferStatuses.SUBMITTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_TRANSFER_SUBMITTED')

    elif validation_status == CreditTransferStatuses.RECOMMEND_APPROVAL:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_TRANSFER_RECOMMEND_APPROVAL')

    elif validation_status == CreditTransferStatuses.RECOMMEND_REJECTION:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_TRANSFER_RECOMMEND_REJECT')

    elif validation_status == CreditTransferStatuses.APPROVED:
        if transfer_sent_back_to_analyst:
            notifications = Notification.objects.values_list('id', flat=True).filter(
                Q(notification_code='CREDIT_TRANSFER_APPROVED'))
        elif not transfer_saved_by_analyst:
            notifications = Notification.objects.values_list('id', flat=True).filter(
                Q(notification_code='CREDIT_TRANSFER_APPROVED') |
                Q(notification_code='CREDIT_TRANSFER_APPROVED_PARTNER'))

    elif validation_status == CreditTransferStatuses.DISAPPROVED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_TRANSFER_REJECTED_PARTNER')

    elif validation_status == CreditTransferStatuses.RESCINDED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='CREDIT_TRANSFER_RESCINDED') |
            Q(notification_code='CREDIT_TRANSFER_RESCINDED_PARTNER'))

    elif validation_status == CreditTransferStatuses.RESCIND_PRE_APPROVAL:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_TRANSFER_RESCINDED_PARTNER')

    elif validation_status == CreditTransferStatuses.REJECTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='CREDIT_TRANSFER_REJECTED_GOVT') |
            Q(notification_code='CREDIT_TRANSFER_REJECTED'))

    if settings.DEBUG:
        print('notifications_credit_transfers', notifications)

    if notifications:
        send_credit_transfer_emails(notifications, transfer)

def notifications_model_year_report(updated_model_year_report, report_returned_to_supplier):
    """
    Handles notifications for model year report events based on the validation status.

    :param updated_model_year_report: The new/updated model year report.
    :param report_returned_to_supplier: Whether the report was returned to the supplier.
    """
    validation_status = updated_model_year_report.validation_status
    notifications = None
    if validation_status == ModelYearReportStatuses.ASSESSED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='MODEL_YEAR_REPORT_ASSESSED_SUPPLIER') |
            Q(notification_code='MODEL_YEAR_REPORT_ASSESSED_GOVT'))
    elif validation_status == ModelYearReportStatuses.SUBMITTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='MODEL_YEAR_REPORT_SUBMITTED')
    elif validation_status == ModelYearReportStatuses.RECOMMENDED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='MODEL_YEAR_REPORT_RECOMMENDED')
    elif validation_status == ModelYearReportStatuses.RETURNED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='MODEL_YEAR_REPORT_RETURNED')
    elif report_returned_to_supplier:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='MODEL_YEAR_REPORT_RETURNED_TO_SUPPLIER')

    if settings.DEBUG:
        print('notifications_model_year_report', notifications)

    if notifications:
        send_model_year_report_emails(notifications, updated_model_year_report)

def notifications_credit_agreement(agreement: object):
    """
    Handles notifications for credit agreement events based on the agreement status.

    :param agreement: The credit agreement object with its current status.
    """
    validation_status = agreement.status
    notifications = None
    if validation_status == CreditAgreementStatuses.ISSUED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='CREDIT_AGREEMENT_ISSUED_SUPPLIER') |
            Q(notification_code='CREDIT_AGREEMENT_ISSUED_GOVT'))
    elif validation_status == CreditAgreementStatuses.RECOMMENDED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_AGREEMENT_RECOMMENDED')
    elif validation_status == CreditAgreementStatuses.RETURNED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_AGREEMENT_RETURNED_WITH_COMMENT')

    if settings.DEBUG:
        print('notifications_credit_agreement', notifications)

    if notifications:
        send_credit_agreement_emails(notifications, agreement)

def notifications_credit_application(submission: object):
    """
    Handles notifications for credit application events based on the submission status.

    :param submission: The credit application submission object with its current status.
    """
    validation_status = submission.validation_status
    notifications = None
    if validation_status == SalesSubmissionStatuses.VALIDATED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='CREDIT_APPLICATION_ISSUED') |
            Q(notification_code='CREDIT_APPLICATION_ISSUED_GOVT'))

    elif validation_status == SalesSubmissionStatuses.CHECKED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_APPLICATION_CHECKED')

    elif validation_status == SalesSubmissionStatuses.RECOMMEND_APPROVAL:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_APPLICATION_RECOMMEND_APPROVAL')

    elif validation_status == SalesSubmissionStatuses.SUBMITTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='CREDIT_APPLICATION_SUBMITTED')

    if settings.DEBUG:
        print('notifications_credit_application', notifications)

    if notifications:
        send_credit_application_emails(notifications, submission)

def notifications_zev_model(vehicle: object, validation_status: str):
    """
    Handles notifications for ZEV model events based on the validation status.

    :param vehicle: The vehicle object containing relevant data.
    :param validation_status: The validation status of the ZEV model.
    """
    notifications = None
    if validation_status == VehicleDefinitionStatuses.VALIDATED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='ZEV_MODEL_VALIDATED')

    elif validation_status == VehicleDefinitionStatuses.REJECTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='ZEV_MODEL_REJECTED')

    elif validation_status == VehicleDefinitionStatuses.SUBMITTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            Q(notification_code='ZEV_MODEL_SUBMITTED') |
            Q(notification_code='ZEV_MODEL_RANGE_REPORT_SUBMITTED'))

    elif validation_status == VehicleDefinitionStatuses.CHANGES_REQUESTED:
        notifications = Notification.objects.values_list('id', flat=True).filter(
            notification_code='ZEV_MODEL_RANGE_REPORT_TEST_RESULT_REQUESTED')

    if settings.DEBUG:
        print('notifications_zev_model', notifications)

    if notifications:
        send_zev_model_emails(notifications, vehicle)

def send_credit_transfer_emails(notifications, transfer):
    """
    Sends emails to subscribed users with credit transfer updates.

    :param notifications: Notification objects for the credit transfer event.
    :param transfer: The credit transfer object.
    """
    if settings.DEBUG:
        print('send_credit_transfer_emails')

    request_type = 'credit_transfer'
    user_emails = get_subscribed_user_emails(notifications, transfer, request_type)
    notification_objects = get_notification_objects(notifications)
    test_info = prepare_test_info(transfer, notification_objects)

    if user_emails:
        email_type = 'credit transfer update'
        send_email(list(user_emails), email_type, test_info)

def send_credit_application_emails(notifications, submission):
    """
    Sends emails to subscribed users with credit application updates.

    :param notifications: Notification objects for the credit application event.
    :param submission: The credit application submission object.
    """
    if settings.DEBUG:
        print('send_credit_application_emails')

    request_type = 'credit_application'
    user_emails = get_subscribed_user_emails(notifications, submission, request_type)
    notification_objects = get_notification_objects(notifications)
    test_info = prepare_test_info(submission, notification_objects)

    if user_emails:
        email_type = 'credit application update'
        send_email(list(user_emails), email_type, test_info)

def send_zev_model_emails(notifications, vehicle):
    """
    Sends emails to subscribed users with ZEV model updates.

    :param notifications: Notification objects for the ZEV model event.
    :param request: The request object containing relevant data.
    """
    if settings.DEBUG:
        print('send_zev_model_emails')

    request_type = 'zev_model'
    user_emails = get_subscribed_user_emails(notifications, vehicle, request_type)
    notification_objects = get_notification_objects(notifications)
    test_info = prepare_test_info(vehicle, notification_objects)

    if user_emails:
        email_type = 'ZEV model update'
        send_email(list(user_emails), email_type, test_info)

def send_model_year_report_emails(notifications, model_year_report):
    """
    Sends emails to subscribed users with model year report updates.

    :param notifications: Notification objects for the model year report event.
    :param request: The request object containing relevant data.
    """
    if settings.DEBUG:
        print('send_model_year_report_emails')

    request_type = 'model_year_report'
    user_emails = get_subscribed_user_emails(notifications, model_year_report, request_type)
    notification_objects = get_notification_objects(notifications)
    test_info = prepare_test_info(model_year_report, notification_objects)

    if user_emails:
        email_type = 'model year report update'
        send_email(list(user_emails), email_type, test_info)

def send_credit_agreement_emails(notifications, agreement):
    """
    Sends emails to subscribed users with credit agreement updates.

    :param notifications: Notification objects for the credit agreement event.
    :param agreement: The credit agreement object.
    """
    if settings.DEBUG:
        print('send_credit_agreement_emails')

    request_type = 'credit_agreement'
    user_emails = get_subscribed_user_emails(notifications, agreement, request_type)
    notification_objects = get_notification_objects(notifications)
    test_info = prepare_test_info(agreement, notification_objects)

    if user_emails:
        email_type = 'credit agreement update'
        send_email(list(user_emails), email_type, test_info)


import urllib.parse
import urllib.request
import urllib.response
import json

def get_metadata(chosen_title: str) -> dict:
    """
    Render resource in JSON format,
    convert JSON string to Pythonic object
    """
    url = f"http://dbpedia.org/data/{chosen_title}.json"
    # format chosen title into URL
    request = urllib.request.urlopen(url)
    request = request.read()
    request_dictionary = json.loads(request)
    for key, value in request_dictionary.items():
        if key == f"http://dbpedia.org/resource/{chosen_title}":
            # this is the "master key"
            metadata_dictionary = value
            # the value of this key is again a dictionary
    return metadata_dictionary

def get_abstract(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return abstract of chosen title in English
    """
    if "http://dbpedia.org/ontology/abstract" in metadata_dictionary:
        # account for the possibility of an abstract not being available
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/abstract":
                # the abstract can be found under this key
                list_of_abstracts = value
                # the value of this key is a list of dictionaries
                for abstract_dictionary in list_of_abstracts:
                    for key, value in abstract_dictionary.items():
                        # each dictionary contains two keys:
                        # "value": the actual abstract
                        # "lang": the language of the abstract
                        if key == "lang":
                            language = value
                            if language == "en":
                                # extract only the abtract in English
                                for key, value in abstract_dictionary.items():
                                    if key == "value":
                                        abstract = value
                                        print("\n")
                                        print("Abstract", ":", abstract)
        return abstract
    else:
        print("\n")
        print("Abstract: Abstract not found.")

def get_publication_date(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return publication date of chosen title
    """
    if "http://dbpedia.org/ontology/publicationDate" in metadata_dictionary:
        # account for the possibility of a publication date not being available
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/publicationDate":
                # the publication date can be found under this key
                list_of_publication_dates = value
                # the value of this key is a list of dictionaries
                for publication_date_dictionary in list_of_publication_dates:
                    for key, value in publication_date_dictionary.items():
                        if key == "value":
                            # the actual publication date can again be found under a key named "value"
                            publication_date = value
                            print("\n")
                            print("Publication date", ":", publication_date)
        return publication_date
    else:
        print("\n")
        print("Publication date: Publication date not found.")

def get_author(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return author of chosen title
    """
    if "http://dbpedia.org/ontology/author" in metadata_dictionary:
        # account for the possibility of an author being unknown
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/author":
                # the author URI can be found under this key
                list_of_authors = value
                # the value of this key is a list of dictionaries
                for author_dictionary in list_of_authors:
                    for key, value in author_dictionary.items():
                        if key == "value":
                            # the actual author URI can again be found under a key named "value"
                            author_URI = value
                            author = author_URI.rsplit('/', 1)[-1]
                            # extract the name of the author by splitting the URI at the last slash
                            # e.g. http://dbpedia.org/resource/J._K._Rowling -> J._K._Rowling
                            author_with_spaces = author.replace("_", " ")
                            # replace all underscores by spaces
                            print("\n")
                            print("Author", ":", author_with_spaces)
        return author
        # return the name of the author with underscores instead of spaces
    else:
        print("\n")
        print("Author: Author unkown.")


from .command import Command
import itertools
from tabulate import tabulate

class OutboxCommand(Command):
    def __init__(self, args):
        super().__init__(args)
        self.offset = args.offset
        self.limit = args.limit

    def run(self):
        actor = self.logged_in_actor()
        if actor is None:
            raise Exception("Not logged in")
        outbox = actor.get("outbox", None)
        if outbox is None:
            raise Exception("No outbox found")
        outbox_id = self.to_id(outbox)
        slice = itertools.islice(
            self.items(outbox_id), self.offset, self.offset + self.limit
        )
        rows = []
        for item in slice:
            id = self.to_id(item)
            type = item.get("type", None)
            summary = self.text_prop(item, "summary")
            published = item.get("published")
            rows.append([id, type, summary, published])
        print(tabulate(rows, headers=["id", "type", "summary", "published"]))


import sqlite3
from flask import Flask, render_template, request, jsonify
import base64
import matplotlib.pyplot as plt
import seaborn as sns
import cv2
import pandas as pd
from io import BytesIO
import base64
import os

app = Flask(__name__)
# connexion à la bdd
def get_db_connection():
    conn = sqlite3.connect('database.db')
   # conn.row_factory = sqlite3.Row
    return conn

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/login', methods=['post'])
def login():
    mail=request.form['mail']
    mdp=request.form['mdp']
    conn = get_db_connection()
    user = conn.execute(f"SELECT * FROM user where adressemail='{mail}' and mdp='{mdp}'").fetchone()
    conn.close()
    if user:
        img = create_figure()
        plot_url = base64.b64encode(img.getvalue())
        print(plot_url)

        img1 = create_data_viz()
        plot_url1 = base64.b64encode(img1.getvalue())
        print(plot_url1)

        img2 = create_data_viz1()
        plot_url2 = base64.b64encode(img2.getvalue())
        print(plot_url2)

        img3 = create_data_viz2()
        plot_url3 = base64.b64encode(img3.getvalue())
        print(plot_url3)

        img4 = create_data_viz3()
        plot_url4 = base64.b64encode(img4.getvalue())
        print(plot_url4)

        return render_template('home.html', plot_url=plot_url.decode(), plot_url1=plot_url1.decode(), plot_url2=plot_url2.decode(), plot_url3=plot_url3.decode(), plot_url4=plot_url4.decode())
    else:
       return render_template('index.html')

def create_data_viz():
    DATASET_PATH = './data_viz/'
    styles = pd.read_csv(os.path.join(DATASET_PATH, "styles.csv"), error_bad_lines=False)

    # Afficher le graphique en utilisant le type de graphique camembert
    sns.catplot(x="gender", kind="count", data=styles, aspect=1.5)

    img1 = BytesIO()

    plt.savefig(img1, format='png')
    plt.close()
    img1.seek(0)
    return img1

def create_data_viz1():
    DATASET_PATH = './data_viz/'
    styles = pd.read_csv(os.path.join(DATASET_PATH, "styles.csv"), error_bad_lines=False)

    # Sélectionner les colonnes souhaitées
    colonnes_voulues = ['gender', 'articleType']
    df_filtre = styles[colonnes_voulues]

    # Filtrer les lignes selon une colonne particulière (ici "gender")
    df_filtre = df_filtre[df_filtre['gender'] == 'Men']

    # Compter le nombre d'occurrences de chaque articleType pour les articles masculins
    comptes = df_filtre['articleType'].value_counts()

    colors = sns.color_palette('bright')

    # Filtrer les comptes pour n'afficher que les pourcentages plus de 5%
    comptes = comptes[comptes / comptes.sum() * 100 > 5]

    plt.pie(comptes, labels=comptes.index, colors=colors, autopct='%0.0f%%')
    plt.legend(loc='right', bbox_to_anchor=(1.25, 0.5), labelspacing=1.5)
    plt.title("La répartition des vêtements pour Homme")

    img2 = BytesIO()

    plt.savefig(img2, format='png')
    plt.close()
    img2.seek(0)
    return img2

def create_data_viz2():
    DATASET_PATH = './data_viz/'
    styles = pd.read_csv(os.path.join(DATASET_PATH, "styles.csv"), error_bad_lines=False)

    # Sélectionner les colonnes souhaitées
    colonnes_voulues = ['gender', 'articleType']
    df_filtre = styles[colonnes_voulues]

    # Filtrer les lignes selon une colonne particulière (ici "gender")
    df_filtre = df_filtre[df_filtre['gender'] == 'Women']

    # Compter le nombre d'occurrences de chaque articleType pour les articles masculins
    comptes = df_filtre['articleType'].value_counts()

    colors = sns.color_palette('bright')

    # Filtrer les comptes pour n'afficher que les pourcentages plus de 5%
    comptes = comptes[comptes / comptes.sum() * 100 > 5]

    plt.pie(comptes, labels=comptes.index, colors=colors, autopct='%0.0f%%')
    plt.legend(loc='right', bbox_to_anchor=(1.25, 0.5), labelspacing=1.5)
    plt.title("La répartition des vêtements pour les Femme")

    img3 = BytesIO()

    plt.savefig(img3, format='png')
    plt.close()
    img3.seek(0)
    return img3

def create_data_viz3():
    DATASET_PATH = './data_viz/'
    styles = pd.read_csv(os.path.join(DATASET_PATH, "styles.csv"), error_bad_lines=False)

    #sns.catplot(x='gender', y='year', data=styles, hue='season')
    sns.pairplot(styles, hue='gender')

    img4 = BytesIO()

    plt.savefig(img4, format='png')
    plt.close()
    img4.seek(0)
    return img4

def create_figure():
    # Charger l'image
    img = cv2.imread("dress.jpg")

    # Convertir l'image en espace de couleurs HSV
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    # Définir les limites de couleurs à détecter (en utilisant les valeurs HSV)
    lower_blue = (110, 50, 50)
    upper_blue = (130, 255, 255)

    lower_red = (0, 50, 50)
    upper_red = (10, 255, 255)

    lower_green = (50, 100, 100)
    upper_green = (70, 255, 255)

    lower_orange = (10, 50, 50)
    upper_orange = (20, 255, 255)

    lower_yellow = (20, 50, 50)
    upper_yellow = (30, 255, 255)

    lower_teal = (70, 100, 100)
    upper_teal = (90, 255, 255)

    lower_purple = (130, 50, 50)
    upper_purple = (140, 255, 255)

    lower_pink = (140, 50, 50)
    upper_pink = (160, 255, 255)

    lower_brown = (20, 100, 100)
    upper_brown = (30, 255, 255)

    lower_gray = (0, 0, 50)
    upper_gray = (179, 50, 220)

    lower_black = (0, 0, 0)
    upper_black = (179, 255, 30)

    lower_white = (0, 0, 221)
    upper_white = (179, 30, 255)

    # Appliquer la détection de couleurs
    mask_red = cv2.inRange(hsv, lower_red, upper_red)
    res_red1 = cv2.bitwise_and(img, img, mask=mask_red)

    mask_orange = cv2.inRange(hsv, lower_orange, upper_orange)
    res_orange = cv2.bitwise_and(img, img, mask=mask_orange)

    mask_yellow = cv2.inRange(hsv, lower_yellow, upper_yellow)
    res_yellow = cv2.bitwise_and(img, img, mask=mask_yellow)

    mask_green = cv2.inRange(hsv, lower_green, upper_green)
    res_green = cv2.bitwise_and(img, img, mask=mask_green)

    mask_teal = cv2.inRange(hsv, lower_teal, upper_teal)
    res_teal = cv2.bitwise_and(img, img, mask=mask_teal)

    mask_blue = cv2.inRange(hsv, lower_blue, upper_blue)
    res_blue = cv2.bitwise_and(img, img, mask=mask_blue)

    mask_purple = cv2.inRange(hsv, lower_purple, upper_purple)
    res_purple = cv2.bitwise_and(img, img, mask=mask_purple)

    mask_pink = cv2.inRange(hsv, lower_pink, upper_pink)
    res_pink = cv2.bitwise_and(img, img, mask=mask_pink)

    mask_brown = cv2.inRange(hsv, lower_brown, upper_brown)
    res_brown = cv2.bitwise_and(img, img, mask=mask_brown)

    mask_gray = cv2.inRange(hsv, lower_gray, upper_gray)
    res_gray = cv2.bitwise_and(img, img, mask=mask_gray)

    mask_black = cv2.inRange(hsv, lower_black, upper_black)
    res_black = cv2.bitwise_and(img, img, mask=mask_black)

    mask_white = cv2.inRange(hsv, lower_white, upper_white)
    res_white = cv2.bitwise_and(img, img, mask=mask_white)

    # Compter les pixels de chaque couleur dans l'image
    red_pixels = cv2.countNonZero(mask_red)
    orange_pixels = cv2.countNonZero(mask_orange)
    yellow_pixels = cv2.countNonZero(mask_yellow)
    green_pixels = cv2.countNonZero(mask_green)
    teal_pixels = cv2.countNonZero(mask_teal)
    blue_pixels = cv2.countNonZero(mask_blue)
    purple_pixels = cv2.countNonZero(mask_purple)
    pink_pixels = cv2.countNonZero(mask_pink)
    brown_pixels = cv2.countNonZero(mask_brown)
    gray_pixels = cv2.countNonZero(mask_gray)
    black_pixels = cv2.countNonZero(mask_black)
    white_pixels = cv2.countNonZero(mask_white)

    # Créer un dataframe avec les données de couleur
    data = {'color': ['red', 'orange', 'yellow', 'green', 'teal', 'blue', 'purple', 'pink', 'brown', 'gray', 'black',
                      'white'],
            'pixels': [red_pixels, orange_pixels, yellow_pixels, green_pixels, teal_pixels, blue_pixels, purple_pixels,
                       pink_pixels, brown_pixels, gray_pixels, black_pixels, white_pixels]}
    df = pd.DataFrame(data)

    img = BytesIO()

    # Créer un graphique à barres avec Seaborn de détection de couleur
    sns.barplot(x='color', y='pixels', data=df)

    # nuage de point
    #sns.relplot(data=df, x='color', y='pixels', kind='scatter', col='color')
    #sns.pairplot(df, hue='color')

    plt.savefig(img, format='png')
    plt.close()
    img.seek(0)
    return img

@app.route('/upload', methods=['POST'])
def upload():
    # Récupération de l'image téléchargée
    image = request.files['image']
    # Enregistrement de l'image dans un dossier local
    image.save(os.path.join('uploads', image.filename))
    # connexion à la bdd
    conn = get_db_connection()
    # Insertion des informations de l'image dans la table
    conn.execute("INSERT INTO images (nom_image, path) VALUES (?,?)", (image.filename, os.path.join('uploads', image.filename)))
    # Enregistrement des modifications et fermeture de la connexion
    conn.commit()
    conn.close()
    return jsonify({'message': 'Image uploaded successfully'})

if __name__ == '__main__':
    app.run()


import collections
import logging
import random
from typing import Tuple, List

import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor as T
from torch import nn
import pdb

from dpr.data.biencoder_data import BiEncoderSample
from dpr.utils.data_utils import Tensorizer
from dpr.utils.model_utils import CheckpointState

logger = logging.getLogger(__name__)

BiEncoderBatch = collections.namedtuple(
    "BiENcoderInput",
    [
        "question_ids",
        "question_segments",
        "context_ids",
        "ctx_segments",
        "is_positive",
        "hard_negatives",
        "encoder_type",
        "questions",
        "positive_contexts",
        "negative_contexts",
        "all_contexts",
    ],
)
# TODO: it is only used by _select_span_with_token. Move them to utils
rnd = random.Random(0)

def dot_product_scores(q_vectors: T, ctx_vectors: T) -> T:
    """
    calculates q->ctx scores for every row in ctx_vector
    :param q_vector:
    :param ctx_vector:
    :return:
    """
    # q_vector: n1 x D, ctx_vectors: n2 x D, result n1 x n2
    print('ctx_vectors.shape', ctx_vectors.shape)
    r = torch.matmul(torch.transpose(q_vectors, 1, 2), ctx_vectors)
    return r

from transformers.modeling_bert import BertEncoder, BertConfig, BertModel

class BiEncoder(nn.Module):
    """Bi-Encoder model component. Encapsulates query/question and context/passage encoders."""

    def __init__(
        self,
        config,
        question_model: nn.Module,
        ctx_model: nn.Module,
        fix_q_encoder: bool = False,
        fix_ctx_encoder: bool = False,

    ):
        self.config = config
        super(BiEncoder, self).__init__()
        self.question_model = question_model
        self.ctx_model = ctx_model
        self.fix_q_encoder = fix_q_encoder
        self.fix_ctx_encoder = fix_ctx_encoder
        trans_layer = nn.TransformerEncoderLayer(768, 12)  # 12是head的个数
        self.trans_encoder = nn.TransformerEncoder(trans_layer, 1)
        cfg = BertConfig.from_pretrained("bert-base-uncased")
        # cfg = BertConfig.from_pretrained("/data/home/scv2223/archive/bert-base-uncased", local_files_only=True)

        cfg.num_hidden_layers = 1
        # self.encoder = BertEncoder(cfg)
        self.encoder = BertModel(cfg)
        self.dense = nn.Linear(768 * 2, 768)
        self.final_dense = nn.Linear(768, 1)

    @staticmethod
    def get_representation(
        sub_model: nn.Module,
        ids: T,
        segments: T,
        attn_mask: T,
        fix_encoder: bool = False,
        representation_token_pos=0,
    ) -> (T, T, T):
        sequence_output = None
        pooled_output = None
        hidden_states = None
        if ids is not None:
            if fix_encoder:
                with torch.no_grad():
                    sequence_output, pooled_output, hidden_states = sub_model(
                        ids,
                        segments,
                        attn_mask,
                        representation_token_pos=representation_token_pos,
                    )

                if sub_model.training:
                    sequence_output.requires_grad_(requires_grad=True)
                    pooled_output.requires_grad_(requires_grad=True)
            else:
                sequence_output, pooled_output, hidden_states = sub_model(
                    ids,
                    segments,
                    attn_mask,
                    representation_token_pos=representation_token_pos,
                )

        return sequence_output, pooled_output, hidden_states

    @staticmethod
    def get_scores(q_vector: T, ctx_vectors: T) -> T:
        f = BiEncoderNllLoss.get_similarity_function()
        return f(q_vector, ctx_vectors)

    def forward(
        self,
        question_ids: T,
        question_segments: T,
        question_attn_mask: T,
        context_ids: T,
        ctx_segments: T,
        ctx_attn_mask: T,
        encoder_type: str = None,
        representation_token_pos=0,
    ) -> Tuple[T, T]:
        # question_ids 和 context_ids 都是加了cls和sep的
        q_encoder = (
            self.question_model
            if encoder_type is None or encoder_type == "question"
            else self.ctx_model
        )
        _q_seq, q_pooled_out, _q_hidden = self.get_representation(
            q_encoder,
            question_ids,
            question_segments,
            question_attn_mask,
            self.fix_q_encoder,
            representation_token_pos=representation_token_pos,
        )
        # todo: 检查question_segments和ctx_segments是什么情况；

        ctx_encoder = (
            self.ctx_model
            if encoder_type is None or encoder_type == "ctx"
            else self.question_model
        )
        _ctx_seq, ctx_pooled_out, _ctx_hidden = self.get_representation(
            ctx_encoder, context_ids, ctx_segments, ctx_attn_mask, self.fix_ctx_encoder
        )  # q_pooled_out: 8 x 768  ctx_pooled_out: 16 x 768

        extended_question_ids = self.double_question(question_ids)

        extended_q_seq = self.double_question(_q_seq)

        # ctx_attn_mask = ctx_attn_mask.unsqueeze(-1)
        extended_question_attn_mask = self.double_question(question_attn_mask)
        # last_question_attn_mask = last_question_attn_mask.unsqueeze(-1)

        # """
       # 再加上全局 position embeddings 和 type embeddings
        q_token_type_ids = torch.zeros_like(extended_question_ids)
        c_token_type_ids = torch.ones_like(context_ids)
        x_type_ids = torch.cat([q_token_type_ids, c_token_type_ids], dim=1).long()
        x_attn_mask = torch.cat([extended_question_attn_mask, ctx_attn_mask], dim=1)
       # """

        x = torch.cat([extended_q_seq, _ctx_seq], dim=1)

        encoder_outputs = self.encoder(inputs_embeds=x, token_type_ids=x_type_ids, attention_mask=x_attn_mask)
        sequence_output = encoder_outputs[0]

        avg_pooled = sequence_output.mean(1)
        max_pooled = torch.max(sequence_output, dim=1)
        pooled = torch.cat((avg_pooled, max_pooled[0]), dim=1)
        pooled = self.dense(pooled)

        predictions = self.final_dense(pooled)

        return predictions

    def double_question(self, quesitons):
        """
        之前是一个问题对应两个context(正负例)，现在要把每个问题扩展成两个，也就是一个问题对应一个context
        """
        cat_result = []
        for single_q in quesitons:
            single_cat_q = torch.cat([single_q.unsqueeze(0)]*2)
            cat_result.append(single_cat_q)
        cat_result = torch.cat(cat_result, dim=0)
        return cat_result

    @classmethod
    def create_biencoder_input2(
        cls,
        samples: List[BiEncoderSample],
        tensorizer: Tensorizer,
        insert_title: bool,
        num_hard_negatives: int = 0,
        num_other_negatives: int = 0,
        shuffle: bool = True,
        shuffle_positives: bool = False,
        hard_neg_fallback: bool = True,
        query_token: str = None,
    ) -> BiEncoderBatch:
        """
        Creates a batch of the biencoder training tuple.
        :param samples: list of BiEncoderSample-s to create the batch for
        :param tensorizer: components to create model input tensors from a text sequence
        :param insert_title: enables title insertion at the beginning of the context sequences
        :param num_hard_negatives: amount of hard negatives per question (taken from samples' pools)
        :param num_other_negatives: amount of other negatives per question (taken from samples' pools)
        :param shuffle: shuffles negative passages pools
        :param shuffle_positives: shuffles positive passages pools
        :return: BiEncoderBatch tuple
        """
        question_tensors = []
        ctx_tensors = []
        positive_ctx_indices = []
        hard_neg_ctx_indices = []
        questions = []
        positive_contexts = []
        negative_contexts = []
        all_contexts = []

        for sample in samples:
            # ctx+ & [ctx-] composition
            # as of now, take the first(gold) ctx+ only

            if shuffle and shuffle_positives:
                positive_ctxs = sample.positive_passages
                positive_ctx = positive_ctxs[np.random.choice(len(positive_ctxs))]
            else:
                positive_ctx = sample.positive_passages[0]

            positive_contexts.append(positive_ctx.text)
            all_contexts.append(positive_ctx.text)

            neg_ctxs = sample.negative_passages
            hard_neg_ctxs = sample.hard_negative_passages

            # question = normalize_question(sample.query)

            if shuffle:
                random.shuffle(neg_ctxs)
                random.shuffle(hard_neg_ctxs)

            if hard_neg_fallback and len(hard_neg_ctxs) == 0:
                hard_neg_ctxs = neg_ctxs[0:num_hard_negatives]

            neg_ctxs = neg_ctxs[0:num_other_negatives]
            hard_neg_ctxs = hard_neg_ctxs[0:num_hard_negatives]
            negative_contexts.append(hard_neg_ctxs[0].text)
            all_contexts.append(hard_neg_ctxs[0].text)

            all_ctxs = [positive_ctx] + neg_ctxs + hard_neg_ctxs
            hard_negatives_start_idx = 1
            hard_negatives_end_idx = 1 + len(hard_neg_ctxs)

            current_ctxs_len = len(ctx_tensors)

            question = sample.query
            questions.append(question)
            # question = cls.build_question_with_special_tokens(question)  # 加上cls标签

            sample_ctxs_tensors = [
                tensorizer.text_to_tensor(
                    ctx.text, title=ctx.title if (insert_title and ctx.title) else None
                )
                for ctx in all_ctxs
            ]

            ctx_tensors.extend(sample_ctxs_tensors)
            positive_ctx_indices.append(current_ctxs_len)
            hard_neg_ctx_indices.append(
                [
                    i
                    for i in range(
                        current_ctxs_len + hard_negatives_start_idx,
                        current_ctxs_len + hard_negatives_end_idx,
                    )
                ]
            )   # 后面好像没用到negative的例子, 但是感觉不能全部注释掉，因为原来的all_ctxs里面也是包含negative的例子的

            if query_token:
                # TODO: tmp workaround for EL, remove or revise
                if query_token == "[START_ENT]":
                    query_span = _select_span_with_token(
                        question, tensorizer, token_str=query_token
                    )
                    question_tensors.append(query_span)
                else:
                    question_tensors.append(
                        tensorizer.text_to_tensor(" ".join([query_token, question]))
                    )
            else:
                question_tensors.append(tensorizer.text_to_tensor(question)) # 看看这里要怎么和cxt连接在一起 看dl4macro那个项目吧

        ctxs_tensor = torch.cat([ctx.view(1, -1) for ctx in ctx_tensors], dim=0) # ctx 是一维的256，ctxs_tensor是(32, 256), 竖着拼接  (batch_size为16的时候
        questions_tensor = torch.cat([q.view(1, -1) for q in question_tensors], dim=0)  # (16, 256)  (batch_size为16

        ctx_segments = torch.zeros_like(ctxs_tensor)
        question_segments = torch.zeros_like(questions_tensor)  # 这里的是不是应该设置成ones_like. 嗷嗷，应该也不用，因为前面的是对偶结构

        return BiEncoderBatch(
            questions_tensor,
            question_segments,
            ctxs_tensor,
            ctx_segments,
            positive_ctx_indices,
            hard_neg_ctx_indices,
            "question",
            questions,
            positive_contexts,
            negative_contexts,
            all_contexts
        )

    @classmethod
    def build_question_with_special_tokens(
            cls, input
    ) -> str:
        if isinstance(input, str):
            return '[CLS]' + input + '[SEP]'
        if isinstance(input, list):
            context_list = [ctx.text + ctx.title for ctx in input]
            return '[CLS]'+''.join(context_list) + '[SEP]'

    def load_state(self, saved_state: CheckpointState):
        # TODO: make a long term HF compatibility fix
        if "question_model.embeddings.position_ids" in saved_state.model_dict:
            del saved_state.model_dict["question_model.embeddings.position_ids"]
            del saved_state.model_dict["ctx_model.embeddings.position_ids"]
        self.load_state_dict(saved_state.model_dict, strict=False)

    def get_state_dict(self):
        return self.state_dict()

class BiEncoderNllLoss(object):
    def calc(
        self,
        input,
        linear_output: T,
        # ctx_vectors: T,
        positive_idx_per_question: list,
        hard_negative_idx_per_question: list = None,
        loss_scale: float = None,
    ) -> Tuple[T, int]:
        """
        # hard_negative_idx_per_question怎么用上的呢
        Computes nll loss for the given lists of question and ctx vectors.
        Note that although hard_negative_idx_per_question in not currently in use, one can use it for the
        loss modifications. For example - weighted NLL with different factors for hard vs regular negatives.
        :return: a tuple of loss value and amount of correct predictions per batch
        """
        # scores = self.get_scores(q_vectors, ctx_vectors)  # 计算相似度 q和c的dot  q:16 x 768, c: 32 x 768

        q_num = linear_output.size(0)
        scores = linear_output.view(q_num, -1)  # [4,2]

        #### manual test 下面的代码 ###
        # questions = input.questions
        # all_contexts = input.all_contexts
        # negative_contexts = input.negative_contexts
        # positive_contexts = input.positive_contexts
        sigmoid_scores = torch.sigmoid(scores)  # [4, 2]
        #### manual test ###

        prediction_scores = scores.view(-1)
        new_positive_idx_per_question = [float(0)]*q_num
        for i in positive_idx_per_question:
            new_positive_idx_per_question[i] = float(1)

        # loss = F.nll_loss(
        #     softmax_scores,
        #     torch.tensor(new_positive_idx_per_question).to(softmax_scores.device),
        #     reduction="mean",
        # )
        loss_fun = nn.BCEWithLogitsLoss()
        # loss_fun = nn.CrossEntropyLoss()
        loss = loss_fun(prediction_scores, torch.tensor(new_positive_idx_per_question).to(scores.device))
        # loss = self.compute_loss(sigmoid_scores, new_positive_idx_per_question)

        correct_predictions_count = [sigmoid_scores[index] > sigmoid_scores[index+1] for index in positive_idx_per_question if index+1 < len(sigmoid_scores)].count(True)

        # correct_predictions_count = torch.tensor([len([i for i in q_vectors if i > 0.5])])

        if loss_scale:
            loss.mul_(loss_scale)

        return loss, correct_predictions_count

    def compute_loss(self, predictions, labels):
        # 将预测和标记的维度展平, 防止出现维度不一致
        predictions = predictions.view(-1)
        labels = labels.view(-1)
        epsilon = 1e-8
        # 交叉熵
        loss =\
            - labels * torch.log(predictions + epsilon) - \
            (torch.tensor(1.0) - labels) * torch.log(torch.tensor(1.0) - predictions + epsilon)
        # 求均值, 并返回可以反传的loss
        # loss为一个实数
        loss = torch.mean(loss)
        return loss

    @staticmethod
    def get_scores(q_vector: T, ctx_vectors: T) -> T:
        f = BiEncoderNllLoss.get_similarity_function()
        return f(q_vector, ctx_vectors)

    @staticmethod
    def get_similarity_function():
        return dot_product_scores

def _select_span_with_token(
    text: str, tensorizer: Tensorizer, token_str: str = "[START_ENT]"
) -> T:
    id = tensorizer.get_token_id(token_str)
    query_tensor = tensorizer.text_to_tensor(text)

    if id not in query_tensor:
        query_tensor_full = tensorizer.text_to_tensor(text, apply_max_len=False)
        token_indexes = (query_tensor_full == id).nonzero()
        if token_indexes.size(0) > 0:
            start_pos = token_indexes[0, 0].item()
            # add some randomization to avoid overfitting to a specific token position

            left_shit = int(tensorizer.max_length / 2)
            rnd_shift = int((rnd.random() - 0.5) * left_shit / 2)
            left_shit += rnd_shift

            query_tensor = query_tensor_full[start_pos - left_shit :]
            cls_id = tensorizer.tokenizer.cls_token_id
            if query_tensor[0] != cls_id:
                query_tensor = torch.cat([torch.tensor([cls_id]), query_tensor], dim=0)

            from dpr.models.reader import _pad_to_len

            query_tensor = _pad_to_len(
                query_tensor, tensorizer.get_pad_id(), tensorizer.max_length
            )
            query_tensor[-1] = tensorizer.tokenizer.sep_token_id
            # logger.info('aligned query_tensor %s', query_tensor)

            assert id in query_tensor, "query_tensor={}".format(query_tensor)
            return query_tensor
        else:
            raise RuntimeError(
                "[START_ENT] toke not found for Entity Linking sample query={}".format(
                    text
                )
            )
    else:
        return query_tensor


"""Base work chain to run an ORCA calculation"""

# Not sure if this is needed? Can we use self.run()?
from aiida.engine import (
    ExitCode,
    ToContext,
    WorkChain,
    append_,
    if_,
    process_handler,
    run,
)
from aiida.orm import (
    Bool,
    Dict,
    Float,
    Int,
    List,
    SinglefileData,
    StructureData,
    TrajectoryData,
    to_aiida_type,
)
from aiida.plugins import CalculationFactory, DataFactory, WorkflowFactory

from .harmonic_wigner import generate_wigner_structures
from .optimization import RobustOptimizationWorkChain
from .utils import (
    ConcatInputsToList,
    add_orca_wf_guess,
    extract_trajectory_arrays,
    pick_structure_from_trajectory,
    structures_to_trajectory,
)

Code = DataFactory("core.code.installed")
OrcaCalculation = CalculationFactory("orca.orca")
OrcaBaseWorkChain = WorkflowFactory("orca.base")

class OrcaExcitationWorkChain(OrcaBaseWorkChain):
    """A simple shim for UV/vis excitation in ORCA."""

    def _build_process_label(self) -> str:
        return "Excitation workflow"

    @classmethod
    def define(cls, spec):
        super().define(spec)
        spec.output(
            "excitations",
            valid_type=Dict,
            required=True,
            help="Excitation energies and oscillator strengths from a single-point excitations",
        )

    def extract_transitions_from_orca_output(self, orca_output_params):
        return {
            "oscillator_strengths": orca_output_params["etoscs"],
            # Orca returns excited state energies in cm^-1
            # Perhaps we should do the conversion here,
            # to make this less ORCA specific.
            "excitation_energies_cm": orca_output_params["etenergies"],
        }

    @process_handler(exit_codes=ExitCode(0), priority=600)
    def add_excitation_output(self, calculation):
        """Extract excitation energies and osc. strengths into a separate output node"""
        transitions = self.extract_transitions_from_orca_output(
            calculation.outputs.output_parameters
        )
        self.out("excitations", Dict(transitions).store())

class OrcaWignerSpectrumWorkChain(WorkChain):
    """Top level workchain for Nuclear Ensemble Approach UV/vis
    spectrum for a single conformer"""

    def _build_process_label(self):
        return "NEA spectrum workflow"

    @classmethod
    def define(cls, spec):
        super().define(spec)
        spec.expose_inputs(
            RobustOptimizationWorkChain,
            namespace="opt",
            exclude=["orca.structure", "orca.code"],
        )
        spec.expose_inputs(
            OrcaExcitationWorkChain,
            namespace="exc",
            exclude=["orca.structure", "orca.code"],
        )
        spec.input("structure", valid_type=(StructureData, TrajectoryData))
        spec.input("code", valid_type=Code)

        # Whether to perform geometry optimization
        spec.input(
            "optimize",
            valid_type=Bool,
            default=lambda: Bool(True),
            serializer=to_aiida_type,
        )

        # Number of Wigner geometries (computed only when optimize==True)
        spec.input(
            "nwigner", valid_type=Int, default=lambda: Int(1), serializer=to_aiida_type
        )

        spec.input(
            "wigner_low_freq_thr",
            valid_type=Float,
            default=lambda: Float(10),
            serializer=to_aiida_type,
        )

        spec.output(
            "franck_condon_excitations",
            valid_type=Dict,
            required=True,
            help="Output parameters from a single-point excitations",
        )
        spec.expose_outputs(
            RobustOptimizationWorkChain,
            namespace="opt",
            include=["output_parameters", "relaxed_structure"],
            namespace_options={"required": False},
        )

        spec.output(
            "wigner_excitations",
            valid_type=List,
            required=False,
            help="Output parameters from all Wigner excited state calculation",
        )

        spec.outline(
            if_(cls.should_optimize)(
                cls.optimize,
                cls.inspect_optimization,
            ),
            cls.excite,
            cls.inspect_excitation,
            if_(cls.should_run_wigner)(
                cls.wigner_sampling,
                cls.wigner_excite,
                cls.inspect_wigner_excitation,
            ),
        )

        spec.exit_code(
            401,
            "ERROR_OPTIMIZATION_FAILED",
            "optimization encountered unspecified error",
        )
        spec.exit_code(
            402, "ERROR_EXCITATION_FAILED", "excited state calculation failed"
        )

    def excite(self):
        """Calculate excited states for a single geometry"""
        inputs = self.exposed_inputs(
            OrcaExcitationWorkChain, namespace="exc", agglomerate=False
        )
        inputs.orca.code = self.inputs.code

        if self.inputs.optimize:
            self.report("Calculating spectrum for optimized geometry")
            inputs.orca.structure = self.ctx.calc_opt.outputs.relaxed_structure

            # Pass in converged SCF wavefunction
            with self.ctx.calc_opt.outputs.retrieved.base.repository.open(
                "aiida.gbw", "rb"
            ) as handler:
                gbw_file = SinglefileData(handler)
            inputs.orca.file = {"gbw": gbw_file}
            inputs.orca.parameters = add_orca_wf_guess(inputs.orca.parameters)
        else:
            self.report("Calculating spectrum for input geometry")
            inputs.orca.structure = self.inputs.structure

        calc_exc = self.submit(OrcaExcitationWorkChain, **inputs)
        calc_exc.label = "franck-condon-excitation"
        return ToContext(calc_exc=calc_exc)

    def wigner_sampling(self):
        self.report(f"Generating {self.inputs.nwigner.value} Wigner geometries")

        n_low_freq_vibs = 0
        for freq in self.ctx.calc_opt.outputs.output_parameters["vibfreqs"]:
            if freq < self.inputs.wigner_low_freq_thr:
                n_low_freq_vibs += 1
        if n_low_freq_vibs > 0:
            self.report(
                f"Ignoring {n_low_freq_vibs} vibrations below {self.inputs.wigner_low_freq_thr.value} cm^-1"
            )

        self.ctx.wigner_structures = generate_wigner_structures(
            self.ctx.calc_opt.outputs.relaxed_structure,
            self.ctx.calc_opt.outputs.output_parameters,
            self.inputs.nwigner,
            self.inputs.wigner_low_freq_thr,
        )

    def wigner_excite(self):
        inputs = self.exposed_inputs(
            OrcaExcitationWorkChain, namespace="exc", agglomerate=False
        )
        inputs.orca.code = self.inputs.code
        # Pass in SCF wavefunction from minimum geometry
        with self.ctx.calc_opt.outputs.retrieved.base.repository.open(
            "aiida.gbw", "rb"
        ) as handler:
            gbw_file = SinglefileData(handler)
        inputs.orca.file = {"gbw": gbw_file}
        inputs.orca.parameters = add_orca_wf_guess(inputs.orca.parameters)
        for i in self.ctx.wigner_structures.get_stepids():
            inputs.orca.structure = pick_structure_from_trajectory(
                self.ctx.wigner_structures, Int(i)
            )
            calc = self.submit(OrcaExcitationWorkChain, **inputs)
            calc.label = f"wigner-excitation-{i}"
            self.to_context(wigner_calcs=append_(calc))

    def optimize(self):
        """Optimize geometry"""
        inputs = self.exposed_inputs(
            RobustOptimizationWorkChain, namespace="opt", agglomerate=False
        )
        inputs.orca.structure = self.inputs.structure
        inputs.orca.code = self.inputs.code

        calc_opt = self.submit(RobustOptimizationWorkChain, **inputs)
        calc_opt.label = "optimization"
        return ToContext(calc_opt=calc_opt)

    def inspect_optimization(self):
        """Check whether optimization succeeded"""
        if not self.ctx.calc_opt.is_finished_ok:
            self.report("Optimization failed :-(")
            return self.exit_codes.ERROR_OPTIMIZATION_FAILED

        self.out_many(
            self.exposed_outputs(
                self.ctx.calc_opt,
                RobustOptimizationWorkChain,
                namespace="opt",
                agglomerate=False,
            )
        )

    def inspect_excitation(self):
        """Check whether excitation succeeded"""
        calc = self.ctx.calc_exc
        if not calc.is_finished_ok:
            self.report("Single point excitation failed :-(")
            return self.exit_codes.ERROR_EXCITATION_FAILED
        self.out("franck_condon_excitations", calc.outputs.excitations)

    def inspect_wigner_excitation(self):
        """Check whether all wigner excitations succeeded"""
        for calc in self.ctx.wigner_calcs:
            if not calc.is_finished_ok:
                self.report("Wigner excitation failed :-(")
                return self.exit_codes.ERROR_EXCITATION_FAILED

        all_wigner_data = [
            wc.outputs.excitations.get_dict() for wc in self.ctx.wigner_calcs
        ]
        self.out("wigner_excitations", List(all_wigner_data).store())

    def should_optimize(self):
        return self.inputs.optimize.value

    def should_run_wigner(self):
        return self.should_optimize() and self.inputs.nwigner > 0

class AtmospecWorkChain(WorkChain):
    """The top-level ATMOSPEC workchain"""

    def _build_process_label(self):
        return "ATMOSPEC workflow"

    @classmethod
    def define(cls, spec):
        super().define(spec)
        spec.expose_inputs(OrcaWignerSpectrumWorkChain, exclude=["structure"])
        spec.input("structure", valid_type=TrajectoryData)

        spec.output(
            "spectrum_data",
            valid_type=List,
            required=True,
            help="All data necessary to construct spectrum in SpectrumWidget",
        )

        spec.output(
            "relaxed_structures",
            valid_type=TrajectoryData,
            required=False,
            help="Minimized structures of all conformers",
        )

        spec.outline(
            cls.launch,
            cls.collect,
        )

    def launch(self):
        inputs = self.exposed_inputs(OrcaWignerSpectrumWorkChain, agglomerate=False)
        self.report(
            f"Launching ATMOSPEC for {len(self.inputs.structure.get_stepids())} conformers"
        )
        for conf_id in self.inputs.structure.get_stepids():
            inputs.structure = self.inputs.structure.get_step_structure(conf_id)
            workflow = self.submit(OrcaWignerSpectrumWorkChain, **inputs)
            workflow.label = f"atmospec-conf-{conf_id}"
            self.to_context(confs=append_(workflow))

    def collect(self):
        for wc in self.ctx.confs:
            if not wc.is_finished_ok:
                return ExitCode(wc.exit_status, wc.exit_message)

        conf_outputs = [wc.outputs for wc in self.ctx.confs]

        # Combine all spectra data
        if self.inputs.optimize and self.inputs.nwigner > 0:
            data = {
                str(i): outputs.wigner_excitations
                for i, outputs in enumerate(conf_outputs)
            }
        else:
            data = {
                str(i): [outputs.franck_condon_excitations.get_dict()]
                for i, outputs in enumerate(conf_outputs)
            }
        all_results = run(ConcatInputsToList, ns=data)
        self.out("spectrum_data", all_results["output"])

        # Combine all optimized geometries into single TrajectoryData
        if self.inputs.optimize:
            relaxed_structures = {}
            orca_output_params = {}
            for i, outputs in enumerate(conf_outputs):
                relaxed_structures[f"struct_{i}"] = outputs.opt.relaxed_structure
                orca_output_params[f"params_{i}"] = outputs.opt.output_parameters

            # For multiple conformers, we're appending relative energies and Boltzmann weights
            array_data = None
            if len(self.ctx.confs) > 1:
                array_data = extract_trajectory_arrays(**orca_output_params)

            trajectory = structures_to_trajectory(
                arrays=array_data, **relaxed_structures
            )
            self.out("relaxed_structures", trajectory)


__author__ = "dougfelt@google.com (Doug Felt)"

import argparse
import os
import os.path
import re
import shutil
import zipfile

from fontTools import ttLib

import notoconfig

def grab_files(dst, files, src_vendor, name_date_re, extract_fn):
  """Get date from each filename in files, create a folder for it, under
  dst/drops, then extract the files to it."""

  # The zip indicates that the corresponding drop is good and built from it. But
  # we might have messed up along the way, so:
  # - if we have a drop and a zip, assume it's already handled
  # - if we have a drop but no zip, assume the drop needs to be rebuilt from the zip
  # - if we have a zip and no drop
  #   - if we have new zip, complain
  #   - else rebuild the drop from the old zip
  # - else build the drop, and if successful, save the zip

  for f in files:
    if not os.path.exists(f):
      print('file \'%s\' does not exist, skipping' % f)
      continue

    filename = os.path.basename(f)
    result = name_date_re.match(filename)
    if not result:
      print('could not parse %s, skipping' % f)
      continue

    name = result.group(1)
    date = '_'.join([d for d in result.group(2,3,4)])
    drop_dir = os.path.join(dst, 'drops', name + '_' + date)

    zip_dir = os.path.join(dst, 'zips')
    zip_filename = os.path.join(zip_dir, filename)
    if os.path.exists(drop_dir):
      if os.path.exists(zip_filename):
        print('already have a %s drop and zip for %s' % (src_vendor, filename))
        continue
      else:
        # clean up, assume needs rebuild
        shutil.rmtree(drop_dir)
    else:
      if os.path.exists(zip_filename):
        if os.path.realpath(f) != os.path.realpath(zip_filename):
          print('already have a zip file named %s for %s' % (zip_filename, f))
          continue

    os.mkdir(drop_dir)
    extract_fn(drop_dir, f)

    if not os.path.exists(zip_filename):
      print('writing %s to %s' % (f, zip_filename))
      shutil.copy2(f, zip_filename)

def matching_files_in_dir(src, namere):
  """Iterate over files in src with names matching namere, returning the list."""
  filelist = []
  for f in os.listdir(src):
    path = os.path.join(src, f)
    if not os.path.isfile(path):
      continue
    if not re.search(namere, f):
      continue
    filelist.append(path)
  if not filelist:
    print("no files in %s matched '%s'" % (src, namere))
  return filelist

def invoke_main(src_vendor, name_date_re, extract_fn, default_params = {}):
  """Grab the files.

  src_vendor is a string, currently either Adobe or Monotype.
  name_date_re is a regex, it should extract name, year, month, and day fields from the filename
  extract_fn is a fn to to extract a file, it takes two args, a dest dir and the zip file name.

  default_params are default values for argparse.  They can be:
  - default_srcdir
  - default_dstdir
  - default_regex

  The default regex and the name_date_re are superficially similar, but different in
  purpose.  The default_regex is used to select files under the src directory. The
  name_date_re is used to extract the date from the file name.  Both apply to the
  file name, but the default_regex can be anything, while name_date_re needs to select
  four groups, where the 2nd, 3rd, and 4th are the year, month, and day (yes this is
  brittle, but all of this is).

  The dest directory must exist and should have 'zips' and 'drops' subdirs."""

  if not src_vendor:
    print('must define src_vendor')
    return
  if not name_date_re:
    print('must define name_date_re')
    return
  if not extract_fn:
    print('must define extract_fn')
    return

  default_srcdir = default_params.get('default_srcdir')
  default_dstdir = default_params.get('default_dstdir')
  default_regex = default_params.get('default_regex')

  parser = argparse.ArgumentParser(description='Copy and extract drop from %s.' %
                                   src_vendor)
  parser.add_argument('-dd', '--dstdir', help='destination directory (default %s)' %
                      default_dstdir, default=default_dstdir, metavar='dst')
  parser.add_argument('-sd', '--srcdir', help='source directory (default %s)' %
                      default_srcdir, default=default_srcdir, metavar='src')
  parser.add_argument('--name', help='file name regex to match (default \'%s\')' %
                      default_regex, default=default_regex, metavar='re')
  parser.add_argument('--srcs', help='source files (if defined, use instead of srcdir+name)',
                      nargs="*", metavar='zip')
  args = parser.parse_args()

  if not os.path.exists(args.dstdir):
    print('%s does not exists or is not a directory' % args.dstdir)
    return

  if not args.srcs:
    if not os.path.isdir(args.srcdir):
      print('%s does not exist or is not a directory' % args.srcdir)
      return
    filelist = matching_files_in_dir(args.srcdir, args.name)
  else:
    filelist = args.srcs

  grab_files(args.dstdir, filelist, src_vendor, name_date_re, extract_fn)


import subprocess
import sys

import pytest

def run(*args, check=True):
    process = subprocess.run(  # noqa: PLW1510
        [sys.executable, "-m", *args],  # noqa: S603
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        encoding="utf-8",
    )
    if check and process.returncode:
        pytest.fail(process.stdout)

    return process.stdout

def append(file, text):
    file.write_text(file.read_text() + text)


import pandas as pd
import torch
df = pd.read_csv('helloRyan.csv')

from simpletransformers.classification import ClassificationModel

# define hyperparameter
train_args ={
             "reprocess_input_data": True,

             "overwrite_output_dir": True,
             "fp16":False,
             "num_train_epochs": 4}

# Create a ClassificationModel
from sklearn.model_selection import train_test_split
new_df = pd.DataFrame()
new_df['LengthofContent'] = df['LengthofContent']
train_df, test_df = train_test_split(new_df, test_size=0.10)

print('train shape: ',train_df.shape)
print('test shape: ',test_df.shape)
model = ClassificationModel(

    "bert", "bert-base-cased",
    num_labels=3,
    use_cuda = False,
    args=train_args
)
model.train_model(train_df)
class_list = ['Short', 'Medium', 'Long']

post = "Short"

predictions, raw_outputs = model.predict([post])
from sklearn.metrics import f1_score, accuracy_score

def f1_multiclass(labels, preds):
    return f1_score(labels, preds, average='micro')

result, model_outputs, wrong_predictions = model.eval_model(test_df, f1=f1_multiclass, acc=accuracy_score)
print(result)
print(class_list[predictions[0]])


from django.urls import path
from . import views
from .views import (AmcListView,AmcUpdateView)

urlpatterns = [
    #Amc urls
    path('amc_add/<int:pk>', views.create_amc, name="AmcCreate"),

    path('amc_list', AmcListView.as_view(), name="AmcList"),
    path('amc_update/<int:pk>', AmcUpdateView.as_view(), name="AmcUpdate"),
    path('amc_detail/<int:pk>', views.AmcDetail, name="AmcDetail"),
    path('load_contact_person', views.load_contact_person, name="LoadContactPerson"),

    ]


import dataclasses
import logging
import csv
import requests
from typing import List
from urllib.request import urlopen

CSV_URL = 'https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv'
TRACK1_PACKAGE_PREFIX = 'azure-mgmt-'
TRACK2_PACKAGE_PREFIX = 'azure-resourcemanager-'

MIGRATION_LINK = 'https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/resourcemanager/docs/MIGRATION_GUIDE.md'

CSV_FILENAME = 'deprecate_java_packages.csv'

SERVICE_NAME_MAP = {
    'cosmosdb': 'cosmos',
    'datalake-analytics': 'datalakeanalytics',
    'datalake-store': 'datalakestore',
    'datalake-store-uploader': 'datalakestore',
    'devices': 'deviceprovisioningservices',
    'devtestlab': 'devtestlabs',
    'documentdb': 'cosmos',
    'eventhub': 'eventhubs',
    'features': 'resources',
    'graph-rbac': 'authorization',
    'insights': 'monitor',
    'locks': 'resources',
    'machinelearning': 'machinelearningservices',
    'media': 'mediaservices',
    'dbformariadb': 'mariadb',
    'policy': 'resources',
    'powerbi': 'powerbidedicated',
    'samples': 'resources',
    'scheduler': 'resources',
    'subscriptions': 'resources',
    'traffic-manager': 'trafficmanager',
    'utility': 'resources',
    'website': 'appservice',
    'websites': 'appservice',
}

@dataclasses.dataclass
class Sdk:
    service: str
    package: str
    artifact: str

@dataclasses.dataclass
class SdkInfo:
    service: str
    t1_package: str
    t1_artifact: str
    t2_artifact: str

    def to_row(self) -> List[str]:
        return [self.service, 'Java', self.t1_maven_url(), self.t2_maven_url(), MIGRATION_LINK]

    def t1_maven_url(self):
        return f'https://mvnrepository.com/artifact/{self.t1_package}/{self.t1_artifact}'

    def t2_maven_url(self):
        return f'https://mvnrepository.com/artifact/com.azure.resourcemanager/{self.t2_artifact}'

def process_java_packages_csv() -> List[SdkInfo]:
    logging.info(f'query csv: {CSV_URL}')
    with urlopen(CSV_URL) as csv_response:
        csv_data = csv_response.read()
        csv_str = csv_data.decode('utf-8')
        csv_reader = csv.DictReader(csv_str.split('\n'), delimiter=',', quotechar='"')

    t1_sdks = []
    t2_sdks = {}
    for row in csv_reader:
        package = row['Package']
        namespace = row['GroupId']
        if package.startswith(TRACK1_PACKAGE_PREFIX):
            service = package[len(TRACK1_PACKAGE_PREFIX):]
            if service in SERVICE_NAME_MAP:
                service = SERVICE_NAME_MAP[service]
            sdk = Sdk(service, namespace, package)
            t1_sdks.append(sdk)
        elif package == 'azure':
            service = ''
            sdk = Sdk(service, namespace, package)
            t1_sdks.append(sdk)

        if package.startswith(TRACK2_PACKAGE_PREFIX) and package != TRACK2_PACKAGE_PREFIX + 'parent':
            service = package[len(TRACK2_PACKAGE_PREFIX):]
            sdk = Sdk(service, namespace, package)
            t2_sdks[service] = sdk
        elif package == 'azure-resourcemanager':
            service = ''
            sdk = Sdk(service, namespace, package)
            t2_sdks[service] = sdk

    t1_sdks.sort(key=lambda r: r.service)

    sdk_info_list = []
    for sdk in t1_sdks:
        t2_sdk = None
        if sdk.service in t2_sdks:
            t2_sdk = t2_sdks[sdk.service]
        if not t2_sdk:
            print(sdk.service)

        sdk_info = SdkInfo(sdk.service, sdk.package, sdk.artifact, t2_sdk.artifact if t2_sdk else '')
        sdk_info_list.append(sdk_info)

    # sdk_info_list.append(SdkInfo('', 'com.microsoft.azure', 'azure', 'azure-resourcemanager'))

    return sdk_info_list

def run():
    logging.basicConfig(level=logging.INFO)
    sdk_info_list = process_java_packages_csv()
    write_csv(sdk_info_list)

    validate_maven_packages(sdk_info_list)

def write_csv(sdk_info_list: List[SdkInfo]):
    logging.info(f'write csv: {CSV_FILENAME}')
    with open(CSV_FILENAME, 'w', newline='') as f:
        writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        writer.writerow(['Service', 'Language',
                         'Legacy package (package being replaced)',
                         'New Packages that replace the legacy package on the left',
                         'Migration Guide link'])
        for item in sdk_info_list:
            writer.writerow(item.to_row())

def validate_maven_packages(sdk_info_list: List[SdkInfo]):
    for sdk_info in sdk_info_list:
        check_maven_url(sdk_info.t1_maven_url())
        check_maven_url(sdk_info.t2_maven_url())

def check_maven_url(url: str):
    url = url\
              .replace('mvnrepository.com', 'mvnrepository#com')\
              .replace('.', '/')\
              .replace('mvnrepository#com', 'mvnrepository.com')\
              .replace('https://mvnrepository.com/artifact/', 'https://repo1.maven.org/maven2/') + '/maven-metadata.xml'
    r = requests.get(url)
    logging.info(f'check url {url}, status code {r.status_code}')
    r.raise_for_status()

if __name__ == "__main__":
    run()


from pymongo.database import Database
from typing import Tuple
from werkzeug.exceptions import BadRequest
from application.handlers.utils_handler import _can_courier_take_order

def _write_to_db(courier_id: int, patch_data: dict, db: Database) -> Tuple[dict, int]:
    courier = db['couriers'].find_one({"_id": courier_id})
    if not courier:
        raise BadRequest('Content must contain courier_id in database')
    allowed_fields = {'courier_type', 'regions', 'working_hours'}
    for key in patch_data:
        if key not in allowed_fields:
            raise BadRequest(f'Unknown key {key}')

    for key, value in patch_data.items():
        courier[key] = value
    prev_orders = courier['orders']
    courier['orders'] = {}
    for order_num, timestamp in prev_orders.items():
        order = db['orders'].find_one({'_id': int(order_num)})
        order['is_used'] = False
        if _can_courier_take_order(courier, order):
            courier['orders'][order_num] = timestamp
        else:
            db['orders'].replace_one({'_id': int(order_num)}, order)

    db['couriers'].replace_one({'_id': courier_id}, courier)
    response_keys = ['courier_type', 'regions', 'working_hours']
    response = {key: courier[key] for key in response_keys}
    response['courier_id'] = courier['_id']
    return response, 200

def patch_courier(courier_id: int, patch_data: dict, db: Database) -> Tuple[dict, int]:
    return _write_to_db(courier_id, patch_data, db)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun  3 19:17:40 2019

@author: ajit
"""
import xlrd

# excel files have rather long names, lets shorten them
sName = {
        'Chemical Engineering' : 'CHD',
        'Civil Engineering' : 'CED',
        'Computer Science and Engineering' : 'CSE',
        'Electrical Engineering' : 'EEE',
        'Mechanical Engineering' : 'MED',
        'Art & Performing Art' : 'ADP',
        'Communication' : 'COM',
        'Design' : 'DES',
        'Economics' : 'ECO',
        'English' : 'ENG',
        'History' : 'HIS',
        'International Relations and Governance Studies' : 'INT',
        'Sociology' : 'SOC',
        'Management': 'MGT',
        'Chemistry' : 'CHY',
        'Life Sciences' : 'BIO',
        'Mathematics' : 'MAT',
        'Physics' : 'PHY',
        'Archaeology' : 'HIS', # TODO: check which dept offers history
        'Biotechnology' : 'BIO',
        'Chemistry' : 'CHY',
        'Dance' : 'ADP',  # TODO: check which dept offers dance
        'Data Analytics': 'MGT', # TODO: check which dept offers data analytics
        'Electronics and Communication Engineering' : 'ECE',
        'International Relations and Public Affairs': 'INT',
        'Electrical and Electronics Engineering': 'EEE',
        'International Relations': 'INT',
        'Art Design and Performing Arts': 'ADP',
        'Bachelor of Management Studies': 'BMS',
        'Under Graduate': 'UGG',
        'OHM': 'MGT',
        'General Management': 'MGT',
        'FAC': 'MGT',
        'STM': 'MGT',
        'DOM': 'MGT'
        }

# read excel file
# data stored as a dictionary in courseList
#---------------------------------------------

COSfilePath = "COS_backUp_Jul05.xlsx"
wb = xlrd.open_workbook(COSfilePath)
sheet = wb.sheet_by_index(0) # get first sheet

# column heading numbers
School	=	0
Department	=	1
CourseCode	=	2
CourseTitle	=	3
Credits	=	4
CourseType	=	5
OpenasUWE	=	6
PartofMinoras	=	7
Programs	=	[8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
CourseCapacity	=	18
LectureHoursPerWeek	=	19
TutorialHoursPerWeek	=	20
PracticalHoursPerWeek	=	21
LectureDuration	=	22
LectureSections	=	23
TutorialSections	=	24
PracticalSections	=	25
LectureInstructors	=	26
TutorialInstructors	=	27
PracticalInstructors	=	28
LabRoomNumber	=	29
LabCapacity	=	30
SpecialRequirement	=	31
LectureSlottedoutsideMonFriand95	=	32
TutorialSlottedoutsideMonFriand95	=	33
PracticalSlottedoutsideMonFriand95	=	34
LectureRemarks	=	35
TutorialRemarks	=	36
PracticalRemarks	=	37
CourseCoordinator	=	38
LectureCredits	=	39
TutorialCredits	=	40
PracticalCredits	=	41
AssessmentTypeWeightage	=	42
CrossListedwith	=	43
Prerequisite	=	44
Crosslistedcourseoffered	=	45
Lasttimeofferedsemester	=	46

courseList = dict()

for i in range(1,sheet.nrows):

    # loop over courseCode column, make a dictionary for each course
    # if a cell in this column is empty, that means that row has details of
    # last defined course code

    # skip over empty cells
    if (sheet.cell_type(i,CourseCode) == xlrd.XL_CELL_EMPTY):
        continue

    j = 0 # How many EXTRA rows for a particular course
    if ((i+j+1) < sheet.nrows):
        while (sheet.cell_type(i+j+1,CourseCode) == xlrd.XL_CELL_EMPTY):
            j = j+1

    cCode = sheet.cell_value(i,CourseCode)

    courseList[cCode] = dict()

    courseI = courseList[cCode]
    courseI['Credits'] = int(sheet.cell_value(i,Credits))
    courseI['Title'] = sheet.cell_value(i,CourseTitle)
    courseI['CourseType'] = sheet.cell_value(i,CourseType)

    if (courseI['CourseType'] != 'CCC'):
        courseI['level'] = cCode[-3]
    else:
        courseI['level'] = 0

    courseI['OpenasUWE'] = sheet.cell_value(i,OpenasUWE)
    courseI['PartofMinoras'] = sheet.cell_value(i,PartofMinoras)
    courseI['CourseCapacity'] = int(sheet.cell_value(i,CourseCapacity))

    # read target audience of the course
    courseI['programs'] = set()
    for p in Programs:
        if (sheet.cell_type(i,p) != xlrd.XL_CELL_EMPTY):
            cellVal = sheet.cell_value(i,p)
#            cellVal = cellVal[:-1]+cellVal[-1]
            courseI['programs'].add(cellVal)

    # read number lecture hours
    if (sheet.cell_type(i,LectureHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['LectureHoursPerWeek']=0
    else:
        courseI['LectureHoursPerWeek']=int(2*float(sheet.cell_value(i,LectureHoursPerWeek)))
        courseI['LectureDuration']=int(2*float(sheet.cell_value(i,LectureDuration)))

    #read number of tutorials hours
    if (sheet.cell_type(i,TutorialHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['TutorialHoursPerWeek']=0
    else:
        courseI['TutorialHoursPerWeek']=int(2*float(sheet.cell_value(i,TutorialHoursPerWeek)))

    #read number of lab hours
    if (sheet.cell_type(i,PracticalHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['PracticalHoursPerWeek']=0
    else:
        courseI['PracticalHoursPerWeek']=int(2*float(sheet.cell_value(i,PracticalHoursPerWeek)))

    # read lecture sections and respective instructors
    if (courseI['LectureHoursPerWeek'] >= 1):

        courseI['lecSections'] = dict()
        if (j == 0):
            seci = 'LEC1'
            courseI['lecSections'][seci] = dict()
            courseI['lecSections'][seci]['instructors'] = sheet.cell_value(i,LectureInstructors).split(',\n')[:-1]

            courseI['lecSections'][seci]['students'] = set()
        else:
            for k in range(i,i+j+1):
                if (sheet.cell_type(k,LectureSections) != xlrd.XL_CELL_EMPTY):
                    seci = sheet.cell_value(k,LectureSections)
                    courseI['lecSections'][seci] = dict()
                    courseI['lecSections'][seci]['instructors'] = sheet.cell_value(k,LectureInstructors).split(',\n')[:-1]
                    courseI['lecSections'][seci]['students'] = set()

    # read tut sections and respective instructors
    if (courseI['TutorialHoursPerWeek'] >= 1):
        courseI['tutSections'] = dict()

        if (j == 0) :
            seci = 'TUT1'
            courseI['tutSections'][seci] = dict()
            courseI['tutSections'][seci]['instructors'] = sheet.cell_value(i,TutorialInstructors).split(',\n')[:-1]
            courseI['tutSections'][seci]['students'] = set()
        else:
            for k in range(i,i+j+1):
                if (sheet.cell_type(k,TutorialSections) != xlrd.XL_CELL_EMPTY):
                    seci = sheet.cell_value(k,TutorialSections)
                    courseI['tutSections'][seci] = dict()
                    courseI['tutSections'][seci]['instructors'] = sheet.cell_value(k,TutorialInstructors).split(',\n')[:-1]
                    courseI['tutSections'][seci]['students'] = set()

    # read Lab sections, respective instructors, and lab room number
    if (courseI['PracticalHoursPerWeek'] >= 1):
       courseI['labSections'] = dict()
       if (j == 0):
           praci = 'PRAC1'
           courseI['labSections'][praci] = dict()
           courseI['labSections'][praci]['instructors'] =  sheet.cell_value(i,PracticalInstructors).split(',\n')[:-1]
           courseI['labSections'][praci]['students'] =  set()
           courseI['labSections'][praci]['room'] =  sheet.cell_value(i,LabRoomNumber)
           courseI['labSections'][praci]['students'] = set()

       else:
           for k in range(i,i+j+1):
               if (sheet.cell_type(k,PracticalSections) != xlrd.XL_CELL_EMPTY):
                    praci = sheet.cell_value(k,PracticalSections)
                    courseI['labSections'][praci] = dict()
                    courseI['labSections'][praci]['instructors'] =  sheet.cell_value(k,PracticalInstructors).split(',\n')[:-1]
                    courseI['labSections'][praci]['room'] =  sheet.cell_value(k,LabRoomNumber)
                    courseI['labSections'][praci]['students'] = set()

# create a dictionary major electives, minor electives and UWEs
majorElectives = dict()
minorElectives = dict()
UWEnotInMinors = dict()
UWEs = dict()

for cIndex, c in courseList.items():
    d = cIndex[0:3] # dept
    y = cIndex[3]   # year

    # combine many business school minor in one
    if (d == 'MKT') or (d == 'DOM') or (d == 'FAC') or (d == 'OHM') or (d == 'STM'):
        d = 'MGT'

    if (int(y) >= 4):
        y = '4'

    #combine year and dept again
    d = d+y

    if 'Yes' in c['OpenasUWE']:
        if d in UWEs:
            UWEs[d].add(cIndex)
        else:
            UWEs[d] = set()
            UWEs[d].add(cIndex)

    # update the list of major electives
    if 'Major Elective' in c['CourseType']:
        if d in majorElectives:
            majorElectives[d].add(cIndex)
        else:
            majorElectives[d] = set()
            majorElectives[d].add(cIndex)

    # update list of minor electives
    # treat UWE which are not part of minors as minor electives
    #    for time-tabling purpose
    if 'Elective' in c['PartofMinoras']:
        if d in minorElectives:
            minorElectives[d].add(cIndex)
        else:
            minorElectives[d] = set()
            minorElectives[d].add(cIndex)

    if ('Not a part' in c['PartofMinoras']) and ('Yes' in c['OpenasUWE']):
        if d in UWEnotInMinors:
            UWEnotInMinors[d].add(cIndex)
        else:
            UWEnotInMinors[d] = set()
            UWEnotInMinors[d].add(cIndex)

# extract a list of CCC
CCCcourses = set()
for cIndex in courseList:
    if 'CCC' in cIndex:
        CCCcourses.add(cIndex)

# read target audience for each major electives
majorElectiveTakers = dict()
for ci in courseList:
    c = courseList[ci]
    if ('Major Elective' in c['CourseType']):

        if (ci in majorElectiveTakers):
            majorElectiveTakers[ci].add(c['programs'])
        else:
            majorElectiveTakers[ci] = set()
            majorElectiveTakers[ci] = majorElectiveTakers[ci].union(c['programs'])

# for each group create a set of major elective course
majorElectivePool = dict()
for ci in majorElectiveTakers:
    stSeti = majorElectiveTakers[ci]

    for sti in stSeti:
        if (sti in majorElectivePool):
            majorElectivePool[sti].add(ci)
        else:
            majorElectivePool[sti] = set()
            majorElectivePool[sti].add(ci)


from assessment.models import Choice
from assessment.utils import markdownify_bold, markdownify_italic, select_label_choice
from ckeditor.widgets import CKEditorWidget
from django import forms
from django.forms import ModelForm, widgets
from django.forms.renderers import get_default_renderer
from django.utils.safestring import mark_safe
from django.utils.translation import gettext_lazy as _

class ChoiceForm(ModelForm):
    """
    The class ChoiceForm defines the form object used for each evaluation element of the user evaluation
    The form is made up of 2 fields, either a radio or checkbox and a charfield for the user notes
    """

    class Meta:
        model = Choice
        fields = []

    def __init__(self, *args, **kwargs):
        # Catch the evaluation element
        evaluation_element = kwargs.pop("evaluation_element")
        super().__init__(*args, **kwargs)

        # If the question_type of the evaluation element is a radio item
        if evaluation_element.master_evaluation_element.question_type == "radio":
            choices_tuple = evaluation_element.get_choices_as_tuple()
            list_choices_ticked = evaluation_element.get_list_choices_ticked()
            question = forms.ChoiceField(
                label="",
                widget=MarkdownifyRadioChoices,
                choices=choices_tuple,
                initial=list_choices_ticked,  # Initial ticked choices according the database
                required=False,
            )  # see if needed
            self.fields[str(evaluation_element.id)] = question

        # If the question_type of the evaluation element is a checkbox, so multiple choices can be selected
        elif evaluation_element.master_evaluation_element.question_type == "checkbox":
            choices_tuple = evaluation_element.get_choices_as_tuple()
            list_choices_ticked = evaluation_element.get_list_choices_ticked()
            question = forms.MultipleChoiceField(
                label="",
                widget=MarkdownifyMultiselectChoices,
                choices=choices_tuple,
                initial=list_choices_ticked,
                required=False,
            )  # see if needed
            self.fields[str(evaluation_element.id)] = question

        self.add_element_justification(evaluation_element)  # Add the justification field

        # If the evaluation element has no user_notes set yet, the default value is displayed
        if evaluation_element.user_notes is None or evaluation_element.user_notes == "":
            notes = forms.CharField(
                label=_("My notes"),
                widget=forms.Textarea(
                    attrs={
                        "rows": 5,
                        "size": 100,
                        "width": "100%",
                        "class": "textarea textarea-empty",
                        "placeholder": _("Enter your notes on the evaluation element here."),
                    }
                ),
                required=False,
            )

        # Elif the user has already registered the champ user_notes for this element, it is displayed as initial
        elif evaluation_element.user_notes is not None:
            form_attr = {
                "rows": 5,
                "size": 100,
                "width": "100%",
                "class": "textarea textarea-data",
            }
            if evaluation_element.user_notes_archived:
                form_attr["disabled"] = "disabled"
                form_attr["class"] += " note-disabled"
            notes = forms.CharField(
                label=_("My notes"),
                widget=forms.Textarea(form_attr),
                initial=evaluation_element.user_notes,
                required=False,
            )

        self.fields["notes"] = notes  # add the notes (empty or not) in the form

    def add_element_justification(self, evaluation_element):
        """
        Used during the initialization of a class instance to add a new field in the form for the
        user_justification field of the evaluation_element class model.
        """
        if (
            evaluation_element.user_justification is None
            or evaluation_element.user_justification == ""
        ):
            justification = forms.CharField(
                label=_("Answer justification"),
                widget=CKEditorWidget(),
                required=False,
            )
        elif evaluation_element.user_justification is not None:
            justification = forms.CharField(
                label=_("Answer justification"),
                widget=CKEditorWidget(),
                initial=evaluation_element.user_justification,
                required=False,
            )
        self.fields["justification"] = justification

class MarkdownifyRadioChoices(widgets.RadioSelect):
    """
    Format the choices of the evaluation elements in order to transform markdown symbols into html tags
    """

    def _render(self, template_name, context, renderer=None):
        if renderer is None:
            renderer = get_default_renderer()
        text = renderer.render(template_name, context)
        for value in select_label_choice(renderer.render(template_name, context)):
            text = text.replace(value, markdownify_italic(markdownify_bold(value)))
        return mark_safe(text)

class MarkdownifyMultiselectChoices(forms.CheckboxSelectMultiple):
    """
    Format the choices of the evaluation elements in order to transform markdown symbols into html tags
    """

    def _render(self, template_name, context, renderer=None):
        if renderer is None:
            renderer = get_default_renderer()
        text = renderer.render(template_name, context)
        for value in select_label_choice(renderer.render(template_name, context)):
            text = text.replace(value, markdownify_italic(markdownify_bold(value)))
        return mark_safe(text)