import logging
import traceback
import os
import re
import requests
import boto3
from botocore.exceptions import ClientError
from storages.backends.s3boto3 import S3Boto3Storage
from django.conf import settings
from django.core.files.storage import get_storage_class

logger = logging.getLogger(__name__)

class CodalabS3BotoStorage(S3Boto3Storage):
    def deconstruct(self):
        path, args, kwargs = super().deconstruct()
        return path, [], {}

s3 = boto3.resource('s3', endpoint_url=settings.AWS_S3_ENDPOINT_URL)

StorageClass = get_storage_class(settings.DEFAULT_FILE_STORAGE)

if hasattr(settings, 'USE_AWS') and settings.USE_AWS:
    BundleStorage = StorageClass(bucket=settings.AWS_STORAGE_PRIVATE_BUCKET_NAME)
    PublicStorage = StorageClass(bucket=settings.AWS_STORAGE_BUCKET_NAME)
    PublicStorage.connection.auth_region_name = settings.S3DIRECT_REGION
    BundleStorage.connection.auth_region_name = settings.S3DIRECT_REGION
elif hasattr(settings, 'BUNDLE_AZURE_ACCOUNT_NAME') and settings.BUNDLE_AZURE_ACCOUNT_NAME:
    BundleStorage = StorageClass(account_name=settings.BUNDLE_AZURE_ACCOUNT_NAME,
                                 account_key=settings.BUNDLE_AZURE_ACCOUNT_KEY,
                                 azure_container=settings.BUNDLE_AZURE_CONTAINER)

    PublicStorage = StorageClass(account_name=settings.AZURE_ACCOUNT_NAME,
                                 account_key=settings.AZURE_ACCOUNT_KEY,
                                 azure_container=settings.AZURE_CONTAINER)
else:
    # No storage provided, like in a test, let's just do something basic
    BundleStorage = StorageClass()
    PublicStorage = StorageClass()

def clean_html_script(html_content):
    # Finds <script and everything between /script>. No scripts for you.
    return re.sub('(<script)(\s*?\S*?)*?(/script>)', "", str(html_content))

def docker_image_clean(image_name):
    if not image_name:
        return ""
    # Remove all excess whitespaces on edges, split on spaces and grab the first word.
    # Wraps in double quotes so bash cannot interpret as an exec
    image_name = '"{}"'.format(image_name.strip().split(' ')[0])
    # Regex acts as a whitelist here. Only alphanumerics and the following symbols are allowed: / . : -.
    # If any not allowed are found, replaced with second argument to sub.
    image_name = re.sub('[^0-9a-zA-Z/.:\-_]+', '', image_name)
    return image_name

def check_bad_scores(score_dict):
    bad_score_count = 0
    bad_scores = list()
    for score in score_dict:
        for subm in score['scores']:
            for i in range(len(subm)):
                if type(subm[i]) is dict:
                    for k, v in subm[i].items():
                        if k == 'values':
                            for result in v:
                                for result_key, result_value in result.items():
                                    if result_value == 'NaN' or result_value == '-':
                                        bad_score_count += 1
                                        bad_scores.append(result)
    return bad_score_count, bad_scores

def _put_blob(url, file_path):
    requests.put(
        url,
        data=open(file_path, 'rb'),
        headers={
            # For Azure but doesn't hurt AWS
            'x-ms-blob-type': 'BlockBlob',
        }
    )

def push_submission_to_leaderboard_if_best(submission):
    from apps.web.models import PhaseLeaderBoard, PhaseLeaderBoardEntry, add_submission_to_leaderboard
    # In this phase get the submission score from the column with the lowest ordering
    score_def = submission.get_default_score_def()
    lb = PhaseLeaderBoard.objects.get(phase=submission.phase)

    # Get our leaderboard entries: Related Submissions should be in our participant's submissions,
    # and the leaderboard should be the one attached to our phase
    entries = PhaseLeaderBoardEntry.objects.filter(result__in=submission.participant.submissions.all(), board=lb)
    submissions = [(entry.result, entry.result.get_default_score()) for entry in entries]
    sorted_list = sorted(submissions, key=lambda x: x[1])
    if sorted_list:
        top_sub, top_score = sorted_list[0]
        score_value = submission.get_default_score()
        if score_def.sorting == 'asc':
            # The last value in ascending is the top score, 1 beats 3
            if score_value <= top_score:
                add_submission_to_leaderboard(submission)
                logger.info("Force best submission added submission to leaderboard in ascending order "
                            "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score,
                            score_value)
        elif score_def.sorting == 'desc':
            # The first value in descending is the top score, 3 beats 1
            if score_value >= top_score:
                add_submission_to_leaderboard(submission)
                logger.info(
                    "Force best submission added submission to leaderboard in descending order "
                    "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value)
    else:
        add_submission_to_leaderboard(submission)
        logger.info(
            "Force best submission added submission: {0} with score: {1} to leaderboard: {2}"
            " because no submission was present".format(
                submission, submission.get_default_score(), lb)
        )

def inheritors(klass):
    subclasses = set()
    work = [klass]
    while work:
        parent = work.pop()
        for child in parent.__subclasses__():
            if child not in subclasses:
                subclasses.add(child)
                work.append(child)
    return subclasses

def get_object_base_url(object, attr):
    if settings.USE_AWS:
        # Boto3 does not like receiving an empty path.
        return getattr(object, attr).storage.url(' ').replace("%20", "")
    else:
        return getattr(object, attr).storage.url('')

def s3_key_from_url(url):
    if not url or url == '':
        logger.error("Received an invalid url to convert to a key: {}".format(url))
        return
    # Remove the beginning of the URL (before bucket name) so we just have the path(key) to the file
    key = url.split("{}/".format(settings.AWS_STORAGE_PRIVATE_BUCKET_NAME))[-1]
    # Path could also be in a format <bucket>.<url> so check that as well
    key = key.split("{}.{}/".format(settings.AWS_STORAGE_PRIVATE_BUCKET_NAME, settings.AWS_S3_HOST))[-1]
    return key

def get_competition_size_data(competition):
    data = {
        'id': competition.id,
        'title': competition.title,
        'creator': "{0} ({1})".format(competition.creator.email, competition.creator.username),
        'is_active': competition.is_active,
        'submissions': 0,
        'datasets': 0,
        'dumps': 0,
        'bundle': 0,
    }
    data_file_attrs = [
        'reference_data_organizer_dataset',
        'input_data_organizer_dataset',
        'scoring_program_organizer_dataset',
        'public_data_organizer_dataset',
        'starting_kit_organizer_dataset',
        'ingestion_program_organizer_dataset',
    ]
    keys_to_total = [
        'submissions',
        'datasets',
        'bundle',
        'dumps',
    ]
    for phase in competition.phases.all():
        # submissions
        for submission in phase.submissions.all():
            # submission.size should call get_submission_size anyway.
            data['submissions'] += submission.size
        # datasets
        for attr in data_file_attrs:
            dataset = getattr(phase, attr)
            if dataset:
                data['datasets'] += dataset.size
    # bundle
    if hasattr(competition, 'bundle'):
        data['bundle'] = competition.bundle.size
    # dumps
    if competition.dumps.exists():
        data['dumps'] = sum([dump.size for dump in competition.dumps.all()])
    data['total'] = sum([data[key] for key in keys_to_total])
    return data

def get_submission_size(submission):
    total = 0
    if not os.environ.get('PYTEST'):
        file_attrs = [
            'inputfile',
            'runfile',
            'output_file',
            'private_output_file',
            'stdout_file',
            'stderr_file',
            'history_file',
            'scores_file',
            'coopetition_file',
            'detailed_results_file',
            'prediction_runfile',
            'prediction_output_file',
            'prediction_stdout_file',
            'prediction_stderr_file',
            'ingestion_program_stdout_file',
            'ingestion_program_stderr_file',
        ]
        total += get_filefield_size(submission, 'file', aws_attr='s3_file', s3direct=True)
        for file_attr in file_attrs:
            total += get_filefield_size(submission, file_attr)
    return total

def get_filefield_size(obj, attr, aws_attr=None, s3direct=False):
    size = None
    attr_obj = getattr(obj, aws_attr) if aws_attr and s3direct else getattr(obj, attr)
    if settings.USE_AWS and s3direct and aws_attr:
        attr_obj = getattr(obj, aws_attr)
        # S3DirectFields are stored as text fields with the full url to the key.
        if attr_obj and attr_obj != '':
            key = s3_key_from_url(attr_obj)
            get_size_from_summary(BundleStorage.bucket.name, key)
    else:
        attr_obj = getattr(obj, attr)
        if attr_obj.name and attr_obj.name != '':
            try:
                size = attr_obj.size
            # This error seems to occur specifically with private_output_file due to 2 different path styles used
            # and the files not actually existing in storage.
            except AttributeError as e:
                logger.error(
                    "An error occurred trying to get a file's size. File: {0}; Object: {1}(ID:{2}); Attr: {3}".format(
                        attr_obj.name, obj, obj.id, attr))
                logger.error(e)
                # If we hit an exception this way, and we're using S3, try the other method.
                if settings.USE_AWS:
                    size = get_size_from_summary(attr_obj.storage.bucket.name, attr_obj.name)
            except ClientError as e:
                logger.error(
                    "An error occurred trying to get a file's size. File: {0}; Object: {1}(ID:{2}); Attr: {3}".format(
                        attr_obj.name, obj, obj.id, attr))
    # Always make sure we return at least 0
    if type(size) == str:
        size = int(size)
    return size or 0

def get_size_from_summary(bucket_name, key):
    size = None
    # Test if file exists, since it's an observed bug that the object exists but no file is present in the storage
    try:
        obj_summary = s3.ObjectSummary(bucket_name, key)
        if obj_summary:
            size = obj_summary.size
    except ClientError as e:
        if e.response['Error']['Code'] == "404":
            # The object does not exist.
            logger.error("File was not found in storage. File: {0}; Storage: {1}".format(bucket_name, key))
        else:
            # Something else has gone wrong.
            logger.error(traceback.format_exc())
    except Exception as e:
        logger.error(traceback.format_exc())
    return size

def delete_key_from_storage(obj, attr, aws_attr=None, s3direct=False, use_boto_method=True):
    """Helper function to do checks and delete a key from storage. Key is FileField.name"""
    if settings.USE_AWS and (use_boto_method or s3direct):
        if not aws_attr:
            aws_attr = attr
        attr_obj = getattr(obj, aws_attr)
        key = None
        if s3direct:
            storage = BundleStorage
            if attr_obj and attr_obj != '':
                key = s3_key_from_url(attr_obj)
        else:
            storage = attr_obj.storage
            key = attr_obj.name
        if key == '' or not key:
            return
        obj_summary = s3.ObjectSummary(storage.bucket.name, key)
        if obj_summary:
                logger.info("Attempting to delete key: {}".format(key))
                obj_summary.delete()
    else:
        attr_obj = getattr(obj, attr)
        storage = attr_obj.storage
        if attr_obj.name and attr_obj.name != '':
            logger.info("Attempting to delete storage file: {}".format(attr_obj.name))
            storage.delete(attr_obj.name)

def storage_recursive_find(storage, dir='', depth=0):
    found_files = []
    if not depth >= 25:
        dirs, files = storage.listdir(dir)
        for file in files:
            found_files.append("{0}/{1}".format(dir, file))
        for new_dir in dirs:
            next_dir = "{}/{}".format(dir, new_dir) if dir != '' else "{}".format(new_dir)
            new_files = storage_recursive_find(storage, next_dir, depth+1)
            found_files += new_files
    else:
        logger.error("Max recursion reached in recursive storage find!")
    return found_files

def storage_get_total_use(storage):
    total_bytes = 0
    if settings.USE_AWS:
        bucket = storage.bucket
        for obj in bucket.objects.all():
            total_bytes += obj.size
    else:
        found_files = set(storage_recursive_find(storage))
        for file_path in found_files:
            total_bytes += storage.size(file_path) or 0
    return float(total_bytes)

def sort_submissions_by_score(submissions):
    # Submissions should always be from the same phase.
    if not len(submissions) >= 1:
        return
    # Grab the first sub's score def. If they're all from the same phase they should be the same
    score_def = submissions[0].get_default_score_def()
    # Make a list of tuples that consist of the submission and it's score for the default field
    submissions_and_scores = [(submission, submission.get_default_score()) for submission in submissions]
    reverse_val = True if score_def.sorting == 'desc' else False
    # Sort the list of tuples based on score
    sorted_list = sorted(submissions_and_scores, key=lambda x: x[1], reverse=reverse_val)
    if sorted_list:
        return sorted_list
    else:
        return

def delete_submissions_except_best_and_or_last(submission):
    from apps.web.models import PhaseLeaderBoardEntry
    subs_to_keep = []
    # Exclude this submission early on so we don't forget
    part_subs = submission.participant.submissions.filter(phase=submission.phase)\
        .exclude(id=submission.id)\
        .exclude(is_public=True) # We do not want to delete a user's public submissions
    # Grab a LeaderBoardEntry from this participant from this phase.
    part_lbe = PhaseLeaderBoardEntry.objects.filter(result__phase=submission.phase,
                                                    result__participant=submission.participant)
    if not part_lbe.exists():
        # We do not have a LBE. Grab the best scored submission
        best_scored_subs = sort_submissions_by_score(part_subs.filter(status__codename='finished'))
        if best_scored_subs:
            top_sub, top_score = best_scored_subs[0]
            subs_to_keep.append(top_sub.id)
        # Add as many of the latest successful submissions as we can
        subs_to_search = part_subs.exclude(id__in=subs_to_keep).filter(status__codename='finished').order_by(
            '-submitted_at'
        )[:2 - len(subs_to_keep)]
        for sub in subs_to_search:
            subs_to_keep.append(sub.id)
        # We do not have enough finished subs to keep at least 2; Regardless of status, grab as many as we need of the latest
        if len(subs_to_keep) < 2:
            for sub in part_subs.exclude(id__in=subs_to_keep).order_by('-submitted_at')[:2 - len(subs_to_keep)]:
                subs_to_keep.append(sub.id)
    else:
        # We do have a LBE. Add it and grab the best submission. If it is not the LBE, keep it (just in-case)
        part_lbe = part_lbe.first()
        subs_to_keep.append(part_lbe.result.id)
        best_scored_subs = sort_submissions_by_score(part_subs.filter(status__codename='finished'))
        top_sub, top_score = best_scored_subs[0]
        # If the best scored submission is not the LBE, keep it
        if top_sub.id != part_lbe.result.id:
            subs_to_keep.append(top_sub.id)
        # Grab the last finished submission
        finished_sub_sorted = part_subs.exclude(id=part_lbe.result.id).filter(status__codename='finsished').order_by(
            '-submitted_at'
        ).first()
        # If we have at least 1 sub
        if finished_sub_sorted:
            subs_to_keep.append(finished_sub_sorted.id)
        else:
            # Try tp add at least 1 more submission (ordered by latest)
            for sub in part_subs.exclude(id__in=subs_to_keep).order_by('-submitted_at')[:1]:
                subs_to_keep.append(sub.id)
    logger.debug("Subs to keep: #{0} {1}".format(part_subs.filter(id__in=subs_to_keep).count(), part_subs.filter(id__in=subs_to_keep)))
    logger.debug("Subs to delete: {}".format(part_subs.exclude(id__in=subs_to_keep)))
    part_subs.exclude(id__in=subs_to_keep).delete()


from contact.forms import ContactForm
from django.conf import settings
from django.contrib import messages
from django.core.mail import send_mail
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404, redirect

#If going to qs, must import model to qs
from .models import Project, Image

# Create your views here.

def projects_home(request):

    # order by active and neg -publish date
    # queryset_list = Project.objects.order_by("-timestamp")
    # if request.user.is_staff or request.user.is_superuser:
    #     queryset_list = Project.objects.all()
    # context = {
    #     'project_list': queryset_list
    # }

    # instance = get_object_or_404(Project, slug=slug)

    # pic_qs = Image.objects.get(name__icontains='head')
    currentUrl = request.get_full_path()

    form = ContactForm(request.POST or None)
    if form.is_valid():
        # print(request.POST)
        # dont just straight up grab data by using 'request.POST' - insecure. Clean it.
        # print(form.cleaned_data['comment'])
        form_email = form.cleaned_data['email']
        form_comment = form.cleaned_data['comment']
        form_full_name = form.cleaned_data['name']
        subject = 'Site contact form'
        from_email = settings.EMAIL_HOST_USER
        to_email = [from_email, 'retsnomis@gmail.com']
        message = '%s:\n%s\nvia %s' %(
            form_full_name,
            form_comment,
            form_email)
        send_mail(
            subject,
            message,
            from_email,
            to_email,
            fail_silently=True)

        messages.success(request, 'Message Submitted\nI\'ll be getting back to you shortly.')
        return HttpResponseRedirect('/')
    # else:
    #     return redirect('/#contact')

        # return HttpResponseRedirect(instance.objects.get_absolute_url())

    recent_projects = []
    for i in range(0, 6):
        try:
            recent_projects.append(Project.objects.order_by('-timestamp')[i])
        except IndexError:
            break
    context = {
        'projects': recent_projects,
        'form': form,
        'currentUrl': currentUrl,
        # 'pic': pic_qs,
    }
    return render(request, 'projects_home.html', context)

def project_detail(request, slug):
    instance = get_object_or_404(Project, slug=slug)

    form = ContactForm(request.POST or None)
    if form.is_valid():
        print(request.POST)

    context = {
        'instance':instance,
        'form': form
    }
    return render(request, 'project_detail.html', context)


import pytest
from pathlib import Path
from helper.storage import get_storage_path, prepare_storage_dir
from tempfile import TemporaryDirectory

STORAGE_DIR = Path(Path(__file__).parent.parent.parent, 'data')

@pytest.mark.parametrize('file_name, expected_output', [
    ('none_existing', 'none_existing'),
    ('existing', 'existing-0'),
    ('existing-twice', 'existing-twice-1')
])
def test_get_storage_path(file_name, expected_output):
    designated_path = Path(STORAGE_DIR, file_name)
    expected_resulting_path = Path(STORAGE_DIR, expected_output)
    assert get_storage_path(designated_path, STORAGE_DIR) == expected_resulting_path

@pytest.mark.parametrize('path', [
    (''),
    ('none_existing'),
])
def test_prepare_storage_dir(path):
    tmp_dir = TemporaryDirectory(prefix='fact_sd_test_')
    storage_destination = '{}/{}'.format(tmp_dir.name, path)
    storage_directory = prepare_storage_dir(storage_destination)
    assert storage_directory.exists()
    assert storage_directory.is_dir()


'''
Validasi model analisis sentimen
'''

import pandas as pd
import numpy as np
import sqlite3
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import confusion_matrix
from sklearn.externals import joblib
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.pipeline import Pipeline
from sklearn.neural_network import MLPClassifier
from sklearn.feature_extraction.text import TfidfVectorizer

def sentimentModelVal(csvfile, conn, cursor, startdate, model):
    '''
    csvfile: string.
        path to csv file that contains preprocessed words and overall sentiment
    conn: sqlite3 connection
    cursor: sqlite3 cursor
    startdate: string. Format -> '2018-03-25'
        cut the dataset from the startdate to the most recent date
    model: string.
        select model available.
        logreg - logistic regression
        mlp    - multilayer perceptron
        mlp2   - multilayer perceptron
        nb     - naive bayes
        pa     - passive agressive
        sgd    - stochastic gradient descent
    '''
    # query
    sqlc = 'SELECT rowid, datereview FROM reviews'
    cursor.execute(sqlc)
    conn.commit()
    # get all the query
    datereview = cursor.fetchall()
    # convert it into dataframe
    datereview = pd.DataFrame(datereview)
    datereview.columns = ['rowid', 'date']
    # change 'date' string into datetime
    datereview['date'] = pd.to_datetime(datereview['date'], format='%d %b %Y')
    # raw format: 26 Jun 2018
    # %d -> Day of the month as a zero-padded decimal number. e.g.: 30
    # %b -> Month as locale’s abbreviated name. e.g.: Sep
    # %Y -> Year with century as a decimal number. e.g.: 2013

    # read csv file
    text = pd.read_csv(csvfile, skip_blank_lines=False)
    # merge text and datereview
    text = pd.merge(text, datereview, on='rowid')

    # select record => march 25 2018
    text = text[text['date'] >= startdate]
    # drop NaN
    text = text[~pd.isna(text['text'])]

    # X_val and y_val
    X_val = text['text'].values
    y_val = text['ovsentiment'].values

    # Model Path
    if model == 'logreg':
        modelpath = './training/sentiment_logreg.pkl'
        modelname = 'Logistic Regression'
    elif model == 'mlp':
        modelpath = './training/sentiment_mlp.pkl'
        modelname = 'MultiLayer Perceptron'
    elif model == 'mlp2':
        modelpath = './training/sentiment_mlp2.pkl'
        modelname = 'MultiLayer Perceptron'
    elif model == 'nb':
        modelpath = './training/sentiment_naivebayes.pkl'
        modelname = 'Naive Bayes'
    elif model == 'pa':
        modelpath = './training/sentiment_passiveagressive.pkl'
        modelname = 'Passive Agressive'
    elif model == 'sgd':
        modelpath = './training/sentiment_sgd.pkl'
        modelname = 'Stochastic Gradient Descent'
    else:
        print('Not Supported')
        return 0

    # load estimator
    estimator = joblib.load(modelpath)
    tfidf = TfidfVectorizer(strip_accents=None, lowercase=False, preprocessor=None)

    # Cross validation
    cvscore = cross_val_score(estimator, X_val, y_val, cv=5, verbose=2, n_jobs=-1)

    # confusion matrix
    y_val_pred = cross_val_predict(estimator, X_val, y_val, cv=5,
            verbose=2, n_jobs=-1)
    conf_mx = confusion_matrix(y_val, y_val_pred)

    # printing
    print('\n')
    print('{0}'.format(modelname))
    print('Cross Validation: {0}'.format(cvscore))
    print('Confusion Matrix: \n{0}'.format(conf_mx))
    print('\n')
    # Precision-recall with average='micro'
    print('Precision-Recall (\'micro\')')
    print('Precision: {0} \nRecall: {1}'.format(
        precision_score(y_val, y_val_pred, average='micro'),
        recall_score(y_val, y_val_pred, average='micro')))
    # Precision-recall with average='macro'
    print('Precision-Recall (\'macro\')')
    print('Precision: {0} \nRecall: {1}'.format(
        precision_score(y_val, y_val_pred, average='macro'),
        recall_score(y_val, y_val_pred, average='macro')))
    # f1 score
    print('F1 Score')
    print('F1 score (\'micro\'): {0} \nF1 score (\'macro\'): {1}'.format(
        f1_score(y_val, y_val_pred, average='micro'),
        f1_score(y_val, y_val_pred, average='macro')))

# make connection to sqlite db
conn = sqlite3.connect('validasi.db')
c = conn.cursor()

# enable foreign keys
c.execute("PRAGMA foreign_keys = ON")
conn.commit()

csvfile = './csvfiles/output1_sentiment_validasi.csv'
startdate = '2018-03-25'
sentimentModelVal(csvfile, conn, c, startdate, model='nb')


from django.urls import path, include
from . import views

urlpatterns = [
    path('', views.dashboard),
    path('events/', include('apps.events.urls')),
    path('profile/<int:id>', views.user_profile),
    path('profile/<int:id>/update', views.update_profile),
    path('schedule', include('apps.scheduler.urls')),

    # path('event'),
    # path('profile')

]


from unittest.mock import patch

import pandas as pd

from rel_bball_analytics.database import db
from rel_bball_analytics.players.data_source import (
    clean_player_data,
    fetch_player_data,
    filter_nba_players,
    get_player_matches,
)
from rel_bball_analytics.tests.fixtures.database import reset_test_db, setup_test_db

from .fixtures.api_responses import (
    empty_response,
    players_nba,
    players_non_nba,
)
from .fixtures.db_records import player_record, players_valid
from .fixtures.expected import df_player_matches

class TestGetPlayerMatches:
    """test suite for get_player_matches"""

    @patch("rel_bball_analytics.players.data_source.fetch_player_data")
    def test_returns_existing_records_from_db(
        self, fetch_player_data, reset_test_db, player_record
    ):
        db.session.add(player_record)
        result = get_player_matches(name="Curry")

        assert type(result) == pd.DataFrame
        assert len(result) > 0
        assert not fetch_player_data.called

    @patch("rel_bball_analytics.players.data_source.save_records")
    @patch("rel_bball_analytics.players.data_source.fetch_player_data")
    def test_returns_data_from_api(
        self, fetch_player_data, save_records, reset_test_db, players_valid
    ):
        fetch_player_data.return_value = pd.DataFrame(players_valid)
        result = get_player_matches(name="Curry")

        assert type(result) == pd.DataFrame
        assert len(result) > 0
        assert fetch_player_data.called
        assert save_records.called

    @patch("rel_bball_analytics.players.data_source.fetch_player_data")
    def test_returns_none_if_no_matches_found(self, fetch_player_data):
        fetch_player_data.return_value = None
        result = get_player_matches(name="Curry")

        assert result is None
        assert fetch_player_data.called

class TestFetchPlayerData:
    """test suite for fetch_player_data"""

    @patch("rel_bball_analytics.players.data_source.get_data_from_api")
    def test_returns_dataframe_for_valid_matches(self, get_data_from_api, players_nba):
        get_data_from_api.return_value = players_nba
        results = fetch_player_data(name="Curry")

        assert type(results) == pd.DataFrame
        assert len(results) > 0

    @patch("rel_bball_analytics.players.data_source.get_data_from_api")
    def test_returns_none_if_no_matches(self, get_data_from_api, empty_response):
        get_data_from_api.return_value = empty_response
        results = fetch_player_data(name="Fake")

        assert results is None

    @patch("rel_bball_analytics.players.data_source.get_data_from_api")
    def test_returns_none_if_no_nba_players(self, get_data_from_api, players_non_nba):
        get_data_from_api.return_value = players_non_nba
        results = fetch_player_data(name="Wembanyama")

        assert results is None

class TestFilterNbaPlayers:
    """test suite for filter_nba_players"""

    def test_returns_nba_players(self, players_nba):
        players = pd.DataFrame(players_nba["response"])
        results = filter_nba_players(players=players)

        assert type(results) == pd.DataFrame
        assert len(results) > 0

    def test_returns_empty_df_if_no_nba_players(self, players_non_nba):
        players = pd.DataFrame(players_non_nba["response"])
        results = filter_nba_players(players=players)

        assert type(results) == pd.DataFrame
        assert results.empty

class TestCleanPlayerData:
    """test suite for clean_player_data"""

    def test_returns_formatted_player_info(self, players_nba, df_player_matches):
        players = pd.DataFrame(players_nba["response"])
        results = clean_player_data(players=players)

        assert results.equals(df_player_matches)


# A small python program to depict the difference between
# classical mechanics and special relativity by creating small visual aids.
# There weren't really any available resources on the internet, so
# programming it myself was a quicker, more customisable approach

import math
from matplotlib import pyplot as plt
import numpy as np
import matplotlib
import copy

# Declaring speed of light
c = 3 * 10**8
# Setting up the plot and customising it so it looks a bit better
fig = plt.figure()
ax = plt.axes()
plt.axhline(y=0, color="black")
# plt.axvline(x=0, label="t", color="black")
plt.ylabel("Tid i år")
plt.xlabel("k: lysår")
plt.grid(True)
plt.title("Rumtidsdiagram")
plt.ylim([0, 5])
"""plt.xlim([-5, 2.375])
plt.xticks([-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5],\
    labels=["-5k", "-4k", "-3k", "-2k", "-1k", "0", "1k", "2k",\
          "3k", "4k", "5k"]) """
plt.xlim([-2, 2])
plt.xticks([-2, -1, 0, 1, 2],\
    labels=["-2k", "-1k", "0", "1k", "2k"])
plt.yticks([0, 1, 2, 3, 4, 5])

#Plotting a spacetime diagram, where time is along the second axis, and distance along the first axis
#Special relativity operates out of a framework where the speed of light is always constant.
#The Lorentz transformations to "switch" perspective only works with movement in one direction.
#Requires a rightward and leftward moving photon to visualize. Their movements are plotted as a function of the distance they've travelled:
# tc = x <=> t = x/c            (or, more generally: t=x/v)

# initialise some particle
# in hindsigt, creating a particle class would definitely have been better
# and then adding the transformations as different methods
def createParticle(starting_point, velocity, starting_t=0):
    t_values = []
    x_values = np.arange(-10, 10, 0.01)
    for i in range(len(x_values)):

        t_values.append((x_values[i]/velocity)+float(starting_t)) #t=x/v
        x_values[i] += starting_point
    return x_values, t_values, velocity

# Apply galilean transformation where x' = x-vt as in classical mechanics
def galileanTransformation(frame_of_reference_velocity, particle):

    for i in range(len(particle[0])):   #Loop through the x_values of the particle that will be adjusted to the frame of reference.
        particle[0][i] = particle[0][i]-frame_of_reference_velocity*particle[1][i]   #x' = x-vt
    new_particle_velocity = round((particle[0][1]-particle[0][0])/(particle[1][1]-particle[1][0]), 2)
    return particle[0], particle[1], new_particle_velocity        #new x and v, whilst t remains the same

# Apply Lorentz Transformation which applies in Special Relativity whre speeds are near c
def lorentzTransformation(frame_of_reference_velocity, particle):
    lorentzFactor = 1/(math.sqrt(1-(frame_of_reference_velocity**2)))

    #For the purposes of this program, c is set to 1, and therefore left out of the equations
    transformed_t = []
    transformed_x = []
    for i in range(len(particle[0])):
        transformed_x.append(lorentzFactor * (particle[0][i]-(frame_of_reference_velocity*particle[1][i])))
        transformed_t.append(lorentzFactor * (particle[1][i]-(frame_of_reference_velocity*particle[0][i])))

    if particle[2] != 1 and particle[2] !=-1:
        own_lorentzFactor = 1/(math.sqrt(1-(particle[2]**2)))
        print(own_lorentzFactor)
        if particle[2] > -0.02 and particle[2] < 0.02:
            for i in range(6):
                plt.scatter(0, i, color="blue")
        else:
            for i in range(len(transformed_t)):
                if round(transformed_t[i], 2)%own_lorentzFactor<0.01:
                    plt.scatter(transformed_x[i], transformed_t[i], color="green")
    print("Done")

    new_particle_velocity = round((particle[2]-frame_of_reference_velocity)/(1-(frame_of_reference_velocity*particle[2])), 2)  #Subtracting the particle's velocity from that of the frame of reference using velocity equation derived from the lorentz-transformation
    return transformed_x, transformed_t, new_particle_velocity

# takes some list of particles along with a colour key and plots them in diagram
def plotAll(some_list_of_particles, colour_key, style="solid"):
    for particle_num in range(len(list_of_particles)):
        if style == "solid":
            ax.plot(list_of_particles[particle_num][0], list_of_particles[particle_num][1], color=colour_key[particle_num], label="v="+str(list_of_particles[particle_num][2]) + "c")
        elif style == "dashed":
            ax.plot(list_of_particles[particle_num][0], list_of_particles[particle_num][1], color=colour_key[particle_num], label="v="+str(list_of_particles[particle_num][2]) + "", linestyle="dashed")

# creating whatever photons are needed for particular diagram
rightward_photon = createParticle(0, 1)
leftward_photon = createParticle(0, -1)
red_car = createParticle(0, 0.001)
blue_car = createParticle(0, 0.8)
blue_car2 = createParticle(2, -0.8, 2.5)

list_of_particles = [red_car, blue_car, blue_car2, rightward_photon, leftward_photon]
corresponding_colour_list = ["red", "blue", "blue", "orange", "orange"]

frame_of_reference_velocity = 0.00

# what type of transformation
lorentz = True
newton = False

if lorentz and not newton:
    for i in range(len(list_of_particles)):
        list_of_particles[i] = lorentzTransformation(frame_of_reference_velocity, list_of_particles[i])
    plotAll(list_of_particles, corresponding_colour_list)
elif newton and not lorentz:
    for i in range(len(list_of_particles)):
        list_of_particles[i] = galileanTransformation(frame_of_reference_velocity, list_of_particles[i])
    plotAll(list_of_particles, corresponding_colour_list)

for k in range(1, 6):
    plt.plot([0, (20-4*k)/9], [k, (20+5*k)/9], color="orange", linestyle="dashed")

plt.legend()

# saving plot
plt.savefig("Tvillingeparadoks.png", dpi=300)
plt.show()


#inspiration from the pygame aliens example
# http://66.media.tumblr.com/2a29a324317a09f9744886bb70f5c124/tumblr_nbq2z80NAq1s5xy3ro1_1280.gif
# http://animizer.net/en/gif-apng-splitter

import random
import os.path
import os
import time

import pygame
from pygame.locals import *

from Explosion import Explosion
from Creep import Creep
from Tower import Tower
from Projectile import Projectile

###############################################################################

SCREENRECT     = Rect(0, 0, 1530, 960)

#main_dir = os.path.split(os.path.abspath(__file__))[0]
main_dir = '.'

###############################################################################

def load_image(file, scale=1.0, transparent_color=None):
	#loads an image, prepares it for play

	#try:
	surface = pygame.image.load(file)

	surface_w, surface_h = surface.get_size()
	surface = pygame.transform.scale(surface, (int(surface_w*scale), int(surface_h*scale)))

	if transparent_color != None:
		surface.set_colorkey( transparent_color, RLEACCEL )

	#except pygame.error:
	#	raise SystemExit('Could not load image "%s" %s'%(file, pygame.get_error()))
	return surface
#END def load_image(file):

def load_images(path_list, scale=1.0, transparent_color=None):
	imgs = []

	dir = main_dir
	for p in path_list:
		dir = os.path.join(dir, p)

	for file in os.listdir(dir):
		full_path = os.path.join(dir, file)
		imgs.append(load_image(full_path, scale, transparent_color))
	return imgs
#END def load_images(*files):

###############################################################################

def main():
	# Initialize pygame
	pygame.init()

	clock = pygame.time.Clock()

	# Set the display mode
	winstyle = 0  # |FULLSCREEN
	bestdepth = pygame.display.mode_ok(SCREENRECT.size, winstyle, 32)
	screen = pygame.display.set_mode(SCREENRECT.size, winstyle, bestdepth)

	#Set the SCREENRECT in the classes
	Creep.SCREENRECT = SCREENRECT
	Tower.SCREENRECT = SCREENRECT
	Projectile.SCREENRECT = SCREENRECT

	#decorate the game window
	#icon = pygame.transform.scale(Creep.images[0], (32, 32))
	#pygame.display.set_icon(icon)							#Change the system image for the display window
	pygame.display.set_caption('Markers Tower Defense')		#window name
	#pygame.mouse.set_visible(0)	#TODO: set cursor as a target

	#create the background, tile the bgd image
	bgdtile = load_image(os.path.join('backgrounds', 'background.jpg'))
	background = pygame.Surface(SCREENRECT.size)
	for x in range(0, SCREENRECT.width, bgdtile.get_width()):
		for y in range(0, SCREENRECT.height, bgdtile.get_height()):
			background.blit(bgdtile, (x, y))
	screen.blit(background, (0,0))
	pygame.display.flip()

	#set the Tower images
	img = load_image(os.path.join('tower', 'tower.png'), 0.2)
	Tower.images = [img]

	#set the Creep images
	Creep.image_map = {}
	for color in ['red', 'green', 'blue']:
		Creep.image_map[color] = load_images(['ufo_' + color], 0.20)

	#set the explosion images
	Explosion.images = load_images(['throwable', 'explosion_all'], 5.0)

	#set the Projectile images
	Projectile.images = load_images(['throwable', 'fireball'], 0.3, (0,0,255))

	# Initialize Game Groups
	creeps = pygame.sprite.Group()
	projectiles = pygame.sprite.Group()
	towers = pygame.sprite.Group()
	explosions = pygame.sprite.Group()
	all = pygame.sprite.RenderUpdates()

	#assign default groups to each sprite class
	Creep.containers = creeps, all
	Projectile.containers = projectiles, all
	Tower.containers = towers, all
	Explosion.containers = explosions, all

	tower_cycle = 5
	tower_load = tower_cycle

	creep_cycle = 25
	creep_load = creep_cycle

	projectile_cycle = 10
	projectile_load = projectile_cycle

	score = 0

	while True:	#player.alive():
		loop_start_time = time.time()

		#get input
		for event in pygame.event.get():
			if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
				return
		#END for event

		#key and mouse state
		keystate = pygame.key.get_pressed()
		mousestate = pygame.mouse.get_pressed()
		mouse_position = pygame.mouse.get_pos()

		# clear/erase the last drawn sprites
		all.clear(screen, background)

		#update all the sprites
		all.update()

		#load new creeps
		creep_load -= 1
		if creep_load < 0:
			#Creep(int(random.random()*10))
			Creep(40)
			creep_load = creep_cycle

		#load new projectiles
		projectile_load -= 1
		if projectile_load < 0:
			for t in towers:
				tower_position = t.center_position()
				projectile_direction = [mouse_position[0] - tower_position[0], mouse_position[1] - tower_position[1]]
				Projectile(t.center_position(), projectile_direction)
			projectile_load = projectile_cycle

		#load new towers
		tower_load -= 1
		if mousestate == (0,0,1) and tower_load < 0:
			Tower()
			tower_load = tower_cycle

		#detect collisions
		"""for p in projectiles:
			aliens_hit = pygame.sprite.spritecollide(p, creeps, False)
			if len(aliens_hit) > 0:
				p.kill()
				for a in aliens_hit:
					Explosion(a)
					a.take_damage(p.get_damage())"""

		for c in creeps:
			for p in pygame.sprite.spritecollide(c, projectiles, False):
				p.kill()
				Explosion(c)
				c.take_damage(p.get_damage())

		#draw the scene
		dirty = all.draw(screen)
		pygame.display.update(dirty)

		loop_end_time = time.time()
		#print loop_end_time - loop_start_time

		#cap the framerate
		clock.tick(30)

	#END while player.alive():

	pygame.time.wait(1000)
	pygame.quit()

#END def main():

if __name__ == '__main__':
	main()


from django.contrib import admin
from role_murid.models import *
from sekolah.models import *
import csv
from django.http import HttpResponse

# Register your models here.
class ExportCsvMixin:
    def export_as_csv(self, request, queryset):

        meta = self.model._meta
        field_names = [field.name for field in meta.fields]

        response = HttpResponse(content_type='text/csv')
        response['Content-Disposition'] = 'attachment; filename={}.csv'.format(meta)
        writer = csv.writer(response)

        writer.writerow(field_names)
        for obj in queryset:
            row = writer.writerow([getattr(obj, field) for field in field_names])

        return response

    export_as_csv.short_description = "Export Selected"

class Profil_Murid_admin(admin.ModelAdmin, ExportCsvMixin):
	list_display = (
			'murid_profil_uuid',
			'murid_nama',
			'murid_nisn',
			'murid_kelas',
			'murid_semester',
			'murid_tahun_ajaran'
		)
	readonly_fields = ['murid_profil_uuid',]
	actions = ['export_as_csv']
	def formfield_for_foreignkey(self, db_field, request, **kwargs):
		if db_field.name == 'murid_user':
			kwargs['queryset'] = Akun.objects.filter(is_murid=True)
		return super().formfield_for_foreignkey(db_field, request, **kwargs)
admin.site.register(Profil_Murid, Profil_Murid_admin)


import pandas as pd
import os
import zipfile

import requests

from os import remove
from io import BytesIO
from flask import flash, Flask, request, render_template, send_file, url_for, redirect
from flask_swagger_ui import get_swaggerui_blueprint
from flask_wtf import FlaskForm
from flask_bootstrap import Bootstrap
from wtforms import StringField

from wtforms.validators import DataRequired, URL

from werkzeug.utils import secure_filename
from config import config

ALLOWED_EXTENSIONS = {'xls', 'xlsx'}
config_name = os.environ.get("APP_MODE") or "development"

app = Flask(__name__)
app.config.from_object(config[config_name])

bootstrap = Bootstrap(app)

SWAGGER_URL = "/api/docs"
API_URL = "/static/swagger.json"
swaggerui_blueprint = get_swaggerui_blueprint(
    SWAGGER_URL,
    API_URL,
    config={
        "app_name": "XLStoCSV"
    }
)

app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

class StartForm(FlaskForm):
    data_url = StringField('URL to xls file', validators=[DataRequired(), URL()], description='Paste URL to a excel file containing data-sheets')

@app.route('/', methods=['GET', 'POST'])
def index():
    logo = './static/resources/MatOLab-Logo.svg'
    start_form = StartForm()
    return render_template("index.html", logo=logo, start_form=start_form)

@app.route("/api", methods=["GET", "POST"])
def api():
    if request.method == 'POST':

        url = request.form.get("data_url")

        if not allowed_file(url):
            flash("cannot convert file with given extension!")
            return redirect(url_for("index"))

        # we need to access the raw file from github, without html code
        if "github.com" in url:
            url = url.replace("github.com", "raw.githubusercontent.com").replace("/blob/", "/")

        filename = secure_filename(url.rsplit("/", maxsplit=1)[1])

        # write the file to local directory, the file is deleted again after conversion
        response = requests.get(url)
        output = open(filename, 'wb')
        output.write(response.content)
        output.close()

        excel_file = pd.ExcelFile(filename)

        memory_file = BytesIO()

        # for each input file, create sheets, for each sheet, try to add csv to zipfile

        with zipfile.ZipFile(memory_file, 'w') as zf:

            prefix = filename.replace(".xls", "")

            for sheetname in excel_file.sheet_names:

                try:
                    df = pd.read_excel(excel_file, sheetname)
                    plain_text = df.to_csv(index=False)

                    tmp = secure_filename(prefix + "_" + sheetname + ".csv")
                    zf.writestr(zinfo_or_arcname=tmp, data=plain_text)

                except:
                    # sometimes, excel sheets can't be turned into csv files,e.g. when
                    # the excel sheet contains a diagram. In this case, omit the current sheet
                    continue

        memory_file.seek(0)

        # we're done with conversion, delete the local file.
        excel_file.close()
        remove(filename)

        return send_file(memory_file, attachment_filename='result.zip', as_attachment=True)

    return render_template("index.html")


from windowsget_pdm import download
import pytest
from pathlib import Path
import shutil

correct_filepaths = [
    ("https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data", "x/y/test.csv"),
    ("https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data", "example.csv"),
]

bad_filepaths = [
    ("https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data", "x/y/test"),
    ("https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data", "example"),
]

@pytest.mark.parametrize("url, filepath", correct_filepaths)
def test_correct_url_and_filepath(url, filepath):
    TEST_DIR = "test_dir"
    test_filepath = f"{TEST_DIR}/{filepath}"
    response = download(url=url, filepath=Path(test_filepath))
    assert isinstance(response, int)
    assert Path(test_filepath).exists()
    shutil.rmtree(TEST_DIR)

@pytest.mark.parametrize("url, filepath", bad_filepaths)
def test_bad_url_and_filepath(url, filepath):
    TEST_DIR = "test_dir"
    test_filepath = f"{TEST_DIR}/{filepath}"
    with pytest.raises(ValueError):
        response = download(url=url, filepath=Path(test_filepath))
    shutil.rmtree(TEST_DIR)


import urllib3
import time

def test_workflow_execution(workflow_api, dataplane_api, stack_resources, testing_env_variables):
    workflow_api = workflow_api()
    dataplane_api = dataplane_api()

    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    test_preprocess_stage = {"Name": "TestPreprocess", "Operations": ["Mediainfo", "Thumbnail"]}
    test_video_stage = {"Name": "TestVideo", "Operations": ["celebrityRecognition", "contentModeration", "faceDetection", "labelDetection",  "personTracking", "shotDetection", "textDetection", "Mediaconvert", "technicalCueDetection"]}
    test_audio_stage = {"Name": "TestAudio", "Operations": ["TranscribeVideo"]}
    test_webcaptions_stage = {"Name": "TestWebCaptions", "Operations": ["WebCaptions"]}
    test_text_stage = {"Name": "TestText", "Operations": ["TranslateWebCaptions", "ComprehendKeyPhrases", "ComprehendEntities"]}

    test_workflow = {
        "Name": "TestingWF",
        "StartAt": "TestPreprocess",
        "Stages": {
            "TestPreprocess": {
                "Next": "TestVideo"
            },
            "TestVideo": {
                "Next": "TestAudio"
            },
            "TestAudio": {
                "Next": "TestWebCaptions"
            },
            "TestWebCaptions": {
                "Next": "TestText"
            },
            "TestText": {
                "End": True
            }
        }
    }

    test_execution = {
        "Name": "TestingWF",
        "Input": {
            "Media": {
                "Video": {
                    "S3Bucket": stack_resources['DataplaneBucket'],
                    "S3Key": 'upload/' + testing_env_variables['SAMPLE_VIDEO']
                }
            }
        }
    }

    # clean up previously incomplete tests
    print("Cleaning up previously incomplete tests")

    delete_preprocess_stage_request = workflow_api.delete_stage_request(test_preprocess_stage["Name"])
    delete_video_stage_request = workflow_api.delete_stage_request(test_video_stage["Name"])
    delete_audio_stage_request = workflow_api.delete_stage_request(test_audio_stage["Name"])
    delete_webcaptions_stage_request = workflow_api.delete_stage_request(test_webcaptions_stage["Name"])
    delete_text_stage_request = workflow_api.delete_stage_request(test_text_stage["Name"])
    if workflow_api.get_workflow_request(test_workflow["Name"]).status_code == 200:
        workflow_api.delete_workflow_request(test_workflow["Name"])
        # Wait for the workflow to delete. It can take several seconds.
        time.sleep(30)

    # create stages

    preprocess_stage_request = workflow_api.create_stage_request(test_preprocess_stage)
    assert preprocess_stage_request.status_code == 200

    video_stage_request = workflow_api.create_stage_request(test_video_stage)
    assert video_stage_request.status_code == 200

    audio_stage_request = workflow_api.create_stage_request(test_audio_stage)
    assert audio_stage_request.status_code == 200

    webcaptions_stage_request = workflow_api.create_stage_request(test_webcaptions_stage)
    assert webcaptions_stage_request.status_code == 200

    text_stage_request = workflow_api.create_stage_request(test_text_stage)
    assert text_stage_request.status_code == 200

    # create workflow

    workflow_create_request = workflow_api.create_workflow_request(test_workflow)
    assert workflow_create_request.status_code == 200

    # create workflow execution

    workflow_execution_request = workflow_api.create_workflow_execution_request(test_execution)
    assert workflow_execution_request.status_code == 200

    workflow_execution_response = workflow_execution_request.json()
    workflow_execution_id = workflow_execution_response["Id"]
    asset_id = workflow_execution_response["AssetId"]

    time.sleep(5)

    # wait for workflow to complete

    workflow_processing = True

    while workflow_processing:
        get_workflow_execution_request = workflow_api.get_workflow_execution_request(workflow_execution_id)
        get_workflow_execution_response = get_workflow_execution_request.json()
        assert get_workflow_execution_request.status_code == 200

        workflow_status = get_workflow_execution_response["Status"]
        print("Workflow Status: {}".format(workflow_status))

        allowed_statuses = ["Started", "Queued", "Complete"]
        assert workflow_status in allowed_statuses

        if workflow_status == "Complete":
            workflow_processing = False
        else:
            print('Sleeping for 10 seconds before retrying')
            time.sleep(10)

    # Get asset mediainfo from dataplane

    asset_mediainfo_request = dataplane_api.get_single_metadata_field(asset_id, "Mediainfo")
    assert asset_mediainfo_request.status_code == 200

    print(asset_mediainfo_request.json())

    # Delete asset

    delete_asset_request = dataplane_api.delete_asset(asset_id)
    assert delete_asset_request.status_code == 200

    # Delete workflow

    delete_workflow_request = workflow_api.delete_workflow_request(test_workflow["Name"])
    assert delete_workflow_request.status_code == 200
    # Wait for the workflow to delete. It can take several seconds.
    time.sleep(30)

    # Delete stages

    delete_preprocess_stage_request = workflow_api.delete_stage_request(test_preprocess_stage["Name"])
    assert delete_preprocess_stage_request.status_code == 200

    delete_video_stage_request = workflow_api.delete_stage_request(test_video_stage["Name"])
    assert delete_video_stage_request.status_code == 200

    delete_audio_stage_request = workflow_api.delete_stage_request(test_audio_stage["Name"])
    assert delete_audio_stage_request.status_code == 200

    delete_webcaptions_stage_request = workflow_api.delete_stage_request(test_webcaptions_stage["Name"])
    assert delete_webcaptions_stage_request.status_code == 200

    delete_text_stage_request = workflow_api.delete_stage_request(test_text_stage["Name"])
    assert delete_text_stage_request.status_code == 200


"""
Author: Anirban Mukhopadhyay, Prof. Anil's magnonics group, IITM, India
Created on: April 9, 2021
Version: 0.2
"""

import numpy as np
import matplotlib.pyplot as plt
import os as os
import fnmatch as fn
from scipy.signal import windows as win

class SWLib:
    class CalcSWProp:
        def __init__(self, h_0, k, film_thickness, saturation_magnetization, sw_type, lambda_ex=None,
                     exchange=False):
            self.h_0 = h_0
            self.k = k
            self.dk = np.average(np.diff(k))
            self.film_thickness = film_thickness
            self.ms = saturation_magnetization
            self.sw_type = sw_type
            self.exch = exchange
            self.f_0_bar = self.h_0 / self.ms
            if self.exch is True:
                self.lambda_ex = lambda_ex
                self.f_ex_bar = self.f_0_bar + self.lambda_ex * self.k ** 2
            else:
                pass

        def bvsw_disp(self):
            with np.errstate(divide='ignore', invalid='ignore'):
                if self.exch is True:
                    buffer = np.sqrt(self.f_ex_bar ** 2 + self.f_ex_bar * (
                            1 - np.exp(-np.abs(self.k) * self.film_thickness)) / (np.abs(self.k) * self.film_thickness))
                else:
                    buffer = np.sqrt(self.f_0_bar ** 2 + self.f_0_bar * (
                            1 - np.exp(-np.abs(self.k) * self.film_thickness)) / (np.abs(self.k) * self.film_thickness))
            buffer[np.where(np.isnan(buffer))] = np.sqrt(self.f_0_bar ** 2 + self.f_0_bar)
            return buffer

        def fvsw_disp(self):
            with np.errstate(divide='ignore', invalid='ignore'):
                if self.exch is True:
                    buffer = np.sqrt(self.f_ex_bar ** 2 + self.f_ex_bar - self.f_ex_bar * (
                            1 - np.exp(-np.abs(self.k) * self.film_thickness)) / (np.abs(self.k) * self.film_thickness))
                else:
                    buffer = np.sqrt(self.f_0_bar ** 2 + self.f_0_bar - self.f_0_bar * (
                            1 - np.exp(-np.abs(self.k) * self.film_thickness)) / (np.abs(self.k) * self.film_thickness))
            buffer[np.where(np.isnan(buffer))] = self.f_0_bar
            return buffer

        def ssw_disp(self):
            with np.errstate(divide='ignore', invalid='ignore'):
                if self.exch is True:
                    buffer = np.sqrt(
                        self.f_ex_bar ** 2 + self.f_ex_bar + (
                                    1 - np.exp(-2 * np.abs(self.k) * self.film_thickness)) / 4)
                else:
                    buffer = np.sqrt(
                        self.f_0_bar ** 2 + self.f_0_bar + (1 - np.exp(-2 * np.abs(self.k) * self.film_thickness)) / 4)
            buffer[np.where(np.isnan(buffer))] = np.sqrt(self.f_0_bar ** 2 + self.f_0_bar)
            return buffer

        def calc_dispersion_curve_lowest_order_mode(self):
            dict = {"bvsw": self.bvsw_disp, "fvsw": self.fvsw_disp, "ssw": self.ssw_disp}
            func = dict.get(self.sw_type, "Invalid spin-wave class")
            fig, ax = plt.subplots(1, 1, squeeze=True)
            ax.plot(self.k, func(), "b--")
            ax.set_xlabel(r"$k$ (rad/nm)", fontsize=15)
            ax.set_ylabel(r"$f/f_{\rm{M}}$", fontsize=15)
            ax.tick_params(labelsize=12)

        def calc_grvel_lowest_order_mode(self):
            dict = {"bvsw": self.bvsw_disp, "fvsw": self.fvsw_disp, "ssw": self.ssw_disp}
            func = dict.get(self.sw_type, "Invalid spin-wave class")
            group_velocity = np.diff(func()) / self.dk
            fig, ax = plt.subplots(1, 1, squeeze=True)
            ax.plot(self.k[:-1], group_velocity, "b--")
            ax.set_xlabel(r"$k$ (rad/nm)", fontsize=15)
            ax.set_ylabel(r"$v_{\rm{g}}/f_{\rm{M}}$ (nm/rad)", fontsize=15)
            ax.tick_params(labelsize=12)

    class MagOvfPostProcessor:
        def __init__(self, folder_path, run_time, dt, x_in, length, dx, window_choice,
                     zero_pad=None, temporal_limits=None, spatial_limits=None):
            # Optional arguments value assignment if None
            if zero_pad is None:
                zero_pad = [2048, 2048]

            if temporal_limits is None:
                temporal_limits = [0, run_time]
            if spatial_limits is None:
                spatial_limits = [x_in, x_in + length]

            self.path = folder_path
            self.run_time, self.dt, self.n_t = run_time, dt, int(run_time / dt + 1)
            self.x_in, self.length, self.dx, self.n_x = x_in, length, dx, int(length / dx)
            self.window_choice = window_choice
            self.zero_pad = zero_pad
            self.temporal_limits = temporal_limits
            self.spatial_limits = spatial_limits

        def importer(self):
            my_file_list = []  # creating a empty list to store paths
            with os.scandir(self.path) as entries:
                for entry in entries:
                    if fn.fnmatch(entry.name, 'm*.ovf'):
                        my_file_list.append(entry.name)

            my_file_list = sorted(my_file_list)  # sorting filenames in increasing order of current value
            mag_data_spacetime = []
            [mag_data_spacetime.append(np.loadtxt(self.path + file, usecols=1)) for file in
             my_file_list]  # choosing y-component

            # reshape(no. if time instances (axis 0), no. of cells along y-axis (axis 1), no. of cells along x-axis (axis
            # 2))
            mag_data_spacetime = np.array(mag_data_spacetime).reshape(self.n_t, -1, self.n_x)
            return mag_data_spacetime

        def slicer(self):
            mag_data_spacetime = self.importer()
            i_upr, i_lwr = int(self.temporal_limits[1] / self.dt), int(self.temporal_limits[0] / self.dt)
            j_upr, j_lwr = int((self.spatial_limits[1] - self.x_in) / self.dx), int(
                (self.spatial_limits[0] - self.x_in) / self.dx)
            return mag_data_spacetime[i_lwr: i_upr + 1, :, j_lwr: j_upr + 1], mag_data_spacetime[0, :, j_lwr: j_upr + 1]

        def kaiser2d(self, shape, beta=14):
            return np.outer(win.kaiser(shape[0], beta), win.kaiser(shape[1], beta))

        def chebwin2d(self, shape, att=95):
            return np.outer(win.chebwin(shape[0], att), win.chebwin(shape[1], att))

        def window2d(self, shape):
            dict = {"kai": self.kaiser2d, "che": self.chebwin2d}
            func = dict.get(self.window_choice, "Invalid window method")
            return func(shape)

        def pre_conditioner(self):
            mag_data_spacetime, init_mag_data_spacetime = self.slicer()
            # removing the intial magnetization state
            mag_data_spacetime = np.array([mag_data_spacetime[i, :, :] - init_mag_data_spacetime for i in
                                           range(mag_data_spacetime.shape[0])])
            # averaging over y-axis & row no. = no. of time instances
            # column no. = no. of cells along x-axis
            mag_data_spacetime = np.average(mag_data_spacetime, axis=1)
            mag_data_spacetime *= self.window2d(mag_data_spacetime.shape)
            mag_pwr = 20 * np.log10(np.abs(np.fft.fftshift(np.fft.fft2(
                mag_data_spacetime, s=self.zero_pad, norm="ortho"))))  # dBm
            fmin, fmax = np.amin(np.fft.fftshift(np.fft.fftfreq(mag_pwr.shape[0], self.dt))), \
                         np.amax(np.fft.fftshift(np.fft.fftfreq(mag_pwr.shape[0], self.dt)))  # GHz
            kmin, kmax = np.amin(np.fft.fftshift(np.fft.fftfreq(mag_pwr.shape[1], self.dx) * 2 * np.pi)), \
                         np.amax(np.fft.fftshift(np.fft.fftfreq(mag_pwr.shape[1], self.dx) * 2 * np.pi))  # rad/nm
            return [kmin, kmax], [fmin, fmax], mag_pwr

        def spatio_temporal_plottr(self, axs_label, axs_range, cmap_choice="bwr", cbar_limits=None):
            mag_data_spacetime, init_mag_data_spacetime = self.slicer()
            # removing the intial magnetization state
            mag_data_spacetime = np.array([mag_data_spacetime[i, :, :] - init_mag_data_spacetime for i in
                                           range(mag_data_spacetime.shape[0])])
            # averaging over y-axis & row no. = no. of time instances
            # column no. = no. of cells along x-axis
            mag_data_spacetime = np.average(mag_data_spacetime, axis=1)
            boundary = [self.spatial_limits, self.temporal_limits]
            boundary = list(np.concatenate(boundary).flat)
            fig, ax = plt.subplots(1, 1)
            if cbar_limits is None:
                im = ax.imshow(mag_data_spacetime, cmap=cmap_choice, vmin=np.amin(mag_data_spacetime),
                               vmax=np.amax(mag_data_spacetime), interpolation='antialiased',
                               extent=boundary, origin='lower', aspect='auto')
            else:
                im = ax.imshow(mag_data_spacetime, cmap=cmap_choice, vmin=cbar_limits[0],
                               vmax=cbar_limits[1], interpolation='antialiased',
                               extent=boundary, origin='lower', aspect='auto')
            ax.set_xlabel(r"$" + axs_label[0] + "$ (nm)", fontsize=15)
            ax.set_ylabel(r"$" + axs_label[1] + "$ (ns)", fontsize=15)
            ax.set_xlim(axs_range[:2])
            ax.set_ylim(axs_range[2:])
            cbar = plt.colorbar(im, shrink=0.95)
            cbar.set_label(label=r"Magnetization (1)", size=12)
            ax.tick_params(labelsize=12)

        def dispersion_plottr(self, axs_label, axs_range, cmap_choice="inferno", cbar_limits=None):
            k, f, mag_pwr = self.pre_conditioner()
            boundary = [k, f]
            boundary = list(np.concatenate(boundary).flat)
            fig, ax = plt.subplots(1, 1)
            if cbar_limits is None:
                im = ax.imshow(mag_pwr, cmap=cmap_choice, vmin=np.amin(mag_pwr),
                                vmax=np.amax(mag_pwr), interpolation='antialiased',
                                extent=boundary, origin='lower', aspect='auto')
            else:
                im = ax.imshow(mag_pwr, cmap=cmap_choice, vmin=cbar_limits[0],
                                vmax=cbar_limits[1], interpolation='antialiased',
                                extent=boundary, origin='lower', aspect='auto')
            ax.set_xlabel(r"$" + axs_label[0] + "$ (rad/nm)", fontsize=15)
            ax.set_ylabel(r"$" + axs_label[1] + "$ (GHz)", fontsize=15)
            ax.set_xlim(axs_range[:2])
            ax.set_ylim(axs_range[2:])
            cbar = plt.colorbar(im, shrink=0.95)
            cbar.set_label(label=r"Power (dB)", size=12)
            ax.tick_params(labelsize=12)


from io import StringIO
import re
import sys
import time

from oasa import converter_base
from oasa import plugin
from oasa import molecule

class Molfile(plugin.Plugin):

    name = "molfile"
    read = 1
    write = 1

    def __init__(self, structure=None):
        "Initializes the Molfile class."""
        self.structure = structure

    def set_structure(self, structure):
        self.structure = structure

    def get_structure(self):
        return self.structure

    def read_file(self, input_file):
        self._read_header(input_file)
        self._read_body(input_file)

    def _read_header(self, input_file):
        for i in range(3):
            input_file.readline()

    def _read_body(self, input_file):
        atoms = read_molfile_value(input_file, 3, conversion=int)
        bonds = read_molfile_value(input_file, 3, conversion=int)
        # nothing more interesting
        input_file.readline()
        # read the structure
        self.structure = molecule.Molecule()
        for i in range(atoms):
            a = self._read_atom(input_file)
            self.structure.add_vertex(v=a)
        for k in range(bonds):
            b, i, j = self._read_bond(input_file)
            self.structure.add_edge(i, j, e=b)
        for line in input_file:
            if line.strip() == "M  END":
                break
            if line.strip().startswith("M  "):
                self._read_property(line.strip())
            # print(line.strip())

    def _read_atom(self, input_file):
        x = read_molfile_value(input_file, 10, conversion=float)
        y = read_molfile_value(input_file, 10, conversion=float)
        z = read_molfile_value(input_file, 10, conversion=float)
        read_molfile_value(input_file, 1)  # empty space
        symbol = read_molfile_value(input_file, 3)
        mass_diff = read_molfile_value(input_file, 2)
        charge, multi = self._read_molfile_charge(
            read_molfile_value(input_file, 3, conversion=int))
        input_file.readline()  # next line please
        ret = self.structure.create_vertex()
        ret.coords = (x, y, z)
        ret.charge = charge
        ret.symbol = symbol
        ret.multiplicity = multi
        return ret

    def _read_bond(self, input_file):
        # molfiles index from 1
        a1 = read_molfile_value(input_file, 3, conversion=int) - 1
        a2 = read_molfile_value(input_file, 3, conversion=int) - 1
        order = read_molfile_value(input_file, 3, conversion=int)
        bond_type = read_molfile_value(input_file, 3, conversion=int)
        type_remap = {0: 'n', 1: 'w', 6: 'h', 4: 'a'}
        bond_type = type_remap.get(bond_type, 'n')
        input_file.readline()  # next line please
        b = self.structure.create_edge()
        b.order = order
        b.type = bond_type
        return b, a1, a2

    def _read_property(self, prop):
        m = re.match("M\s+RAD\s+(\d+)(.*)", prop)
        if m:
            for at, rad in re.findall("(\d+)\s+(\d+)", m.group(2)):
                index = int(at)
                multi = int(rad)
                self.structure.vertices[index-1].multiplicity = multi
        m = re.match("M\s+CHG\s+(\d+)(.*)", prop)
        if m:
            for at, chg in re.findall("(\d+)\s+(-?\d+)", m.group(2)):
                index = int(at)
                charge = int(chg)
                self.structure.vertices[index-1].charge = charge

    def write_file(self, output_file):
        """file should be a writable file object"""
        if not self.structure:
            raise Exception("No structure to write")
        self._write_header(output_file)
        self._write_counts_line(output_file)
        self._write_body(output_file)
        self._write_m_lines(output_file)
        output_file.write('M  END\n')

    def get_text(self):
        return ''.join((
            self._get_header(), self._get_counts_line(), self._get_body(), 'M  END'
        ))

    def _write_header(self, input_file):
        input_file.write(self._get_header())

    def _get_header(self):
        return "\n\n\n"

    def _write_counts_line(self, input_file):
        input_file.write(self._get_counts_line())

    def _get_counts_line(self):
        atoms = len(self.structure.vertices)
        bonds = len(self.structure.edges)
        atom_lists = 0
        fff = 0  # obsolete
        chiral = 0
        stexts = 0
        obsolete = "  0  0  0  0"
        extras = 999
        mol_version = " V2000"
        return "%3d%3d%3d%3d%3d%3d%s%s%s\n" % (
            atoms, bonds, atom_lists, fff, chiral,
            stexts, obsolete, extras, mol_version
        )

    def _write_body(self, input_file):
        input_file.write(self._get_body())

    def _get_body(self):
        lines = []
        for a in self.structure.vertices:
            lines.append(self._get_atom_line(a))
        for b in self.structure.edges:
            lines.append(self._get_bond_line(b))
        return '\n'.join(lines)+'\n'

    def _get_atom_line(self, a):
        x = a.get_x()
        y = a.get_y()
        z = a.get_z()
        symbol = a.symbol
        mass_diff = 0
        charge = self._get_molfile_charge(a.charge, a.multiplicity)
        rest = "  0  0  0  0  0  0  0  0  0  0"
        return "%10.4f%10.4f%10.4f %-3s%2d%3d%s" % (x, y, z, symbol, mass_diff, charge, rest)

    def _get_bond_line(self, b):
        v1, v2 = b.get_vertices()
        a1 = self.structure.vertices.index(v1) + 1
        a2 = self.structure.vertices.index(v2) + 1
        order = b.order
        type_remap = {'n': 0, 'w': 1, 'h': 6, 'a': 4, 'b': 0, 'd': 0}
        bond_type = type_remap.get(b.type, 0)
        rest = "  0  0  0"
        return "%3d%3d%3d%3d%s" % (a1, a2, order, bond_type, rest)

    def _write_m_lines(self, input_file):
        m_lines = self._get_m_lines()
        if m_lines:
            input_file.write("\n".join(m_lines) + "\n")

    def _get_m_lines(self):
        # radicals
        radicals = {}
        i = 1
        for v in self.structure.vertices:
            if v.multiplicity != 1:
                radicals[i] = v.multiplicity
            i += 1
        if radicals:
            assert len(radicals) <= 8
            nums = sorted(radicals.keys())
            rads = " ".join(["%3d %3d" % (n, radicals[n]) for n in nums])
            rad_line = "M  RAD%3d %s" % (len(nums), rads)
        else:
            rad_line = None
        # return m_lines
        m_lines = [_f for _f in [rad_line] if _f]
        return m_lines

    def _read_molfile_charge(self, value):
        if value == 0:
            return (0, 1)
        ch = -(value-4)
        multi = ch == 0 and 2 or 1
        return (ch, multi)

    def _get_molfile_charge(self, charge, multiplicity):
        if charge == 0:
            if multiplicity == 1:
                return 0
            else:
                return 4
        else:
            return 4-charge

def read_molfile_value(input_file, length, strip=1, conversion=None):
    """reads specified number of characters, if strip strips whitespace,
    if conversion (a fuction taking one string argument) applies it;
    if empty string is obtained after stripping 0 is returned"""
    str = input_file.read(length)
    if strip:
        str = str.strip()
    if str == "":
        return 0
    if conversion:
        str = conversion(str)
    return str

reads_text = 1
reads_files = 1
writes_text = 1
writes_files = 1

def mol_to_text(molfile):
    molfile_object = Molfile()
    molfile_object.structure = molfile
    return molfile_object.get_text()

def mol_to_file(mol, output_file):
    molfile_object = Molfile()
    molfile_object.structure = mol
    molfile_object.write_file(output_file)

def file_to_mol(input_file):
    molfile_object = Molfile()
    molfile_object.read_file(input_file)
    return molfile_object.structure

def text_to_mol(text):
    return file_to_mol(StringIO(text))

class MolfileConverter(converter_base.ConverterBase):

    # standard converter attrs
    reads_text = True
    writes_text = True
    reads_files = True
    writes_files = True

    default_configuration = {
    }

    def __init__(self):
        pass

    def mols_to_text(self, structures):
        output_file = StringIO()
        self.mols_to_file(structures, output_file)
        return output_file.getvalue()

    def read_text(self, text):
        super().read_text(text)
        mf = StringIO(text)
        for molfile in self.read_file(mf):
            yield mol

    def read_file(self, input_file):
        super().read_file(input_file)
        chunk = []
        m = Molfile()
        for line in input_file:
            if line.strip() != "$$$$":
                chunk.append(line)
            else:
                m.read_file(StringIO("".join(chunk)))
                yield m.structure
                chunk = []
        if [line for line in chunk if line.strip()]:  # non empty lines
            m.read_file(StringIO("".join(chunk)))
            yield m.structure

    def mols_to_file(self, structures, output_file):
        super().mols_to_file(structures, output_file)
        molfile_object = Molfile()
        first = output_file.tell() == 0
        for mol in structures:
            if not first:
                output_file.write("$$$$\n")
            first = False
            molfile_object.structure = mol
            molfile_object.write_file(f)
        self.last_status = self.STATUS_OK

if __name__ == "__main__":

    if len(sys.argv) < 1:
        print("you must supply a filename")
        sys.exit()

    file_name = sys.argv[1]
    with open(file_name, 'r') as f:
        mol = file_to_mol(f)

    for a in mol.atoms:
        print(a.x, a.y)

    t = time.time()
    lens = sorted(map(len, mol.get_smallest_independent_cycles()))
    print(lens)
    print(time.time() - t)
    print("total %d rings" % len(lens))
    print(mol)


import numpy as np
import cv2
import argparse
import time

ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video", help = "path to the (optional) video file" )
args = vars(ap.parse_args())

blueLower = np.array([100, 67, 0], dtype = "uint8")
blueUpper = np.array([255, 128, 50], dtype = "uint8")

camera = cv2.VideoCapture(args["video"])

while True:
    (ok, frame) = camera.read()
    if not ok:
        break
    blue = cv2.inRange(frame, blueLower, blueUpper)
    blue = cv2.GaussianBlur(blue, (3, 3), 0)

    (cnts, _) = cv2.findContours(blue.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    cv2.drawContours(frame, cnts, -1, (0, 255, 0), 2)

    cv2.imshow("Tracking color blue", frame)
    cv2.imshow("Binary", blue)

    time.sleep(0.255)
    if cv2.waitKey(1) & 0xFF == ord("q"):
        break

camera.realease()
cv2.destroyAllWindows()


import os
import telebot

from parser import Parser
from loader import Connector

TOKEN = os.getenv('TELEGRAM_BOT_TOKEN')
bot = telebot.TeleBot(TOKEN)
bot.remove_webhook()

@bot.message_handler(func=lambda message: True, commands=['firstnews', 'lastnews', 'listnews'])
def update_news_list(message):
    urls = parser.get_detailed()
    objs = [parser.follow(u) for u in urls]
    print(objs)
    connector.insert(objs)
    command = message.html_text
    if command == '/firstnews':
        payload = f"Title: {objs[-1]['title']}\nDate: {objs[-1]['date']}\nUrl: {objs[-1]['url']}\n"
    elif command == '/lastnews':
        payload = f"Title: {objs[0]['title']}\nDate: {objs[0]['date']}\nUrl: {objs[0]['url']}\n"
    elif command == '/listnews':
        payload = '\n\n'.join(['\n'.join([str(obj['date']), obj['title']]) for obj in objs])
    else:
        payload = 'wtf??'
    bot.send_message(message.chat.id, payload)

if __name__ == '__main__':
    connector = Connector()
    parser = Parser()
    bot.polling(none_stop=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Project ：PythonAdvanced
@File    ：multiprocessing_module_02.py
@Author  ：Toby
@Date    ：2022/11/22 17:35
@Description：进程pid
"""
import os
from multiprocessing import Process

def run_proc():
    """子进程运行的代码"""
    print('子进程运行中，pid=%s....' % os.getpid())
    print('子进程结束')

if __name__ == '__main__':
    print("父进程pid=%s" % os.getpid())
    p = Process(target=run_proc)
    p.start()


from django.conf import settings

from django.utils.translation import ugettext as _
from django.shortcuts import render, redirect

from django.contrib import messages
from django.contrib.auth import login as auth_login, logout as auth_logout
from django.contrib.auth.decorators import login_required

from main import forms

def login(request, template_name='base_form.html'):
    """Allow a user to log in with either username/email and password."""

    if request.user.is_authenticated():
        message = _('You are already logged in as') + u' %s!' % request.user.email
        messages.info(request, message)
        return redirect('home')

    if request.method == 'POST':
        form = forms.AuthenticationForm(data=request.POST)
        if form.is_valid():
            user = form.get_user()
            auth_login(request, user)

            redirect_url = form.cleaned_data.get('redirect_url', 'home')
            return redirect(redirect_url)
    else:
        form = forms.AuthenticationForm(initial={'redirect_url': request.GET.get('next', '/home')})

    template_vars = {
        'form': form,
    }

    return render(request, template_name, template_vars)

@login_required
def logout(request):
    auth_logout(request)
    return redirect('landing_page')

def sign_up(request, template_name='base_form.html'):
    """Allow a user to log in with either username/email and password."""
    form = forms.UserCreationForm()

    if request.user.is_authenticated():
        messages.success(request, _('You are already logged in!'))
        return redirect('home')

    if request.method == 'POST':
        form = forms.UserCreationForm(request.POST)

        if form.is_valid():

            user = form.save(commit=False)
            user.email = user.username
            user.save()

            user.backend = settings.AUTHENTICATION_BACKENDS[0]
            auth_login(request, user)

            return redirect('home')

    template_vars = {
        'form': form,
    }

    return render(request, template_name, template_vars)


# -*- coding: utf-8 -*-
import scrapy
from tangspiderframe.items import TangspiderframeItem

class TextPhilippinesAbanteLinkSpider(scrapy.Spider):
    name = 'text_philippines_abante_link'
    start_urls = ["https://www.abante.com.ph/"]

    def parse(self, response):
        links = response.xpath('//ul[@class="main-menu menu bsm-pure clearfix"]/li/a/@href').extract()
        for i in range(1, 3000):
            for link in links:
                link = link + "/page/{i}".format(i=i)
                yield scrapy.Request(url=link, callback=self.parse_url, dont_filter=True)

    def parse_url(self, response):
        # next_links = response.xpath('//a[(contains(@class, "btn-bs-pagination"))]/@href').extract()
        # for next_link in next_links:
        #     yield scrapy.Request(url=next_link, callback=self.parse_url, dont_filter=True)

        links = response.xpath('//h2/a[@class="post-title post-url"]/@href').extract()
        for link in links:
            item = TangspiderframeItem()
            item['url'] = link
            yield item


import pygame as pg
from libs.libs_FSM.Classic_FSM import FSM
from libs.utils.class_draw_utils import Text

# -------------------------------------------------------------------------------------------

SCREEN_DIM =        400,400

DIS_LUZ_PULS_X =    50
DIS_LUZ_PULS_Y =    150

POS_LUZ =       200,50
POS_LED_P1 =    DIS_LUZ_PULS_X, POS_LUZ[1] + DIS_LUZ_PULS_Y
POS_LED_P2 =    SCREEN_DIM[0] - DIS_LUZ_PULS_X, POS_LUZ[1] +DIS_LUZ_PULS_Y

POS_TEXT_P1 =    POS_LED_P1[0], POS_LED_P1[1] - 30
POS_TEXT_P2 =    POS_LED_P2[0], POS_LED_P2[1] - 30

DIM_PULS    = 10,20
RECT_P1 =     POS_LED_P1[0],POS_LED_P1[1],DIM_PULS[0], DIM_PULS[1]
RECT_P2 =     POS_LED_P2[0],POS_LED_P2[1],DIM_PULS[0], DIM_PULS[1]

RADIO_LUZ =     30
RADIO_PULS =    5

TEXT_PULS =     (255,255,0)

LED_LUZ_ON =    (255,255,255)
LED_LUZ_OFF =   (64,64,64)

LED_P1_ON =     (0,255,0)
LED_P1_OFF =    (0,64,0)
LED_P2_ON =     (0,255,0)
LED_P2_OFF =    (0,64,0)

# -------------------------------------------------------------------------------------------

class Luz(FSM):
    """
    ---------------------------------------------------------------------------------
      Problema: Control de la luz en los dos extremos de un pasillo público
    --------------------------------------------------------------------------------

      1 - El pasillo dispone de dos pulsadores, uno al lado de cada puerta, de
      manera que se pueda encender y apagar la luz desde cada extremo. Cada
      pulsador produce un ‘1’ lógico mientras está pulsado, y un ‘0’ lógico
      cuando no lo está.

      2 - Se desea que, cada vez que se pulse cualquier pulsador, la luz cambie
      de estado: si está apagada se debe encender, y viceversa.

      3 - Se debe tener en cuenta el caso en el que, mientras se pulsa un
      interruptor, se pulse el otro. Por ejemplo, si estando apagada la luz,
      alguien pulsa P1 se enciende la luz. Pero si mientras está pulsado P1
      alguien pulsa P2, entonces se apagará nuevamente la luz.

      4 - Hay que considerar la simultaneidad de pulsaciones

    """

    name = 'clt_luz'

    # -------------------------------------------------------------------------------------------

    def __init__(self, screen, fsm_data_dict):
        """

        :param screen:
        """

        # ===================================================================================

        super().__init__(fsm_data_dict)

        self.screen =       screen

        self.sw_P1 = False      # entrada
        self.sw_P2 = False      # entrada

        self.sw_luz = False     # salida

        self.txt_P1 = Text( FontSize= 20)
        self.txt_P2 = Text( FontSize= 20)

    # ----------------------------------------------------------------------------------------------

    def get_push_button_state(self, event):
        """
            Detecta actividad del botón de start del motor (tecla <s> del teclado)

        :param event:
        :return:
        """
        if event.type == pg.KEYDOWN:
            if event.key == pg.K_s:
                self.sw_P1 = True
            if event.key == pg.K_d:
                self.sw_P2 = True

        elif event.type == pg.KEYUP:
            if event.key == pg.K_s:
                self.sw_P1 = False
            if event.key == pg.K_d:
                self.sw_P2 = False

    # -------------------------------------------------------------------------------------------
    #   A C C I O N E S

    def set_action(self, out_event_dict):
        """
            activa o para giro del motor, prende o apaga luz de alarma

        :param event_dict:
        :return:
        """

        for key, value in out_event_dict.items():
            self.sw_luz = value
        self.sw_luz = out_event_dict['luz']

    # ----------------------------------------------------------------------------------------------

    def simulation_step(self):
        """
            Ejecuta un paso de la máquina de estados adquiriendo el código
            de evento co condición.

        :return: ninguno
        """

        event_dict = self.get_event_dict()
        in_event_dict = event_dict['inputs']
        in_event_dict['sw_p1'] = int(self.sw_P1)
        in_event_dict['sw_p2'] = int(self.sw_P2)

        self.out_event_dict = self.step(self.gets_coded_events(in_event_dict))
        self.set_action(self.out_event_dict)

    # ----------------------------------------------------------------------------------------------
    #   D I B U J O S

    def draw_light(self):
        """
            Dibuja simulación gráfica de la  luz con los leds de los pulsadores P1 y P2

        :return: ninguno
        """

        self.txt_P1.render(self.screen,'P1 (Tecla s)',POS_TEXT_P1, TEXT_PULS)
        self.txt_P2.render(self.screen,'P2 (Tecla d)',POS_TEXT_P2, TEXT_PULS)

        if self.sw_luz:
            pg.draw.circle(self.screen,LED_LUZ_ON, POS_LUZ, RADIO_LUZ)      # luz
        else:
            pg.draw.circle(self.screen,LED_LUZ_OFF, POS_LUZ, RADIO_LUZ)

        if self.sw_P1:
            pg.draw.rect(self.screen, LED_P1_ON, RECT_P1)
            # pg.draw.circle(self.screen,LED_P1_ON, POS_LED_P1, RADIO_PULS)      # mPulsador P1
        else:
            pg.draw.rect(self.screen, LED_P1_OFF, RECT_P1)
            # pg.draw.circle(self.screen,LED_P1_OFF, POS_LED_P1, RADIO_PULS)

        if self.sw_P2:
            pg.draw.rect(self.screen, LED_P2_ON, RECT_P2)
            # pg.draw.circle(self.screen,LED_P2_ON, POS_LED_P2, RADIO_PULS)      # mPulsador P2
        else:
            pg.draw.rect(self.screen, LED_P2_OFF, RECT_P2)
            # pg.draw.circle(self.screen,LED_P2_OFF, POS_LED_P2, RADIO_PULS)


import yaml
import sys

class Scanner:

    def __init__(self,filename,dictionary='forbidden.yaml',badwords=[]):
        self.filename=filename
        self.dict=dictionary
        self.badwords=badwords

    def getData(self):
        try:
            with open(self.filename,'rb') as f:
                data=f.read()
        except FileNotFoundError as e:
            print("File was not found.")
            print(e)
            raise FileNotFoundError
        except Exception as e:
           print(e)
        else:
            return data

    def getDict(self):
        try:
            self.badwords=[]
            with open(self.dict,'r') as d:
                data=yaml.load(d, Loader=yaml.Loader)
            for i in range(len(data)):
                self.badwords+= list(data.values())[i]
        except FileNotFoundError or OSError:
            print("You did not supply a dictionary. Please fix this.")
            raise FileNotFoundError
        except Exception as e:
            print("There was a problem loading the dictionary:")
            print(e)
            exit()
        else:
            return self.badwords

    def checkAll(self):
        try:
            badwords=self.getDict()
            self.matchedWords=0
            wordsFound=[]
            matchedEncodings=[]
            found =[]
            allEncodes=['ascii','utf-8','big5','big5hkscs','cp037','cp273','cp424','cp437','cp500','cp720','cp737','cp775','cp850','cp852','cp855','cp856','cp857','cp858',
                            'cp860','cp861','cp862','cp863','cp864','cp865','cp866','cp869','cp874','cp875','cp932','cp949','cp950','cp1006','cp1026','cp1125','cp1140',
                            'cp1250','cp1251','cp1252','cp1253','cp1254','cp1255','cp1256','cp1257','cp65001','euc_jp','euc_jis_2004','euc_jisx0213','euc_kr',
                            'gb2312','gbk','gb18030','hz','iso2022_jp','iso2022_jp_1','iso2022_jp_2','iso2022_jp_2004','iso2022_jp_3','iso2022_jp_ext','iso2022_kr',
                            'latin_1','iso8859_2','iso8859_3','iso8859_4','iso8859_5','iso8859_6','iso8859_7','iso8859_8','iso8859_9','iso8859_10','iso8859_11','iso8859_13',
                            'iso8859_14','iso8859_15','iso8859_16','johab','koi8_r','koi8_t','koi8_u','kz1048','mac_cyrillic','mac_greek','mac_iceland','mac_latin2',
                            'mac_roman','mac_turkish','ptcp154','shift_jis','shift_jis_2004','shift_jisx0213','utf_32','utf_32_be','utf_32_le','utf_16','utf_16_be',
                            'utf_16_le','utf_7','utf_8','utf_8_sig','cp1258']
            with open('FoundIn'+self.filename+'.txt', 'w') as f:
                print('Now checking '+self.filename+' for forbidden words:',file=f)
                print('Results are a tuple of (word, offset, encoding)', file=f)
                for e in allEncodes:
                    newBytes=[b.encode(encoding=e,errors='replace') for b in badwords]
                    for b in newBytes:
                        if (b in self.getData()) and (b):
                            if ((b.decode(encoding=e)) not in wordsFound) and ((b.decode(encoding=e))!='??'):
                                offset=self.getData().find(b)
                                wordsFound.append(b.decode(encoding=e))
                                if e not in matchedEncodings:
                                    matchedEncodings.append(e)
                                found.append((b.decode(encoding=e,errors='ignore'), offset, e))
                                self.matchedWords+=1
        except UnicodeEncodeError or UnicodeDecodeError or UnicodeError:
            print("Issues with Unicode")
        except Exception as e:
            print("Other error: "+str(e))
            raise LookupError
        else:
            with open('FoundIn'+self.filename+'.txt', 'a') as f:
                for i in found:
                    print(i, file=f)
                print("Number of matched words: "+str(self.matchedWords),file=f)
                print("Words found: "+str(wordsFound),file=f)
                print("Encodings matched" +str(matchedEncodings),file=f)
                if (self.matchedWords==0):
                    print("No matching words found, program is clean.",file=f)
                else: print("Please remove the matched words before submitting the file.",file=f)
            if self.matchedWords==0:
                print("No matching words, test passed.")
                return True
            else:
                print ("Test failed due to matching words found, see file "+str(f.name))
                return False

    def scanReturn(self):
        try:
            self.checkAll()
        except Exception as e:
            print("Program failed.")
            if e!='':
                print('Exception information: '+str(type(e)))
            exit()
        else:
            return 0

if len(sys.argv)==2:
    file=sys.argv[1]
else: file='test5'

newScan=Scanner(str(file))
newScan.scanReturn()


import copy
import math

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from epc.utils.resnet_encoder import SimpleResNetEncoder
from epc.utils.smt_state_encoder import SMTStateEncoder

def init(module, weight_init, bias_init, gain=1):
    weight_init(module.weight.data, gain=gain)
    bias_init(module.bias.data)
    return module

init_ = lambda m: init(
    m,
    nn.init.orthogonal_,
    lambda x: nn.init.constant_(x, 0),
    nn.init.calculate_gain("relu"),
)

class VisualEncoder(nn.Module):
    def __init__(self, cfg):
        super().__init__()
        self.cfg = cfg
        self._feat_size = None
        self._feat_shape = None
        self.encoder_type = self.cfg.encoder_type

        self.visual_encoder = SimpleResNetEncoder(
            cfg.ENCODER.input_shape,
            baseplanes=cfg.ENCODER.baseplanes,
            ngroups=cfg.ENCODER.baseplanes // 2,
            backbone_type=cfg.ENCODER.backbone,
            normalize_visual_inputs=cfg.ENCODER.normalize_visual_inputs,
        )
        fc_input_size = np.prod(self.visual_encoder.output_shape)
        self.visual_fc = nn.Sequential(
            nn.Flatten(),
            nn.Linear(fc_input_size, cfg.ENCODER.hidden_size),
            nn.ReLU(True),
        )
        self._feat_size = cfg.ENCODER.hidden_size
        self._feat_shape = (self._feat_size,)
        if cfg.ENCODER.pretrained_path != "":
            encoder_state_dict = torch.load(
                cfg.ENCODER.pretrained_path, map_location="cpu"
            )["state_dict"]
            keyword = "module.encoder_q.visual_encoder."
            visual_encoder_states = {
                k[len(keyword) :]: v
                for k, v in encoder_state_dict.items()
                if k.startswith(keyword)
            }
            self.visual_encoder.load_state_dict(visual_encoder_states)
            keyword = "module.encoder_q.visual_fc."
            visual_fc_states = {
                k[len(keyword) :]: v
                for k, v in encoder_state_dict.items()
                if k.startswith(keyword)
            }
            self.visual_fc.load_state_dict(visual_fc_states)

    def forward(self, x):
        x_cat = [x["rgb"], x["depth"]]
        x_cat = torch.cat(x_cat, dim=1)
        feats = self.visual_encoder(x_cat)
        feats = self.visual_fc(feats)
        return feats

    @property
    def feat_size(self):
        return self._feat_size

    @property
    def feat_shape(self):
        return self._feat_shape

class SelfSupervisedEncoder(SMTStateEncoder):
    def forward(self, x, memory, *args, **kwargs):
        """
        Single input case:
            Inputs:
                x - (N, input_size)
                memory - (M, N, input_size)
                memory_masks - (N, M)

        Multiple input case:
            Inputs:
                x - (K, N, input_size)
                memory - (M, N, input_size)
                memory_masks - (N, M)
            Each of the K inputs per batch element are treated independetly by generating
            an identity target mask.
        """
        assert x.dim() == 2 or x.dim() == 3
        output = None
        if x.dim() == 2:
            assert x.size(0) == memory.size(1)
            output = self.single_forward(x, memory, *args, **kwargs)
        elif x.dim() == 3:
            output = self.multi_forward(x, memory, *args, **kwargs)
        return output

    def single_forward(self, x, memory, memory_masks, goal=None, **kwargs):
        r"""Forward for a non-sequence input

        Args:
            x: (N, input_size) Tensor
            memory: The memory of encoded observations in the episode. It is a
                (M, N, input_size) Tensor.
            memory_masks: The masks indicating the set of valid memory locations
                for the current observations. It is a (N, M) Tensor.
            goal: (N, goal_dims) Tensor (optional)
        """
        # If memory_masks is all zeros for a data point, x_att will be NaN.
        # In these cases, just set memory_mask to ones and replace x_att with x.
        all_zeros_mask = (memory_masks.sum(dim=1) == 0).float().unsqueeze(1)
        memory_masks = 1.0 * all_zeros_mask + memory_masks * (1 - all_zeros_mask)

        # Compute relative pose encoding if applicable
        if self._use_pose_encoding:
            pi, pj = self._pose_indices
            x_pose = x[..., pi:]
            memory_poses = memory[..., pi:]
            x_pose_enc, memory_poses_enc = self._encode_pose(x_pose, memory_poses)
            # Update memory and observation encodings with the relative encoded poses
            x = torch.cat([x[..., :pi], x_pose_enc], dim=-1)
            memory = torch.cat([memory[..., :pi], memory_poses_enc], dim=-1)

        if self._use_goal_encoding:
            x = self.goal_observation_mlp(torch.cat([x, goal], 1))

        # Compress features
        x = self.fusion_encoder(x)
        M, bs = memory.shape[:2]
        memory = self.fusion_encoder(memory.view(M * bs, -1)).view(M, bs, -1)
        # Transformer operations
        t_masks = self._convert_masks_to_transformer_format(memory_masks)
        x_enc = self.transformer.encoder(memory, src_key_padding_mask=t_masks, **kwargs)
        x_att = self.transformer.decoder(
            x.unsqueeze(0), x_enc, memory_key_padding_mask=t_masks, **kwargs
        ).squeeze(0)
        # Mask out elements with no attention.
        x_att = x * all_zeros_mask + (1 - all_zeros_mask) * x_att

        if self._use_goal_encoding:
            x_att = torch.cat([x_att, goal], 1)

        return x_att

    def multi_forward(self, x, memory, memory_masks, **kwargs):
        r"""Forward for multiple inputs per batch

        Args:
            x: (K, N, input_size) Tensor where K is the number of inputs per batch
            memory: The memory of encoded observations in the episode. It is a
                (M, N, input_size) Tensor.
            memory_masks: The masks indicating the set of valid memory locations
                for the current observations. It is a (N, M) Tensor.

        Note: This assumes that each input per batch element will attend to the same
        set of input features.
        """
        # If memory_masks is all zeros for a data point, x_att will be NaN.
        # In these cases, just set memory_mask to ones and replace x_att with x.
        all_zeros_mask = (memory_masks.sum(dim=1) == 0).float().unsqueeze(1)
        memory_masks = 1.0 * all_zeros_mask + memory_masks * (1 - all_zeros_mask)

        # Compute relative pose encoding if applicable
        if self._use_pose_encoding:
            pi, pj = self._pose_indices
            # All inputs of a batch element are assumed to be reasonably close to
            # each other. So, the average pose is computed across inputs in a batch element.
            # The relative encoding is performed w.r.t that average pose.
            x_pose = x[..., pi:]  # (K, N, 4)
            x_pose_avg = x_pose.mean(dim=0)  # (N, 4)
            memory_poses = memory[..., pi:]  # (M, N, 4)
            _, x_pose_enc = self._encode_pose(x_pose_avg, x_pose)  # (K, N, G)
            _, memory_poses_enc = self._encode_pose(
                x_pose_avg, memory_poses
            )  # (M, N, G)

            # Update memory and observation encodings with the relative encoded poses
            x = torch.cat([x[..., :pi], x_pose_enc], dim=-1)  # (K, N, F)
            memory = torch.cat(
                [memory[..., :pi], memory_poses_enc], dim=-1
            )  # (K, N, F)

        # Compress features
        K, bs = x.shape[:2]
        M = memory.shape[0]
        x = self.fusion_encoder(x.view(K * bs, -1)).view(K, bs, -1)
        memory = self.fusion_encoder(memory.view(M * bs, -1)).view(M, bs, -1)

        # Transformer operations
        t_masks = self._convert_masks_to_transformer_format(memory_masks)
        ## Identity mask to prevent attention b/w the decoder inputs
        d_masks = self._convert_masks_to_transformer_format(
            torch.eye(K, K, device=x.device)
        )
        x_enc = self.transformer.encoder(memory, src_key_padding_mask=t_masks, **kwargs)
        x_att = self.transformer.decoder(
            x, x_enc, memory_key_padding_mask=t_masks, tgt_mask=d_masks, **kwargs
        )  # (K, N, F)

        # Mask out elements with no attention.
        all_zeros_mask = all_zeros_mask.unsqueeze(0)  # (1, N, 1)
        x_att = x * all_zeros_mask + (1 - all_zeros_mask) * x_att

        return x_att


import time
from functools import wraps

def retry(max_tries=3, delay_seconds=1):
    def decorator_retry(func):
        @wraps(func)
        def wrapper_retry(*args, **kwargs):
            tries = 0
            while tries < max_tries:
                try:
                    return func(*args, **kwargs)
                except Exception as e:
                    tries += 1
                    if tries == max_tries:
                        raise e
                    time.sleep(delay_seconds)

        return wrapper_retry

    return decorator_retry

def memoize(fn):
    cache = {}
    miss = object()

    @wraps(fn)
    def wrapper(*args, **kwargs):
        result = cache.get(args, miss)
        if result is miss:
            result = fn(*args, **kwargs)
            cache[args] = result
        return result

    def clear_cache():
        cache.clear()

    wrapper.clear_cache = clear_cache
    return wrapper

def timing_decorator(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        result = func(*args, **kwargs)
        end_time = time.time()
        print(f"Function {func.__name__} took {end_time - start_time} seconds to run.")
        return result

    return wrapper

def log_execution(func):
    import logging

    logging.basicConfig(level=logging.INFO)

    @wraps(func)
    def wrapper(*args, **kwargs):
        logging.info(f"Executing {func.__name__}")
        result = func(*args, **kwargs)
        logging.info(f"Finished executing {func.__name__}")
        return result

    return wrapper