import torch.nn as nn
import torch.optim as optim
from torch.utils.data.sampler import RandomSampler
from tqdm import tqdm
import pandas as pd

import attack_utils
import utils
import model.net as net
from attack_utils import AttackLoss
from dataloader import *

logger = logging.getLogger('DeepAR.Eval')

class AttackModule(nn.Module):

    def __init__(self, model, params, c, data,id_batch, v_batch, hidden, cell):

        super(AttackModule, self).__init__()

        self.model = model
        self.params = params
        self.c = c
        self.data = data
        self.id_batch = id_batch
        self.v_batch = v_batch
        self.hidden = hidden
        self.cell = cell
        self.n_inputs = data.shape[1]
        self.attack_loss = AttackLoss(params,c,v_batch)

        # Initialize perturbation
        self.perturbation = nn.Parameter(torch.zeros(self.data.shape[:2], device=self.params.device))

    def generate_target(self,labels,mode):

        if mode == "over":
            target = 1.5*labels
        elif mode == "under":
            target = 0.5*labels
        else:
            raise Exception("No such mode")

        return target

    # Returns mean and std
    def forward(self):

        perturbed_data = torch.zeros(self.data.shape).to(self.params.device)
        perturbed_data[:,:,0] = self.data[:,:,0]  * (1 + self.perturbation)
        perturbed_data[:,:,1:] = self.data[:,:,1:]

        samples, sample_mu, sample_sigma = attack_utils.forward_model(model,
                                                                      perturbed_data,
                                                                      self.id_batch,
                                                                      self.v_batch,
                                                                      self.hidden,
                                                                      self.cell,
                                                                      self.params)

        return samples,sample_mu,sample_sigma

    # Not clear yet how to compute that
    def forward_naive(self):

        perturbed_data = torch.zeros(self.data.shape).to(self.params.device)
        perturbed_data[:, :, 0] = self.data[:, :, 0] * (1 + self.perturbation)
        perturbed_data[:, :, 1:] = self.data[:, :, 1:]

        with torch.no_grad():
            samples, sample_mu,_ = attack_utils.forward_model(model,
                                                              perturbed_data,
                                                              self.id_batch,
                                                              self.v_batch,
                                                              self.hidden,
                                                              self.cell,
                                                              self.params)

        aux_estimate = torch.zeros(self.data.shape[1], device=self.params.device)

        hidden = self.hidden
        cell = self.cell
        for t in range(self.params.test_predict_start):

            mu, sigma, hidden, cell = model(perturbed_data[t].unsqueeze(0),
                                            self.id_batch,
                                            hidden,
                                            cell)

        for i in range(samples.shape[0]):
            sample = samples[i]
            log_prob = attack_utils.forward_log_prob(model,
                                                     sample,
                                                      perturbed_data,
                                                      self.id_batch,
                                                      self.v_batch,
                                                      hidden,
                                                      cell,
                                                      self.params)
            aux_estimate += sample[:,self.params.target] * log_prob

        aux_estimate /= float(samples.shape[0])
        aux_estimate = aux_estimate.sum(0)

        return sample_mu,aux_estimate

class Attack():

    def __init__(self,model, loss_fn, test_loader, params, plot_num):
        self.model = model
        self.loss_fn = loss_fn
        self.test_loader = test_loader
        self.params = params
        self.plot_num = plot_num

        # Set the model to eval mode
        # Replaced model.eval() to deal with
        # RuntimeError: cudnn RNN backward can only be called in training mode
        # (https: // github.com / pytorch / pytorch / issues / 10006)
        for name, module in self.model.named_modules():
            if isinstance(module, nn.Dropout):
                module.p = 0

            elif isinstance(module, nn.LSTM):
                module.dropout = 0

        print('model: ', model)

        self.max_pert_len = len(params.tolerance)

    def print(self,i,norm,distance,loss,batch):

        print(i,norm.detach().item()/batch,
              distance.detach().item()/batch,
              loss.detach().item()/batch)

    def project_perturbation(self,attack_module):

        aux = torch.tensor([-1.], device=self.params.device)

        attack_module.perturbation.data = torch.max(attack_module.perturbation.data, aux)

    def attack_step_ours(self, attack_module, optimizer, i, target):

        attack_module.zero_grad()

        _,prediction,_ = attack_module()

        norm_per_sample, distance_per_sample, loss_per_sample, norm, distance, loss = \
            attack_module.attack_loss(attack_module.perturbation, prediction, target)

        # Differentiate loss with respect to input
        loss.backward()

        # Apply one step of optimizer
        optimizer.step()

        self.project_perturbation(attack_module)

    def attack_step_naive(self, attack_module, optimizer, i, target):

        attack_module.zero_grad()

        mean, aux_estimate = attack_module.forward_naive()

        aux_estimate.backward()

        # Compute the derivative of the loss with respect to the mean
        mean.requires_grad = True
        attack_module.perturbation.requires_grad = False
        _,_,_,_,_, loss = attack_module.attack_loss(attack_module.perturbation, mean, target)

        # This propagates the gradient to the mean
        loss.backward()

        # Multiply the two, and set it in perturbation
        attack_module.perturbation.grad *= mean.grad[:,self.params.target]

        # Compute the derivative of the loss with respect to the norm
        mean.requires_grad = False
        attack_module.perturbation.requires_grad = True
        norm_per_sample,_,_,norm,distance, loss = \
            attack_module.attack_loss(attack_module.perturbation, mean, target)

        # This propagates the gradient to the norm
        loss.backward()

        # Apply one step of optimizer
        optimizer.step()

        self.project_perturbation(attack_module)

    def attack_batch(self, batch_number, data, id_batch, v_batch, labels, hidden, cell, estimator):

            batch_size = data.shape[1]

            with torch.no_grad():
                _, original_mu, original_sigma = attack_utils.forward_model(model,
                                                                          data,
                                                                          id_batch,
                                                                          v_batch,
                                                                          hidden,
                                                                          cell,
                                                                          self.params)

            shape = (self.max_pert_len,) + data.shape[:2]

            best_perturbation = {"over": np.zeros(shape),
                                 "under": np.zeros(shape)}

            c_shape = (self.max_pert_len, data.shape[1])
            best_c = {"over": np.zeros(c_shape),
                      "under": np.zeros(c_shape)}
            best_distance = {"over": np.full(c_shape, np.inf),
                             "under": np.full(c_shape, np.inf)}

            out_shape = (self.max_pert_len,)+ original_mu.shape
            perturbed_output_mu = {"over": np.zeros(out_shape),
                                   "under": np.zeros(out_shape)}
            perturbed_output_sigma = {"over": np.zeros(out_shape),
                                      "under": np.zeros(out_shape)}

            modes = ["under","over"]
            targets = {}

            lines = []

            for mode in modes:

                print("mode",mode)

                # Loop on values of c to find successful attack with minimum perturbation

                for i in range(0, len(self.params.c), self.params.batch_c):

                    bound = min(self.params.batch_c,len(self.params.c)-i)

                    batched_data = data.repeat(1,bound,1)

                    batched_id_batch = id_batch.repeat(1,bound)
                    batched_v_batch = v_batch.repeat(bound,1)
                    batched_labels = labels.repeat(bound)
                    batched_hidden = hidden.repeat(1,bound,1)
                    batched_cell = cell.repeat(1,bound,1)

                    batched_c = torch.cat([self.params.c[i+j]*\
                                           torch.ones(batch_size,device=self.params.device)\
                                           for j in range(bound)],dim = 0)

                    # Update the lines
                    attack_module = AttackModule(self.model,
                                                 self.params,
                                                 batched_c,
                                                 batched_data,
                                                 batched_id_batch,
                                                 batched_v_batch,
                                                 batched_hidden,
                                                 batched_cell)

                    batched_target = attack_module.generate_target(batched_labels,mode)

                    optimizer = optim.Adam([attack_module.perturbation], lr=self.params.learning_rate)

                    # Iterate steps
                    for k in range(self.params.n_iterations):

                        if estimator == "reparam":
                            self.attack_step_ours(attack_module, optimizer, k, batched_target)
                        elif estimator == "score":
                            self.attack_step_naive(attack_module, optimizer, k, batched_target)
                        else:
                            raise Exception("No such estimator")

                    # Evaluate the attack
                    # Run full number of samples on perturbed input to obtain perturbed output
                    with torch.no_grad():
                        _,batched_perturbed_output,_ = attack_module()

                        # Unbatch c to run everything from this
                        for j in range(bound):

                            c = self.params.c[i+j]

                            left = batch_size*j
                            right = batch_size*(j+1)

                            target = batched_target[left:right]
                            targets[mode] = target

                            perturbed_output = batched_perturbed_output[left:right]
                            v_batch = batched_v_batch[left:right]

                            loss = attack_utils.AttackLoss(self.params,c,v_batch)

                            norm_per_sample, distance_per_sample, loss_per_sample, norm, distance, loss = \
                                loss(attack_module.perturbation[:,left:right],
                                                          perturbed_output,
                                                          target)

                            # Find
                            numpy_norm = np.sqrt(utils.convert_from_tensor(norm_per_sample))
                            numpy_distance = utils.convert_from_tensor(distance_per_sample)
                            numpy_perturbation = utils.convert_from_tensor(
                                attack_module.perturbation.data[:,left:right])

                            for l in range(self.max_pert_len):

                                indexes_best_c = np.logical_and(numpy_norm <= self.params.tolerance[l],
                                                                numpy_distance < best_distance[mode][l])

                                best_perturbation[mode][l][:, indexes_best_c] = \
                                    numpy_perturbation[:, indexes_best_c]
                                best_distance[mode][l, indexes_best_c] = \
                                    numpy_distance[indexes_best_c]
                                best_c[mode][l, indexes_best_c] = c

                            # Save norm and distance for c plot
                            mean_numpy_norm = np.mean(numpy_norm)
                            mean_distance = np.mean(np.sqrt(numpy_distance))

                            lines.append([batch_number,estimator,mode,c,mean_numpy_norm,mean_distance])

                with torch.no_grad():

                    # Update the lines
                    attack_module = AttackModule(self.model,
                                                 self.params,
                                                 0,
                                                 data,
                                                 id_batch,
                                                 v_batch,
                                                 hidden,
                                                 cell)

                    for l in range(self.max_pert_len):

                        attack_module.perturbation.data = \
                            torch.tensor(best_perturbation[mode][l],
                                         device=self.params.device).float()
                        _,aux1,aux2 = attack_module()

                        perturbed_output_mu[mode][l] = aux1.cpu().numpy()
                        perturbed_output_sigma[mode][l] = aux2.cpu().numpy()

            return original_mu,original_sigma,best_c, best_perturbation, \
                   best_distance, perturbed_output_mu, perturbed_output_sigma,\
                   targets,lines

    def attack(self):

        lines = []

        # For each test sample
        # Test_loader:
        # test_batch ([batch_size, train_window, 1+cov_dim]): z_{0:T-1} + x_{1:T}, note that z_0 = 0;
        # id_batch ([batch_size]): one integer denoting the time series id;
        # v ([batch_size, 2]): scaling factor for each window;
        # labels ([batch_size, train_window]): z_{1:T}.
        for i, (test_batch, id_batch, v, labels) in enumerate(tqdm(self.test_loader)):

                print("\n")
                print("Batch",i)

                index = v[:,0] > 0
                test_batch = test_batch[index]
                id_batch = id_batch[index]
                v = v[index]
                labels = labels[index]

                # Prepare batch data
                test_batch = test_batch.permute(1, 0, 2).to(torch.float32).to(params.device)
                id_batch = id_batch.unsqueeze(0).to(params.device)
                v_batch = v.to(torch.float32).to(params.device)
                test_labels = labels.to(torch.float32).to(params.device)[:,params.target]

                batch_size = test_batch.shape[1]
                hidden = model.init_hidden(batch_size)
                cell = model.init_cell(batch_size)

                for estimator in ["score", "reparam"]:

                    print("estimator",estimator)

                    original_mu,original_sigma,best_c,best_perturbation,best_distance,\
                        perturbed_output_mu, perturbed_output_sigma,targets, l = \
                        self.attack_batch(i,test_batch,id_batch,v_batch,
                                          test_labels,hidden,cell,estimator)
                    lines += l

                    # Save results
                    saver = attack_utils.H5pySaver(os.path.join(params.output_folder,"batch_"+str(i)+"/"))

                    saver.save_to_file(original_mu, estimator + '_original_mu')
                    saver.save_to_file(original_sigma, estimator + '_original_sigma')
                    saver.save_dict_to_file(best_c, estimator + '_best_c')
                    saver.save_dict_to_file(best_perturbation, estimator + '_best_perturbation')
                    saver.save_dict_to_file(best_distance, estimator + '_best_distance')
                    saver.save_dict_to_file(perturbed_output_mu, estimator + '_perturbed_output_mu')
                    saver.save_dict_to_file(perturbed_output_sigma, estimator + '_perturbed_output_sigma')
                    saver.save_dict_to_file(targets, estimator + '_targets')
                    saver.save_to_file(labels, estimator + '_labels')

        df = pd.DataFrame(lines,columns=["batch","estimator","mode","c","norm","distance"])
        df.to_csv(os.path.join(params.output_folder,"results.csv"))

if __name__ == '__main__':

    params,model_dir,args,data_dir = attack_utils.set_params()

    model = attack_utils.set_cuda(params,logger)

    test_set = TestDataset(data_dir, args.dataset, params.num_class)
    test_loader = DataLoader(test_set, batch_size=params.predict_batch, sampler=RandomSampler(test_set), num_workers=4)

    loss_fn = net.loss_fn

    # Reload weights from the saved file
    utils.load_checkpoint(os.path.join(model_dir, args.restore_file + '.pth.tar'), model)

    attack = Attack(model, loss_fn, test_loader, params, -1,)

    test_metrics = attack.attack()
    save_path = os.path.join(model_dir, 'metrics_test_{}.json'.format(args.restore_file))


from flask import *
from . import db
from .models import *
import datetime
teacher = Blueprint('teacher',__name__)

@teacher.route('/',methods=["GET"])
def dashboard():
    if session.get('name'):
        c_s = Course.query.filter_by(teacher_id = session.get('id')).all()
        return render_template('teacher/dashboard.html', courses=c_s,name=session.get('name'))
    else:
        return redirect('auth.login')

@teacher.route('/course',methods=['GET'])
def course():
    if session.get('name'):
        if request.args.get('id'):
            session['course']= request.args.get('id')
            c = Course.query.get(request.args.get('id'))
            e_s = Exam.query.filter_by(course_id=c.id)
            return render_template('teacher/course.html',course = c,exams=e_s)
        else:
            c = Course.query.get(session['course'])
            e_s = Exam.query.filter_by(course_id=c.id)
            return render_template('teacher/course.html',course = c,exams=e_s)
    else:
        return redirect(url_for('auth.login'))

@teacher.route('/exam',methods=['GET','POST'])
def exam():
    if session.get('name'):
        if request.method == 'GET':
            return render_template('teacher/exam.html')
        else:
            for key,value in request.form.items():
                if not value or value=='':
                    flash('Missing '+key+'!')
                    return redirect(url_for('teacher.exam'))
            e = Exam(
                course_id = session.get('course'),
                title = request.form.get('title'),
                date = datetime.datetime(
                    int(request.form.get('date').split('-')[0]),
                    int(request.form.get('date').split('-')[1]),
                    int(request.form.get('date').split('-')[2]),
                    int(request.form.get('time').split(':')[0]),
                    int(request.form.get('time').split(':')[1])),
                number_of_questions = request.form.get('number_of_questions'),
                duration = request.form.get('duration'),
                max_score = request.form.get('max_score')
            )
            db.session.add(e)
            db.session.commit()
            flash('Added Exam Successfully!')
            session['nq'] = int(request.form.get('number_of_questions'))
            session['q'] = 1
            session['exam'] = e.id
            return redirect(url_for('teacher.question'))
    else:
        return redirect(url_for('auth.login'))

@teacher.route('/question',methods=['GET','POST'])
def question():
    if session.get('name'):
        if request.method == 'GET':
            if session.get('q') <= session.get('nq'):
                return render_template('teacher/question.html')
            else:
                flash('All Questions Saved!')
                return redirect(url_for('teacher.question'))
        else:
            for key,value in request.form.items():
                if not value:
                    flash('Missing '+key+' !')
                    return redirect(url_for('teacher.question'))
            q = Question(
                exam_id = session.get('exam'),
                order = session.get('q'),
                body = request.form.get('body'),
                option_1 = request.form.get('option1'),
                option_2 = request.form.get('option2'),
                option_3 = request.form.get('option3'),
                option_4 = request.form.get('option4'),
                answer = request.form.get('answer')
            )
            db.session.add(q)
            db.session.commit()
            flash('Question '+str(session.get('q'))+' Saved Successfully!')
            session['q'] = session.get('q')+1
            if session.get('q') > session.get('nq'):
                flash('All Question Saved!')
                return redirect(url_for('teacher.exam'))
            return redirect(url_for('teacher.question'))
    else:
        return redirect(url_for('auth.login'))

@teacher.route('/scores',methods=['GET'])
def scores():
    if session.get('name'):
        e_i = request.args.get('exam_id')
        scs = Test_score.query.filter_by(exam_id=e_i)
        scs_list = []
        for i in scs:
            scs_list.append((i,User.query.get(i.student_id)))
        return render_template('teacher/scores.html', scores=scs_list)
    else:
        return redirect(url_for('auth.login'))


import os
import glob
import json

TARGETS = [
    'Java',
    'Perl',
    'POSIX',
    'Python',
    'JavaScript',
    'PHP',
    'Golang',
    'Rust',
    'C#',
    'Scala',
    'Kotlin'
]

def read_resources(component: str, directory: str):
    base = os.path.dirname(os.path.realpath(__file__))

    for file in sorted(glob.glob('{}/resources/{}/{}/*.json'.format(base,
                                                                    component,
                                                                    directory))):
        with open('{}'.format(file), 'r') as f:
            info = json.load(f)

        with open(file.replace('.json', '.out'), 'r') as f:
            expected = {}
            for i, line in enumerate(f.readlines()):
                line = line.strip()

                if i == 0 or i % 3 == 0:
                    expected[line] = {}
                    lang = line
                elif (i - 1) == 0 or (i - 1) % 3 == 0:
                    expected[lang]['compiled'] = line
                else:
                    expected[lang]['regex'] = line

        yield info, expected


#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import mozunit
import pytest
from jsonschema.exceptions import ValidationError

from mozperftest.metrics.utils import validate_intermediate_results

def test_results_with_directory():
    test_result = {"results": "path-to-results", "name": "the-name"}
    validate_intermediate_results(test_result)

def test_results_with_measurements():
    test_result = {
        "results": [
            {"name": "metric-1", "values": [0, 1, 1, 0]},
            {"name": "metric-2", "values": [0, 1, 1, 0]},
        ],
        "name": "the-name",
    }
    validate_intermediate_results(test_result)

def test_results_with_suite_perfherder_options():
    test_result = {
        "results": [
            {"name": "metric-1", "values": [0, 1, 1, 0]},
            {"name": "metric-2", "values": [0, 1, 1, 0]},
        ],
        "name": "the-name",
        "extraOptions": ["an-extra-option"],
        "value": 9000,
    }
    validate_intermediate_results(test_result)

def test_results_with_subtest_perfherder_options():
    test_result = {
        "results": [
            {"name": "metric-1", "shouldAlert": True, "values": [0, 1, 1, 0]},
            {"name": "metric-2", "alertThreshold": 1.0, "values": [0, 1, 1, 0]},
        ],
        "name": "the-name",
        "extraOptions": ["an-extra-option"],
        "value": 9000,
    }
    validate_intermediate_results(test_result)

def test_results_with_bad_suite_property():
    test_result = {
        "results": "path-to-results",
        "name": "the-name",
        "I'll cause a failure,": "an expected failure",
    }
    with pytest.raises(ValidationError):
        validate_intermediate_results(test_result)

def test_results_with_bad_subtest_property():
    test_result = {
        "results": [
            # Error is in "shouldalert", it should be "shouldAlert"
            {"name": "metric-1", "shouldalert": True, "values": [0, 1, 1, 0]},
            {"name": "metric-2", "alertThreshold": 1.0, "values": [0, 1, 1, 0]},
        ],
        "name": "the-name",
        "extraOptions": ["an-extra-option"],
        "value": 9000,
    }
    with pytest.raises(ValidationError):
        validate_intermediate_results(test_result)

def test_results_with_missing_suite_property():
    test_result = {
        # Missing "results"
        "name": "the-name"
    }
    with pytest.raises(ValidationError):
        validate_intermediate_results(test_result)

def test_results_with_missing_subtest_property():
    test_result = {
        "results": [
            # Missing "values"
            {"name": "metric-2", "alertThreshold": 1.0}
        ],
        "name": "the-name",
        "extraOptions": ["an-extra-option"],
        "value": 9000,
    }
    with pytest.raises(ValidationError):
        validate_intermediate_results(test_result)

if __name__ == "__main__":
    mozunit.main()


# -*- coding: utf-8 -*-
import re
import click
import logging
from pathlib import Path
from dotenv import find_dotenv, load_dotenv
import pandas as pd
import numpy as np

@click.command()
@click.argument("input_filepath", type=click.Path(exists=True))
@click.argument("output_filepath", type=click.Path())
def main(
    input_filepath: str = "../data/raw/train.csv.zip",
    output_filepath: str = "../data/processed/train_processed.csv",
) -> None:
    """Runs data processing scripts to turn raw data from (../raw) into
    cleaned data ready to be analyzed (saved in ../processed).
    """
    print("Loading zip files..")
    data = pd.read_csv(input_filepath, compression="zip")
    # labels = ['toxic', 'severe_toxic', 'obscene', 'threat','insult', 'identity_hate']
    data["total_classes"] = data.iloc[:, 2:8].apply(lambda x: sum(x), axis=1)
    data["non_toxic"] = data.iloc[:, 2:8].apply(
        lambda x: 1 if (sum(x) == 0) else 0, axis=1
    )
    cleaned_data = data.copy()

    # Removing Hyperlinks from text
    # The below code is a re pattern that matches URLs beginning with http or https, followed by :// and one or more
    # non-whitespace characters, or URLs beginning with www., followed by one or more non-whitespace characters.

    print("Cleaning the data..")
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"https?://\S+|www\.\S+", "", x)
    )
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(
            "["
            "\U0001F600-\U0001F64F"
            "\U0001F300-\U0001F5FF"
            "\U0001F680-\U0001F6FF"
            "\U0001F1E0-\U0001F1FF"
            "\U00002702-\U000027B0"
            "\U000024C2-\U0001F251"
            "]+",
            "",
            x,
            flags=re.UNICODE,
        )
    )

    # matches of css style,class elements and attributes
    idxs_css = []
    for i in range(len(cleaned_data)):
        if re.findall(r"[{][|].+\n", cleaned_data.loc[i, "comment_text"]):
            idxs_css.append(i)

    # Comments Containing Css style, class and attributes
    cleaned_data.loc[idxs_css, "total_classes"].value_counts()
    cleaned_data.drop(idxs_css, inplace=True)

    # Removing html tags from text
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"<.*?>", "", x)
    )
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"\"\"", '"', x)
    )  # replacing "" with "
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"^\"", "", x)
    )  # removing quotation from start and the end of the string
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"\"$", "", x)
    )

    # Removing Punctuation / Special characters (;:'".?@!%&*+) which appears more than twice in the text
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"[^a-zA-Z0-9\s][^a-zA-Z0-9\s]+", " ", x)
    )

    # Removing Special characters
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"[^a-zA-Z0-9\s\"\',:;?!.()]", " ", x)
    )

    # Removing extra spaces in text
    cleaned_data["comment_text"] = cleaned_data["comment_text"].map(
        lambda x: re.sub(r"\s\s+", " ", x)
    )
    cleaned_data.reset_index(inplace=True)
    cleaned_data.drop(columns=["index"], inplace=True)
    Final_data = cleaned_data.copy()

    # Keeping 15291 comments from non_toxic comments and dropping the rest
    C_indexs = cleaned_data[cleaned_data["total_classes"] == 0].index
    drop_indxs = np.random.choice(C_indexs, size=127696, replace=False)

    # Randomly selecting and dropping non_toxic comments from datasets and leaving 5000 behind
    Final_data.drop(drop_indxs, inplace=True)

    Final_data.to_csv(output_filepath, index=False)
    print("Saved file in data\processed")

    logger = logging.getLogger(__name__)
    logger.info("making final data set from raw data")

if __name__ == "__main__":
    log_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # not used in this stub but often useful for finding various files
    project_dir = Path(__file__).resolve().parents[2]

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    main()


"""
xepr_link.py
------------

Xepr interface functions, communicate with Xepr usingXeprAPI.

Pause times necesary to let Xepr process command are accessed through the
global variable ``COMPILATION_TIME`` (s, ``1`` by default).
It applies pauses of lengths:

 - ``COMPILATION_TIME`` after compilation of .exp file

 - ``COMPILATION_TIME`` after compilation of .def file

 - ``COMPILATION_TIME/4`` after compilation of .shp file

 - ``2*COMPILATION_TIME`` before and after Xepr reset

SPDX-License-Identifier: GPL-3.0-or-later

"""

import time
import XeprAPI         # load the Xepr API module
from typing import List

# global variable to control Xepr files compilation time
COMPILATION_TIME = 1  # (s)

def load_xepr():
    """
    Initialise the Xepr module and return the Xepr object thus created.

    Parameters
    ----------
    None

    Returns
    -------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object, used for communication with
        Xepr-the-programme.
    """
    xepr = XeprAPI.Xepr()  # start Xepr API module

    return xepr

def load_exp(xepr, exp_file: str) -> None:
    """
    Load and compile an Xepr experiment file.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    exp_file : str
        The Xepr experiment file (the full path with .exp extension).

    Returns
    -------
    None
    """
    try:
        xepr.XeprCmds.aqPgLoad(exp_file)
        xepr.XeprCmds.aqPgShowPrg()
        xepr.XeprCmds.aqPgCompValid()
        xepr.XeprCmds.aqPgCompile()

        # wait for Xepr to finish compiling
        time.sleep(COMPILATION_TIME)
    except Exception:
        raise RuntimeError("Error loading and compiling experiment file")

def modif_exp(xepr, exp_file: str, line_nb: int, new_line: str) -> None:
    """
    Modify the Xepr .exp file by overwriting a line.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    exp_file : str
        Name of the Xepr definition file (full path with .def extension).
    line_nb : int
        Number of the lines to overwrite
    new_line : str
        DESCRIPTION.

    Returns
    -------
    None
    """
    with open(exp_file, 'r') as exp_f:
        fullExp = exp_f.read()

    # write new line
    fullExp = fullExp.split("\n")
    fullExp[line_nb-1] = new_line

    # replace .exp file with modifications
    with open(exp_file, 'w') as exp_f:
        exp_f.write('\n'.join(fullExp))

    if xepr is not None:  # to allow test without Xepr
        load_exp(xepr, exp_file)

def load_def(xepr, def_file: str) -> None:
    """
    Load and compile an Xepr definition file.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    def_file : str
        Name of the Xepr definition file (full path with .def extension).

    Returns
    -------
    None
    """
    try:
        xepr.XeprCmds.aqPgDefLoad(def_file)
        xepr.XeprCmds.aqPgShowDef()
        xepr.XeprCmds.aqPgCompile()

        # wait for Xepr to finish compiling
        time.sleep(COMPILATION_TIME)
    except Exception:
        raise RuntimeError("Error loading and compiling definition file")

def modif_def_PlsSPELGlbTxt(xepr, def_file: str,
                            var_name: List[str], var_value: List[str]) -> None:
    """
    Directly modify definitions in the current experiment.

    !will not modify parameters which rely on dependency in the .def (e.g.
                                                                   'p0 = 2*p1')
    !can lead to bugs if forbiden charachters are used in the .def file (e.g.
                                                                           '%')
    !can create a bug where the Xepr stays stuck at the .def file modification
                                           (cf. documentation for more details)
    but faster than modif_def() as no compilation required

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    def_file : str
        Name of the Xepr definition file (full path with .def extension).
    var_name : list of strings
        Variable names as named in the .def file.
    var_value : list of strings
        List of variable values to be input.

    Returns
    -------
    None
    """
    try:
        currentExp = xepr.XeprExperiment()
    except Exception:
        raise RuntimeError("No experiment has been selected in the"
                           " primary viewport of Xepr.")

    # get the start value for the variable
    # search for our desired variable
    # first get the text of the full PulseSpel def
    fullDefs = currentExp.getParam("PlsSPELGlbTxt").value
    # need to check if fullDefs is empty and exit cause pulsespel not being
    # loaded

    fullDefs = fullDefs.split("\n")

    for i, var_name_i in enumerate(var_name):
        cmdStr = (var_name_i
                  + " = "
                  + var_value[i])

        for line in fullDefs:
            line = line.replace(" ", "")  # getting rid of spaces
            line = line[0:len(var_name_i) + 1]  # selecting first characters
            if var_name_i + "=" == line:
                currentExp["ftEPR.PlsSPELSetVar"].value = cmdStr

def modif_def(xepr, def_file: str,
              var_name: List[str], var_value: List[str]) -> None:
    """
    Modify definitions by modifying the .def file and reloading it.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    def_file : str
        Name of the Xepr definition file (full path with .def extension).
    var_name : list of strings
        Variable names as named in the .def file.
    var_value : list of strings
        List of variable values to be input.

    Returns
    -------
    None
    """
    with open(def_file, 'r') as def_f:
        fullDefs = def_f.read()
    fullDefs = fullDefs.split("\n")

    for name, value in zip(var_name, var_value):

        for j, line in enumerate(fullDefs):

            equal_partition = line.partition("=")

            if name == equal_partition[0].replace(" ", ""):
                fullDefs[j] = equal_partition[0] + "= " + value

                # preserve possible comment at the end of the line
                comment_partition = equal_partition[-1].partition(";")
                if comment_partition[1] == ";":
                    fullDefs[j] += " " ";" + comment_partition[-1]

    # replace definition file with modifications
    with open(def_file, 'w') as def_f:
        def_f.write('\n'.join(fullDefs))

    if xepr is not None:  # to allow test without Xepr
        load_def(xepr, def_file)

def load_shp(xepr, shp_file: str) -> None:
    """
    Load and compile an Xepr shape file.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    shp_file : str
        Shape file name (full path, including .shp extension).

    Returns
    -------
    None
    """
    try:
        xepr.XeprCmds.aqPgShpLoad(shp_file)
        xepr.XeprCmds.aqPgShowShp()
        xepr.XeprCmds.aqPgCompile()

        # wait for Xepr to finish compiling
        time.sleep(COMPILATION_TIME*0.25)
    except Exception:
        raise RuntimeError("Error loading and compiling Xepr shape file")

def run2getdata_exp(xepr, SignalType: str = None, exp_name: str = None):
    """
    Run an experiment and get the data from it.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.
    SignalType : str, by default None
        To change the signal type (pick from {"TM", "Signal", "RM"})
    exp_name : str, by default None
        The name of the experiment to run, usually "AWGTransient" for the
        Signal or "AWG_1pulseSHP" for the TM.

    Returns
    -------
    data : XeprAPI.Dataset
        The data retrieved from Xepr. Attributes of interest are:
            - data.X      : the X-abscissa values of the dataset
            - data.O      : complex-valued numpy.ndarray of the signal
            - data.O.real : real part of the signal
            - data.O.imag : imaginary part of the signal
    """
    try:
        currentExp = xepr.XeprExperiment()
        hiddenExp = xepr.XeprExperiment("AcqHidden")
    except Exception:
        raise RuntimeError("No experiment has been selected in the"
                           " primary viewport of Xepr.")

    # change detection mode and experiment select if needed
    try:
        if SignalType is not None and \
                hiddenExp["ftBridge.Detection"].value != SignalType:
            hiddenExp["ftBridge.Detection"].value = SignalType
        if exp_name is not None and \
                currentExp["ftEpr.PlsSPELEXPSlct"].value != exp_name:
            currentExp["ftEpr.PlsSPELEXPSlct"].value = exp_name
    except Exception:
        raise RuntimeError("Error changing detection mode to TM,"
                           " or in experiment selection")

    # run new experiment
    try:
        currentExp.aqExpRunAndWait()
    except Exception:
        raise RuntimeError("Error running current experiment")

    # retrieve data
    data = xepr.XeprDataset()

    # no data? -- try to run current experiment (if possible)
    if not data.datasetAvailable():
        try:
            print("Trying to run current experiment to create some data...")
            xepr.XeprExperiment().aqExpRunAndWait()
        except XeprAPI.ExperimentError:
            raise RuntimeError("No dataset available and no (working)"
                               " experiment to run; aborting")
    if not data.datasetAvailable():
        raise RuntimeError("No dataset available; aborting")

    return data

def reset_exp(xepr) -> None:
    """
    Copy the current experiment and use it to replace the current experiment.

    Needed to reset the AWG after 114 sequential shape load and run.

    Parameters
    ----------
    xepr : instance of XeprAPI.Xepr
        The instantiated Xepr object.

    Returns
    -------
    None
    """
    # wait for Xepr to be ready to reset the experiment
    time.sleep(2*COMPILATION_TIME)

    # get current experiment name
    curr_exp = xepr.XeprExperiment()
    expt_name = curr_exp.aqGetExpName()

    # duplicate current experiment
    print(f"replacing experiment <{expt_name}> with new"
          " instance of its copy...")
    xepr.XeprCmds.aqExpCut(expt_name)

    print(f"new instance of experiment <{expt_name}> created...")
    xepr.XeprCmds.aqExpPaste()

    # select new experiment
    xepr.XeprCmds.aqExpSelect(expt_name)
    print(f"new instance of experiment <{expt_name}> selected...")

    # activate new experiment
    xepr.XeprCmds.aqExpActivate(expt_name)
    print(f"new instance of experiment <{expt_name}> activated.")

    # open parameter panel
    xepr.XeprCmds.aqParOpen()

    # ask user to allow .exp file to be loaded
    print('Experiment reset')
    print('clik twice the PulseSPEL button to load the .exp file')
    print('(located at the bottom of FT EPR parameters window)')
    input('when done, press enter in python console to continue:')

    # wait for Xepr to reset the experiment
    time.sleep(2*COMPILATION_TIME)

    # prevent Xepr from reseting high power attenuation value
    xepr.XeprCmds.aqParStep("AcqHidden", "ftBridge.Attenuation", "Fine 1")
    xepr.XeprCmds.aqParStep("AcqHidden", "ftBridge.Attenuation", "Fine -1")


import os
import pathlib
import platform
import shutil
import subprocess
import sys
import warnings
from copy import deepcopy
from dataclasses import dataclass
from io import TextIOWrapper
from typing import Any, List, Optional, Tuple

from mlir_quantum.compiler_driver import run_compiler_driver

from catalyst._configuration import INSTALLED
from catalyst.utils.exceptions import CompileError
from catalyst.utils.filesystem import Directory

package_root = os.path.dirname(__file__)

@dataclass
class CompileOptions:
    """Generic compilation options, for which reasonable default values exist.

    Args:
        verbose (Optional[bool]): flag indicating whether to enable verbose output.
            Default is ``False``
        logfile (Optional[TextIOWrapper]): the logfile to write output to.
            Default is ``sys.stderr``
        keep_intermediate (Optional[bool]): flag indicating whether to keep intermediate results.
            Default is ``False``
        pipelines (Optional[List[Tuple[str,List[str]]]]): A list of tuples. The first entry of the
            tuple corresponds to the name of a pipeline. The second entry of the tuple corresponds
            to a list of MLIR passes.
        autograph (Optional[bool]): flag indicating whether experimental autograph support is to
            be enabled.
        lower_to_llvm (Optional[bool]): flag indicating whether to attempt the LLVM lowering after
            the main compilation pipeline is complete. Default is ``True``.
    """

    verbose: Optional[bool] = False
    logfile: Optional[TextIOWrapper] = sys.stderr
    target: Optional[str] = "binary"
    keep_intermediate: Optional[bool] = False
    pipelines: Optional[List[Any]] = None
    autograph: Optional[bool] = False
    lower_to_llvm: Optional[bool] = True

    def __deepcopy__(self, memo):
        """Make a deep copy of all fields of a CompileOptions object except the logfile, which is
        copied directly"""
        return CompileOptions(
            **{
                k: (deepcopy(v) if k != "logfile" else self.logfile)
                for k, v in self.__dict__.items()
                if k != "logfile"
            }
        )

    def get_pipelines(self) -> List[Tuple[str, List[str]]]:
        """Get effective pipelines"""
        return self.pipelines if self.pipelines is not None else DEFAULT_PIPELINES

def run_writing_command(command: List[str], compile_options: Optional[CompileOptions]) -> None:
    """Run the command after optionally announcing this fact to the user.

    Args:
        command (List[str]): command to be sent to a subprocess.
        compile_options (Optional[CompileOptions]): compile options.
    """

    if compile_options.verbose:
        print(f"[SYSTEM] {' '.join(command)}", file=compile_options.logfile)
    subprocess.run(command, check=True)

default_lib_paths = {
    "llvm": os.path.join(package_root, "../../mlir/llvm-project/build/lib"),
    "runtime": os.path.join(package_root, "../../runtime/build/lib"),
    "enzyme": os.path.join(package_root, "../../mlir/Enzyme/build/Enzyme"),
}

def get_lib_path(project, env_var):
    """Get the library path."""
    if INSTALLED:
        return os.path.join(package_root, "lib")  # pragma: no cover
    return os.getenv(env_var, default_lib_paths.get(project, ""))

DEFAULT_PIPELINES = [
    (
        "HLOLoweringPass",
        [
            "canonicalize",
            "func.func(chlo-legalize-to-hlo)",
            "stablehlo-legalize-to-hlo",
            "func.func(mhlo-legalize-control-flow)",
            "func.func(hlo-legalize-to-linalg)",
            "func.func(mhlo-legalize-to-std)",
            "convert-to-signless",
            "func.func(scalarize)",
            "canonicalize",
            "scatter-lowering",
        ],
    ),
    (
        "QuantumCompilationPass",
        [
            "lower-gradients",
            "adjoint-lowering",
        ],
    ),
    (
        "BufferizationPass",
        [
            "one-shot-bufferize{dialect-filter=memref}",
            "inline",
            "gradient-bufferize",
            "scf-bufferize",
            "convert-tensor-to-linalg",  # tensor.pad
            "convert-elementwise-to-linalg",  # Must be run before --arith-bufferize
            "arith-bufferize",
            "empty-tensor-to-alloc-tensor",
            "func.func(bufferization-bufferize)",
            "func.func(tensor-bufferize)",
            "func.func(linalg-bufferize)",
            "func.func(tensor-bufferize)",
            "quantum-bufferize",
            "func-bufferize",
            "func.func(finalizing-bufferize)",
            "func.func(buffer-hoisting)",
            "func.func(buffer-loop-hoisting)",
            "func.func(buffer-deallocation)",
            "convert-arraylist-to-memref",
            "convert-bufferization-to-memref",
            "canonicalize",
            # "cse",
            "cp-global-memref",
        ],
    ),
    (
        "MLIRToLLVMDialect",
        [
            "convert-gradient-to-llvm",
            "func.func(convert-linalg-to-loops)",
            "convert-scf-to-cf",
            # This pass expands memref ops that modify the metadata of a memref (sizes, offsets,
            # strides) into a sequence of easier to analyze constructs. In particular, this pass
            # transforms ops into explicit sequence of operations that model the effect of this
            # operation on the different metadata. This pass uses affine constructs to materialize
            # these effects. Concretely, expanded-strided-metadata is used to decompose
            # memref.subview as it has no lowering in -finalize-memref-to-llvm.
            "expand-strided-metadata",
            "lower-affine",
            "arith-expand",  # some arith ops (ceildivsi) require expansion to be lowered to llvm
            "convert-complex-to-standard",  # added for complex.exp lowering
            "convert-complex-to-llvm",
            "convert-math-to-llvm",
            # Run after -convert-math-to-llvm as it marks math::powf illegal without converting it.
            "convert-math-to-libm",
            "convert-arith-to-llvm",
            "finalize-memref-to-llvm{use-generic-functions}",
            "convert-index-to-llvm",
            "convert-quantum-to-llvm",
            "emit-catalyst-py-interface",
            # Remove any dead casts as the final pass expects to remove all existing casts,
            # but only those that form a loop back to the original type.
            "canonicalize",
            "reconcile-unrealized-casts",
        ],
    ),
]

class LinkerDriver:
    """Compiler used to drive the linking stage.
    In order to avoid relying on a single linker at run time and allow the user some flexibility,
    this class defines a compiler resolution order where multiple known compilers are attempted.
    The order is defined as follows:
    1. A user specified compiler via the environment variable CATALYST_CC. It is expected that the
        user provided compiler is flag compatilble with GCC/Clang.
    2. clang: Priority is given to clang to maintain an LLVM toolchain through most of the process.
    3. gcc: Usually configured to link with LD.
    4. c99: Usually defaults to gcc, but no linker interface is specified.
    5. c89: Usually defaults to gcc, but no linker interface is specified.
    6. cc: Usually defaults to gcc, however POSIX states that it is deprecated.
    """

    _default_fallback_compilers = ["clang", "gcc", "c99", "c89", "cc"]

    @staticmethod
    def get_default_flags():
        """Re-compute the path where the libraries exist.

        The use case for this is if someone is in a python jupyter notebook and
        needs to change the environment mid computation.
        Returns
            (List[str]): The default flag list.
        """
        mlir_lib_path = get_lib_path("llvm", "MLIR_LIB_DIR")
        rt_lib_path = get_lib_path("runtime", "RUNTIME_LIB_DIR")

        lib_path_flags = [
            f"-Wl,-rpath,{mlir_lib_path}",
            f"-L{mlir_lib_path}",
        ]

        if rt_lib_path != mlir_lib_path:
            lib_path_flags += [
                f"-Wl,-rpath,{rt_lib_path}",
                f"-L{rt_lib_path}",
            ]
        else:
            pass  # pragma: nocover

        system_flags = []
        if platform.system() == "Linux":
            system_flags += ["-Wl,-no-as-needed"]
        elif platform.system() == "Darwin":  # pragma: nocover
            system_flags += ["-Wl,-arch_errors_fatal"]

        default_flags = [
            "-shared",
            "-rdynamic",
            *system_flags,
            *lib_path_flags,
            "-lrt_backend",
            "-lrt_capi",
            "-lpthread",
            "-lmlir_c_runner_utils",  # required for memref.copy
        ]

        return default_flags

    @staticmethod
    def _get_compiler_fallback_order(fallback_compilers):
        """Compiler fallback order"""
        preferred_compiler = os.environ.get("CATALYST_CC", None)
        preferred_compiler_exists = LinkerDriver._exists(preferred_compiler)
        compilers = fallback_compilers
        emit_warning = preferred_compiler and not preferred_compiler_exists
        if emit_warning:
            msg = f"User defined compiler {preferred_compiler} is not in PATH. Using fallback ..."
            warnings.warn(msg, UserWarning)
        else:
            compilers = [preferred_compiler] + fallback_compilers
        return compilers

    @staticmethod
    def _exists(compiler):
        if compiler is None:
            return None
        return shutil.which(compiler)

    @staticmethod
    def _available_compilers(fallback_compilers):
        for compiler in LinkerDriver._get_compiler_fallback_order(fallback_compilers):
            if LinkerDriver._exists(compiler):
                yield compiler

    @staticmethod
    def _attempt_link(compiler, flags, infile, outfile, options):
        try:
            command = [compiler] + flags + [infile, "-o", outfile]
            run_writing_command(command, options)
            return True
        except subprocess.CalledProcessError as e:
            # Only warn in verbose mode, as users might see it otherwise in regular use.
            if options.verbose:
                msg = f"Compiler {compiler} failed to link executable and returned with exit code "
                msg += f"{e.returncode}. Output was: {e.output}.\nCommand: {command}"
                warnings.warn(msg, UserWarning)
            return False

    @staticmethod
    def get_output_filename(infile):
        """Rename object file to shared object

        Args:
            infile (str): input file name
            outfile (str): output file name
        """
        path = pathlib.Path(infile)
        if not path.exists():
            raise FileNotFoundError(f"Cannot find {infile}.")
        return str(path.with_suffix(".so"))

    @staticmethod
    def run(infile, outfile=None, flags=None, fallback_compilers=None, options=None):
        """
        Link the infile against the necessary libraries and produce the outfile.

        Args:
            infile (str): input file
            outfile (Optional[str]): output file
            flags (Optional[List[str]]): flags to be passed down to the compiler
            fallback_compilers (Optional[List[str]]): name of executables to be looked for in PATH
            compile_options (Optional[CompileOptions]): generic compilation options.
        Raises:
            EnvironmentError: The exception is raised when no compiler succeeded.
        """
        if outfile is None:
            outfile = LinkerDriver.get_output_filename(infile)
        if flags is None:
            flags = LinkerDriver.get_default_flags()
        if fallback_compilers is None:
            fallback_compilers = LinkerDriver._default_fallback_compilers
        if options is None:
            options = CompileOptions()
        for compiler in LinkerDriver._available_compilers(fallback_compilers):
            success = LinkerDriver._attempt_link(compiler, flags, infile, outfile, options)
            if success:
                return outfile
        msg = f"Unable to link {infile}. Please check the output for any error messages. If no "
        msg += "compiler was found by Catalyst, please specify a compatible one via $CATALYST_CC."
        raise CompileError(msg)

class Compiler:
    """Compiles MLIR modules to shared objects by executing the Catalyst compiler driver library."""

    def __init__(self, options: Optional[CompileOptions] = None):
        self.options = options if options is not None else CompileOptions()
        self.last_compiler_output = None

    def run_from_ir(self, ir: str, module_name: str, workspace: Directory):
        """Compile a shared object from a textual IR (MLIR or LLVM).

        Args:
            ir (str): Textual MLIR to be compiled
            module_name (str): Module name to use for naming
            workspace (Directory): directory that holds output files and/or debug dumps.

        Returns:
            output_filename (str): Output file name. For the default pipeline this would be the
                                   shard object library path.
            out_IR (str): Output IR in textual form. For the default pipeline this would be the
                          LLVM IR.
            A list of:
               func_name (str) Inferred name of the main function
               ret_type_name (str) Inferred main function result type name
        """
        assert isinstance(
            workspace, Directory
        ), f"Compiler expects a Directory type, got {type(workspace)}."
        assert workspace.is_dir(), f"Compiler expects an existing directory, got {workspace}."

        lower_to_llvm = (
            self.options.lower_to_llvm if self.options.lower_to_llvm is not None else False
        )

        if self.options.verbose:
            print(f"[LIB] Running compiler driver in {workspace}", file=self.options.logfile)

        try:
            compiler_output = run_compiler_driver(
                ir,
                str(workspace),
                module_name,
                keep_intermediate=self.options.keep_intermediate,
                verbose=self.options.verbose,
                pipelines=self.options.get_pipelines(),
                lower_to_llvm=lower_to_llvm,
            )
        except RuntimeError as e:
            raise CompileError(*e.args) from e

        if self.options.verbose:
            for line in compiler_output.get_diagnostic_messages().strip().split("\n"):
                print(f"[LIB] {line}", file=self.options.logfile)

        filename = compiler_output.get_object_filename()
        out_IR = compiler_output.get_output_ir()
        func_name = compiler_output.get_function_attributes().get_function_name()
        ret_type_name = compiler_output.get_function_attributes().get_return_type()

        if lower_to_llvm:
            output = LinkerDriver.run(filename, options=self.options)
            output_filename = str(pathlib.Path(output).absolute())
        else:
            output_filename = filename

        self.last_compiler_output = compiler_output
        return output_filename, out_IR, [func_name, ret_type_name]

    def run(self, mlir_module, *args, **kwargs):
        """Compile an MLIR module to a shared object.

        .. note::

            For compilation of hybrid quantum-classical PennyLane programs,
            please see the :func:`~.qjit` decorator.

        Args:
            mlir_module: The MLIR module to be compiled

        Returns:
            (str): filename of shared object
        """

        return self.run_from_ir(
            mlir_module.operation.get_asm(
                binary=False, print_generic_op_form=False, assume_verified=True
            ),
            str(mlir_module.operation.attributes["sym_name"]).replace('"', ""),
            *args,
            **kwargs,
        )

    def get_output_of(self, pipeline) -> Optional[str]:
        """Get the output IR of a pipeline.
        Args:
            pipeline (str): name of pass class

        Returns
            (Optional[str]): output IR
        """
        if len(dict(self.options.get_pipelines()).get(pipeline, [])) == 0:
            warnings.warn("Requesting an output of an empty pipeline")  # pragma: no cover

        if not self.last_compiler_output:
            return None

        return self.last_compiler_output.get_pipeline_output(pipeline)

    def print(self, pipeline):
        """Print the output IR of pass.
        Args:
            pipeline (str): name of pass class
        """
        print(self.get_output_of(pipeline))  # pragma: no cover


from logger import log, GameError
from maptraveler import MapTraveler
from pysikuli import Region
from src.config import CONFIG
import pyautogui as pyag
from time import sleep
import random

class Character:
    def __init__(self, maptraveler):
        self.maptraveler = maptraveler
        self.matcher = Region()

    def teleport_to(self, direction, distance, sleep_time=0.25, offset=None, mode="normal"):
        log.debug("Teleport start")
        direction_location_x, direction_location_y = MapTraveler.MOVE_DIRECTIONS[direction]
        center_x, center_y = CONFIG["CENTER_LOCATION"]

        if offset is not None:
            direction_location_x, direction_location_y = direction_location_x + offset[0], \
                                                         direction_location_y + offset[1]
        teleport_location = MapTraveler.get_move_location(direction_location_x - center_x,
                                                          direction_location_y - center_y, (distance, distance))

        log.debug("Teleport location: " + str(teleport_location))
        if mode == "normal":
            self.matcher.click(teleport_location, button="right")
        elif mode == "continuous":
            pyag.moveTo(teleport_location)
            pyag.mouseDown(button='right')
        sleep(sleep_time)
        if mode == "continuous":
            pyag.mouseUp(button='right')
        log.debug("Teleport end")

    def go_to_destination(self, destination, shift=None, move_step=(250, 350), accepted_distance=20, steps_timeout=30,
                          critical=True, map_filter=False, button="left", move_sleep=0.1):
        step = 0
        error_counter = 0
        waypoints_found = 0
        log.info("Going to destination " + str(destination))
        while True:
            step += 1
            log.info("Started moving loop nr " + str(step))
            if step > steps_timeout:
                log.error("Timeout when going to " + str(destination))
                if critical:
                    raise GameError("Timeout when going to " + str(destination))
                else:
                    log.warning("This is not critical.")
                    return False
            # Checking blockers
            try:
                if Region(*CONFIG["WAYPOINT_REGION"]).exists("images/waypoint.png", 0.1, debug=False):
                    log.warning("Waypoint image found. Closing it.")
                    pyag.press("esc")
                    waypoints_found += 1
                    if waypoints_found % 5 == 4:
                        Region().click((1251,1007))
                        sleep(2)
            except:
                log.warning("Waypoint issue found. Repeating.")
            log.info("Waypoint checked.")

            self.maptraveler.update_screen()
            diff_location = self.maptraveler.get_diff_from_destination(destination, shift=shift,
                                                                       map_filter=map_filter)
            # Error counter is used to do not fail function if we do not detect image in single iteration.
            if diff_location is None:
                log.warning("Cannot find destination.")
                sleep(0.2)
                if error_counter == 30:
                    return False
                else:
                    error_counter += 1
                    continue
            else:
                diff_location_x, diff_location_y = diff_location

            error_counter = 0
            if abs(diff_location_x) > accepted_distance or abs(diff_location_y) > accepted_distance:
                move_location = MapTraveler.get_move_location(diff_location_x, diff_location_y, move_step)
                self.matcher.click(move_location, button=button)
                sleep(move_sleep)
            else:
                log.info("Destination reached.")
                return True

    def hover_destination(self, destination, hover_image, dest_shift=None, special_shift=None):
        hover_destination_timeout = 0
        while True:
            log.info("hover_destination nr " + str(hover_destination_timeout))
            hover_destination_timeout += 1
            if hover_destination_timeout > 5:
                log.error("Failed to find " + str(hover_image))
                return False
            find_map_destination_timeout = 0
            while True:
                find_map_destination_timeout += 1
                if find_map_destination_timeout > 20:
                    log.error("Timeout when hovering destination" + str(destination))
                    raise GameError("Timeout when hovering destination" + str(destination))

                log.debug("Trying to find location nr " + str(find_map_destination_timeout))
                diff_location = self.maptraveler.get_diff_from_destination(destination,
                                                                           shift=dest_shift)
                if diff_location is not None:
                    diff_location_x, diff_location_y = diff_location
                    break

            center_x, center_y = CONFIG["CENTER_LOCATION"]

            finding_shifts = [(0, 0), (100, 0), (-100, 0), (0, 100), (0, -100), (100, 100), (100, -100), (-100, 100),
                              (-100, -100),
                              (200, 0), (-200, 0), (0, 200), (0, -200), (200, 200), (200, -200), (-200, 200),
                              (-200, -200),
                              (200, 100), (-200, 100), (100, 200), (100, -200), (200, -100), (-200, -100), (-100, 200),
                              (-100, -200),
                              (0, 250), (250, 0), (-250, 0), (0, -250), (250, 250), (250, -250), (-250, 250),
                              (-250, -250)]

            if special_shift is not None:
                special_shift_x, special_shift_y = special_shift
            else:
                special_shift_x, special_shift_y = (0, 0)

            for shift in finding_shifts:
                shift_x, shift_y = shift
                checking_location = (center_x + (diff_location_x * 15) + shift_x + special_shift_x,
                                     center_y + (diff_location_y * 15) + shift_y + special_shift_y)
                self.matcher.hover(checking_location)
                sleep(0.05)
                if self.matcher.exists(hover_image, 0.05):
                    log.info("Found {} on location {}".format(hover_image, checking_location))
                    return True

    def enter_destination(self, destination, hover_image, final_image, dest_shift=None, special_shift=None):
        enter_max_attempts = 5
        enter_timeout = 0
        while True:
            enter_timeout += 1
            # Check if no other npc was clicked not intentionally
            if Region(856, 219, 1268, 673).exists("images/cancel.png", 0.1):
                pyag.press("esc")
            if enter_timeout > enter_max_attempts:
                log.error("Timeout when entering " + str(destination))
                raise GameError("Timeout when entering " + str(destination))
            if self.hover_destination(destination, hover_image, dest_shift, special_shift):
                log.info("Destination found.")
                pyag.click()
                sleep(1)
                if Region().exists(final_image, 5):
                    log.info("Destination entered correctly.")
                    return True
                else:
                    log.warning("Failed to enter destination.")

    def goto_entrance(self):
        log.info("Going to located entrance")
        timeout_counter = 0
        while True:
            char_location = self.maptraveler.get_char_location()

            if timeout_counter > 20:
                log.error("Timeout when going to entrance.")
                return False, None

            # Check entrance position
            entrance_location = self.maptraveler.get_entrance_location()
            if entrance_location is None:
                timeout_counter += 1
                sleep(0.3)
                log.warning("Cannot find entrance location nr " + str(timeout_counter))
                continue

            log.info("Character location: {} Entrance location: {}".format(char_location, entrance_location))

            # Go to entrance possition
            entrance_x, entrance_y = entrance_location
            char_x, char_y = char_location
            if abs(entrance_x - char_x) > 40 or abs(entrance_y - char_y) > 30:
                tele_location = self.maptraveler.get_tele_location(char_location, entrance_location)
                self.maptraveler.click(tele_location, button='right')
                sleep(0.7)
                if timeout_counter > 5:
                    self.matcher.click(random.choice(list(MapTraveler.MOVE_DIRECTIONS.values())), button="right")
                    sleep(1)
            else:
                sleep(1)
                if self.hover_destination(([149, 41, 98], [235, 101, 223]), "images/dur.png"):
                    log.info("Entrance hovered")
                    if self.matcher.exists("images/3.png", threshold=0.9):
                        log.info("Entering entrance")
                        self.matcher.click(None)
                    else:
                        log.warning("Wrong entrance found.")
                        return False, self.maptraveler.get_entrance_image()
                    if Region(2162, 17, 383, 131).exists("images/indurance3.png", 5, threshold=0.9):
                        log.info("Destination entered correctly.")
                        return True, None
                    else:
                        log.info("Failed to enter durance level 3.")
                        return False, None
                else:
                    log.error("Cannot find durance description of entrance")
            timeout_counter += 1

def main():
    log.info("Character test")
    sleep(2)
    traveler = MapTraveler()
    character = Character(traveler)
    character.enter_destination("images/malah.png", "images/malah_destination.png", "images/trade.png",
                                special_shift=(70, 200))

def waypoint_test():
    log.info("Waypoint test start.")
    sleep(2)
    if Region(*CONFIG["WAYPOINT_REGION"]).exists("images/waypoint.png", 0.1):
        log.warning("Waypoint image found. Closing it.")
        pyag.press("esc")

if __name__ == '__main__':
    waypoint_test()
    #main()


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
  File: test.py
  Desc: 新闻类
  Author: CoderPig
  Date: 2019/3/22 0022 14:46
-------------------------------------------------
"""

import time
from collections import OrderedDict

class News:
    def __init__(self, _id, title=None, overview=None, url=None, image=None, publish_time=None, origin=None,
                 create_time=int(round(time.time() * 1000)), update_time=None):
        self._id = _id    # 新闻id
        self.title = title  # 新闻标题
        self.overview = overview  # 新闻概述
        self.url = url  # 新闻链接
        self.image = image  # 新闻配图
        self.publish_time = publish_time  # 新闻发布时间
        self.origin = origin  # 新闻来源
        self.create_time = create_time  # 新闻创建时间
        self.update_time = update_time  # 新闻更新时间

    def to_dict(self):
        news_dict = OrderedDict()
        news_dict['_id'] = self._id
        news_dict['title'] = self.title
        news_dict['overview'] = self.overview
        news_dict['url'] = self.url
        news_dict['image'] = self.image
        news_dict['publish_time'] = self.publish_time
        news_dict['origin'] = self.origin
        news_dict['create_time'] = self.create_time
        news_dict['update_time'] = self.update_time
        return news_dict


#**********************************************************************
# FILE: mpi_pi_reduce.py
# DESCRIPTION:
#   MPI pi Calculation Example - Python mpi4py version.
#   Collective Communication example:
#   This program calculates pi using a "dartboard" algorithm.  See
#   Fox et al.(1988) Solving Problems on Concurrent Processors, vol.1
#   page 207.  All processes contribute to the calculation, with the
#   master averaging the values for pi. This version uses mpi_reduce to
#   collect results
# AUTHOR: Blaise Barney. Adapted from Ros Leibensperger, Cornell Theory
#   Center. Converted to MPI: George L. Gusciora, MHPCC (1/95)
# LAST REVISED: 06/13/13 Blaise Barney
#   Adapted to Python / mpi4py by SH (27/10/18)
#**********************************************************************
from mpi4py import MPI
import numpy as np

DARTS = 50000     # number of throws at dartboard
ROUNDS = 100      # number of times "darts" is iterated
MASTER = 0        # task ID of master task

#**************************** functions ************************************

def dboard (darts):
    """
    Used in pi calculation example codes.
    Throw darts at board.  Done by generating random numbers
    between 0 and 1 and converting them to values for x and y
    coordinates and then testing to see if they "land" in
    the circle."  If so, score is incremented.  After throwing the
    specified number of darts, pi is calculated.  The computed value
    of pi is returned as the value of this function, dboard.
    """
    r = np.random.rand(darts,2)  # generate set of random numbers for x and y
    score = 0
    for n in range(darts):  # "throw darts at board"
        x_coord = (2.0 * r[n,0]) - 1.0
        y_coord = (2.0 * r[n,1]) - 1.0
        if (x_coord*x_coord + y_coord*y_coord) <= 1.0:
            score += 1  # if dart lands in circle, increment score
    pi = 4.0 * score/darts
    return pi

#**************************** initializations ************************************
#
comm = MPI.COMM_WORLD
numtasks = comm.Get_size()
taskid = comm.Get_rank()
print ("MPI task %d has started..." % taskid)
#
# Set seed for random number generator equal to task ID, to get different
# random numbers for each task.
#
np.random.seed(taskid)
#
#**************************** all tasks ************************************
avepi = 0
for i in range(ROUNDS):
#
# All tasks calculate pi using dartboard algorithm
#
    homepi = dboard(DARTS)
#
# Use MPI_Reduce to sum values of homepi across all tasks
# Master will store the accumulated value in pisum
# homepi is the send buffer
# pisum is the receive buffer (used by the receiving task only)
#
#    pisum = comm.reduce(homepi, None, root=MASTER)
    pisum = comm.reduce(homepi, op=MPI.SUM, root=MASTER)
#
# Master computes average for this iteration and all iterations
#
    if (taskid == MASTER):
        pi = pisum/numtasks
        avepi = ((avepi * i) + pi)/(i + 1)
        print("   After %8d throws, average value of pi = %10.8f" % (DARTS * (i + 1)*numtasks,avepi))

if (taskid == MASTER):
    print ("Real value of PI: 3.1415926535897")


import os
import os.path as osp
import random
import glob
import gast as ast

import torch
import numpy as np
from tqdm import tqdm
import scipy.stats as stats

from networkx.algorithms.isomorphism import GraphMatcher
from torch_geometric.data import Dataset
from torch.utils.data import DataLoader
from typing import Optional, Callable, List

from codescholar.utils.graph_utils import save_as_json, program_graph_to_nx
from codescholar.utils.train_utils import featurize_graph
from codescholar.sast.simplified_ast import get_simplified_ast

def my_collate(batch):
    """dummy collate fn to just return data as is"""
    return batch

class Corpus:
    """Generator for graphs saved in a dataset file to train models.

    Iteratively, new batch of graphs (positive and negative)
    are generated by sampling from the dataset.
    """

    def __init__(self, dataset, n_train, n_test, train=True, save_json=False):
        if train:
            self.train_dataset = ProgramDataset(
                root=f"./tmp/{dataset}/train", name=dataset, task="train", n_samples=n_train, save_json=save_json
            )

        self.test_dataset = ProgramDataset(root=f"./tmp/{dataset}/test", name=dataset, task="test", n_samples=n_test, save_json=save_json)

    def gen_data_loader(self, batch_size, train=True):
        if train:
            return DataLoader(self.train_dataset, collate_fn=my_collate, batch_size=batch_size, sampler=None, shuffle=False)
        else:
            return DataLoader(self.test_dataset, collate_fn=my_collate, batch_size=batch_size, sampler=None, shuffle=False)

class ProgramDataset(Dataset):
    def __init__(
        self,
        root: str,
        name: str,
        task: str,
        n_samples: int,
        transform: Optional[Callable] = None,
        pre_transform: Optional[Callable] = None,
        pre_filter: Optional[Callable] = None,
        save_json: Optional[bool] = False,
        min_size=2,
        max_size=20,
    ):
        if not os.path.exists(root):
            raise ValueError(f"Dataset for {name} not found in {root}")

        self.name = name
        self.graph_dir = osp.join(root, "graphs")

        if not os.path.exists(self.graph_dir):
            os.makedirs(self.graph_dir)

        self.task = task
        self.n_samples = n_samples
        self.min_size = min_size
        self.max_size = max_size
        self.save_json = save_json
        self.json_dir = osp.join(root, "networkx")

        if self.save_json and not os.path.exists(self.json_dir):
            os.makedirs(self.json_dir)

        super().__init__(root, transform, pre_transform, pre_filter)

    def download(self):
        pass

    @property
    def num_node_labels(self) -> int:
        return self.sizes["num_node_labels"]

    @property
    def num_node_attributes(self) -> int:
        return self.sizes["num_node_attributes"]

    @property
    def num_edge_labels(self) -> int:
        return self.sizes["num_edge_labels"]

    @property
    def num_edge_attributes(self) -> int:
        return self.sizes["num_edge_attributes"]

    @property
    def raw_file_names(self) -> List[str]:
        names = sorted(glob.glob(osp.join(self.raw_dir, "*.py")))
        names = [os.path.basename(file) for file in names]
        return names

    @property
    def processed_file_names(self) -> str:
        names = sorted(glob.glob(osp.join(self.processed_dir, "*.pt")))
        names = [os.path.basename(file) for file in names]
        return names
        # return 'not_implemented.pt'

    def pre_transform(self, data):
        return data

    def process(self):
        """process and store python graphs as a Data object onto the disk"""
        count = 0
        program_size = []

        # Convert each python file to networkx graphs and save in self.graph_dir
        for raw_path in tqdm(self.raw_paths, desc=f"Preprocess-{self.task}"):
            data = self.create_prog_datapoint(raw_path)

            if data is None:
                continue

            if self.pre_filter is not None and self.prefilter(data):
                continue

            if self.pre_transform is not None:
                data = self.pre_transform(data)

            if self.save_json is True:
                save_as_json(data, osp.join(self.json_dir, f"data_{count}.json"))

            torch.save(data, osp.join(self.graph_dir, f"data_{count}.pt"))
            program_size.append(len(data))
            count += 1

        # Create randomly mutated examples and store in self.processed_dir
        self.random_samples_generator(program_size, count)

    def random_samples_generator(self, program_size, count):
        """Steps:
        1. choose a random size for graph
        2. choose a random target graph
        3. perform random bfs traversal in neighborhood = size
        4. choose a random but anchored query = positive e.g.
        5. repeat for negative e.g. but with different random graph
        for the query"""

        idx = 0
        pbar = tqdm(total=self.n_samples + 1, desc=f"RandBFSSample-{self.task}")

        while idx < self.n_samples:
            size = random.randint(self.min_size + 1, self.max_size)
            graph, t = self.sample_neigh(program_size, count, size)
            q = t[: random.randint(self.min_size, len(t) - 1)]

            anchor = list(graph.nodes)[0]
            pos_t_anchor = anchor
            pos_q_anchor = anchor
            pos_t, pos_q = graph.subgraph(t), graph.subgraph(q)

            size = random.randint(self.min_size + 1, self.max_size)
            graph_t, t = self.sample_neigh(program_size, count, size)
            graph_q, q = self.sample_neigh(program_size, count, random.randint(self.min_size, size - 1))

            neg_t_anchor = list(graph_t.nodes)[0]
            neg_q_anchor = list(graph_q.nodes)[0]
            neg_t, neg_q = graph_t.subgraph(t), graph_q.subgraph(q)

            # translate to DeepSnap Graph
            pos_t = featurize_graph(pos_t, pos_t_anchor)
            pos_q = featurize_graph(pos_q, pos_q_anchor)
            neg_t = featurize_graph(neg_t, neg_t_anchor)
            neg_q = featurize_graph(neg_q, neg_q_anchor)

            data = [pos_t, pos_q, neg_t, neg_q]
            torch.save(data, osp.join(self.processed_dir, f"data_{idx}.pt"))
            idx += 1
            pbar.update(1)

    def sample_neigh(self, ps, count, size):
        """random bfs walk to find neighborhood graphs of a set size"""
        ps = np.array(ps, dtype=np.float)
        ps /= np.sum(ps)
        dist = stats.rv_discrete(values=(np.arange(count), ps))

        while True:
            idx = dist.rvs()
            graph = torch.load(osp.join(self.graph_dir, f"data_{idx}.pt"))

            start_node = random.choice(list(graph.nodes))
            neigh = [start_node]
            frontier = list(set(graph.neighbors(start_node)) - set(neigh))
            visited = set([start_node])

            while len(neigh) < size and frontier:
                new_node = random.choice(list(frontier))

                assert new_node not in neigh

                neigh.append(new_node)
                visited.add(new_node)
                frontier += list(graph.neighbors(new_node))
                frontier = [x for x in frontier if x not in visited]

            if len(neigh) == size:
                return graph, neigh

    def create_prog_datapoint(self, path):
        with open(path, "r") as fp:
            source = fp.read()
        try:
            program = ast.parse(source)
        except:
            return None

        # prog_graph = program_graph.get_program_graph(path)
        prog_graph = get_simplified_ast(source, dfg=False, cfg=False)
        if prog_graph is None:
            return None

        digraph = program_graph_to_nx(prog_graph, directed=True)

        return digraph

    def len(self):
        return self.n_samples

    def get(self, idx):
        """load a python method (graph) from disk."""
        data = torch.load(osp.join(self.processed_dir, f"data_{idx}.pt"))
        return data


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import division
import numpy as np
import pandas as pd
import json
from pymystem3 import Mystem
import re
import io
import itertools

analyzer = Mystem()
strip_characters = '!`«»‰~@\'▲▼„"#$%^&*()_-+=[]\{\}°:;|•.,<>?/————–'
split_reg = re.compile('[\s\(\)\.,;"«»\'\[\]&<>]+')
punctuation = set(strip_characters)
num_reg = re.compile(r'[0-9)(:\-/№#]+?')
pos_map = {'A': 'ADJ',
            'ADV': 'ADV',
            'ADVPRO': 'PRON',
            'ANUM': 'ADJ',
            'APRO': 'PRON',
            'COM': 'NOUN',
            'CONJ': 'CCONJ',
            'INTJ': 'PART',
            'NUM': 'NUM',
            'PART': 'PART',
            'PR': 'ADP',
            'S': 'NOUN',
            'SPRO':'PRON',
            'V': 'VERB'}

with open('stop_words.json') as file:
        stop_words = json.load(file)

adress_reg = re.compile('(?:г\.|город|ул\.|улица|дом|д\.|стр\.|строение|проезд|бульвар)\s*[А-ЯЁа-яёa-z0-9\-\.]+?[,\s$][0-9]*')
adress_keywords_reg = re.compile('(?:г\.|город|ул\.|улица|дом|д\.|стр\.|строение|проезд|бульвар)')
nickname_reg = re.compile('@[a-zA-Z_\-0-9]+?[\s\.,]')

def remove_nickname_(string):
    string = nickname_reg.sub(string=string, repl=' ')
    return string

def tokenize(string, need_pos=False, remove_nickname=True):
    if remove_nickname:
        tokens = [lemmatize_pos_tag(i.lower().strip(strip_characters), need_pos=need_pos)
                  for i in re.split(split_reg, remove_nickname_(string)) if i.strip(strip_characters) and i not in punctuation
                  and i not in stop_words and not num_reg.match(i)]
    else:
        tokens = [lemmatize_pos_tag(i.lower().strip(strip_characters), need_pos=need_pos)
                  for i in re.split(split_reg, string) if i.strip(strip_characters) and i not in punctuation
                  and i not in stop_words and not num_reg.match(i)]
    valid_tokens = [token for token in tokens if token]
    return valid_tokens

def lemmatize_pos_tag(token, need_pos):
    if need_pos:
        analysis = analyzer.analyze(token)
        try:
            lemma = analysis[0]['analysis'][0]['lex']
            pre_pos = re.split('[,=]+?', analysis[0]['analysis'][0]['gr'])[0]
            pos = pos_map[pre_pos]
            lemma = lemma+'_'+pos
        except (KeyError, IndexError) as e:
            lemma = token
            pos = 'NOUN'
            lemma = lemma+'_'+pos
    else:
        try:
            lemma = analyzer.lemmatize(token)[0]
        except (KeyError, IndexError) as e:
            lemma = token
    return lemma


import time

from selenium.webdriver.common.by import By

link = "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/"

def test_find_the_button(browser):
    # Check if "add to basket" button exist
    browser.get(link)
    time.sleep(30)
    btn = browser.find_element(By.XPATH, "//form[@id='add_to_basket_form']/button[@type='submit']")

    assert btn, "Button does not found"


#!/usr/bin/env python
import numba
import numpy as np

__all__ = ["frame", "closing", "opening", "resample", "dcm", "overlapsave", "hamming"]

def frame(x, frame_length, frame_step, pad=True, pad_value=0j):
    """
    Creates frames out of the 1D array x that are frame_length
    long, and by stepping frame_step samples.

    Parameters
    ----------
    x : numpy array of shape (N,) where N is the number of samples

    frame_length : int
        Length of each frame

    frame_step : int
        Specifies how many samples to step for each frame
        this does not have to be equal to frame_length

    pad : bool (optional, default is True)
        Whether the end of x should be padded before forming the
        frames. If false, the frames will only be created out to the
        last frame that contains samples, if true it will be padded out
        so that all samples are contained in a frame but the last frame
        may contain some pad values

    pad_value : float or complex (optional, defaults to 0j)
        If ``pad`` is True, this is the value that should be used to pad

    Returns
    -------
    x_frame : a (M, frame_length) sized array
        the framed data

    Examples
    --------
    >>> import numpy as np
    >>> from siglib import frame
    >>> x = np.arange(10)
    >>> frame(x, 5, 5)
    array([[0.+0.j, 1.+0.j, 2.+0.j, 3.+0.j, 4.+0.j],
           [5.+0.j, 6.+0.j, 7.+0.j, 8.+0.j, 9.+0.j]])
    """
    if pad:
        n_frames = int(np.ceil((x.shape[-1] - frame_length) / frame_step) + 1)
        n_pad = int((n_frames - 1) * frame_step + frame_length - x.shape[-1])
        x = np.concatenate([x, np.full(n_pad, pad_value)], axis=-1)
    else:
        n_frames = int(np.floor((x.shape[-1] - frame_length) / frame_step) + 1)

    # Explanation for the below:
    # * ``np.arange(frame_length).reshape(1, -1)`` creates a row vector from [0, frame_length)
    # * ``np.arange(n_frames).reshape(-1, 1)`` creates a column vector from [0, n_frames)
    # * multiplying by frame_step makes the sliding window slide frame_steps at a time
    # * adding the two together gives us a n_frames x frame_length matrix,
    #   where each row is a window
    x_frame_idx = np.arange(frame_length).reshape(1, -1) + frame_step * np.arange(
        n_frames
    ).reshape(-1, 1)
    return x[x_frame_idx]

def closing(x, ntaps):
    """
    Computes the closing morphological filter over ntaps of the input x

    Parameters
    -----------
    x : array of shape (N,)

    ntaps : int
        The width over which the min/max functions are calculated

    Returns
    --------
    y: array of shape (N,)
        The closing function over x

    Examples
    --------
    >>> import numpy as np
    >>> from siglib import closing
    >>> x = np.arange(10)
    >>> closing(x, 5)
    array([4.+0.j, 4.+0.j, 4.+0.j, 4.+0.j, 4.+0.j, 5.+0.j, 6.+0.j, 7.+0.j,
           8.+0.j, 9.+0.j])
    """
    x = np.concatenate([x, np.ones(ntaps - 1) * x[-1]])
    xf = frame(x, frame_length=ntaps, frame_step=1, pad=False)
    d = np.max(xf, axis=-1)
    d = np.concatenate([np.ones(ntaps - 1) * d[0], d])
    df = frame(np.flip(d), frame_length=ntaps, frame_step=1, pad=False)
    c = np.min(df, axis=-1)
    return np.flip(c)

def opening(x, ntaps):
    """
    Computes the opening morphological filter over ntaps of the input x

    Parameters
    -----------
    x: array of shape (N,)

    ntaps: int
        The width over which the min max functions  are calculated

    Returns
    --------
    y: array of shape (N,)
        The opening function over x

    Examples
    --------
    >>> import numpy as np
    >>> from siglib import opening
    >>> x = np.arange(10)
    >>> opening(x, 5)
    array([0.+0.j, 1.+0.j, 2.+0.j, 3.+0.j, 4.+0.j, 5.+0.j, 6.+0.j, 7.+0.j,
           8.+0.j, 9.+0.j])
    """
    x = np.concatenate([x, np.ones(ntaps - 1) * x[-1]])
    xf = frame(x, frame_length=ntaps, frame_step=1, pad=False)
    e = np.min(xf, axis=-1)
    e = np.concatenate([np.ones(ntaps - 1) * e[0], e])
    ef = frame(np.flip(e), frame_length=ntaps, frame_step=1, pad=False)
    o = np.max(ef, axis=-1)
    return np.flip(o)

def resample(x, idx, ntaps):
    """
    Resamples the array x to the indicies in idx using sinc interpolation with
    ntaps number of points

    Parameters
    -----------
    x: array of shape (N,)

    idx: array of shape (M,)

    ntaps: int
        How many points to use for the sinc interpolation

    Returns
    --------
    y: array of shape (M,)
        The sinc interpolated signal at the indicies idx

    Example
    -------
    >>> import numpy as np
    >>> from siglib import resample
    >>> x = np.arange(10**2)
    >>> idx = np.array([45.567])
    >>> resample(x, idx, 5)
    array([44.96565413])
    """
    d_idx = np.ceil(np.arange(-ntaps / 2, ntaps / 2))
    idx_array = np.round(idx[:, np.newaxis]) + d_idx
    idx_array = np.asarray(idx_array, dtype=np.int32)
    sinc_array = idx[:, np.newaxis] - idx_array

    y = np.sum(x[idx_array] * np.sinc(sinc_array), axis=-1)
    return y

@numba.njit(fastmath=True)
def dcm(x, delay=1, pad=True, pad_value=1 + 0j):
    """
    Perform a delay-conjugate-multiply on
    some input signal ``x`` with delay ``delay``

    Parameters
    ----------
    x : array of shape (N,)

    delay : int (optional, defaults to 1)

    pad : bool (optional, defaults to True)
        Whether or not to pad the right end of ``x``

    pad_value : complex (optional, defaults to 1+0j)
        If ``pad`` is True, this is the value that should be used to pad.
        Note: defaults to ``1+0j`` to avoid issues in subsequent ``np.atan2``.

    Returns
    -------
    y : array of shape (N,)
        Delay-conjugate-multiply result

    Example
    -------
    >>> from siglib import dcm
    >>> x = np.array([1 + 3j, 4 + 2j, 5 + 6j, 1 + 0j])
    >>> dcm(x)
    array([10.-10.j, 32.+14.j,  5. -6.j,  1. +0.j])
    """
    if pad:
        x_pad = np.concatenate((x, np.full(delay, pad_value)), axis=-1)
        return x_pad[delay:] * np.conjugate(x_pad[:-delay])
    else:
        return x[delay:] * np.conjugate(x[:-delay])

def overlapsave(x, H, step):
    """
    Perform convolution of the 1D array x with a bank of fourier domain filters
    H. The convolution is done (as you might expect) by the overlapsave method.
    I know I could have the zero pad and fourier operations to get to H in this
    function, but this way you have to know a little about what you are doing.

    Parameters
    ----------
    x : array of shape (N,)

    H : array of shape (L,M,)
        The rows are zero padded and Fourier Transformed filters. There are L
        different filters each of length M << N. M specifies the length of the
        frames into which x is broken.

    step : int
        Specifies what the step is for each of the frame

    Returns
    -------
    xh : array of shape (L, N+M-step,)
        Each row is the convolution of x with the respective filter in H.

    Example
    -------
    >>> from siglib import overlapsave
    >>> x = np.array([ 0.+3.j, -4.+2.j, -4.+1.j, -5.+2.j, -4.-3.j,  2.+1.j,
    ...               -1.-2.j,  3.+3.j, -3.-1.j, -3.+2.j, -4.+0.j, -4.-4.j,
    ...               -3.-4.j, -4.-3.j, -4.+3.j, -3.+4.j, -4.-1.j, -5.+0.j,
    ...                4.+2.j,  2.-3.j])
    >>> h = np.array([-4.+4.j, -4.+3.j])
    >>> overlap = h.shape[-1] - 1
    >>> len_fft = int(2**(np.ceil(np.log2(8 * overlap))))
    >>> H = np.fft.fft(np.concatenate([h, np.zeros(len_fft - h.shape[-1])]))
    >>> step = H.shape[-1] - overlap
    >>> overlapsave(x, H, step)
    array([[-12.-12.j,  -1.-36.j,  22.-40.j,  25.-44.j,  42.-27.j,  13. +4.j,
          1. +6.j, -14. +5.j,  -5.-11.j,  19.-25.j,  22.-33.j,  48.-12.j,
         56. +8.j,  52. +3.j,  29.-28.j,   3.-52.j,  20.-37.j,  39.-28.j,
         -4. -7.j, -18.+24.j,   1.+18.j]])
    """
    # lets accept a 1D, 2D, or 3D H, in the first two cases we need to add a
    #   dimension for the array broadcasting in the X * H operation
    if H.ndim == 1:
        H = H[np.newaxis, np.newaxis, :]
    elif H.ndim == 2:
        H = H[:, np.newaxis, :]

    overlap = H.shape[-1] - step

    # since we will drop the first overlap samples in each row, lets zero pad
    # x by the overlap value, this will also make it so that the output
    # matches that of scipy.signal
    x = np.concatenate([np.zeros(overlap), x], axis=-1)

    N = x.shape[-1]

    x = frame(x, H.shape[-1], step, pad=True, pad_value=0j)
    X = np.fft.fft(x, axis=-1)
    XH = X * H
    xh = np.fft.ifft(XH, axis=-1)

    # for each row and for each filter take from index overlap till the end
    xh = xh[:, :, overlap:]

    # now reshape to get L rows and then drop the padding that happend at the
    # end during the frame operation
    xh = np.reshape(xh, (H.shape[0], xh.shape[-1] * xh.shape[-2]))
    xh = xh[:, :N]
    return xh

@numba.njit(fastmath=True)
def hamming(window_length):
    """
    A simple hamming window generator, I know, I know it is in scipy.signal
    with a bunch more OO windows and such but I just wanted something simple

    Parameters
    ----------
    window_length : int
        Length of the window

    Returns
    -------
    h : array of shape (N,)
        Contains the coeficients of the Hamming window

    Example
    -------
    >>> from siglib import hamming
    >>> hamming(5)
    array([0.08, 0.54, 1.  , 0.54, 0.08])
    """
    n = np.arange(0, window_length)
    return 0.54 - 0.46 * np.cos(2 * np.pi * n / (window_length - 1))


from django.shortcuts import render, redirect
from django.contrib import messages
from django.core.mail import send_mail
from django.conf import settings

def home(request):
    """
    View function to render the home page.

    Args:
        request: The HTTP request object.

    Returns:
        A rendered HTML template for the home page.
    """
    return render(request, "frontPage/home.html")

def about(request):
    """
    View function to render the about page.

    Args:
        request: The HTTP request object.

    Returns:
        A rendered HTML template for the about page.
    """
    return render(request, "frontPage/about.html")

def contact(request):
    """
    View function to handle the contact form submission.

    Args:
        request: The HTTP request object.

    Returns:
        A redirection to the contact page after submitting the form,
        or a rendered HTML template for the contact page if it's a GET request.
    """
    if request.method == 'POST':
        first_name = request.POST.get('firstname')
        last_name = request.POST.get('lastname')
        email = request.POST.get('email')
        country = request.POST.get('country')
        from_message = request.POST.get('message')

        send_mail(
            subject="Message from Contact Form",
            message=f'Name: {first_name} {last_name}\nEmail: {email}\nCountry: {country}\n\nMessage: {from_message}',
            from_email=settings.EMAIL_HOST_USER,
            recipient_list=[settings.EMAIL_HOST_USER],
            fail_silently=False,
        )
        messages.add_message(request, messages.SUCCESS,
                                         'Your email has been successfully sent')
        return redirect('contact')

    context = {
        'page': 'contact'
    }
    return render(request, "frontPage/contact.html", context)


from parsel import Selector
import requests
from argparse import ArgumentParser

def run():
    parser = ArgumentParser()
    parser.add_argument(
        "base_url",
        help="Base URL to find file on",
        default="https://familieretshuset.dk/navne/navne/godkendte-fornavne",
    )

    args = parser.parse_args()

    resp = requests.get(args.base_url)
    doc = Selector(text=resp.text)

if __name__ == "__main__":
    run()


fout = open('oops.txt','wt')
print('Oops, I created Imgoos.', file=fout)
fout.close()

import os
os.path.exists('Oops.txt')
os.path.exists('./oops.txt')
os.path.exists('waffles')
os.path.exists('.')
os.path.exists('..')

name = 'oops.txt'
os.path.isfile(name)
os.path.isdir(name)
os.path.isdir('.')
os.path.isabs('name')
os.path.isabs('/big/fake/name')
os.path.isabs('big/fake/name/withoua/a/leading/slash')
import shutil
shutil.copy('oops.txt','ohno.txt')
shutil.move('ohno.txt','ohno2.txt')
os.rename('ohno2.txt', 'ohno.txt')
os.link('oops.txt','yikes.txt')
os.path.isfile('yikes.txt')
os.path.islink('yikes.txt')
os.symlink('oops.txt','jeepers.txt')
os.path.islink('jeepers.txt')
os.chmod('oops.txt',0o400)
import stat
os.chmoe('oops.txt', stat.S_IRUSR)
os.path.abspath('oops.txt')
os.remove('oops.txt')
os.path.exists('oops.txt')
os.mkdir('poems')
os.path.exists('poems')
os.rmdir('poems')
os.path.exists('poems')
os.mkdir('poems')
os.listdir('poems')
os.mkdir('poems/mcintyre')
os.listdir('poems')
fout = open('poems/mcintyre/the_good_man','wt')
fout.write('Imgoos is king')
fout.close()
os.listdir('poems/mcintyre')
os.chdir('poems')
os.listdir('.')
import glob
glob.glob('m*')
glob.glob('??')
glob.glob('m??????e')
glob.glob('[klm]*e')
os.getpid()
os.getcwd()

import subprocess
import multiprocessing
import calendar
calendar.isleap(1900)
from datetime import date
halloween = date(2015,10,31)
halloween
halloween.day
halloween.month
halloween.isoformat()
now = date.today()
from datetime import timedelta
one_day = timedelta(days=1)
tomorrow = now + one_day
tomorrow
now + 17 * one_day
now - one_day
from datetime import time
noon = time(12,0,0)
noon
noon.hour
noon.microsecond
from datetime import datetime
some_day = datetime(2015,1,2,3,4,5,6)
some_day
some_day.isoformat()
now = datetime.now()
now.minute
now.microsecond
noon = time(12)
this_day = date.today()
noon_today = datetime.combine(this_day, noon)
noon_today
noon_today.date()
noon_today.time()
import time
now = time.time()
now
time.ctime(now)
time.localtime(now)
time.gmtime(now)
tm = time.localtime(now)
time.mktime(tm)
time.ctime(time.time())
fmt = "It's %A, %B %d, %Y, local time %I:%M:%S%p"
t = time.localtime()
t
time.strftime(fmt, t)
fmt = "%Y-%m-%d"
time.strptime("2015-06-02", fmt)

import locale
from datetime import date
halloween = date(2015, 10, 31)
for lang_country in ['ko-kr', 'en-us']:
    locale.setlocale(locale.LC_TIME, lang_country)
    halloween.strftime('%A, %B %d')

names = locale.locale_alias.keys()
good_names = [name for name in names if len(name) == 5 and name[2] == '_']
good_names[:5]
de = [name for name in names if name.startswith('de')]
de
# arrow dateutil iso8601 fleming

Python/처음 시작하는 파이썬(Introducing Python)/Chap_10.py