from scipy.interpolate import make_interp_spline

import matplotlib.pyplot as plt
import numpy as np

import random
import math

# M = int(input("Enter the number of experiments(M) : "))
# N = int(input("Enter the number of alternatives in each experiment(N) : "))

M = 1000
N = 100

# min_val = int(input("Random Mim: "))
# max_val = int(input("Random Max: "))

min_val = 1
max_val = 2000

delta_input = int(input("Start delta: "))
# delta_input = 0

experiments, results = [[random.randint(min_val, max_val) for _ in range(N)] for _ in range(M)], []

t = round(N / math.e)

results_check = []

# Monte Carlo Method
def Monte_carlo_Method(delta_input, t):
    results = []
    for j in range(M):
        opt_max, all_max = max([experiments[j][k] for k in range(t)]), max(experiments[j])

        first_max, delta = 0, (delta_input / 100) * all_max

        for i in range(t, N):
            if opt_max <= experiments[j][i]:
                first_max = experiments[j][i]
                if abs(first_max - all_max) <= delta:
                    results.append(1)
                    results_check.append(1)
                    break
                else:
                    results.append(0)
                    results_check.append(0)
                    break

    P = (sum(results) / M)
    return P

# Виведення масиву чисел (з поступкою 0%)
res_1 = Monte_carlo_Method(0, 37)
print(f'{res_1}, {results_check}')

P_delta_zero = Monte_carlo_Method(delta_input, t)
print(f"Probability(P(∆)) ∆ = 0: {int(P_delta_zero * 100)}%")

deltas, different_t = [0, 2, 4, 6, 8, 10], [i * 10 - 5 for i in range(1, 11)]
results_dict = {}

for delta in deltas:
    results_dict[delta] = []
    for t in different_t:
        res = Monte_carlo_Method(delta, t)
        results_dict[delta].append(res)

j = 0
for i in results_dict:
    x_values = different_t
    y_values = results_dict[i]

    plt.figure(figsize=(12, 6))
    plt.plot(x_values, y_values, marker='o')
    plt.xlabel('t')
    plt.ylabel('P')
    plt.title(f'Concession = {j}')

    plt.xticks(range(min(x_values), max(x_values) + 1, 5))
    j += 2
    plt.grid(True)
    plt.show()

results_p, t = [], round(N / math.e)
for delta in deltas:
    res1 = Monte_carlo_Method(delta, t)
    results_p.append(res1)
print(results_p)

baseline = results_p[0]

# Обчислення відсоткової різниці між значеннями
for value in results_p[1:]:
    percentage_difference = ((value - baseline) / baseline) * 100
    print(f" % difference between {baseline} and {value}: {percentage_difference:.2f}%")

x = deltas
y, res_data = results_p, results_p
xnew = np.linspace(min(x), max(x), 1000)
spl = make_interp_spline(x, y, k=2)
y_smooth = spl(xnew)
plt.plot(xnew, y_smooth, label='∆ = 0')
plt.plot(deltas, res_data, marker='o')
plt.title('Залежність максимальної ймовірності успіху від велечини поступки')
plt.xlabel("Поступка")
plt.ylabel("Ймовірність")
plt.grid(True)
plt.ylim(0, 0.70)
plt.show()

deltas, result_t = [0, 2, 4, 6, 8, 10], []
for delta in deltas:
    max_el = max(results_dict[delta])
    t = (results_dict[delta].index(max_el) + 1) * 10 - 5
    result_t.append(t)
x = deltas
y, res_data = result_t, result_t
xnew = np.linspace(min(x), max(x), 700)
spl = make_interp_spline(x, y, k=2)
y_smooth = spl(xnew)
plt.plot(xnew, y_smooth, label='∆ = 0')
plt.plot(deltas, res_data, marker='o')
plt.title('Залежність номеру зупинки від велечини поступки')
plt.xlabel("Поступка")
plt.ylabel("Крок зупинки (t*)")
plt.grid(True)
plt.ylim(0, 35)
plt.show()

print(result_t)

baseline_2 = result_t[0]

# Обчислення відсоткової різниці між значеннями
for value in result_t[1:]:
    percentage_difference = ((value - baseline_2) / baseline_2) * 100
    print(f" % difference between {baseline_2} and {value}: {percentage_difference:.2f}%")


from django.urls import path
from weather.geoapps.views import show_geoapp, show_goeadmin, show_api_v1, show_api_v2, show_api_v3, show_api_history, show_api_station

app_name = 'weather.geoapps'

urlpatterns = [
    path('', show_geoapp, name='home'),
    path('dashboard/show/geoapp', show_goeadmin, name='show'),
    path('api/v1', show_api_v1, name='apiv1'),
    path('api/v2', show_api_v2, name='apiv2'),
    path('api/v3', show_api_v3, name='apiv3'),
    path('api/v4', show_api_history, name='history'),
    path('api/v5', show_api_station, name='station'),
]


import pytest
import pytorch_lightning as pl
import torch
from torch.nn import CrossEntropyLoss

from kale.embed.mogonet import MogonetGCN
from kale.loaddata.multiomics_datasets import SparseMultiomicsDataset
from kale.pipeline.multiomics_trainer import MultiomicsTrainer
from kale.predict.decode import LinearClassifier, VCDN
from kale.prepdata.tabular_transform import ToOneHotEncoding, ToTensor
from kale.utils.seed import set_seed

binary_class_data_url = "https://github.com/pykale/data/raw/main/multiomics/ROSMAP.zip"
multi_class_data_url = "https://github.com/pykale/data/raw/main/multiomics/TCGA_BRCA.zip"

@pytest.fixture
def test_model(num_classes, url):
    num_modalities = 3
    gcn_hidden_dim = [200, 200, 100]
    vcdn_hidden_dim = pow(num_classes, num_modalities)
    gcn_dropout_rate = 0.5
    gcn_lr = 5e-4
    vcdn_lr = 1e-3
    loss_function = CrossEntropyLoss(reduction="none")

    unimodal_encoder = []
    unimodal_decoder = []
    multimodal_decoder = None

    if num_classes > 2:
        root = "tests/test_data/multiomics/trainer/multi_class"
    else:
        root = "tests/test_data/multiomics/trainer/binary_class"

    file_names = []
    for modality in range(1, num_modalities + 1):
        file_names.append(f"{modality}_tr.csv")
        file_names.append(f"{modality}_lbl_tr.csv")
        file_names.append(f"{modality}_te.csv")
        file_names.append(f"{modality}_lbl_te.csv")

    dataset = SparseMultiomicsDataset(
        root=root,
        raw_file_names=file_names,
        num_modalities=num_modalities,
        num_classes=num_classes,
        edge_per_node=10,
        url=url,
        random_split=False,
        train_size=0.7,
        equal_weight=False,
        pre_transform=ToTensor(dtype=torch.float),
        target_pre_transform=ToOneHotEncoding(dtype=torch.float),
    )

    for modality in range(num_modalities):
        unimodal_encoder.append(
            MogonetGCN(
                in_channels=dataset.get(modality).num_features,
                hidden_channels=gcn_hidden_dim,
                dropout=gcn_dropout_rate,
            )
        )

        unimodal_decoder.append(LinearClassifier(in_dim=gcn_hidden_dim[-1], out_dim=num_classes))

    if num_modalities >= 2:
        multimodal_decoder = VCDN(num_modalities=num_modalities, num_classes=num_classes, hidden_dim=vcdn_hidden_dim)

    trainer = MultiomicsTrainer(
        dataset=dataset,
        num_modalities=num_modalities,
        num_classes=num_classes,
        unimodal_encoder=unimodal_encoder,
        unimodal_decoder=unimodal_decoder,
        loss_fn=loss_function,
        multimodal_decoder=multimodal_decoder,
        train_multimodal_decoder=True,
        gcn_lr=gcn_lr,
        vcdn_lr=vcdn_lr,
    )

    return trainer

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_init(test_model, num_classes, url):
    assert isinstance(test_model.dataset, SparseMultiomicsDataset)
    assert test_model.num_modalities == 3
    assert test_model.num_classes == num_classes
    assert all(isinstance(encoder, MogonetGCN) for encoder in test_model.unimodal_encoder)
    assert all(isinstance(decoder, LinearClassifier) for decoder in test_model.unimodal_decoder)
    assert isinstance(test_model.loss_fn, CrossEntropyLoss)
    assert isinstance(test_model.multimodal_decoder, VCDN)
    assert test_model.train_multimodal_decoder
    assert test_model.gcn_lr == 5e-4
    assert test_model.vcdn_lr == 1e-3

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_configure_optimizers(test_model, num_classes, url):
    optimizers = test_model.configure_optimizers()
    assert len(optimizers) == 4
    assert all(isinstance(optimizer, torch.optim.Adam) for optimizer in optimizers)
    for modality in range(test_model.num_modalities):
        assert optimizers[modality].param_groups[0]["lr"] == 5e-4
    assert optimizers[test_model.num_modalities].param_groups[0]["lr"] == 1e-3

@pytest.mark.parametrize(
    "num_classes, url, multimodal",
    [
        (2, binary_class_data_url, False),
        (2, binary_class_data_url, True),
        (5, multi_class_data_url, False),
        (5, multi_class_data_url, True),
    ],
)
def test_forward2(test_model, num_classes, url, multimodal):
    x = []
    adj_t = []
    for modality in range(test_model.num_modalities):
        data = test_model.dataset.get(modality)
        x.append(data.x[data.train_idx])
        adj_t.append(data.adj_t_train)

    outputs = test_model.forward(x, adj_t, multimodal)

    assert isinstance(outputs, list) != multimodal
    if not multimodal:
        assert len(outputs) == test_model.num_modalities
        for modality in range(test_model.num_modalities):
            assert outputs[modality].shape == (test_model.dataset.get(modality).num_train, test_model.num_classes)
    else:
        assert isinstance(outputs, torch.Tensor)
        assert outputs.shape == (test_model.dataset.get(0).num_train, test_model.num_classes)
        test_model.multimodal_decoder = None
        with pytest.raises(TypeError):
            _ = test_model.forward(x, adj_t, multimodal)

@pytest.mark.parametrize("num_classes, url", [(2, binary_class_data_url), (5, multi_class_data_url)])
def test_pipeline(test_model, num_classes, url):
    set_seed(2023)
    trainer_pretrain = pl.Trainer(
        default_root_dir="./tests/outputs",
        max_epochs=2,
        accelerator="cpu",
        enable_model_summary=False,
    )
    trainer_pretrain.fit(test_model)
    result = trainer_pretrain.test(test_model)
    if num_classes > 2:
        assert 0 <= result[0]["Accuracy"] <= 1
        assert 0 <= result[0]["F1 weighted"] <= 1
        assert 0 <= result[0]["F1 macro"] <= 1
    else:
        assert 0 <= result[0]["Accuracy"] <= 1
        assert 0 <= result[0]["F1"] <= 1
        assert 0 <= result[0]["AUC"] <= 1

    assert str(test_model) is not None
    assert len(str(test_model)) > 0


import torch
import torch.nn as nn
import torch.nn.functional as F
from collections import namedtuple

Hypothesis = namedtuple('Hypothesis', ['value', 'score'])

class Encoder(nn.Module):
    def __init__(self, config):
        super(Encoder, self).__init__()
        self.embed = nn.Embedding(config.enc_vocab_size, config.embed_size)

        self.rnn = nn.GRU(config.embed_size, config.enc_hidden_size, batch_first=True, bidirectional=True)
        self.dropout = nn.Dropout(config.dropout)
        # 将encoder的输出转为decoder的输入，* 2 是使用了bidirectional
        self.fc = nn.Linear(config.enc_hidden_size * 2, config.dec_hidden_size)

    def forward(self, x, lengths):
        embedded = self.dropout(self.embed(x))

        # 新版pytorch增加了batch里的排序功能，默认需要强制倒序
        # pack_padded_sequence里面的length需要放在cpu上
        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, lengths.cpu(), batch_first=True)
        # hid 【2, batch, enc_hidden_size】
        packed_out, hid = self.rnn(packed_embedded)
        # 【batch, seq, 2 * enc_hidden_size】
        out, _ = nn.utils.rnn.pad_packed_sequence(packed_out, batch_first=True, total_length=max(lengths))

        # 将hid双向叠加 【batch, 2*enc_hidden_size】
        hid = torch.cat([hid[-2], hid[-1]], dim=1)
        # 转为decoder输入hidden state 【1,batch,dec_hidden_size】
        hid = torch.tanh(self.fc(hid)).unsqueeze(0)

        return out, hid

class Attention(nn.Module):
    """  """

    def __init__(self, config):
        super(Attention, self).__init__()

        self.enc_hidden_size = config.enc_hidden_size
        self.dec_hidden_size = config.dec_hidden_size

        self.liner_in = nn.Linear(2 * config.enc_hidden_size, config.dec_hidden_size)
        self.liner_out = nn.Linear(2 * config.enc_hidden_size + config.dec_hidden_size, config.dec_hidden_size)

    def forward(self, output, context, mask):
        # context 上下文输出，即encoder的gru hidden state 【batch,enc_seq,enc_hidden*2】
        # output  decoder的gru hidden state  【batch,dec_seq, dec_hidden】
        # mask 【batch, dec_seq, enc_seq】mask在decoder中创建

        batch_size = context.shape[0]
        enc_seq = context.shape[1]
        dec_seq = output.shape[1]

        # score计算公式使用双线性模型 h*w*s
        context_in = self.liner_in(context.reshape(batch_size * enc_seq, -1).contiguous())
        context_in = context_in.view(batch_size, enc_seq, -1).contiguous()
        atten = torch.bmm(output, context_in.transpose(1, 2))
        # 【batch,dec_seq,enc_seq】

        atten.data.masked_fill(mask, -1e6)  # mask置零
        atten = F.softmax(atten, dim=2)

        # 将score和value加权求和，得到输出
        # 【batch, dec_seq, 2*enc_hidden】
        context = torch.bmm(atten, context)
        # 将attention + output 堆叠获取融合信息
        output = torch.cat((context, output), dim=2)

        # 最终输出 batch,dec_seq,dec_hidden_size
        output = torch.tanh(self.liner_out(output.view(batch_size * dec_seq, -1))).view(batch_size, dec_seq, -1)

        return output, atten

class Decoder(nn.Module):
    """"""

    def __init__(self, config):
        super(Decoder, self).__init__()
        self.config = config
        self.embed = nn.Embedding(config.dec_vocab_size, config.embed_size)
        self.atten = Attention(config)
        # decoder不使用bidirectional
        self.rnn = nn.GRU(config.embed_size, config.dec_hidden_size, batch_first=True)
        self.out = nn.Linear(config.dec_hidden_size, config.dec_vocab_size)
        self.dropout = nn.Dropout(config.dropout)

    def create_mask(self, x_len, y_len):
        # 最长句子的长度
        max_x_len = x_len.max()
        max_y_len = y_len.max()
        # 句子batch
        batch_size = len(x_len)

        # 将超出自身序列长度的元素设为False
        x_mask = (torch.arange(max_x_len.item())[None, :].to(self.config.device) < x_len[:, None]).float()  # [batch,max_x_len]
        y_mask = (torch.arange(max_y_len.item())[None, :].to(self.config.device) < y_len[:, None]).float()  # [batch,max_y_len]

        # y_mask[:, :, None] size: [batch,max_y_len,1]
        # x_mask[:, None, :] size:  [batch,1,max_x_len]
        # 需要mask的地方设置为true
        mask = (1 - y_mask[:, :, None] * x_mask[:, None, :]) != 0
        # [batch_size, max_y_len, max_x_len]
        return mask

    def forward(self, ctx, ctx_lengths, y, y_lengths, hid):
        '''
        :param ctx:encoder层的输出 ： 【batch, enc_seq, 2*enc_hidden】
        :param ctx_lengths: encoder层输入句子的长度list
        :param y: decoder层的输入 【batch, dec_seq, dec_hidden】
        :param y_lengths: decoder输入的句子长度
        :param hid: encoder层输出的最后一个hidden state 【1, batch, dec_hidden】
        :return:
        '''
        y_embed = self.dropout(self.embed(y))
        # 这里没法保证译文也是排倒序
        y_packed = nn.utils.rnn.pack_padded_sequence(y_embed, y_lengths.cpu(), batch_first=True, enforce_sorted=False)
        # 将emcoder的hidden state作为decoder的第一个hidden state
        pack_output, hid = self.rnn(y_packed, hid)
        output_seq, _ = nn.utils.rnn.pad_packed_sequence(pack_output, batch_first=True, total_length=max(y_lengths))

        # 做attention之前需要创建mask
        mask = self.create_mask(ctx_lengths, y_lengths)
        # annention处理
        output, atten = self.atten(output_seq, ctx, mask)
        # 将输出转为vocab_size的softmax概率分布并取对数
        output = F.log_softmax(self.out(output), dim=-1)

        return output, atten, hid

class Seq2Seq(nn.Module):
    '''
        模型架构
    '''

    def __init__(self, encoder, decoder):
        super(Seq2Seq, self).__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, x, x_lengths, y, y_lengths):
        context, hid = self.encoder(x, x_lengths)

        output, atten, hid = self.decoder(
            context, x_lengths,
            y, y_lengths,
            hid
        )
        # output: 【batch,output_len,vocab_size】
        # atten   【batch,output_len,input_len】
        return output, atten

    def beam_search(self, x, x_lengths, y, EOS_id, topk=5, max_length=100):
        # hid: [1,1,512]，第二位为batchsize
        encoder_out, hid = self.encoder(x, x_lengths)
        # batch_size = x.shape[0]
        # preds = []
        # attens = []
        # for i in range(max_length):
        #
        #     output,atten,hid = self.decoder(
        #         encoder_out,x_lengths,
        #         y,torch.ones(batch_size).long().to(y.device),
        #         hid
        #     )
        #     # 取出预测概率最大index
        #     y = output.argmax(2).view(batch_size,1)
        #     preds.append(y)
        #     attens.append(atten)

        BOS_id = y[0][0].item()  # 2
        hypotheses = [[BOS_id]]  # [[2]]
        hyp_scores = torch.zeros(len(hypotheses), dtype=torch.float, device=y.device)
        completed_hypotheses = []
        t = 0
        while len(completed_hypotheses) < topk and t < max_length:
            t += 1
            hyp_num = len(hypotheses)  # 初始值为1
            # 扩展成batch
            # [1, x_lengths, 1024]
            exp_src_encodings = encoder_out.expand(hyp_num, encoder_out.shape[1], encoder_out.shape[2])
            exp_x_lengths = x_lengths.expand(hyp_num)  # [x_lengths]
            exp_hid = hid.expand(hid.shape[0], hyp_num, hid.shape[2])  # [1,1,512]
            output_t, atten_t, exp_hid = self.decoder(
                exp_src_encodings, exp_x_lengths,
                torch.tensor(hypotheses).long().to(y.device), torch.ones(hyp_num).long().to(y.device) * t,
                exp_hid
            )
            live_hyp_num = topk - len(completed_hypotheses)

            # 这里把num * vocab 展开来方便取topk
            contiuating_hyp_scores = (
                    hyp_scores.unsqueeze(1).expand(hyp_num, output_t.shape[-1]) + output_t[:, -1, :].squeeze(
                1)).view(-1)
            top_cand_hyp_scores, top_cand_hyp_pos = torch.topk(contiuating_hyp_scores, k=live_hyp_num)
            # 标记当前概率最大的k个，其是跟在哪个单词的后面
            prev_hyp_ids = top_cand_hyp_pos // (output_t.shape[-1])
            hyp_word_ids = top_cand_hyp_pos % (output_t.shape[-1])

            new_hypotheses = []
            live_hyp_ids = []
            new_hyp_scores = []

            for prev_hyp_id, hyp_word_id, cand_new_hyp_score in zip(prev_hyp_ids, hyp_word_ids, top_cand_hyp_scores):
                prev_hyp_id = prev_hyp_id.item()
                hyp_word_id = hyp_word_id.item()
                cand_new_hyp_score = cand_new_hyp_score.item()

                # 将当前最大概率的k个，拼接在正确的prev单词后面
                new_hyp_sent = hypotheses[prev_hyp_id] + [hyp_word_id]
                if hyp_word_id == EOS_id:
                    # 搜寻终止
                    completed_hypotheses.append(Hypothesis(value=new_hyp_sent[1:-1],
                                                           score=cand_new_hyp_score))
                else:
                    new_hypotheses.append(new_hyp_sent)
                    live_hyp_ids.append(prev_hyp_id)
                    new_hyp_scores.append(cand_new_hyp_score)

            if len(completed_hypotheses) == topk:
                break

            hypotheses = new_hypotheses
            # print(hypotheses)
            hyp_scores = torch.tensor(new_hyp_scores, dtype=torch.float, device=y.device)

        # 若搜寻了max_len后还没有一个到达EOS则取第一个
        if len(completed_hypotheses) == 0:
            completed_hypotheses.append(Hypothesis(value=hypotheses[0][1:],
                                                   score=hyp_scores[0].item()))
        completed_hypotheses.sort(key=lambda hyp: hyp.score, reverse=True)

        return completed_hypotheses

class LanguageModelCriterion(nn.Module):
    def __init__(self):
        '''损失函数'''
        super(LanguageModelCriterion, self).__init__()

    def forward(self, inuptY, target, mask):
        # inputY: batch, seq_len, vocab_size
        # target/mask: batch, seq_len
        inuptY = inuptY.contiguous().view(-1, inuptY.shape[2])
        target = target.contiguous().view(-1, 1)
        mask = mask.contiguous().view(-1, 1)
        # 模型seq2seq的输出已经经过log-softmax了，只需将target对应index值收集后在mask
        output = - inuptY.gather(1, target) * mask
        return torch.sum(output) / torch.sum(mask)


from django.shortcuts import render
from django.http import HttpResponse
from django.views import View
from django.views.generic import TemplateView, CreateView

from .models import SiteStatus
from .forms import  SiteStatusForm

# Create your views here.

class HomeView(TemplateView):
    template_name = "home.html"

    def get_context_data(self, *args, **kwargs):
        context = super(HomeView, self).get_context_data(*args, *kwargs)
        mylist = {"0", "1", "2", "3", "4", 5, 6, 7, 8}
        context = {"var": "string of words", "yy": mylist}
        return context

class AboutView(TemplateView):
    template_name = "about.html"

class ContactView(TemplateView):
    template_name = "contact.html"

class SiteStatusView(CreateView):
    form_class = SiteStatusForm
    template_name = "sitestatus_form.html"


"""Package configuration for Dwolla."""
import os

from setuptools import find_packages, setup

def read(filename):
    """Read file contents."""
    path = os.path.realpath(os.path.join(os.path.dirname(__file__), filename))
    with open(path, 'rb') as f:
        return f.read().decode('utf-8')

dependencies = read('requirements.txt').split()
setup(
    name='dwolla',
    version='1.0.1',
    description='A Python package for a technical interview problem',
    url='https://github.com/nickolashkraus/dwolla',
    author='Nickolas Kraus <0x@nickolaskraus.io>',
    install_requires=dependencies,
    packages=find_packages(exclude=['*.test', '*.test.*', 'test.*', 'test']),
    entry_points={
        'console_scripts': ['dwolla=dwolla.cli:cli'],
    },
    include_package_data=True,
    zip_safe=False)


import numpy as np
import tensorflow as tf
import tensorflow.contrib.rnn as rnn
import matplotlib.pyplot as plt

TIME_STEPS=10
BATCH_SIZE=128
HIDDEN_UNITS1=30
HIDDEN_UNITS=1
LEARNING_RATE=0.001
EPOCH=50

TRAIN_EXAMPLES=11000
TEST_EXAMPLES=1100

#------------------------------------Generate Data-----------------------------------------------#
#generate data
def generate(seq):
    X=[]
    y=[]
    for i in range(len(seq)-TIME_STEPS):
        X.append([seq[i:i+TIME_STEPS]])
        y.append([seq[i+TIME_STEPS]])
    return np.array(X,dtype=np.float32),np.array(y,dtype=np.float32)

#s=[i for i in range(30)]
#X,y=generate(s)
#print(X)
#print(y)

seq_train=np.sin(np.linspace(start=0,stop=100,num=TRAIN_EXAMPLES,dtype=np.float32))
seq_test=np.sin(np.linspace(start=100,stop=110,num=TEST_EXAMPLES,dtype=np.float32))

#plt.plot(np.linspace(start=0,stop=100,num=10000,dtype=np.float32),seq_train)

#plt.plot(np.linspace(start=100,stop=110,num=1000,dtype=np.float32),seq_test)
#plt.show()

X_train,y_train=generate(seq_train)
#print(X_train.shape,y_train.shape)
X_test,y_test=generate(seq_test)

#reshape to (batch,time_steps,input_size)
X_train=np.reshape(X_train,newshape=(-1,TIME_STEPS,1))
X_test=np.reshape(X_test,newshape=(-1,TIME_STEPS,1))

#draw y_test
plt.plot(range(1000),y_test[:1000,0],"r*")
#print(X_train.shape)
#print(X_test.shape)

#-----------------------------------------------------------------------------------------------------#

#--------------------------------------Define Graph---------------------------------------------------#
graph=tf.Graph()
with graph.as_default():

    #------------------------------------construct LSTM------------------------------------------#
    #place hoder
    X_p=tf.placeholder(dtype=tf.float32,shape=(None,TIME_STEPS,1),name="input_placeholder")
    y_p=tf.placeholder(dtype=tf.float32,shape=(None,1),name="pred_placeholder")

    #lstm instance
    lstm_cell1=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS1)
    lstm_cell=rnn.BasicLSTMCell(num_units=HIDDEN_UNITS)

    multi_lstm=rnn.MultiRNNCell(cells=[lstm_cell1,lstm_cell])

    #自己初始化state
    #第一层state
    lstm_layer1_c=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1))
    lstm_layer1_h=tf.zeros(shape=(BATCH_SIZE,HIDDEN_UNITS1))
    layer1_state=rnn.LSTMStateTuple(c=lstm_layer1_c,h=lstm_layer1_h)

    #第二层state
    lstm_layer2_c = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS))
    lstm_layer2_h = tf.zeros(shape=(BATCH_SIZE, HIDDEN_UNITS))
    layer2_state = rnn.LSTMStateTuple(c=lstm_layer2_c, h=lstm_layer2_h)

    init_state=(layer1_state,layer2_state)
    print(init_state)

    #自己展开RNN计算
    outputs = list()                                    #用来接收存储每步的结果
    state = init_state
    with tf.variable_scope('RNN'):
        for timestep in range(TIME_STEPS):
            if timestep > 0:
                tf.get_variable_scope().reuse_variables()
            # 这里的state保存了每一层 LSTM 的状态
            (cell_output, state) = multi_lstm(X_p[:, timestep, :], state)
            outputs.append(cell_output)
    h = outputs[-1]

    #---------------------------------define loss and optimizer----------------------------------#
    mse=tf.losses.mean_squared_error(labels=y_p,predictions=h)
    #print(loss.shape)
    optimizer=tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss=mse)

    init=tf.global_variables_initializer()

#-------------------------------------------Define Session---------------------------------------#
with tf.Session(graph=graph) as sess:
    sess.run(init)
    for epoch in range(1,EPOCH+1):
        results = np.zeros(shape=(TEST_EXAMPLES, 1))
        train_losses=[]
        test_losses=[]
        print("epoch:",epoch)
        for j in range(TRAIN_EXAMPLES//BATCH_SIZE):
            _,train_loss=sess.run(
                    fetches=(optimizer,mse),
                    feed_dict={
                            X_p:X_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE],
                            y_p:y_train[j*BATCH_SIZE:(j+1)*BATCH_SIZE]
                        }
            )
            train_losses.append(train_loss)
        print("average training loss:", sum(train_losses) / len(train_losses))

        for j in range(TEST_EXAMPLES//BATCH_SIZE):
            result,test_loss=sess.run(
                    fetches=(h,mse),
                    feed_dict={
                            X_p:X_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE],
                            y_p:y_test[j*BATCH_SIZE:(j+1)*BATCH_SIZE]
                        }
            )
            results[j*BATCH_SIZE:(j+1)*BATCH_SIZE]=result
            test_losses.append(test_loss)
        print("average test loss:", sum(test_losses) / len(test_losses))
        plt.plot(range(1000),results[:1000,0])
    plt.show()


import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import torch.nn.functional as F
import spacy
from torch.utils.data import Dataset, DataLoader
import re
import subprocess as call
import json
import pandas as pd
from collections import Counter
from tqdm.auto import tqdm, trange
tqdm.pandas(desc='Progress')

# ******************************************* #
# * Make batches through PyTorch Dataloader * #
# ******************************************* #

def collate_fn(data):
    """This function will be used to pad the tweets to max length
       in the batch and transpose the batch from
       batch_size x max_seq_len to max_seq_len x batch_size.
       It will return padded vectors, labels and lengths of each tweets (before padding)
       It will be used in the Dataloader
    """
    data.sort(key=lambda x: len(x[0]), reverse=True)
    lens = [len(sent) for sent, label in data]
    labels = []
    padded_sents = torch.zeros(len(data), max(lens)).long()
    for i, (sent, label) in enumerate(data):
        padded_sents[i,:lens[i]] = torch.LongTensor(sent)
        labels.append(label)

    padded_sents = padded_sents.transpose(0,1)
    return padded_sents, torch.tensor(labels).long(), lens

# ************************ #
# * Custom class for data  #
# ************************ #
PAD = 0
UNK = 1

print('Pytorch version:', torch.__version__)
print('Spacy version:', spacy.__version__)

class SentimentDataset(Dataset):
    """Define the pytorch Dataset to process the tweets
       This class can be used for both training and validation dataset
       Run it for training data and pass the word2idx and idx2word when running
       for validation data
    """

    def __init__(self, df=None, word2idx=None, idx2word=None, max_vocab_size=50000):
        print('Processing Data')
        self.df = df
        if(self.df is not None):
            print('Removing white space...')
            self.df.SentimentText = self.df.SentimentText.progress_apply(lambda x: x.strip())
        try:
            self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])
        except IOError:
            call.check_output("python -m spacy download en", shell=True)
            self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])
        finally:
            pass
        if word2idx is None and df is not None:
            print('Building Counter...')
            word_counter = self.build_counter()
            print('Building Vocab...')
            self.word2idx, self.idx2word = self.build_vocab(word_counter, max_vocab_size)
        else:
            self.word2idx, self.idx2word = word2idx, idx2word
        print('*'*100)
        print('Dataset info:')
        if df is not None:
            print(f'Number of Tweets: {self.df.shape[0]}')
            print(f'Vocab Size: {len(self.word2idx)}')
        print('*'*100)

    def __len__(self):
        return self.df.shape[0]

    def __getitem__(self, idx):
        sent = self.df.SentimentText[idx]
        tokens = [w.text.lower() for w in self.nlp(self.tweet_clean(sent))]
        vec = self.vectorize(tokens, self.word2idx)
        return vec, self.df.Sentiment[idx]

    def tweet_clean(self, text):
        """Very basic text cleaning. This function can be built upon for
           better preprocessing
        """
        text = re.sub(r'[\s]+', ' ', text) # replace multiple white spaces with single space
#         text = re.sub(r'@[A-Za-z0-9]+', ' ', text) # remove @ mentions
        text = re.sub(r'https?:/\/\S+', ' ', text) # remove links
        text = re.sub(r'[^A-Za-z0-9]+', ' ', text) # remove non alphanumeric character
        return text.strip()

    def build_counter(self):
        """Tokenize the tweets using spacy and build vocabulary
        """
        words_counter = Counter()
        for sent in tqdm(self.df.SentimentText.values):
            words_counter.update(w.text.lower() for w in self.nlp(self.tweet_clean(sent)))
        return words_counter

    def build_vocab(self, words_counter, max_vocab_size):
        """Add pad and unk tokens and build word2idx and idx2word dictionaries
        """
        word2idx = {'<PAD>': PAD, '<UNK>': UNK}
        word2idx.update({word:i+2 for i, (word, count) in tqdm(enumerate(words_counter.most_common(max_vocab_size)))})
        idx2word = {idx: word for word, idx in tqdm(word2idx.items())}
        return word2idx, idx2word

    def vectorize(self, tokens, word2idx):
        """Convert tweet to vector
        """
        vec = [word2idx.get(token, UNK) for token in tokens]
        return vec

    def save_vocabulary(self, PATH='./models/sd', filenames=('w2i','i2w')):
        """ Save the word2idx & idx2word dicts
        """
        f = open(PATH+filenames[0]+".json","w")
        f.write(json.dumps(self.word2idx))
        f.close()
        print("wrote word2idx to {}".format(PATH+filenames[0]+".json"))
        f = open(PATH+filenames[1]+".json","w")
        f.write(json.dumps(self.idx2word))
        f.close()
        print("wrote word2idx to {}".format(PATH+filenames[1]+".json"))
        return

    def load_vocabulary(self, PATH='./models/sd', filenames=('w2i','i2w')):
        f = open(PATH+filenames[0]+".json","r")
        self.word2idx = json.load(f)
        f.close()
        f = open(PATH+filenames[1]+".json","r")
        self.idx2word = json.load(f)
        f.close()
        print('!'*100)
        print(f'Updated Vocab Sizes (word2idx, idx2word): {len(self.word2idx)}, {len(self.idx2word)}')
        print('!'*100)
        return

    def load_json_as_df(self, json_obj):
        assert 'SentimentText' in json.loads(json_obj), "key `SentimentText` should appear in hash [dict] argument"
        assert 'Sentiment' in json.loads(json_obj), "key `Sentiment` should appear in hash [dict] argument"
        self.df = pd.read_json(json_obj)
        self.df.SentimentText.progress_apply(lambda x: x.strip())
        print('!'*100)
        print('Notice Updated Dataset info:')
        print(f'Number of %Tweets% / data entries: {self.df.shape[0]}')
        print('!'*100)
        return
    @staticmethod
    def get_pdl(ds, collator=collate_fn):
        """Get the Pytorch DataLoader for a SentimentDataset
            used for predictions
        """
        return DataLoader(ds, collate_fn=collator)

# *********************************** #
# * Max Pooling and Average Pooling * #
#   with Concat pooling GRU model     #
# *********************************** #

resolve_model_output = lambda a : 0 if a[0] == 0 and a[1] != 0 else (1 if a[0] != 0 and a[1] == 0 else None)
class ConcatPoolingGRUAdaptive(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size = vocab_size
        self.embedding_dim = embedding_dim
        self.n_hidden = n_hidden
        self.n_out = n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.emb_drop = nn.Dropout(0.3)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden, dropout=0.3)
        self.out = nn.Linear(self.n_hidden*3, self.n_out)
        self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

    def forward(self, seq, lengths):
        self.h = self.init_hidden(seq.size(1))
        embs = self.emb_drop(self.emb(seq))
        embs = pack_padded_sequence(embs, lengths)
        gru_out, self.h = self.gru(embs, self.h)
        gru_out, lengths = pad_packed_sequence(gru_out)

        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)
        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)

        outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))
        return F.log_softmax(outp, dim=-1) # it will return log of softmax

    def init_hidden(self, batch_size):
        return torch.zeros((1, batch_size,self.n_hidden), requires_grad=True).to(self.device)

    def import_dict(self, PATH='./models/text_gru_concat_sentiment_10.pth'):
        """# Load the saved state into the model
        """
        self.load_state_dict(torch.load(PATH))

    @staticmethod
    def prediction(model, dl):
        """ static method used on `ConcatPoolingGRUAdaptive` class to
            predict sentiment
        """
        sentiment = []
        model.eval() # run if you only want to use it for inference
        with torch.no_grad():
            for x, y, lens in dl:
                x = x.to(model.device) # , y.to(model.device)
                y_pred = model(x, lens)
                pred = torch.round(y_pred.squeeze()).tolist()
                print(pred)
                sentiment.append(resolve_model_output(pred))
            return sentiment


import datetime
import sys
import unittest

try:
    import simplejson as json
except ImportError:
    import json

from libcloud.utils.py3 import httplib

from libcloud.test import MockHttp, LibcloudTestCase
from libcloud.common.google import (GoogleAuthError,
                                    GoogleBaseAuthConnection,
                                    GoogleInstalledAppAuthConnection,
                                    GoogleServiceAcctAuthConnection,
                                    GoogleGCEServiceAcctAuthConnection,
                                    GoogleBaseConnection)
from libcloud.test.secrets import GCE_PARAMS

# Skip some tests if PyCrypto is unavailable
try:
    from Crypto.Hash import SHA256
except ImportError:
    SHA256 = None

class MockJsonResponse(object):
    def __init__(self, body):
        self.object = body

class GoogleBaseAuthConnectionTest(LibcloudTestCase):
    """
    Tests for GoogleBaseAuthConnection
    """
    GoogleBaseAuthConnection._now = lambda x: datetime.datetime(2013, 6, 26,
                                                                19, 0, 0)

    def setUp(self):
        GoogleBaseAuthConnection.conn_classes = (GoogleAuthMockHttp,
                                                 GoogleAuthMockHttp)
        self.mock_scopes = ['foo', 'bar']
        kwargs = {'scopes': self.mock_scopes}
        self.conn = GoogleInstalledAppAuthConnection(*GCE_PARAMS,
                                                     **kwargs)

    def test_scopes(self):
        self.assertEqual(self.conn.scopes, 'foo bar')

    def test_add_default_headers(self):
        old_headers = {}
        expected_headers = {
            'Content-Type': 'application/x-www-form-urlencoded',
            'Host': 'accounts.google.com'}
        new_headers = self.conn.add_default_headers(old_headers)
        self.assertEqual(new_headers, expected_headers)

    def test_token_request(self):
        request_body = {'code': 'asdf', 'client_id': self.conn.user_id,
                        'client_secret': self.conn.key,
                        'redirect_uri': self.conn.redirect_uri,
                        'grant_type': 'authorization_code'}
        new_token = self.conn._token_request(request_body)
        self.assertEqual(new_token['access_token'], 'installedapp')
        self.assertEqual(new_token['expire_time'], '2013-06-26T20:00:00Z')

class GoogleInstalledAppAuthConnectionTest(LibcloudTestCase):
    """
    Tests for GoogleInstalledAppAuthConnection
    """
    GoogleInstalledAppAuthConnection.get_code = lambda x: '1234'

    def setUp(self):
        GoogleInstalledAppAuthConnection.conn_classes = (GoogleAuthMockHttp,
                                                         GoogleAuthMockHttp)
        self.mock_scopes = ['https://www.googleapis.com/auth/foo']
        kwargs = {'scopes': self.mock_scopes}
        self.conn = GoogleInstalledAppAuthConnection(*GCE_PARAMS,
                                                     **kwargs)

    def test_refresh_token(self):
        # This token info doesn't have a refresh token, so a new token will be
        # requested
        token_info1 = {'access_token': 'tokentoken', 'token_type': 'Bearer',
                       'expires_in': 3600}
        new_token1 = self.conn.refresh_token(token_info1)
        self.assertEqual(new_token1['access_token'], 'installedapp')

        # This token info has a refresh token, so it will be able to be
        # refreshed.
        token_info2 = {'access_token': 'tokentoken', 'token_type': 'Bearer',
                       'expires_in': 3600, 'refresh_token': 'refreshrefresh'}
        new_token2 = self.conn.refresh_token(token_info2)
        self.assertEqual(new_token2['access_token'], 'refreshrefresh')

        # Both sets should have refresh info
        self.assertTrue('refresh_token' in new_token1)
        self.assertTrue('refresh_token' in new_token2)

class GoogleBaseConnectionTest(LibcloudTestCase):
    """
    Tests for GoogleBaseConnection
    """
    GoogleBaseConnection._get_token_info_from_file = lambda x: None
    GoogleBaseConnection._write_token_info_to_file = lambda x: None
    GoogleInstalledAppAuthConnection.get_code = lambda x: '1234'
    GoogleServiceAcctAuthConnection.get_new_token = \
        lambda x: x._token_request({})
    GoogleGCEServiceAcctAuthConnection.get_new_token = \
        lambda x: x._token_request({})
    GoogleBaseConnection._now = lambda x: datetime.datetime(2013, 6, 26,
                                                            19, 0, 0)

    def setUp(self):
        GoogleBaseAuthConnection.conn_classes = (GoogleAuthMockHttp,
                                                 GoogleAuthMockHttp)
        self.mock_scopes = ['https://www.googleapis.com/auth/foo']
        kwargs = {'scopes': self.mock_scopes, 'auth_type': 'IA'}
        self.conn = GoogleBaseConnection(*GCE_PARAMS, **kwargs)

    def test_auth_type(self):
        self.assertRaises(GoogleAuthError, GoogleBaseConnection, *GCE_PARAMS,
                          **{'auth_type': 'XX'})

        kwargs = {'scopes': self.mock_scopes}

        if SHA256:
            kwargs['auth_type'] = 'SA'
            conn1 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
            self.assertTrue(isinstance(conn1.auth_conn,
                                       GoogleServiceAcctAuthConnection))

        kwargs['auth_type'] = 'IA'
        conn2 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
        self.assertTrue(isinstance(conn2.auth_conn,
                                   GoogleInstalledAppAuthConnection))

        kwargs['auth_type'] = 'GCE'
        conn3 = GoogleBaseConnection(*GCE_PARAMS, **kwargs)
        self.assertTrue(isinstance(conn3.auth_conn,
                                   GoogleGCEServiceAcctAuthConnection))

    def test_add_default_headers(self):
        old_headers = {}
        new_expected_headers = {'Content-Type': 'application/json',
                                'Host': 'www.googleapis.com'}
        new_headers = self.conn.add_default_headers(old_headers)
        self.assertEqual(new_headers, new_expected_headers)

    def test_pre_connect_hook(self):
        old_params = {}
        old_headers = {}
        new_expected_params = {}
        new_expected_headers = {'Authorization': 'Bearer installedapp'}
        new_params, new_headers = self.conn.pre_connect_hook(old_params,
                                                             old_headers)
        self.assertEqual(new_params, new_expected_params)
        self.assertEqual(new_headers, new_expected_headers)

    def test_encode_data(self):
        data = {'key': 'value'}
        json_data = '{"key": "value"}'
        encoded_data = self.conn.encode_data(data)
        self.assertEqual(encoded_data, json_data)

    def test_has_completed(self):
        body1 = {"endTime": "2013-06-26T10:05:07.630-07:00",
                 "id": "3681664092089171723",
                 "kind": "compute#operation",
                 "status": "DONE",
                 "targetId": "16211908079305042870"}
        body2 = {"endTime": "2013-06-26T10:05:07.630-07:00",
                 "id": "3681664092089171723",
                 "kind": "compute#operation",
                 "status": "RUNNING",
                 "targetId": "16211908079305042870"}
        response1 = MockJsonResponse(body1)
        response2 = MockJsonResponse(body2)
        self.assertTrue(self.conn.has_completed(response1))
        self.assertFalse(self.conn.has_completed(response2))

    def test_get_poll_request_kwargs(self):
        body = {"endTime": "2013-06-26T10:05:07.630-07:00",
                "id": "3681664092089171723",
                "kind": "compute#operation",
                "selfLink": "https://www.googleapis.com/operations-test"}
        response = MockJsonResponse(body)
        expected_kwargs = {'action':
                           'https://www.googleapis.com/operations-test'}
        kwargs = self.conn.get_poll_request_kwargs(response, None, {})
        self.assertEqual(kwargs, expected_kwargs)

    def test_morph_action_hook(self):
        self.conn.request_path = '/compute/apiver/project/project-name'
        action1 = ('https://www.googleapis.com/compute/apiver/project'
                   '/project-name/instances')
        action2 = '/instances'
        expected_request = '/compute/apiver/project/project-name/instances'
        request1 = self.conn.morph_action_hook(action1)
        request2 = self.conn.morph_action_hook(action2)
        self.assertEqual(request1, expected_request)
        self.assertEqual(request2, expected_request)

class GoogleAuthMockHttp(MockHttp):
    """
    Mock HTTP Class for Google Auth Connections.
    """
    json_hdr = {'content-type': 'application/json; charset=UTF-8'}

    def _o_oauth2_token(self, method, url, body, headers):
        token_info = {'access_token': 'tokentoken',
                      'token_type': 'Bearer',
                      'expires_in': 3600}
        refresh_token = {'access_token': 'refreshrefresh',
                         'token_type': 'Bearer',
                         'expires_in': 3600}
        ia_token = {'access_token': 'installedapp',
                    'token_type': 'Bearer',
                    'expires_in': 3600,
                    'refresh_token': 'refreshrefresh'}
        if 'code' in body:
            body = json.dumps(ia_token)
        elif 'refresh_token' in body:
            body = json.dumps(refresh_token)
        else:
            body = json.dumps(token_info)
        return (httplib.OK, body, self.json_hdr, httplib.responses[httplib.OK])

if __name__ == '__main__':
    sys.exit(unittest.main())


from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _

def numbers_only_validator(value):
    if not all([x.isdigit() for x in str(value)]):
        raise ValidationError(_('Please, enter digits only.'), code='invalid')

def phone_number_validator(value):
    allowed_chars = ['+', ' ']
    for c in str(value):
        if not c.isdigit() and c not in allowed_chars:
            raise ValidationError(_('The allowed characters are: +, space and digits.'), code='invalid')


import os
import pdfplumber
import pytesseract
import logging
from PIL import Image
import traceback

# Ensure pytesseract uses the correct installation path to Tesseract
pytesseract.pytesseract.tesseract_cmd = r'/usr/bin/tesseract'

# Initialize logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

def prompt_user_for_directories():
    """
    Prompt the user for source and target directory paths.
    """
    source_dir = input("Please enter the source directory path: ")
    target_dir = input("Please enter the target directory path: ")
    return source_dir, target_dir

def extract_text_from_page(page):
    """
    Extract text from a page.
    """
    text = page.extract_text()
    if text is None:
        text = ""
    return text

def extract_text_from_image(image):
    """
    Use OCR to extract text from a PIL image.
    """
    return pytesseract.image_to_string(image)

def extract_text_from_pdf(pdf_path, target_dir):
    """
    Extract text from a PDF file. The content from all pages, including text and OCR results from images,
    is combined and written to a single text file in the target directory.
    """
    with pdfplumber.open(pdf_path) as pdf:
        # Get the basename for the output file
        base_filename = os.path.splitext(os.path.basename(pdf_path))[0]
        logging.info(f"Extracting text from PDF: {base_filename}.pdf")

        all_text_content = []  # List to store all text content

        for page_num, page in enumerate(pdf.pages):
            text_content = extract_text_from_page(page)
            text_content += " \n\n"  # Add space between page text and image text

            # Extract text from images on the page
            image = page.to_image().original.convert('RGB')
            ocr_text = extract_text_from_image(image)
            if ocr_text:  # Append only if OCR text is extracted
                text_content += "Image Text:\n" + ocr_text + "\n\n"

            all_text_content.append(text_content)

        # Combine all text into one string
        combined_text = "\n\n".join(all_text_content)

        # Write the combined text to a single file named after the PDF
        output_filename = os.path.join(target_dir, f"{base_filename}.txt")
        with open(output_filename, 'w', encoding='utf-8') as output_file:
            output_file.write(combined_text)
            logging.info(f"Saved extracted text to {output_filename}")

def main():
    source_dir, target_dir = prompt_user_for_directories()

    # Validate source and target directories
    if not os.path.exists(source_dir) or not os.path.isdir(source_dir):
        logging.error(f"Source directory does not exist or is not a directory: {source_dir}")
        return

    if not os.path.exists(target_dir):
        try:
            os.makedirs(target_dir)
            logging.info(f"Created target directory: {target_dir}")
        except Exception as e:
            logging.error(f"Unable to create target directory: {e}")
            return

    # Attempt to read and list all files in the source directory for debugging
    try:
        file_list = os.listdir(source_dir)
        logging.info(f"All files found in source directory: {file_list}")
    except Exception as e:
        logging.error(f"Unable to read the source directory: {e}")
        return

    # Process PDF files in the source directory
    pdf_files = [f for f in file_list if f.lower().endswith('.pdf')]
    logging.info(f"Number of PDF files found: {len(pdf_files)}")
    if len(pdf_files) == 0:
        logging.info("No PDF files found in source directory.")
        return

    for filename in pdf_files:
        pdf_path = os.path.join(source_dir, filename)
        try:
            extract_text_from_pdf(pdf_path, target_dir)
            logging.info(f"Successfully extracted text from {filename}")
        except Exception as e:
            logging.error(f"Failed to extract text from {filename}: {e}")
            traceback.print_exc()  # Print the full traceback

    logging.info("Extraction completed. Check the target directory for the output files.")

if __name__ == "__main__":
    main()


#!/usr/bin/python
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils import common_koji

ANSIBLE_METADATA = {
    'metadata_version': '1.0',
    'status': ['preview'],
    'supported_by': 'community'
}

DOCUMENTATION = '''
---
module: koji_external_repo

short_description: Create and manage Koji external repos
description:
   - Create and manage Koji external repos
options:
   name:
     description:
       - The name of the Koji external repo to create and manage.
     required: true
   url:
     description:
       - The URL to the Koji external repo.
       - Note, this uses "$arch", not the common "$basearch" you may find in a
         typical Yum repository file.
       - For idempotency, please ensure your url always ends with a "/"
         character. If you leave it out, Koji Hub will automatically add a "/"
         slash when storing this value in the database, and every subsequent
         Ansible run will appear to be "changing" the external repo's URL.
     required: true
requirements:
  - "python >= 2.7"
  - "koji"
'''

EXAMPLES = '''
- name: create a koji tag with an external repo.
  hosts: localhost
  tasks:
    - name: Create an external repo for CentOS "CR"
      koji_external_repo:
        name: centos7-cr
        url: http://mirror.centos.org/centos/7/cr/$arch/
        state: present

    - name: Create a koji tag that uses the CentOS CR repo
      koji_tag:
        name: storage7-ceph-nautilus-el7-build
        state: present
        external_repos:
        - repo: centos7-cr
          priority: 5
'''

RETURN = ''' # '''

def ensure_external_repo(session, name, check_mode, url):
    """
    Ensure that this external repo exists in Koji.

    :param session: Koji client session
    :param name: Koji external repo name
    :param check_mode: don't make any changes
    :param url: URL to this external repo
    """
    repoinfo = session.getExternalRepo(name)
    result = {'changed': False, 'stdout_lines': []}
    if not repoinfo:
        if check_mode:
            result['stdout_lines'].append('would create repo %s' % name)
            result['changed'] = True
            return result
        common_koji.ensure_logged_in(session)
        repoinfo = session.createExternalRepo(name, url)
        result['stdout_lines'].append('created repo id %d' % repoinfo['id'])
        result['changed'] = True
        return result
    if repoinfo['url'] != url:
        result['stdout_lines'].append('set url to %s' % url)
        result['changed'] = True
        if not check_mode:
            common_koji.ensure_logged_in(session)
            session.editExternalRepo(info=repoinfo['id'], url=url)
    return result

def delete_external_repo(session, name, check_mode):
    """ Ensure that this external_repo is deleted from Koji. """
    repoinfo = session.getExternalRepo(name)
    result = dict(
        stdout='',
        changed=False,
    )
    if repoinfo:
        result['stdout'] = 'deleted external repo %s' % name
        result['changed'] = True
        if not check_mode:
            common_koji.ensure_logged_in(session)
            session.deleteExternalRepo(name)
    return result

def run_module():
    module_args = dict(
        koji=dict(),
        name=dict(required=True),
        state=dict(choices=['present', 'absent'], default='present'),
        url=dict(),
    )
    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True
    )

    if not common_koji.HAS_KOJI:
        module.fail_json(msg='koji is required for this module')

    check_mode = module.check_mode
    params = module.params
    profile = params['koji']
    name = params['name']
    state = params['state']
    url = params['url']

    session = common_koji.get_session(profile)

    if state == 'present':
        if not url:
            module.fail_json(msg='you must set a url for this external_repo')
        result = ensure_external_repo(session, name, check_mode, url)
    elif state == 'absent':
        result = delete_external_repo(session, name, check_mode)

    module.exit_json(**result)

def main():
    run_module()

if __name__ == '__main__':
    main()