import gensim
import numpy as np
import pandas as pd
from sklearn.linear_model import RidgeClassifier
from sklearn.base import ClassifierMixin, BaseEstimator
from .._types import validate_text_input
from ..base import AnySklearnEstimator

def _gensim_preprocess(X: pd.Series):
    """Lowers and splits texts"""
    X_pre = X.apply(gensim.utils.simple_preprocess).tolist()
    X_pre = [gensim.models.doc2vec.TaggedDocument(texts, [i]) for i, texts in enumerate(X_pre)]

    return X_pre

class Doc2VecClassifier(BaseEstimator, ClassifierMixin):
    """Text Classifier using Doc2Vec feateures given any Sklearn compatible estimator.

    Parameters
    ----------
    estimator : AnySklearnEstimator, default=RidgeClassifier
        Estimator to fit on TF-IDF vector. Prefer estimators with sparse vector support.

    vector_size : int, default=100
        Dimensionality of the feature vectors.

    min_count : int, default=1
        Ignores all words with total frequency lower than this.

    window : int, default=5
        The maximum distance between the current and predicted word within a sentence.

    workers : int, default=1
        Use these many worker threads to train the model (=faster training with multicore machines).

    epochs : int, default=10
        Number of iterations (epochs) over the corpus.
    """

    def __init__(self,
                 estimator: AnySklearnEstimator = RidgeClassifier(),
                 vector_size: int = 100,
                 min_count: int = 1,
                 window: int = 5,
                 epochs: int = 10,
                 workers: int = 1,
                 ):

        self.estimator = estimator
        self.vector_size = vector_size
        self.min_count = min_count
        self.window = window
        self.epochs = epochs
        self.workers = workers

        self.vectorizer = gensim.models.doc2vec.Doc2Vec(
            vector_size=self.vector_size,
            min_count=self.min_count,
            window=self.window,
            epochs=self.epochs,
            workers=self.workers,
        )
        self.is_prefit = False

    def prefit(
        self,
        X: pd.Series,
        y=None
    ):
        """Prefit Doc2Vec vectorizer with unlabeled text data.

        Parameters
        ----------
        X : pd.Series
            Pandas series containing texts.

        y : None
             This parameter is not needed.

        Returns
        -------
        self : object
            Fitted estimator.
        """

        validate_text_input(X)

        if self.is_prefit:
            print("Overwriting previous prefit.")

        train_corpus = _gensim_preprocess(X)

        self.vectorizer.build_vocab(train_corpus)
        self.vectorizer.train(train_corpus, total_examples=self.vectorizer.corpus_count, epochs=self.vectorizer.epochs)

        self.is_prefit = True

    def fit(
        self,
        X: pd.Series,
        y: pd.Series
    ):
        """Build Doc2Vec vectors from training set and fit the provided estimator.

        Parameters
        ----------
        X : pd.Series
            Pandas text series containing texts.

        y : pd.Series
            Pandas text series containing targets.

        Returns
        -------
        self : object
            Fitted estimator.
        """
        validate_text_input(X)

        if self.is_prefit:
            X_pre = _gensim_preprocess(X)
            vectors = [self.vectorizer.infer_vector(doc.words) for doc in X_pre]
            X_vec = pd.DataFrame(np.stack(vectors))
        else:
            train_corpus = _gensim_preprocess(X)
            self.vectorizer.build_vocab(train_corpus)
            self.vectorizer.train(train_corpus, total_examples=self.vectorizer.corpus_count, epochs=self.vectorizer.epochs)

            vectors = [self.vectorizer.infer_vector(doc.words) for doc in train_corpus]
            X_vec = pd.DataFrame(np.stack(vectors))

        self.estimator.fit(X_vec, y)

        return self

    def predict(
        self,
        X: pd.Series
    ):
        """Predict class labels for samples in `X`.

        Parameters
        ----------
        X : pd.Series
            Pandas text series containing texts.

        Returns
        -------
        y_pred : ndarray of shape (n_samples,) or (n_samples, n_outputs)
            Vector or matrix containing the predictions.
        """
        validate_text_input(X)

        X_pre = _gensim_preprocess(X)
        vectors = [self.vectorizer.infer_vector(doc.words) for doc in X_pre]
        X_vec = pd.DataFrame(np.stack(vectors))

        return self.estimator.predict(X_vec)

    def predict_proba(
        self,
        X: pd.Series
    ):
        """Probability estimates.

        Parameters
        ----------
        X : pd.Series
            Pandas text series containing texts.

        Returns
        -------
        y_prob : ndarray of shape (n_samples, n_classes)
            The predicted probability of the sample for each class in the
            model, where classes are ordered as they are in `self.classes_`.
        """
        validate_text_input(X)

        if not hasattr(self.estimator, "predict_proba"):
            raise AttributeError(f"{self.estimator} does not have a predict_proba method.")

        X_pre = _gensim_preprocess(X)
        vectors = [self.vectorizer.infer_vector(doc.words) for doc in X_pre]
        X_vec = pd.DataFrame(np.stack(vectors))

        return self.estimator.predict_proba(X_vec)


"""
Remove overlapping words such as *read* and *reading*...
R IY D .
R IY D IH NG .
using a trie. Because of how tries are constructed, smaller
overlapping words will become part of larger ones, and ultimately
all words will be found at nodes with an out degree of zero.

===

# Subsets and supersets/affixes
In an orthographic sense, *run* is a subset of *run*ning.
Three kinds of subsets: prefixes, suffixes, and infixes.
We already have a prefix tree
We can make a suffix tree as well (this would be a much more literal copying of the rhyme trie)
The words we then want are the leaves of the tree: in degree of one and an out degree of zero.

~Infixes require a little more elbow grease; we can find them through a function that finds any affix~
Going for infixes at this stage isn't worth it, we should just handle them during the main stage.
"""

# import nltk
# nltk.download('cmudict')

from difflib import SequenceMatcher
from typing import Iterable, List
import networkx as nx
from nltk.corpus import cmudict

def leaves(G: nx.DiGraph) -> List[str]:
	return [tuple(node.split(' ')) for node in G.nodes if G.in_degree(node) == 1 and G.out_degree(node) == 0]

def deduplicate(pronunciations: Iterable[str]) -> List[str]:
	trie = nx.DiGraph()

	for pronunciation in pronunciations:
		stages = [''] + [' '.join(pronunciation[:index]) for index in range(1, len(pronunciation) + 1)]
		for i in range(len(stages) - 1):
			trie.add_edge(stages[i], stages[i + 1], phone=pronunciation[i])

	return leaves(trie)

def is_affix(subset, superset):
	matcher = SequenceMatcher(a=subset, b=superset)
	block = matcher.find_longest_match(0, len(subset), 0, len(superset))
	return block.a == 0 and block.size == len(subset)

pronunciations = [tuple(pronunciation) for _, pronunciation in cmudict.entries()]
# 133737
# print(len(pronunciations))

unique_pronunciations = set(pronunciations)
# 114966 (14% reduction)
# print(len(unique_pronunciations))

deduplicated_pronunciations_by_prefix = deduplicate(unique_pronunciations)
# 87627 (34% reduction)
# print(len(deduplicated_pronunciations_by_prefix))

deduplicated_pronunciations_by_suffix = {
	pronunciaition[::-1]
	for pronunciaition in
	(deduplicate(pronunciation[::-1] for pronunciation in deduplicated_pronunciations_by_prefix))
}
# 81187 (39% reduction)
# print(len(deduplicated_pronunciations_by_suffix))

deduplicated_pronunciations = deduplicated_pronunciations_by_suffix


import time
import logging
import re
import torch
from torch.utils.data import Dataset, DataLoader
from transformers import T5ForConditionalGeneration, T5Tokenizer, AdamW
import pytorch_lightning as pl

from transformers import AutoTokenizer, AutoModelWithLMHead

tokenizer = AutoTokenizer.from_pretrained("deep-learning-analytics/triviaqa-t5-base")

# tokenizer = T5Tokenizer.from_pretrained('t5-base')

class QADataset(Dataset):
    def __init__(
            self,
            data, #:pd.DataFrame
            source_max_token_len: int = 396,
            target_max_token_len: int = 32,
        ):
        self.data =  data
        self.source_max_token_len =  source_max_token_len
        self.target_max_token_len =  target_max_token_len
        self.tokenizer = tokenizer

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index: int):
        data_row = self.data.iloc[index]
        source_encoding = self.tokenizer(
            data_row['question'],
            data_row['context'],
            max_length=self.source_max_token_len,
            padding='max_length',
            truncation="only_second",
            return_attention_mask=True,
            add_special_tokens=True,
            return_tensors="pt"
            )
        target_encoding = self.tokenizer(
            data_row['answer_text'],
            max_length=self.target_max_token_len,
            padding='max_length',
            truncation=True,
            return_attention_mask=True,
            add_special_tokens=True,
            return_tensors="pt"
            )
        labels = target_encoding['input_ids']
        labels[labels==0] = -100
        return dict(
            question=data_row['question'],
            context=data_row['context'],
            answer_text=data_row['answer_text'],
            input_ids=source_encoding["input_ids"].flatten(),
            attention_mask=source_encoding['attention_mask'].flatten(),
            labels=labels.flatten()
        )

class DataModule(pl.LightningDataModule):

    def __init__(
            self,
            train_df,
            test_df,
            batch_size: int = 8,
            source_max_token_len: int = 396,
            target_max_token_len: int = 32,
        ):
        super().__init__()
        self.train_df = train_df
        self.test_df = test_df
        self.batch_size = batch_size
        self.source_max_token_len = source_max_token_len
        self.target_max_token_len = target_max_token_len

    def setup(self):
        self.train_dataset = QADataset(
            self.train_df,
            self.source_max_token_len,
            self.target_max_token_len
        )
        self.test_dataset = QADataset(
            self.test_df,
            self.source_max_token_len,
            self.target_max_token_len
        )

    def train_dataloader(self):
        return DataLoader(
            self.train_dataset,
            batch_size=self.batch_size,
            shuffle=True,
            num_workers=4
        )

    def val_dataloader(self):
        return DataLoader(
            self.test_dataset,
            batch_size=self.batch_size,
            num_workers=4
        )

    def test_dataloader(self):
        return DataLoader(
            self.test_dataset,
            batch_size=1,
            num_workers=4
        )

class QAModel(pl.LightningModule):

    def __init__(self):
        super().__init__()
        # self.model = T5ForConditionalGeneration.from_pretrained('t5-base', return_dict=True)
        self.model = AutoModelWithLMHead.from_pretrained("deep-learning-analytics/triviaqa-t5-base", return_dict=True)

    def forward(self, input_ids, attention_mask, labels=None):
        output = self.model(
            input_ids,
            attention_mask=attention_mask,
            labels=labels)
        return output.loss, output.logits

    def training_step(self, batch, batch_idx):
        input_ids = batch['input_ids']
        attention_mask=batch['attention_mask']
        labels = batch['labels']
        loss, outputs = self(input_ids, attention_mask, labels)
        self.log("train_loss", loss, prog_bar=True, logger=True)
        return {
            "loss": loss,
            "predictions":outputs,
            "labels": labels
        }

    def validation_step(self, batch, batch_idx):
        input_ids = batch['input_ids']
        attention_mask=batch['attention_mask']
        labels = batch['labels']
        loss, outputs = self(input_ids, attention_mask, labels)
        self.log("val_loss", loss, prog_bar=True, logger=True)
        return loss

    def test_step(self, batch, batch_idx):
        input_ids = batch['input_ids']
        attention_mask=batch['attention_mask']
        labels = batch['labels']
        loss, outputs = self(input_ids, attention_mask, labels)
        self.log("test_loss", loss, prog_bar=True, logger=True)
        return loss

    def configure_optimizers(self):
        optimizer = AdamW(self.parameters(), lr=0.0001)
        return optimizer

def generate_answer(question, trained_model):
    source_encoding = tokenizer(
        question["question"],
        question['context'],
        max_length = 396,
        padding="max_length",
        truncation="only_second",
        return_attention_mask=True,
        add_special_tokens=True,
        return_tensors="pt"
    )
    generated_ids = trained_model.model.generate(
        input_ids=source_encoding["input_ids"],
        attention_mask=source_encoding["attention_mask"],
        num_beams=10,  # greedy search
        max_length=20,
        repetition_penalty=2.5,
        early_stopping=True,
        use_cache=True)
    preds = [
            tokenizer.decode(generated_id, skip_special_tokens=True, clean_up_tokenization_spaces=True)
            for generated_id in generated_ids
    ]
    return "".join(preds)


#!/usr/bin/env python

from __future__ import print_function

# ------------------------------------------------------------------------
# Test client for the pyserv project. Encrypt test.

import  os, sys, getopt, signal, select, socket, time, struct
import  random, stat

#sys.path.append('../common')
#import support, pycrypt, pyservsup, pyclisup, syslog, comline, pypacker

# Set parent as module include path
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from common import support, pycrypt, pyservsup, pyclisup
from common import pysyslog, comline, pypacker

version = "1.0"

optarr =  comline.optarr
optarr.append ( ["p:",  "port",  6666,   None, "Port to use (default: 6666)"] )

conf = comline.Config(optarr)

# ------------------------------------------------------------------------

if __name__ == '__main__':

    args = conf.comline(sys.argv[1:])

    if len(args) == 0:
        ip = '127.0.0.1'
    else:
        ip = args[0]

    hand = pyclisup.CliSup()
    hand.verbose = conf.verbose
    hand.pgdebug = conf.pgdebug

    try:
        resp2 = hand.connect(ip, conf.port)
    except:
        print( "Cannot connect to:", ip + ":" + str(conf.port), sys.exc_info()[1])
        sys.exit(1)

    #print ("Server initial:", resp2)

    resp = hand.client(["ver"])

    if conf.quiet == False:
        print ("Server ver response:", resp[1])

    print("Waiting for timeout ...")
    response = hand.getreply()

    print ("Server timeout response:", response[1])

    # Here the server response should be none (disconnected)
    #qr = hand.client(["quit"])
    #hand.close()
    #print ("Server quit response:", qr[1])

    sys.exit(0)

# EOF


from django.urls import path
from . import views
from watchlist import views as wlv
from exam import views as EView
from register import views as RView
from django.contrib.auth import views as auth_views

urlpatterns = [
    path("", views.home, name="home"),
    path("home/", views.home, name="home"),
    path("BuyAndSell/", views.BuyAndSell, name="BuyAndSell"),
    path('accounts/login/', auth_views.LoginView.as_view(), name='login'),
    path("log-out/", views.logout, name="log-out"),
    path("cryptoList/", views.cryptoList, name="CryptoList"),
    path("tickets/", views.tickets, name="tickets"),
    path("personal_info/", views.ViewPersonalInformation.as_view(), name="personal_info"),
    path("register/", RView.register, name="register"),
    path('FillPersonalInfo/<int:user_id>/', RView.personal_info, name='personal_info'),
    path("watchlist/", wlv.watchList, name="watchlist"),
    path("deleteItem/<int:item_id>/", wlv.deleteItem,name="deleteItem"),
    path("previousTrades", views.previousTrades, name="previousTrades"),
    path("exam/", EView.take_quiz, name="exam"),
    path ('exam/results/', EView.result, name='results'),
    path("create-ticket", views.create_ticket, name='create-ticket'),
    path('deposit_withdraw/', views.show_deposit_withdraw_form, name='deposit_withdraw'),
    path('deposit/', views.deposit, name='deposit'),
    path('withdraw/', views.withdraw, name='withdraw'),
]


"""lokaleplan URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin

from lokaleplan.views import (
    SessionList, SessionDelete, Home, PerlView, AddUser,
    ParticipantDetail, ParticipantPlans, ParticipantMessageUpdate,
    EventTable, EventList, EventDelete, LocationList,
    EventUpdate, EventUpdateExternal, EventCreate, EventCreateExternal,
    SessionEdit, SessionCreate,
)

sessions = [
    url(r'^$', Home.as_view(), name='home'),
    url(r'^delete/$', SessionDelete.as_view(), name='session_delete'),
    url(r'^import/$', PerlView.as_view(), name='import'),
    url(r'^edit/$', SessionEdit.as_view(), name='edit-csv'),
    url(r'^adduser/$', AddUser.as_view(), name='add_user'),
    url(r'^hold/(?P<pk>[0-9]+)/$', ParticipantDetail.as_view(), name='participant_detail'),
    url(r'^hold/(?P<pk>[0-9]+)/message/$', ParticipantMessageUpdate.as_view(), name='participant_message'),
    url(r'^lokaleplan-(?P<pk>[0-9]+)\.tex$', ParticipantPlans.as_view(),
        name='participant-tex', kwargs=dict(mode='source')),
    url(r'^lokaleplan-(?P<pk>[0-9]+)\.pdf$', ParticipantPlans.as_view(),
        name='participant-pdf', kwargs=dict(mode='pdf')),
    url(r'^locations/$', LocationList.as_view(), name='location_list'),
    url(r'^events/$', EventList.as_view(), name='events'),
    url(r'^events/locations/$', EventTable.as_view(),
        name='locations', kwargs=dict(mode='locations')),
    url(r'^events/participants/$', EventTable.as_view(),
        name='participants', kwargs=dict(mode='participants')),
    url(r'^event/(?P<pk>\d+)/$', EventUpdate.as_view(), name='event_update'),
    url(r'^event/(?P<pk>\d+)/delete/$', EventDelete.as_view(), name='event_delete'),
    url(r'^event/external/new/$', EventCreateExternal.as_view(),
        name='event_create_external'),
    url(r'^event/external/(?P<pk>\d+)/$', EventUpdateExternal.as_view(),
        name='event_update_external'),
    url(r'^event/new/$', EventCreate.as_view(), name='event_create'),
    url(r'^lokaleplan\.tex$', ParticipantPlans.as_view(),
        name='participants-tex', kwargs=dict(mode='source')),
    url(r'^lokaleplan\.pdf$', ParticipantPlans.as_view(),
        name='participants-pdf', kwargs=dict(mode='pdf')),
]

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url('^', include('django.contrib.auth.urls')),
    url(r'^$', SessionList.as_view(), name='session_list'),
    url(r'^new/$', SessionCreate.as_view(), name='session_create'),
    url(r'^session/(?P<session>\d+)/', include(sessions)),
]


import pytesseract
from tqdm import tqdm
from mydatabase import SemPdf
from pdf2image import convert_from_path

def extract_text_from_pdf(pdf_path):
    # Convert PDF to image
    pages = convert_from_path(pdf_path)

    # Extract text from each page using Tesseract OCR
    text_data = ""
    for page in pages:
        text = pytesseract.image_to_string(page)
        text_data += text + "\n"

    # Return the text data
    return text_data

def extract_text_for_sempdfs(sempdfs: list[SemPdf]):
    pbar = tqdm(sempdfs)
    for sempdf in pbar:
        sempdf.text = extract_text_from_pdf(sempdf.paths[0])
        pbar.set_description("Processing %s" % sempdf.paths[0].split("/")[-1])
    return sempdfs


import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
matplotlib.use('TkAgg')

if __name__ == '__main__':
    files = os.listdir("differentMDOperatorsResults")
    print(files)
    dfs = {}
    outliers = {}
    for file in files:
        df = pd.read_csv(f"differentMDOperatorsResults/{file}", delimiter=",")
        dfs[file.replace('.csv','')] = df
        all_averages = []
        total_mutants = 0
        total_methods = 0
        for idx, row in df.iterrows():
            sloc = row['Source Lines of Code']
            md = row["Mutant Density"]
            if sloc > 99:
                continue
            # all_diffs.append(sloc-md)
            all_averages.append(md / sloc)
            total_methods += 1
            total_mutants += md

        mean = np.mean(all_averages)
        std = np.std(all_averages)
        current_outliers = []
        for idx, row in df.iterrows():
            sloc = row['Source Lines of Code']
            md = row["Mutant Density"]
            average = md/sloc
            if sloc > (mean+std*2) or sloc < (mean-std*2):
                current_outliers.append(f"{row['File']};{row['Method Name']}")
        outliers[file.replace('.csv','')] = current_outliers
        # sns.displot(all_averages, kde=True).set(title='opencv distribution plot of average MD : ' + file.replace('.csv',''))
        # plt.xlim(-50, 50)
        # plt.show()
        # print(f"Mutations: {file.replace('.csv','')}, Methods: {total_methods}, Total Mutants: {total_mutants}")
        print(f"Mutations: {file.replace('.csv','')}, Mean: {mean}, Standard deviation: {std}")

    temp = []
    keys = []
    for key, val in outliers.items():
        temp.append(set(val))
        keys.append(key)

    results = set.intersection(*map(set, temp))
    print("------- Outliers over all sets ---------")
    print(len(results))
    print("------ Single mutation outliers --------")
    for index, list in enumerate(temp):

        for index2, second_list in enumerate(temp):
            if index != index2:
                list = list.difference(second_list)
        print(f"{keys[index]} : {list}")


from functools import reduce
employees=[
    {"eid":1000,"name":"ajay","salary":25000,"designation":"developer"},
    {"eid": 1001, "name": "vjay", "salary": 22000, "designation": "developer"},
     {"eid": 1002, "name": "arun", "salary": 26000, "designation": "qa"},
     {"eid": 1003, "name": "varun", "salary": 27000, "designation": "ba"},
     {"eid": 1004, "name": "ram", "salary": 20000, "designation": "mrkt"},
]
# print employees name only
#
# for employee in employees:
#     print(employee["name"])
e_names=list(map(lambda emp:emp["name"],employees))
up_names=list(map(lambda name:name.upper(),e_names))
print(e_names)
print(up_names)
a_name=list(filter(lambda emp:emp["name"][0]=='a',employees))
print(a_name)
sal_emp=list(filter(lambda emp:emp["salary"]>23000,employees))
print(sal_emp)
developers=list(filter(lambda emp:emp["designation"]=="developer,",employees))
print(developers)
high_salary=reduce(lambda sal1,sal2:sal1 if sal1>sal2 else sal2,
                   list(map(lambda emp:emp["salary"],employees)))
print(high_salary)


"""
LAST VERSION TO COMPUTE BAG OF WORDS
"""
import json
from itertools import cycle
import numpy as np
import time

from sklearn.manifold import Isomap, SpectralEmbedding, TSNE
from sklearn.metrics import silhouette_score
from cluster.hkmeans import HKMeans
import matplotlib.pyplot as plt
# from sklearn.decomposition import PCA, SparsePCA
from cluster.distance_functions import hamming

paths = []
paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s0/0_1fps')
paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s0/1_1fps')

# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s1/0_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s1/1_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s1/2_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s1/3_1fps')

# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s2/0_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s2/1_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s2/2_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s2/3_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s2/4_1fps')

# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s3/0_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s3/1_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s3/2_1fps')
# paths.append('/media/arvc/50324D7B324D6756/datasets/VIRB360/euroc/s3/3_1fps')

def read_descriptors(path):
    descriptors_input_path = path # + '/mav0/descriptors.json'
    with open(descriptors_input_path, 'r') as json_file:
        data = json.load(json_file)
        desc1 = data['descriptors_front']
        desc2 = data['descriptors_back']
        # extend list 1
        desc1.extend(desc2)
        return desc1

def flatten_list(all_descriptors):
    """
    Flattens list of descriptors: original list is of size
        n_landmarks list with [1500 x 32] in each
    """
    ret_descriptors = []
    for descriptors_image in all_descriptors:
        for desc in descriptors_image:
            ret_descriptors.append(desc)
    return ret_descriptors

# def plot_result(X, y, centroids):
#     # Plot init seeds along side sample data
#     plt.figure()
#     colors = cycle(['r', 'g', 'b', 'k', 'c', 'm', 'y'])
#     for k in set(y):
#         cluster_data = (y == k)
#         plt.scatter(X[cluster_data, 0], X[cluster_data, 1], c=next(colors), marker='.', s=50)
#         # plt.scatter(centroids[k][0], centroids[k][1], c='k', marker='.', s=200)
#     plt.title("RESULT! Data clustered")
#     plt.show()
#     return

def plot_results_projecting(X, y, centroids):
    from cluster.distance_functions import compute_hamming_distances
    # Plot init seeds along side sample data
    plt.figure()
    colors = cycle(['r', 'g', 'b', 'k', 'c', 'm', 'y'])
    on = np.unpackbits(np.array([255, 255, 255, 255, 255, 255, 255, 255,
                                 255, 255, 255, 255, 255, 255, 255, 255,
                                 255, 255, 255, 255, 255, 255, 255, 255,
                                 255, 255, 255, 255, 255, 255, 255, 255], dtype=np.uint8))
    off = np.unpackbits(np.array([0, 0, 0, 0, 0, 0, 0, 0,
                                  0, 0, 0, 0, 0, 0, 0, 0,
                                  0, 0, 0, 0, 0, 0, 0, 0,
                                  0, 0, 0, 0, 0, 0, 0, 0], dtype=np.uint8))
    for k in set(y):
        cluster_data = (y == k)
        # project all data of that cluster to to two dimensions, using two hamming distances.
        # d1 is the distance to [0  0 00 0 ]
        # d2 is the distance to [11111]
        d1 = compute_hamming_distances(X[cluster_data], on)
        d2 = compute_hamming_distances(X[cluster_data], off)
        # plt.scatter(X[cluster_data, 0], X[cluster_data, 1], c=next(colors), marker='.', s=50)
        plt.scatter(d1, d2, c=next(colors), marker='.', s=50)
        d1c = hamming(centroids[k], on)
        d2c = hamming(centroids[k], off)
        # plt.scatter(centroids[k][0], centroids[k][1], c='k', marker='.', s=200)
        plt.scatter(d1c, d2c, c=next(colors), marker='*', s=50)
        plt.annotate(str(k), (d1c, d2c))

    plt.title("RESULT! Data clustered")
    plt.show()
    return

def plot_results_isomap(X, y, centroids):
    embedding = Isomap(n_components=2, metric='manhattan', p=1)
    X_red = embedding.fit_transform(X)
    plot_results(X_red, y, title='RESULT! Data clustered. Using ISOMAP CLUSTERING TO VISUALIZE')

    # centroids_red = embedding.transform(centroids)
    # colors = cycle(['r', 'g', 'b', 'k', 'c', 'm', 'y'])
    #
    # for k in set(y):
    #     cluster_data = (y == k)
    #     plt.scatter(X_red[cluster_data, 0], X_red[cluster_data, 1], c=next(colors), marker='.', s=50)
    #     plt.scatter(centroids_red[k][0], centroids_red[k][1], c='k', marker='.', s=200)
    # plt.title("RESULT! Data clustered. Using ISOMAP TO VISUALIZE")
    # plt.show()

def plot_results_spectral(X, y, centroids):
    embedding = SpectralEmbedding(n_components=2)
    X_red = embedding.fit_transform(X)
    plot_results(X_red, y, title='RESULT! Data clustered. Using SPECTRAL CLUSTERING TO VISUALIZE')

    #
    # # centroids_red = embedding.transform(centroids)
    # colors = cycle(['r', 'g', 'b', 'k', 'c', 'm', 'y'])
    #
    # for k in set(y):
    #     cluster_data = (y == k)
    #     plt.scatter(X_red[cluster_data, 0], X_red[cluster_data, 1], c=next(colors), marker='.', s=50)
    #     # plt.scatter(centroids_red[k][0], centroids_red[k][1], c='k', marker='.', s=200)
    # plt.title("RESULT! Data clustered. Using SPECTRAL CLUSTERING TO VISUALIZE")
    # plt.show()

def plot_results_TSNE(X, y, centroids):
    embedding = TSNE(n_components=2,
                     init='pca',
                     perplexity=500)
    X_red = embedding.fit_transform(X)
    plot_results(X_red, y, title='RESULT! Data clustered. Using tsne TO VISUALIZE')

def plot_results(X, y, title):
    colors = cycle(['r', 'g', 'b', 'k', 'c', 'm', 'y'])
    markers = cycle(['.', '+', 'o', '*', '1', 'p', 's', 'x', 'X'])
    strs = []
    i = 0
    for k in set(y):
        cluster_data = (y == k)
        color = next(colors)
        marker = next(markers)
        plt.scatter(X[cluster_data, 0], X[cluster_data, 1], c=color, marker=marker, s=50)
        # plt.scatter(centroids_red[k][0], centroids_red[k][1], c='k', marker='.', s=200)
        strs.append(str(i))
        i += 1
    plt.legend(strs)
    plt.title(title)
    plt.show()

def precompute_distances(X, labels, output_size_ratio):
    idx = np.random.choice(len(X), int(len(X)*output_size_ratio), replace=False)
    X = X[idx]
    labels = labels[idx]
    # create a square distance matrix
    D = np.zeros((len(X), len(X)))
    for i in range(len(X)):
        for j in range(len(X)):
            a = X[i, :]
            b = X[j, :]
            d = hamming(a, b)
            D[i, j] = d
    return D, labels

def load_data(paths, sampling=None, max_index=None):
    all_descriptors = []
    if paths is None:
        all_descriptors = read_descriptors('../data/descriptors.json')
    else:
        # read several paths
        for path in paths:
            descriptors = read_descriptors(path + '/mav0/descriptors.json')
            all_descriptors.extend(descriptors)
    # next, flatten list in order to obtain a flat list of ORB descriptors
    all_descriptors = flatten_list(all_descriptors)
    if max_index is None:
        max_index = len(all_descriptors)
    if sampling is None:
        sampling = 1
    all_descriptors = all_descriptors[0:max_index:sampling]
    X = np.array(all_descriptors, np.uint8)
    return X

def save_bow_vectors(bow_vector, words, n_words, filename='bagofwords.json'):
    """
    Save the bag of words
    """
    data = {'bow_vector': bow_vector.tolist(),
            'words': words.tolist(),
            'n_words': n_words.tolist()}
    with open(filename, 'w') as outfile:
        json.dump(data, outfile)

if __name__ == '__main__':
    # number of samples to process
    # caution may be in the number of
    # 600000
    n_samples = 5000
    kw = 2
    lw = 9
    # load ORB DESCRIPTORS
    X = load_data(paths=paths, sampling=1, max_index=n_samples)
    # X = load_data(paths=None, sampling=1, max_index=n_samples)
    # transform to binary descriptors (length is 256 bits)
    X = np.unpackbits(X, axis=1)

    kmeans_params = {'tol': 0.001,
                     'max_iter': 300,
                     'distance_function': 'hamming',
                     'centroid_replacement': False,
                     'averaging_function': 'mean-round', # this is majority voting for binary descriptors if unpacked
                     'init_method': 'kmeans++',
                     'plot_progress': True}
    start_time = time.time()
    hk = HKMeans(kw=kw, lw=lw, kmeans_params=kmeans_params)
    hk.print_tree()
    hk.fit(X)
    elapsed_time = time.time() - start_time
    hk.print_tree()
    hk.print_words()
    # hk.plot_tree_data(X)
    print('Clustering took: ', elapsed_time, '(s)')

    print('Finding predictions via brute-force:')
    start_time = time.time()
    new_labels1, _ = hk.predict_brute_force(X)
    elapsed_time = time.time() - start_time
    print('Took: ', elapsed_time, '(s)')
    print('Done predicting brute Force. found: ', len(set(new_labels1)), ' different labels')

    print('Finding predictions via top-down:')
    start_time = time.time()
    new_labels2, _ = hk.predict_top_down(X)
    elapsed_time = time.time() - start_time
    print('Took: ', elapsed_time)
    print('Done predicting top down. found: ', len(set(new_labels2)), ' different labels')

    print("Clustered ", len(X), " n descriptors")
    print('Trained a tree with (kw, lw): ', kw, ', ', lw)
    print('Total number of nodes: ', hk.get_number_of_nodes())
    print('Total number of leaves: ', hk.get_number_of_leaves())
    print('Total number of words expected: ', hk.get_expected_number_of_words())
    print('Total number of words found: ', hk.count_number_of_words())
    print('Total cost of fit (at last hierarchy level): ', hk.get_total_cost())
    print('Number of datapoints per leaf (WORD): ', hk.get_n_datapoints_per_word())
    print('Fit time: ', elapsed_time, '(s)')
    # use 10 percent of the smaples
    # precompute distance matrix using hamming distance
    D, sampled_labels = precompute_distances(X, new_labels2, .1)
    # sil_score = silhouette_score(X, new_labels, sample_size=n_samples)
    # using precomputed distance
    sil_score = silhouette_score(D, sampled_labels, sample_size=n_samples, metric='precomputed')
    print("Silhouette Coefficient of tree classification: ", sil_score)

    # Xred = SparsePCA(n_components=2).fit_transform(X)
    # plot_results_projecting(X, new_labels2, hk.leaf_centroids)
    plot_results_isomap(X, new_labels1, hk.leaf_centroids)
    plot_results_spectral(X, new_labels1, hk.leaf_centroids)
    plot_results_TSNE(X, new_labels1, hk.leaf_centroids)

    print('Creating vocabulary object!')
    # create the vocabulary!!!
    voc = hk.create_vocabulary()
    print('Replacing centroids by true prototypes')
    # optionally, replace each original word by the closest, real, existing descripotr
    # no need to replace centroids if centroid_replacement=True
    # voc.replace_centroids(X)
    print('Saving vocabulary')
    voc.save_json('voc.json')
    print('Computing BOW vectors')
    bow_vector_st, words, n_words = voc.compute_bows(X, option='standard')
    print(bow_vector_st)
    # bow_vector_bin, words, n_words = voc.compute_bows(X, option='binary')
    # bow_vector_tfidf, words, n_words = voc.compute_bows(X, option='tfidf')

    # save bag of word vectors
    save_bow_vectors(bow_vector_st, words, n_words, 'bagofwords.json')


from pathlib import Path
import shutil

import pandas as pd
import torch
import torchaudio
import wget

URL = 'http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz'

class SpeechCommandsDataset(torch.utils.data.Dataset):
    def __init__(
        self,
        path2dir,
        transform=None,
        keywords=None,
        csv=None
    ):
        if keywords is None and csv is None:
            raise ValueError('You must specify either keywords or csv')

        self.transform = transform

        path2dir = Path(path2dir)
        self.path2dir = path2dir
        if not path2dir.exists() or len(list(path2dir.iterdir())) == 0:
            self._download(path2dir)

        if csv is None:
            keywords = keywords if isinstance(keywords, list) else [keywords]

            all_keywords = [
                p.stem for p in path2dir.glob('*')
                if p.is_dir() and not p.stem.startswith('_')
            ]

            triplets = []
            for keyword in all_keywords:
                paths = (path2dir / keyword).rglob('*.wav')
                if keyword in keywords:
                    for path2wav in paths:
                        triplets.append((str(path2wav.relative_to(path2dir)),
                                         keyword, 1))
                else:
                    for path2wav in paths:
                        triplets.append((str(path2wav.relative_to(path2dir)),
                                         keyword, 0))

            self.csv = pd.DataFrame(
                triplets,
                columns=['path', 'keyword', 'label']
            )
        else:
            self.csv = csv

        self.csv.sort_values(by='path', inplace=True, ignore_index=True)

    def __getitem__(self, index: int):
        instance = self.csv.iloc[index]

        path2wav = self.path2dir / instance['path']
        wav, sr = torchaudio.load(path2wav)
        wav = wav.sum(dim=0)

        if self.transform:
            wav = self.transform(wav)

        return {
            'wav': wav,
            'keywors': instance['keyword'],
            'label': instance['label']
        }

    def __len__(self):
        return len(self.csv)

    def train_val_split(self, train_frac, augs=None):
        indices = torch.randperm(len(self))
        train_sz = int(len(self) * train_frac)
        train_indices = indices[:train_sz]
        val_indices = indices[train_sz:]

        train_df = self.csv.iloc[train_indices].reset_index(drop=True)
        val_df = self.csv.iloc[val_indices].reset_index(drop=True)
        train_set = SpeechCommandsDataset(self.path2dir, csv=train_df,
                                          transform=augs)
        val_set = SpeechCommandsDataset(self.path2dir, csv=val_df)
        return train_set, val_set

    def _download(self, path2dir: Path):
        path2dir.mkdir(parents=True, exist_ok=True)
        archive_path = path2dir / 'archive.tar.gz'
        if archive_path.exists():
            raise RuntimeError('archive.tar.gz exists in root directory')
        wget.download(URL, str(archive_path))
        shutil.unpack_archive(str(archive_path), str(path2dir))
        archive_path.unlink()


import stage
import game
import theme
import curses

screen = None

def drawTile(x, y, tile='', color=None):
    color = color or theme.get_color('default')

    x = x * 2 + stage.padding[3] * 2 + stage.width / 2
    y += stage.padding[0] + stage.height / 2

    screen.addstr(y, x, tile, color)
    if (len(tile) < 2):
        screen.addstr(y, x + 1, tile, color)

def drawGameOver():
    drawTile(-4, -1, " GAME OVER ", theme.get_color('border'))
    drawTile(-7, 1, " Press ENTER to restart ", theme.get_color('border'))

def drawScore():
    score_formatted = str(game.score).zfill(2)
    drawTile(
        (stage.width / 2) - 1,
        (-stage.height / 2) - 1,
        score_formatted,
        theme.get_color('border')
        )

def drawLives():
    posx = (-stage.width / 2) + 3
    for x in xrange(1, game.lives + 1):
        posx += 1
        drawTile(
            posx,
            (-stage.height / 2) - 1,
            theme.get_tile('lives'),
            theme.get_color('lives')
            )
        posx += 1
        drawTile(
            posx,
            (-stage.height / 2) - 1,
            theme.get_tile('border-h'),
            theme.get_color('border')
            )

def drawSnake():
    for part in game.snake:
        drawTile(
            part[0],
            part[1],
            theme.get_tile('snake-body'),
            theme.get_color('snake')
            )
    # Clean last tile
    drawTile(
        game.lastPos[0],
        game.lastPos[1],
        theme.get_tile('bg'),
        theme.get_color('bg')
        )

def drawApples():
    for apple in game.apples:
        drawTile(
            apple[0],
            apple[1],
            theme.get_tile('apple'),
            theme.get_color('apple')
            )

def drawGame():
    for y in range(stage.boundaries['top'], stage.boundaries['bottom']):
        for x in range(stage.boundaries['left'], stage.boundaries['right']):
            drawTile(x, y, theme.get_tile('bg'), theme.get_color('bg'))
    drawBorders()
    drawText()

def drawBorders():
    tile_v = theme.get_tile('border-v')
    tile_h = theme.get_tile('border-h')
    tile_c = theme.get_tile('border-c')
    color = theme.get_color('border')

    x_left = stage.boundaries['left']
    x_right = stage.boundaries['right']

    y_top = stage.boundaries['top']
    y_bottom = stage.boundaries['bottom']

    for y in range(y_top, y_bottom):
        drawTile(x_left - 1, y, tile_v, color)
        drawTile(x_right, y, tile_v, color)

    for x in range(x_left, x_right):
        drawTile(x, y_top - 1, tile_h, color)
        drawTile(x, y_bottom, tile_h, color)

    drawTile(x_left - 1, y_top - 1, tile_c, color)
    drawTile(x_left - 1, y_bottom, tile_c, color)
    drawTile(x_right, y_top - 1, tile_c, color)
    drawTile(x_right, y_bottom, tile_c, color)

def drawText():
    color = theme.get_color('border')
    drawTile((stage.width / 2) - 4, (-stage.height / 2) - 1, "score:", color)
    drawTile((-stage.width / 2), (-stage.height / 2) - 1, "lives:", color)
    drawTile(-5, (stage.height / 2), " Press Q to quit ", color)

def update():

    drawSnake()
    drawApples()
    drawScore()
    drawLives()

def init():
    global screen

    screen = curses.initscr()
    curses.noecho()
    curses.cbreak()
    curses.curs_set(0)
    curses.start_color()
    screen.nodelay(1)

def exit():
    screen.clear()
    screen.keypad(0)
    curses.echo()
    curses.nocbreak()
    curses.endwin()


from __future__ import unicode_literals
from django.utils.encoding import smart_text
from django.db import models
import pdb

from .. import User

class Foup(models.Model):
    foupname=models.CharField(max_length=16, verbose_name='Foup Name',primary_key=True)
    note=models.CharField(max_length=200,null=True,blank=True)
    owner=models.ForeignKey(User,on_delete=models.SET_NULL,null=True,blank=True)
    timestamp=models.DateTimeField(auto_now_add=True,auto_now=False)    #creation time
    updated=models.DateTimeField(auto_now_add=False,auto_now=True)  #update time
    def __str__(self):
        return self.foupname

    @classmethod
    def create(cls,foupname,note,owner):
        #for foup in cls.objects.all():
        foup=cls(foupname=foupname,note=note,owner=owner)
        foup.save()
        for i in range(1,26):
            foup_slot=foup.foup_slot_set.create(slot=i)
            foup_slot.save()

        return foup
    def delete(self):
        #delete a foup from database
        for foup_slot in self.foup_slot_set.all():
            if foup_slot.wafer!=None:
                foup_slot.wafer.isused='Used'
            foup_slot.delete()
        self.delete()
        return

    def show_slots(self):
        #show the 25 slots of the foup, whether it is vacant, whether the contained wafer is new or used
        for i in range(1,26):
            foup_slot=self.foup_slot_set.filter(slot=i)[0]
            if foup_slot.wafer==None:
                print(self.foupname+'\t'+'S'+str(i)+'\t'+'Vacant')
            else:
                print(self.foupname+'\t'+'S'+str(i)+'\t'+foup_slot.wafer.isused)
        return

class Wafer(models.Model):
    waferType=(
        ('PT', 'Particle Grade Wafer'),
        ('LR', 'Low Res Particle Grade Wafer'),
        ('MC', 'Mechanical Grade Wafer'),
        ('ER', 'External Reclaimed Wafer'),
        ('IR', 'Internal Reclaimed Wafer'),
    )

    used_choices=(
        ('N','New'),
        ('U','Used'),
    )
    isUsed=models.CharField(max_length=10,choices=used_choices)

    wafertype=models.CharField(max_length=20,choices=waferType,default='Internal Reclaimed Wafer')
    note=models.CharField(max_length=200,null=True,blank=True)
    timestamp=models.DateTimeField(auto_now_add=True,auto_now=False)    #creation time
    updated=models.DateTimeField(auto_now_add=False,auto_now=True)  #update time
    def __str__(self):
        return smart_text(self.wafertype+' '+self.isUsed)

class Foup_slot(models.Model):
    foup=models.ForeignKey(Foup,on_delete=models.CASCADE)
    wafer=models.OneToOneField(Wafer,on_delete=models.SET_NULL,blank=True,null=True)    #setting on_delete=models.SET_NULL, means at times of deleting a wafer instance, the system sets this field to NULL
    slot=models.IntegerField(default=0)
    note=models.CharField(max_length=200,null=True,blank=True)
    updated=models.DateTimeField(auto_now_add=False,auto_now=True)  #update time
    def __str__(self):
        foup_name=self.foup.foupname
        s=foup_name+' S'+str(self.slot)+'\t'
        status=''
        if self.wafer is None:
            status='Vacant'
        else:
            status='Occupied\t'+'Wafer:\t'+self.wafer.isUsed
        return s+status
    def load_new_wafers(self,wafer_type):
        #load wafer into the current foup_slot: 1. check whether this slot is vacant; 2. load new wafers
        if self.wafer!=None:
            print("Slot: "+str(self.slot)+" still has wafer. Load unsuccessfull.")
            return
        new_wafer=Wafer(isUsed='New',wafertype=wafer_type,note=self.foup.foupname+' S'+str(self.slot))
        #pdb.set_trace()
        new_wafer.save()
        self.wafer=new_wafer
        self.save()
        print("Load sucessfully.")
        return
    def reclaim_wafer(self):
        if self.wafer==None:
            print("Slot: "+str(self.slot)+" is vacant.")
        else:
            self.wafer=None
            self.save()
            print("Reclaim successfully")
        return
    def has_wafer(self):
        return self.wafer!=None
    def wafer_used(self):
        if self.has_wafer():
            if self.wafer.isUsed=='New':
                return False
            else:
                return True
        else:
            return False


from confluent_kafka import Consumer, KafkaError
import cv2
import numpy as np

# Kafka configuration
kafka_conf = {
    'bootstrap.servers': 'localhost:9092',  # Replace with your Kafka broker address
    'group.id': 'camera_consumer_group',    # Consumer group ID
    'auto.offset.reset': 'earliest'        # Start reading from the beginning of the topic
}

# Kafka topic for frames
kafka_topic = 'camera_frames'

# Create Kafka consumer
consumer = Consumer(kafka_conf)
consumer.subscribe([kafka_topic])

def read_frames():
    while True:
        msg = consumer.poll(1.0)

        if msg is None:
            break

        if msg.error():
            if msg.error().code() == KafkaError._PARTITION_EOF:
                print("Reached end of partition")
            else:
                print(f"Error while consuming: {msg.error()}")
        else:
            # Decode the frame data
            frame_data = msg.value()

            # Convert the frame data back to an image
            frame = cv2.imdecode(np.frombuffer(frame_data, np.uint8), cv2.IMREAD_COLOR)

            # Process the frame as needed (e.g., display, save, etc.)
            cv2.imshow("IP Camera Footage", frame)

            # Exit the program if 'q' is pressed
            if cv2.waitKey(1) == ord('q'):
                break

    # Destroy all windows when done
    cv2.destroyAllWindows()

def main():
    read_frames()


import json
import os

from collections import Mapping

import google.auth

from decouple import config
from google.cloud.storage.client import Client  # pylint: disable=ungrouped-imports
from google.oauth2.service_account import Credentials  # pylint: disable=ungrouped-imports

from ..exceptions import DblueStoresException
from ..logger import logger
from .base import BaseClient

DEFAULT_SCOPES = ('https://www.googleapis.com/auth/cloud-platform',)

class GCPClient(BaseClient):

    @classmethod
    def get_client(cls, *args, **kwargs):
        return cls._get_client(*args, **kwargs)

    @staticmethod
    def get_credentials(key_path=None, keyfile_dict=None, scopes=None):
        """
        Returns the Credentials object for Google API
        """
        key_path = key_path or config("GCP_KEY_FILE_PATH", default=None)
        keyfile_dict = keyfile_dict or config("GCP_KEY_FILE_DICT", default=None)
        scopes = scopes or config("GCP_SCOPES", default=None)

        if scopes is not None:
            scopes = [s.strip() for s in scopes.split(',')]
        else:
            scopes = DEFAULT_SCOPES

        if not key_path and not keyfile_dict:
            logger.info('Getting connection using `google.auth.default()` '
                        'since no key file is defined for hook.')
            credentials, _ = google.auth.default(scopes=scopes)
        elif key_path:
            # Get credentials from a JSON file.
            if key_path.endswith('.json'):
                logger.info('Getting connection using a JSON key file.')
                credentials = Credentials.from_service_account_file(
                    os.path.abspath(key_path), scopes=scopes)
            else:
                raise DblueStoresException('Unrecognised extension for key file.')
        else:
            # Get credentials from JSON data.
            try:
                if not isinstance(keyfile_dict, Mapping):
                    keyfile_dict = json.loads(keyfile_dict)

                # Convert escaped newlines to actual newlines if any.
                keyfile_dict['private_key'] = keyfile_dict['private_key'].replace('\\n', '\n')

                credentials = Credentials.from_service_account_info(keyfile_dict, scopes=scopes)
            except ValueError:  # json.decoder.JSONDecodeError does not exist on py2
                raise DblueStoresException('Invalid key JSON.')

        return credentials

    @classmethod
    def get_access_token(cls, key_path=None, keyfile_dict=None, credentials=None, scopes=None):
        credentials = credentials or cls.get_credentials(key_path=key_path,
                                                         keyfile_dict=keyfile_dict,
                                                         scopes=scopes)
        return credentials.token

    @classmethod
    def _get_client(cls, project_id=None, key_path=None, keyfile_dict=None, credentials=None, scopes=None):
        credentials = credentials or cls.get_credentials(key_path=key_path,
                                                         keyfile_dict=keyfile_dict,
                                                         scopes=scopes)

        project_id = project_id or config("GCP_PROJECT_ID", default=None)

        return Client(
            project=project_id,
            credentials=credentials
        )


import boto3
from requests_aws4auth import AWS4Auth
from opensearchpy import OpenSearch, RequestsHttpConnection
import curator
import os

retention_in_days = int(os.environ.get('RETENTION_IN_DAYS'))
host = os.environ.get('OPENSEARCH_HOST')
region = os.environ.get('AWS_REGION')
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key,
                   credentials.secret_key,
                   region,
                   service,
                   session_token=credentials.token)

def lambda_handler(event, context):
    print("Checking {} OpenSearch domain".format(host))
    client = OpenSearch(
        hosts=[{'host': host, 'port': 443}],
        http_auth=awsauth,
        use_ssl=True,
        verify_certs=True,
        connection_class=RequestsHttpConnection
    )

    # Listing all indexes
    index_list = curator.IndexList(client)

    # Filtering out Kibana
    index_list.filter_kibana()

    # Filtering index list to be deleted
    index_list.filter_by_age(source='name',
                             direction='older', timestring='%Y.%m.%d', unit='days', unit_count=retention_in_days)
    print("Found {} indices that are older than {} to delete".format(len(index_list.indices), retention_in_days))

    # Deleting indexes
    if len(index_list.indices) > 0:
        curator.DeleteIndices(index_list).do_action()
    else:
        print("There is no index to delete")

if __name__ == "__main__":
    event = ''
    context = []
    lambda_handler(event, context)


import logging
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
from scapy.all import *
from scapy.layers.inet import IP,TCP
from random import randint
import pickle
from concurrent.futures import ThreadPoolExecutor

# -------------------------------------- GLOBALS --------------------------------------
opt_to_num = {opt:i  for i,opt  in enumerate(scapy.layers.inet.TCPOptions[1].keys())}
opt_to_num["NONE"] = -1
opts_set = set()
# -------------------------------------------------------------------------------------

def get_tcp_opt(p, opt, default):
    if opt in map(lambda tup : tup[0], p['TCP'].options):
        return reduce(lambda opt1,opt2 : opt1 if opt1[0]==opt else opt2, p['TCP'].options)[1]
    return default

def get_features_of(addr):
    global opts_set

    # attempt to get SYN_ACK packet from server
    syn = IP(dst=addr)/TCP(dport=443, flags='S', seq=randint(0,2**32))
    rst = IP(dst=addr)/TCP(dport=443, flags='R')
    SA  = sr1(syn, retry=5, timeout=3, verbose=0)
    tmp = send(rst, verbose=0)

    if SA == None:
        return addr, None

    opts_set.add(''.join([o[0] for o in SA['TCP'].options]))

    # extract features from the response
    # some feature names are taken from p0f in order to have reference for their meaning
    features = {
        'df'      : 1 if 'DF' in str(SA['IP'].flags) else 0 ,
        'ttl'     : SA['IP'].ttl ,
        'w_size'  : SA['TCP'].window ,
        'mss'     : get_tcp_opt(SA, 'MSS', default=-1) ,
        'w_scale' : get_tcp_opt(SA, 'WScale', default=1) ,

        'df+'  : 1  if (    'DF' in str(SA['IP'].flags) and not SA['IP'].id)    else 0 ,
        'df-'  : 1  if (not 'DF' in str(SA['IP'].flags) and     SA['IP'].id)    else 0 ,
        'fo+'  : 1  if (not 'DF' in str(SA['IP'].flags) and     SA['IP'].frag)  else 0 ,
        'fo-'  : 1  if (    'DF' in str(SA['IP'].flags) and not SA['IP'].frag)  else 0 ,

        'ecn'  : 1  if ('E' in SA['TCP'].flags)                                else 0 ,
        'seq0' : 1  if (SA['TCP'].seq == 0)                                    else 0
    }

    for i,opt in enumerate(SA['TCP'].options):
        if opt[0] in opt_to_num:
            features['opt%d' % (i)] = opt_to_num[opt[0]]
        else:
            features['opt%d' % (i)] = int(opt[0])
    for i in range(len(SA['TCP'].options), 10):
        features['opt%d' % (i)] = -1

    return addr, features

def main():
    global opts_set

    src_dir, dst_dir = sys.argv[1].replace('\\', '/'), sys.argv[2]
    skip = int(sys.argv[3])
    for filename in sorted(os.listdir(src_dir))[skip:]:
    # for filename in sorted(os.listdir(src_dir)):
        print('collecting features for file %s...', filename)
        ips, features = [], {}

        # read ips from current file
        with open('%s\\%s' % (src_dir, filename), 'rt') as curr_input_file:
            ips = list(map(lambda line : line.rstrip('\n'), curr_input_file))

        # collect features
        with ThreadPoolExecutor(max_workers = 100) as executor:
            features = {addr:data  for addr, data  in executor.map(get_features_of, ips)  if data != None}

        # save features to file
        dst_filename = '%s/%s%s' % (dst_dir, filename, '.pkl')
        with open(dst_filename, 'wb') as curr_dst_file:
            pickle.dump(features, curr_dst_file)

    print(opts_set)

if __name__ == "__main__":
    main()


from ..dba import *
from chinese_calendar import *
import os
import numpy as np
import logging
from catboost import CatBoostRegressor
from sklearn.model_selection import KFold

os.environ['NUMEXPR_MAX_THREADS'] = '16'
# 定义logger日志输出
logger = logging.getLogger(__name__)

class FactoryPredict:
    def __init__(self):
        self.predict_days = 3
        self.history_days = 35
        self.dir_path = "data/生产安排"
        self.date_col = 'DS'
        self.group_col = ['FACTORY_ID', "PRODUCT_ID"]
        self.target_col = "Y"
        self.summary_col = self.group_col + [self.date_col]
        self.plu = 'SELECT PRODUCT_ID FROM PLUVIEW WHERE PRODUCT_CATEGORY_NAME = \'散装食品\''
        self.org = "SELECT FACTORY_ID, ORGCODE STORE_ID FROM TZYORG_YH_PS_MODE"
        self.sql_holiday = """
        SELECT RPTDATE DS, FESTIVALS_NL, FESTIVALS_GL
        FROM ZCB_DAY_INFO
        ORDER BY RPTDATE
        """
        self.unit = """
        SELECT PRODUCT_STORE_ID FACTORY_ID, PRODUCT_ID, PACK_WEGIHT YH_UNIT
        FROM FACTORY_PRODUCT_ENABLE
        """

    def update_data(self, full=True):
        if full:
            date_range = pd.date_range(
                start='2019-01-01', end=datetime.datetime.now())
        else:
            from datetime import date, timedelta
            today = date.today()
            d = date(today.year, today.month, 1) - timedelta(1)
            date_range = pd.date_range(start=date(
                d.year, d.month, 1), end=datetime.datetime.now())
        df_date = pd.DataFrame(date_range, columns=["DS"])
        df_date["YEAR"] = df_date["DS"].dt.year
        df_date["MONTH"] = df_date["DS"].dt.month
        try:
            os.makedirs("data/生产安排")
        except FileExistsError:
            logger.info(f"Path exists: {self.dir_path} ")
        for (year, month), g_data in df_date.groupby(["YEAR", "MONTH"], as_index=False):
            start_date, end_date = g_data["DS"].min(), g_data["DS"].max()
            sql_ps = f"""
            SELECT ORGCODE STORE_ID,
                    PLUCODE PRODUCT_ID,
                    RPTDATE DS,
                    PLUCOUNT Y
            FROM TZYPLUPRO_PS
            WHERE RPTDATE >= TO_DATE('{start_date}','yyyy-mm-dd hh24:mi:ss') AND RPTDATE <= TO_DATE('{end_date}','yyyy-mm-dd hh24:mi:ss')
            """
            df_ps = oracle97.read(sql_ps)
            df_ps.to_feather(os.path.join(
                self.dir_path, "%04d%02d.feather" % (year, month)))

    def get_data(self):
        df_plu = oracle97.read(self.plu)
        df_org = oracle97.read(self.org)
        df_input = []
        if len(os.listdir(self.dir_path)) < 7:
            self.update_data(full=True)
        for file in os.listdir(self.dir_path):
            df_part = pd.read_feather(os.path.join(self.dir_path, file))
            df_part = df_part.merge(df_org, how="left", left_on=[
                                    "STORE_ID"], right_on=["STORE_ID"])
            df_part = df_part[df_part["PRODUCT_ID"].isin(df_plu["PRODUCT_ID"])]
            df_part = df_part.groupby(self.summary_col, as_index=False).sum()
            df_input.append(df_part)
        df_input = pd.concat(df_input, ignore_index=True, sort=False)
        df_input = df_input[df_input.groupby(
            ["FACTORY_ID", "PRODUCT_ID"]).Y.transform("mean") > 100]
        df_input = df_input[df_input.groupby(
            ["FACTORY_ID", "PRODUCT_ID"]).Y.transform("std") > 1]
        df_input = df_input.sort_values(
            by=self.summary_col).reset_index(drop=True)
        df_group = df_input.groupby(self.group_col, sort=False)
        df_input['Q10'] = df_group[self.target_col].rolling(
            self.history_days, min_periods=21).quantile(0.1).values
        df_input['Q90'] = df_group[self.target_col].rolling(
            self.history_days, min_periods=21).quantile(0.9).values
        df_input['TRUE'] = df_group[self.target_col].shift(
            -self.predict_days).values
        df_input['TRUE_STD'] = (
            df_input['TRUE'] - df_input['Q10']) / (df_input['Q90'] - df_input['Q10'])
        df_input[self.date_col] = df_input[self.date_col] + \
            datetime.timedelta(days=self.predict_days)
        return df_input

    def get_holiday(self, df_input):
        df_holiday = Oracle().read(self.sql_holiday)
        for var in ['FESTIVALS_NL', 'FESTIVALS_GL']:
            for day_num in range(-1, 2):
                if day_num > 0:
                    df_holiday["%s(t-%02d)" % (var, day_num)
                               ] = df_holiday[var].shift(day_num)
                if day_num < 0:
                    df_holiday["%s(t+%02d)" % (var, np.abs(day_num))
                               ] = df_holiday[var].shift(day_num)
        df_holiday["HOLIDAY"] = df_holiday['DS'].apply(
            is_holiday).replace([True, False], ["1", "0"])
        df_holiday["HOLIDAY_T1"] = df_holiday["HOLIDAY"].shift(-1).fillna("0")
        df_holiday["HOLIDAY_T-1"] = df_holiday["HOLIDAY"].shift(1).fillna("0")
        df_holiday["WEEK_ADJ"] = df_holiday['DS'].dt.month.astype(
            str) + df_holiday["HOLIDAY_T-1"] + df_holiday["HOLIDAY"] + df_holiday["HOLIDAY_T1"]
        df_holiday = df_holiday.drop(
            ['HOLIDAY', 'HOLIDAY_T1', 'HOLIDAY_T-1'], axis=1)

        df_input = df_input.merge(
            df_holiday, how="left", left_on=self.date_col, right_on=self.date_col)
        df_input["HOLIDAY_AFFECT"] = np.nan
        for var in df_holiday.columns[1:]:
            df_input[var] = df_input.groupby(
                self.group_col + [var], sort=False)["TRUE_STD"].transform('mean').values
            df_input[var] = df_input[var].fillna(
                df_input.groupby(self.date_col)[var].transform('mean'))
            df_input["HOLIDAY_AFFECT"] = np.where(df_input["HOLIDAY_AFFECT"].isnull(),
                                                  df_input[var].values,
                                                  df_input["HOLIDAY_AFFECT"].values)
        df_input = df_input.sort_values(
            by=self.group_col + ["WEEK_ADJ", "DS"]).reset_index(drop=True)
        for day in [1, 2, 3, 4]:
            df_input["T%02d" % day] = df_input.groupby(
                self.group_col + ["WEEK_ADJ"])["TRUE_STD"].shift(day)
        df_input = df_input.sort_values(
            by=self.summary_col).reset_index(drop=True)
        return df_input

    def predict(self, df_input):
        feature = [x for x in df_input.columns if
                   x not in self.summary_col + ['Q10', 'Q90', 'TRUE', 'TRUE_STD', 'Y']]
        df_input = df_input.replace([np.inf, np.nan])
        x = df_input.dropna(subset=["TRUE_STD"])[feature].values
        y = df_input.dropna(subset=["TRUE_STD"])['TRUE_STD'].values
        y_predict = np.zeros(len(df_input)).flatten()
        kf = KFold(5, shuffle=True, random_state=1)
        for k, (train_index, test_index) in enumerate(kf.split(x, y)):
            x_train, x_test = x[train_index], x[test_index]
            y_train, y_test = y[train_index], y[test_index]
            model = CatBoostRegressor(iterations=500,
                                      task_type="GPU",
                                      loss_function='MAE',
                                      min_data_in_leaf=10,
                                      use_best_model=True)
            model.fit(
                x_train, y_train,
                eval_set=(x, y),
                early_stopping_rounds=500,
                verbose=0
            )
            y_predict = model.predict(
                df_input[feature]).flatten() / 5 + y_predict
        # 做强输出控制
        df_input["PREDICT"] = y_predict * \
            (df_input["Q90"] - df_input["Q10"]) + df_input["Q10"]
        df_input["PREDICT"] = np.maximum(df_input["PREDICT"], 0)
        df_input["TRUE"] = df_input["TRUE_STD"] * \
            (df_input["Q90"] - df_input["Q10"]) + df_input["Q10"]
        return df_input

    def run(self):
        self.update_data(full=False)
        df_input = self.get_data()
        df_input = self.get_holiday(df_input)
        df_input = self.predict(df_input)
        df_input = df_input[["FACTORY_ID", "PRODUCT_ID",
                             "DS", "TRUE", "PREDICT", "HOLIDAY_AFFECT"]]
        df_unit = oracle97.read(self.unit)
        df_yh = df_input.merge(df_unit, how="inner",
                               left_on=self.group_col, right_on=self.group_col)
        df_yh.columns = ["FACTORY_ID", "PLUCODE", "RPTDATE",
                         "TRUE", "PREDICT", "HOLIDAY_AFFECT", "YH_UNIT"]
        df_yh = df_yh[df_yh['RPTDATE'] >
                      df_yh['RPTDATE'].max() - datetime.timedelta(days=94)]
        df_yh["SUPPORT"] = df_yh['RPTDATE'] - pd.to_datetime("1899-12-30")
        df_yh["SUPPORT"] = df_yh["SUPPORT"].dt.days
        df_yh["SUPPORT"] = df_yh["SUPPORT"].astype(str)
        df_yh["SUPPORT"] = df_yh["SUPPORT"] + df_yh["PLUCODE"].astype(str)
        df_yh['PREDICT'] = np.ceil(df_yh['PREDICT'] / df_yh['YH_UNIT'])
        df_yh['TRUE'] = np.ceil(df_yh['TRUE'] / df_yh['YH_UNIT'])
        df_yh["ERROR"] = df_yh["PREDICT"].fillna(0) - df_yh['TRUE'].fillna(0)
        df_yh["ERROR"] = np.where(df_yh['TRUE'] > 10, df_yh["ERROR"], np.nan)
        df_yh["ABS_ERROR"] = np.abs(df_yh["ERROR"])
        df_yh["ERROR_PERCENTAGE"] = df_yh["ERROR"].values / np.where(df_yh['TRUE'].values == 0,
                                                                     df_yh['TRUE'].values + 1,
                                                                     df_yh['TRUE'].values)
        df_yh["ABS_ERROR_PERCENTAGE"] = np.abs(df_yh["ERROR_PERCENTAGE"])
        df_yh["SAFE_STOCK"] = np.abs(
            df_yh.groupby(['FACTORY_ID', 'PLUCODE', 'HOLIDAY_AFFECT'])["ERROR_PERCENTAGE"].transform('mean').fillna(
                0.05))
        df_yh["SAFE_STOCK"] = np.where(
            df_yh["SAFE_STOCK"] > 0.2, 0.2, df_yh["SAFE_STOCK"])
        df_yh["SAFE_STOCK"] = np.where(
            df_yh["SAFE_STOCK"] < 0.05, 0.05, df_yh["SAFE_STOCK"])
        df_yh["SAFE_STOCK"] = df_yh["PREDICT"] / df_yh["YH_UNIT"] * df_yh["HOLIDAY_AFFECT"] * df_yh[
            "SAFE_STOCK"]  # * df_yh['YH_STORE_NUM']
        df_yh["SAFE_STOCK"] = df_yh["SAFE_STOCK"].apply(lambda x: max(x, 1))
        df_yh["SAFE_STOCK"] = 0
        df_yh["ERROR1"] = df_yh["PREDICT"].fillna(
            0) + df_yh["SAFE_STOCK"] - df_yh['TRUE'].fillna(0)
        df_yh["ERROR1"] = np.where(df_yh['TRUE'] > 1, df_yh["ERROR1"], np.nan)
        df_yh["ABS_ERROR1"] = np.abs(df_yh["ERROR1"])
        df_yh["综合波动"] = df_yh["ERROR1"].values / np.where(df_yh['TRUE'].values == 0,
                                                          df_yh['TRUE'].values + 1,
                                                          df_yh['TRUE'].values)
        df_yh = df_yh.replace([np.inf, -np.inf], [0, 0])
        df_yh['EXEDATE'] = pd.to_datetime(
            datetime.datetime.today().strftime('%Y-%m-%d'))
        df_yh = df_yh[df_yh['RPTDATE'] >
                      df_yh['RPTDATE'].max() - datetime.timedelta(days=34)]
        df_label = []
        for g_name, g_data in df_yh[(df_yh["TRUE"] > 1)].groupby(["FACTORY_ID", "PLUCODE"]):
            error_percentage = g_data["综合波动"].values
            error_percentage = np.sort(error_percentage)
            error_percentage = error_percentage[3:-3]
            count_small = np.sum(error_percentage < -0.05) / len(g_data)
            count_large = np.sum(error_percentage > 0.13) / len(g_data)
            error_sum = (np.sum(g_data["PREDICT"] + g_data["SAFE_STOCK"]) - np.sum(g_data["TRUE"])) / np.where(
                np.sum(g_data["TRUE"]) > 0, np.sum(g_data["TRUE"]), 1)
            if (count_small < 0.15) & (count_large < 0.15) & (error_sum > 0) & (error_sum < 0.1):
                label = "A"
            elif (count_small < 0.3) & (count_large < 0.3) & (error_sum > 0.1) & (error_sum < 0.2):
                label = "B"
            else:
                label = "C"
            df_label.append([g_name[0], g_name[1], label])
        df_label = pd.DataFrame(
            df_label, columns=["FACTORY_ID", "PLUCODE", "LABEL"])
        df_yh = df_yh.merge(df_label, how="left", left_on=[
                            "FACTORY_ID", "PLUCODE"], right_on=["FACTORY_ID", "PLUCODE"])
        df_yh['LABEL'] = df_yh['LABEL'].fillna("空")
        df_yh = df_yh[["FACTORY_ID", "PLUCODE", "RPTDATE", "SUPPORT", "TRUE", "PREDICT", "HOLIDAY_AFFECT", "SAFE_STOCK",
                       "EXEDATE", "ERROR", "ABS_ERROR", "ERROR_PERCENTAGE", "ABS_ERROR_PERCENTAGE", "LABEL", "综合波动",
                       "YH_UNIT"]]
        df_factory_yh = df_yh.drop("综合波动", axis=1)
        Oracle().write_dataframe(key_col=["FACTORY_ID", "PLUCODE", "RPTDATE"],
                                 table_name="ZCB_PREDICT_FACTORY_YH",
                                 df=df_factory_yh,
                                 add_ts=False)
        return df_yh


import argparse

def main(model_name, layers, acts_name, version):
    print(model_name)
    print(layers)
    print(acts_name)
    print(version)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Save activations')
    parser.add_argument('model_name', type=str,
                        help='Specify the model of which activations are saved')
    parser.add_argument('layers', type=int, nargs='+',
                        help='Specify the layers of which activations are saved')
    parser.add_argument('acts_name', type=str,
                        help='Specify what activations are saved')
    parser.add_argument('version', type=int,
                        help='Version number. Error will be raised if the activations for the specified version' \
                             'already exists and --overwrite flag is not provided.')
    args = parser.parse_args()
    main(args.model_name, args.layers, args.acts_name, args.version)


# Matthew Hilton
# July 6th, 2021
'''
Module: Vector.py

This is the port of the Vector module in the C++ Standard
Template Library. Anything involving pointers or move semantics
Are either modified or removed. Const iterators are also removed.
However vector.data() has been implemented
'''
from typing import Any, List, Tuple

class Vector():

    '''
    This is the main Vector class. It is statically typed but dynamically sized. The type of the
    items in this vector are guarenteed to all be the same. For more information visit:

    https://www.cplusplus.com/reference/vector/vector/
    '''

    __type: type = None
    __data: List = []
    __size: int = 0

    class ConstructorFailure(Exception):
        '''
        Rasied if the constructor does not receive the proper arguments
        '''

    class OutOfRangeError(Exception):
        '''
        Rasied if the user attempts to access not in range of the Vector
        '''

    class ArgumentError(Exception):
        '''
        Rasied if a memeber function does not receive thee proper arguments
        '''

    class VectorIterator:
        '''
        An iterator class for vector
        '''

        def __init__(self, vector: 'Vector', index: int = -1) -> None:
            self.__vector: 'Vector' = vector
            self.__size: int = len(vector)
            self.__index: int = index

        def __next__(self) -> None:
            self.__size: int = len(self.__vector)
            self.__index += 1
            if self.__index < self.__size:
                return self.__vector[self.__index]
            raise StopIteration

    class VectorReverseIterator:
        '''
        An iterator class for vector that will iterate backwards
        '''

        def __init__(self, vector: 'Vector', end: bool = False) -> None:
            self.__vector: 'Vector' = vector
            self.__size: int = len(vector)
            self.__index: int = self.__size if end is False else 0

        def __next__(self) -> None:
            self.__size: int = len(self.__vector)
            self.__index -= 1
            if self.__index > -1:
                return self.__vector[self.__index]
            raise StopIteration

    def __init__(self, *args) -> None:
        '''
        Written to work similar to C++11's std::vector.

        Parameters:
            * Type:    Sets the type of vector. Must be supplied with
                       no other arguments
            * Vector:  Must be specified with no other arguments. If
                       a vector is passed, then the it will copy the
                       contents of the passed vector to the new vector.
            * List of
              values:  A list obj of values that are to be added to the
                       vector. All items in the list must match the
                       type of the first element of the list.
            * Size:    Ammount to fill vector. Must be in the first
                       position
            * Value:   Value to fill vector. Must be the last argument
                       requires a size
        '''
        # Empty Ctor
        if len(args) == 0:
            self.__type = None
            self.__data = []
            self.__size = 0
            return

        # Copy ctor
        if len(args) == 1 and isinstance(args[0], Vector):
            self.__type = args[0].__type
            self.__size = len(args[0])
            self.__data = args[0].__data
            return

        # Type instantiation ctor
        if len(args) == 1 and isinstance(args[0], type):
            self.__type = args[0]
            self.__data = []
            self.__size = 0
            return

        # List obj Ctor
        if len(args) == 1 and isinstance(args[0], list):
            self.__type = type(args[0][0])
            self.__size = len(args[0])
            self.__data = []
            for val in args[0]:
                if not isinstance(val, self.__type):
                    raise TypeError(
                        f"Vector Error (init): Value '{val}' is of type '{type(val)}' and is not \
                            of type '{self.__type}'")
                self.__data.append(val)
            return

        # Fill ctor
        if len(args) == 2 and isinstance(args[0], int):
            self.__type = type(args[1])
            self.__size = args[0]
            self.__data = [args[1] for _ in range(self.__size)]
            return

        # Fallthrough Error
        raise self.ConstructorFailure(
            f"Vector Error (init): Invalid Argument(s). ({args})")

    # Python Magic Methods
    # Magic Python Method to allow for len(vector)
    #
    # Returns the length of the vector
    def __len__(self) -> int:
        return self.__size

    # Magic Python Method that customizes the representation of print(vector)
    def __str__(self) -> str:
        return self.__data.__str__()

    # Magic Python Method that overloads the == method
    def __eq__(self, other: 'Vector') -> bool:
        if self.__size == other.__size and self.__type is other.__type:
            return self.__data == other.__data
        return False

    def type(self) -> type:
        '''Hidden Helper that returns the type of the vector'''
        return self.__type

    # Iterator Methods
    # rbegin, rend, cbegin, cend, crbegin, crend = delete

    def __iter__(self) -> Any:
        '''Magic Python Method that allows for iter(vector)
           Useful for foreach type for loops (ie: for elem in vector)'''
        return self.VectorIterator(self)

    def begin(self) -> Any:
        '''Returns the iterator to the first element in the vector'''
        return self.VectorIterator(self)

    def end(self) -> Any:
        '''Returns the iterator to the last element in the vector'''
        return self.VectorIterator(self, self.__size - 1)

    def rbegin(self) -> 'VectorReverseIterator':
        '''
        Returns the iterator to the last element in the vector.

        When next is called, you will go to the previous element
        in the vector.
        '''
        return self.VectorReverseIterator(self)

    def rend(self) -> 'VectorReverseIterator':
        '''
        Returns the iterator to the last element in the vector

        When next is called, you will go to the previous element
        in the vector.
        '''
        return self.VectorReverseIterator(self, end=True)

    # Capacity Methods

    # The following methods have not been implemented because the underlying
    # data structure is a list, not a statically sized array:
    #
    # max_size()
    # resize()
    # capacity()
    # reserve()
    # shrink_to_fit()

    def size(self) -> int:
        '''Returns the number of elements in the vector'''
        return self.__size

    def empty(self) -> bool:
        '''Checks to see if the vector is empty or not'''
        return self.__size == 0

    # Element Access
    # The following two dunder methods effectivly overload the [] operator

    # Magic Python Method that overloads operator[] access
    #
    # Returns the value at the given index
    def __getitem__(self, i: int) -> Any:
        if i >= self.__size:
            raise IndexError(
                f"VectorError (Operator[]): Index '{i}' is not in the bounds of the Vector. \
                    (Max index: {self.__size - 1})")
        return self.__data[i]

    # Magic Python Method that overloads operator[] assign
    #
    # Sets the value in the vector to the passed value at the passed index
    def __setitem__(self, i: int, value: Any) -> None:
        if isinstance(None, self.__type):
            self.__type = type(value)
        if i >= self.__size:
            raise IndexError(
                f"VectorError (Operator[]): Index '{i}' is not in the bounds of the Vector. \
                    (Max index: {self.__size - 1})")
        if not isinstance(value, self.__type):
            raise TypeError(
                f"VectorError (Operator[]): Value '{value}' is of type '{type(value)}' when type \
                    '{self.__type}' was expected.")
        self.__data[i] = value

    # Simialar to the previous Dunder methods; please just use the [] operator
    def at(self, i: int, *value: Any) -> Any:
        '''Returns the element at the given location. Will not accept assignments
           (ie: vector.at() = value) however if you want to set the value at
           that location add the value as an argument (ie: vector.at(index, value))'''
        if len(value) == 0:
            return self.__getitem__(i)

        if len(value) == 1:
            self.__setitem__(i, value[0])
            return None

        raise self.ArgumentError(
            f"Vector Error (at): Invalid Arguments. Expected Either an index or an index and a \
                value. (args: {i}, {value})")

    def front(self) -> Any:
        '''Returns the first element in the vector'''
        if self.__size < 1:
            raise self.OutOfRangeError("Vector Error (front): Vector is empty")
        return self.__data[0]

    def back(self) -> Any:
        '''Returns the last element in the vector'''
        if self.__size < 1:
            raise self.OutOfRangeError("Vector Error (back): Vector is empty")
        return self.__data[-1]

    def data(self) -> Tuple[Any]:
        '''
        Returns the underlying list as a tuple to maintain the state of the vector when this
        method is called.

        Order of elements is preserved.
        '''
        return tuple(self.__data)

    # Modifiers
    # These functions are not implemented becuase
    # assign(): Relates to iterables. Iterables are
    #           fundamentally different in python than c++

    def push_back(self, val: Any) -> None:
        '''
        Adds an element to the end of the vector.

        Raises a type error if passed value is not of the same type as the Vector
        '''
        if not isinstance(val, self.__type):
            raise TypeError(
                f"Vector Error (push_back): Value '{val}' is of type '{type(val)}' and is not of \
                    type '{self.__type}'")
        self.__data.append(val)
        self.__size += 1

    def pop_back(self) -> None:
        '''
        Pops of (removes) the last element in the vector

        If the vector is empty, raises Vector.EmptyError
        '''
        if self.__size < 1:
            raise self.OutOfRangeError(
                "Vector Error (pop_back): Unable to pop back the last element, Vector is empty!")
        self.__data.pop()
        self.__size -= 1

    def insert(self, i: int, val: Any) -> None:
        '''
        Inserts the value (val) at the location (i)

        i must be 0 <= i < Vector.size()
        val must be of the same type as the other elements in the vector
        '''

        if i < 0 or i >= self.__size:
            raise self.OutOfRangeError(
                f"Vector Error (insert): Invalid Index (Value: {i} is not 0 <= i < {self.__size})")

        if not isinstance(val, self.__type):
            raise TypeError(
                f"Vector Error (insert): Value '{val}' is of type '{type(val)}' and is not of \
                    type '{self.__type}'")

        self.__data.insert(i, val)
        self.__size += 1

    def erase(self, i: int, r: int = 1) -> None:
        '''
        Deletes the value at the location (i)

        i must be 0 <= i < Vector.size()
        '''
        if i < 0 or i >= self.__size:
            raise self.OutOfRangeError(
                f"Vector Error (insert): Invalid Index (Value: {i} is not 0 <= i < {self.__size})")

        if i + r > self.__size:
            raise self.OutOfRangeError(
                f"Vector Error (insert): Invalid Range, index + range must not exceed \
                    vector.size! ({i} + {r} > {self.__size})")

        for _ in range(r):
            self.__data.pop(i)
            self.__size -= 1

    def swap(self, v: 'Vector') -> None:
        '''Swaps the LHS elements with the RHS elements'''
        # This version has excpetions due to the fact that there are no compile
        # time errors for type checking and bounds checking.
        if self.__type is not v.__type:
            raise TypeError(
                f"ArrayError (swap): LHS vector is of type '{self.__type}' and RHS vector is of \
                    type '{v.__type}.'")

        # Generate temp list
        temp_li = self.__data

        # Clear values in lists, then -replace them with the items in the other list
        self.__data.clear()
        for item in v.__data:
            self.__data.append(item)

        v.__data.clear()
        for item in temp_li:
            v.__data.append(item)

        # Setting the sizes of the respective vectors.
        temp = self.__size
        self.__size = v.__size
        v.__size = temp

    def clear(self) -> None:
        '''
        Clears the vector of any items
        '''
        self.__size = 0
        self.__data.clear()

    def emplace(self, i: int, *args) -> None:
        '''
        Creates a new obj based on Vector.type and inserts it into the Vector at
        index (i).

        If i is not in range, this function will raise an OutOfRangeError
        '''
        if i < 0 or i >= self.__size:
            raise self.OutOfRangeError(
                f"Vector Error (emplace): Invalid Index (Value: {i} is not 0 <= i < {self.__size})")

        self.__data.insert(i, self.__type(*args))
        self.__size += 1

    def emplace_back(self, *args) -> None:
        '''
        Creates a new obj based on Vector.type and appends it to the end of the vector
        '''
        self.__data.append(self.__type(*args))
        self.__size += 1


import decimal
import functools
import json
import jsonschema
import lxml.etree as etree
import os
import re

import reframe as rfm
import reframe.core.exceptions as errors
import reframe.utility.jsonext as jsonext
import reframe.utility.osext as osext
from reframe.core.logging import getlogger
from reframe.core.warnings import suppress_deprecations

# The schema data version
# Major version bumps are expected to break the validation of previous schemas

DATA_VERSION = '3.1'
_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')

class _RunReport:
    '''A wrapper to the run report providing some additional functionality'''

    def __init__(self, report):
        self._report = report
        self._fallbacks = []    # fallback reports

        # Index all runs by test case; if a test case has run multiple times,
        # only the last time will be indexed
        self._cases_index = {}
        for run in self._report['runs']:
            for tc in run['testcases']:
                c, p, e = tc['unique_name'], tc['system'], tc['environment']
                self._cases_index[c, p, e] = tc

        # Index also the restored cases
        for tc in self._report['restored_cases']:
            c, p, e = tc['unique_name'], tc['system'], tc['environment']
            self._cases_index[c, p, e] = tc

    def __getitem__(self, key):
        return self._report[key]

    def __getattr__(self, name):
        with suppress_deprecations():
            return getattr(self._report, name)

    def add_fallback(self, report):
        self._fallbacks.append(report)

    def slice(self, prop, when=None, unique=False):
        '''Slice the report on property ``prop``.'''

        if unique:
            returned = set()

        for tc in self._report['runs'][-1]['testcases']:
            val = tc[prop]
            if unique and val in returned:
                continue

            if when is None:
                if unique:
                    returned.add(val)

                yield val
            elif tc[when[0]] == when[1]:
                if unique:
                    returned.add(val)

                yield val

    def case(self, check, part, env):
        c, p, e = check.unique_name, part.fullname, env.name
        ret = self._cases_index.get((c, p, e))
        if ret is None:
            # Look up the case in the fallback reports
            for rpt in self._fallbacks:
                ret = rpt._cases_index.get((c, p, e))
                if ret is not None:
                    break

        return ret

    def restore_dangling(self, graph):
        '''Restore dangling dependencies in graph from the report data.

        Returns the updated graph.
        '''

        restored = []
        for tc, deps in graph.items():
            for d in deps:
                if d not in graph:
                    restored.append(d)
                    self._do_restore(d)

        return graph, restored

    def _do_restore(self, testcase):
        tc = self.case(*testcase)
        if tc is None:
            raise errors.ReframeError(
                f'could not restore testcase {testcase!r}: '
                f'not found in the report files'
            )

        dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
        try:
            with open(dump_file) as fp:
                testcase._check = jsonext.load(fp)
        except (OSError, json.JSONDecodeError) as e:
            raise errors.ReframeError(
                f'could not restore testcase {testcase!r}') from e

def next_report_filename(filepatt, new=True):
    if '{sessionid}' not in filepatt:
        return filepatt

    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
    new_id = -1
    basedir = os.path.dirname(filepatt) or '.'
    for filename in os.listdir(basedir):
        match = re.match(search_patt, filename)
        if match:
            found_id = int(match.group(1))
            new_id = max(found_id, new_id)

    if new:
        new_id += 1

    return filepatt.format(sessionid=new_id)

def _load_report(filename):
    try:
        with open(filename) as fp:
            report = json.load(fp)
    except OSError as e:
        raise errors.ReframeError(
            f'failed to load report file {filename!r}') from e
    except json.JSONDecodeError as e:
        raise errors.ReframeError(
            f'report file {filename!r} is not a valid JSON file') from e

    # Validate the report
    with open(_SCHEMA) as fp:
        schema = json.load(fp)

    try:
        jsonschema.validate(report, schema)
    except jsonschema.ValidationError as e:
        try:
            found_ver = report['session_info']['data_version']
        except KeyError:
            found_ver = 'n/a'

        raise errors.ReframeError(
            f'invalid report {filename!r} '
            f'(required data version: {DATA_VERSION}), found: {found_ver})'
        ) from e

    return _RunReport(report)

def load_report(*filenames):
    primary = filenames[0]
    rpt = _load_report(primary)

    # Add fallback reports
    for f in filenames[1:]:
        rpt.add_fallback(_load_report(f))

    return rpt

def write_report(report, filename, compress=False, link_to_last=False):
    with open(filename, 'w') as fp:
        if compress:
            jsonext.dump(report, fp)
        else:
            jsonext.dump(report, fp, indent=2)
            fp.write('\n')

        if not link_to_last:
            return

        # Add a symlink to the latest report
        basedir = os.path.dirname(filename)
        with osext.change_dir(basedir):
            link_name = 'latest.json'
            create_symlink = functools.partial(
                os.symlink, os.path.basename(filename), link_name
            )
            if not os.path.exists(link_name):
                create_symlink()
            else:
                if os.path.islink(link_name):
                    os.remove(link_name)
                    create_symlink()
                else:
                    getlogger().warning('could not create a symlink '
                                        'to the latest report file: '
                                        'path exists and is not a symlink')

def junit_xml_report(json_report):
    '''Generate a JUnit report from a standard ReFrame JSON report.'''

    xml_testsuites = etree.Element('testsuites')
    for run_id, rfm_run in enumerate(json_report['runs']):
        xml_testsuite = etree.SubElement(
            xml_testsuites, 'testsuite',
            attrib={
                'errors': '0',
                'failures': str(rfm_run['num_failures']),
                'hostname': json_report['session_info']['hostname'],
                'id': str(run_id),
                'name': f'ReFrame run {run_id}',
                'package': 'reframe',
                'tests': str(rfm_run['num_cases']),
                'time': str(json_report['session_info']['time_elapsed']),

                # XSD schema does not like the timezone format, so we remove it
                'timestamp': json_report['session_info']['time_start'][:-5],
            }
        )
        testsuite_properties = etree.SubElement(xml_testsuite, 'properties')
        for tc in rfm_run['testcases']:
            casename = (
                f"{tc['unique_name']}[{tc['system']}, {tc['environment']}]"
            )
            testcase = etree.SubElement(
                xml_testsuite, 'testcase',
                attrib={
                    'classname': tc['filename'],
                    'name': casename,

                    # XSD schema does not like the exponential format and since
                    # we do not want to impose a fixed width, we pass it to
                    # `Decimal` to format it automatically.
                    'time': str(decimal.Decimal(tc['time_total'] or 0)),
                }
            )
            if tc['result'] == 'failure':
                testcase_msg = etree.SubElement(
                    testcase, 'failure', attrib={'type': 'failure',
                                                 'message': tc['fail_phase']}
                )
                testcase_msg.text = f"{tc['fail_phase']}: {tc['fail_reason']}"

        testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out')
        testsuite_stdout.text = ''
        testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err')
        testsuite_stderr.text = ''

    return xml_testsuites

def junit_dump(xml, fp):
    fp.write(
        etree.tostring(xml, encoding='utf8', pretty_print=True,
                       method='xml', xml_declaration=True).decode()
    )