import SimpleITK as sitk
import os
import re

path = "\VSD.Brain_3more.XX.O.OT.54517.mha"
img = sitk.ReadImage((os.getcwd()+path))
array = sitk.GetArrayFromImage(img)
vsdId = re.findall("\d+", path)[1]

# name = "VSD.your_description."+sss+".dcm"
name = "VSD.your_description."+vsdId+".mha"
imgs = sitk.GetImageFromArray(array*255)
sitk.WriteImage(sitk.Cast(img, sitk.sitkUInt8), name)

from loadMHA.readMHATrain import BRATS2015

brats = BRATS2015()

for i in range(2):
    img, label = brats.next_train_batch(20)
    print(i, label.shape, img.shape)


from tkinter import *
import pyperclip
import pyshorteners

def url_shortener():

    shortener = pyshorteners.Shortener()

    while True:
        try:
            url = shortener.tinyurl.short(original_url.get())
            # set global short url
            short_url.set(url)
        except ReadTimeout:
            continue
        break

def copy_url():
    # copy short url to clipboard
    pyperclip.copy(short_url.get())

if __name__ == "__main__":

    root = Tk()
    root.geometry("700x700")
    root.title("Mark's URL Shortener")
    root.configure(bg="#49A")

    original_url = StringVar()
    short_url = StringVar()

    # Pack is used to align widget in the center of the frame
    # padx and pady determine the extra space around the button box
    # padx adds empty space vertically
    # pady adds empty space horizontally

    Label(root, text="Enter the url you would like to shorten",
          font="poppins").pack(pady=5)
    Entry(root, textvariable=original_url, width=100).pack(pady=5)
    Button(root, text="Generate short URL", command=url_shortener).pack(pady=5)

    Label(root, text="Shortened URL", font="poppins").pack(pady=5)
    Entry(root, textvariable=short_url, width=50).pack(pady=5)
    Button(root, text="Copy short URL", command=copy_url).pack(pady=5)

    root.mainloop()


import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from matplotlib import pyplot as plt
from cleverhans.utils_keras import KerasModelWrapper
from cleverhans.attacks import FastGradientMethod
from cleverhans.utils_tf import model_eval
import tensorflow as tf

sess = tf.Session()
keras.backend.set_session(sess)

(x_train, y_train), (x_test, y_test) = mnist.load_data()

#tensorflow expects explicit channel dimension so we add it here (x1)
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)

#as floats
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')

#between 0 and 1
x_train /= 255
x_test /= 255

#categorical classes (vectors of 1x10, one-hot)
y_train = np_utils.to_categorical(y_train, 10)
y_test = np_utils.to_categorical(y_test, 10)

model = Sequential()
model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=32, epochs=1, verbose=1)
score = model.evaluate(x_test, y_test, verbose=0)

wrap = KerasModelWrapper(model)
fgsm = FastGradientMethod(wrap, sess=sess)
fgsm_params = {'eps': 0.3,
               'clip_min': 0.,
               'clip_max': 1.}
x = tf.placeholder(tf.float32, shape=(None, 28, 28, 1))
y = tf.placeholder(tf.float32, shape=(None, 10))
adv_x = fgsm.generate(x, **fgsm_params)
adv_x = tf.stop_gradient(adv_x) #Consider the attack to be constant
preds_adv = model(adv_x)
eval_par = {'batch_size': 32}
acc = model_eval(sess, x, y, preds_adv, x_test, y_test, args=eval_par)
print('Test accuracy on adversarial examples: %0.4f\n' % acc)


#! env python
# -*- coding: utf-8 -*-

import unittest
from typing import List, Optional

# 9.7. Implement the "paint fill" function
# Date: 2021/01/27

def fill(img: List, cache: List, x: int, y: int, new_color: int, old_color: int) -> Optional[List]:
    if img[x][y] == new_color or cache[x][y]:
        if not cache[x][y]:
            cache[x][y] = True
        return None

    cache[x][y] = True

    if img[x][y] == old_color:
        img[x][y] = new_color
        if x > 0:
            fill(img, cache, x - 1, y, new_color, old_color)
        if x < (len(img) - 1):
            fill(img, cache, x + 1, y, new_color, old_color)
        if y > 0:
            fill(img, cache, x, y - 1, new_color, old_color)
        if y < (len(img[0]) - 1):
            fill(img, cache, x, y + 1, new_color, old_color)

    return img

def paint_fill(img: List, x: int, y: int, color: int) -> Optional[List]:
    if img[x][y] == color:
        return img
    old_color = img[x][y]
    cache = [[False] * len(img[0]) for i in range(len(img))]
    ans = list(map(list, img))
    return fill(ans, cache, x, y, color, old_color)

class Test(unittest.TestCase):
    def test_1(self):
        p1 = [[0, 0],
              [0, 2]]
        a1 = [[3, 3],
              [3, 2]]
        self.assertEqual(a1, paint_fill(p1, 0, 0, 3))

        p2 = [[0, 0, 0, 1],
              [0, 1, 0, 2],
              [0, 1, 1, 2]]
        a2 = [[4, 4, 4, 1],
              [4, 1, 4, 2],
              [4, 1, 1, 2]]
        self.assertEqual(a2, paint_fill(p2, 1, 2, 4))

        p3 = [[2, 1, 1, 2, 2],
              [2, 1, 1, 0, 0],
              [2, 1, 2, 1, 2],
              [2, 2, 2, 2, 2]]
        a3 = [[0, 1, 1, 2, 2],
              [0, 1, 1, 0, 0],
              [0, 1, 0, 1, 0],
              [0, 0, 0, 0, 0]]
        self.assertEqual(a3, paint_fill(p3, 2, 2, 0))

        return

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan 14 12:43:32 2022

@author: akash
"""

import pandas as pd
from sklearn.model_selection import cross_val_score, StratifiedKFold, GridSearchCV
from sklearn.neighbors import KNeighborsClassifier

df = pd.read_csv("/media/akash/5E820BDA820BB595/CDAC ACTS/Machine Learning/Cases/Bankruptcy/Bankruptcy.csv")
dum_df = pd.get_dummies(df, drop_first=True)
param = dum_df.iloc[:,2:]
result = dum_df.iloc[:,1]

np_param = param.values
np_result = result.values

params = {'n_neighbors':[1,3,5,7,9,11,13,15,17,19,21,23,25]}
kfold = StratifiedKFold(n_splits=5,shuffle=True,random_state=2022)
knn=KNeighborsClassifier()
cv = GridSearchCV(estimator=knn, param_grid=params, scoring = 'roc_auc', cv=kfold)
cv.fit(np_param, np_result)
pd.cv = pd.DataFrame(cv.cv_results_)

print(cv.best_params_)
print(cv.best_score_)


import math
import random

import pygame as pg

BLACK = (0, 0, 0)
GRAY = (224, 224, 224)
WHITE = (255, 255, 255)
GREEN = (23, 232, 215)
GREEN1 = (172, 255, 112)
# Initialize the pygame
pg.init()

# parameter
NumberOfTracks = 5
weight = 400
high = 750
slowSpeed = 0.01
fastSpeed = 10
space = 1
track = weight / NumberOfTracks
index = 0
boxSpeed = slowSpeed
backgroundTopMargin = 50
boxMoving = backgroundTopMargin - 10

# Box rect size
sizeBox = track - 4 * space

# Box
boxScore = []
boxX = []
boxY = []
positionY = []
# # how many tracks
# for i in range(NumberOfTracks):
#     boxScore.append([])
#     boxX.append([])
#     boxY.append([])
#     positionY.append(high - sizeBox)

# Create the screen
screen = pg.display.set_mode((weight, high))

# Background
background = pg.image.load('image/background.jpg')
background = pg.transform.scale(background, (weight, high))

# font
font_32 = pg.font.Font('freesansbold.ttf', 32)
font_16 = pg.font.Font('freesansbold.ttf', 16)
text_size_x = 9.5
text_size_y = 16
# get score
total_score = 0

# computing length of text
def compute_length_of_text(text):
    return len(str(text)) * text_size_x

# restart game
def restart_game():
    global boxScore, boxY, boxX, positionY, total_score
    boxScore = []
    boxX = []
    boxY = []
    positionY = []
    total_score = 0
    # how many tracks
    for k in range(NumberOfTracks):
        boxScore.append([])
        boxX.append([])
        boxY.append([])
        positionY.append(high - sizeBox)

# get score
def get_score():
    score_str = font_32.render(str(total_score), True, WHITE)
    screen.blit(score_str, (weight / 2 - compute_length_of_text(total_score), text_size_y))

# background
def set_background():
    screen.blit(background, (0, 0))
    for i in range(NumberOfTracks - 1):
        pg.draw.rect(screen, GRAY, pg.Rect((i + 1) * (track - space), backgroundTopMargin, 2 * space, high))
    pg.draw.rect(screen, BLACK, pg.Rect(0, backgroundTopMargin, weight, 5 * space))

# Create Box
def create_box(score, x, y, size_box):
    global text_size_x, text_size_y
    pg.draw.rect(screen, color(score), pg.Rect(x, round(y), size_box, size_box))
    score_str = font_32.render(str(score), True, WHITE)
    screen.blit(score_str, (x + size_box / 2 - compute_length_of_text(score), y + size_box / 2 - text_size_y))
    # pg.display.flip()

# Computer color by score
def color(score):
    green = 229 - score * 2 if 229 - score * 2 > 128 else 128
    blue = 204 - score * 2 if 204 - score * 2 > 0 else 0
    red = 255 - score * 2 if 204 - score * 2 > 0 else 0
    return red, green, blue

# check box collision
# score 分數
# y 座標
# x 座標
# tracks_index 第幾軌道
# item_index 上一個item位置
def is_collision(score, box_y, box_x, tracks_index, item_index):
    global total_score
    if item_index < 0:
        return len(score[tracks_index])
    is_mix = False  # 是否有合併
    high_box = len(score[tracks_index]) - 1
    if high_box < 0:
        return len(score[tracks_index])
    temp_get_score = 0  # 得分
    target = score[tracks_index][high_box]  # 目標分數
    save_index = item_index  # 最後儲存的座標
    # 合併規則
    # EX 2(L)2(T)
    if tracks_index - 1 >= 0 and high_box < len(score[tracks_index - 1]) and target == score[tracks_index - 1][
        high_box]:
        score[tracks_index][high_box] *= 2
        temp_get_score = score[tracks_index][high_box]
        del_box(score, box_x, box_y, tracks_index - 1, high_box)  # delete box
        is_mix = True
    # EX 2(T)2(R)
    if tracks_index + 1 < len(score) and high_box < len(score[tracks_index + 1]) and target == score[tracks_index + 1][
        high_box]:
        score[tracks_index][high_box] *= 2
        temp_get_score = score[tracks_index][high_box]
        del_box(score, box_x, box_y, tracks_index + 1, high_box)  # delete box
        is_mix = True
    # EX:   2(T)
    #       2
    if high_box > 0 and target == score[tracks_index][high_box - 1]:
        score[tracks_index][high_box] *= 2
        score[tracks_index][high_box - 1] = score[tracks_index][high_box]
        del_box(score, box_x, box_y, tracks_index, high_box)  # delete box
        is_mix = True
        save_index -= 1
        temp_get_score = score[tracks_index][high_box - 1]

    total_score += temp_get_score  # 得分
    if not is_mix:
        return len(score[tracks_index])

    return is_collision(score, box_y, box_x, tracks_index, save_index)

# delete box
def del_box(score, box_x, box_y, x, y):
    del score[x][y]
    del box_x[x][y]
    del box_y[x][len(box_y[x]) - 1]

# game over
def is_game_over(box_y):
    global total_score
    if len(box_y) - 1 >= 0 and box_y[len(box_y) - 1] <= sizeBox:
        top_score_text = "Top Score"
        restart_text = "RESTART"
        game_over = font_16.render(top_score_text, True, WHITE)
        restart = font_32.render(restart_text, True, WHITE)
        total = font_32.render(str(total_score), True, WHITE)
        pg.draw.rect(screen, GREEN, pg.Rect(weight / 4, high / 3, weight / 2, high / 4))
        screen.blit(game_over, (weight / 2 - len(str(top_score_text)) * 4.5, high / 2 - 7 * text_size_y))
        screen.blit(total, (weight / 2 - compute_length_of_text(total_score), high / 2 - 5.5 * text_size_y))
        screen.blit(restart, (weight / 2 - compute_length_of_text(restart_text) - 7, high / 2 - 2 * text_size_y))
        return True
    return False

# Game Loop
# restart game
restart_game()
running = True
game_pass = True
initial_score = [int(math.pow(2, random.randint(1, 5))), int(math.pow(2, random.randint(1, 5)))]
while running:
    screen.fill(BLACK)
    set_background()
    # get score
    get_score()
    # Listen event
    for event in pg.event.get():
        # Close screen
        if event.type == pg.QUIT:
            running = False
        # Right Click
        if event.type == pg.MOUSEBUTTONDOWN:
            if game_pass:
                index = int(pg.mouse.get_pos()[0] / track)
                boxSpeed = fastSpeed
            elif 120 < pg.mouse.get_pos()[0] in range(120, 290) and pg.mouse.get_pos()[1] in range(340, 370):
                game_pass = True
                restart_game()

    # Show all of box
    for i in range(len(boxScore)):
        for j in range(len(boxScore[i])):
            create_box(boxScore[i][j], boxX[i][j], boxY[i][j], sizeBox)

    # Show next box
    create_box(initial_score[1], weight / 5, 10, sizeBox / 2)

    # check game over
    if is_game_over(boxY[index]):
        game_pass = False
        pg.display.update()
        continue

    # initial Box moving
    boxMoving += boxSpeed
    positionX = 2 * space + index * track
    create_box(initial_score[0], positionX, boxMoving, sizeBox)

    # initial box and save box
    if positionY[index] < boxMoving:
        # save box
        boxScore[index].append(initial_score[0])
        boxX[index].append(positionX)
        boxY[index].append(positionY[index])
        # initial box
        boxMoving = 50
        boxSpeed = slowSpeed
        initial_score[0]=initial_score[1]
        initial_score[1] = int(math.pow(2, random.randint(1, 5)))

    # check box collision
    number = is_collision(boxScore, boxY, boxX, index, len(boxScore) - 1)
    # 計算boxY軸最高線 軌道高-(目前軌道item數量+1)*box高 - box和box之間的距離
    positionY[index] = high - (number + 1) * sizeBox - 2 * number * space

    # update draw
    # pg.display.flip()
    # update screen
    pg.display.update()


# https://exchangeratesapi.io/
import requests

def main():
    # response = requests.get("https://api.exchangeratesapi.io/latest?base=USD&symbols=GBP")

    prima_valuta = input("Inserisci prima valuta : ")
    seconda_valuta = input("Inserisci seconda valuta : ")

    payload = {'base': prima_valuta, 'symbols': seconda_valuta}
    response = requests.get("https://api.exchangeratesapi.io/latest",
                        params=payload)

    if response.status_code != 200:
        print(f"Status code: {response.status_code}")
        raise Exception("La richiesta API non e' andata a buon fine")

    response_data = response.json()
    tasso_di_cambio = response_data["rates"][seconda_valuta]

    print("Data:", response_data["date"])
    print(f" 1 {prima_valuta} corrisponda a {tasso_di_cambio} {seconda_valuta} ")

if __name__ == "__main__":
    main()


import unittest
from itertools import chain
from typing import Dict, Optional, Tuple
from unittest.mock import MagicMock, patch

import pytest
from confluent_kafka.schema_registry.schema_registry_client import (
    RegisteredSchema,
    Schema,
)

from datahub.configuration.common import ConfigurationError
from datahub.emitter.mce_builder import (
    make_dataplatform_instance_urn,
    make_dataset_urn_with_platform_instance,
)
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.kafka import KafkaSource, KafkaSourceConfig
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.schema_classes import (
    BrowsePathsClass,
    DataPlatformInstanceClass,
    KafkaSchemaClass,
    SchemaMetadataClass,
)

class KafkaSourceTest(unittest.TestCase):
    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_kafka_source_configuration(self, mock_kafka):
        ctx = PipelineContext(run_id="test")
        kafka_source = KafkaSource(
            KafkaSourceConfig.parse_obj({"connection": {"bootstrap": "foobar:9092"}}),
            ctx,
        )
        kafka_source.close()
        assert mock_kafka.call_count == 1

    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_kafka_source_workunits_wildcard_topic(self, mock_kafka):
        mock_kafka_instance = mock_kafka.return_value
        mock_cluster_metadata = MagicMock()
        mock_cluster_metadata.topics = ["foobar", "bazbaz"]
        mock_kafka_instance.list_topics.return_value = mock_cluster_metadata

        ctx = PipelineContext(run_id="test")
        kafka_source = KafkaSource(
            KafkaSourceConfig.parse_obj(
                {"connection": {"bootstrap": "localhost:9092"}}
            ),
            ctx,
        )
        workunits = list(kafka_source.get_workunits())

        first_mce = workunits[0].metadata
        assert isinstance(first_mce, MetadataChangeEvent)
        mock_kafka.assert_called_once()
        mock_kafka_instance.list_topics.assert_called_once()
        assert len(workunits) == 4

    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_kafka_source_workunits_topic_pattern(self, mock_kafka):
        mock_kafka_instance = mock_kafka.return_value
        mock_cluster_metadata = MagicMock()
        mock_cluster_metadata.topics = ["test", "foobar", "bazbaz"]
        mock_kafka_instance.list_topics.return_value = mock_cluster_metadata

        ctx = PipelineContext(run_id="test1")
        kafka_source = KafkaSource.create(
            {
                "topic_patterns": {"allow": ["test"]},
                "connection": {"bootstrap": "localhost:9092"},
            },
            ctx,
        )
        workunits = [w for w in kafka_source.get_workunits()]

        mock_kafka.assert_called_once()
        mock_kafka_instance.list_topics.assert_called_once()
        assert len(workunits) == 2

        mock_cluster_metadata.topics = ["test", "test2", "bazbaz"]
        ctx = PipelineContext(run_id="test2")
        kafka_source = KafkaSource.create(
            {
                "topic_patterns": {"allow": ["test.*"]},
                "connection": {"bootstrap": "localhost:9092"},
            },
            ctx,
        )
        workunits = [w for w in kafka_source.get_workunits()]
        assert len(workunits) == 4

    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_kafka_source_workunits_with_platform_instance(self, mock_kafka):

        PLATFORM_INSTANCE = "kafka_cluster"
        PLATFORM = "kafka"
        TOPIC_NAME = "test"

        mock_kafka_instance = mock_kafka.return_value
        mock_cluster_metadata = MagicMock()
        mock_cluster_metadata.topics = [TOPIC_NAME]
        mock_kafka_instance.list_topics.return_value = mock_cluster_metadata

        ctx = PipelineContext(run_id="test1")
        kafka_source = KafkaSource.create(
            {
                "connection": {"bootstrap": "localhost:9092"},
                "platform_instance": PLATFORM_INSTANCE,
            },
            ctx,
        )
        workunits = [w for w in kafka_source.get_workunits()]

        # We should only have 1 topic + sub-type wu.
        assert len(workunits) == 2
        assert isinstance(workunits[0], MetadataWorkUnit)
        assert isinstance(workunits[0].metadata, MetadataChangeEvent)
        proposed_snap = workunits[0].metadata.proposedSnapshot
        assert proposed_snap.urn == make_dataset_urn_with_platform_instance(
            platform=PLATFORM,
            name=TOPIC_NAME,
            platform_instance=PLATFORM_INSTANCE,
            env="PROD",
        )

        # DataPlatform aspect should be present when platform_instance is configured
        data_platform_aspects = [
            asp
            for asp in proposed_snap.aspects
            if type(asp) == DataPlatformInstanceClass
        ]
        assert len(data_platform_aspects) == 1
        assert data_platform_aspects[0].instance == make_dataplatform_instance_urn(
            PLATFORM, PLATFORM_INSTANCE
        )

        # The default browse path should include the platform_instance value
        browse_path_aspects = [
            asp for asp in proposed_snap.aspects if type(asp) == BrowsePathsClass
        ]
        assert len(browse_path_aspects) == 1
        assert (
            f"/prod/{PLATFORM}/{PLATFORM_INSTANCE}/{TOPIC_NAME}"
            in browse_path_aspects[0].paths
        )

    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_close(self, mock_kafka):
        mock_kafka_instance = mock_kafka.return_value
        ctx = PipelineContext(run_id="test")
        kafka_source = KafkaSource.create(
            {
                "topic_patterns": {"allow": ["test.*"]},
                "connection": {"bootstrap": "localhost:9092"},
            },
            ctx,
        )
        kafka_source.close()
        assert mock_kafka_instance.close.call_count == 1

    def test_kafka_source_stateful_ingestion_requires_platform_instance(
        self,
    ):
        class StatefulProviderMock:
            def __init__(self, config, ctx):
                self.ctx = ctx
                self.config = config

            def is_stateful_ingestion_configured(self):
                return self.config.stateful_ingestion.enabled

        kafka_source_patcher = unittest.mock.patch.object(
            KafkaSource, "__bases__", (StatefulProviderMock,)
        )

        ctx = PipelineContext(run_id="test", pipeline_name="test")
        with pytest.raises(ConfigurationError):
            with kafka_source_patcher:
                # prevent delattr on __bases__ on context __exit__
                kafka_source_patcher.is_local = True
                KafkaSource.create(
                    {
                        "stateful_ingestion": {"enabled": "true"},
                        "connection": {"bootstrap": "localhost:9092"},
                    },
                    ctx,
                )

    @patch(
        "datahub.ingestion.source.kafka.confluent_kafka.schema_registry.schema_registry_client.SchemaRegistryClient",
        autospec=True,
    )
    @patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True)
    def test_kafka_source_workunits_schema_registry_subject_name_strategies(
        self, mock_kafka_consumer, mock_schema_registry_client
    ):
        # Setup the topic to key/value schema mappings for all types of schema registry subject name strategies.
        # <key=topic_name, value=(<key_schema>,<value_schema>)
        topic_subject_schema_map: Dict[
            str, Tuple[RegisteredSchema, RegisteredSchema]
        ] = {
            # TopicNameStrategy is used for subject
            "topic1": (
                RegisteredSchema(
                    schema_id="schema_id_2",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic1Key", "namespace": "test.acryl", "fields": [{"name":"t1key", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="topic1-key",
                    version=1,
                ),
                RegisteredSchema(
                    schema_id="schema_id_1",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic1Value", "namespace": "test.acryl", "fields": [{"name":"t1value", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="topic1-value",
                    version=1,
                ),
            ),
            # RecordNameStrategy is used for subject
            "topic2": (
                RegisteredSchema(
                    schema_id="schema_id_3",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic2Key", "namespace": "test.acryl", "fields": [{"name":"t2key", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="test.acryl.Topic2Key",
                    version=1,
                ),
                RegisteredSchema(
                    schema_id="schema_id_4",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic2Value", "namespace": "test.acryl", "fields": [{"name":"t2value", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="test.acryl.Topic2Value",
                    version=1,
                ),
            ),
            # TopicRecordNameStrategy is used for subject
            "topic3": (
                RegisteredSchema(
                    schema_id="schema_id_4",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic3Key", "namespace": "test.acryl", "fields": [{"name":"t3key", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="topic3-test.acryl.Topic3Key-key",
                    version=1,
                ),
                RegisteredSchema(
                    schema_id="schema_id_5",
                    schema=Schema(
                        schema_str='{"type":"record", "name":"Topic3Value", "namespace": "test.acryl", "fields": [{"name":"t3value", "type": "string"}]}',
                        schema_type="AVRO",
                    ),
                    subject="topic3-test.acryl.Topic3Value-value",
                    version=1,
                ),
            ),
        }

        # Mock the kafka consumer
        mock_kafka_instance = mock_kafka_consumer.return_value
        mock_cluster_metadata = MagicMock()
        mock_cluster_metadata.topics = list(topic_subject_schema_map.keys())
        mock_cluster_metadata.topics.append("schema_less_topic")
        mock_kafka_instance.list_topics.return_value = mock_cluster_metadata

        # Mock the schema registry client
        # - mock get_subjects: all subjects in topic_subject_schema_map
        mock_schema_registry_client.return_value.get_subjects.return_value = [
            v.subject for v in chain(*topic_subject_schema_map.values())
        ]

        # - mock get_latest_version
        def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]:
            for registered_schema in chain(*topic_subject_schema_map.values()):
                if registered_schema.subject == subject_name:
                    return registered_schema
            return None

        mock_schema_registry_client.return_value.get_latest_version = (
            mock_get_latest_version
        )

        # Test the kafka source
        source_config = {
            "connection": {"bootstrap": "localhost:9092"},
            # Setup the topic_subject_map for topic2 which uses RecordNameStrategy
            "topic_subject_map": {
                "topic2-key": "test.acryl.Topic2Key",
                "topic2-value": "test.acryl.Topic2Value",
            },
        }
        ctx = PipelineContext(run_id="test")
        kafka_source = KafkaSource.create(source_config, ctx)
        workunits = list(kafka_source.get_workunits())

        mock_kafka_consumer.assert_called_once()
        mock_kafka_instance.list_topics.assert_called_once()
        assert len(workunits) == 8
        i: int = -1
        for wu in workunits:
            assert isinstance(wu, MetadataWorkUnit)
            if not isinstance(wu.metadata, MetadataChangeEvent):
                continue
            mce: MetadataChangeEvent = wu.metadata
            i += 1

            if i < len(topic_subject_schema_map.keys()):
                # First 3 workunits (topics) must have schemaMetadata aspect
                assert isinstance(mce.proposedSnapshot.aspects[1], SchemaMetadataClass)
                schemaMetadataAspect: SchemaMetadataClass = (
                    mce.proposedSnapshot.aspects[1]
                )
                assert isinstance(schemaMetadataAspect.platformSchema, KafkaSchemaClass)
                # Make sure the schema name is present in topic_subject_schema_map.
                assert schemaMetadataAspect.schemaName in topic_subject_schema_map
                # Make sure the schema_str matches for the key schema.
                assert (
                    schemaMetadataAspect.platformSchema.keySchema
                    == topic_subject_schema_map[schemaMetadataAspect.schemaName][
                        0
                    ].schema.schema_str
                )
                # Make sure the schema_str matches for the value schema.
                assert (
                    schemaMetadataAspect.platformSchema.documentSchema
                    == topic_subject_schema_map[schemaMetadataAspect.schemaName][
                        1
                    ].schema.schema_str
                )
                # Make sure we have 2 fields, one from the key schema & one from the value schema.
                assert len(schemaMetadataAspect.fields) == 2
            else:
                # Last topic('schema_less_topic') has no schema defined in the registry.
                # The schemaMetadata aspect should not be present for this.
                for aspect in mce.proposedSnapshot.aspects:
                    assert not isinstance(aspect, SchemaMetadataClass)


#import necessary modules
import csv
import datetime as dt
#use the dictionary for state abbreviations from Special Hint (thank you!)
us_state_abbrev = {
    'Alabama': 'AL',
    'Alaska': 'AK',
    'Arizona': 'AZ',
    'Arkansas': 'AR',
    'California': 'CA',
    'Colorado': 'CO',
    'Connecticut': 'CT',
    'Delaware': 'DE',
    'Florida': 'FL',
    'Georgia': 'GA',
    'Hawaii': 'HI',
    'Idaho': 'ID',
    'Illinois': 'IL',
    'Indiana': 'IN',
    'Iowa': 'IA',
    'Kansas': 'KS',
    'Kentucky': 'KY',
    'Louisiana': 'LA',
    'Maine': 'ME',
    'Maryland': 'MD',
    'Massachusetts': 'MA',
    'Michigan': 'MI',
    'Minnesota': 'MN',
    'Mississippi': 'MS',
    'Missouri': 'MO',
    'Montana': 'MT',
    'Nebraska': 'NE',
    'Nevada': 'NV',
    'New Hampshire': 'NH',
    'New Jersey': 'NJ',
    'New Mexico': 'NM',
    'New York': 'NY',
    'North Carolina': 'NC',
    'North Dakota': 'ND',
    'Ohio': 'OH',
    'Oklahoma': 'OK',
    'Oregon': 'OR',
    'Pennsylvania': 'PA',
    'Rhode Island': 'RI',
    'South Carolina': 'SC',
    'South Dakota': 'SD',
    'Tennessee': 'TN',
    'Texas': 'TX',
    'Utah': 'UT',
    'Vermont': 'VT',
    'Virginia': 'VA',
    'Washington': 'WA',
    'West Virginia': 'WV',
    'Wisconsin': 'WI',
    'Wyoming': 'WY',
}

def import_reformat_export():
    '''This function requires no parameters imports a (local to script) CSV containing employee data and
    reformats it into an appropriate format then exports it to a new CSV'''
    print("Welcome to the Employee Reformatron 9000!")
    csvname = input("Please enter the name of the CSV file you would like to reformat:  ")
    #Import the employee data file and convert to ordered dict
    with open(csvname,'r',newline='') as csvfile:
        new_list = []
        dictreader = csv.DictReader(csvfile)
        #loop over each row in the csv dictreader
        for row in dictreader:
            #split first and last names into two strings
            first_name, last_name = row['Name'].split(" ")
            #update the dict with two new key:value pairs for each name variable
            row.update([('First Name',first_name)])
            row.update([('Last Name',last_name)])
            #remove the name doublet
            row.pop('Name')
            #extract the DoB to datetime for reformatting
            row_dt = dt.datetime.strptime(row['DOB'],"%Y-%m-%d")
            formatted_date = row_dt.strftime("%m/%d/%Y")
            #add formatted date string back into dictionary
            row.update([('DoB',formatted_date)])
            #remove DOB doublet
            row.pop('DOB')
            #slice into strings to extract the first six characters (5 digits and hyphen)
            first_six_characters = row['SSN'][:6]
            #because the row['SSN'] is immutable, assign the string replace value to a new string variable
            new_SSN = row['SSN'].replace(first_six_characters,'***-**')
            #update the ordered dictionary with the new_SSN string
            row.update([('SSN', new_SSN)])
            #assign abbrev_state the value returned by searching the state/abbrev dict with row['State'] as the key
            abbrev_state = us_state_abbrev[row['State']]
            #update the row with the new abbrev_state value
            row.update([('State',abbrev_state)])
            #add each row into a list that can be read outside this IO event
            new_list.append(row)
        #output the reformatted ordereddict to acsv output file for HR
        with open(csvname.replace('.csv','')+'_output.csv','w') as csvfile:
            fieldnames = ['Emp ID','First Name','Last Name','DoB','SSN','State']
            writer = csv.DictWriter(csvfile, fieldnames)
            writer.writeheader()
            for i in new_list:
                writer.writerow(i)
if __name__ == "__main__":
    import_reformat_export()


from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
from django.core.paginator import Paginator
from django.views.decorators.http import require_GET, require_POST, require_http_methods
from django.views.generic import *
from django.db.models import Q
from django.urls import reverse_lazy
from django.utils.translation import get_language
from .models import Corpora, Genres
from .apps import toGenres, toResult, toModal, toNLPjson, optimizeAbstract, generateID
from CorporaD.execute import *
from CorporaD.CorporaDB import corporaDB
from CorporaD.Global.summa import summarizer
from CorporaD.Global import bigram, sozgebolu
import time, os, re, json
from opencorpora.settings import BASE_DIR

temp_folder = BASE_DIR/'static/main/tmp'
remjs = os.path.join(temp_folder, "remove.js")
nlpjson = os.path.join(temp_folder, "nlp.json")
start = float(0)
end   = float(0)

genres_lang = {'kk':'kz', 'ru':'rus', 'en':'en'}

@require_GET
def index(request):
    til = get_language()
    current_genres_column = genres_lang.get(til, 'kz')
    start = time.monotonic()
    if os.path.exists(remjs):
        os.remove(remjs)
    if os.path.exists(nlpjson):
        os.remove(nlpjson)
    context = {}
    CPI = 5
    genres = Genres.objects.all().order_by(current_genres_column)
    toGenres(genres, lang=til)
    context['genres'] = genres
    context['check_kk'] = True if til == 'kk' else False
    context['check_ru'] = True if til == 'ru' else False
    context['check_en'] = True if til == 'en' else False
    if request.is_ajax or request.method == 'GET':
        if request.GET.get('source', None):
            source = request.GET.get('source', None)
            context['source'] = source
            if request.GET.get('filters', None):
                filters = request.GET.get('filters', None)
                if filters != 'all' or filters != '':
                    context['filters'] = filters
                    filterSid = {}
                    if(filters.find(',')):
                        filters = str(filters).split(',')
                        for genre in filters:
                            p = Genres.objects.filter(Q(kz=genre) | Q(rus=genre) | Q(en=genre))
                            if p:
                                filterSid[str(p[0].pk)] = genre
                    else:
                        p = Genres.objects.filter(Q(kz=filters) | Q(rus=filters) | Q(en=filters))
                        if p:
                            filterSid[str(p[0].pk)] = filters
                    context['filterSid'] = filterSid
                    start = time.monotonic()
                    corp = Corpora.objects.raw("SELECT * FROM corpora WHERE MATCH(title,text,annotation,keywords) AGAINST ('%s' IN IN BOOLEAN MODE) and genre_id in(%s)"%(source,",".join(filterSid.keys())))
                    end = time.monotonic()
                else:
                    start = time.monotonic()
                    corp = Corpora.objects.raw("SELECT * FROM corpora WHERE MATCH(title,text,annotation,keywords) AGAINST ('%s' IN IN BOOLEAN MODE)"%source)
                    end = time.monotonic()

            corp = Corpora.objects.raw("SELECT * FROM corpora WHERE MATCH(title,text,annotation,keywords) AGAINST ('%s' IN BOOLEAN MODE)"%source)
            if len(corp) == 0:
                context['results'] = None
                return render(request, 'main/index.html', context)
            elif len(corp) < CPI:
                CPI = len(corp)
            else:
                context['countitem'] = len(corp)
                paginator = Paginator(corp, CPI)
                if 'page' in request.GET:
                    page_num = request.GET['page']
                else:
                    page_num = 1
                page = paginator.get_page(page_num)
                context['page_obj'] = page
                if page.object_list:
                    cid         = generateID(CPI)
                    title       = [obj.title for obj in page.object_list]
                    text        = [obj.text for obj in page.object_list]
                    lemmas      = [obj.lemmas for obj in page.object_list]
                    morphanaliz = [obj.morphanaliz for obj in page.object_list]
                    keywords    = [obj.keywords for obj in page.object_list]
                    annotation  = [obj.annotation for obj in page.object_list]
                    url         = [obj.url for obj in page.object_list]
                    context['results']   = toResult(CPI= CPI, cid= cid, text=text, title= title, keywords= keywords, annotation= annotation, url= url, lang= til)
                    toNLPjson(CPI= CPI, cid= cid, title= title, text= text, lemmas= lemmas, morphanaliz= morphanaliz, annotation= annotation)
            end = time.monotonic()
            context['dttime'] = round(end-start, 5)
            context['null'] = False
            return render(request, 'main/index.html', context)
        context['null'] = True
        return render(request, 'main/index.html', context)
    return render(request, 'main/index.html', context)

@require_POST
def ressumma(request):
    if request.is_ajax and request.method == 'POST':
        if request.POST.get('id', None) and request.POST.get('type', None):
            changeid = request.POST['id']
            types = request.POST['type']
            data = {}
            with open(nlpjson) as f:
                myjson = json.load(f)
            try:
                res = myjson[changeid]
            except KeyError:
                render(request, 'main/index.html')
            data['id'] = changeid
            response = ""
            annotation = res['annotation']
            mydict = {
                'kk': ' Жабу.',
                'en': ' Close.',
                'ru': ' Закрыть.',
                'kk1': 'ары қарай оқу',
                'en1': 'read more',
                'ru1': 'читать далее'
            }
            if types == 'show':
                a = str(annotation) if str(annotation).endswith('.') else str(annotation) + ".\t"
                response = a + ''+mydict[get_language()]+''
            else:
                if ('' or '\n' or ' ' or None) != annotation :
                    obj = optimizeAbstract(annotation)
                else:
                    obj = optimizeAbstract(res['text'])
                response = obj.shortAbstract +''+mydict[get_language()+'1']+''
            data['summa'] = response
            return JsonResponse(data=data)
    return render(request, 'main/index.html')

@require_POST
def handlerpost(request):
    if request.method == 'POST':
        if request.POST.get('id', None):
            changeid = request.POST.get('id')
            data = {}
            with open(nlpjson) as f:
                myjson = json.load(f)
            res = myjson[changeid]
            data['token'] = toModal(res['token'])
            data['lemmas'] = toModal(res['lemmas'])
            data['morphanaliz'] = toModal(res['morphanaliz'])
            data['id'] = changeid
            return JsonResponse(data=data)
    return render(request, 'main/index.html')

@require_POST
def download(request):
    if request.is_ajax and request.method == 'POST':
        if request.POST.get('id', None) and request.POST.get('type', None):
            changeid = request.POST.get('id')
            types = request.POST.get('type', None)
            data = {}
            with open(nlpjson) as f:
                myjson = json.load(f)
            res = myjson[changeid]
            data['title'] = res['title']
            data['fileval'] = res[types]
            return JsonResponse(data=data)
    return render(request, 'main/index.html')

@require_POST
def execpost(request):
    context = {}
    if request.POST.get('text', None) and request.POST.get('text') != '' and request.POST.get('text') != '\n':
        input_val = request.POST['text']
        keywords = ""
        annotation = ""
        lemmatization = ""
        morphanaliz = ""
        #connect db
        ob = corporaDB()
        lencorp = int(ob.Count_corpora()[0]['sany'])

        with open(os.path.join(BASE_DIR,'CorporaD/tmp/text.tmp'),'w',encoding="utf-8") as f:
            f.write(input_val)
        os.system('''cd $HOME/sources/apertium-kaz-rus\ncat "{0}" | apertium -n -d. kaz-rus-tagger > "{1}"'''.format(os.path.join(BASE_DIR,'CorporaD/tmp/text.tmp'), os.path.join(BASE_DIR,'CorporaD/tmp/app.tmp')))
        text = open(os.path.join(papka_korpus,'tmp/text.tmp'),'r',encoding="utf8").read()
        apertium = open(os.path.join(papka_korpus,'tmp/app.tmp'),'r',encoding="utf-8").read()
        morphanaliz = str(EditedApertium_DB(text = text, apertium = apertium))
        outtexts = str(outtexts_DB(aptext = str(morphanaliz)))
        lemmatization = str(train_DB(outtexts=str(outtexts)))
        annotation = summarizer.summarize(text, language="kazakh")
        soz = sozgebolu(outtexts)

        #TF_IDF klasssyndagy konstruktordy qoldanyluy
        TfIdf = tf_idf(text = soz, len_corp = lencorp, objectCorporaDB = ob, length_keywords=5)
        #esepteuler
        tf = TfIdf.tf_esepteu()
        idf = TfIdf.idf_esepteu()
        tfidf = TfIdf.tf_idf_esepteu()

        #bigram klasssyndagy konstruktordy qoldanyluy
        bi = bigram(text = str(outtexts))
        bi_text = [bi.newlemm, bi.lastlemm]

        #bigram TF_IDF klasssyndagy konstruktordy qoldanyluy
        BiTfIdf = bi_tf_idf(text = bi_text, len_corp = lencorp, objectCorporaDB = ob, length_keywords=5)
        bi_tf = BiTfIdf.bi_tf_esepteu()
        bi_idf = BiTfIdf.bi_idf_esepteu()
        bi_tfidf = BiTfIdf.bi_tf_idf_esepteu()
        spis = []
        for x in tfidf:
            spis.append(str(x))
        for x in bi_tfidf:
            spis.append(str(x))

        keywords = ", ".join(spis)

        context['keywords'] = toModal(keywords)
        context['annotation'] = toModal(annotation)
        context['lemma'] = toModal(lemmatization)
        context['morph'] = toModal(morphanaliz)
        context['text'] = input_val
    return JsonResponse(context)

@require_GET
def execute(request):
    return render(request, 'main/execute.html')

@require_GET
def about(request):
    return render(request, 'main/about.html')

@require_GET
def privacy(request):
    return render(request, 'main/privacy.html')


# from notebooks.opencv_install.opencv_contrib.modules.dnn_objdetect.scripts.pascal_preprocess import rescale
import PIL
import numpy
from numpy.core.fromnumeric import argmax
import pandas as pd
import open3d as o3d
import random
import math
import re
import h5py
import io
import multiprocessing

# from multiprocessing import Pool
from multiprocessing import get_context

# multiprocessing.set_start_method('spawn') #Fork does not play nice with h5py!
# Reference: https://pythonspeed.com/articles/python-multiprocessing/

import numpy as np
import cupy as cp
import os

from tqdm import tqdm

from deco import concurrent, synchronized
from tqdm.utils import RE_ANSI

import siampose.data.objectron.data_transforms
import siampose.data.objectron.sequence_parser
import siampose.data.utils
import siampose.geometry as geo

from siampose.data.objectron.dataset import iou
from siampose.data.objectron.dataset import box

from google.protobuf import text_format

# The annotations are stored in protocol buffer format.
from siampose.data.objectron.schema import annotation_data_pb2 as annotation_protocol

# The AR Metadata captured with each frame in the video
from siampose.data.objectron.schema import (
    a_r_capture_metadata_pb2 as ar_metadata_protocol,
)

from PIL import Image, ImageDraw

EDGES = (
    [1, 5],
    [2, 6],
    [3, 7],
    [4, 8],  # lines along x-axis
    [1, 3],
    [5, 7],
    [2, 4],
    [6, 8],  # lines along y-axis
    [1, 2],
    [3, 4],
    [5, 6],
    [7, 8],  # lines along z-axis
)

FACES = np.array(
    [
        [5, 6, 8, 7],  # +x on yz plane
        [1, 3, 4, 2],  # -x on yz plane
        [3, 7, 8, 4],  # +y on xz plane = top
        [1, 2, 6, 5],  # -y on xz plane = bottom
        [2, 4, 8, 6],  # +z on xy plane = front
        [1, 5, 7, 3],  # -z on xy plane
    ]
)

BOTTOM_POINTS = [1, 2, 6, 5]

use_cupy = True

def get_center_ajust(result_bbox, projected_center):
    cx_result = (result_bbox[0] + result_bbox[2]) / 2
    cy_result = (result_bbox[1] + result_bbox[3]) / 2
    adjust_x = projected_center[0] - cx_result
    adjust_y = projected_center[1] - cy_result
    adjust_x_rel = adjust_x / (result_bbox[2] - result_bbox[0])
    adjust_y_rel = adjust_y / (result_bbox[3] - result_bbox[1])
    return adjust_x_rel, adjust_y_rel

def estimate_object_center_in_query_image(
    query_intrinsics, query_bbox, points_2d_px_result
):
    cx = (query_bbox[0] + query_bbox[2]) / 2
    cy = (query_bbox[1] + query_bbox[3]) / 2
    result_bbox = geo.get_bbox(points_2d_px_result, 360, 480)
    projected_center = points_2d_px_result[0]
    adjust_x_rel, adjust_y_rel = get_center_ajust(result_bbox, projected_center)
    adjust_x_px = adjust_x_rel * (query_bbox[2] - query_bbox[0])
    adjust_y_px = adjust_y_rel * (query_bbox[3] - query_bbox[1])
    cx = cx + adjust_x_px
    cy = cy + adjust_y_px
    b = (cx - query_intrinsics[1, 2]) / query_intrinsics[1, 1]
    a = (cy - query_intrinsics[0, 2]) / query_intrinsics[0, 0]
    return cx, cy, a, b

def align_box_with_plane(points3d, plane_normal_query, plane_normal_result):
    box_rotation = geo.rotation_matrix_from_vectors(
        plane_normal_query, plane_normal_result
    )
    # plane_normal_query, plane_normal_result, plane_center_query, plane_center_result
    points_3d_axis = np.dot(points3d - points3d[0], box_rotation) + points3d[0]
    return points_3d_axis

def get_scale_factor(query_bbox, points3d_scaled, intrinsics, width=360, height=480):
    points2d_px_result = geo.project_3d_to_2d(points3d_scaled, intrinsics)
    # points2d_px_query = geo.project_3d_to_2d(points_3d_query, intrinsics)
    result_bbox = geo.get_bbox(points2d_px_result, width, height)
    # query_bbox = geo.get_bbox(points2d_px_query, width, height)
    scale = geo.get_bbox_area(query_bbox) / geo.get_bbox_area(result_bbox)
    return scale

def get_smooth_scale_factor(query_bbox, points3d_scaled, intrinsics, alpha):
    factor = get_scale_factor(query_bbox, points3d_scaled, intrinsics)
    return (alpha + factor - 1) / alpha

def get_bounding_box(idx, match_idx, experiment, adjust_scale=False):
    points_2d_result, points_3d_result = experiment.get_points(match_idx, train=True)
    points_2d_px_result = geo.points_2d_to_points2d_px(points_2d_result, 360, 480)
    points_2d_query, _ = experiment.get_points(idx, train=False)
    points_2d_px_query = geo.points_2d_to_points2d_px(points_2d_query, 360, 480)
    plane_center_query, plane_normal_query = experiment.get_plane(idx, train=False)
    plane_center_result, plane_normal_result = experiment.get_plane(
        match_idx, train=True
    )
    result_bbox = geo.get_bbox(points_2d_px_result, 360, 480)
    # query_camera = experiment.get_camera(idx, train=False)
    query_intrinsics = experiment.get_intrinsics(idx, train=False)
    query_intrinsics = geo.scale_intrinsics(query_intrinsics, 0.25, 0.25)
    result_bbox = geo.get_bbox(points_2d_px_result, 360, 480)
    query_bbox = geo.get_bbox(points_2d_px_query, 360, 480)
    cx, cy, a, b = estimate_object_center_in_query_image(
        query_intrinsics, query_bbox, points_2d_px_result
    )
    center_ray = np.array([a, b, -1])
    points_3d_axis = align_box_with_plane(
        points_3d_result, plane_normal_query, plane_normal_result
    )
    obj_radius = geo.get_dist_from_plane(
        plane_normal_result, plane_center_result, points_3d_axis[0]
    )
    points_3d_result_snapped = geo.snap_to_plane(
        points_3d_axis, plane_normal_query, plane_center_query, center_ray, obj_radius
    )
    if adjust_scale:
        scale = get_smooth_scale_factor(
            query_bbox, points_3d_result_snapped, query_intrinsics, 2
        )
        points3d_scaled = points_3d_result_snapped
        for i in range(0, 4):
            # print(scale, obj_radius)
            obj_radius = geo.get_dist_from_plane(
                plane_normal_query, plane_center_query, points3d_scaled[0]
            )
            points3d_scaled = geo.snap_to_plane(
                geo.scale_3d_bbox(points3d_scaled, scale),
                plane_normal_query,
                plane_center_query,
                center_ray,
                obj_radius=obj_radius * scale,
            )
            scale = get_smooth_scale_factor(
                query_bbox, points3d_scaled, query_intrinsics, 2
            )
            # print(i, get_iou_between_bbox(np.array(points_3d_query), np.array(points3d_scaled)))
        return points3d_scaled
    return points_3d_result_snapped

def create_objectron_bbox_from_points(points, color=(0, 0, 0)):
    assert len(color) == 3
    # EDGES = (
    #    [1, 5], [2, 6], [3, 7], [4, 8],  # lines along x-axis
    #    [1, 3], [5, 7], [2, 4], [6, 8],  # lines along y-axis
    #    [1, 2], [3, 4], [5, 6], [7, 8]   # lines along z-axis
    # )
    pcd = o3d.geometry.PointCloud()
    pcd = o3d.geometry.PointCloud()
    pcd.points = o3d.utility.Vector3dVector(points)
    lineset = o3d.geometry.LineSet.create_from_point_cloud_correspondences(
        pcd, pcd, EDGES
    )
    colors = np.array(color * len(lineset.lines)).reshape(len(lineset.lines), 3)
    lineset.colors = o3d.utility.Vector3dVector(colors)
    return lineset

BOTTOM_POINTS = [1, 2, 6, 5]

def get_match_aligned_points(
    idx: int,
    match_idx: int,
    info_df: pd.DataFrame,
    train_info_df: pd.DataFrame,
    ground_truth=False,
):
    """Get 3D IoU for a specific query image, using the kth neighbor.

    Returns:
        [type]: [description]
    """
    assert type(match_idx) == int
    points2d_train, points3d_train = get_points(train_info_df, match_idx)
    points2d_valid, points3d_valid = get_points(info_df, idx)

    valid_image = Image.open(info_df.iloc[idx]["filepath_full"])
    train_image = Image.open(train_info_df.iloc[match_idx]["filepath_full"])
    points2d_valid_px = geo.points_2d_to_points2d_px(
        points2d_valid, valid_image.width, valid_image.height
    )
    points2d_train_px = geo.points_2d_to_points2d_px(
        points2d_train, train_image.width, train_image.height
    )
    valid_bbox = geo.get_bbox(points2d_valid_px, valid_image.width, valid_image.height)
    train_bbox = geo.get_bbox(points2d_train_px, train_image.width, train_image.height)
    points3d_train_rotated, _ = geo.align_with_bbox_3d(
        points3d_train, train_bbox, valid_bbox
    )
    if ground_truth:
        return np.array(points3d_train_rotated), np.array(points3d_valid)
    else:
        return np.array(points3d_train_rotated), np.array(points3d_train)

def get_objectron_bbox_colors():
    bbox_point_colors = np.zeros((9, 3))

    for bottom_point in BOTTOM_POINTS:
        bbox_point_colors[bottom_point] = np.array([1, 0, 0])
    bbox_point_colors[0] = np.array([0, 0, 1])
    return o3d.utility.Vector3dVector(bbox_point_colors)

def fix_idx(idx):
    if type(idx) is not int and type(idx) is not numpy.ndarray:
        idx = int(idx)  # Just make sure this is the right
    return idx

def visualize(points3d_query1, points3d_results1, points3d_results2):
    """Small visualisation helper. Interface will change!"""

    pcd_results1 = o3d.geometry.PointCloud()
    pcd_results2 = o3d.geometry.PointCloud()
    # points3d_train_results1 = points3d_results1
    # points3d_train_rotated, points3d_train_aligned = align_with_bbox_3d(points3d_train, train_bbox, valid_bbox)
    pcd_results1.points = o3d.utility.Vector3dVector(points3d_results1)
    pcd_results2.points = o3d.utility.Vector3dVector(points3d_results2)

    # pcd_train_aligned.points = o3d.utility.Vector3dVector(points3d_train_aligned)

    pcd_valid = o3d.geometry.PointCloud()
    pcd_valid.points = o3d.utility.Vector3dVector(points3d_query1)

    vis = o3d.visualization.Visualizer()
    vis.create_window()
    bbox_3d_results1 = pcd_results1.get_oriented_bounding_box()
    bbox_3d_results2 = pcd_results2.get_oriented_bounding_box()
    bbox_3d_valid = pcd_valid.get_oriented_bounding_box()
    # bbox_3d.create_from_point_cloud_poisson(pcd_train)
    bbox_3d_results1.color = [0, 0, 255]
    bbox_3d_results2.color = [255, 0, 0]
    vis.add_geometry(bbox_3d_results1)
    vis.add_geometry(bbox_3d_results2)
    vis.add_geometry(bbox_3d_valid)
    vis.add_geometry(pcd_results1)
    vis.add_geometry(pcd_results2)
    vis.add_geometry(pcd_valid)

    print(bbox_3d_results1.extent, bbox_3d_results1.center)
    print(bbox_3d_valid.extent, bbox_3d_valid.center)

    vis.run()
    ctr = vis.get_view_control()
    pinhole_parameters = ctr.convert_to_pinhole_camera_parameters()
    vis.destroy_window()
    # pinhole_parameters.intrinsic
    # vis.draw_geometries()
    return

def get_match_snapped_points(
    idx,
    match_idx,
    info_df,
    train_info_df,
    ground_truth=False,
    rescale=True,
    align_axis=True,
):
    assert type(match_idx) == int
    plane_center, plane_normal = get_plane(info_df, idx)
    points2d, points3d = get_points(info_df, idx)
    points3d_result_rotated, points3d_result = get_match_aligned_points(
        idx, match_idx, info_df, train_info_df, ground_truth=ground_truth
    )
    snapped, intersect = geo.snap_box_to_plane(
        points3d_result_rotated, plane_normal, plane_center, align_axis=align_axis
    )

    result = snapped
    if rescale:
        points2d_result, _ = get_points(train_info_df, match_idx)
        # points3d_result = np.array(points3d_result)
        camera = get_camera(info_df, idx)
        intrinsics = get_intrinsics(camera)
        intrinsics = geo.scale_intrinsics(intrinsics, 0.25, 0.25)
        points2d_px = geo.project_3d_to_2d(snapped, intrinsics)
        dest_bbox = geo.get_bbox(points2d_px, 360, 480)

        points2d_valid, points3d_valid = get_points(info_df, idx)

        valid_image = Image.open(info_df.iloc[idx]["filepath_full"])
        points2d_valid_px = geo.points_2d_to_points2d_px(
            points2d_valid, valid_image.width, valid_image.height
        )
        valid_bbox = geo.get_bbox(
            points2d_valid_px, valid_image.width, valid_image.height
        )
        scale_x, scale_y = geo.get_scale_factors(dest_bbox, valid_bbox)
        scale_factor = (scale_x + scale_y) / 2

        # print(scale_factor)
        fixed_point = snapped[0]
        snapped_rotated = snapped.copy()
        # scale_factor = 1.2
        snapped_rotated = (snapped_rotated - fixed_point) * scale_factor + fixed_point
        snapped_rotated, intersect = geo.snap_box_to_plane(
            snapped_rotated, plane_normal, plane_center
        )
        result = snapped_rotated

    return result, points3d_result

def get_iou_between_bbox(points_train, points_valid):
    try:
        v_rotated = np.array(points_train)
        v_valid = np.array(points_valid)
        w_rotated = box.Box(vertices=v_rotated)
        w_valid = box.Box(vertices=v_valid)
        loss = iou.IoU(w_rotated, w_valid)
        return float(loss.iou())
    except:
        print("Error computing IoU!")
        return 0

def get_iou(
    idx: int,
    embeddings: np.array,
    info_df: pd.DataFrame,
    train_embeddings: np.array,
    train_info_df: pd.DataFrame,
    symmetric=False,
    rescale=False,
    k=0,
    show=False,
    compute_aligned=False,
    ground_plane=True,
    align_axis=True,
):
    """Get 3D IoU for a specific query image, using the kth neighbor.

    Args:
        idx (int): Query index in meta dataframe.
        embeddings (np.array): Query embeddings.
        info_df (pd.DataFrame): Query embeddings metadata.
        train_embeddings (np.array): Trainin set embeddings. Will be used for lookup.
        train_info_df (pd.DataFrame): Training set embeddings metadata.
        k (int, optional): kth neighbor to use as result.. Defaults to 0.
        show (bool, optional): Visualize the result in 3D. Defaults to False.

    Returns:
        [type]: [description]
    """
    # @concurrent
    match_idx = find_match_idx(idx, embeddings, train_embeddings, k)
    if ground_plane:
        points3d_processed, points3d_valid = get_match_snapped_points(
            idx,
            match_idx,
            info_df,
            train_info_df,
            ground_truth=True,
            rescale=rescale,
            align_axis=align_axis,
        )
    else:
        points3d_processed, points3d_valid = get_match_aligned_points(
            idx, match_idx, info_df, train_info_df, ground_truth=True
        )

    # if show:
    #    visualize(points3d_valid, points3d_train_rotated, points3d_train_aligned)
    iou_value = get_iou_between_bbox(points3d_valid, points3d_processed)
    best_iou = iou_value
    if symmetric:
        for angle in np.arange(5, 360, 5):
            theta = (angle / 180) * np.pi
            pivoted = geo.rotate_bbox_around_its_center(points3d_processed, theta)
            iou_at_theta = get_iou_between_bbox(pivoted, points3d_valid)
            if iou_at_theta > best_iou:
                best_iou = iou_at_theta
        # _get_iou.wait()
    return best_iou, match_idx

def get_iou_rotated(points3d_processed, points3d_valid, initial_iou):
    """Get the IoU metric for a symmetric object by rotating it around its axis.

    Args:
        points3d_processed (np.array): Result Bounding box
        points3d_valid (np.array): Query Bounding box
        initial_iou (float): Initial IoU when the object is not rotated.

    Returns:
        float: Best IoU obtained while rotating the object around its axis.
    """
    best_iou = initial_iou
    for angle in np.arange(5, 360, 5):
        theta = (angle / 180) * np.pi
        pivoted = geo.rotate_bbox_around_its_center(points3d_processed, theta)
        iou_at_theta = get_iou_between_bbox(pivoted, points3d_valid)
        if iou_at_theta > best_iou:
            best_iou = iou_at_theta
    return best_iou

def find_match_idx(
    idx, query_embeddings: np.ndarray, train_embeddings: np.ndarray, k=0, score=False
):
    """Find the nearest neighbor of a single query embedding in the test set in the training set.

    Args:
        idx (int): Index in the test set.
        query_embeddings (np.ndarray): Test set embeddings.
        train_embeddings (np.ndarray): Train set embeddings.
        k (int, optional): kth neighbor to use. Defaults to 0.

    Returns:
        int: Match index in the train set.
    """
    global use_cupy
    idx = fix_idx(idx)
    lib = cp
    if not use_cupy:
        lib = np

    similarity = lib.dot(query_embeddings[idx].T, train_embeddings)
    if k != 0:
        best_matches = lib.argsort(-similarity)
        match_idx = best_matches[k]
    else:
        match_idx = lib.argmin(-similarity)
    if not score:
        return int(match_idx)
    else:
        return int(match_idx), float(similarity[match_idx])

def find_all_match_idx(query_embeddings: np.ndarray, train_embeddings: np.ndarray, k=0):
    """Find all matches for the test set in the training set at the sametime, using cupy.

    This is solely for optimisation purpose in order to get the code to run faster on machines
    with GPU.

    Args:
        query_embeddings (np.ndarray): Test set embeddings.
        train_embeddings (np.ndarray): Train set embeddings.
        k (int, optional): [description]. Defaults to 0.

    Raises:
        ValueError: The case where "k!=0" is not yet implemeted.

    Returns:
        [type]: [description]
    """
    global use_cupy
    print("Using GPU to compute matches!")

    if k != 0:
        # best_matches = cp.argsort(-cp.dot(query_embeddings.T,train_embeddings))
        # match_idx = best_matches[k]
        raise ValueError("The case where k is not 0 must be implemented.")
    else:
        match_idxs = []
        query_chunk_size = 1024
        train_chunk_size = 65536 * 2
        for i in tqdm(range(0, math.ceil(len(query_embeddings) / query_chunk_size))):
            query_start = i * query_chunk_size
            query_end = query_start + query_chunk_size
            if query_end > len(query_embeddings):
                query_end = len(query_embeddings)
            cuda_query_embeddings = cp.asarray(query_embeddings[query_start:query_end])

            matches = []
            scores = []
            best_match_idx_chunk_score = np.zeros((query_end - query_start, 1))
            best_match_idx_chunk = np.zeros(
                (query_end - query_start, 1), dtype=np.uint64
            )
            for j in range(0, math.ceil(train_embeddings.shape[1] / train_chunk_size)):
                train_start = j * train_chunk_size
                train_end = train_start + train_chunk_size
                if train_end > train_embeddings.shape[1]:
                    train_end = train_embeddings.shape[1]
                cuda_train_embeddings = cp.asarray(
                    train_embeddings[:, train_start:train_end]
                )
                similarity = cp.dot(cuda_query_embeddings, cuda_train_embeddings)
                match_idx_chunk = cp.argmax(similarity, axis=1).get()
                similarity = similarity.get()
                match_idx_chunk_score = np.take_along_axis(
                    similarity, np.expand_dims(match_idx_chunk, axis=1), axis=1
                )
                match_idx_chunk += train_start
                best_match_idx_chunk = np.where(
                    match_idx_chunk_score > best_match_idx_chunk_score,
                    np.expand_dims(match_idx_chunk, axis=1),
                    best_match_idx_chunk,
                ).astype(np.uint64)
                best_match_idx_chunk_score = np.where(
                    match_idx_chunk_score > best_match_idx_chunk_score,
                    match_idx_chunk_score,
                    best_match_idx_chunk_score,
                )

                # if use_cupy:
                # match_idx_chunk=match_idx_chunk.get()

                matches.append(match_idx_chunk)
            match_idxs += best_match_idx_chunk.squeeze().tolist()
        return match_idxs

def describe_intrinsics(intrinsics: np.array):
    """Describe the intrinsics matrix in their common names.

    Args:
        intrinsics (np.array): Intrinsics
    """
    alpha_x = intrinsics[0, 0]
    alpha_y = intrinsics[1, 1]
    center_x = intrinsics[0, 2]
    center_y = intrinsics[1, 2]
    return float(alpha_x), float(alpha_y), float(center_x), float(center_y)

# embeddings = None
# info_df = None
# train_embeddings = None
# train_info_df = None
experiment = None

args = None
all_match_idxs = None
import argparse
import time

def get_iou_mp(
    idx: int,  # , symmetric=False, rescale=False,
    # k=0, show=False, compute_aligned=False, ground_plane=True,
):
    """Get 3D IoU for a specific query image, using the kth neighbor.

    Args:
        idx (int): Query index in meta dataframe.
        embeddings (np.array): Query embeddings.
        info_df (pd.DataFrame): Query embeddings metadata.
        train_embeddings (np.array): Trainin set embeddings. Will be used for lookup.
        train_info_df (pd.DataFrame): Training set embeddings metadata.
        k (int, optional): kth neighbor to use as result.. Defaults to 0.
        show (bool, optional): Visualize the result in 3D. Defaults to False.

    Returns:
        [type]: [description]
    """

    global args, experiment, all_match_idxs
    experiment.load_hdf5_file()  # Just make sure the hdf5 file is not loader prior to forking.
    category = experiment.get_category(idx, train=False)

    # all_match_idxs = find_match_idx(idx, embeddings, train_embeddings,0)
    match_idx = all_match_idxs[idx]
    if args.random_bbox:
        match_idx = random.randint(0, len(experiment.train_info_df) - 1)
    if args.random_bbox_same:
        match_idx = random.sample(
            list(experiment.train_info_df.query(f"category=='{category}'").index), 1
        )
        match_idx = match_idx[0]

    # if args.legacy:
    #    if args.ground_plane:
    #        points3d_processed, points3d_valid = get_match_snapped_points(idx, match_idx, experiment.info_df, train_info_df, ground_truth=True, rescale=args.rescale, align_axis=not args.no_align_axis)
    #    else:
    #        points3d_processed, points3d_valid = get_match_aligned_points(idx, match_idx, experiment.info_df, train_info_df, ground_truth=True)
    # else:
    _, points3d_valid = experiment.get_points(idx, train=False)
    points3d_processed = get_bounding_box(idx, match_idx, experiment, adjust_scale=True)
    # if show:
    #    visualize(points3d_valid, points3d_train_rotated, points3d_train_aligned)
    iou_value = get_iou_between_bbox(points3d_valid, points3d_processed)
    best_iou = iou_value
    if args.symmetric:
        if category == "cup" or category == "bottle":
            best_iou = get_iou_rotated(points3d_processed, points3d_valid, best_iou)
        # _get_iou.wait()
    return best_iou, idx, match_idx

class ExperimentHandlerFile:
    def __init__(self, experiment: str = None, test=False):
        self.info_df = None
        self.train_info_df = None
        self.hdf5_dataset = None
        self.hdf5_test_dataset = None
        self.test = test
        if experiment:
            self.read_experiment(experiment, test)
        self.source_image_width = 1440
        self.source_image_height = 1920

    def read_experiment(self, experiment: str, test: bool):
        """Read the output of a SimSiam experiment folder. (Embeddings and metadata)

        Args:
            experiment (str): Output folder location

        Returns:
            tuple: validation embeddings, validation meta data, train embeddings, train meta data
        """
        global use_cupy
        load_fn = cp.load
        if not use_cupy:
            load_fn = np.load

        info_prefix = ""
        if test:
            info_prefix = "test_"
            print(
                "Loading experiment with test set instead of validation set embeddings!"
            )
        embeddings = load_fn(f"{experiment}/{info_prefix}embeddings.npy")
        if embeddings.shape[1] > embeddings.shape[0]:
            embeddings = embeddings.T
        info = numpy.load(f"{experiment}/{info_prefix}info.npy")
        assert info.shape[0] == 2
        train_new_filename = f"{experiment}/train_embeddings.npy"
        train_old_filename = f"{experiment}/training_embeddings.npy"
        if os.path.exists(train_new_filename):
            train_embeddings = load_fn(train_new_filename)
        else:
            train_embeddings = load_fn(train_old_filename)
        train_info = numpy.load(f"{experiment}/train_info.npy")
        assert train_info.shape[0] == 2

        # lines = content
        self.init_experiment(embeddings, info, train_embeddings, train_info)
        try:
            if not self.test:
                with open(f"{experiment}/train_sequences.txt", "r") as f:
                    train_sequences_t = f.read().split("\n")
                    valid_sequences_g = sorted(
                        list((self.info_df["sequence_uid"].unique()))
                    )
                    for valid_sequence in valid_sequences_g:
                        if valid_sequence in train_sequences_t:
                            raise ValueError(
                                f"The validation sequence {valid_sequence} was part of the training set!"
                            )
        except FileNotFoundError:
            print(
                f"Warning, unable to find {experiment}/train_sequences.txt.\n We will not be able to make sure there is no overlap between training and validation!"
            )

    def load_hdf5_file(self):
        if self.hdf5_dataset is None:
            self.hdf5_dataset = h5py.File(
                "/home/raphael/datasets/objectron/extract_s5_raw.hdf5", "r"
            )
        if self.test and self.hdf5_test_dataset is None:
            self.hdf5_test_dataset = h5py.File(
                "/home/raphael/datasets/objectron/extract_s5_raw_test.hdf5", "r"
            )

    def init_experiment(self, embeddings, info, train_embeddings, train_info):
        info_df = pd.DataFrame(info.T)
        train_info_df = pd.DataFrame(train_info.T)
        info_df_columns = {0: "category_id", 1: "uid"}
        train_info_df = train_info_df.rename(columns=info_df_columns)
        info_df = info_df.rename(columns=info_df_columns)
        if "hdf5" in train_info_df.iloc[0][1]:
            self.mode = "hdf5"
            self.hdf5_dataset = None
        else:
            self.mode = "raw"
        self.parse_info_df(info_df)
        self.parse_info_df(train_info_df, subset="train")
        assert train_embeddings.shape[0] == embeddings.shape[1]
        self.embeddings = embeddings
        self.train_embeddings = train_embeddings
        self.info_df = info_df
        self.train_info_df = train_info_df

    def _pick_hdf5_dataset(self, train):
        if not self.test:
            return self.hdf5_dataset
        if not train and self.test:
            return self.hdf5_test_dataset
        if train:
            return self.hdf5_dataset
        raise ValueError("We should get here!")

    def get_points(self, idx: int, train=False):
        if self.mode == "raw":
            return self._raw_get_points(idx, train)
        elif self.mode == "hdf5":
            category, sequence, image_id = self._hdf5_parse_uid(idx, train)
            hdf5_dataset = self._pick_hdf5_dataset(train)
            image_idx = list(hdf5_dataset[category][sequence]["IMAGE_ID"]).index(
                image_id
            )
            return hdf5_dataset[category][sequence]["POINT_2D"][image_idx].reshape(
                9, 3
            ), hdf5_dataset[category][sequence]["POINT_3D"][image_idx].reshape(9, 3)
        else:
            raise ValueError(f"Mode not supported: {self.mode}")

    def get_image(self, idx, train=False):
        df = self._get_df(train)
        if self.mode == "hdf5":
            category, sequence, image_id = self._hdf5_parse_uid(idx, train)
            hdf5_dataset = self._pick_hdf5_dataset(train)
            image_idx = list(hdf5_dataset[category][sequence]["IMAGE_ID"]).index(
                image_id
            )
            jpeg_data = hdf5_dataset[category][sequence]["IMAGE"][image_idx]
            # image = Image.frombytes('RGBA', (128,128), jpeg_data, 'raw')
            image = Image.open(io.BytesIO(jpeg_data))
            # img= cv2.imdecode(self.hdf5_dataset[category][sequence]["IMAGE"][image_idx],cv2.IMREAD_ANYCOLOR)
            return image
        else:
            raise ValueError("To be implemented!")

    def _get_df(self, train):
        if train:
            df = self.train_info_df
        else:
            df = self.info_df
        return df

    def _hdf5_parse_uid(self, idx, train=False):
        df = self._get_df(train)
        uid = df.iloc[idx]["uid"]
        m = re.match("hdf5_(\w+)/(\d+)_(\d+)", uid)
        category, sequence, image_id = m[1], m[2], int(m[3])
        return category, sequence, image_id

    def _raw_get_points(self, idx: int, train=False):
        """Get 2d and 3d points for a specific frame.

        Args:
            df (pd.DataFrame): Embeddings meta data frame.
            idx (int): Index of the frame in the dataframe.

        Returns:
            tuple: 2d points, 3d points relative to camera.
        """
        if train:
            df = self.train_info_df
        else:
            df = self.info_df

        idx = fix_idx(idx)
        sequence_annotations = self._raw_get_sequence_annotations(
            df.iloc[idx]["category"],
            df.iloc[idx]["batch_number"],
            df.iloc[idx]["sequence_number"],
        )
        frame = int(df.iloc[idx]["frame"])
        object_id = int(df.iloc[idx]["object_id"])
        keypoints = (
            sequence_annotations.frame_annotations[frame]
            .annotations[object_id]
            .keypoints
        )
        points_2d = []
        points_3d = []
        for keypoint in keypoints:
            point_2d = (
                keypoint.point_2d.x,
                keypoint.point_2d.y,
                keypoint.point_2d.depth,
            )
            point_3d = (keypoint.point_3d.x, keypoint.point_3d.y, keypoint.point_3d.z)
            points_2d.append(point_2d)
            points_3d.append(point_3d)
        return np.array(points_2d), np.array(points_3d)

    def _raw_get_sequence_annotations(
        self,
        category: str,
        batch_number: str,
        sequence_number: str,
        annotations_path="/home/raphael/datasets/objectron/annotations",
    ):
        """Get annotation data for a video sequence.

        Args:
            category (str): Category in the objectron dataset.
            batch_number (str): Batch number.
            sequence_number (str): Sequence number.
            annotations_path (str, optional): Location of the Objectron annotation files. Defaults to "/home/raphael/datasets/objectron/annotations".

        Returns:
            annotations: Google's annotations in their format.
        """
        sequence = annotation_protocol.Sequence()
        annotation_file = (
            f"{annotations_path}/{category}/{batch_number}/{sequence_number}.pbdata"
        )
        with open(annotation_file, "rb") as pb:
            sequence.ParseFromString(pb.read())
        return sequence

    def get_plane(self, idx: int, train=False):
        if self.mode == "raw":
            return self._raw_get_plane(idx, train)
        elif self.mode == "hdf5":
            category, sequence, image_id = self._hdf5_parse_uid(idx, train)
            hdf5_dataset = self._pick_hdf5_dataset(train)
            image_idx = list(hdf5_dataset[category][sequence]["IMAGE_ID"]).index(
                image_id
            )
            return (
                hdf5_dataset[category][sequence]["PLANE_CENTER"][image_idx],
                hdf5_dataset[category][sequence]["PLANE_NORMAL"][image_idx],
            )
        else:
            raise ValueError(f"Mode not supported: {self.mode}")

    def _raw_get_plane(self, idx: int, train=False):
        """Get object plane for a specific camera frame.

        Args:
            df (pd.DataFrame): Embeddings meta data frame.
            idx (int): Index of the frame in the dataframe.

        Returns:
            tuple: 2d points, 3d points relative to camera.
        """
        if train:
            df = self.train_info_df
        else:
            df = self.info_df
        idx = int(idx)
        sequence_annotations = self._raw_get_sequence_annotations(
            df.iloc[idx]["category"],
            df.iloc[idx]["batch_number"],
            df.iloc[idx]["sequence_number"],
        )
        frame = int(df.iloc[idx]["frame"])

        plane_center = sequence_annotations.frame_annotations[frame].plane_center
        plane_normal = sequence_annotations.frame_annotations[frame].plane_normal

        return np.array(plane_center), np.array(plane_normal)

    def _raw_get_camera(self, idx: int, train=False):
        """Get camera information for a specific trame.

        Args:
            df (pd.DataFrame): Embeddings meta data frame.
            idx (int): Index of the frame in the dataframe.

        Returns:
            object: Google's camera information for the frame.
        """
        if train:
            df = self.train_info_df
        else:
            df = self.info_df
        idx = fix_idx(idx)
        sequence_annotations = self._raw_get_sequence_annotations(
            df.iloc[idx]["category"],
            df.iloc[idx]["batch_number"],
            df.iloc[idx]["sequence_number"],
        )
        frame = int(df.iloc[idx]["frame"])
        frame_annotation = sequence_annotations.frame_annotations[frame]
        return frame_annotation.camera

    def _raw_get_intrinsics(self, idx, train=False):
        camera = self._raw_get_camera(idx, train)
        return np.array(camera.intrinsics).reshape(3, 3)

    def get_intrinsics(self, idx, train=False):
        if self.mode == "raw":
            return self._raw_get_intrinsics(idx, train)
        elif self.mode == "hdf5":
            category, sequence, image_id = self._hdf5_parse_uid(idx, train)
            hdf5_dataset = self._pick_hdf5_dataset(train)
            image_idx = list(hdf5_dataset[category][sequence]["IMAGE_ID"]).index(
                image_id
            )
            return hdf5_dataset[category][sequence]["INTRINSIC_MATRIX"][
                image_idx
            ].reshape(3, 3)
        else:
            raise ValueError(f"Mode not supported: {self.mode}")

    @staticmethod
    def _get_subset(info_df, embeddings, ratio):
        if ratio == 1:
            return info_df, embeddings
        # info_df["video_uid"]=info_df["category"]+"_"+info_df["video_id"]
        video_uids = list(info_df["sequence_uid"].unique())
        videos_uids_subset = random.sample(video_uids, int(len(video_uids) * ratio))
        print(
            f"Using a subset of {len(videos_uids_subset)} out of {len(video_uids)} total sequences for evaluation."
        )

        info_df_subset = info_df[info_df["sequence_uid"].isin(videos_uids_subset)]
        embeddings_subset = embeddings[:, list(info_df_subset.index)]
        info_df_subset = info_df_subset.reset_index()
        return info_df_subset, embeddings_subset

    def set_trainsubset(self, ratio):
        train_info_df, train_embeddings = self._get_subset(
            self.train_info_df, self.train_embeddings, ratio
        )
        self.train_info_df = train_info_df
        self.train_embeddings = train_embeddings

    def get_category(self, idx, train=False):
        df = self._get_df(train)
        if self.mode == "raw":
            return df.iloc[idx]["category"]
        elif self.mode == "hdf5":
            category, _, _ = self._hdf5_parse_uid(idx, train)
            return category
        else:
            raise ValueError(f"Mode not supported: {self.mode}")

    def get_sequence_uid(self, idx, train=False):
        df = self._get_df(train)
        return df.iloc[idx]["sequence_uid"]

    def parse_info_df(self, info_df, subset="valid"):
        """Parses an embedding meta-data dataframe. (The output of SimSiam)

        Args:
            info_df (pd.DataFrame): Pandas Dataframe, as output by the "read experiment" function.
            subset (str, optional): Which subset to use. Can be "train", "valid" or test. Defaults to "valid".
        """
        info_df["category"] = info_df["uid"].str.extract("(.*?)-")
        info_df["sequence_uid"] = info_df["uid"].str.extract("(batch\-\d+_\d+_\d+)")
        info_df["frame"] = info_df["uid"].str.extract("-(\d+)$")
        info_df["video_id"] = info_df["uid"].str.extract("(batch\-\d+_\d+)")
        info_df["object_id"] = info_df["uid"].str.extract("batch\-\d+_\d+_(\d+)")
        info_df["batch_number"] = info_df["uid"].str.extract("(batch\-\d+)")
        info_df["sequence_number"] = info_df["uid"].str.extract("batch\-\d+_(\d+)_\d+")
        info_df["filepath"] = (
            f"/home/raphael/datasets/objectron/96x96/{subset}/"
            + info_df["category"]
            + "/"
            + info_df["sequence_uid"]
            + "."
            + info_df["frame"]
            + ".jpg"
        )
        info_df["filepath_full"] = (
            "/home/raphael/datasets/objectron/640x480_full/"
            + info_df["category"]
            + "/"
            + info_df["sequence_uid"]
            + "."
            + info_df["frame"]
            + ".jpg"
        )
        if self.mode == "hdf5":
            info_df["sequence_uid"] = info_df["uid"].str.extract("hdf5_(\w+/\d+)_")

def main():
    global args, experiment, ground_plane, symmetric, rescale, all_match_idxs, use_cupy
    """Command line tool for evaluating zero shot pose estimation
    on objectron."""
    parser = argparse.ArgumentParser(
        description="Command line tool for evaluating zero shot pose estimation on objectron."
    )
    parser.add_argument(
        "experiment",
        type=str,
        help="Experiment folder location. i.e. ../SimSiam/outputs/objectron_96x96_experiment_synchflip_next",
    )
    parser.add_argument(
        "--subset_size",
        type=int,
        default=1000,
        help="Number of samples for 3D IoU evaluation",
    )
    parser.add_argument(
        "--ground_plane", default=True, help="If enabled, snap to ground plane"
    )
    parser.add_argument(
        "--iou_t",
        default=0.5,
        help="IoU threshold required to consider a positive match",
    )
    parser.add_argument(
        "--symmetric",
        action="store_true",
        help="Rotate symetric objects (cups, bottles) and keep maximum IoU.",
    )
    parser.add_argument(
        "--rescale", action="store_true", help="Rescale 3d bounding box"
    )
    parser.add_argument(
        "--random_bbox",
        action="store_true",
        help="Fit a randomly selected bbox instead of the nearest neighbor",
    )
    parser.add_argument(
        "--random_bbox_same",
        action="store_true",
        help="Fit a randomly selected bbox from same category instead of the nearest neighbor",
    )
    parser.add_argument(
        "--trainset-ratio",
        type=float,
        default=1,
        help="Ratio of the training set sequences used for inference",
    )
    parser.add_argument(
        "--single_thread", action="store_true", help="Disable multithreading."
    )
    parser.add_argument("--cpu", action="store_true", help="Disable cuda accelaration.")
    parser.add_argument(
        "--no_align_axis",
        action="store_true",
        help="Don't to to align axis with ground plane.",
    )
    parser.add_argument(
        "--legacy", action="store_true", help="Deprecated legacy evalution mode"
    )
    parser.add_argument(
        "--test", action="store_true", help="Evaluate on test set embeddings"
    )
    args = parser.parse_args()
    symmetric = args.symmetric
    rescale = args.rescale

    if args.cpu:
        use_cupy = False

    experiment = ExperimentHandlerFile(args.experiment, args.test)

    if args.trainset_ratio > 0 and args.trainset_ratio <= 1:
        experiment.set_trainsubset(args.trainset_ratio)
    else:
        raise ValueError("Training set ratio must be between 0 and 1!")

    all_match_idxs = find_all_match_idx(
        experiment.embeddings, experiment.train_embeddings, 0
    )
    # get_iou_mp(1)
    if args.subset_size < len(experiment.info_df):
        subset = random.sample(range(0, len(experiment.info_df)), args.subset_size)
    else:
        print(
            f"Evaluating on all samples size subset size ({args.subset_size}) is larger that test set size ({len(experiment.info_df)})."
        )
        subset = list(range(0, len(experiment.info_df)))
        random.shuffle(subset)

    ious = {}
    results = []
    ious_aligned = []
    threshold = args.iou_t
    valid_count = 0
    get_iou_params = []

    for idx in subset:
        # params = (idx, args.symmetric, args.rescale)
        get_iou_params.append(idx)
    if not args.single_thread:
        with get_context("fork").Pool(4) as p:
            results = list(tqdm(p.imap(get_iou_mp, get_iou_params), total=len(subset)))
    else:
        for idx in tqdm(get_iou_params):
            results.append(get_iou_mp(idx))

    for iou, idx, match_idx in results:
        category = experiment.get_category(idx, train=False)
        if not category in ious:
            ious[category] = []
        if iou > threshold:
            valid_count += 1
        ious[category].append(float(iou))

    ious = pd.DataFrame.from_dict(ious, orient="index").T
    ious.to_csv("raw_results.txt")
    print(f"{'category' : <10}\tmean iou\tmedian iou\tAP at iou_t")
    for category in sorted(ious.columns):
        column = ious[category]
        ap_at_iout = (column > args.iou_t).sum() / column.notnull().sum()
        print(
            f"{category: <10}\t{column.mean():0.2f}\t\t{column.median():0.2f}\t\t{ap_at_iout:0.2f}"
        )

if __name__ == "__main__":
    main()


import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import chi2_contingency

# fetch annotated data
in_data = pd.read_excel(os.path.join('regex_data', 'regex_search_output.xlsx'))#, nrows=400)

# create figure directory if it doesn't exist
figs = os.path.join(os.getcwd(), 'figs')
if not os.path.exists(figs):
    os.mkdir(figs)

# get frequency counts based on word and category
try:
    counts = in_data.value_counts(subset=['word', 'category', 'count']).to_frame().reset_index()
except:
    print('No data found. The spreadsheet may be unannotated. A "category" column is required, as well as a "count" column for instances with multiple uses.')
    exit()

# rename column
counts.rename(columns={0: 'freq'}, inplace=True)

# easier to read
counts.sort_values(by=['word', 'category'], inplace=True)

# add up instances with multiple or single uses
counts['total_freq'] = counts['count'] * counts['freq']

# no longer need this separately
counts.drop(columns=['count', 'freq'], inplace=True)

# merge counts for instances with multiple or single uses
counts = counts.groupby(['word', 'category'], as_index=False).agg('sum')

def affix_ly(w): # function to add -ly to a potential zero form
    if w == 'good':
        return 'well' # special case: good -> well
    elif w.endswith('y'): # special case: happy -> happily
        return w[:-1] + 'ily'
    elif w.endswith('le'): # special case: terrible -> terribly
        return w[:-1] + 'y'
    elif w.endswith('ic'): # special case: tragic -> tragically
        return w + 'ally'
    elif w.endswith('ue'): # special case: true -> truly
        return w[:-1] + 'ly'
    else:
        return w + 'ly' # general case: sad -> sadly

# Create the figure and axes objects for the subplots
fig, axs = plt.subplots(2, 3, figsize=(10, 5))

# Adjust the spacing between the subplots
plt.subplots_adjust(wspace=0.4, hspace=0.9)

fig_ly, axs_ly = plt.subplots(2, 3, figsize=(10, 5))

# Adjust the spacing between the subplots
plt.subplots_adjust(wspace=0.4, hspace=0.9)

contingency = counts.pivot(index='word', columns='category', values='total_freq').fillna(0)

loc = [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]

i = 0
for w in ('bad', 'good', 'real', 'different', 'quick', 'sure'):
    #print(contingency.loc[w].values.reshape(1, -1))

    # grab data
    word_ly = affix_ly(w)
    c = counts[counts['word'] == w]
    c_ly = counts[counts['word'] == word_ly]
    bins = ['A', 'PREVB', 'POSTVB', 'PREJJ', 'POSTJJ', 'NPAM']
    heights = []
    heights_ly = []
    for bin in bins:
        try:
            heights.append(c[c['category'] == bin]['total_freq'].values[0])
        except:
            heights.append(0)
        try:
            heights_ly.append(c_ly[c_ly['category'] == bin]['total_freq'].values[0])
        except:
            heights_ly.append(0)

    # Create the first subplot
    ax = sns.barplot(x=bins, y=heights, ax=axs[loc[i][0],loc[i][1]])
    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, horizontalalignment='right')
    axs[loc[i][0],loc[i][1]].set_title(w)
    axs[loc[i][0],loc[i][1]].set_xlabel('Categories')
    axs[loc[i][0],loc[i][1]].set_ylabel('Counts')

    # Create the second subplot
    ax_ly = sns.barplot(x=bins, y=heights_ly, ax=axs_ly[loc[i][0],loc[i][1]])
    ax_ly.set_xticklabels(ax.get_xticklabels(), rotation=45, horizontalalignment='right')
    if (w == 'quick'):
        ax_ly.set_yticklabels((0,3,6,9,12))
    axs_ly[loc[i][0],loc[i][1]].set_title(word_ly)
    axs_ly[loc[i][0],loc[i][1]].set_xlabel('Categories')
    axs_ly[loc[i][0],loc[i][1]].set_ylabel('Counts')

    i += 1

fig.savefig(os.path.join(figs, 'ZAM.png'), dpi=450, bbox_inches='tight')
fig_ly.savefig(os.path.join(figs, 'OAM.png'), dpi=450, bbox_inches='tight')


import json
import sys
from traceback import print_tb

from bson.json_util import dumps
from palmettopy.palmetto import Palmetto
from pymongo import MongoClient
from tweepy import *

from analysis.topic_modelling import PalmettoHelper
from twitter_data.api import TweepyHelper

client = MongoClient('localhost', 27017)
db = client['twitter_db']
tweet_collection = db['tweet_collection']
user_collection = db['user_collection']
following_collection = db['following_collection']
followers_collection = db['followers_collection']
friendship_collection = db['friendship_collection']
lexicon_so_collection = db['lexicon_so_collection']
anew_lexicon_collection = db['anew_lexicon_collection']

npmi_collection = db['npmi_collection']

UNAVAILABLE_KEY = 'unavailable'

# NPMI word-similarity related
def get_or_add_coherence_score(word1, word2, coherence_type="npmi"):

    word1 = word1.lower()
    word2 = word2.lower()

    if sorted([word1, word2])[0] == word2:
        temp = word1
        word1 = word2
        word2 = temp

    from_db = npmi_collection.find_one({"word1": word1, "word2":word2, "coherence_type":coherence_type})
    if from_db:
        score = from_db["score"]
    else:
        score = PalmettoHelper.get_similarity_score([word1, word2], coherence_type)
        npmi_collection.insert_one({"word1":word1,"word2":word2,"coherence_type":coherence_type, "score":score}, True)

    return score

# Lexicon-related
def get_lexicon_so(lexicon_id):
    # return lexicon_so_collection.find_one()
    return lexicon_so_collection.find_one({"id":lexicon_id})

def add_lexicon_so_entries(lexicon_entries):
    add_batch_entries(lexicon_so_collection, lexicon_entries)

# ANEW Lexicon-related

def get_anew_lexicon(anew_lexicon_id):
    return anew_lexicon_collection.find_one({"id":anew_lexicon_id})

def add_anew_lexicon_entries(anew_lexicon_entries):
    add_batch_entries(anew_lexicon_collection, anew_lexicon_entries)

# Friendship-related
def get_or_add_friendship(source_id, target_id):
    if source_id > target_id:
        return get_or_add_friendship(target_id, source_id)

    friendship_identifier = str(source_id)+"-"+str(target_id)

    try:
        from_db = json.loads(dumps(friendship_collection.find_one({"id":friendship_identifier})))
        if from_db:
            return from_db

        from_api = TweepyHelper.show_friendship(source_id, target_id)
        if from_api:
            new_dict = {"id": friendship_identifier, "following": from_api[0].following, "followed_by":from_api[0].followed_by}
            friendship_collection.insert_one(new_dict)

        return new_dict

    except:
        print("Unexpected error:", sys.exc_info()[0])
        return None

# Tweet-related
def get_or_add_tweet_db_given_json(tweet_json):
    return add_or_update_db_given_json(tweet_json, tweet_collection, Status.parse)

def get_or_add_tweet(tweet_id):
    return get_or_add(tweet_id, tweet_collection, TweepyHelper.retrieve_tweet, Status.parse)

def delete_tweet(tweet_id):
    tweet_collection.delete_one({"id":tweet_id})

# User-related

def get_or_add_user(user_id):
    return get_or_add(user_id, user_collection, TweepyHelper.retrieve_user, User.parse)

def delete_user(user_id):
    user_collection.delete_one({"id":user_id})

def get_or_add_following_ids(user_id):
    return get_or_add_list(user_id, following_collection, TweepyHelper.retrieve_following_ids, 'following_ids')

def get_or_add_followers_ids(user_id):
    return get_or_add_list(user_id, followers_collection, TweepyHelper.retrieve_followers_ids, 'followers_ids')

def delete_following_ids(user_id):
    following_collection.delete_one({"id":user_id})

def delete_followers_ids(user_id):
    followers_collection.delete_one({"id":user_id})

# Helper functions

def add_batch_entries(collection, entries):
     for entry in entries:
        collection.insert(entry)

def get_or_add_list(id, collection, retrieve_func, list_name):
    try:
        from_db = collection.find_one({'id':id})

        if from_db:
            # this means the API cannot give the information needed (based on a past query, so don't retry the query anymore); TODO: review this, might miss out on info not retrieved due to connectivity issues
            if UNAVAILABLE_KEY in from_db:
                return None
            else:
                return from_db[list_name]
        else:
            add_or_update_list_db(id, collection, retrieve_func, list_name)
        # return from_db[list_name] if from_db else add_or_update_list_db(id, collection, retrieve_func, list_name)
    except Exception as e:
        print("Get or add list exception: {}".format(e))
        return None

def add_or_update_list_db(id, collection, retrieve_func, list_name):
    from_api = retrieve_func(id)
    if from_api:
        from_api = [str(x) for x in from_api]
        json = {"id":id, list_name:from_api}
        collection.update({"id":id}, json, True)
    else:
        collection.update({"id":id}, {"id":id, UNAVAILABLE_KEY:True}, True)
    return from_api

def get_or_add(id, collection, retrieve_func, obj_constructor):
    try:
        from_db = json.loads(dumps(collection.find_one({"id":id})))

        if from_db:
            if UNAVAILABLE_KEY in from_db:
                return None
            else:
                return obj_constructor(TweepyHelper.api, from_db)
        else:
            add_or_update_db(id, collection, retrieve_func)
        # return obj_constructor(TweepyHelper.api, from_db) if from_db else add_or_update_db(id, collection, retrieve_func)
    except:
        print("Unexpected error:", sys.exc_info()[0])
        return None

def add_or_update_db(id, collection, retrieve_func):
    from_api = retrieve_func(id)
    if from_api:
        collection.update({"id":id}, from_api._json, True)
    else:
        collection.update({"id":id}, {"id":id, UNAVAILABLE_KEY:True}, True)
    return from_api

def add_or_update_db_given_json(json, collection, obj_constructor):
    try:
        collection.update({"id":json["id"]}, json, True)
        return obj_constructor(TweepyHelper.api, json)
    except Exception as e:
        print("Exception in add_or_update_db_given_json: {}".format(e))


from django.test import TestCase, Client
from django.urls import reverse

class TestViews(TestCase):

    def setUp(self):
        self.client = Client()
        self.driver_list_url = reverse("driver_list")
        self.constructor_list_url = reverse("constructor_list")
        self.circuit_list_url = reverse("circuit_list")

    def test_driver_list_GET(self):
        response = self.client.get(self.driver_list_url)
        self.assertEquals(response.status_code, 200)
        self.assertTemplateUsed(response, "f1app/driver_list.html")

    def test_constructor_list_GET(self):
        response = self.client.get(self.constructor_list_url)
        self.assertEquals(response.status_code, 200)
        self.assertTemplateUsed(response, "f1app/constructor_list.html")

    def test_circuit_list_GET(self):
        response = self.client.get(self.circuit_list_url)
        self.assertEquals(response.status_code, 200)
        self.assertTemplateUsed(response, "f1app/circuit_list.html")


from setuptools import setup
import os
import codecs

def fpath(name):
    return os.path.join(os.path.dirname(__file__), name)

def read(fname):
    return codecs.open(fpath(fname), encoding='utf-8').read()

requirements = read(fpath('requirements.txt'))

setup(name='scikit-dsp-comm',
      version='0.0.4',
      description='DSP and Comm package.',
      long_description=read(fpath('README.md')),
      author='Mark Wickert',
      author_email='mwickert@uccs.edu',
      url='https://github.com/mwickert/scikit-dsp-comm',
      package_dir={'sk_dsp_comm': 'sk_dsp_comm'},
      packages=['sk_dsp_comm'],
      package_data={'sk_dsp_comm': ['ca1thru37.txt']},
      license='BSD',
      install_requires=requirements.split(),
      test_suite='nose.collector',
      tests_require=['nose','numpy', 'tox'],
      extras_require={
            'helpers': ['pyaudio', 'pyrtlsdr']
      }
     )


import sys
from collections import deque

def BFS(baked, starts, n, m):
    flag = False  # 익을 수 있는 모든 토마토가 익은 후, 0이 남아있다면 True
    max_cnt = 0  # 가장 오래 걸리는 일수 저장
    q = deque(starts)
    dr = [(-1, 0), (1, 0), (0, -1), (0, 1)]

    while q:
        i, j = q.popleft()
        for di, dj in dr:
            if 0 <= i+di < n and 0 <= j+dj < m and not baked[i+di][j+dj]:
                baked[i+di][j+dj] = baked[i][j] + 1
                q.append((i+di, j+dj))
                max_cnt = max(max_cnt, baked[i+di][j+dj])

    for i in range(N):
        for j in range(M):
            if not baked[i][j]:
                flag = True
                break

    return -1 if flag else (max_cnt-1 if max_cnt > 0 else 0)

M, N = map(int, sys.stdin.readline().split())
tomato_info = [list(map(int, sys.stdin.readline().split())) for _ in range(N)]
start_positions = []
for i in range(N):
    for j in range(M):
        if tomato_info[i][j] == 1: start_positions.append((i, j))

print(BFS(tomato_info, start_positions, N, M))