import os
import numpy as np
from matplotlib import pyplot as plt, cm
from matplotlib.ticker import LinearLocator
import csv

def results_path(clip_path, file_name=''):
    """
    Creates a results directory for a given clip.
    :param clip_path:
    :return: Path to the results directory
    """

    clip_path = os.path.splitext(clip_path)[0]
    base_path, name = os.path.split(clip_path)
    res = base_path + '\\' + name + '\\'
    if not os.path.exists(res):
        os.makedirs(res)
    return res + file_name

def save_surface_plot(output_array: np.array, clip_file_name: str, output_file_name: str):
    """
    Shows and saves a surface plot, to be used with emd_response_mid_row.
    :param output_array:
    :param clip_file_name:
    :param output_file_name:
    :return:
    """
    fig, ax = plt.subplots(subplot_kw={"projection": "3d"})
    FRAMES, RESPONSES = output_array.shape
    x = np.array(range(FRAMES))
    y = np.array(range(RESPONSES))
    X, Y = np.meshgrid(x, y)
    surf = ax.plot_surface(np.transpose(X), np.transpose(Y), output_array, cmap=cm.coolwarm, linewidth=0,
                           antialiased=False)
    ax.set_ylabel('time [frames]')
    ax.set_xlabel('EMD')
    ax.set_zlabel('Amplitude')
    # Customize the z axis.
    # ax.set_zlim(-.015, .015)
    ax.zaxis.set_major_locator(LinearLocator(10))
    # A StrMethodFormatter is used automatically
    ax.zaxis.set_major_formatter('{x:.02f}')
    ax.set_title(clip_file_name + ' middle row EMD response')
    # Add a color bar which maps values to colors.
    fig.colorbar(surf, shrink=0.5, aspect=5)
    plt.savefig(output_file_name)
    plt.show()
    print("Done")

def write_dict_to_csv(d, file_name):
    """
    Writing a dictionary to a csv format
    :param d: A dictionary, whose fields are lists *of the same length*
    :param file_name:
    :return:
    """
    with open(file_name, "w", newline='') as outfile:
        writer = csv.writer(outfile)
        key_list = list(d.keys())
        writer.writerow(d.keys())
        for i in range(len(d[key_list[0]])):
            writer.writerow(d[key][i] for key in key_list)

def progress_bar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='█', printEnd="\r"):
    """
    Call in a loop to create terminal progress bar
    Print the value returned using ` end='' `
    Example: print(progress_bar(i, 100, prefix='Process: '), end='')
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    res = f'\r{prefix} |{bar}| {percent}% {suffix}'
    # Print New Line on Complete
    if iteration == total:
        res = res + '\n'
    return res


import subprocess
import argparse
from yt_dlp import YoutubeDL
import re
import ffmpeg
import logging

from transcripts.utils import (
    transcripts_dir,
    model_dir,
    whispercpp_dir,
    timeit,
)

log_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
logging.basicConfig(level=logging.INFO, format=log_fmt)
logger = logging.getLogger(__name__)

def extract_video_id(url):
    # Check for YouTube URL pattern
    youtube_match = re.search(r"v=([a-zA-Z0-9_-]+)", url)
    if youtube_match:
        return youtube_match.group(1)

    # Check for Vimeo URL pattern
    vimeo_match = re.search(r"vimeo.com/(\d+)", url)
    if vimeo_match:
        return vimeo_match.group(1)

    else:
        # If neither pattern is found, use yt-dlp to get the id
        ydl_opts = {"quiet": True}
        with YoutubeDL(ydl_opts) as ydl:
            info_dict = ydl.extract_info(url, download=False)
        return info_dict["id"]

def download_audio(url, outfile_name):
    try:
        # Extract the video ID from the URL using the same logic as extract_video_id
        video_id = extract_video_id(url)

        # Determine the appropriate URL prefix based on the source
        if "youtube.com" in url:
            url_prefix = "https://www.youtube.com/watch?v="
            full_url = url_prefix + video_id
        elif "vimeo.com" in url:
            url_prefix = "https://vimeo.com/"
            full_url = url_prefix + video_id
        else:
            # Use the original URL if it doesn't match known patterns
            full_url = url

        options = {
            "format": "bestaudio/best",
            "postprocessors": [
                {
                    "key": "FFmpegExtractAudio",
                    "preferredcodec": "wav",
                    "preferredquality": "192",
                }
            ],
            "outtmpl": str(transcripts_dir / outfile_name),
        }
        with YoutubeDL(options) as ydl:
            print(f"Downloading: {full_url}")
            ydl.download([full_url])
    except Exception as e:
        print(f"Error in download_audio: {e}")

# Extract audio from video
def extract_audio(video_file, audio_file):
    """
    Extract audio from video

    Parameters
    ----------
    video_file : str
        Path to video file
    audio_file : str
        Path to audio file

    Returns
    -------
    None
    """
    logging.info(
        f"Using ffmpeg to extract audio from {video_file} to {audio_file}"
    )
    try:
        (
            ffmpeg.input(video_file)
            .output(audio_file, acodec="mp3", ac=2, ar="48k", ab="192k")
            .run(overwrite_output=True)
        )
        return audio_file
    except Exception as e:
        print(f"Error extracting audio: {e}")
        return None

def ensure_wav_16k(filename):
    input_path = transcripts_dir / f"{filename}.wav"
    output_path = transcripts_dir / f"{filename}_16k.wav"
    # always override with yes
    command = ["ffmpeg", "-y", "-i", input_path, "-ar", "16000", output_path]
    subprocess.run(command)

def is_transcript_valid(filename, directory=transcripts_dir):
    input_path = directory / f"{filename}.txt"

    try:
        with open(input_path, "r", encoding="utf-8") as f:
            f.read()  # Attempt to read the entire file
        return True
    except UnicodeDecodeError:
        return False

@timeit
def run_whisper(filename, model_name, num_threads=7, num_processors=1):
    model_path = model_dir / model_name
    input_path = transcripts_dir / f"{filename}_16k.wav"
    output_path = transcripts_dir / f"{filename}.txt"

    whisper_main = whispercpp_dir / "main"
    command = f"{whisper_main} -m {model_path} -t {num_threads} -p {num_processors} -f {input_path} > {output_path}"
    subprocess.run(command, shell=True)

    # if is_transcript_valid(filename):
    #     logging.info(f"Transcript for {filename} is valid.")
    # else:
    #     logging.info(f"Transcript for {filename} contains invalid bytes.")
    #     # delete transcript

def clean_up_wav_files(base_filename):
    """
    Deletes the .wav and _16k.wav files for the given base filename.

    Parameters
    ----------
    base_filename : str
        The base name of the .wav files to delete.
    """
    try:
        wav_path = transcripts_dir / f"{base_filename}.wav"
        wav_16k_path = transcripts_dir / f"{base_filename}_16k.wav"

        # Remove the original wav file
        if wav_path.exists():
            wav_path.unlink()
            logging.info(f"Deleted {wav_path}")

        # Remove the 16kHz wav file
        if wav_16k_path.exists():
            wav_16k_path.unlink()
            logging.info(f"Deleted {wav_16k_path}")

    except Exception as e:
        logging.error(f"Error during file cleanup: {e}")

def main():
    parser = argparse.ArgumentParser(
        description="Automate downloading and processing audio from YouTube/Vimeo."
    )
    parser.add_argument("url", type=str, help="URL of the video")
    parser.add_argument(
        "--model_name",
        type=str,
        default="ggml-large-v2.bin",
        help="Name of the whisper model to use",
    )
    args = parser.parse_args()

    # Variable for .wav file name
    wav_file_name = extract_video_id(args.url)

    logging.info("Download audio")
    download_audio(args.url, wav_file_name)

    logging.info("Ensure the wav file is 16 kHz")
    ensure_wav_16k(wav_file_name)

    logging.info("Run whisper.cpp using Metal")
    run_whisper(wav_file_name, args.model_name)

    # Clean up WAV files after processing
    clean_up_wav_files(wav_file_name)

    logging.info("WAV files cleanup complete.")

if __name__ == "__main__":
    main()


"""
word frequencies statistic
"""
import jieba
from pathlib import Path
data_path = Path('../data')
stat_path = data_path.joinpath('stat')
query_data_path = data_path.joinpath('byte_camp_query.txt')

def run():
    freq_dict = {}
    with open(query_data_path) as fin:
        for line in fin.readlines():
            line = line.strip('\n').strip()
            fileds = line.split('\t')

            if len(fileds) != 3:
                continue

            spt_word = jieba.cut(fileds[0])
            for word in spt_word:
                if len(word) < 2:
                    continue
                if word in freq_dict:
                    freq_dict[word] = freq_dict[word] + 1
                else:
                    freq_dict[word] = 1

    sorted_word = sorted(freq_dict.items(),key=lambda x:x[1], reverse=True) # sort dict with frequency

    stat_path.mkdir(exist_ok=True)
    filename = stat_path.joinpath('word_freq.txt')
    with open(filename, 'w') as fout:
        for key, value in sorted_word:
            fout.write(key + ' ' + str(value) + '\n')

if __name__ == '__main__':
    run()


from flask import Flask, render_template
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import MetaData

import config

naming_convention = {
    "ix": 'ix_%(column_0_label)s',
    "uq": "uq_%(table_name)s_%(column_0_name)s",
    "ck": "ck_%(table_name)s_%(column_0_name)s",
    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
    "pk": "pk_%(table_name)s"
}
db = SQLAlchemy(metadata=MetaData(naming_convention=naming_convention))
migrate = Migrate()

def page_not_found(e):
    return render_template('404.html'), 404

def create_app():
    app = Flask(__name__)
    app.config.from_object(config)

    #app.run(host='0.0.0.0', port=5000)

    db.init_app(app)
    if app.config['SQLALCHEMY_DATABASE_URI'].startswith("sqlite"):
        migrate.init_app(app, db, render_as_batch=True)
    else:
        migrate.init_app(app, db)

    from . import models
    from .views import main_views, board_views, auth_views, comment_views, guide_views, chart_views, check_views, flask_upload, test
    app.register_blueprint(main_views.bp)
    app.register_blueprint(board_views.bp)
    app.register_blueprint(auth_views.bp)
    app.register_blueprint(comment_views.bp)
    app.register_blueprint(guide_views.bp)
    app.register_blueprint(chart_views.bp)
    app.register_blueprint(check_views.bp)
    app.register_blueprint(flask_upload.bp)
    app.register_blueprint(test.bp)

    from .filter import format_datetime
    app.jinja_env.filters['datetime'] = format_datetime

    app.register_error_handler(404, page_not_found)

    return app


# 나이브 베이즈는 분류기를 만들 수 있는 간단한 기술로써
# 단일 알고리즘을 통한 훈련이 아닌 일반적인 원칙에 근거한 여러 알고리즘들을 이용하여 훈련된다.
# 모든 나이브 베이즈 분류기는 공통적으로 모든 특성 값은 서로 독립임을 가정한다.
# 예를 들어, 특정 과일을 사과로 분류 가능하게 하는 특성들 (둥글다, 빨갛다, 지름 10cm)은
# 나이브 베이즈 분류기에서 특성들 사이에서 발생할 수 있는 연관성이 없음을 가정하고
# 각각의 특성들이 특정 과일이 사과일 확률에 독립적으로 기여 하는 것으로 간주한다.

# 조건부 확률 :feature가 발생했다는 가정하에 label이 발생할 확률
# P(Label:feature)=P(Feature|Label) * P(label) / P(Feature)
from sklearn.naive_bayes import GaussianNB
import numpy as np
from sklearn import metrics

x=np.array([1,2,3,4,5])
x=x[:,np.newaxis] #newaxis: 차원확장
print(x)
y=np.array([1,3,5,7,9])

model=GaussianNB().fit(x,y)
pred=model.predict(x)
print('분류정확도: ', metrics.accuracy_score(y,pred))

# 새로운 값으로 예측
new_x=np.array([[0.1],[0.5],[5],[12]])
new_pred=model.predict(new_x)
print(new_pred)

#.....


import pandas as pd
import glob
import json
import os

print(os.getcwd())
files = glob.glob('tweet/*')
length = len(files)

print(length)

dictlist = []

for file in files:
    json_string = open(file, 'r', encoding="utf-8").read()
    json_dict = json.loads(json_string)
    dictlist.append(json_dict)

df = pd.DataFrame(dictlist)
print(df)

df = df.replace({'\n': ' '}, regex=True)
df = df.replace({'\t': ' '}, regex=True)
df = df.replace({'\r': ' '}, regex=True)

df
print(df)

df.to_csv("data.csv")

print('Completed writing to csv')


from __future__ import annotations
from ipaddress import IPv4Address, IPv4Network
from validators import ValidatorIpV4

class CalcSubnetIpv4:
    """
    Класс калькулятора подсети
    """
    def __init__(self):
        self.validator = ValidatorIpV4()

    def run(self, file: str) -> tuple:
        """
        Основной метод.
        Достаёт список ip-адресов из файла, вызывает валидации и находит подсеть
        """
        with open(file) as ips_file:
            ips_list = ips_file.read().splitlines()
            result_validating = self.validator.validating(ips_list)
            if result_validating[0]:
                return result_validating[0], self.calc_subnet(ips_list)
            return result_validating

    def calc_subnet(self, ips_list: list[str]) -> IPv4Network:
        """
        Метод расчёта подсети для списка ip-адресов
        """
        ips_list = [IPv4Address(ip) for ip in ips_list]
        min_ip = min(ips_list)
        max_ip = max(ips_list)
        ip_range = 0
        net = IPv4Network(f'{min_ip}/32', strict=False)
        while max_ip not in net:
            ip_range += 1
            net = IPv4Network(f'{min_ip}/{32 - ip_range}', strict=False)
        return net


"""
Letter Finder
Name : Min Soo (Mike) HWANG
Github ID : Delivery_KiKi
"""

from typing import TextIO
from io import StringIO
from itertools import combinations

def a_finder(filename: str, letter: str) -> None:
    """ Open a text file, preprocess the data, and find probability that a set of letters contains the certain letter.

    :precondition: the functions in a_finder must be correctly operated
    :postcondition: open a text file, process the data, and find probability that
                    a set of letters among the second line of text contains the certain letter
    :return: nothing
    """
    with open(filename) as file_object:
        text_list = process_data(file_object)
    calculate_probability(letter, text_list)

def process_data(file_object: TextIO) -> list:
    """ Take the text file, split a line of text by a space, and store the value to a list.

    :param file_object: an object of the text file.
    :precondition: each line of text must be separated by space and each line must be separated by <Enter> key
    :postcondition: make a list of the lists that contain the information of each line of text separating by a space
    :return: a list of lists that contains the information of each line of text

    >>> text_file = StringIO('')
    >>> process_data(text_file)
    []
    >>> text_file = StringIO('\\n\\n\\n\\n\\n')
    >>> process_data(text_file)
    [[''], [''], [''], [''], ['']]
    >>> text_file = StringIO('1.3 3.4\\n2 4.2\\n-1 1\\n')
    >>> process_data(text_file)
    [['1.3', '3.4'], ['2', '4.2'], ['-1', '1']]
    >>> text_file = StringIO('4\\na a b c\\n2\\n')
    >>> process_data(text_file)
    [['4'], ['a', 'a', 'b', 'c'], ['2']]
    """
    text_list = [line.replace('\n', '').split(' ') for line in file_object]
    return text_list

def calculate_probability(letter: str, text_list: list) -> None:
    """ Find the probability that the sequence of letters contains the certain letter.

    :param letter: a letter that will be used for calculating the probability
    :param text_list: a list of the lists that contain the information of each row of the text file
    :precondition: letter can be any letters but spaces
    :precondition: the length of text_list must be 3
    :precondition: text_list[0] must be a string of a number between 1 to 10 inclusive
    :precondition: the length of text_list[1] must be the same as the numerical value of text_list[0]
                   and made up of any characters but spaces
    :precondition: text_list[1] must be a string of a number between 1 to the numerical value of text_list[0] inclusive
    :postcondition: print the probability that sequence of letters in text_list[1] contains letter
    :return: nothing

    >>> string_to_find = 'a'
    >>> line_list = [['4'], ['a', 'a', 'b', 'c'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more 'a'
    >>> string_to_find = 'b'
    >>> line_list = [['1'], ['a'], ['1']]
    >>> calculate_probability(string_to_find, line_list)
    0.0 probability that a pair of items selected will contain one or more 'b'
    >>> string_to_find = 'c'
    >>> line_list = [['5'], ['c', 'b', 'a', 't', 'a'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.4 probability that a pair of items selected will contain one or more 'c'
    >>> string_to_find = '&'
    >>> line_list = [['4'], ['&', '&', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more '&'
    >>> string_to_find = 'A'
    >>> line_list = [['4'], ['A', 'A', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more 'A'
    >>> string_to_find = 'Z'
    >>> line_list = [['4'], ['Z', 'Z', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more 'Z'
    >>> string_to_find = '0'
    >>> line_list = [['4'], ['0', '0', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more '0'
    >>> string_to_find = '9'
    >>> line_list = [['4'], ['9', '9', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more '9'
    >>> string_to_find = '1234567890'
    >>> line_list = [['4'], ['1234567890', '1234567890', 'a', 't'], ['2']]
    >>> calculate_probability(string_to_find, line_list)
    0.8333 probability that a pair of items selected will contain one or more '1234567890'
    """
    try:
        letters = text_list[1]
        n_list = list(range(len(letters)))
        number_k = int(text_list[2][0])
        result = set()
        combination_list = list(combinations(n_list, number_k))
        for index in combination_list[:]:
            for number in index:
                if letters[number] == letter:
                    result.add(index)
        probability = round(len(result) / len(combination_list), 4)
        print(f"{probability} probability that a pair of items selected will contain one or more '{letter}'")
    except ZeroDivisionError:
        print('CHECK THE TEXT FILE.')
        print("The second line in the text file must include letters divided by a space.")
    except IndexError:
        print('CHECK THE TEXT FILE.')
        print("The third line in the text file must include an integer.")

def main() -> None:
    """
    Drives the program.
    """
    # import doctest
    # doctest.testmod(verbose=True)
    a_finder('find_probability.txt', 'a')

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
GeomMop configuration file parsers

This file contains the parsing functions for configuration files of Flow123d.
Currently supports .con format (as specified by Flow123d manual v1.8.2).

Future support for .yaml files for Flow123d v2 is planned.

@author: Tomas Krizek
"""

import demjson
import re
from copy import copy

from .model import DataNode, RefError

def parse_con(filename):
    """
    Parses a configuration file of Flow123d in .con format with given filename.
    If the input file includes references, they are resolved and represented
    by actual python references in the output.

    Returns the root of the resulting tree DataNode structure.
    """
    con = open(filename).read()
    data = _decode_con(con)
    return _resolve_references(data)

def con_to_yaml(filename_in, filename_out):
    """
    Converts the con file into yaml file.
    Does not resolve references.
    """
    import yaml
    with open(filename_in) as con_file:
        con = con_file.read()
        data = _decode_con(con)
        with open(filename_out, 'w') as stream:
            yaml.dump(data, stream)

def _resolve_references(data):
    """
    Resolves references in data. Replaces REF keys with actual Python
    references.
    """
    refs = _extract_references(data)
    root = DataNode(data)
    while len(refs) > 0:
        length = len(refs)
        for path, ref_path in copy(refs).items():
            try:
                node = root.get(path)
                if ref_path.startswith('/'):  # absolute ref
                    node.ref = root.get(ref_path)
                else:  # relative ref
                    node.ref = node.get(ref_path)
            except LookupError:
                continue
            else: del refs[path]
        if length == len(refs):
            # no reference removed -> impossible to resolve references
            raise RefError("Can not resolve references.")
    return root

def _decode_con(con):
    """Reads .con format and returns read data in form of dicts and lists."""
    pattern = re.compile(r"\s?=\s?")  # TODO can I replace = simply like this?
    con = pattern.sub(':', con)
    return demjson.decode(con)

def _extract_references(json):
    """
    Crawl through the data and find all REF keys.

    Returns a dictionary of their locations and where they point.
    """
    def crawl(data, path):
        if isinstance(data, dict):
            try:
                ref_path = data['REF']
            except KeyError:
                pass
            else:
                refs[path] = ref_path
                del data['REF']
            # crawl for all keys
            for key, value in data.items():
                crawl(value, '{path}/{key}'.format(path=path, key=key))
        elif isinstance(data, list):
            # crawl for all records
            for i, item in enumerate(data):
                crawl(item, '{path}/{key}'.format(path=path, key=i))

    refs = {}
    crawl(json, '')
    return refs


import logging

from pyramid.view import view_config
from pyramid.events import ApplicationCreated
from pyramid.events import subscriber

from . import registration
from .appenv import Node

log = logging.getLogger(__name__)

@view_config(route_name='services', renderer='json')
def services(request):
    web_apps_dir = request.registry.settings['web_apps_dir']
    node = Node(web_apps_dir)

    return node.repo_data

@view_config(route_name='tag', renderer='json')
def tag(request):
    web_apps_dir = request.registry.settings['web_apps_dir']
    node = Node(web_apps_dir)
    apps = [s.strip() for s in request.POST['apps'].split(',')]
    return node.tag_apps(
        apps,
        request.POST['tag'],
        request.POST['description'])

@view_config(route_name='mark_as_deployed', renderer='json')
def mark_as_deployed(request):
    web_apps_dir = request.registry.settings['web_apps_dir']
    node = Node(web_apps_dir)
    apps = [s.strip() for s in request.POST['apps'].split(',')]

    return node.mark_as_deployed(apps, request.POST['tag'])

@subscriber(ApplicationCreated)
def register_me(event):
    """
    Register this Bambino node with Doula.
    """
    settings = event.app.registry.settings

    machine_info = Node.get_machine_info()
    machine_info['url'] = 'http://%s:%s' % (
        machine_info['ip'], settings['port'])

    registration.register_bambino(
        machine_info,
        settings['register_url'],
        settings['registration_interval'])


import numpy as np
import math as m
from PIL import Image, ImageDraw

MAX_ITERATIONS = 75

def convert_in_ones(Y):
    #
    Y_new = Y
    for n in range(len(Y_new[0])):
        if 0 <= Y[0][n][0]:
            Y_new[0][n][0] = 1
        else:
            Y_new[0][n][0] = -1
    return Y_new

def if_equal(Y, X):
    for j in range(len(X)):
        k = 0
        for i in range(len(Y[0])):
            if Y[0][i][0] == X[j][i][0]:
                k = k + 1
        if k == len(Y[0]):
            return True
    return False

def draw_image(path, Y, SIZE):
    image = Image.new('1', (SIZE, SIZE), "white")
    draw = ImageDraw.Draw(image)
    index = 0
    for i in range(SIZE):
        for j in range(SIZE):
            if Y[index][0] < 0:
                clr = 0
            else:
                clr = 1
            index = index + 1
            draw.point((i, j), clr)
    image.save(path)
    image.show()

def activation_function(Y):
    for i in range(len(Y)):
        for j in range(len(Y[i])):
            Y[i][j] = th(Y[i][j])
    return Y

def sh(x):
    return m.exp(x) - m.exp(-x)

def ch(x):
    return m.exp(x) + m.exp(-x)

def th(x):
    return sh(x) / ch(x)

def transparent(X):
    return X.T

def get_weights(X, SIZE):
    W = np.zeros((SIZE**2, SIZE**2))
    for n in range(len(X)):
        ups = (W @ X[n] - X[n]) @ transparent(W @ X[n] - X[n])
        W = W + (ups / (transparent(X[n]) @ X[n] - transparent(X[n]) @ W @ X[n]))
    return W

def get_image(img_path, SIZE):
    X = []
    one = 1
    negative_one = -1
    for n in range(len(img_path)):
        index = 0
        image = Image.open(img_path[n])
        pixels = image.load()
        x = np.zeros((SIZE*SIZE, 1))
        for i in range(SIZE):
            for j in range(SIZE):
                if pixels[i, j][0] == 255:
                    required_pixel = one
                else:
                    required_pixel = negative_one
                x[index] = required_pixel
                index = index + one
        X.append(x)
    return X

def get_image_and_weights(img_path, SIZE):
    X = get_image(img_path, SIZE)
    W = get_weights(X, SIZE)
    return X, W

def while_cycle(X, W, SIZE):
    input_image = input("Введите название цифры которую будет распозновать: ")
    input_image_show = Image.open("curved_images/%s.png" % input_image)
    print("Открываю картинку цифры %s.." % input_image)
    input_image_show.show()
    Y = get_image(["curved_images/%s.png" % input_image], SIZE)

    iteration = 0
    flag = True

    while flag:
        if iteration >= MAX_ITERATIONS:
            print("Восстановить не удалось :(")
            break
        Y = [activation_function((W @ Y[0]))]
        Y_new = convert_in_ones(Y)
        if if_equal(Y_new, X):
            print("Распознал!")
            print("Восстановленное изображение: image_result/img.png")
            draw_image(f"output_images/img.png", Y[0], SIZE)
            flag = False
        iteration = iteration + 1
        pass

def main():
    SIZE = 30
    img_path = ["normal_images/0.png", "normal_images/1.png",
                "normal_images/2.png", "normal_images/3.png",
                "normal_images/4.png", "normal_images/5.png",
                "normal_images/6.png", "normal_images/7.png",
                "normal_images/8.png", "normal_images/9.png"]

    X, W = get_image_and_weights(img_path, SIZE)

    while_cycle(X, W, SIZE)

if __name__ == "__main__":
    main()


import qrcode
#setting the qr code size and width
qr=qrcode.QRCode(version=1, box_size=10, border=5)
# setting the data to the qr code
data = 'hii i am Niranjan'

qr.add_data(data)
qr.make(fit=True)
# saving the qr code as image
myqrcode = qr.make_image(fill='black', back_color='white')

myqrcode.save('Myqrcode.png')


import re

from datetime import datetime as dt
import pandas as pd
from pathlib import Path

def extract_datetime_from_filename(filename: str):
    """Extract date and time from a filename.

    The filename should have the format 'YYYYMMDD_HHMMSS' somewhere in its name.

    Parameters
    ----------
    filename : str
        The filename to extract the date and time from.

    Returns
    -------
    tuple(datetime.datetime, datetime.datetime)
        A tuple containing the date and time as datetime objects.
    """
    datetime_str = re.search(r'\d{8}_\d{6}', filename)
    if datetime_str:
        datetime_str = datetime_str.group()
        date_obj = dt.strptime(datetime_str[:8], '%Y%m%d')
        time_obj = dt.strptime(datetime_str[9:], '%H%M%S').time()
        return date_obj, time_obj
    else:
        return None, None

def list_audio_files(path: str) -> pd.DataFrame:
    """List all audio files (wav) in a directory and its subdirectories.

    The files should be in wav format, and the filename should have the format 'YYYYMMDD_HHMMSS'
    somewhere in its name.

    Parameters
    ----------
    path : str
        The path to the directory containing audio files.

    Returns
    -------
    pd.DataFrame
        A DataFrame with the following columns:
            * 'Name': Name of each audio file.
            * 'Folder': Name of the parent folder for each audio file.
            * 'Date': Date extracted from the filename in format (YYYY-MM-DD).
            * 'Time': Time extracted from the filename in format (HH:MM:SS).
            * 'Path': Full path of each audio file.
    """
    audio_data = []

    try:
        for audio_path in Path(path).rglob('*.[Ww][Aa][Vv]'):
            try:
                file_name = audio_path.name
                parent_folder = audio_path.parent.name
                date, time = extract_datetime_from_filename(file_name)

                audio_data.append({
                    'Name': file_name,
                    'Folder': parent_folder,
                    'Date': date,
                    'Time': time,
                    'Path': str(audio_path)
                })
            except Exception as e:
                print(f"Error processing file: {str(audio_path)} - {e}")

    except Exception as e:
        print(f"Error accessing path: {path} - {e}")

    return pd.DataFrame(audio_data)


# -------------------------------------------------------------------------------------------------------------------------------------
# Following code curated for Bio-Diffusion (https://github.com/BioinfoMachineLearning/bio-diffusion):
# -------------------------------------------------------------------------------------------------------------------------------------

import glob
import os
import imageio
import random
import torch
import matplotlib
import wandb
import numpy as np
import matplotlib.pyplot as plt
import prody as pr

from torch import nn
from torch_scatter import scatter
from torch_geometric.data import Batch
from matplotlib.axes._subplots import Axes
from typing import Any, Callable, Dict, List, Optional, Union, Tuple
from wandb.sdk.wandb_run import Run
from pathlib import Path
from rdkit import Chem

from torchtyping import TensorType, patch_typeguard
from typeguard import typechecked

from src.datamodules.components.edm import get_bond_order
from src.utils.pylogger import get_pylogger

log = get_pylogger(__name__)

try:
    from pymol import cmd
except ImportError:
    log.warning("PyMOL not found.")

patch_typeguard()  # use before @typechecked

pr.confProDy(verbosity="none")

matplotlib.use("Agg")

@typechecked
def centralize(
    batch: Batch,
    key: str,
    batch_index: torch.Tensor,
    node_mask: Optional[TensorType["batch_num_nodes"]] = None,
    edm: bool = False
) -> Tuple[torch.Tensor, torch.Tensor]:  # note: cannot make assumptions on output shape
    if node_mask is not None:
        # derive centroid of each batch element
        if edm:
            masked_max_abs_value = batch[key][~node_mask].abs().sum().item()
            assert masked_max_abs_value < 1e-5, f"Masked CoG error {masked_max_abs_value} is too high"

            num_entities = scatter(
                node_mask.float(),
                batch_index,
                dim=0,
                reduce="sum"
            ).unsqueeze(-1)
            entities_sum = scatter(
                batch[key],
                batch_index,
                dim=0,
                reduce="sum"
            )
            entities_centroid = entities_sum / num_entities
        else:
            entities_centroid = scatter(
                batch[key][node_mask],
                batch_index[node_mask],
                dim=0,
                reduce="mean"
            )  # e.g., [batch_size, 3]

        # center entities using corresponding centroids
        if edm:
            entities_centered = batch[key] - (entities_centroid[batch_index] * node_mask.float().unsqueeze(-1))
        else:
            masked_values = (
                torch.zeros_like(batch[key])
                if edm
                else torch.ones_like(batch[key]) * torch.inf
            )
            values = batch[key][node_mask]
            masked_values[node_mask] = (values - entities_centroid[batch_index][node_mask])
            entities_centered = masked_values

    else:
        # derive centroid of each batch element, and center entities using corresponding centroids
        entities_centroid = scatter(batch[key], batch_index, dim=0, reduce="mean")  # e.g., [batch_size, 3]
        entities_centered = batch[key] - entities_centroid[batch_index]

    return entities_centroid, entities_centered

@typechecked
def decentralize(
    batch: Batch,
    key: str,
    batch_index: torch.Tensor,
    entities_centroid: torch.Tensor,
    node_mask: Optional[torch.Tensor] = None,
    edm: bool = False
) -> torch.Tensor:  # note: cannot make assumptions on output shape
    if node_mask is not None:
        if edm:
            entities_centered = batch[key] + (entities_centroid[batch_index] * node_mask.float().unsqueeze(-1))
        else:
            masked_values = torch.ones_like(batch[key]) * torch.inf
            masked_values[node_mask] = (batch[key][node_mask] + entities_centroid[batch_index])
            entities_centered = masked_values
    else:
        entities_centered = batch[key] + entities_centroid[batch_index]
    return entities_centered

@typechecked
def localize(
    x: TensorType["batch_num_nodes", 3],
    edge_index: TensorType[2, "batch_num_edges"],
    norm_x_diff: bool = True,
    node_mask: Optional[torch.Tensor] = None
) -> TensorType["batch_num_edges", 3, 3]:
    row, col = edge_index[0], edge_index[1]

    if node_mask is not None:
        edge_mask = node_mask[row] & node_mask[col]

        x_diff = torch.ones((edge_index.shape[1], 3), device=edge_index.device) * torch.inf
        x_diff[edge_mask] = x[row][edge_mask] - x[col][edge_mask]

        x_cross = torch.ones((edge_index.shape[1], 3), device=edge_index.device) * torch.inf
        x_cross[edge_mask] = torch.cross(x[row][edge_mask], x[col][edge_mask])
    else:
        x_diff = x[row] - x[col]
        x_cross = torch.cross(x[row], x[col])

    if norm_x_diff:
        # derive and apply normalization factor for `x_diff`
        if node_mask is not None:
            norm = torch.ones((edge_index.shape[1], 1), device=x_diff.device)
            norm[edge_mask] = (
                torch.sqrt(torch.sum((x_diff[edge_mask] ** 2), dim=1).unsqueeze(1))
            ) + 1
        else:
            norm = torch.sqrt(torch.sum((x_diff) ** 2, dim=1).unsqueeze(1)) + 1
        x_diff = x_diff / norm

        # derive and apply normalization factor for `x_cross`
        if node_mask is not None:
            cross_norm = torch.ones((edge_index.shape[1], 1), device=x_cross.device)
            cross_norm[edge_mask] = (
                torch.sqrt(torch.sum((x_cross[edge_mask]) ** 2, dim=1).unsqueeze(1))
            ) + 1
        else:
            cross_norm = (torch.sqrt(torch.sum((x_cross) ** 2, dim=1).unsqueeze(1))) + 1
        x_cross = x_cross / cross_norm

    if node_mask is not None:
        x_vertical = torch.ones((edge_index.shape[1], 3), device=edge_index.device) * torch.inf
        x_vertical[edge_mask] = torch.cross(x_diff[edge_mask], x_cross[edge_mask])
    else:
        x_vertical = torch.cross(x_diff, x_cross)

    f_ij = torch.cat((x_diff.unsqueeze(1), x_cross.unsqueeze(1), x_vertical.unsqueeze(1)), dim=1)
    return f_ij

@typechecked
def scalarize(
    vector_rep: TensorType["batch_num_entities", 3, 3],
    edge_index: TensorType[2, "batch_num_edges"],
    frames: TensorType["batch_num_edges", 3, 3],
    node_inputs: bool,
    dim_size: int,
    node_mask: Optional[TensorType["batch_num_nodes"]] = None
) -> TensorType["effective_batch_num_entities", 9]:
    row, col = edge_index[0], edge_index[1]

    # gather source node features for each `entity` (i.e., node or edge)
    # note: edge inputs are already ordered according to source nodes
    vector_rep_i = vector_rep[row] if node_inputs else vector_rep

    # project equivariant values onto corresponding local frames
    if vector_rep_i.ndim == 2:
        vector_rep_i = vector_rep_i.unsqueeze(-1)
    elif vector_rep_i.ndim == 3:
        vector_rep_i = vector_rep_i.transpose(-1, -2)

    if node_mask is not None:
        edge_mask = node_mask[row] & node_mask[col]
        local_scalar_rep_i = torch.zeros((edge_index.shape[1], 3, 3), device=edge_index.device)
        local_scalar_rep_i[edge_mask] = torch.matmul(
            frames[edge_mask], vector_rep_i[edge_mask]
        )
        local_scalar_rep_i = local_scalar_rep_i.transpose(-1, -2)
    else:
        local_scalar_rep_i = torch.matmul(frames, vector_rep_i).transpose(-1, -2)

    # reshape frame-derived geometric scalars
    local_scalar_rep_i = local_scalar_rep_i.reshape(vector_rep_i.shape[0], 9)

    if node_inputs:
        # for node inputs, summarize all edge-wise geometric scalars using an average
        return scatter(
            local_scalar_rep_i,
            # summarize according to source node indices due to the directional nature of GCP2's equivariant frames
            row,
            dim=0,
            dim_size=dim_size,
            reduce="mean"
        )

    return local_scalar_rep_i

@typechecked
def vectorize(
    gate: TensorType["batch_num_entities", 9],
    edge_index: TensorType[2, "batch_num_edges"],
    frames: TensorType["batch_num_edges", 3, 3],
    node_inputs: bool,
    dim_size: int,
    node_mask: Optional[TensorType["batch_num_nodes"]] = None
) -> TensorType["effective_batch_num_entities", 3, 3]:
    row, col = edge_index

    frames = frames.reshape(frames.shape[0], 1, 9)
    x_diff, x_cross, x_vertical = frames[:, :, :3].squeeze(
    ), frames[:, :, 3:6].squeeze(), frames[:, :, 6:].squeeze()

    # gather source node features for each `entity` (i.e., node or edge)
    gate = gate[row] if node_inputs else gate  # note: edge inputs are already ordered according to source nodes

    # derive edge mask if provided node mask
    if node_mask is not None:
        edge_mask = node_mask[row] & node_mask[col]

    # use invariant scalar features to derive new vector features using each neighboring node
    gate_vector = torch.zeros_like(gate)
    for i in range(0, gate.shape[-1], 3):
        if node_mask is not None:
            gate_vector[edge_mask, i:i + 3] = (
                gate[edge_mask, i:i + 1] * x_diff[edge_mask]
                + gate[edge_mask, i + 1:i + 2] * x_cross[edge_mask]
                + gate[edge_mask, i + 2:i + 3] * x_vertical[edge_mask]
            )
        else:
            gate_vector[:, i:i + 3] = (
                gate[:, i:i + 1] * x_diff
                + gate[:, i + 1:i + 2] * x_cross
                + gate[:, i + 2:i + 3] * x_vertical
            )
    gate_vector = gate_vector.reshape(gate_vector.shape[0], 3, 3)

    # for node inputs, summarize all edge-wise geometric vectors using an average
    if node_inputs:
        return scatter(
            gate_vector,
            # summarize according to source node indices due to the directional nature of GCP2's equivariant frames
            row,
            dim=0,
            dim_size=dim_size,
            reduce="mean"
        )

    return gate_vector

@typechecked
def safe_norm(
    x: torch.Tensor,
    dim: int = -1,
    eps: float = 1e-8,
    keepdim: bool = False,
    sqrt: bool = True
):
    norm = torch.sum(x ** 2, dim=dim, keepdim=keepdim)
    if sqrt:
        norm = torch.sqrt(norm + eps)
    return norm + eps

@typechecked
def is_identity(nonlinearity: Optional[Union[Callable, nn.Module]] = None):
    return nonlinearity is None or isinstance(nonlinearity, nn.Identity)

@typechecked
def norm_no_nan(
    x: torch.Tensor,
    dim: int = -1,
    keepdim: bool = False,
    eps: float = 1e-8,
    sqrt: bool = True
):
    """
    From https://github.com/drorlab/gvp-pytorch

    L2 norm of tensor clamped above a minimum value `eps`.

    :param sqrt: if `False`, returns the square of the L2 norm
    """
    out = torch.clamp(torch.sum(torch.square(x), dim=dim, keepdim=keepdim), min=eps)
    return torch.sqrt(out) if sqrt else out

@typechecked
def num_nodes_to_batch_index(
    num_samples: int,
    num_nodes: Union[int, TensorType["batch_size"]],
    device: Union[torch.device, str]
) -> TensorType["batch_num_nodes"]:
    assert isinstance(num_nodes, int) or len(num_nodes) == num_samples
    sample_inds = torch.arange(num_samples, device=device)
    return torch.repeat_interleave(sample_inds, num_nodes)

@typechecked
def save_xyz_file(
    path: str,
    positions: TensorType["batch_num_nodes", 3],
    one_hot: TensorType["batch_num_nodes", "num_atom_types"],
    charges: torch.Tensor,  # TODO: incorporate charges within saved XYZ file
    dataset_info: Dict[str, Any],
    id_from: int = 0,
    name: str = "molecule",
    batch_index: Optional[TensorType["batch_num_nodes"]] = None
):
    try:
        os.makedirs(path)
    except OSError:
        pass

    if batch_index is None:
        batch_index = torch.zeros(len(one_hot))

    for batch_i in torch.unique(batch_index):
        current_batch_index = (batch_index == batch_i)
        num_atoms = int(torch.sum(current_batch_index).item())
        f = open(path + name + "_" + "%03d.xyz" % (batch_i + id_from), "w")
        f.write("%d\n\n" % num_atoms)
        atoms = torch.argmax(one_hot[current_batch_index], dim=-1)
        batch_pos = positions[current_batch_index]
        for atom_i in range(num_atoms):
            atom = atoms[atom_i]
            atom = dataset_info["atom_decoder"][atom]
            f.write("%s %.9f %.9f %.9f\n" % (atom, batch_pos[atom_i, 0], batch_pos[atom_i, 1], batch_pos[atom_i, 2]))
        f.close()

@typechecked
def write_xyz_file(
    positions: TensorType["num_nodes", 3],
    atom_types: TensorType["num_nodes"],
    filename: str
):
    out = f"{len(positions)}\n\n"
    assert len(positions) == len(atom_types)
    for i in range(len(positions)):
        out += f"{atom_types[i]} {positions[i, 0]:.3f} {positions[i, 1]:.3f} {positions[i, 2]:.3f}\n"
    with open(filename, "w") as f:
        f.write(out)

@typechecked
def write_sdf_file(sdf_path: Path, molecules: List[Chem.Mol], verbose: bool = True):
    w = Chem.SDWriter(str(sdf_path))
    for m in molecules:
        if m is not None:
            w.write(m)
    if verbose:
        log.info(f"Wrote generated molecules to SDF file {sdf_path}")

@typechecked
def load_molecule_xyz(
    file: str,
    dataset_info: Dict[str, Any]
) -> Tuple[
    TensorType["num_nodes", 3],
    TensorType["num_nodes", "num_atom_types"]
]:
    with open(file, encoding="utf8") as f:
        num_atoms = int(f.readline())
        one_hot = torch.zeros(num_atoms, len(dataset_info["atom_decoder"]))
        positions = torch.zeros(num_atoms, 3)
        f.readline()
        atoms = f.readlines()
        for i in range(num_atoms):
            atom = atoms[i].split(" ")
            atom_type = atom[0]
            one_hot[i, dataset_info["atom_encoder"][atom_type]] = 1
            position = torch.Tensor([float(e) for e in atom[1:]])
            positions[i, :] = position
        return positions, one_hot

@typechecked
def load_files_with_ext(path: str, ext: str, shuffle: bool = True) -> List[str]:
    files = glob.glob(path + f"/*.{ext}")
    if shuffle:
        random.shuffle(files)
    return files

@typechecked
def visualize_mol(
    path: str,
    dataset_info: Dict[str, Any],
    max_num: int = 25,
    wandb_run: Optional[Run] = None,
    spheres_3d: bool = False,
    mode: str = "molecule",
    verbose: bool = True
):
    files = load_files_with_ext(path, ext="xyz")[0: max_num]
    for file in files:
        positions, one_hot = load_molecule_xyz(file, dataset_info)
        atom_types = torch.argmax(one_hot, dim=-1).numpy()
        dists = torch.cdist(positions.unsqueeze(0), positions.unsqueeze(0)).squeeze(0)
        dists = dists[dists > 0]

        if verbose:
            log.info(f"Average distance between atoms: {dists.mean().item()}")

        plot_data3d(
            positions=positions,
            atom_types=atom_types,
            dataset_info=dataset_info,
            save_path=f"{file[:-4]}.png",
            spheres_3d=spheres_3d
        )

        if wandb_run is not None:
            # log image(s) via WandB
            path = f"{file[:-4]}.png"
            im = plt.imread(path)
            wandb_run.log({mode: [wandb.Image(im, caption=path)]})

@typechecked
def visualize_mol_chain(
    path: str,
    dataset_info: Dict[str, Any],
    wandb_run: Optional[Run] = None,
    spheres_3d: bool = False,
    mode: str = "chain",
    verbose: bool = True
):
    files = load_files_with_ext(path, ext="xyz")
    files = sorted(files)
    save_paths = []

    for file in files:
        positions, one_hot = load_molecule_xyz(file, dataset_info=dataset_info)

        atom_types = torch.argmax(one_hot, dim=-1).numpy()
        fn = f"{file[:-4]}.png"
        plot_data3d(
            positions=positions,
            atom_types=atom_types,
            dataset_info=dataset_info,
            save_path=fn,
            spheres_3d=spheres_3d,
            alpha=1.0
        )

        save_paths.append(fn)

    imgs = [imageio.imread(fn) for fn in save_paths]
    dirname = os.path.dirname(save_paths[0])
    gif_path = os.path.join(dirname, "output.gif")

    if verbose:
        log.info(f"Creating GIF with {len(imgs)} images")

    # add the last frame 10 times so that the final result remains temporally
    # imgs.extend([imgs[-1]] * 10)

    imageio.mimsave(gif_path, imgs, subrectangles=True)

    if wandb_run is not None:
        wandb_run.log({mode: [wandb.Video(gif_path, caption=gif_path)]})

@typechecked
def draw_sphere(
    ax: plt.axis,
    x: float,
    y: float,
    z: float,
    size: float,
    color: str,
    alpha: float
):
    u = np.linspace(0, 2 * np.pi, 100)
    v = np.linspace(0, np.pi, 100)

    xs = size * np.outer(np.cos(u), np.sin(v))
    ys = size * np.outer(np.sin(u), np.sin(v)) * 0.8  # Correct for matplotlib.
    zs = size * np.outer(np.ones(np.size(u)), np.cos(v))

    ax.plot_surface(
        x + xs,
        y + ys,
        z + zs,
        rstride=2,
        cstride=2,
        color=color,
        linewidth=0,
        alpha=alpha
    )

@typechecked
def plot_molecule(
    ax: Axes,
    positions: TensorType["num_nodes", 3],
    atom_types: np.ndarray,
    alpha: float,
    spheres_3d: bool,
    hex_bg_color: str,
    dataset_info: Dict[str, Any]
):
    x = positions[:, 0]
    y = positions[:, 1]
    z = positions[:, 2]

    # Hydrogen, Carbon, Nitrogen, Oxygen, Flourine

    colors_dic = np.array(dataset_info["colors_dic"])
    radius_dic = np.array(dataset_info["radius_dic"])
    area_dic = 1500 * radius_dic ** 2

    areas = area_dic[atom_types]
    radii = radius_dic[atom_types]
    colors = colors_dic[atom_types]

    if spheres_3d:
        for i, j, k, s, c in zip(x, y, z, radii, colors):
            draw_sphere(ax, i.item(), j.item(), k.item(), 0.7 * s, c, alpha)
    else:
        ax.scatter(x, y, z, s=areas, alpha=0.9 * alpha,
                   c=colors)  # , linewidths=2, edgecolors="#FFFFFF")

    for i in range(len(x)):
        for j in range(i + 1, len(x)):
            p1 = np.array([x[i], y[i], z[i]])
            p2 = np.array([x[j], y[j], z[j]])
            dist = np.sqrt(np.sum((p1 - p2) ** 2))

            s = (atom_types[i], atom_types[j])

            draw_edge_int = get_bond_order(
                dataset_info["atom_decoder"][s[0]],
                dataset_info["atom_decoder"][s[1]],
                dist
            )
            line_width = 2

            draw_edge = draw_edge_int > 0
            if draw_edge:
                if draw_edge_int == 4:
                    linewidth_factor = 1.5
                else:
                    # linewidth_factor = draw_edge_int  # note: proportional to number of edges
                    linewidth_factor = 1
                ax.plot(
                    [x[i], x[j]],
                    [y[i], y[j]],
                    [z[i], z[j]],
                    linewidth=line_width * linewidth_factor,
                    c=hex_bg_color,
                    alpha=alpha
                )

@typechecked
def plot_data3d(
    positions: TensorType["num_nodes", 3],
    atom_types: np.ndarray,
    dataset_info: Dict[str, Any],
    camera_elev: int = 0,
    camera_azim: int = 0,
    save_path: str = None,
    spheres_3d: bool = False,
    bg: str = "black",
    alpha: float = 1.0
):
    black = (0, 0, 0)
    white = (1, 1, 1)
    hex_bg_color = "#FFFFFF" if bg == "black" else "#666666"

    fig = plt.figure()
    ax = fig.add_subplot(projection="3d")
    ax.set_aspect("auto")
    ax.view_init(elev=camera_elev, azim=camera_azim)
    if bg == "black":
        ax.set_facecolor(black)
    else:
        ax.set_facecolor(white)

    ax.xaxis.pane.set_alpha(0)
    ax.yaxis.pane.set_alpha(0)
    ax.zaxis.pane.set_alpha(0)
    ax._axis3don = False

    if bg == "black":
        ax.w_xaxis.line.set_color("black")
    else:
        ax.w_xaxis.line.set_color("white")

    plot_molecule(
        ax=ax,
        positions=positions,
        atom_types=atom_types,
        alpha=alpha,
        spheres_3d=spheres_3d,
        hex_bg_color=hex_bg_color,
        dataset_info=dataset_info
    )

    if "GEOM" in dataset_info["name"]:
        max_value = positions.abs().max().item()

        axis_lim = min(40, max(max_value / 1.5 + 0.3, 3.2))
        ax.set_xlim(-axis_lim, axis_lim)
        ax.set_ylim(-axis_lim, axis_lim)
        ax.set_zlim(-axis_lim, axis_lim)
    else:
        max_value = positions.abs().max().item()

        axis_lim = min(40, max(max_value / 1.5 + 0.3, 3.2))
        ax.set_xlim(-axis_lim, axis_lim)
        ax.set_ylim(-axis_lim, axis_lim)
        ax.set_zlim(-axis_lim, axis_lim)

    dpi = 120 if spheres_3d else 50

    if save_path is not None:
        plt.savefig(save_path, bbox_inches="tight", pad_inches=0.0, dpi=dpi)

        if spheres_3d:
            img = imageio.imread(save_path)
            img_brighter = np.clip(img * 1.4, 0, 255).astype("uint8")
            imageio.imsave(save_path, img_brighter)
    else:
        plt.show()
    plt.close()

class ScalarVector(tuple):
    """
    From https://github.com/sarpaykent/GBPNet
    """
    def __new__(cls, scalar, vector):
        return tuple.__new__(cls, (scalar, vector))

    def __getnewargs__(self):
        return self.scalar, self.vector

    @property
    def scalar(self):
        return self[0]

    @property
    def vector(self):
        return self[1]

    # Element-wise addition
    def __add__(self, other):
        if isinstance(other, tuple):
            scalar_other = other[0]
            vector_other = other[1]
        else:
            scalar_other = other.scalar
            vector_other = other.vector

        return ScalarVector(self.scalar + scalar_other, self.vector + vector_other)

    # Element-wise multiplication or scalar multiplication
    def __mul__(self, other):
        if isinstance(other, tuple):
            other = ScalarVector(other[0], other[1])

        if isinstance(other, ScalarVector):
            return ScalarVector(self.scalar * other.scalar, self.vector * other.vector)
        else:
            return ScalarVector(self.scalar * other, self.vector * other)

    def concat(self, others, dim=-1):
        dim %= len(self.scalar.shape)
        s_args, v_args = list(zip(*(self, *others)))
        return torch.cat(s_args, dim=dim), torch.cat(v_args, dim=dim)

    def flatten(self):
        flat_vector = torch.reshape(self.vector, self.vector.shape[:-2] + (3 * self.vector.shape[-2],))
        return torch.cat((self.scalar, flat_vector), dim=-1)

    @staticmethod
    def recover(x, vector_dim):
        v = torch.reshape(x[..., -3 * vector_dim:], x.shape[:-1] + (vector_dim, 3))
        s = x[..., : -3 * vector_dim]
        return ScalarVector(s, v)

    def vs(self):
        return self.scalar, self.vector

    def idx(self, idx):
        return ScalarVector(self.scalar[idx], self.vector[idx])

    def repeat(self, n, c=1, y=1):
        return ScalarVector(self.scalar.repeat(n, c), self.vector.repeat(n, y, c))

    def clone(self):
        return ScalarVector(self.scalar.clone(), self.vector.clone())

    def mask(self, node_mask: TensorType["num_nodes"]):
        return ScalarVector(
            self.scalar * node_mask[:, None],
            self.vector * node_mask[:, None, None]
        )

    def __setitem__(self, key, value):
        self.scalar[key] = value.scalar
        self.vector[key] = value.vector

    def __repr__(self):
        return f"ScalarVector({self.scalar}, {self.vector})"

class VectorDropout(nn.Module):
    """
    From https://github.com/drorlab/gvp-pytorch
    """

    def __init__(self, drop_rate):
        super().__init__()
        self.drop_rate = drop_rate

    def forward(self, x):
        """
        :param x: `torch.Tensor` corresponding to vector channels
        """
        device = x[0].device
        if not self.training:
            return x
        mask = torch.bernoulli((1 - self.drop_rate) * torch.ones(x.shape[:-1], device=device)).unsqueeze(-1)
        x = mask * x / (1 - self.drop_rate)
        return x

class GCPDropout(nn.Module):
    """
    From https://github.com/drorlab/gvp-pytorch
    """

    def __init__(self, drop_rate: float, use_gcp_dropout: bool = True):
        super().__init__()
        self.scalar_dropout = nn.Dropout(drop_rate) if use_gcp_dropout else nn.Identity()
        self.vector_dropout = VectorDropout(drop_rate) if use_gcp_dropout else nn.Identity()

    def forward(self, x: Union[torch.Tensor, ScalarVector]):
        if isinstance(x, torch.Tensor) and x.shape[0] == 0:
            return x
        elif isinstance(x, ScalarVector) and (x.scalar.shape[0] == 0 or x.vector.shape[0] == 0):
            return x
        elif isinstance(x, torch.Tensor):
            return self.scalar_dropout(x)
        return ScalarVector(self.scalar_dropout(x[0]), self.vector_dropout(x[1]))

class GCPLayerNorm(nn.Module):
    """
    From https://github.com/drorlab/gvp-pytorch
    """

    def __init__(self, dims: ScalarVector, eps: float = 1e-8, use_gcp_norm: bool = True):
        super().__init__()
        self.scalar_dims, self.vector_dims = dims
        self.scalar_norm = nn.LayerNorm(self.scalar_dims) if use_gcp_norm else nn.Identity()
        self.use_gcp_norm = use_gcp_norm
        self.eps = eps

    @staticmethod
    def norm_vector(v: torch.Tensor, use_gcp_norm: bool = True, eps: float = 1e-8) -> torch.Tensor:
        v_norm = v
        if use_gcp_norm:
            vector_norm = torch.clamp(torch.sum(torch.square(v), dim=-1, keepdim=True), min=eps)
            vector_norm = torch.sqrt(torch.mean(vector_norm, dim=-2, keepdim=True))
            v_norm = v / vector_norm
        return v_norm

    def forward(self, x: Union[torch.Tensor, ScalarVector]):
        if isinstance(x, torch.Tensor) and x.shape[0] == 0:
            return x
        elif isinstance(x, ScalarVector) and (x.scalar.shape[0] == 0 or x.vector.shape[0] == 0):
            return x
        elif not self.vector_dims:
            return self.scalar_norm(x)
        s, v = x
        return ScalarVector(self.scalar_norm(s), self.norm_vector(v, use_gcp_norm=self.use_gcp_norm, eps=self.eps))


from patchify import patchify
import tifffile as tiff
import os
from PIL import Image, ImageDraw
from math import floor
import shutil
from sklearn.model_selection import train_test_split
from torchvision.transforms import v2
import random
from torchvision import transforms
from osgeo import ogr, osr, gdal

def make_split(
    class_name, dataset_dir="Datasets/Crow_classify", classify=False, split_p=0.2
):
    """
    Saves the data in a given path into train and test folders
    """

    class_dir = os.path.join(dataset_dir, class_name)
    samples_list = os.listdir(class_dir)

    print("Sample list: ", len(samples_list))

    samples_train, samples_test = train_test_split(
        samples_list, test_size=split_p, random_state=42
    )

    train_dir = os.path.join(dataset_dir, "train")
    test_dir = os.path.join(dataset_dir, "test")

    if classify:
        train_dir = os.path.join(dataset_dir, "train", class_name)
        test_dir = os.path.join(dataset_dir, "test", class_name)

    if not os.path.exists(train_dir):
        os.makedirs(train_dir)

    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    print(len(samples_train), " in ", train_dir)
    print(len(samples_test), " in ", test_dir)

    for sample in samples_test:
        sample_path = os.path.join(class_dir, sample)
        save_path = os.path.join(test_dir, sample)

        shutil.copyfile(sample_path, save_path)

    for sample in samples_train:
        sample_path = os.path.join(class_dir, sample)
        save_path = os.path.join(train_dir, sample)

        shutil.copyfile(sample_path, save_path)

def make_patches(dataset_dir, save_dir, patch_size=224, img_format="jpg"):
    """
    Generate patches of the given size from the images into the given directory
    """

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    original_sample_names = os.listdir(dataset_dir)

    for name in original_sample_names:
        sample_path = os.path.join(dataset_dir, name)

        if os.path.isdir(sample_path):
            continue

        image = tiff.imread(sample_path, key=0)

        h, w, c = image.shape

        if h >= patch_size and w >= patch_size:
            h_new = floor(h / patch_size) * patch_size
            w_new = floor(w / patch_size) * patch_size
            image = image[:h_new, :w_new, :]

            patches_imgs = patchify(image, (224, 224, 3), step=224)
            print("N patches ", patches_imgs.shape)

            for i in range(len(patches_imgs)):
                for j in range(len(patches_imgs[i])):
                    im = Image.fromarray(patches_imgs[i][j][0])
                    save_path = os.path.join(
                        save_dir, f"{name.split('.')[0]}_{i}_{j}.{img_format}"
                    )
                    im.save(save_path)
        else:
            print("Skipping image: patch size too big!")

    return os.listdir(save_dir)

def get_patch_info(patch_path):
    patch_name = os.path.basename(patch_path).split("_")[0]
    patch_row = int(os.path.basename(patch_path).split("_")[1])
    patch_col = int(os.path.basename(patch_path).split("_")[2].split(".")[0])

    return patch_name, patch_row, patch_col

def get_WGS84(path_tif, x=None, y=None):
    """
    Returns WGS84 coordinates of pixels x,y

    Source https://stackoverflow.com/questions/50191648/gis-geotiff-gdal-python-how-to-get-coordinates-from-pixel
    """

    ### Get native coordinates of pixels x,y
    ds = gdal.Open(path_tif)
    xoff, a, b, yoff, d, e = ds.GetGeoTransform()

    if not x:
        x = int(ds.RasterXSize / 2)
    if not y:
        y = int(ds.RasterYSize / 2)

    xp = a * x + b * y + xoff
    yp = d * x + e * y + yoff

    ### Convert to WGS84
    # get CRS from dataset
    crs = osr.SpatialReference()
    crs.ImportFromWkt(ds.GetProjectionRef())
    # create lat/long crs with WGS84 datum
    crsGeo = osr.SpatialReference()
    crsGeo.ImportFromEPSG(4326)  # 4326 is the EPSG id of lat/long crs
    t = osr.CoordinateTransformation(crs, crsGeo)
    long, lat, z = t.TransformPoint(xp, yp)
    return lat, long

def get_patch_coordinates(
    patch_path,
    data_tif_path,
    patch_xy=None,
    patch_size=224,
):
    """
    Returns the patch ID (patch-name_row_col) and coordinates of a given patch
    """

    # Get files info
    patch_name, patch_row, patch_col = get_patch_info(patch_path)
    tif_path = os.path.join(data_tif_path, patch_name + ".tif")

    if not patch_xy:
        patch_xy = (int(patch_size / 2), int(patch_size / 2))

    # Get coordinates
    tif_x = patch_xy[0] + patch_col * patch_size
    tif_y = patch_xy[1] + patch_row * patch_size
    c1, c2 = get_WGS84(tif_path, tif_x, tif_y)

    return f"{patch_name}_{patch_row}_{patch_col}", c1, c2


import json

def write_contacts_to_file(filename, contacts):
    with open(filename, 'w') as f:
        json.dump({'contacts': contacts}, f)

def read_contacts_from_file(filename):
    with open(filename, 'r') as f:
        unpacked = json.load(f)
        return unpacked['contacts']

p = {
    "name": "Allen Raymond",
    "email": "nulla.ante@vestibul.co.uk",
    "phone": "(992) 914-3792",
    "favorite": False,
}
p1 = {
    "name": "Allen Raymond",
    "email": "nulla.ante@vestibul.co.uk",
    "phone": "(992) 914-4792",
    "favorite": False,
}
l = [p, p1]
write_contacts_to_file('kok', l)
print(read_contacts_from_file('kok'))


import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("lightnum", type=int)
parser.add_argument("lightval", type=int)

args = parser.parse_args()

#print(args.lightnum)

if args.lightnum == 1:
	if args.lightval == 1:
		os.system('mycroft-say-to lights on')
	elif args.lightval == 0:
		os.system('mycroft-say-to lights off')


import gym
from gym import spaces
from gym.utils import seeding

from gym.envs.classic_control.acrobot import *

class AcrobotTrainEnv(AcrobotEnv):
	def __init__(self):
		self.viewer = None

		self.dt = 0.05
		self.min_mass = 0.2
		self.max_mass = 0.5

		self.min_action = -1.0
		self.max_action = 1.0

		high = np.array([1.0, 1.0, 1.0, 1.0, self.MAX_VEL_1, self.MAX_VEL_2])
		low = np.array([-1.0, -1.0, -1.0, -1.0, -self.MAX_VEL_1, -self.MAX_VEL_2])

		self.observation_space = spaces.Box(low=low, high=high, dtype=np.float32)
		self.action_space = spaces.Box(low=self.min_action, high=self.max_action,
									   shape=(1,), dtype=np.float32)
		self.state = None
		self.goal_err = 0
		self.seed()

	def reset(self):
		low = np.array([-0.05, -0.05, -0.05, -0.05, self.min_mass, self.min_mass])
		high =  np.array([0.05, 0.05, 0.05, 0.05, self.max_mass, self.max_mass])
		self.state = self.np_random.uniform(low=low, high=high)
		return self._get_ob()

	def step(self, a):
		s = self.state
		torque = np.clip(a, self.min_action, self.max_action)[0]

		# Now, augment the state with our force action so it can be passed to
		# _dsdt
		s_augmented = np.append(s, torque)

		ns = rk4(self._dsdt, s_augmented, [0, self.dt])
		# only care about final timestep of integration returned by integrator
		ns = ns[-1]
		ns = ns[:-1]  # omit action
		# ODEINT IS TOO SLOW!
		# ns_continuous = integrate.odeint(self._dsdt, self.s_continuous, [0, self.dt])
		# self.s_continuous = ns_continuous[-1] # We only care about the state
		# at the ''final timestep'', self.dt

		ns[0] = wrap(ns[0], -pi, pi)
		ns[1] = wrap(ns[1], -pi, pi)
		ns[2] = bound(ns[2], -self.MAX_VEL_1, self.MAX_VEL_1)
		ns[3] = bound(ns[3], -self.MAX_VEL_2, self.MAX_VEL_2)
		self.state = ns
		terminal = self._terminal()
		self.goal_err = self._goal_error()
		reward = -self.goal_err

		return (self._get_ob(), reward, terminal, {})

	def _get_ob(self):
		s = self.state
		return np.array([cos(s[0]), np.sin(s[0]), cos(s[1]), sin(s[1]), s[2], s[3]])

	def _terminal(self):
		s = self.state
		return bool(-np.cos(s[0]) - np.cos(s[1] + s[0]) > 1.)

	def get_safe_error(self):
		return 0.0

	def get_goal_error(self):
		return self.goal_err

	def _goal_error(self):
		s = self.state
		h = -np.cos(s[0]) - np.cos(s[1] + s[0])
		if h < 1.0:
			return 1.0 - h
		return 0.0

	def get_dt(self):
		return self.dt

	def _dsdt(self, s_augmented, t):

		g = 9.8
		a = s_augmented[-1]
		s = s_augmented[:-1]
		m1 = s[-2]
		m2 = s[-1]
		l1 = self.LINK_LENGTH_1
		lc1 = self.LINK_COM_POS_1
		lc2 = self.LINK_COM_POS_2
		I1 = self.LINK_MOI * m1
		I2 = self.LINK_MOI * m2
		theta1 = s[0]
		theta2 = s[1]
		dtheta1 = s[2]
		dtheta2 = s[3]
		d1 = m1 * lc1 ** 2 + m2 * \
			(l1 ** 2 + lc2 ** 2 + 2 * l1 * lc2 * np.cos(theta2)) + I1 + I2
		d2 = m2 * (lc2 ** 2 + l1 * lc2 * np.cos(theta2)) + I2
		phi2 = m2 * lc2 * g * np.cos(theta1 + theta2 - np.pi / 2.)
		phi1 = - m2 * l1 * lc2 * dtheta2 ** 2 * np.sin(theta2) \
			   - 2 * m2 * l1 * lc2 * dtheta2 * dtheta1 * np.sin(theta2)  \
			+ (m1 * lc1 + m2 * l1) * g * np.cos(theta1 - np.pi / 2) + phi2
		if self.book_or_nips == "nips":
			# the following line is consistent with the description in the
			# paper
			ddtheta2 = (a + d2 / d1 * phi1 - phi2) / \
				(m2 * lc2 ** 2 + I2 - d2 ** 2 / d1)
		else:
			# the following line is consistent with the java implementation and the
			# book
			ddtheta2 = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * dtheta1 ** 2 * np.sin(theta2) - phi2) \
				/ (m2 * lc2 ** 2 + I2 - d2 ** 2 / d1)
		ddtheta1 = -(d2 * ddtheta2 + phi1) / d1

		return (dtheta1, dtheta2, ddtheta1, ddtheta2, 0., 0., 0.)


from flask import Flask, render_template, request, session, redirect, url_for
import os
import visualization as viz

app = Flask(__name__)
app.secret_key = os.urandom(24)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0

@app.route("/")
def index():

    if 'filename' in session and 'filepath' in session:
        filepath = session['filepath']
        filename = session['filename']
    else:
        filepath = None
        filename = "Choose image"

    return render_template("index.html", filename=filename, filepath=filepath)

@app.route("/upload/", methods=["GET", "POST"])
def upload():
    if request.method == "POST":
        if request.files:
            image = request.files["image"]
            filename = image.filename
            print(len(filename))
            if (len(filename) > 0):
                filepath = os.path.join('static/img/', image.filename)
                image.save(filepath)

                session["filepath"] = filepath
                session["filename"] = filename

    return redirect(url_for('index'))

@app.route("/detect/", methods=["POST"])
def start_detection():

    if request.method == "POST":
        if 'filename' in session and 'filepath' in session:
            filepath = session['filepath']
            viz.detect_img(filepath)

    return redirect(url_for('index'))

# No caching at all for API endpoints.
@app.after_request
def add_header(r):
    r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
    r.headers["Pragma"] = "no-cache"
    r.headers["Expires"] = "0"
    r.headers['Cache-Control'] = 'public, max-age=0'
    return r

if __name__ == "__main__":
    app.run(debug=True)


"""
copy and past this function into AWS Lambda > Functions > {FUNCTION_NAME}
https://www.notion.so/datasaur/AWS-Comprehend-63149417ef3c4ebb9d84d945d6a5bc31?pvs=4#b692240c9cb84063999db3bc435188da
"""

import json
import boto3

def lambda_handler(event, context):

    try:
        body = json.loads(event["body"])
        final_resp_list = []
        for data in body:
            id_ = data["id"]
            text = data["text"]

            # Get Comprehend client
            comprehend = boto3.client("comprehend")
            # Use detect entities function to get a list of entities
            resp = comprehend.classify_document(
                Text=text,
                EndpointArn="{YOUR_ENDPOINT_ARN}",
            )

            conf_score = [v["Score"] for v in resp["Classes"]]
            conf_score_max = max(conf_score)
            index_max_score = conf_score.index(conf_score_max)

            pred_label = resp["Classes"][index_max_score]["Name"]
            final_resp = {"id": id_, "label": pred_label}
            final_resp_list.append(final_resp)

        return {
            "isBase64Encoded": True,
            "statusCode": "200",
            "headers": {"Content-Type": "text/html"},
            "body": json.dumps(final_resp_list),
        }

    except Exception as e:
        print(e.__class__, " occurred :", e)
        resp = json.dumps({"error": str(e)}, sort_keys=True, indent=4)
        return {"statusCode": "500", "body": resp}


import pickle
from time import sleep
from unittest.mock import patch
import flask
from src.utils.redis_cache import cache

def test_cache(redis_mock):
    """Test that the redis cache decorator works"""

    @patch('src.utils.redis_cache.extract_key')
    def get_mock_cache(extract_key):

        # cache requires a request object
        app = flask.Flask(__name__)
        with app.test_request_context('/'):

            mock_key_1 = 'mock_key'
            extract_key.return_value = mock_key_1

            # Test a mock function returning two items
            @cache(ttl_sec=1)
            def mock_func():
                return {'name': 'joe'}, 200

            res = mock_func()
            assert res[0] == {'name': 'joe'}
            assert res[1] == 200

            cached_resp = redis_mock.get(mock_key_1)
            deserialized = pickle.loads(cached_resp)
            assert deserialized == {'name': 'joe'}

            # This should call the function and return the cached response
            res = mock_func()
            assert res[0] == {'name': 'joe'}
            assert res[1] == 200

            # Sleep to wait for the cache to expire
            sleep(1)

            cached_resp = redis_mock.get(mock_key_1)
            assert cached_resp is None

            # Test the single response
            def transform(input):
                return {'music': input}

            @cache(ttl_sec=1, transform=transform)
            def mock_func_transform():
                return 'audius'

            res = mock_func_transform()
            assert res == {'music': 'audius'}

            cached_resp = redis_mock.get(mock_key_1)
            deserialized = pickle.loads(cached_resp)
            assert deserialized == 'audius'

            # This should call the function and return the cached response
            res = mock_func_transform()
            assert res == {'music': 'audius'}

            # Sleep to wait for the cache to expire
            sleep(1)

            cached_resp = redis_mock.get(mock_key_1)
            assert cached_resp is None

    get_mock_cache() # pylint: disable=no-value-for-parameter


import json
import os
from datetime import datetime, timezone
from itertools import takewhile
from typing import Callable

import dateutil.parser
import requests
from bs4 import BeautifulSoup
from loguru import logger
from sqlalchemy import func
from sqlalchemy.orm import Session
from telethon.errors.rpcerrorlist import ChannelInvalidError, ChannelPrivateError
from telethon.helpers import add_surrogate, del_surrogate
from telethon.sync import TelegramClient
from telethon.tl import types

from cisticola.base import Channel, ChannelInfo, Post, RawChannelInfo, ScraperResult
from cisticola.transformer.base import Transformer

class TelegramTelethonTransformer(Transformer):
    __version__ = "TelegramTelethonTransformer 0.0.4"

    # TODO cache
    # cache channels for which we cannot get the name from the web interface
    bad_channels = {}

    # cache channels for which we have already looked up the name
    channels_cache_by_platformid = {}
    channels_cache_by_id = {}
    channels_cache_by_screenname = {}

    # cache the ID of posts that get replies, avoiding database lookups when possible
    posts_cache = {}

    get_screenname_cache = {}

    def can_handle(self, data: ScraperResult) -> bool:
        scraper = data.scraper.split(" ")
        if scraper[0] == "TelegramTelethonScraper":
            return True

        return False

    def __init__(self, telethon_session_name=None):
        super().__init__()

        api_id = os.environ["TELEGRAM_API_ID"]
        api_hash = os.environ["TELEGRAM_API_HASH"]
        phone = os.environ["TELEGRAM_PHONE"]

        if telethon_session_name is None:
            telethon_session_name = phone

        # set up a persistent client for Telethon
        self.client = TelegramClient(telethon_session_name, api_id, api_hash)
        self.client.connect()

    def get_screenname_from_id(self, channel_id):
        if channel_id in self.get_screenname_cache:
            return self.get_screenname_cache[channel_id]
        else:
            output = ("", "", None)

            try:
                data = self.client.get_entity(channel_id)
                if isinstance(data, types.User):
                    output = (
                        data.username,
                        str(data.first_name or "") + " " + str(data.last_name or ""),
                        "",
                    )
                else:
                    output = (data.username, data.title, "")
            except ChannelPrivateError:
                output = ("", "", "ChannelPrivateError")
            except ChannelInvalidError:
                output = ("", "", "ChannelInvalidError")
            except ValueError:
                output = ("", "", "ValueError")

            self.get_screenname_cache[channel_id] = output
            return output

    def get_name_from_web_interface(self, orig_screenname, id):
        url = "https://t.me/s/" + orig_screenname + "/" + str(id)

        # this doesn't work for chat channels
        if orig_screenname in self.bad_channels:
            logger.debug(
                f"Skipping screenname because it is not accessible for channel {orig_screenname}"
            )
            return ""

        logger.info(f"Finding channel from URL {url}")
        r = requests.get(url)

        if r.url != url:
            self.bad_channels[orig_screenname] = True
            return ""

        soup = BeautifulSoup(r.content, features="lxml")
        post = soup.findAll("div", {"data-post": orig_screenname + "/" + str(id)})
        name = ""

        # multiple posts can be combined into one result in the web interface
        decrement = 0
        while len(post) == 0:
            decrement += 1
            if decrement > 8:
                break

            logger.info(
                f"Could not find post from {url}, looking for id {id - decrement}"
            )
            post = soup.findAll(
                "div", {"data-post": orig_screenname + "/" + str(id - decrement)}
            )

        if len(post) == 0:
            logger.warning(f"Could not find post from {url}")
        else:
            fwd_tag = post[0].findAll(
                "a", {"class", "tgme_widget_message_forwarded_from_name"}
            )

            if len(fwd_tag) == 0:
                fwd_tag = post[0].findAll(
                    "span", {"class", "tgme_widget_message_forwarded_from_name"}
                )

            if len(fwd_tag) >= 1:
                name = fwd_tag[0].text

        return name

    def transform_info(
        self, data: RawChannelInfo, insert: Callable, session, channel=None
    ):
        raw = json.loads(data.raw_data)

        chat_raw = raw["chats"][0]

        transformed = ChannelInfo(
            raw_channel_info_id=data.id,
            channel=data.channel,
            platform_id=raw["full_chat"]["id"],
            platform=data.platform,
            scraper=data.scraper,
            transformer=self.__version__,
            screenname=chat_raw["username"],
            name=chat_raw["title"],
            description=raw["full_chat"]["about"],
            description_url="",  # does not exist for Telegram
            description_location="",  # does not exist for Telegram
            followers=raw["full_chat"]["participants_count"],
            following=-1,  # does not exist for Telegram
            verified=False,  # does not exist for Telegram
            date_created=dateutil.parser.parse(chat_raw["date"]),
            date_archived=data.date_archived,
            date_transformed=datetime.now(timezone.utc),
        )

        transformed = insert(transformed)

        if channel.platform_id is None:
            logger.info(
                f"Missing platform ID on {channel}, setting to {raw['full_chat']['id']}"
            )

            new_channel = session.query(Channel).where(Channel.id == channel.id).one()
            new_channel.platform_id = raw["full_chat"]["id"]
            session.flush()
            session.commit()

        if len(raw["chats"]) > 1:
            for chat in raw["chats"][1:]:
                new_chat = Channel(
                    name=chat["title"],
                    platform_id=chat["id"],
                    category=channel.category,  # this should be the same as the "parent"
                    platform=channel.platform,  # this should be the same as the "parent"
                    url=("https://t.me/s/" + chat["username"])
                    if "username" in chat
                    else "",
                    screenname=chat["username"] if "username" in chat else "",
                    country=channel.country,  # this should be the same as the "parent"
                    influencer=channel.influencer,  # this should be the same as the "parent"
                    public=None,
                    chat=not chat["broadcast"],
                    notes=channel.id,  # this should be the channel ID of the parent
                    source="linked_channel",
                )

                insert(new_chat)

    def transform(
        self,
        data: ScraperResult,
        insert: Callable,
        session: Session,
        flush_posts: Callable,
    ):
        raw = json.loads(data.raw_data)

        if raw["_"] != "Message":
            logger.warning(f"Cannot convert type {raw['_']} to post")
            return

        fwd_from = None

        if (
            raw["fwd_from"]
            and raw["fwd_from"]["from_id"]
            and "channel_id" in raw["fwd_from"]["from_id"]
        ):
            # use cache to look up channel instead of a DB request if possible
            if (
                str(raw["fwd_from"]["from_id"]["channel_id"])
                not in self.channels_cache_by_platformid
            ):
                channel = (
                    session.query(Channel)
                    .filter_by(
                        platform_id=str(raw["fwd_from"]["from_id"]["channel_id"]),
                        platform="Telegram",
                    )
                    .first()
                )

                if channel is None:
                    (screenname, name, notes) = self.get_screenname_from_id(
                        raw["fwd_from"]["from_id"]["channel_id"]
                    )

                    if name == "":
                        logger.info("Trying fallback web interface")
                        orig_channel = (
                            session.query(Channel).filter_by(id=data.channel).first()
                        )
                        if orig_channel.screenname is not None:
                            name = self.get_name_from_web_interface(
                                orig_channel.screenname, raw["id"]
                            )

                    channel = Channel(
                        name=name,
                        platform_id=raw["fwd_from"]["from_id"]["channel_id"],
                        platform=data.platform,
                        url="https://t.me/s/" + screenname
                        if screenname is not None
                        else "",
                        screenname=screenname,
                        category="forwarded",
                        source=self.__version__,
                        notes=notes,
                    )

                    channel = insert(channel)
                    logger.info(f"Added {channel}")

                self.channels_cache_by_platformid[
                    str(raw["fwd_from"]["from_id"]["channel_id"])
                ] = channel

            fwd_from = self.channels_cache_by_platformid[
                str(raw["fwd_from"]["from_id"]["channel_id"])
            ].id

        reply_to = None
        if raw["reply_to"]:
            reply_to_id = str(raw["reply_to"]["reply_to_msg_id"])

            # use cache to find post ID instead of a DB request, if possible
            if (data.channel, reply_to_id) not in self.posts_cache:
                session.commit()
                # this is necessary because the post we are looking for could be batched but not yet committed to the DB
                flush_posts()
                post = (
                    session.query(Post)
                    .filter_by(channel=data.channel, platform_id=reply_to_id)
                    .first()
                )
                if post is None:
                    reply_to = -1
                else:
                    reply_to = post.id

                self.posts_cache[(data.channel, reply_to_id)] = reply_to
            else:
                reply_to = self.posts_cache[(data.channel, reply_to_id)]

        mentions = []

        for mention_entity in [
            entity
            for entity in raw["entities"]
            if entity["_"] == "MessageEntityMention"
        ]:
            offset = mention_entity["offset"]
            length = mention_entity["length"]

            screenname = (
                add_surrogate(raw["message"])[offset : offset + length]
                .strip("@")
                .strip()
            )

            # use cache rather than a DB request if possible
            if screenname.lower() not in self.channels_cache_by_screenname:
                channel = (
                    session.query(Channel)
                    .filter(func.lower(Channel.screenname) == func.lower(screenname))
                    .first()
                )

                if channel is None:
                    channel = Channel(
                        name=None,
                        platform_id=None,
                        platform="Telegram",
                        url="https://t.me/s/" + screenname,
                        screenname=screenname,
                        category="mentioned",
                        source=self.__version__,
                    )

                    channel = insert(channel)
                    logger.info(f"Added {channel}")

                self.channels_cache_by_screenname[screenname.lower()] = channel

            channel = self.channels_cache_by_screenname[screenname.lower()]

            mentions.append(channel.id)

        # use cache rather than a DB request if possible
        if int(data.channel) not in self.channels_cache_by_id:
            channel = session.query(Channel).filter_by(id=int(data.channel)).first()
            self.channels_cache_by_id[int(data.channel)] = channel

        channel = self.channels_cache_by_id[int(data.channel)]

        if channel is not None and channel.url:
            url = channel.url.strip("/") + f"/{raw['id']}"
            author_username = channel.screenname
        else:
            url = ""
            author_username = ""

        author_id = raw.get("peer_id", {}).get("channel_id")
        if raw["from_id"] and "user_id" in raw["from_id"]:
            author_id = raw["from_id"]["user_id"]
            author_username = ""
            (screenname, name, notes) = self.get_screenname_from_id(author_id)
            if screenname:
                author_username = screenname

        transformed = Post(
            raw_id=data.id,
            platform_id=raw["id"],
            scraper=data.scraper,
            transformer=self.__version__,
            platform=data.platform,
            channel=data.channel,
            date=dateutil.parser.parse(raw["date"]),
            date_archived=data.date_archived,
            date_transformed=datetime.now(timezone.utc),
            url=url,
            content=add_markdown_links(raw),
            author_id=author_id,
            author_username=author_username,
            forwarded_from=fwd_from,
            reply_to=reply_to,
            mentions=mentions,
            forwards=raw.get("forwards"),
            views=raw.get("views"),
        )

        insert(transformed)

def stripped(s):
    """https://stackoverflow.com/a/29933716"""

    lstripped = "".join(takewhile(str.isspace, s))
    rstripped = "".join(reversed(tuple(takewhile(str.isspace, reversed(s)))))

    return lstripped + rstripped

def add_markdown_links(raw_post):
    """This function is necessary because Telethon's markdown.unparse doesn't
    correctly handle trailing whitespace or multi-line links"""

    global_offset = 0
    transformed_content = add_surrogate(raw_post["message"])
    links = [
        entity
        for entity in raw_post["entities"]
        if entity["_"] == "MessageEntityTextUrl"
    ]

    for link in links:
        offset = global_offset + link["offset"]
        length = link["length"]
        url = link["url"]

        before_link = transformed_content[:offset]
        inner_text = transformed_content[offset : offset + length]

        # skip creation of link if inner link text is only whitespace
        if inner_text.replace("\u200b", "").strip():
            processed_inner_text = inner_text.strip().replace("\n", "\\\n")
            link_text = f"[{processed_inner_text}]"
            trailing_whitespace = stripped(
                transformed_content[offset : offset + length]
            )
            link_href = f"({url})"
            after_link = transformed_content[offset + length :]

            transformed_content = (
                before_link + link_text + link_href + trailing_whitespace + after_link
            )
            global_offset += 4 + len(url) + inner_text.strip().count("\n")

    return del_surrogate(transformed_content)


"""
`sklearn.pipeline.Pipeline` subclass which applies all transformations currently
defined in the `preprocessers.py` and `regex_transformations.py` modules.
"""

from overrides import overrides

from sub_clf.preprocess import preprocessors as pp
from sub_clf.preprocess import regex_transformations as rt
from sub_clf.preprocess.base import MultiplePreprocessorPipeline

class KitchenSinkPreprocessor(MultiplePreprocessorPipeline):
    """
    Apply all available preprocesing steps to comments.

    Note: Multiple `pp.RegexTransformer` instances are instantiated because while each
    instance's regex transformations are applied in parallel, some transformations must
    occur before others lest they conflict. Therefore, splitting transformations across
    multiple instances balances the desired parallelism with the required sequentialism.
    """

    regex_transformers = [

        # convert HTML codes to literal characters
        pp.RegexTransformer(
            transformations=[
                rt.HTMLConverter(),
            ],
            name='HTMLConverter'
        ),

        # standardize certain individual characters (transformations require literals)
        pp.RegexTransformer(
            transformations=[
                rt.HyphenNormalizer(),
                rt.QuotationMarkNormalizer(),
            ],
            name='HyphenQuotesStandardizer'
        ),

        # remove spans of unwanted characters
        pp.RegexTransformer(
            transformations=[
                rt.CodeBlockRemover(),
                rt.HyperlinkRemover(),
                rt.InlineCodeRemover(),
                rt.QuoteRemover()
            ],
            name='GarbageSpanRemover'
        ),

        # standardize apostrophes (would interfere with `InlineCodeRemover` because ` -> ')
        pp.RegexTransformer(
            transformations=[
                rt.ApostropheNormalizer(),
            ],
            name='ApostropheNormalizer'
        ),

        # remove all remaining punctuation characters
        pp.RegexTransformer(
            transformations=[
                rt.PunctuationRemover()
            ],
            name='PunctuationRemover'
        ),

        # standardize and collapse whitespace (`PunctuationRemover` may result in "  ")
        pp.RegexTransformer(
            transformations=[
                rt.WhitespaceNormalizer()
            ],
            name='WhitespaceNormalizer'
        )

    ]

    preprocessors = [
        pp.CaseNormalizer(),
        pp.AccentRemover(),
        *regex_transformers,
        pp.StopwordRemover(lemmatize=True)
    ]

    @overrides
    def __init__(self, verbose: bool = False):
        super().__init__(*self.preprocessors, verbose=verbose)


from google.cloud import bigquery

SERVICE_ACCOUNT_JSON = r'C:\Users\shrijha\Desktop\testproject_credentials.json'

client = bigquery.Client.from_service_account_json(SERVICE_ACCOUNT_JSON)

table_id = "testproject123456789-313307.beam_samples.test_table"

job_config = bigquery.LoadJobConfig(
        autodetect=True, source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
    )
uri = "gs://bqtest987654321/site.json"
load_job = client.load_table_from_uri(
    uri, table_id, job_config=job_config
)
load_job.result()
destination_table = client.get_table(table_id)


import csv
import datetime
import markdown
import pandas as pd
import re
import urllib.request

def markdown_to_html(input: str):
    return markdown.markdown(
        input,
        extensions=['markdown.extensions.tables']
    )

def now_in_jst():
    jst = datetime.timezone(datetime.timedelta(hours=9))
    return datetime.datetime.now(jst).isoformat(timespec='seconds')

MD_PATH: str = './FORKED_SITES.md'
LOG_PATH: str = './audit-forked-sites/error.csv'

with open(MD_PATH, 'r', encoding='utf-8') as f:
    md: str = f.read()

df = pd.read_html(markdown_to_html(md))

forked_sites: list = [
    {
        'timestamp': None,
        'url': url,
        'exists': None,
        'status': None
    }
    for url in df[0]['公開サイト']
]

for website in forked_sites:
    req = urllib.request.Request(
        website['url'],
        data=None,
        headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0'
        }
    )

    try:
        with urllib.request.urlopen(req) as r:
            website['exists'] = True
            website['status'] = str(r.status)
    except urllib.error.HTTPError as e:
        website['exists'] = False
        website['status'] = str(e.code)
    except urllib.error.URLError as e:
        website['exists'] = False
        website['status'] = str(e.reason)
    finally:
        website['timestamp'] = now_in_jst()

alive_sites: list = [
    website for website in forked_sites if website['exists'] is True]

if len(alive_sites) > 0:
    for website in alive_sites:
        # alive_sites の内，アクセスが復帰した URL を含む行を検出する
        row_find: str = r'(\n\| *\[\]\(\d*\)[^\W\s]+ *\| *' + \
            website['url'] + r' *\|.*\|.*\| *)\*\*リンク切れ\*\*( *\|)'

        # 該当する行の「リンク切れ」表記を削除する
        row_replace: str = r'\1\2'

        md = re.sub(
            row_find,
            row_replace,
            md
        )
else:
    print('There is no website which has been alive or recovered.')

dead_sites: list = [
    website for website in forked_sites if website['exists'] is False]

if len(dead_sites) > 0:
    for website in dead_sites:
        # dead_sites の URL を含む行を検出する
        row_find: str = r'(\n\| *\[\]\(\d*\)[^\W\s]+ *\| *' + \
            website['url'] + r' *\|.*\|.*\| *)( *\|)'

        # 該当する URL を含む行に「リンク切れ」を追記
        row_replace: str = r'\1**リンク切れ**\2'

        md = re.sub(
            row_find,
            row_replace,
            md
        )

    with open(LOG_PATH, 'a', encoding='utf-8', newline='\n') as f:
        writer = csv.writer(f)

        for data in dead_sites:
            writer.writerow(data.values())
else:
    print('There is no website which has been dead.')

with open(MD_PATH, 'w', encoding='utf-8', newline='\n') as f:
    f.write(md)


import logging
import os
import threading
import ecdsa

from mini_core.block import Block
from mini_core.chain import load_from_disk, get_active_chain
from mini_core.transaction import OutPoint, Transaction, TxIn, TxOut, UnspentTxOut, SignatureScript
from mini_core.networking import GetActiveChainMsg, GetMempoolMsg, GetUTXOsMsg, GetBlocksMsg, InvMsg, TCPHandler, ThreadedTCPServer, send_to_peer, get_ibd_done, get_peer_hostnames
from mini_core.proof_of_work import mine_forever
from mini_core.validation import build_spend_message

logging.basicConfig(
    level=getattr(logging, os.environ.get('TC_LOG_LEVEL', 'INFO')),
    format='[%(asctime)s][%(module)s:%(lineno)d] %(levelname)s %(message)s')

logger = logging.getLogger(__name__)

PORT = os.environ.get('TC_PEER', 8888)

def make_txin(signing_key, outpoint: OutPoint, txout: TxOut) -> TxIn:
    sequence = 0
    pk = signing_key.verifying_key.to_string()
    spend_msg = build_spend_message(outpoint, pk, sequence, [txout])

    return TxIn(signature=SignatureScript(unlock_sig=signing_key.sign(spend_msg), unlock_pk=pk), outpoint=outpoint, sequence=sequence)

def main():
	load_from_disk()

	workers = []
	server = ThreadedTCPServer(('0.0.0.0', PORT), TCPHandler)

	def start_worker(fnc):
		workers.append(threading.Thread(target=fnc, daemon=True))
		workers[-1].start()

	logger.info(f'[p2p] listening on PORT {PORT}')
	start_worker(server.serve_forever)

	if get_peer_hostnames():
		logger.info(f'start initial block download from {len(get_peer_hostnames())} peers')
		send_to_peer(GetBlocksMsg(get_active_chain()[-1].id))
		get_ibd_done().wait(60.)

	start_worker(mine_forever)
	[w.join() for w in workers]

if __name__ == '__main__':
	from mini_core.wallet import init_wallet
	signing_key, verifying_key, my_address = init_wallet()
	main()