import requests
import nltk
import operator
import math
import json
import numpy as np
import pandas as pd
from time import time
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import wordnet
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, precision_recall_fscore_support
from sklearn.model_selection import train_test_split, cross_val_score
from itertools import combinations
from collections import Counter
from statistics import mean
from treatment import diseaseDetail

class DiseasePredictor:
    def __init__(self):
        # nltk.download('all')

        self.stop_words = stopwords.words('english')

        # Load Dataset scraped from NHP (https://www.nhp.gov.in/disease-a-z) & Wikipedia
        # Scrapping and creation of dataset csv is done in a separate program
        self.df_comb = pd.read_csv("./dataset/dis_sym_dataset_comb.csv") # Disease combination
        self.df_norm = pd.read_csv("./dataset/dis_sym_dataset_norm.csv") # Individual Disease

        self.X = self.df_comb.iloc[:, 1:]
        self.Y = self.df_comb.iloc[:, 0:1]

        # Using Logistic Regression (LR) Classifier as it gives better accuracy compared to other
        # classification models as observed in the comparison of model accuracies in Model_latest.py
        # Cross validation is done on dataset with cv = 5
        self.lr = LogisticRegression()
        self.lr = self.lr.fit(self.X, self.Y)
        self.scores = cross_val_score(self.lr, self.X, self.Y, cv=5)

        # List of symptoms
        self.dataset_symptoms = list(self.X.columns)

        # Preprocess
        self.lemmatizer = WordNetLemmatizer()
        self.splitter = RegexpTokenizer(r'\w+')

    # returns the list of synonyms of the input word from thesaurus.com (https://www.thesaurus.com/)
    # and wordnet (https://www.nltk.org/howto/wordnet.html)
    def synonyms(self, term):
        synonyms = []
        response = requests.get('https://www.thesaurus.com/browse/{}'.format(term))
        soup = BeautifulSoup(response.content,  "html.parser")
        try:
            container=soup.find('section', {'class': 'MainContentContainer'})
            row=container.find('div',{'class':'css-191l5o0-ClassicContentCard'})
            row = row.find_all('li')
            for x in row:
                synonyms.append(x.get_text())
        except:
            None
        for syn in wordnet.synsets(term):
            synonyms+=syn.lemma_names()
        return set(synonyms)

    def setInitialSymptoms(self, initial_symptoms):
        # Taking symptoms from user as input str(input("Please enter symptoms separated by comma(,):\n"))
        self.user_symptoms = initial_symptoms.lower().split(',')
        # Preprocessing the input symptoms
        self.processed_user_symptoms=[]
        for sym in self.user_symptoms:
            sym=sym.strip()
            sym=sym.replace('-',' ')
            sym=sym.replace("'",'')
            sym = ' '.join([self.lemmatizer.lemmatize(word) for word in self.splitter.tokenize(sym)])
            self.processed_user_symptoms.append(sym)

        # Taking each user symptom and finding all its synonyms and appending it to the pre-processed symptom string
        user_symptoms = []
        for user_sym in self.processed_user_symptoms:
            user_sym = user_sym.split()
            str_sym = set()
            for comb in range(1, len(user_sym)+1):
                for subset in combinations(user_sym, comb):
                    subset=' '.join(subset)
                    subset = self.synonyms(subset)
                    str_sym.update(subset)
            str_sym.add(' '.join(user_sym))
            user_symptoms.append(' '.join(str_sym).replace('_',' '))

        # Loop over all the symptoms in dataset and check its similarity score to the synonym string of the user-input
        # symptoms. If similarity>0.5, add the symptom to the final list
        found_symptoms = set()
        for idx, data_sym in enumerate(self.dataset_symptoms):
            data_sym_split=data_sym.split()
            for user_sym in user_symptoms:
                count=0
                for symp in data_sym_split:
                    if symp in user_sym.split():
                        count+=1
                if count/len(data_sym_split)>0.5:
                    found_symptoms.add(data_sym)
        found_symptoms = list(found_symptoms)

        # Show the related symptoms found in the dataset and ask user to select among them
        # select_list = input("\nPlease select the relevant symptoms. Enter indices (separated-space):\n").split()

        # Find other relevant symptoms from the dataset based on user symptoms based on the highest co-occurance with the
        # ones that is input by the user
        dis_list = set()
        self.final_symp = []
        counter_list = []
        for idx, itm in enumerate(found_symptoms):
            symp=found_symptoms[idx]
            self.final_symp.append(symp)
            dis_list.update(set(self.df_norm[self.df_norm[symp]==1]['label_dis']))

        for dis in dis_list:
            row = self.df_norm.loc[self.df_norm['label_dis'] == dis].values.tolist()
            row[0].pop(0)
            for idx,val in enumerate(row[0]):
                if val!=0 and self.dataset_symptoms[idx] not in self.final_symp:
                    counter_list.append(self.dataset_symptoms[idx])

        # Symptoms that co-occur with the ones selected by user
        dict_symp = dict(Counter(counter_list))
        self.dict_symp_tup = sorted(dict_symp.items(), key=operator.itemgetter(1),reverse=True)
        return found_symptoms

    def getRelatedSymptoms(self):
        found_symptoms=[]
        for tup in self.dict_symp_tup:
            found_symptoms.append(tup[0])
        return found_symptoms

    def setSymptom(self, symptom):
        self.final_symp = symptom
        return self.final_symp

    def predict(self):
        # Create query vector based on symptoms selected by the user
        sample_x = [0 for x in range(0,len(self.dataset_symptoms))]
        for val in self.final_symp:
            sample_x[self.dataset_symptoms.index(val)]=1

        # Predict disease
        lr = LogisticRegression()
        lr = lr.fit(self.X, self.Y)
        prediction = lr.predict_proba([sample_x])

        k = 10
        diseases = list(set(self.Y['label_dis']))
        diseases.sort()
        topk = prediction[0].argsort()[-k:][::-1]

        topk_dict = {}
        # Show top 10 highly probable disease to the user.
        for idx,t in  enumerate(topk):
            match_sym=set()
            row = self.df_norm.loc[self.df_norm['label_dis'] == diseases[t]].values.tolist()
            row[0].pop(0)

            for idx,val in enumerate(row[0]):
                if val!=0:
                    match_sym.add(self.dataset_symptoms[idx])
            prob = (len(match_sym.intersection(set(self.final_symp)))+1)/(len(set(self.final_symp))+1)
            prob *= mean(self.scores)
            topk_dict[t] = prob

        topk_sorted = dict(sorted(topk_dict.items(), key=lambda kv: kv[1], reverse=True))

        diseaseNames = []
        diseaseProbabilities = []

        for key in topk_sorted:
            prob = topk_sorted[key]*100
            diseaseNames.append(diseases[key])
            diseaseProbabilities.append(round(prob, 2))

        predictions = [{"name": d, "probability": p} for d, p in zip(diseaseNames, diseaseProbabilities)]
        return predictions


#!/usr/bin/env python
# coding: utf-8

# In[12]:
import cv2
def grayscale(input,output):
    #gray the photos
    image = cv2.imread(input)
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    cv2.imwrite(output, gray)
    return
'''
input="/Users/YiHung/Downloads/0002_com.png"
output="/Users/YiHung/Downloads/後來灰階.png"
grayscale(input,output)
'''


import os
import shutil
from urllib3 import PoolManager, Retry
from urllib3.exceptions import HTTPError
from typing import Union
from .store import BinaryStore

http = PoolManager(retries=Retry(connect=5, read=2, redirect=5))

class WebStore:
    """
    Downloads files into `BinaryStore`s.
    """
    @staticmethod
    def get(url: str, file_name: str = None, download_store: BinaryStore = None) -> Union[str, None]:
        """
        Download a file into the web store.

        :param url: The URL to download
        :param file_name: The name to give to the downloaded file. `None` for name extraction from the URL
        :param download_store: The store to download the file in. `None` for temporary store.
        :return: The downloaded file path
        """
        if file_name is None:
            file_name = url
            if '?' in file_name:
                file_name, _ = file_name.split('?', 1)
            if '/' in file_name:
                _, file_name = file_name.rsplit('/', 1)

        if download_store is None:
            download_store = BinaryStore()

        # Check if file was downloaded previously
        if download_store.isfile(file_name):
            return download_store.get_path(file_name)

        # Download the file
        try:
            with http.request('GET', url, preload_content=False) as response:
                if response.status != 200:
                    # Server couldn't return the file
                    return None

                with download_store.open(file_name, 'wb') as file_obj:
                    shutil.copyfileobj(response, file_obj)

        except HTTPError:
            return None

        # Download finished, return the file path
        return download_store.get_path(file_name)

    @staticmethod
    def decompress(url: str, compression: str = None, store: BinaryStore = None, **kwargs) -> Union[BinaryStore, None]:
        """
        Downloads a zip file and unzips it.

        :param url: The url of the archive.
        :param compression: The type of the compression. If `None` then the compression is decided based on the suffix
            of the url.
        :param store: The store to unzip the file into it. `None` for temporary store.
        :param kwargs: Extra keyword arguments to pass to `BinaryStore`'s relevant extraction function
        :return: The store the archive was extracted to.
        """
        if compression is None:
            if os.path.extsep not in url:
                raise ValueError(f'Compression not specified')

        if store is None:
            store = BinaryStore()

        compressed = WebStore.get(url)
        if compressed is None:
            return None

        store.decompress(compressed, compression, **kwargs)
        return store


import flask
import neomodel
import redis
import uuid

from flask import Flask, jsonify, request
from flask_api import status
from data_models import Person, Place, WentToPlaceRel, ContactWithRel
from neomodel import config
from utilities import make_response

def create_new_user():
    content = request.get_json()
    try:
        phone_number = content['phoneNumber']
    except:
        phone_number = None
    if (phone_number == None):
        return jsonify({'response': 'bad request, please try again and specify a \'phoneNumber\' parameter in the JSON request body'}, status.HTTP_400_BAD_REQUEST)
    else:
        try:
            #iF THE PERSON IS Found properly, this applies
            existingPerson = Person.nodes.get(phoneNumber=phone_number)
        except (neomodel.exceptions.MultipleNodesReturned):
            return jsonify({'response': 'The Phone Number' + phone_number + 'is already registered with multiple users'}, status.HTTP_403_FORBIDDEN)
        except (neomodel.exceptions.DoesNotExist):
            #iF THE PERSON IS NOT FOUND, this applies
            newUser = Person(phoneNumber=phone_number).save()
            return jsonify({'response': 'new number created - ' + phone_number}, status.HTTP_200_OK)
        return jsonify({'response': 'The Phoene Number' + phone_number + 'is already a registered user'}), status.HTTP_403_FORBIDDEN

def report_contact(personA, personB):
    return "contact!!!"

def generateID():
    custom_id = uuid.uuid1()
    return custom_id

def ingestPushNotificationAndCreateID(r):
    content = request.get_json()
    #Step 1, check to ensure that push_token field is present
    try:
        push_token = content['push_token']
    except:
        push_token = None
    if (push_token == None):
        return make_response({'response': 'bad request, there was no \'push_token\' property in the request body'}, status.HTTP_400_BAD_REQUEST)
    else:
        #Step 2, generate a custom ID
        custom_ID = generateID()
        #Step 3, store ID in our redis cluster
        r.set(str(custom_ID), push_token)

    return make_response({ "identifier": custom_ID}, status.HTTP_200_OK)


import re
from typing import List, Tuple

import pytest

from flake8_annotations.checker import FORMATTED_ERROR
from testing.helpers import check_source
from testing.test_cases.variable_formatting_test_cases import (
    variable_formatting_test_cases,
)

SIMPLE_ERROR_CODE = Tuple[str, str]

# Error type specific matching patterns
TEST_ARG_NAMES = {"ANN001": "some_arg", "ANN002": "some_args", "ANN003": "some_kwargs"}
RE_DICT = {"ANN001": r"'(\w+)'", "ANN002": r"\*(\w+)", "ANN003": r"\*\*(\w+)"}

def _simplify_error(error_code: FORMATTED_ERROR) -> SIMPLE_ERROR_CODE:
    """
    Simplify the error yielded by the flake8 checker into an (error type, argument name) tuple.

    Input error codes are assumed to be tuples of the form:
    (line number, column number, error string, checker class)

    Where the error string begins with "ANNxxx" and contains the arg name in the following form:
    ANN001: '{arg name}'
    ANN002: *{arg name}
    ANN003: **{arg name}
    """
    error_type = error_code[2].split()[0]
    arg_name = re.findall(RE_DICT[error_type], error_code[2])[0]
    return error_type, arg_name

class TestArgumentFormatting:
    """Testing class for containerizing parsed error codes & running the fixtured tests."""

    @pytest.fixture(
        params=variable_formatting_test_cases.items(), ids=variable_formatting_test_cases.keys()
    )
    def parsed_errors(self, request) -> Tuple[List[SIMPLE_ERROR_CODE], str]:  # noqa: ANN001
        """
        Create a fixture for the error codes emitted by the test case source code.

        Error codes for the test case source code are simplified into a list of
        (error code, argument name) tuples.
        """
        test_case_name, test_case = request.param
        simplified_errors = [_simplify_error(error) for error in check_source(test_case.src)]

        return simplified_errors, test_case_name

    def test_arg_name(self, parsed_errors: Tuple[List[SIMPLE_ERROR_CODE], str]) -> None:
        """
        Check for correctly formatted argument names.

        Simplified error code information is provided by the fixture as a list of
        (yielded error, test case name) tuples
        """
        assert all(
            TEST_ARG_NAMES[error_type] == arg_name for error_type, arg_name in parsed_errors[0]
        )


import feedparser
import validators
import sys
import pprint
import fnmatch
import platform
import time
import os

class Feed:
    """
    `Author`: Eliakah kakou

    This class allows for more functionality in regards
    to the dictionary entered in the constructor
    """

    def __init__(self, feed):
        """
        `Author`: Eliakah kakou

        The constructor

        `feed`: dict instance containing set of information about feed
        """
        self.feed = feed

    def extract(self):
        """
        `Author`: Eliakah kakou

        This method returns a subset of the dictionary
        """
        return {k: self.feed[k] for k in ('id', 'published', 'title', 'link', 'summary')}

    def record_content(self):
        return self.feed['link'], self.feed['published'];

class Feeder:
    """
    `Author`: Bill Clark

    An interface to fulfill a strategy pattern design in the Grapher with feeders.
    An implementation of this interface will be usable with the Grapher so long as
    it provides the appropriate returns.

    Feeder instances should be created with a link to some sort of file. The result
    to come out of it's methods will be used by the Parser Module to generate topics
    about the information in the Feed. A Feed represents a single article of related
    data. The implementation should generate Feed objects as defined above in order
    to be encapsulated.
    """

    def __init__(self, file):
        """
        A feeder should be initialized to read from it's source when commanded to
        via the load_feeds method. Feeds is a list of Feed objects specifically,
        and is interacted with in the feeds generator call.

        `file`: The file to process later.
        """
        self.feeds = []
        self.path = file

    def contents(self):
        """
        Returns the feeds the Feeder is holding on to via a generator structure.
        """
        for feed in self.feeds:
            yield feed
            # feeds = []

    def fetch(self):
        """
        Calling this method should retrieve information using the file provided
        at initialization. Following that, that information will be standardized
        into a Feed object and stored in the feeds instance variable. This is going
        to be implemented in very different ways, depending on the file type used.
        """
        pass

class RssFeeder(Feeder):
    """
    `Author`: Eliakah kakou

    This class generates a list of feed instances containing relevant data about the file
    """

    def __init__(self, file):
        """
        `Author`: Eliakah kakou

        The constructor, initializes the RssFeeded instance

        `file`: path to file containing list of links
        """
        Feeder.__init__(self, file)
        self.feeds = []
        self.links = []
        self.path = file
        self.__getLinks(file)

    def load_feeds(self):
        """
        `Author`: Eliakah kakou

        returns 'feeds' which contains all of the feed instances
        """
        self.fetch()
        return self.feeds

    def __getLinks(self, file):
        """
        `Author`: Eliakah kakou

        This method inserts each link from the file as an entry into the 'links' list

        `file`:path to file containing list of links
        """
        input_file = open(file)
        try:
            for i, line in enumerate(input_file):
                self.links.append(line)
                # print line,
        finally:
            input_file.close()

    def fetch(self):
        """
        `Author`: Eliakah kakou

        This method appends all of the feed instances to 'feeds'
        """
        for i in range(len(self.links)):
            self.feeds.extend(self.__getFeeds(self.links[i]))

    def __getFeeds(self, url):
        """
        `Author`: Eliakah kakou

        This method generates a Feed instance from the url given

        `file`: full file path

        `return`: Feed generated from file
        """
        flag = validators.url(url)
        if flag:
            feeds = feedparser.parse(url)
        else:
            sys.exit("Invalid Url: Please try again!")

        list = feeds['entries']
        for i in range(len(list)):
            list[i] = Feed(list[i])
            feeds = list

        return feeds

class DocFeeder(Feeder):
    """
     `Author`: Eliakah kakou

     This class recursively access each file in a given directory,
     then generates a list of feed instances containing relevant data about the file
    """

    def __init__(self, file):
        """
        `Author`: Eliakah kakou

        The constructor, initializes the DocFeeded instance

        `file`: directory path
        """
        Feeder.__init__(self, file)
        self.feeds = []
        self.paths = []
        self.path = file
        self.__getPaths(file)

    def __getPaths(self, file):
        """
        `Author`: Eliakah kakou

        This method inserts each link from the file as an entry into the 'paths' list

        `file`: path to original directory
        """
        #print "getting Paths"
        matches = []
        for filename in self.__find_files(file, "*.*"):
            matches.append(filename)
        self.paths = matches

    def __find_files(self, directory, pattern):
        """
        `Author`: Eliakah kakou

        This method yields every single file in the directory matching the pattern

        `directory`: directory path

        `pattern`: file name pattern

        `return`: file paths matching pattern
        """
        #print "finding files"
        for root, dirs, files in os.walk(directory):
            for basename in files:
                if fnmatch.fnmatch(basename, pattern):
                    filename = os.path.join(root, basename)
                    yield filename

    def fetch(self):
        """
        `Author`: Eliakah kakou

        This method appends all of the feed instances to 'feeds'
        """
        #print "fetching ..."
        for i in range(len(self.paths)):
            self.feeds.append(self.__getFeeds(self.paths[i]))

    def load_feeds(self):
        """
        `Author`: Eliakah kakou

        `return`: 'feeds' which contains all of the feed instances
        """
        #print "Loading feeds"
        self.fetch()
        return self.feeds

    def __getFileName(self, path):
        """
        `Author`: Eliakah kakou

        This method extracts the file name from the path

        `path`: full path to the file

        `return`: the file name
        """
        drive, path = os.path.splitdrive(path)
        path, filename = os.path.split(path)
        name = filename.split(".")
        return name[0]

    def __getFeeds(self, file):
        """
        `Author`: Eliakah kakou

        This method generates a Feed instance from the file given

        `file`: full file path

        `return`: Feed generated from file
        """
        title = self.__getFileName(file)
        input_file = open(file)
        try:
            content = input_file.read()
        finally:
            input_file.close()

        stamp = time.ctime(self.__creation_date(file))

        f = {'id': file, 'published': stamp, 'title': title, 'link': file, 'summary': content}
        f = Feed(f)
        return f

    def __creation_date(self, file):
        """
        `Author`: Eliakah kakou

        This method returns the creation date of the file,
        or replaces with the last modified if the former is unavailable

        `return`: date on which file was created
    ...
        """
        if platform.system() == 'Windows':
            return os.path.getctime(file)
        else:
            stat = os.stat(file)
            try:
                return stat.st_birthtime
            except AttributeError:
                return stat.st_mtime

if __name__ == "__main__":
    feeder = RssFeeder('links.txt')
    feeder.fetch()
    feeds = feeder.feeds

    for i in range(len(feeds)):
        pprint.pprint(feeds[i].extract())

        # feeder = DocFeeder('C:\Users\Research\Desktop\parking')
        # feeds = feeder.load_feeds()
        #
        # for i in range(len(feeds)):
        #     pprint.pprint(feeds[i].extract())


import requests,base64,glob,json
from os import system,walk
from time import sleep
from random import choice
from colorama import Fore
from shutil import copyfileobj
from pystyle import Center,Colors,Colorate,System

System.Size(200,40)

b = Fore.LIGHTBLUE_EX
w = Fore.LIGHTYELLOW_EX
rw = Fore.LIGHTWHITE_EX

def clear():
    system('cls')

def title(text):
    system('title '+ text)

def getImagePath():
    file_path_type = ["ChangeToken/Images/*.jpg"]
    images = glob.glob(choice(file_path_type))
    random_image = choice(images)
    file_path = str(random_image).replace('\\','/')
    return file_path

def getImages(image_path):
    with open(image_path, "rb") as img_file:
        b64_string = base64.b64encode(img_file.read())
        return str(b64_string).replace('b\'','')

clear()
title(f'Discord Username ^& PFP ^& Bio Scraper  ^|  Made by seadhy#9999')
logo = Colorate.Vertical(Colors.yellow_to_red, Center.XCenter("""

  ██████ ▓█████ ▄▄▄        ██████  ███▄ ▄███▓ ▄▄▄        ██████  ██░ ██      ██████  ▄████▄   ██▀███   ▄▄▄       ██▓███  ▓█████  ██▀███
▒██    ▒ ▓█   ▀▒████▄    ▒██    ▒ ▓██▒▀█▀ ██▒▒████▄    ▒██    ▒ ▓██░ ██▒   ▒██    ▒ ▒██▀ ▀█  ▓██ ▒ ██▒▒████▄    ▓██░  ██▒▓█   ▀ ▓██ ▒ ██▒
░ ▓██▄   ▒███  ▒██  ▀█▄  ░ ▓██▄   ▓██    ▓██░▒██  ▀█▄  ░ ▓██▄   ▒██▀▀██░   ░ ▓██▄   ▒▓█    ▄ ▓██ ░▄█ ▒▒██  ▀█▄  ▓██░ ██▓▒▒███   ▓██ ░▄█ ▒
  ▒   ██▒▒▓█  ▄░██▄▄▄▄██   ▒   ██▒▒██    ▒██ ░██▄▄▄▄██   ▒   ██▒░▓█ ░██      ▒   ██▒▒▓▓▄ ▄██▒▒██▀▀█▄  ░██▄▄▄▄██ ▒██▄█▓▒ ▒▒▓█  ▄ ▒██▀▀█▄
▒██████▒▒░▒████▒▓█   ▓██▒▒██████▒▒▒██▒   ░██▒ ▓█   ▓██▒▒██████▒▒░▓█▒░██▓   ▒██████▒▒▒ ▓███▀ ░░██▓ ▒██▒ ▓█   ▓██▒▒██▒ ░  ░░▒████▒░██▓ ▒██▒
▒ ▒▓▒ ▒ ░░░ ▒░ ░▒▒   ▓▒█░▒ ▒▓▒ ▒ ░░ ▒░   ░  ░ ▒▒   ▓▒█░▒ ▒▓▒ ▒ ░ ▒ ░░▒░▒   ▒ ▒▓▒ ▒ ░░ ░▒ ▒  ░░ ▒▓ ░▒▓░ ▒▒   ▓▒█░▒▓▒░ ░  ░░░ ▒░ ░░ ▒▓ ░▒▓░
░ ░▒  ░ ░ ░ ░  ░ ▒   ▒▒ ░░ ░▒  ░ ░░  ░      ░  ▒   ▒▒ ░░ ░▒  ░ ░ ▒ ░▒░ ░   ░ ░▒  ░ ░  ░  ▒     ░▒ ░ ▒░  ▒   ▒▒ ░░▒ ░      ░ ░  ░  ░▒ ░ ▒░
░  ░  ░     ░    ░   ▒   ░  ░  ░  ░      ░     ░   ▒   ░  ░  ░   ░  ░░ ░   ░  ░  ░  ░          ░░   ░   ░   ▒   ░░          ░     ░░   ░
      ░     ░  ░     ░  ░      ░         ░         ░  ░      ░   ░  ░  ░         ░  ░ ░         ░           ░  ░            ░  ░   ░
                                                                                ░
                                ⌜――――――――――――――――――――――――――――――――――――――――――――――――――――⌝
                                ┇      [Discord] seadhy#9999                         ┇
                                ┇      [Github]  https://github.com/seadhy           ┇
                                ⌞――――――――――――――――――――――――――――――――――――――――――――――――――――⌟

"""))

class Tools:
    def __init__(self, want_proxy: bool):
        if want_proxy: self.proxies = open('proxies.txt','r',encoding='utf-8').read().splitlines()
        self.usernames = open('Scrapes/usernames.txt','r',encoding='utf-8').read().splitlines()
        self.token = json.load(open('config.json','r',encoding='utf-8'))['account_token']
        self.xsup = 'eyJvcyI6IldpbmRvd3MiLCJicm93c2VyIjoiQ2hyb21lIiwiZGV2aWNlIjoiIiwic3lzdGVtX2xvY2FsZSI6ImVuLVVTIiwiYnJvd3Nlcl91c2VyX2FnZW50IjoiTW96aWxsYS81LjAgKFdpbmRvd3MgTlQgMTAuMDsgV2luNjQ7IHg2NCkgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzEwNi4wLjAuMCBTYWZhcmkvNTM3LjM2IiwiYnJvd3Nlcl92ZXJzaW9uIjoiMTA2LjAuMC4wIiwib3NfdmVyc2lvbiI6IjEwIiwicmVmZXJyZXIiOiIiLCJyZWZlcnJpbmdfZG9tYWluIjoiIiwicmVmZXJyZXJfY3VycmVudCI6IiIsInJlZmVycmluZ19kb21haW5fY3VycmVudCI6IiIsInJlbGVhc2VfY2hhbm5lbCI6InN0YWJsZSIsImNsaWVudF9idWlsZF9udW1iZXIiOjE1MTYzOCwiY2xpZW50X2V2ZW50X3NvdXJjZSI6bnVsbH0='

        self.scraped_counter = 0
        self.param = {
            "limit": 100
        }

        self.p = Fore.LIGHTMAGENTA_EX
        self.w = Fore.LIGHTWHITE_EX
        self.b = Fore.LIGHTBLUE_EX
        self.g = Fore.LIGHTGREEN_EX
        self.r = Fore.LIGHTRED_EX
        self.c = Fore.LIGHTCYAN_EX
        self.y = Fore.LIGHTYELLOW_EX

        title(f'Discord Username ^& PFP ^& Bio Scraper  ^| Total Scraped: {self.scraped_counter}  ^|  Made by seadhy#9999')
        if self.token == 'enter account token':
            print(f'{self.b}[{self.p}Debug Mode{self.b}] {self.w}[{self.y}i{self.w}] Error -> Enter Account Token in config.json!')
            sleep(9999)
    def scrapeInfo(self,channelid: str, want_proxy: bool):
        file_number = len(next(walk('Scrapes/Images/'))[2])

        while True:
            try:
                headers = {
                    'accept': '*/*',
                    'authorization': self.token,
                    'cache-control': 'no-cache',
                    'pragma': 'no-cache',
                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
                    'x-debug-options': 'bugReporterEnabled',
                    'x-super-properties': self.xsup,
                }
                r = requests.get(f'https://discord.com/api/v9/channels/{channelid}/messages',params=self.param,headers=headers)

                for info in r.json():

                    username = info['author']['username']
                    userid = info['author']['id']
                    userpfp = info['author']['avatar']
                    if username not in self.usernames:
                        try:
                            self.usernames.append(username)
                        except:
                            continue
                        r2 = requests.get(url=f'https://cdn.discordapp.com/avatars/{userid}/{userpfp}.jpg?size=512',stream=True)
                        if want_proxy:
                            proxy = choice(self.proxies)
                            proxies = {
                                "http": f'http://{proxy}'
                            }

                            headers = {
                                "accept": "*/*",
                                "accept-encoding": "gzip, deflate, br",
                                'authorization': self.token,
                                "dnt": "1",
                                "sec-fetch-dest": "empty",
                                "sec-fetch-mode": "cors",
                                "sec-fetch-site": "same-origin",
                                "sec-gpc": "1",
                                "x-debug-options": "bugReporterEnabled",
                                "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36",
                                "x-super-properties": self.xsup
                            }

                            r3 = requests.get(url=f'https://discord.com/api/v9/users/{userid}/profile',headers=headers, proxies=proxies, timeout=10)
                            try:
                                userbio = r3.json()['user']['bio']
                            except KeyError:
                                print(f'{self.b}[{self.p}Debug Mode{self.b}] {self.w}[{self.y}i{self.w}] Error -> {self.r}No biography')
                                userbio = False
                            except requests.exceptions.JSONDecodeError as e:
                                print(r3.text)
                                print(f'{self.b}[{self.p}Debug Mode{self.b}] {self.w}[{self.y}i{self.w}] Error -> {self.r}Rate Limited!')
                                userbio = False

                        if want_proxy:
                            print(f'{self.b}[{self.p}Seasmash{self.b}] {self.w}[{self.g}+{self.w}] Username, PFP and Bio of the account {self.c}{username}{self.w} are scraped.')
                        else:
                            print(f'{self.b}[{self.p}Seasmash{self.b}] {self.w}[{self.g}+{self.w}] Username and PFP of the account {self.c}{username}{self.w} are scraped.')

                        file_number += 1
                        self.scraped_counter += 1
                        title(f'Discord Username ^& PFP ^& Bio Scraper  ^|  Total Scraped: {self.scraped_counter}  ^|  Made by seadhy#9999')
                        if userpfp != None:
                            with open(f'Scrapes/Images/images-{file_number}.jpg','wb') as down_file:
                                copyfileobj(r2.raw, down_file)

                        else:
                            file_number -= 1
                        with open('Scrapes/usernames.txt','a',encoding='utf-8') as file:
                            file.write(f"{username}\n")
                        if want_proxy and userbio != False:
                            if userbio != '[]' and not "\n" in userbio and not 'discord.gg/' in userbio and "" != userbio and not '.gg/' in userbio:
                                with open('Scrapes/bio.txt','a',encoding='utf-8') as file:
                                    file.write(f"{userbio}\n")
                self.param = {
                        'before': r.json()[-1]['id'],
                        'limit': 100
                    }
            except Exception as e:
                if 'Unauthorized'in r.text:
                    print(f'{self.b}[{self.p}Debug Mode{self.b}] {self.w}[{self.y}i{self.w}] Error -> {self.r}Token Invalid! Change token in \'config.json\'')
                    sleep(10)
                    break
                else:
                    print(f'{self.b}[{self.p}Debug Mode{self.b}] {self.w}[{self.y}i{self.w}] Error -> {self.r}{str(e).capitalize()}')
                sleep(5)

print(logo)
while True:
    print(f"""
{b}[{w}1{b}]{rw} Username & PFP & Bio Scraper
{b}[{w}2{b}]{rw} Escape
    """)
    user_choice = int(input('> '))
    if user_choice == 1:
        print(f"\n{b}[{w}?{b}]{rw} Channel ID: ")
        channel_id = input('> ')
        print(f"\n{b}[{w}?{b}]{rw} Do you want to use a proxy? (y/n)")
        print(f"{b}[{w}*{b}]{rw} WARNING: If you don't want to use it, the program won't collect the bio!")
        want_proxy = input('> ')
        if want_proxy.lower() == 'y':
            want_proxy = True
        elif want_proxy.lower() == 'n':
            want_proxy = False
        clear()
        print(logo)
        tool = Tools(want_proxy)
        tool.scrapeInfo(channel_id, want_proxy)
    elif user_choice == 2:
        print("Exiting...")
        sleep(3)
        break


from tensorflow.python.data.util import nest as data_nest
from tf_agents.replay_buffers import table
from tf_agents.replay_buffers.tf_uniform_replay_buffer import TFUniformReplayBuffer, _valid_range_ids

import collections
import logging
import numpy as np
import tensorflow as tf

logger = logging.getLogger(__name__)

# this is set so that mathematical operations on NaN are raised.
np.seterr(all='raise')

# NOTE: storing the weights in 'probabilities' to be consistent with uniform buffer
BufferInfo = collections.namedtuple('BufferInfo', ['ids', 'probabilities'])

MIN_PRIORITY = 0.0001

class TfPrioritizedReplayBuffer(TFUniformReplayBuffer):
    """
    This class implements a prioritized replay buffer based on the paper linked below.
    Link to Paper: https://arxiv.org/pdf/1511.05952.pdf

    This replay buffer will track priorities, sample a batch according to them, calculate weights and let us
    update priorities after the loss has become known.

    For simplicity, sampling the experiences is implemented in the O(N). For more efficient way to perform sampling,
    the use of trees is required: https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py#L71

    This priority buffer is designed to only handle 1 step mini_batch extractions. Time stacking is therefore not
    supported either.

    This priority buffer is also design to only 1 max_length(Segments of batches).
    This simplifies the way the priorities are tracked. Our current use case requires adding a batch once after
    initializing a buffer which makes it possible to use max_length == 1.
    """

    def __init__(self, data_spec,
                 batch_size,
                 max_length=1,
                 alpha=0.6,
                 scope='TFUniformReplayBuffer',
                 device='cpu:*',
                 table_fn=table.Table):
        """
        Args:
            data_spec: A TensorSpec or a list/tuple/nest of TensorSpecs describing a
                       single item that can be stored in this buffer.
            batch_size: Batch dimension of tensors when adding to buffer.
            max_length: The maximum number of items that can be stored in a single
                        batch segment of the buffer.
            alpha: α determines how much prioritization is used, with α = 0 corresponding to the uniform case.
            scope: Scope prefix for variables and ops created by this class.
            device: A TensorFlow device to place the Variables and ops.
            table_fn: Function to create tables `table_fn(data_spec, capacity)` that
                      can read/write nested tensors.
        Raises:
            ValueError: If batch_size does not evenly divide capacity.
        """
        if max_length != 1:
            raise NotImplementedError('TfPrioritizedReplayBuffer only supports a batch segments '
                                      'size of 1, but received `max_length` {}.'.format(max_length))

        super(TfPrioritizedReplayBuffer, self).__init__(data_spec, batch_size, max_length, scope, device, table_fn)
        logger.info("Creating an instance of %s. Params: data_spec: %s, capacity: %s, alpha: %s, segment_length: %s" %
                    (str(type(self).__name__), str(data_spec), str(batch_size), str(alpha), str(max_length)))

        self._alpha = alpha

        # an array in which we keep track of the priorities. The size of this array is equal to the size of the replay
        # buffer. Items stored at a given index in the Priority array map to the experience at the same index in the
        # buffer. The content of the items in the priority array represent the loss of their respective experience the
        # last time that experience was used for training.
        self._priority_table = np.zeros((self._capacity_value,), dtype=np.float32)

    def _add_batch(self, items):
        """Adds a batch of items to the replay buffer.

        Args:
          items: A tensor or list/tuple/nest of tensors representing a batch of
          items to be added to the replay buffer. Each element of `items` must match
          the data_spec of this class. Should be shape [batch_size, data_spec, ...]
        Returns:
          An op that adds `items` to the replay buffer.
        Raises:
          ValueError: If called more than once.
        """
        super()._add_batch(items)
        logger.info("Adding a batch of %s experiences to Replay buffer" % str(self._capacity_value))
        # WARNING: This is implemented under the assumption that add batch is only called once
        self._priority_table = np.ones(self._capacity_value, dtype=np.float32)

    @staticmethod
    def compute_beta(beta, curr_iter, total_iter):
        return min(1.0, beta + curr_iter * (1.0 - beta) / total_iter)

    def get_next(self,
                 sample_batch_size=None,
                 beta=0.4,
                 num_steps=1,
                 time_stacked=False):
        """Returns an item or batch of items sampled based on the experience's priorities from the buffer.

            Only num_steps = 1 and time_stacked=False are supported.

            Args:
                sample_batch_size: (Optional.) An optional batch_size to specify the
                    number of items to return. See get_next() documentation.
                beta (float): ratio to use when computing the importance sampling weights
                num_steps: (Optional.)  Optional way to specify that sub-episodes are
                    desired. See get_next() documentation.
                time_stacked: Bool, when true and num_steps > 1 get_next on the buffer
                    would return the items stack on the time dimension. The outputs would be
                    [B, T, ..] if sample_batch_size is given or [T, ..] otherwise.
            Returns:
                A 2 tuple, containing:
                    - An item, sequence of items, or batch thereof sampled uniformly
                    from the buffer.
                    - BufferInfo NamedTuple, containing:
                        - The items' ids.
                        - The importance sampling weight of each item.
            Raises:
                ValueError: if num_steps is bigger than the capacity.
        """
        num_steps_value = num_steps if num_steps is not None else 1

        if num_steps_value != 1:
            raise NotImplementedError('TfPrioritizedReplayBuffer only supports a batches with num_step '
                                      'size of 1, but received batch with num_steps'
                                      'size {}.'.format(num_steps_value))
        with tf.device(self._device), tf.name_scope(self._scope):
            with tf.name_scope('get_next'):
                min_val, max_val = _valid_range_ids(
                    self._get_last_id(), self._max_length, num_steps)
                tf.compat.v1.assert_greater(
                    max_val,
                    min_val,
                    message='TFUniformReplayBuffer is empty. Make sure to add items '
                            'before sampling the buffer.')

        # select index based on the priorities of the experiences in the buffer
        selected_idx, selected_weight = self._sample_ids(sample_batch_size, beta)

        # convert ids to tensor
        rows_to_get = tf.convert_to_tensor(selected_idx)
        importance_sampling_weigths = tf.convert_to_tensor(selected_weight)
        data = self._data_table.read(rows_to_get)

        buffer_info = BufferInfo(ids=rows_to_get, probabilities=importance_sampling_weigths)
        return data, buffer_info

    def _sample_ids(self, sample_size, beta):
        """
        select the indices of the experience to pull from the prioritize replay buffer in addition to compute it's
        importance sampling weight.

        Params:
            sample_size (int): number of ids to retrieve
            beta (float): This a factor to which the Important Sampling is present
        """
        priorities = self._priority_table

        # update the priorities by the power of alpha
        probabilities = priorities ** self._alpha
        probabilities /= probabilities.sum()

        # select an index stochastically
        idx = np.random.choice(len(probabilities), sample_size, p=probabilities)

        # compute important sampling weights of the selected indices
        selected_probabilities = probabilities[idx]
        importance_sampling_weights = (self._capacity_value * selected_probabilities) ** (-beta)
        importance_sampling_weights /= importance_sampling_weights.max()

        return idx, importance_sampling_weights

    def update_priorities(self, indices, new_priorities):
        """
        Updates new priorities of the experiences in the given indices.

        Params:
            batch_indices: the indices to update
            batch_priorities: the new priorities
        """
        for idx, priority in zip(indices, new_priorities):
            ceil_priority = max(priority, MIN_PRIORITY)
            self._priority_table[idx] = ceil_priority

    def as_dataset(self,
                   sample_batch_size=None,
                   beta=0.4,
                   num_steps=None,
                   num_parallel_calls=None):
        """Creates a dataset that returns entries from the buffer.

            Args:
                sample_batch_size: (Optional.) An optional batch_size to specify the
                    number of items to return. See as_dataset() documentation.
                beta: This a factor to which the Important Sampling is present
                num_steps: (Optional.)  Optional way to specify that sub-episodes are
                    desired. See as_dataset() documentation.
                num_parallel_calls: (Optional.) Number elements to process in parallel.
                See as_dataset() documentation.
            Returns:
                A dataset of type tf.data.Dataset, elements of which are 2-tuples of:
                    - An item or sequence of items or batch thereof
                    - Auxiliary info for the items (i.e. ids, probs).
            Raises:
                ValueError: If the data spec contains lists that must be converted to
                tuples.
        """
        # data_tf.nest.flatten does not flatten python lists, nest.flatten does.
        if tf.nest.flatten(self._data_spec) != data_nest.flatten(self._data_spec):
            raise ValueError(
                'Cannot perform gather; data spec contains lists and this conflicts '
                'with gathering operator.  Convert any lists to tuples.  '
                'For example, if your spec looks like [a, b, c], '
                'change it to (a, b, c).  Spec structure is:\n  {}'.format(
                    tf.nest.map_structure(lambda spec: spec.dtype, self._data_spec)))

        def get_next(_):
            return self.get_next(sample_batch_size, beta=beta, num_steps=num_steps, time_stacked=False)

        return tf.data.experimental.Counter().map(
            get_next,
            num_parallel_calls=num_parallel_calls)


import os
import numpy as np
import matplotlib.pyplot as mp
from mpl_toolkits.mplot3d import axes3d

colors = ('r', 'g', 'b', 'k')

data_list = []
with open('celiang.txt','r') as f:
    data_list = [i.strip().split(' ') for i in f.readlines()]

x = [-250,0,250]

data_x = [float(i[0]) for i in data_list]
data_y = [float(i[1]) for i in data_list]
data_z = [float(i[2]) for i in data_list]

mp.figure("3D scatter", facecolor="lightgray")
ax3d = mp.gca(projection="3d")  # 创建三维坐标

ax3d.set_xlabel('x', fontsize=14)
ax3d.set_ylabel('y', fontsize=14)
ax3d.set_zlabel('z', fontsize=14)

ax3d.scatter(data_x, data_y, data_z, s=20, cmap="jet", marker="o")

mp.show()


"""
Cases and functions for testing the non- scraping and plotting functions in the
helpers.py file
"""
import pytest
import pandas as pd

from helpers import (
    average_data,
    clean_gdp_data,
    clean_population_data,
    merge_dataframes,
    pivot
)

clean_gdp_data_cases = [
    ("test_data/gdp_test_data1_raw.csv", "test_data/gdp_test_data1_clean.csv"),
    ("test_data/gdp_test_data2_raw.csv", "test_data/gdp_test_data2_clean.csv")
]

@pytest.mark.parametrize("raw,clean", clean_gdp_data_cases)
def test_clean_gdp_data(raw, clean):
    """
    Test the clean_gdp_data() function in helpers.py.

    Args:
        raw: the filepath for the CSV containing the test data to be cleaned.
        clean: the filepath for the CSV containing the test data correctly
            cleaned.
    """
    df_clean = pd.read_csv(clean)
    assert df_clean.equals(clean_gdp_data(raw))

clean_pop_data_cases = [
    ("test_data/pop_test_data1_raw.csv", "test_data/pop_test_data1_clean.csv"),
    ("test_data/pop_test_data2_raw.csv", "test_data/pop_test_data2_clean.csv")
]

@pytest.mark.parametrize("raw,clean", clean_pop_data_cases)
def test_clean_population_data(raw, clean):
    """
    Test the clean_population_data() function in helpers.py.

    Args:
        raw: the filepath for the CSV containing the test data to be cleaned.
        clean: the filepath for the CSV containing the test data correctly
            cleaned.
    """
    df_clean = pd.read_csv(clean)
    assert df_clean.equals(clean_population_data(raw))

def test_merge_dataframe():
    """
    Test the merge_dataframe() function in helpers.py.
    """
    # Load the correctly merged dataframe to check against
    df_merged = pd.read_csv("test_data/merge_test_data.csv")
    # Load the three test dataframes to be merged
    test_medals = pd.read_csv("test_data/medals_test_data_clean.csv")
    test_gdp = pd.read_csv("test_data/gdp_test_data1_clean.csv")
    test_pop = pd.read_csv("test_data/pop_test_data1_clean.csv")
    # Assert the merge done properly
    assert df_merged.equals(merge_dataframes(
        [test_medals, test_gdp, test_pop]))

def test_pivot():
    """
    Test the pivot() function in helpers.py.
    when we saved our CSVs the data type of year was automatically changed
    from object to int64. As a work around to this, we save our CSV and then
    read it back in for the test. In our actual files, we never save the pivot
    data as a csv so this doesn't affect the actual function.
    """
    df_clean = pd.read_csv("test_data/pivoting_test_data_clean.csv")
    df_pivot = pd.read_csv("test_data/pivoting_test_data.csv")
    df_pivot = pivot(df_pivot)
    df_pivot.to_csv("test_data/pivoting_test_data_intermediate.csv", index=False)
    df_pivot = pd.read_csv("test_data/pivoting_test_data_intermediate.csv")
    pd.testing.assert_frame_equal(df_clean, df_pivot)

def test_average_data():
    """
    Test the average_data() function in helpers.py.
    """
    # Load the correctly averaged dataframe to check against
    df_done = pd.read_csv("test_data/averaging_test_data_done.csv")
    # Load test dataframe to be averaged
    df_raw = pd.read_csv("test_data/averaging_test_data.csv")
    # Assert the averaging done properly
    assert df_done.equals(average_data(df_raw))


import torch
import pdb
import torch.nn as nn
import math
from torch.autograd import Variable
from torch.autograd import Function

import numpy as np

def Binarize(tensor,quant_mode='det'):
    if quant_mode=='det':
        return tensor.sign()
    else:
        return tensor.add_(1).div_(2).add_(torch.rand(tensor.size()).add(-0.5)).clamp_(0,1).round().mul_(2).add_(-1)

def Ternarize(tensor):
    th = 0.5
    t = tensor.clone().zero_()
    t[tensor.ge(th)] = 1.
    t[tensor.le(-th)] = -1.
    return t

class STETernery(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input, inplace):
        if inplace:
            ctx.mark_dirty(input)

        output = Ternarize(input)
        return output

    @staticmethod
    def backward(ctx, grad_output):
        """
        Straight Through Estimator
        """
        return grad_output, None, None, None, None

class TerneryHardTanh(nn.Module):
    def __init__(self, inplace=True):
        super(TerneryHardTanh, self).__init__()
        self.inplace = inplace
    def forward(self, input):
        input = torch.nn.functional.hardtanh(input)
        input = STETernery.apply(input, self.inplace)
        return input

class HingeLoss(nn.Module):
    def __init__(self):
        super(HingeLoss,self).__init__()
        self.margin=1.0

    def hinge_loss(self,input,target):
            #import pdb; pdb.set_trace()
            output=self.margin-input.mul(target)
            output[output.le(0)]=0
            return output.mean()

    def forward(self, input, target):
        return self.hinge_loss(input,target)

class SqrtHingeLossFunction(Function):
    def __init__(self):
        super(SqrtHingeLossFunction,self).__init__()
        self.margin=1.0

    def forward(self, input, target):
        output=self.margin-input.mul(target)
        output[output.le(0)]=0
        self.save_for_backward(input, target)
        loss=output.mul(output).sum(0).sum(1).div(target.numel())
        return loss

    def backward(self,grad_output):
       input, target = self.saved_tensors
       output=self.margin-input.mul(target)
       output[output.le(0)]=0
       import pdb; pdb.set_trace()
       grad_output.resize_as_(input).copy_(target).mul_(-2).mul_(output)
       grad_output.mul_(output.ne(0).float())
       grad_output.div_(input.numel())
       return grad_output,grad_output

def non_zeroQ(input, numBits=4):
    output = input / 2 / torch.max(torch.abs(input)) + 1 / 2
    output=output.mul(2**numBits-1).round().div(2**numBits-1)
    output = 2 * output - 1
    return output

def Quantize(tensor,quant_mode='det',  params=None, numBits=8):
    if quant_mode=='det':
        tensor.clamp_(-2**(numBits-1),2**(numBits-1))
        tensor = non_zeroQ(tensor, numBits=numBits)
    else:
        tensor=tensor.mul(2**(numBits-1)).round().add(torch.rand(tensor.size()).add(-0.5)).div(2**(numBits-1))
        quant_fixed(tensor, params)
    return tensor
# import torch.nn._functions as tnnf

class BinarizeLinear(nn.Linear):

    def __init__(self, *kargs, **kwargs):
        super(BinarizeLinear, self).__init__(*kargs, **kwargs)

    def forward(self, input):
        # if input.size(1) != 784:
        #     # input.data=Quantize(input.data, numBits=2)
        #     # input.data=Binarize(input.data)
        #     input = STETernery.apply(input, True)
        if not hasattr(self.weight,'org'):
            self.weight.org=self.weight.data.clone()
        self.weight.data=Binarize(self.weight.org)
        out = nn.functional.linear(input, self.weight)
        if not self.bias is None:
            self.bias.org=self.bias.data.clone()
            out += self.bias.view(1, -1).expand_as(out)

        return out

class BinarizeConv2d(nn.Conv2d):

    def __init__(self, *kargs, **kwargs):
        super(BinarizeConv2d, self).__init__(*kargs, **kwargs)

    def forward(self, input):
        # if input.size(1) != 3:
        #     # input.data=Quantize(input.data, numBits=2)
        #     # input.data = Binarize(input.data)
        #     input = STETernery.apply(input, True)
        if not hasattr(self.weight,'org'):
            self.weight.org=self.weight.data.clone()
        self.weight.data=Binarize(self.weight.org)

        out = nn.functional.conv2d(input, self.weight, None, self.stride,
                                   self.padding, self.dilation, self.groups)

        if not self.bias is None:
            self.bias.org=self.bias.data.clone()
            out += self.bias.view(1, -1, 1, 1).expand_as(out)

        return out


import logging
import unittest

import pack.test_module

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

logger.critical('Display critical infomration')

''' Practice on invoking function '''
pack.test_module.test_invoke('Jaron','Anny')

''' Practice on invoking class '''
acct = pack.test_module.Account('Justin', '123-4567', 1000)
acct.name = 'Marco'

check_acct = pack.test_module.CheckingAccount('Anny', '123-4567', 100, 'Female')
logger.debug('acct = {}'.format(acct))
logger.debug('check_acct = {}'.format(check_acct))

acct.deposit(500)

check_acct.deposit(100)

logger.debug('acct = {}'.format(acct))
logger.debug('check_acct = {}'.format(check_acct))
logger.debug("{0}'s balance = {1}".format(acct.name, acct.balance))

check_acct.set_gendor('Male')
check_acct.set_gendor('Female')

logger.debug("check_acct = {}".format(check_acct))


import numpy as np
from scipy.spatial.distance import cityblock
from itertools import product
import pytest
from alibi.utils.distance import cityblock_batch

dims = np.array([1, 10, 50])
shapes = list(product(dims, dims))
n_tests = len(dims) ** 2

@pytest.fixture
def random_matrix(request):
    shape = shapes[request.param]
    matrix = np.random.rand(*shape)
    return matrix

@pytest.mark.parametrize('random_matrix', list(range(n_tests)), indirect=True)
def test_cityblock_batch(random_matrix):
    X = random_matrix
    y = X[np.random.choice(X.shape[0])]

    batch_dists = cityblock_batch(X, y)
    single_dists = np.array([cityblock(x, y) for x in X]).reshape(X.shape[0], -1)

    assert np.allclose(batch_dists, single_dists)


# -*- coding: utf-8 -*-
import pytest

from django.conf import settings

from web.favorites_utils import decode_favorites, encode_favorites, toggle_favorite

TEST_OK_SIZES = [(3), (settings.MAX_FAVORITES)]

@pytest.mark.parametrize("size", TEST_OK_SIZES)
def test_ok_encode_decode(size):
    favorites = []
    for id in range(size):
        favorites.append(id)

    encoded = encode_favorites(favorites)
    decoded = decode_favorites(encoded)

    assert len(favorites) == len(decoded)

    for i in range(size):
        assert favorites.pop(0) == decoded.pop(0), "items at %d not same" % i

def test_nok_toggle_decode():
    buffer = "X" * (settings.MAX_FAVORITES * 8 + 1)
    with pytest.raises(ValueError) as e_info:
        decoded = decode_favorites(buffer)

def test_nok_toggle_favorite():
    favorites = []
    for id in range(settings.MAX_FAVORITES):
        favorites.append(id)
    result = toggle_favorite(favorites, settings.MAX_FAVORITES + 1)
    assert result == "error"


from django.test import TestCase

# Create your tests here.

print("hello"[0:150])

from bs4 import BeautifulSoup

s="
hello
123"

soup=BeautifulSoup(s,"html.parser")

# print(soup.text)

print(soup.find_all())

for tag in soup.find_all():

    print(tag.name)
    if tag.name=="script":
        tag.decompose()

print(str(soup))


"""Class for Job Badges resource generator."""

from PIL import Image, ImageDraw
from utils.TextBoxDrawer import TextBox, TextBoxDrawer
from django.utils.translation import ugettext_lazy as _
from resources.utils.BaseResourceGenerator import BaseResourceGenerator
from resources.utils.coords import calculate_box_vertices

FONT_PATH = "static/fonts/PatrickHand-Regular.ttf"
FONT_SIZE = 200
TEXTBOX_WIDTH = 1000
TEXTBOX_HEIGHT = 275
LABEL_DATA = {
    "programmer": _("Hello, I'm a Programmer"),
    "tester": _("Hello, I'm a Tester"),
    "bot": _("Hello, I'm a Bot"),
}

class JobBadgesResourceGenerator(BaseResourceGenerator):
    """Class for Job Badges resource generator."""

    def data(self):
        """Create data for a copy of the Job Badges resource.

        Returns:
            A dictionary of the one page for the resource.
        """
        image_path = "static/img/resources/job-badges/job-badges.png"
        image = Image.open(image_path)
        draw = ImageDraw.Draw(image)
        textbox_drawer = TextBoxDrawer(image, draw)

        # coordinates of top left point of text box
        top_left_x_coords = [50, 1200]
        top_left_y_coord = 100

        # Add text labels
        for label, label_text in LABEL_DATA.items():
            for top_left_x_coord in top_left_x_coords:
                vertices = calculate_box_vertices(
                    (top_left_x_coord, top_left_y_coord),
                    TEXTBOX_WIDTH,
                    TEXTBOX_HEIGHT
                )
                box = TextBox(
                    vertices=vertices,
                    width=TEXTBOX_WIDTH,
                    height=TEXTBOX_HEIGHT,
                    font_path=FONT_PATH,
                    font_size=FONT_SIZE,
                )
                textbox_drawer.write_text_box(
                    box,
                    label_text,
                    horiz_just="center",
                    vert_just="center",
                )

            # increase y coord for next name tag down
            top_left_y_coord += 675

        return {"type": "image", "data": image}


from directory import Directory
import os
import shutil
import pwd
import grp
import time
import stat
import platform
import shutil
import tarfile
import fnmatch

class FileSystem:
    # The FileSystem class is a class that represents a file system.
    def __init__(self):
        self.current_directory = Directory(os.getcwd(), None, None, None)
        os.chdir(self.current_directory.path)
        self.home_directory = Directory(
            os.path.expanduser('~'), None, None, None)

    def resolve_path(self, path):
        """
        The function resolves a given path based on the current directory and returns the absolute path.

        :param path: The `path` parameter is a string that represents a file or directory path
        :return: the resolved path based on the given input path.
        """
        if isinstance(self.current_directory, Directory):
            self.current_directory = self.current_directory.path
        if isinstance(path, Directory):
            path = path.path
        if path == '.':
            # Do nothing and return the current directory
            return self.current_directory
        elif path == '..':
            # Go up one directory level
            return os.path.dirname(self.current_directory)
        elif path == '~' or path == '':
            # Expand ~ or an empty string to the user's home directory
            return os.path.expanduser('~')
        else:
            # For other cases, join the current directory with the given path
            return os.path.join(self.current_directory, path)

    def change_directory(self, new_directory):

        print(f"Changing directory: {new_directory}")

        new_directory = self.resolve_path(new_directory)

        try:
            # If new_directory is a Directory object, get its path
            if isinstance(new_directory, Directory):
                new_directory = new_directory.path

            os.chdir(new_directory)
            self.current_directory = new_directory
            print(
                f"Successfully changed directory to {self.current_directory}")
        except Exception as e:
            print(f"Error: {e}")

    def create_directory(self, directory, recursive=False):
        """
        The function creates a directory at the specified path, with an option to create parent
        directories recursively.

        :param directory: The "directory" parameter is the path of the directory that you want to
        create. It can be either a relative or an absolute path
        :param recursive: The "recursive" parameter is a boolean flag that determines whether the
        directory should be created recursively or not, defaults to False (optional)
        """
        directory = self.resolve_path(directory)
        try:
            if recursive:
                os.makedirs(directory, exist_ok=True)
            else:
                os.mkdir(directory)
            print(f'Successfully created directory: {directory}')
        except Exception as e:
            print(f'Error: Could not create directory "{directory}". {str(e)}')

    def remove_directory(self, directory, recursive=False):
        """
        The function removes a directory and its contents if recursive is set to True, otherwise it only
        removes an empty directory.

        :param directory: The "directory" parameter is the path of the directory that you want to
        remove. It can be either a relative or an absolute path
        :param recursive: The "recursive" parameter is a boolean flag that determines whether to remove
        the directory and its contents recursively or not. If set to True, the function will remove the
        directory and all its subdirectories and files. If set to False, the function will only remove
        the directory if it is empty, defaults to False (optional)
        """
        directory = self.resolve_path(directory)
        try:
            if recursive:
                shutil.rmtree(directory)
            else:
                os.rmdir(directory)
            print(f'Successfully removed directory: {directory}')
        except FileNotFoundError:
            print(f'Error: Directory "{directory}" does not exist.')
        except OSError as e:
            # If the directory is not empty, an OSError will be thrown
            print(f'Error: Directory "{directory}" is not empty. {str(e)}')
        except Exception as e:
            print(f'Error: Could not remove directory "{directory}". {str(e)}')

    def create_file(self, filename):
        """
        The function `create_file` creates a new file with the given filename, handling exceptions for
        file already existing or other errors.

        """
        filename = self.resolve_path(filename)
        try:
            with open(filename, 'x') as f:  # 'x' mode creates a new file and opens it for writing
                pass  # We don't need to write anything to the file, so we just pass
            print(f'Successfully created file: {filename}')
        except FileExistsError:
            print(f'Error: File "{filename}" already exists.')
        except Exception as e:
            print(f'Error: Could not create file "{filename}". {str(e)}')

    def remove_file(self, filename):
        """
        The function removes a file specified by the filename, handling different error cases.

        """
        filename = self.resolve_path(filename)
        try:
            os.remove(filename)
            print(f'Successfully removed file: {filename}')
        except FileNotFoundError:
            print(f'Error: File "{filename}" does not exist.')
        except PermissionError:
            print(f'Error: Permission denied for deleting "{filename}".')
        except Exception as e:
            print(f'Error: Could not remove file "{filename}". {str(e)}')

    def list_directory(self, path='.', long_format=False, show_hidden=False):
        """
        The function `list_directory` lists the files and directories in a given path, with options for
        long format and showing hidden files.
        cluded in the listing, defaults to False (optional)
        """
        resolved_path = self.resolve_path(path)
        if os.path.isfile(resolved_path):
            if long_format:
                self.print_file_info(resolved_path)
            else:
                print(os.path.basename(resolved_path))
        else:
            for filename in os.listdir(resolved_path):
                if filename.startswith('.') and not show_hidden:
                    continue
                if long_format:
                    self.print_file_info(os.path.join(resolved_path, filename))
                else:
                    print(filename)

    def print_file_info(self, filename):
        """
        The function `print_file_info` prints information about a file, including its mode, number of
        links, owner, group, size, modification time, and filename.

        """
        info = os.stat(filename)
        file_mode = stat.filemode(info.st_mode)
        n_links = info.st_nlink
        owner = pwd.getpwuid(info.st_uid).pw_name
        group = grp.getgrgid(info.st_gid).gr_name
        size = info.st_size
        mtime = time.ctime(info.st_mtime)
        print(f'{file_mode} {n_links} {owner} {group} {size} {mtime} {filename}')

    def move_file(self, source, destination):
        """
        The function `move_file` moves a file from a source location to a destination location, handling
        exceptions for file not found and general errors.

        """

        source = self.resolve_path(source)
        destination = self.resolve_path(destination)
        try:
            shutil.move(source, destination)
            print(f'Successfully moved {source} to {destination}')
        except FileNotFoundError:
            print(f'Error: File "{source}" does not exist.')
        except Exception as e:
            print(
                f'Error: Could not move file "{source}" to "{destination}". {str(e)}')

    def print_working_directory(self):
        """
        The function prints the current working directory.
        """
        print(os.getcwd())

    def copy(self, source, destination):
        """
        The `copy` function copies a file or directory from a source path to a destination path,
        handling errors and providing feedback on the operation.

        """
        source = self.resolve_path(source)
        destination = self.resolve_path(destination)
        try:
            if os.path.isdir(source):
                destination = os.path.join(
                    destination, os.path.basename(source))
                shutil.copytree(source, destination)
            else:
                shutil.copy2(source, destination)
            print(f'Successfully copied {source} to {destination}')
        except FileNotFoundError:
            print(f'Error: Source "{source}" does not exist.')
        except Exception as e:
            print(
                f'Error: Could not copy "{source}" to "{destination}". {str(e)}')

    def clear_console(self):
        """
        The function clears the console screen by using the appropriate clear command based on the
        platform.
        """
        # Check the platform and use the appropriate clear command
        print('Clearing console')
        if platform.system().lower() == "windows":
            os.system("cls")
        else:
            os.system("clear")

    def change_permission(self, filename, permission):
        """
        The function `change_permission` changes the permissions of a file specified by `filename` to
        the value specified by `permission`.

        """
        filename = self.resolve_path(filename)
        try:
            # Convert permission to an integer base 8
            os.chmod(filename, int(permission, 8))
            print(
                f'Successfully changed permissions of {filename} to {permission}')
        except Exception as e:
            print(
                f'Error: Could not change permissions of "{filename}". {str(e)}')

    def change_owner_group(self, filename, owner, group):
        """
        The function `change_owner_group` changes the owner and group of a file in Python.

        """
        filename = self.resolve_path(filename)
        try:
            uid = pwd.getpwnam(owner).pw_uid
            gid = grp.getgrnam(group).gr_gid
            os.chown(filename, uid, gid)
            print(
                f'Successfully changed owner/group of {filename} to {owner}:{group}')
        except Exception as e:
            print(
                f'Error: Could not change owner/group of "{filename}". {str(e)}')

    def show_stats(self, filename):
        """
        The function `show_stats` retrieves and displays various statistics of a file specified by the
        `filename` parameter.

        """
        filename = self.resolve_path(filename)
        try:
            stat_info = os.stat(filename)

            print(f'Statistics for {filename}:')
            print(f'File Type: {self.get_file_type(stat_info.st_mode)}')
            print(f'Size: {stat_info.st_size} bytes')
            print(f'Permissions: {oct(stat.S_IMODE(stat_info.st_mode))}')
            print(f'Owner ID: {stat_info.st_uid}')
            print(f'Group ID: {stat_info.st_gid}')
            print(f'Number of Hard Links: {stat_info.st_nlink}')
            print(f'Last Modified: {time.ctime(stat_info.st_mtime)}')
            print(f'Last Accessed: {time.ctime(stat_info.st_atime)}')
            print(f'Last Metadata Change: {time.ctime(stat_info.st_ctime)}')
            return stat_info
        except Exception as e:
            print(
                f'Error: Could not retrieve statistics for "{filename}". {str(e)}')

    def get_file_type(self, mode):
        """
        The function `get_file_type` takes a file mode as input and returns the corresponding file type
        as a string.
        """
        if stat.S_ISDIR(mode):
            return 'Directory'
        elif stat.S_ISREG(mode):
            return 'Regular File'
        elif stat.S_ISLNK(mode):
            return 'Symbolic Link'
        # Add more types as needed
        else:
            return 'Unknown'

    def edit_file(self, filename, new_content=None):
        """
        The `edit_file` function allows for editing the content of a file or printing its content if no
        new content is provided.

        """
        filename = self.resolve_path(filename)
        try:
            if new_content is not None:
                with open(filename, 'w') as f:
                    f.write(new_content)
                print(f'Successfully edited {filename}')
            else:
                with open(filename, 'r') as f:
                    print(f.read())
        except FileNotFoundError:
            print(f'Error: File "{filename}" does not exist.')
        except PermissionError:
            print(f'Error: Permission denied for editing "{filename}".')
        except Exception as e:
            print(f'Error: Could not edit file "{filename}". {str(e)}')

    def write_to_file(self, message, filename, mode):
        """
        The function `write_to_file` writes a message to a file specified by the filename and mode,
        handling various exceptions.

        """
        filename = self.resolve_path(filename)
        try:
            with open(filename, mode) as f:
                f.write(message + '\n')
            print(f'Successfully wrote to {filename}')
        except FileNotFoundError:
            print(f'Error: File "{filename}" does not exist.')
        except PermissionError:
            print(f'Error: Permission denied for writing to "{filename}".')
        except Exception as e:
            print(f'Error: Could not write to file "{filename}". {str(e)}')

    def gzip(self, command_parts):
        """
        The function checks the mode of the gzip command and performs compression or extraction
        accordingly.

        """
        if len(command_parts) < 3:
            print('Error: The gzip command requires at least two arguments.')
            return

        mode = command_parts[1]
        if mode not in ['-c', '-x']:
            print(
                'Error: The gzip command requires a mode: -c for compression or -x for extraction.')
            return

        if mode == '-c':
            self.compress_files(command_parts[2:])
        elif mode == '-x':
            self.extract_file(command_parts[2])

    def edit_file_interactive(self, filename):
        """
        The function `edit_file_interactive` allows the user to interactively edit a file by adding new
        lines of text.

        """

        filename = self.resolve_path(filename)
        try:
            with open(filename, 'r') as f:
                lines = f.readlines()

            print("Enter your text (enter an empty line to save and quit):")
            i = 0
            while i < len(lines):
                print(f"{i+1}: {lines[i]}", end="")
                i += 1
            while True:
                print("Enter new line (leave empty to save and quit):")
                line = input(f"{i+1}: ")
                if line.strip() == "":
                    break
                lines.append(line + '\n')
                i += 1

            with open(filename, 'w') as f:
                f.writelines(lines)
            print(f"Successfully edited {filename}")
        except FileNotFoundError:
            print(f'Error: File "{filename}" does not exist.')
        except PermissionError:
            print(f'Error: Permission denied for editing "{filename}".')
        except Exception as e:
            print(f'Error: Could not edit file "{filename}". {str(e)}')

            with open(filename, 'w') as f:
                f.writelines(lines)
        except FileNotFoundError:
            print(f'Error: File "{filename}" does not exist.')
        except PermissionError:
            print(f'Error: Permission denied for editing "{filename}".')
        except Exception as e:
            print(f'Error: Could not edit file "{filename}". {str(e)}')

    def compress_files(self, file_names):
        """
        The function compresses a list of files into a .tar.gz archive.
        """

        output_file_name = file_names[-1]
        if not output_file_name.endswith('.tar.gz'):
            print('Error: The output file must have a .tar.gz extension.')
            return

        with tarfile.open(output_file_name, 'w:gz') as tar:
            for file_name in file_names[:-1]:
                file_name = self.resolve_path(file_name)
                try:
                    tar.add(file_name)
                    print(f'Successfully added {file_name} to archive')
                except FileNotFoundError:
                    print(f'Error: File "{file_name}" does not exist.')
                except IsADirectoryError:
                    print(f'Error: "{file_name}" is a directory, not a file.')
                except Exception as e:
                    print(
                        f'Error: Could not add file "{file_name}" to archive. {str(e)}')

        print(f'Successfully created archive: {output_file_name}')

    def extract_file(self, file_name):
        """
        The function extracts a .tar.gz file to the same directory as the input file.

        """
        if not file_name.endswith('.tar.gz'):
            print('Error: The input file must have a .tar.gz extension.')
            return

        file_name = self.resolve_path(file_name)
        try:
            with tarfile.open(file_name, 'r:gz') as tar:
                tar.extractall(path=os.path.dirname(file_name))
            print(f'Successfully extracted {file_name}')
        except FileNotFoundError:
            print(f'Error: File "{file_name}" does not exist.')
        except Exception as e:
            print(f'Error: Could not extract file "{file_name}". {str(e)}')

    @staticmethod
    def format_size(size):
        """
        The function "format_size" takes a size in bytes and returns a human-readable format with the
        appropriate unit (B, KB, MB, GB, or TB).
        """

        # formats the size in a human readable format
        units = ['B', 'KB', 'MB', 'GB', 'TB']
        unit = 0
        while size >= 1024:
            size /= 1024
            unit += 1
        return f'{size:.2f} {units[unit]}'

    def search(self, pattern, path="~"):
        """
        The `search` function searches for files matching a given pattern in a specified directory and
        its subdirectories.
        """

        path = os.path.expanduser('~/Desktop')

        if not os.path.isdir(path):
            print(f'Error: "{path}" is not a directory.')
            return

        matches = []
        for root, dirnames, filenames in os.walk(path, followlinks=True):
            # print(f'Looking in: {root}')  # Debugging line
            for filename in fnmatch.filter(filenames, '*' + pattern + '*'):
                matches.append(os.path.join(root, filename))

        if matches:
            print(f'Found {len(matches)} matches:')
            for match in matches:
                print(match)
            return matches
        else:
            print(f'No files found matching "{pattern}".')

    def disk_usage(self, path='.'):
        """
        The `disk_usage` function calculates the size of a file or directory and returns the size or
        total size respectively.
        """

        path = self.resolve_path(path)
        if os.path.isfile(path):
            size = os.path.getsize(path)
            return size  # Make sure to return the size
        elif os.path.isdir(path):
            total_size = 0
            for dirpath, dirnames, filenames in os.walk(path):
                for f in filenames:
                    fp = os.path.join(dirpath, f)
                    total_size += os.path.getsize(fp)
            print(
                f'Total size of directory {path}: {self.format_size(total_size)}')

            return total_size  # Make sure to return the total size
        else:
            print(f'Error: "{path}" is neither a file nor a directory.')


import logging
from functools import partial, reduce

import numpy as np
import pandas as pd
import pypsa
import scipy as sp
from _helpers import configure_logging, update_p_nom_max
from add_electricity import load_costs
from cluster_network import cluster_regions, clustering_for_n_clusters
from pypsa.clustering.spatial import (
    aggregateoneport,
    busmap_by_stubs,
    get_clustering_from_busmap,
)
from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
from scipy.sparse.csgraph import connected_components, dijkstra

logger = logging.getLogger(__name__)

def simplify_network_to_380(n):
    """
    Fix all lines to a voltage level of 380 kV and remove all transformers.

    The function preserves the transmission capacity for each line while
    updating its voltage level, line type and number of parallel bundles
    (num_parallel).

    Transformers are removed and connected components are moved from
    their starting bus to their ending bus. The corresponding starting
    buses are removed as well.
    """
    logger.info("Mapping all network lines onto a single 380kV layer")

    n.buses["v_nom"] = 380.0

    (linetype_380,) = n.lines.loc[n.lines.v_nom == 380.0, "type"].unique()
    n.lines["type"] = linetype_380
    n.lines["v_nom"] = 380
    n.lines["i_nom"] = n.line_types.i_nom[linetype_380]
    n.lines["num_parallel"] = n.lines.eval("s_nom / (sqrt(3) * v_nom * i_nom)")

    trafo_map = pd.Series(n.transformers.bus1.values, n.transformers.bus0.values)
    trafo_map = trafo_map[~trafo_map.index.duplicated(keep="first")]
    several_trafo_b = trafo_map.isin(trafo_map.index)
    trafo_map[several_trafo_b] = trafo_map[several_trafo_b].map(trafo_map)
    missing_buses_i = n.buses.index.difference(trafo_map.index)
    missing = pd.Series(missing_buses_i, missing_buses_i)
    trafo_map = pd.concat([trafo_map, missing])

    for c in n.one_port_components | n.branch_components:
        df = n.df(c)
        for col in df.columns:
            if col.startswith("bus"):
                df[col] = df[col].map(trafo_map)

    n.mremove("Transformer", n.transformers.index)
    n.mremove("Bus", n.buses.index.difference(trafo_map))

    return n, trafo_map

def _prepare_connection_costs_per_link(n, costs, renewable_carriers, length_factor):
    if n.links.empty:
        return {}

    return {
        tech: (
            n.links.length
            * length_factor
            * (
                n.links.underwater_fraction
                * costs.at[tech + "-connection-submarine", "capital_cost"]
                + (1.0 - n.links.underwater_fraction)
                * costs.at[tech + "-connection-underground", "capital_cost"]
            )
        )
        for tech in renewable_carriers
        if tech.startswith("offwind")
    }

def _compute_connection_costs_to_bus(
    n,
    busmap,
    costs,
    renewable_carriers,
    length_factor,
    connection_costs_per_link=None,
    buses=None,
):
    if connection_costs_per_link is None:
        connection_costs_per_link = _prepare_connection_costs_per_link(
            n, costs, renewable_carriers, length_factor
        )

    if buses is None:
        buses = busmap.index[busmap.index != busmap.values]

    connection_costs_to_bus = pd.DataFrame(index=buses)

    for tech in connection_costs_per_link:
        adj = n.adjacency_matrix(
            weights=pd.concat(
                dict(
                    Link=connection_costs_per_link[tech].reindex(n.links.index),
                    Line=pd.Series(0.0, n.lines.index),
                )
            )
        )

        costs_between_buses = dijkstra(
            adj, directed=False, indices=n.buses.index.get_indexer(buses)
        )
        connection_costs_to_bus[tech] = costs_between_buses[
            np.arange(len(buses)), n.buses.index.get_indexer(busmap.loc[buses])
        ]

    return connection_costs_to_bus

def _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output):
    connection_costs = {}
    for tech in connection_costs_to_bus:
        tech_b = n.generators.carrier == tech
        costs = (
            n.generators.loc[tech_b, "bus"]
            .map(connection_costs_to_bus[tech])
            .loc[lambda s: s > 0]
        )
        if not costs.empty:
            n.generators.loc[costs.index, "capital_cost"] += costs
            logger.info(
                "Displacing {} generator(s) and adding connection costs to capital_costs: {} ".format(
                    tech,
                    ", ".join(
                        "{:.0f} Eur/MW/a for `{}`".format(d, b)
                        for b, d in costs.items()
                    ),
                )
            )
            connection_costs[tech] = costs
    pd.DataFrame(connection_costs).to_csv(output.connection_costs)

def _aggregate_and_move_components(
    n,
    busmap,
    connection_costs_to_bus,
    output,
    aggregate_one_ports={"Load", "StorageUnit"},
    aggregation_strategies=dict(),
    exclude_carriers=None,
):
    def replace_components(n, c, df, pnl):
        n.mremove(c, n.df(c).index)

        import_components_from_dataframe(n, df, c)
        for attr, df in pnl.items():
            if not df.empty:
                import_series_from_dataframe(n, df, c, attr)

    _adjust_capital_costs_using_connection_costs(n, connection_costs_to_bus, output)

    generator_strategies = aggregation_strategies["generators"]

    carriers = set(n.generators.carrier) - set(exclude_carriers)
    generators, generators_pnl = aggregateoneport(
        n,
        busmap,
        "Generator",
        carriers=carriers,
        custom_strategies=generator_strategies,
    )

    replace_components(n, "Generator", generators, generators_pnl)

    for one_port in aggregate_one_ports:
        df, pnl = aggregateoneport(n, busmap, component=one_port)
        replace_components(n, one_port, df, pnl)

    buses_to_del = n.buses.index.difference(busmap)
    n.mremove("Bus", buses_to_del)
    for c in n.branch_components:
        df = n.df(c)
        n.mremove(c, df.index[df.bus0.isin(buses_to_del) | df.bus1.isin(buses_to_del)])

def simplify_links(
    n,
    costs,
    renewables,
    length_factor,
    p_max_pu,
    exclude_carriers,
    output,
    aggregation_strategies=dict(),
):
    ## Complex multi-node links are folded into end-points
    logger.info("Simplifying connected link components")

    if n.links.empty:
        return n, n.buses.index.to_series()

    # Determine connected link components, ignore all links but DC
    adjacency_matrix = n.adjacency_matrix(
        branch_components=["Link"],
        weights=dict(Link=(n.links.carrier == "DC").astype(float)),
    )

    _, labels = connected_components(adjacency_matrix, directed=False)
    labels = pd.Series(labels, n.buses.index)

    G = n.graph()

    def split_links(nodes):
        nodes = frozenset(nodes)

        seen = set()
        supernodes = {m for m in nodes if len(G.adj[m]) > 2 or (set(G.adj[m]) - nodes)}

        for u in supernodes:
            for m, ls in G.adj[u].items():
                if m not in nodes or m in seen:
                    continue

                buses = [u, m]
                links = [list(ls)]  # [name for name in ls]]

                while m not in (supernodes | seen):
                    seen.add(m)
                    for m2, ls in G.adj[m].items():
                        if m2 in seen or m2 == u:
                            continue
                        buses.append(m2)
                        links.append(list(ls))  # [name for name in ls])
                        break
                    else:
                        # stub
                        break
                    m = m2
                if m != u:
                    yield pd.Index((u, m)), buses, links
            seen.add(u)

    busmap = n.buses.index.to_series()

    connection_costs_per_link = _prepare_connection_costs_per_link(
        n, costs, renewables, length_factor
    )
    connection_costs_to_bus = pd.DataFrame(
        0.0, index=n.buses.index, columns=list(connection_costs_per_link)
    )

    for lbl in labels.value_counts().loc[lambda s: s > 2].index:
        for b, buses, links in split_links(labels.index[labels == lbl]):
            if len(buses) <= 2:
                continue

            logger.debug("nodes = {}".format(labels.index[labels == lbl]))
            logger.debug("b = {}\nbuses = {}\nlinks = {}".format(b, buses, links))

            m = sp.spatial.distance_matrix(
                n.buses.loc[b, ["x", "y"]], n.buses.loc[buses[1:-1], ["x", "y"]]
            )
            busmap.loc[buses] = b[np.r_[0, m.argmin(axis=0), 1]]
            connection_costs_to_bus.loc[buses] += _compute_connection_costs_to_bus(
                n,
                busmap,
                costs,
                renewables,
                length_factor,
                connection_costs_per_link,
                buses,
            )

            all_links = [i for _, i in sum(links, [])]

            lengths = n.links.loc[all_links, "length"]
            name = lengths.idxmax() + "+{}".format(len(links) - 1)
            params = dict(
                carrier="DC",
                bus0=b[0],
                bus1=b[1],
                length=sum(
                    n.links.loc[[i for _, i in l], "length"].mean() for l in links
                ),
                p_nom=min(n.links.loc[[i for _, i in l], "p_nom"].sum() for l in links),
                underwater_fraction=sum(
                    lengths
                    / lengths.sum()
                    * n.links.loc[all_links, "underwater_fraction"]
                ),
                p_max_pu=p_max_pu,
                p_min_pu=-p_max_pu,
                underground=False,
                under_construction=False,
            )

            logger.info(
                "Joining the links {} connecting the buses {} to simple link {}".format(
                    ", ".join(all_links), ", ".join(buses), name
                )
            )

            n.mremove("Link", all_links)

            static_attrs = n.components["Link"]["attrs"].loc[lambda df: df.static]
            for attr, default in static_attrs.default.items():
                params.setdefault(attr, default)
            n.links.loc[name] = pd.Series(params)

            # n.add("Link", **params)

    logger.debug("Collecting all components using the busmap")

    _aggregate_and_move_components(
        n,
        busmap,
        connection_costs_to_bus,
        output,
        aggregation_strategies=aggregation_strategies,
        exclude_carriers=exclude_carriers,
    )
    return n, busmap

def remove_stubs(
    n,
    costs,
    renewable_carriers,
    length_factor,
    simplify_network,
    output,
    aggregation_strategies=dict(),
):
    logger.info("Removing stubs")

    across_borders = simplify_network["remove_stubs_across_borders"]
    matching_attrs = [] if across_borders else ["country"]
    busmap = busmap_by_stubs(n, matching_attrs)

    connection_costs_to_bus = _compute_connection_costs_to_bus(
        n, busmap, costs, renewable_carriers, length_factor
    )

    _aggregate_and_move_components(
        n,
        busmap,
        connection_costs_to_bus,
        output,
        aggregation_strategies=aggregation_strategies,
        exclude_carriers=simplify_network["exclude_carriers"],
    )

    return n, busmap

def aggregate_to_substations(n, aggregation_strategies=dict(), buses_i=None):
    # can be used to aggregate a selection of buses to electrically closest neighbors
    # if no buses are given, nodes that are no substations or without offshore connection are aggregated

    if buses_i is None:
        logger.info(
            "Aggregating buses that are no substations or have no valid offshore connection"
        )
        buses_i = list(set(n.buses.index) - set(n.generators.bus) - set(n.loads.bus))

    weight = pd.concat(
        {
            "Line": n.lines.length / n.lines.s_nom.clip(1e-3),
            "Link": n.links.length / n.links.p_nom.clip(1e-3),
        }
    )

    adj = n.adjacency_matrix(branch_components=["Line", "Link"], weights=weight)

    bus_indexer = n.buses.index.get_indexer(buses_i)
    dist = pd.DataFrame(
        dijkstra(adj, directed=False, indices=bus_indexer), buses_i, n.buses.index
    )

    dist[
        buses_i
    ] = np.inf  # bus in buses_i should not be assigned to different bus in buses_i

    for c in n.buses.country.unique():
        incountry_b = n.buses.country == c
        dist.loc[incountry_b, ~incountry_b] = np.inf

    busmap = n.buses.index.to_series()
    busmap.loc[buses_i] = dist.idxmin(1)

    line_strategies = aggregation_strategies.get("lines", dict())
    generator_strategies = aggregation_strategies.get("generators", dict())
    one_port_strategies = aggregation_strategies.get("one_ports", dict())

    clustering = get_clustering_from_busmap(
        n,
        busmap,
        aggregate_generators_weighted=True,
        aggregate_generators_carriers=None,
        aggregate_one_ports=["Load", "StorageUnit"],
        line_length_factor=1.0,
        line_strategies=line_strategies,
        generator_strategies=generator_strategies,
        one_port_strategies=one_port_strategies,
        scale_link_capital_costs=False,
    )
    return clustering.network, busmap

def cluster(
    n,
    n_clusters,
    focus_weights,
    solver_name,
    algorithm="hac",
    feature=None,
    aggregation_strategies=dict(),
):
    logger.info(f"Clustering to {n_clusters} buses")

    clustering = clustering_for_n_clusters(
        n,
        n_clusters,
        custom_busmap=False,
        aggregation_strategies=aggregation_strategies,
        solver_name=solver_name,
        algorithm=algorithm,
        feature=feature,
        focus_weights=focus_weights,
    )

    return clustering.network, clustering.busmap

if __name__ == "__main__":
    if "snakemake" not in globals():
        from _helpers import mock_snakemake

        snakemake = mock_snakemake("simplify_network", simpl="")
    configure_logging(snakemake)

    params = snakemake.params
    solver_name = snakemake.config["solving"]["solver"]["name"]

    n = pypsa.Network(snakemake.input.network)
    Nyears = n.snapshot_weightings.objective.sum() / 8760

    n, trafo_map = simplify_network_to_380(n)

    technology_costs = load_costs(
        snakemake.input.tech_costs,
        params.costs,
        params.max_hours,
        Nyears,
    )

    n, simplify_links_map = simplify_links(
        n,
        technology_costs,
        params.renewable_carriers,
        params.length_factor,
        params.p_max_pu,
        params.simplify_network["exclude_carriers"],
        snakemake.output,
        params.aggregation_strategies,
    )

    busmaps = [trafo_map, simplify_links_map]

    if params.simplify_network["remove_stubs"]:
        n, stub_map = remove_stubs(
            n,
            technology_costs,
            params.renewable_carriers,
            params.length_factor,
            params.simplify_network,
            snakemake.output,
            aggregation_strategies=params.aggregation_strategies,
        )
        busmaps.append(stub_map)

    if params.simplify_network["to_substations"]:
        n, substation_map = aggregate_to_substations(n, params.aggregation_strategies)
        busmaps.append(substation_map)

    # treatment of outliers (nodes without a profile for considered carrier):
    # all nodes that have no profile of the given carrier are being aggregated to closest neighbor
    if params.simplify_network["algorithm"] == "hac":
        carriers = params.simplify_network["feature"].split("-")[0].split("+")
        for carrier in carriers:
            buses_i = list(
                set(n.buses.index) - set(n.generators.query("carrier == @carrier").bus)
            )
            logger.info(
                f"clustering preparation (hac): aggregating {len(buses_i)} buses of type {carrier}."
            )
            n, busmap_hac = aggregate_to_substations(
                n, params.aggregation_strategies, buses_i
            )
            busmaps.append(busmap_hac)

    if snakemake.wildcards.simpl:
        n, cluster_map = cluster(
            n,
            int(snakemake.wildcards.simpl),
            params.focus_weights,
            solver_name,
            params.simplify_network["algorithm"],
            params.simplify_network["feature"],
            params.aggregation_strategies,
        )
        busmaps.append(cluster_map)

    # some entries in n.buses are not updated in previous functions, therefore can be wrong. as they are not needed
    # and are lost when clustering (for example with the simpl wildcard), we remove them for consistency:
    remove = [
        "symbol",
        "tags",
        "under_construction",
        "substation_lv",
        "substation_off",
        "geometry",
        "underground",
    ]
    n.buses.drop(remove, axis=1, inplace=True, errors="ignore")
    n.lines.drop(remove, axis=1, errors="ignore", inplace=True)

    update_p_nom_max(n)

    n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
    n.export_to_netcdf(snakemake.output.network)

    busmap_s = reduce(lambda x, y: x.map(y), busmaps[1:], busmaps[0])
    busmap_s.to_csv(snakemake.output.busmap)

    cluster_regions(busmaps, snakemake.input, snakemake.output)


from collections import namedtuple
import config
import dataset
import graph
import random
import tensorflow as tf

RunInfo = namedtuple("RunInfo", [
    "dataset",
    "graph",
    "session",
])

BatchInfo = namedtuple("BatchInfo", [
    "epoch_idx",
    "batch_idx",
])

def run_batch(run_info, batch_info):
    d, g = run_info.dataset, run_info.graph

    examples = random.sample(d.training_idx_examples, config.BATCH_SIZE)
    _, loss = run_info.session.run(
        [g.training_op, g.training_loss],
        feed_dict = {
            g.input_sequence: [e[0] for e in examples],
            g.output_sequence: [e[1] for e in examples],
            g.learning_rate: config.LEARNING_RATE
        }
    )

    print(f"Epoch {batch_info.epoch_idx:02d} | "
          f"Batch {batch_info.batch_idx:02d} | "
          f"Loss: {loss:.3f}")

def run_validation(run_info, epoch_idx):
    print("Beginning validation")
    d, g = run_info.dataset, run_info.graph

    examples = random.sample(
        d.validation_idx_examples, config.BATCH_SIZE
    )
    accuracy = run_info.session.run(
        g.accuracy,
        feed_dict = {
            g.input_sequence: [e[0] for e in examples],
            g.output_sequence: [e[1] for e in examples],
        }
    )

    print(f"Epoch {epoch_idx:02d} | "
          f"Accuracy: {accuracy:.3f}")

def run(session):
    d = dataset.dataset(
        max_length = config.EXAMPLE_MAX_LEN,
        num_examples = config.NUM_EXAMPLES
    )

    g = graph.graph(
        batch_size = config.BATCH_SIZE,
        sequence_length = config.EXAMPLE_MAX_LEN,
        vocab_size = len(d.word_to_idx),
        num_embedding_dimensions = config.NUM_EMBEDDING_DIMENSIONS,
        num_lstm_layers = config.NUM_LSTM_LAYERS,
        num_lstm_units = config.NUM_LSTM_UNITS,
        start_word_idx = d.word_to_idx[dataset.start_word],
        stop_word_idx = d.word_to_idx[dataset.stop_word],
    )

    session.run(tf.global_variables_initializer())

    file_writer = tf.summary.FileWriter('logs/', session.graph)

    run_info = RunInfo(
        dataset = d,
        graph = g,
        session = session
    )
    for epoch_idx in range(1, config.NUM_EPOCHS):
        num_batches = len(d.training_idx_examples) // config.BATCH_SIZE
        for batch_idx in range(1, num_batches):
            batch_info = BatchInfo(
                epoch_idx = epoch_idx,
                batch_idx = batch_idx,
            )
            run_batch(run_info, batch_info)
        run_validation(run_info, epoch_idx)

with tf.Session() as session:
    run(session)