from flask import Flask
from waitress import serve
from process_tilt_ciphersmaug.logging import TiltLogger
import requests
from opentelemetry import trace
#from opentelemetry.instrumentation.requests import RequestsInstrumentor
import sqlite3

def initialize_db():
    db = sqlite3.connect("user.sqlite")
    cursor = db.cursor()
    with open("user.sql",mode="r") as f:
        cursor.executescript(f.read())
    db.close()

tracer = trace.get_tracer(__name__)
app = Flask(__name__)
initialize_db()
#RequestsInstrumentor().instrument()
# Create the tilt Logger
tl = TiltLogger("TILT",tracer)

@app.route('/user/<id>')
def user_information(id: int):
    address = request_address(id)
    user_information = get_user_information(id)
    return user_information | address

@tl.log(concept_name = "Combine User Data", tilt = {
    "data_disclosed": ["user.firstname","user.lastname","user.birthday","address.street","address.postcode","address.number"],
    "purposes": ["personal data access"],
    "legal_bases": ["GDPR-6-1-b"]
    }, msg = "Combine User Data")
def get_user_information(id: int):

    if int(id) > 1000:
        return {}
    else:
        db = sqlite3.connect("user.sqlite")
        result = {}
        for row in db.execute(f"SELECT * from user WHERE id == {id}"):
            id,lastname,firstname,email,sex,ip = row
            result["firstname"]=firstname
            result["lastname"]=lastname
            result["birthday"]=ip
            result["description"]=email
            break
        db.close()
        return result

@tl.log(concept_name = "Request User Address", tilt = {
    "data_disclosed": ["user.id"],
    "purposes": ["personal data access"],
    "legal_bases": ["GDPR-6-1-b"]
    }, msg = "Request User Address")
def request_address(id: int):
    with tracer.start_as_current_span("Request User Address"):
        response = requests.get(f"http://address-service/address/{id}")
        return response.json()

if __name__ == "__main__":
    serve(app, host="0.0.0.0", port=8082)


import datetime

nowdatetime = datetime.datetime.now()
print(nowdatetime)

class Calculator:
    def __init__(self, num1, num2):
        self.num1 = num1
        self.num2 = num2

    def add(self):
        return self.num1+self.num2

    def substract(self):
        return self.num1-self.num2

    def multiply(self):
        return self.num1*self.num2

    def divide(self):
        return self.num1/self.num2

def inputsprinter(cal):
    for x in cal:
        print(x.num1, " ", x.num2)

f = open("input.txt", "a+")
f.write("=> {} \n".format(nowdatetime))

input_list = []
x = 0
while x < 2:
    print(" ")
    print("************************** CT CALCULATOR ******************************** ")
    print(" ")
    print("{} inputs \n".format(x))
    num1 = int(input("Enter the First Number : "))
    num2 = int(input("Enter the Second Number : "))

    cal = Calculator(num1, num2)
    f.write("{} {} \n".format(num1, num2))
    input_list.append(cal)
    total = cal.add()

    balance = cal.substract()
    multiply = cal.multiply()
    division = cal.divide()
    f.write(("Total : {},  Balance :  {}, Multipication is : {}, Division is : , {} \n").format(
        total, balance, multiply, division))
    x = x+1

print(" ")
f.write("{} inputs \n".format(x))

print(" ")
print("************************** THE END ******************************** ")

print(" ")
inputsprinter(input_list)
f.write(" \n")
f.close()


"""Pipeline compiler for bundling modern JavaScript with rollup.js.

Version Added:
    4.0
"""

import os
import re
from pathlib import Path
from typing import List, Optional, Tuple

from pipeline.conf import settings
from pipeline.compilers import SubProcessCompiler

from djblets.pipeline.compilers.mixins import SourceMapStaleCheckMixin

#: The default regular expression used for rollup.js index files.
#:
#: Version Added:
#:     4.0
DEFAULT_FILE_PATTERN = r'index\.(js|es6|es6\.js|ts)$'

_match_regex = re.compile(settings.get('ROLLUP_FILE_PATTERN') or
                          DEFAULT_FILE_PATTERN)

class RollupCompiler(SourceMapStaleCheckMixin, SubProcessCompiler):
    """A Pipeline compiler for interfacing with rollup.js.

    `rollup.js <https://rollupjs.org/>`_ is a module bundler for JavaScript
    that compiles down a JavaScript codebase, linked together through module
    imports (ES6, CommonJS etc.), into a single JavaScript file. It can manage
    compilation with Babel or TypeScript, and supports a wide variety of
    plugins.

    This compiler makes it easy to develop modern JavaScript while still
    taking advantage of the best of Pipeline, such as managing separate
    bundles, loading bundles into Django templates, and triggering automatic
    recompilation on demand if any files (including those imported via modules)
    in the bundle are stale.

    To use this compiler, you will need an ``index`` file in your Pipeline
    JavaScript bundle, which imports any other modules that should be part
    of that compiled rollup.js bundle. An ``index`` file can be in the
    following forms:

    * :file:`index.js`
    * :file:`index.es6`
    * :file:`index.es6.js`
    * :file:`index.ts`

    Custom filename patterns can be specified by setting
    ``settings.PIPELINE['ROLLUP_FILE_PATTERN']`` to a regular expression string
    (defaults to :py:data:`DEFAULT_FILE_PATTERN`).

    Your Pipeline bundle may consist of just this one ``index`` file, or it
    may include other JavaScript as well. Each ``index`` will be compiled as
    its own rollup.js bundle, and its compiled contents included as part of the
    Pipeline bundle.

    Sourcemaps are used to check if any parts of the bundle are stale,
    triggering automatic recompilation when loading pages. Please note that
    any modules that you want included but are not actively being used within
    that rollup.js bundle may be excluded from automatic recompilation
    detection if you have treeshaking enabled (which is enabled by default).

    You will need to set ``settings.PIPELINE['ROLLUP_BINARY']`` to the path
    of :file:`rollup` and then set any command line arguments needed
    (such as a path to your configuration file) in
    ``settings.PIPELINE['ROLLUP_ARGUMENTS']``.

    The arguments should *not* include ``-c`` / ``--config`` to specify the
    ``rollup.config.js`` path. This will be computed automatically, in order
    to ensure the right file is used based on whichever source tree may be
    hosting the input file (such as when a project is consuming another
    project's source files and compiling them).

    Version Added:
        4.0
    """

    output_extension = 'js'

    def match_file(
        self,
        path: str,
    ) -> bool:
        """Return whether this compiler matches a source file.

        This will look for the following filenames by default:

        * :file:`index.js`
        * :file:`index.es6`
        * :file:`index.es6.js`
        * :file:`index.ts`

        To customize this, set ``settings.PIPELINE['ROLLUP_FILE_PATTERN']``
        to a regular expression string.

        Args:
            path (str):
                The path to the source file being considered.

        Returns:
            bool:
            ``True`` if this compiler will invoke rollup.js on this file.
            ``False`` if it will not.
        """
        return _match_regex.match(os.path.basename(path)) is not None

    def compile_file(
        self,
        infile: str,
        outfile: str,
        outdated: bool = False,
        force: bool = False,
    ) -> None:
        """Compile a file using rollup.js.

        This will cause the file and anything it imports in the tree into
        a rollup.js bundle, for inclusion in the Pipeline bundle.

        Args:
            infile (str):
                The source file.

            outfile (str):
                The destination file.

            outdated (bool, optional):
                Whether the destination file is known to be outdated.

            force (bool, optional):
                Whether to force re-running rollup.js on this file.
        """
        if outdated or force:
            # Look for the root of a tree, containing any of a number of
            # build configuration files.
            tree_root, rollup_config_path = \
                self._find_tree_root(os.path.dirname(infile))

            args: List[str] = [
                settings.ROLLUP_BINARY,
            ]

            if rollup_config_path:
                # We found a rollup.config.js. Make sure we tell rollup.js to
                # use it explicitly.
                args += [
                    '--bundleConfigAsCjs',
                    '-c',
                    rollup_config_path,
                ]

            args += [
                settings.ROLLUP_ARGUMENTS,
                '--sourcemap',
                '-i',
                infile,
                '-o',
                outfile,
            ]

            self.execute_command(args, cwd=tree_root)

    def _find_tree_root(
        self,
        start_dir: str,
    ) -> Tuple[Optional[str], Optional[str]]:
        """Return the root of a source tree for an input file.

        This will scan up the tree, looking for a :file:`rollup.config.js` or
        :file:`.babelrc`. This is used to try to find the proper working
        directory needed to successfully compile a file.

        This is important when a project is consuming and building another
        project's static media, so that the consumed project's configuration
        is applied.

        Args:
            start_dir (str):
                The starting directory for the search.

        Returns:
            str:
            The root of the source tree, or ``None`` if it could not be found.
        """
        path = Path(start_dir)
        root = Path(path.root)

        while path != root:
            rollup_config_path = Path(path / 'rollup.config.js')

            if rollup_config_path.exists():
                # This is the ideal result. We found the top-level of the
                # tree and the rollup.config.js file.
                return str(path), str(rollup_config_path)
            elif Path(path / '.babelrc').exists():
                # We didn't find rollup.config.js, but we found .babelrc.
                # Consider this the top of the tree.
                return str(path), None

            path = path.parent

        return None, None


"""

"""
import os
import re
import json
import urllib
import logging
import datetime
import sqlite3
import requests
import collections
from queue import Queue
from bs4 import BeautifulSoup

class Crawler:
    """
    A web crawler that fetches and stores webpages from a given base URL.

    Attributes:
        base_url (str): The base URL from which the crawler starts.
        data_path (str): Path to the directory where data will be stored.
        logs_path (str): Path to the directory where logs will be stored.
        errors_path (str): Path to the directory where error logs will be stored.
        database (str): Name of the SQLite database to use for storing webpages.
    """

    def __init__(self, **kwargs):
        """
        Initializes the crawler with necessary parameters and establishes a database connection.

        Args:
            **kwargs: Keyword arguments for configuration parameters.
        """
        logging.basicConfig(
            level=logging.INFO,
            filename=os.path.join(kwargs.get("logs_path"), "crawler.log"),
            filemode="a",
            format="%(asctime)s - %(levelname)s - %(message)s"
        )
        self.logger = logging.getLogger(__name__)

        self.base_url: str = kwargs.get("base_url")
        self.data_path: str = kwargs.get("data_path", ".")
        self.logs_path: str = kwargs.get("logs_path", ".")
        self.errors_path: str = kwargs.get("errors_path", ".")
        self.database: str = kwargs.get("database", "webpages.db")

        self.errors = collections.defaultdict(lambda: [])
        self.session = requests.Session()
        self.session.headers.update({"Accept-Language": "pt-br,pt-BR"})

        # Establish connection and create table webpages
        self.conn = sqlite3.connect(self.database)
        self.cursor = self.conn.cursor()
        self.cursor.execute("""
            CREATE TABLE IF NOT EXISTS webpages (
                url TEXT PRIMARY KEY,
                webpage BLOB,
                metadata BOOLEAN NOT NULL DEFAULT 0,
                document BOOLEAN NOT NULL DEFAULT 0,
                parsed BOOLEAN NOT NULL DEFAULT 0
            );
        """)
        self.conn.commit()

    def execute(self):
        """
        Executes the web crawling operation, starting from the base URL and traversing its children URLs.
        """
        self.logger.info("started execution")

        queue = Queue()

        visited = set()
        # If database is empty, add base_url to queue, otherwise add all already visited urls
        if self.cursor.execute("SELECT url FROM webpages LIMIT 1").fetchone() is None:
            success = self.database_insert(self.base_url)
            if not success:
                return self.stop()
            queue.put(self.base_url)
            visited.add(self.base_url)
        else:
            res = self.cursor.execute("SELECT url FROM webpages")
            visited_urls = [[_] for _ in res.fetchall()]
            for url in visited_urls:
                queue.put(url)
                visited.add(url)

        # Traverse the website as BFS
        while not queue.empty():
            current: str = queue.get()
            [webpage] = self.database_select_instance(column="url", value=current)
            soup = BeautifulSoup(webpage, "html.parser")

            # Obtain children urls of current webpage
            children = []
            a_tags = soup.find_all("a")
            for a_tag in a_tags:
                url: str = a_tag.attrs.get("href")
                if url is not None:
                    url = urllib.parse.urljoin(self.base_url, url)
                    children.append(url)

            # Traverse children urls
            for child in children:
                parsed_url = urllib.parse.urlparse(child)

                # Ignore the website in other languages
                query: str = parsed_url.query
                if "lang" in query and "lang=pt" not in query:
                    self.logger.warning(f"found lang!=pt: {child}, continuing")
                    continue

                # Ignore links that are not urls
                if parsed_url.scheme and (parsed_url.scheme != "http" or parsed_url.scheme != "https"):
                    self.logger.warning(f"found link!=url: {child}, continuing")
                    continue

                url = urllib.parse.urljoin(self.base_url, child)
                if url not in visited:
                    success = self.database_insert(url)
                    if success:
                        queue.put(url)
                        visited.add(url)

        self.stop()

    def stop(self) -> None:
        """
        Stops the web crawler, closes the database connection and logs the errors encountered during
        execution.
        """
        self.conn.close()
        with open(os.path.join(self.errors_path, "errors_crawler.json"), "w") as f:
            f.write(json.dumps(self.errors))
        self.logger.info(f"Stopping crawler. #Errors: {len(self.errors)}")

    def database_insert(self, url: str) -> bool:
        """
        Fetches the webpage at the given URL and inserts it into the database.

        Args:
            url (str): The URL of the webpage to fetch and insert into the database.

        Returns:
            bool: True if the webpage was successfully inserted into the database, False otherwise.
        """
        response = self.session.get(url)
        if not response.ok:
            self.logger.error(f"unable to obtain {url}. Code: {response.status_code}")
            self.errors["request"].append((url, response.status_code))
            return False

        webpage: bytes = response.content
        metadata: bool = False
        document: bool = False
        if re.search("tde-\d+-\d+", url) is not None:
            if re.search(".pdf$", url) is not None:
                document = True
            else:
                metadata = True
        self.cursor.execute("""
            INSERT INTO webpages VALUES(?, ?, ?, ?, ?)
        """, (url, webpage, metadata, document, 0))
        self.conn.commit()

        self.logger.info(f"successfully inserted {url} to webpages")

        return True

    def database_contains(self, url: str) -> bool:
        """
        Checks if the database contains a webpage at the given URL.
        Args:
            url (str): The URL to check in the database.
        Returns:
            bool: True if the database contains a webpage at the given URL, False otherwise.
        """
        return self.database_select_instance(column="url", value=url) is None

    def database_select_instance(self, column: str, value: str) -> tuple:
        """
        Fetches a row from the database where the given column matches the given value.
        Args:
            column (str): The column to match.
            value (str): The value to match in the column.
        Returns:
            tuple: The row from the database where the column matches the value, or None if no such
            row exists.
        """
        query = f"SELECT * FROM webpages WHERE {column} = ?"
        self.cursor.execute(query, (value, ))
        return self.cursor.fetchone()


import os
import glob
import importlib, importlib.util
import logging

default_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "default")
loaded = {}

def dummy_callback():
    pass

def on(event_name, *args, **kwargs):
    global loaded
    cb_name = 'on_%s' % event_name
    for plugin_name, plugin in loaded.items():
        if cb_name in plugin.__dict__:
            # print("calling %s %s(%s)" %(cb_name, args, kwargs))
            try:
                plugin.__dict__[cb_name](*args, **kwargs)
            except Exception as e:
                logging.error("error while running %s.%s : %s" % (plugin_name, cb_name, e))

def load_from_file(filename):
    plugin_name = os.path.basename(filename.replace(".py", ""))
    spec = importlib.util.spec_from_file_location(plugin_name, filename)
    instance = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(instance)
    return plugin_name, instance

def load_from_path(path, enabled=()):
    global loaded
    for filename in glob.glob(os.path.join(path, "*.py")):
        name, plugin = load_from_file(filename)
        if name in loaded:
            raise Exception("plugin %s already loaded from %s" % (name, plugin.__file__))
        elif name not in enabled:
            # print("plugin %s is not enabled" % name)
            pass
        else:
            loaded[name] = plugin

    return loaded

def load(config):
    enabled = [name for name, options in config['main']['plugins'].items() if 'enabled' in options and options['enabled']]
    custom_path = config['main']['custom_plugins'] if 'custom_plugins' in config['main'] else None
    # load default plugins
    loaded = load_from_path(default_path, enabled=enabled)
    # set the options
    for name, plugin in loaded.items():
        plugin.__dict__['OPTIONS'] = config['main']['plugins'][name]
    # load custom ones
    if custom_path is not None:
        loaded = load_from_path(custom_path, enabled=enabled)
        # set the options
        for name, plugin in loaded.items():
            plugin.__dict__['OPTIONS'] = config['main']['plugins'][name]

    on('loaded')


import numpy as np
import time
import scipy.sparse as sparse
from collections import defaultdict

from lib.PoissonFactorModel import PoissonFactorModel
from lib.MultiGaussianModel import MultiGaussianModel
from lib.TimeAwareMF import TimeAwareMF
from lib.metrics import precisionk, recallk, ndcgk, mapk

def read_poi_coos():
    poi_coos = {}
    poi_data = open(poi_file, 'r').readlines()
    for eachline in poi_data:
        lid, lat, lng = eachline.strip().split()
        lid, lat, lng = int(lid), float(lat), float(lng)
        poi_coos[lid] = (lat, lng)
    return poi_coos

def read_training_data():
    # load train data
    train_data = open(train_file, 'r').readlines()
    sparse_training_matrix = sparse.dok_matrix((user_num, poi_num))
    training_tuples = set()
    for eachline in train_data:
        uid, lid, freq = eachline.strip().split()
        uid, lid, freq = int(uid), int(lid), int(freq)
        sparse_training_matrix[uid, lid] = freq
        training_tuples.add((uid, lid))

    # load checkins
    # time_list_hour = open("./result/time_hour" + ".txt", 'w')
    check_in_data = open(check_in_file, 'r').readlines()
    training_tuples_with_day = defaultdict(int)
    training_tuples_with_time = defaultdict(int)
    for eachline in check_in_data:
        uid, lid, ctime = eachline.strip().split()
        uid, lid, ctime = int(uid), int(lid), float(ctime)
        if (uid, lid) in training_tuples:
            hour = time.gmtime(ctime).tm_hour
            training_tuples_with_time[(hour, uid, lid)] += 1.0
            if 8 <= hour < 18:
                # working time
                hour = 0
            elif hour >= 18 or hour < 8:
                # leisure time
                hour = 1

            training_tuples_with_day[(hour, uid, lid)] += 1.0

    # Default setting: time is partitioned to 24 hours.
    sparse_training_matrices = [sparse.dok_matrix((user_num, poi_num)) for _ in range(24)]
    for (hour, uid, lid), freq in training_tuples_with_time.items():
        sparse_training_matrices[hour][uid, lid] = 1.0 / (1.0 + 1.0 / freq)

    # Default setting: time is partitioned to WD and WE.
    sparse_training_matrix_WT = sparse.dok_matrix((user_num, poi_num))
    sparse_training_matrix_LT = sparse.dok_matrix((user_num, poi_num))

    for (hour, uid, lid), freq in training_tuples_with_day.items():
        if hour == 0:
            sparse_training_matrix_WT[uid, lid] = freq
        elif hour == 1:
            sparse_training_matrix_LT[uid, lid] = freq

    print ("Data Loader Finished!")
    return sparse_training_matrices, sparse_training_matrix, sparse_training_matrix_WT, sparse_training_matrix_LT, training_tuples

def read_ground_truth():
    ground_truth = defaultdict(set)
    truth_data = open(test_file, 'r').readlines()
    for eachline in truth_data:
        uid, lid, _ = eachline.strip().split()
        uid, lid = int(uid), int(lid)
        ground_truth[uid].add(lid)
    print("The loading of Ground Truth Finished.")
    return ground_truth

def main():
    sparse_training_matrices, sparse_training_matrix, sparse_training_matrix_WT, sparse_training_matrix_LT, training_tuples = read_training_data()
    ground_truth = read_ground_truth()
    poi_coos = read_poi_coos()

    start_time = time.time()

    PFM.train(sparse_training_matrix, max_iters=10, learning_rate=1e-4)
    # Multi-Center Weekday
    MGMWT.multi_center_discovering(sparse_training_matrix_WT, poi_coos)
    # Multi-Center Weekend
    MGMLT.multi_center_discovering(sparse_training_matrix_LT, poi_coos)

    TAMF.train(sparse_training_matrices, max_iters=30, load_sigma=False)

    elapsed_time = time.time() - start_time
    print("Done. Elapsed time:", elapsed_time, "s")

    execution_time = open("./result/execution_time" + ".txt", 'w')
    execution_time.write(str(elapsed_time))

    rec_list = open("./result/reclist_top_" + str(top_k) + ".txt", 'w')
    result_5 = open("./result/result_top_" + str(5) + ".txt", 'w')
    result_10 = open("./result/result_top_" + str(10) + ".txt", 'w')
    result_15 = open("./result/result_top_" + str(15) + ".txt", 'w')
    result_20 = open("./result/result_top_" + str(20) + ".txt", 'w')

    all_uids = list(range(user_num))
    all_lids = list(range(poi_num))
    np.random.shuffle(all_uids)

    # list for different ks
    precision_5, recall_5, nDCG_5, MAP_5 = 0, 0, 0, 0
    precision_10, recall_10, nDCG_10, MAP_10 = 0, 0, 0, 0
    precision_15, recall_15, nDCG_15, MAP_15 = 0, 0, 0, 0
    precision_20, recall_20, nDCG_20, MAP_20 = 0, 0, 0, 0

    for cnt, uid in enumerate(all_uids):
        if uid in ground_truth:
            # What is the meaning of the following structure?
            overall_scores = [PFM.predict(uid, lid) * (MGMWT.predict(uid, lid) + MGMLT.predict(uid, lid)) * TAMF.predict(uid, lid)
                              if (uid, lid) not in training_tuples else -1
                              for lid in all_lids]

            overall_scores = np.array(overall_scores)

            predicted = list(reversed(overall_scores.argsort()))[:top_k]
            actual = ground_truth[uid]

            # calculate the average of different k
            precision_5 = precisionk(actual, predicted[:5])
            recall_5 = recallk(actual, predicted[:5])
            nDCG_5 = ndcgk(actual, predicted[:5])
            MAP_5 = mapk(actual, predicted[:5], 5)

            precision_10 = precisionk(actual, predicted[:10])
            recall_10 = recallk(actual, predicted[:10])
            nDCG_10 = ndcgk(actual, predicted[:10])
            MAP_10 = mapk(actual, predicted[:10], 10)

            precision_15 = precisionk(actual, predicted[:15])
            recall_15 = recallk(actual, predicted[:15])
            nDCG_15 = ndcgk(actual, predicted[:15])
            MAP_15 = mapk(actual, predicted[:15], 15)

            precision_20 = precisionk(actual, predicted[:20])
            recall_20 = recallk(actual, predicted[:20])
            nDCG_20 = ndcgk(actual, predicted[:20])
            MAP_20 = mapk(actual, predicted[:20], 20)

            rec_list.write('\t'.join([
                str(cnt),
                str(uid),
                ','.join([str(lid) for lid in predicted])
            ]) + '\n')

            # write the different ks
            result_5.write('\t'.join([str(cnt), str(uid), str(precision_5), str(recall_5), str(nDCG_5), str(MAP_5)]) + '\n')
            result_10.write('\t'.join([str(cnt), str(uid), str(precision_10), str(recall_10), str(nDCG_10), str(MAP_10)]) + '\n')
            result_15.write('\t'.join([str(cnt), str(uid), str(precision_15), str(recall_15), str(nDCG_15), str(MAP_15)]) + '\n')
            result_20.write('\t'.join([str(cnt), str(uid), str(precision_20), str(recall_20), str(nDCG_20), str(MAP_20)]) + '\n')

    print("<< STACP is Finished >>")

if __name__ == '__main__':
    data_dir = "../gowalla_u5628/"

    size_file = data_dir + "Gowalla_data_size.txt"
    check_in_file = data_dir + "Gowalla_checkins.txt"
    train_file = data_dir + "Gowalla_train.txt"
    tune_file = data_dir + "Gowalla_tune.txt"
    test_file = data_dir + "Gowalla_test.txt"
    poi_file = data_dir + "Gowalla_poi_coos.txt"

    user_num, poi_num = open(size_file, 'r').readlines()[0].strip('\n').split()
    user_num, poi_num = int(user_num), int(poi_num)

    top_k = 100

    PFM = PoissonFactorModel(K=30, alpha=20.0, beta=0.2)
    MGMWT = MultiGaussianModel(alpha=0.2, theta=0.02, dmax=15)
    MGMLT = MultiGaussianModel(alpha=0.2, theta=0.02, dmax=15)
    TAMF = TimeAwareMF(K=100, Lambda=1.0, beta=2.0, alpha=2.0, T=24)

    main()


#!/usr/bin/python2.7

#######################################################################################################################
# slack_plugin.py
# Description: Slack operator to send message on a channel
#######################################################################################################################

from airflow.exceptions import AirflowException
from airflow.models import BaseOperator

import logging
from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin
from airflow.configuration import conf
import requests
import json

class SlackOperator(BaseOperator):
    template_fields = tuple()
    ui_color = '#ededef'

    @apply_defaults
    def __init__(self,
                 channel=None,
                 message=None,
                 *args, **kwargs):
        super(SlackOperator, self).__init__(*args, **kwargs)
        if channel is None or message is None:
            if channel is None:
                logging.error('Need to provide slack channel')
            else:
                logging.error('Need to provide message')
            raise AirflowException("Missing channel/message for Slack Operator")

        self.channel = "#" + channel
        self.message = message

    def get_slack_url(self):
        return conf.get('slack', 'url')

    def get_cluster_stamp(self):
        return "[from airflow-{cluster}] ".format(cluster=conf.get('core', 'cluster'))

    def send_slack_message(self, webhook_url, slack_data):
        response = requests.post(
            webhook_url, data=json.dumps(slack_data),
            headers={'Content-Type': 'application/json'}
        )
        if response.status_code != 200:
            raise AirflowException(
                'Request to slack returned an error %s, the response is:\n%s'
                % (response.status_code, response.text)
            )

    def execute(self, context):
        webhook_url = self.get_slack_url()
        cluster_stamp = self.get_cluster_stamp()
        message = str(cluster_stamp) + self.message
        slack_data = {'text': message, 'channel': self.channel}
        logging.info('sending slack message {message} to {channel}'.format(message=self.message, channel=self.channel))
        self.send_slack_message(webhook_url, slack_data)
        logging.info('message sent successfully')

class SlackPlugin(AirflowPlugin):
    name = "slack_plugin"
    operators = [SlackOperator]


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import sqlite3 as sq3
import pandas.io.sql as pds
import pandas as pd

# # Database connections

# In[2]:

path = 'classic_rock.db'     # Initialize path to SQLite database
con = sq3.Connection(path)   # We now have a live connection to our SQL database

# # Reading data

# In[8]:

# Write the query
query = '''
SELECT *
FROM rock_songs;
'''

# Execute the query
observations = pds.read_sql(query, con)

# In[6]:

observations.head()

# # SQL Supported query

# In[11]:

# We can also run any supported SQL query
# Write the query
query = '''
SELECT Artist, Release_Year, COUNT(*) AS num_songs, AVG(PlayCount) AS avg_plays
    FROM rock_songs
    GROUP BY Artist, Release_Year
    ORDER BY num_songs desc;
'''

# Execute the query
observations = pds.read_sql(query, con)

observations.head()

# # Common parameters

# There are a number of common paramters that can be used to read in SQL data with formatting:
#
#   1. coerce_float: Attempt to force numbers into floats
#   2. parse_dates: List of columns to parse as dates
#   3.chunksize: Number of rows to include in each chunk
#
# Let's have a look at using some of these parameters

# In[12]:

query='''
SELECT Artist, Release_Year, COUNT(*) AS num_songs, AVG(PlayCount) AS avg_plays
    FROM rock_songs
    GROUP BY Artist, Release_Year
    ORDER BY num_songs desc;
'''

# Execute the query
observations_generator = pds.read_sql(query,
                            con,
                            coerce_float=True,            # Doesn't efefct this dataset, because floats were correctly parsed
                            parse_dates=['Release_Year'], # Parse Release_Year as a date
                            chunksize=5                   # Allows for streaming results as a series of shorter tables
                           )

for index, observations in enumerate(observations_generator):
    if index < 5:
        print(f'Observations index: {index}'.format(index))
        display(observations)


import os
import statistics

from bs4 import BeautifulSoup

data_dir = './Corpus_detourage/html/'
save_path = 'BS/'
data = []
nbChar =0

# parcours des fichiers & lecture
for file in os.listdir(data_dir):

    nbline = 0
    nbChar += 1

    f = open(data_dir + file, "r", encoding="utf-8", errors='ignore')

    soup = BeautifulSoup(f, 'html.parser')
    paragraphs= soup.get_text().split('\n')

    # création d'un fichier en écriture
    nameBS = os.path.join(save_path, file)
    cf = open(nameBS, "w+")

    # écriture de chaque paragraphes dans le fichier
    for paragraph in paragraphs:
        cf.write("<p>"+paragraph+"</p>"+"\n")
        nbline+=1
    data.append(nbline)

print(nbChar)
print("La moyenne :" + str(statistics.mean(data)))
print("l'écart-type :"+ str(statistics.stdev(data)))


from typing import Dict
from functools import lru_cache

memo: Dict[int, int] = {0: 0, 1: 1}  # casos base

# Fibonacci com recursao infinita, nao deve ser chamada!
def fib1(n: int) -> int:
    return fib1(n-1) + fib1(n-2)

# Fibonacci com recursao finita (tem caso base), mas para n ~= 50 nao deve funcionar!
def fib2(n: int) -> int:
    if n < 2:  # caso base
        return n
    return fib2(n-1) + fib2(n-2)

# Fibonacci com memoizacao
def fib3(n: int) -> int:
    if n not in memo:
        memo[n] = fib3(n-1) + fib3(n-2)  # memoizacao
    return memo[n]

# Fibonacci com memoizacao de funcao embutida
@lru_cache(maxsize=None)
def fib4(n: int) -> int:
    if n < 2:  # caso base
        return n
    return fib4(n-1) + fib4(n-2)

if __name__ == "__main__":
    # print(fib2(5))
    # print(fib2(10))
    # print(fib2(40))
    print(fib3(5))
    print(fib3(10))
    print(fib3(50))
    print(fib3(80))
    print(fib4(5))
    print(fib4(10))
    print(fib4(50))


from pyspecdata import *
import os, time
import SpinCore_pp
from pyspecdata.file_saving.hdf_save_dict_to_group import hdf_save_dict_to_group
import h5py
from datetime import datetime

fl = figlist_var()
target_directory = getDATADIR(exp_type="ODNP_NMR_comp/COSY")
raise RuntimeError(
    "This pulse program has been updated to use active.ini, but not the ppg functions.. Before running again, it should be possible to replace a lot of the code below with a call to the function provided by the 'generic' pulse program inside the ppg directory!"
)
# {{{importing acquisition parameters
config_dict = SpinCore_pp.configuration("active.ini")
nPoints = int(config_dict["acq_time_ms"] * config_dict["SW_kHz"] + 0.5)
# }}}
# NOTE: Number of segments is nEchoes * nPhaseSteps
# {{{create filename and save to config file
date = datetime.now().strftime("%y%m%d")
config_dict["type"] = "COSY"
config_dict["date"] = date
config_dict["COSY_counter"] += 1
filename = f"{config_dict['date']}_{config_dict['chemical']}_{config_dict['type']}"
# }}}
# {{{phase cycling and tx phases
tx_phases = r_[0.0, 90.0, 180.0, 270.0]
# NOTE: Number of segments is nEchoes * nPhaseSteps
phase_cycling = True
ph1 = r_[0, 1, 2, 3]
ph2 = r_[0, 2]
if phase_cycling:
    nPhaseSteps = 8
if not phase_cycling:
    nPhaseSteps = 1
# }}}
data_length = 2 * nPoints * nEchoes * nPhaseSteps
# NOTE: Number of segments is nEchoes * nPhaseSteps
# cannot have a delay of 0, so set to minimum SpinCore can take
# {{{making t1 list
min_t1 = 0.065  # us (lower limit of SpinCore)
max_t1 = 200 * 1e3
t1_step = 1.25 * 1e3
t1_list = r_[min_t1:max_t1:t1_step]
# }}}
# {{{ run ppg
for index, val in enumerate(t1_list):
    t1 = val
    for x in range(config_dict["nScans"]):
        SpinCore_pp.configureTX(
            config_dict["adcOffset"],
            config_dict["carrierFreq_MHz"],
            tx_phases,
            config_dict["amplitude"],
            nPoints,
        )
        SpinCore_pp.init_ppg()
        if phase_cycling:
            SpinCore_pp.load(
                [
                    ("marker", "start", 1),
                    ("phase_reset", 1),
                    ("delay_TTL", config_dict["deblank_us"]),
                    ("pulse_TTL", config_dict["p90_us"], "ph1", r_[0, 1, 2, 3]),
                    ("delay", t1),
                    ("delay_TTL", config_dict["deblank_us"]),
                    ("pulse_TTL", config_dict["p90_us"], "ph2", r_[0, 2]),
                    ("delay", config_dict["deadtime_us"]),
                    ("acquire", config_dict["acq_time_ms"]),
                    ("delay", config_dict["repetition_us"]),
                    ("jumpto", "start"),
                ]
            )
        if not phase_cycling:
            SpinCore_pp.load(
                [
                    ("marker", "start", config_dict["nScans"]),
                    ("phase_reset", 1),
                    ("delay_TTL", config_dict["deblank_us"]),
                    ("pulse_TTL", config_dict["p90_us"], 0),
                    ("delay", t1),
                    ("delay_TTL", config_dict["deblank_us"]),
                    ("pulse_TTL", config_dict["p90_us"], 0),
                    ("delay", config_dict["deadtime_us"]),
                    ("acquire", config_dict["acq_time"]),
                    ("delay", config_dict["repetition_us"]),
                    ("jumpto", "start"),
                ]
            )
        SpinCore_pp.stop_ppg()
        SpinCore_pp.runBoard()
        raw_data = SpinCore_pp.getData(
            data_length, nPoints, config_dict["nEchoes"], nPhaseSteps
        )
        raw_data.astype(float)
        data = []
        data[::] = np.complex128(raw_data[0::2] + 1j * raw_data[1::2])
        dataPoints = float(np.shape(data)[0])
        time_axis = np.linspace(
            0.0,
            config_dict["nEchoes"] * nPhaseSteps * config_dict["acq_time_ms"] * 1e-3,
            dataPoints,
        )
        data = nddata(np.array(data), "t")
        data.setaxis("t", time_axis).set_units("t", "s")
        data.name("signal")
        data.set_prop("acq_params", config_dict.asdict())
        data.set_prop("t1_list", t1_list)
        if index == 0 and x == 0:
            COSY_data = ndshape(
                [len(t1_list), config_dict["nScans"], len(time_axis)],
                ["t1", "nScans", "t"],
            ).alloc(dtype=np.complex128)
            COSY_data.setaxis("t1", t1_list * 1e-6).set_units("t1", "s")
            COSY_data.setaxis("nScans", r_[0 : config_dict["nScans"]])
            COSY_data.setaxis("t", time_axis).set_units("t", "s")
        COSY_data["t1", index]["nScans", x] = data
        COSY_data.set_prop("acq_params", config_dict.asdict())
SpinCore_pp.stopBoard()
# }}}
# {{{saving data
config_dict.write()
if phase_cycling:
    COSY_data.chunk("t", ["ph2", "ph1", "t2"], [2, 4, -1])
    COSY_data.setaxis("ph1", r_[0, 1, 2, 3] / 4.0)
    COSY_data.setaxis("ph2", r_[0, 2] / 4.0)
else:
    COSY_data.rename("t", "t2")
nodename = config_dict["type"] + "_" + str(config_dict["cpmg_counter"])
filename_out = filename + "_" + ".h5"
with h5py.File(
    os.path.normpath(os.path.join(target_directory, f"{filename_out}"))
) as fp:
    if nodename in fp.keys():
        print("this nodename already exists, so I will call it temp_%d" % j)
        COSY_data.name("temp_COSY_%d" % config_dict["cpmg_counter"])
        nodename = "temp_COSY_%d" % config_dict["cpmg_counter"]
        COSY_data.hdf5_write(f"{filename_out}", directory=target_directory)
    else:
        COSY_data.hdf5_write(f"{filename_out}", directory=target_directory)
print("\n*** FILE SAVED IN TARGET DIRECTORY ***\n")
print(("Name of saved data", COSY_data.name()))
print(("Shape of saved data", ndshape(COSY_data)))
# }}}
# {{{image data
fl.next("raw data")
fl.image(COSY_data.setaxis("t1", "#"))
fl.next("abs raw data")
fl.image(abs(COSY_data).setaxis("t1", "#"))
COSY_data.ft("t2", shift=True)
fl.next("FT raw data")
fl.image(COSY_data.setaxis("t1", "#"))
fl.next("FT abs raw data")
fl.image(abs(COSY_data).setaxis("t1", "#"))
fl.show()
# }}}


"""FIX message validation"""

from typing import Any, List, Mapping, Tuple

from ..meta_data import (
    ProtocolMetaData,
    FieldMetaData
)

from .errors import FieldValueError
from .common import calc_checksum, calc_body_length
from .value_encoders import encode_value

def _assert_field_value_matches(
        field: FieldMetaData,
        expected: bytes,
        received: bytes
) -> None:
    if expected != received:
        raise FieldValueError(field, expected, received)

def assert_message_valid(
        protocol: ProtocolMetaData,
        buf: bytes,
        encoded_message: List[Tuple[bytes, bytes]],
        decoded_message: Mapping[str, Any],
        sep: bytes,
        convert_sep_to_soh_for_checksum: bool
) -> None:
    """Check the message is valid

    Args:
        protocol (ProtocolMetaData): The protocol meta data
        buf (bytes): The FIX message as bytes
        encoded_message (List[Tuple[bytes, bytes]]): The encoded message.
        decoded_message (Mapping[str, Any]): The decoded message
        sep (bytes): The field separator
        convert_sep_to_soh_for_checksum (bool): If true convert the separator
            before calculating the checksum.

    Raises:
        FieldValueError: If the message is invalid.
    """
    # Check the begin string.
    begin_string_field = protocol.fields_by_name['BeginString']
    begin_string_value = encode_value(
        protocol,
        begin_string_field,
        decoded_message[begin_string_field.name]
    )
    _assert_field_value_matches(
        begin_string_field,
        protocol.begin_string,
        begin_string_value
    )

    # Check the body length.
    body_length_field = protocol.fields_by_name['BodyLength']
    body_length_value = encode_value(
        protocol,
        body_length_field,
        decoded_message[body_length_field.name]
    )
    body_length = calc_body_length(buf, encoded_message, sep)
    _assert_field_value_matches(
        body_length_field,
        body_length_value,
        encode_value(protocol, body_length_field, body_length)
    )

    # Check the checksum.
    check_sum_field = protocol.fields_by_name['CheckSum']
    check_sum_value = encode_value(
        protocol,
        check_sum_field,
        decoded_message[check_sum_field.name]
    )
    check_sum = calc_checksum(buf, sep, convert_sep_to_soh_for_checksum)
    _assert_field_value_matches(
        check_sum_field,
        check_sum,
        check_sum_value
    )


"""
## SPIRAL PRINTING

Print 2-D array (nxm) in spiral order (clockwise)

INPUT:

1 2 3            Order goes in this format   1 2 3  next 6 9 next 7 8 next 4 5.
4 5 6
7 8 9

OUTPUT:

1 2 3 6 9 8 7 4 5

Execution: python spiral_order_clockwise.py spiral_order_input.txt

"""

import os
import sys
import itertools

def matrix_transpose_and_yield_top(matrix):
    """
    :param matrix:
    :return: yield from the top matrix
    """
    while matrix:
        yield matrix[0]
        matrix = list(reversed(list(zip(*matrix[1:]))))

def spiral_printing(file_path):
    """
    :param file_path:
    :return: spiral printing from the top
    """
    matrix = []
    with open(file_path, 'r') as opened_file:
        for line in opened_file:
            matrix.append(list(map(int, line.split(' '))))
    print(list(itertools.chain(*matrix_transpose_and_yield_top(matrix))))

if __name__ == '__main__':
    if len(sys.argv) == 2:
        file_path = sys.argv[1]
        if os.path.isfile(file_path):
            spiral_printing(file_path)
        else:
            print("File path not exists.\n")


import torch
from transformers import BertTokenizer, BertForMaskedLM, BertModel, BertConfig
from torch.utils.data import Dataset, DataLoader
from torch.nn.utils.rnn import pad_sequence
import torch.distributed as dist
from sklearn.model_selection import train_test_split
import tqdm
import json
import os

device = "cuda" if torch.cuda.is_available() else "cpu"
model_name = "bert-base-chinese"
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForMaskedLM.from_pretrained(model_name)
config = BertConfig.from_pretrained(model_name)
if torch.cuda.device_count() > 1:
    dist.init_process_group(backend='nccl')
    model = torch.nn.DataParallel(model)
model = model.to(device)

with open("../Raw_data/new_chinese_poems_cws.txt", "r", encoding="utf-8") as f:
    lines = f.readlines()

poems_data = [{"text": line.strip()} for line in lines]
poems_train, poems_test = train_test_split(poems_data, test_size=0.2, random_state=42)

class PoemsDataset(Dataset):
    def __init__(self, data,limit=None):
        self.data = data[:limit] if limit is not None else data
        # self.data = data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        inputs = tokenizer.encode_plus(
            self.data[index]["text"],
            add_special_tokens=True,
            return_attention_mask=True,
            return_tensors="pt",
            max_length=128,
            truncation=True,
        )

        # Apply whole word masking
        inputs_whole_word = tokenizer.prepare_for_model(
            inputs["input_ids"].squeeze().tolist(),
            max_length=128,
            padding="max_length",
            truncation=True,
            return_attention_mask=True,
            return_tensors="pt",
            whole_word_masking=True,
        )
        input_ids = inputs_whole_word["input_ids"].squeeze()
        attention_mask = inputs_whole_word["attention_mask"].squeeze()
        masked_indices = torch.where(input_ids == tokenizer.mask_token_id)
        labels = input_ids.clone()
        labels[masked_indices] = -100  # ignore loss for masked tokens
        return {
            "input_ids": input_ids.to(device),
            "attention_mask": attention_mask.to(device),
            "labels": labels.to(device),
            "masked_indices": masked_indices[0].to(device),
        }

def collate_fn(batch):
    input_ids = pad_sequence([data["input_ids"] for data in batch], batch_first=True, padding_value=0)
    attention_mask = pad_sequence([data["attention_mask"] for data in batch], batch_first=True, padding_value=0)
    labels = pad_sequence([data["labels"] for data in batch], batch_first=True, padding_value=-100)
    masked_indices = [data["masked_indices"] for data in batch]
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
        "masked_indices": masked_indices,
    }

train_dataset = PoemsDataset(poems_train)
eval_dataset = PoemsDataset(poems_test)

train_loader = DataLoader(train_dataset, batch_size=128, collate_fn=collate_fn)
eval_loader = DataLoader(eval_dataset, batch_size=128, collate_fn=collate_fn)

# print("Number of samples in the train dataset:", len(train_loader))
# print("Number of samples in the eval dataset:", len(eval_loader))
#
# print("First 5 samples from train dataset:")
# for i in range(5):
#     sample = train_dataset[i]
#     text = tokenizer.decode(sample["input_ids"].tolist(), skip_special_tokens=True)
#     masked_token = tokenizer.decode(sample["input_ids"][sample["masked_indices"]].tolist())
#     print(f"Text: {sample}")
#     print(f"Text: {text}")
#     print(f"Masked token: {masked_token}")
#     print(f"Labels: {sample['labels'].tolist()}")
#     print("=" * 50)

mlm_loss = torch.nn.CrossEntropyLoss(ignore_index=-100)
optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)
epoch_num = 50
for epoch in range(epoch_num):
    model.train()
    total_loss = 0
    for batch in tqdm.tqdm(train_loader):
        optimizer.zero_grad()
        outputs = model(
            input_ids=batch["input_ids"],
            attention_mask=batch["attention_mask"],
            labels=batch["labels"],
        )
        loss = mlm_loss(outputs.logits.view(-1, tokenizer.vocab_size), batch["labels"].view(-1))
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
    avg_train_loss = total_loss / len(train_loader)
    print(f"Epoch {epoch + 1} - Average training loss: {avg_train_loss:.4f}")
    # evaluation loop
    model.eval()
    total_eval_loss = 0
    with torch.no_grad():
        for batch in tqdm.tqdm(eval_loader):
            outputs = model(
                input_ids=batch["input_ids"],
                attention_mask=batch["attention_mask"],
                labels=batch["labels"],
            )
            loss = mlm_loss(outputs.logits.view(-1, tokenizer.vocab_size), batch["labels"].view(-1))
            total_eval_loss += loss.item()

    avg_eval_loss = total_eval_loss / len(eval_loader)
    print(f"Epoch {epoch + 1} - Average evaluation loss: {avg_eval_loss:.4f}")

    # Save the model and optimizer
    folder_path = "../checkpoints/model_round2/"
    if (epoch + 1) % 10 == 0:
        if isinstance(model, torch.nn.DataParallel):
            model.module.save_pretrained(folder_path)
        else:
            model.save_pretrained(folder_path)
        tokenizer.save_pretrained(folder_path)

    # Save the logs
    logs = {"epoch": epoch + 1, "avg_train_loss": avg_train_loss, "avg_eval_loss": avg_eval_loss}
    logs_path = f"{folder_path}logs_{epoch + 1}.json"
    with open(logs_path, "w") as f:
        json.dump(logs, f)

# CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch wwm_pretrain_torch.py
# CUDA_VISIBLE_DEVICES=0,1,2,3 torchrun wwm_pretrain_torch.py


import torch as T
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np

class Policy(nn.Module):
    def __init__(self, lr, input_dims, h1, h2, n_actions):
        super(Policy,self).__init__()
        self.input_dims = input_dims
        self.lr = lr
        self.h1 = h1
        self.h2 = h2
        self.n_actions = n_actions
        self.linear1 = nn.Linear(*self.input_dims, self.h1)
        self.linear2 = nn.Linear(self.h1, self.h2)
        self.linear3 = nn.Linear(self.h2, self.n_actions)
        self.optimizer = optim.Adam(self.parameters(), lr=lr)

        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu:0')
        self.to(self.device)

    def forward(self,obs):
        x = T.tensor(obs,dtype=T.float).to(self.device)
        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = self.linear3(x)

        return x

class Agent(object):
    def __init__(self, lr, input_dims, gamma=0.99, n_actions=2, h1=128, h2=128):
        self.gamma = gamma
        self.reward_memory = []
        self.action_memory = []
        self.policy = Policy(lr, input_dims, h1, h2, n_actions)

    def choose_action(self, observation):
        probs = F.softmax(self.policy(observation),dim=0)
        action_probs = T.distributions.Categorical(probs)
        action = action_probs.sample()
        log_probs = T.log(probs[action])
        self.action_memory.append(log_probs)

        return action.item()

    def store_rewards(self, reward):
        self.reward_memory.append(reward)

    def improve(self):
        self.policy.optimizer.zero_grad()
        G = np.zeros_like(self.reward_memory, dtype=np.float64)
        for t in range(len(self.reward_memory)):
            g_sum = 0
            disc = 1
            for i in range(t, len(self.reward_memory)):
                g_sum += self.reward_memory[i]*disc
                disc *= self.gamma
            G[t] = g_sum
        G = (G - np.mean(G))/(np.std(G) if np.std(G) > 0 else 1)

        G = T.tensor(G, dtype=T.float).to(self.policy.device)

        loss = 0
        for g,log_prob in zip(G, self.action_memory):
            loss += -g * log_prob

        loss.backward()
        self.policy.optimizer.step()

        self.action_memory = []
        self.reward_memory = []


import csv
import matplotlib.pyplot as plt
import numpy as np
import os

# Increase the maximum field size to handle large files
csv.field_size_limit(10000000)

def calc_length(s):
    """Calculate the number of characters excluding delimiters and encapsulators."""

    # Preconditions
    assert isinstance(s, list)

    return sum([len(x) for x in s])

def calc_doc_lengths(filepath, delimiter, encapsulator):
    """Calculate the length of the documents."""

    # Preconditions
    assert os.path.exists(filepath), f"File doesn't exist: {filepath}"
    assert isinstance(delimiter, str)
    assert isinstance(encapsulator, str)

    # List of the lengths of each document
    doc_lengths = []

    with open(filepath, 'r', encoding='utf8') as fp:
        reader = csv.reader(fp, delimiter=delimiter, quotechar=encapsulator)

        num_rows_read = 0

        for row in reader:
            doc_lengths.append(calc_length(row))
            num_rows_read += 1

    print(f"Read {num_rows_read} rows from {filepath}")

    return doc_lengths

def spacing_ratio(n):
    """Calculate n-1 even ratios."""

    # Preconditions
    assert n > 1

    return [x/n for x in list(range(1, n))]

def calc_break_points(cum_doc_lengths, num_files):
    """Calculate breaks (row indicies) in the input file."""

    # Preconditions
    assert isinstance(doc_lengths, list)
    assert num_files > 1

    # Total number of characters
    total = cum_doc_lengths[-1]

    return [np.argmin(abs(cum_doc_lengths - r*total)) for r in spacing_ratio(num_files)]

class BatchedCsvWriter:
    def __init__(self, folder, prefix, delimiter, encapsulator, header):

        # Preconditions
        assert os.path.exists(folder)
        assert isinstance(prefix, str)
        assert isinstance(delimiter, str)
        assert encapsulator is None or isinstance(encapsulator, str)
        assert isinstance(header, list)

        self.folder = folder
        self.prefix = prefix
        self.delimiter = delimiter
        self.encapsulator = encapsulator
        self.header = header

        # File pointer
        self.fp = None

        # CSV writer
        self.writer = None

        # Current index of the file
        self.current_index = -1

    def start_new_file(self):
        """Start a new output file."""

        self.current_index += 1

        # Create the filepath
        filepath = os.path.join(
            self.folder, f"{self.prefix}-{self.current_index}.csv")
        print(f"Writing to file: {filepath}")

        # Open the file for writing
        self.fp = open(filepath, 'w', encoding='utf8', newline='')
        self.writer = csv.writer(self.fp)

        # Add the header to the file
        self.add_row(self.header)

    def add_row(self, row):
        """Add a row of data to the output file."""

        # Preconditions
        assert isinstance(row, list)

        # Start a new file if this is the first row
        if self.fp is None:
            self.start_new_file()

        self.writer.writerow(row)

    def close(self):
        """Close the file."""

        if self.fp is not None:
            self.fp.close()

def split(filepath, delimiter, encapsulator, output_folder, prefix, cuts):
    """Split a CSV file into smaller files."""

    # Preconditions
    assert os.path.exists(filepath)
    assert isinstance(delimiter, str)
    assert encapsulator is None or isinstance(encapsulator, str)
    assert os.path.exists(output_folder)
    assert isinstance(prefix, str)
    assert isinstance(cuts, list) and len(cuts) > 0

    with open(filepath, 'r', encoding='utf8') as fp:
        reader = csv.reader(fp, delimiter=delimiter, quotechar=encapsulator)

        # Read the header
        header = next(reader)

        # Initialise the CSV writer
        writer = BatchedCsvWriter(
            output_folder, prefix, delimiter, encapsulator, header)

        num_rows_read = 0

        for row in reader:

            num_rows_read += 1

            # Should a new file be started?
            if num_rows_read in cuts:
                writer.start_new_file()

            writer.add_row(row)

        print(f"Read {num_rows_read} data rows from {filepath}")
        writer.close()

def plot_doc_lengths(doc_lengths):
    """Plot the length of each document as a function of document index."""

    # Preconditions
    assert isinstance(doc_lengths, list)
    assert len(doc_lengths) > 0

    plt.plot(doc_lengths)
    plt.xlabel("Row index")
    plt.ylabel("Number of characters")

    plt.savefig("doc_lengths.png")
    plt.show()

def plot_break_points(cum_doc_lengths, cuts):
    """Plot the break points for the files"""

    # Preconditions
    assert isinstance(doc_lengths, list)
    assert len(doc_lengths) > 0
    assert isinstance(cuts, list)
    assert len(cuts) > 0

    plt.plot(cum_doc_lengths, '-g')

    for cut in cuts:
        plt.plot([0, len(cum_doc_lengths)], [
                 cum_doc_lengths[cut], cum_doc_lengths[cut]], '--r')

    plt.xlabel("Row index")
    plt.ylabel("Cumulative total number of characters")

    plt.savefig("break_points.png")
    plt.show()

if __name__ == '__main__':

    # Location of the input CSV file to split
    filepath = r"./data/input.csv"

    # CSV file parameters
    delimiter = ","
    encapsulator = "|"

    # Number of files to split the input file into
    num_files = 4

    # Location of the output (split) files
    output_folder = r"./data/"
    prefix = "split"

    # Calculate the length of each document (row) in the CSV file
    print(f"Calculating the lengths of each row in {filepath} ...")
    doc_lengths = calc_doc_lengths(filepath, delimiter, encapsulator)
    plot_doc_lengths(doc_lengths)

    # Cumulative sum of the document lengths
    # Using int64 to avoid overflow issues
    cum_total = np.cumsum(np.int64(doc_lengths))

    # Calculate the 'break' indices
    cuts = calc_break_points(cum_total, num_files)
    plot_break_points(cum_total, cuts)

    # Split the CSV file
    split(filepath, delimiter, encapsulator, output_folder, prefix, cuts)


import pytest
import tests.nimbleclientbase as nimosclientbase
from tests.nimbleclientbase import SKIPTEST, log_to_file as log
from nimbleclient import exceptions

'''GroupsTestCase tests the subnet object functionality '''

@pytest.fixture(scope='module')
def before_running_all_testcase(request):
    log("**** Starting Tests for Groups TestCase *****\n")

    def after_running_all_testcase():
        log("**** Completed Tests for Groups TestCase *****\n")
    request.addfinalizer(after_running_all_testcase)

@pytest.fixture(scope='function')
def setup_teardown_for_each_test(before_running_all_testcase, request):
    # setup operations before yield is called
    nimosclientbase.log_header(request.function.__name__)
    yield setup_teardown_for_each_test
    # teardown operations below
    nimosclientbase.log_footer(request.function.__name__)

@pytest.mark.skipif(SKIPTEST is True,
                    reason="skipped this test as SKIPTEST variable is true")
def test_get_groups(setup_teardown_for_each_test):
    resp = nimosclientbase.get_nimos_client().groups.list(
        detail=True, limit=2)
    assert resp is not None

@pytest.mark.skipif(SKIPTEST is True,
                    reason="skipped this test as SKIPTEST variable is true")
def test_get_group_discovered_list(setup_teardown_for_each_test):
    try:
        resp = nimosclientbase.get_nimos_client().groups.get()
        assert resp is not None
        resp = nimosclientbase.get_nimos_client(
        ).groups.get_group_discovered_list(id=resp.attrs.get("id"))
        assert resp is not None
    except exceptions.NimOSAPIError as ex:
        if "SM_array_not_found" in str(ex):
            log("Failed as expected. Array id given is invalid")
        else:
            log(f"Failed with exception message : {str(ex)}")
            raise ex

@pytest.mark.skipif(SKIPTEST is True,
                    reason="skipped this test as SKIPTEST variable is true")
def test_test_alert(setup_teardown_for_each_test):
    try:
        resp = nimosclientbase.get_nimos_client().groups.get()
        assert resp is not None
        test_resp = nimosclientbase.get_nimos_client().groups.test_alert(
            id=resp.attrs.get("id"), level="notice")
        test_resp is not None
    except exceptions.NimOSAPIError as ex:
        if "SM_array_not_found" in str(ex):
            log("Failed as expected. Array id given is invalid")
        else:
            log(f"Failed with exception message : {str(ex)}")
            raise ex

@pytest.mark.skipif(SKIPTEST is True,
                    reason="skipped this test as SKIPTEST variable is true")
def test_update_group(setup_teardown_for_each_test):
    try:
        resp = nimosclientbase.get_nimos_client().groups.get()
        assert resp is not None
        # save the orig value
        orig_name = resp.attrs.get("name")
        alert_to_email_addrs = resp.attrs.get(
            "alert_to_email_addrs")  # "abc@hpe.com"
        send_alert_to_support = resp.attrs.get("send_alert_to_support")
        isns_enabled = resp.attrs.get("isns_enabled")

        update_resp = nimosclientbase.get_nimos_client().groups.update(
            id=resp.attrs.get("id"),
            name="testname",
            alert_to_email_addrs="alok.ranjxxx@hpe.com",
            send_alert_to_support=False,
            isns_enabled=True)
        update_resp is not None
        # assert the values
        assert update_resp.attrs.get("name") == "testname"
        assert update_resp.attrs.get(
            "alert_to_email_addrs") == "alok.ranjxxx@hpe.com"
        assert update_resp.attrs.get("send_alert_to_support") is False
        assert update_resp.attrs.get("isns_enabled") is True

        # revert to original
        update_resp = nimosclientbase.get_nimos_client().groups.update(
            id=resp.attrs.get("id"),
            name=orig_name,
            alert_to_email_addrs=alert_to_email_addrs,
            send_alert_to_support=send_alert_to_support,
            isns_enabled=isns_enabled)
    except exceptions.NimOSAPIError as ex:
        if "SM_array_not_found" in str(ex):
            log("Failed as expected. Array id given is invalid")
        else:
            log(f"Failed with exception message : {str(ex)}")
            raise ex


"""
Tests for the base classes.
"""
import unittest

import pandas as pd
from sklearn.exceptions import NotFittedError

from automatminer.base import DFTransformer
from automatminer.utils.pkg import check_fitted, set_fitted

class TestTransformerGood(DFTransformer):
    """
    A test transformer and logger.

    Args:
        config_attr: Some attr to be set at initialization
    """

    def __init__(self, config_attr):
        self.config_attr = config_attr
        self.target = None
        super(TestTransformerGood, self).__init__()

    @set_fitted
    def fit(self, df, target):
        """
        Determine the target of the dataframe.

        Args:
            df (pandas.DataFrame): The dataframe to be transformed.
            target (str): The fit target

        Returns:
            TestTransformer
        """
        if target in df.columns:
            self.target = target
        else:
            raise ValueError("Target {} not in dataframe.".format(target))
        return self

    @check_fitted
    def transform(self, df, target):
        """
        Drop the target set during fitting.

        Args:
            df (pandas.DataFrame): The dataframe to be transformed.
            target (str): The transform target (not the same as fit target
                necessarily)

        Returns:
            df (pandas.DataFrame): The transformed dataframe.
        """
        df = df.drop(columns=self.target)
        return df

class TestTransformerBad(DFTransformer):
    """
    A test transformer, implemented incorrectly.
    """

    def __init__(self):
        pass

class TestBaseTransformers(unittest.TestCase):
    def setUp(self):
        self.df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})

    def test_DFTransformer(self):
        ttg = TestTransformerGood(5)
        self.assertTrue(hasattr(ttg, "config_attr"))
        self.assertTrue(ttg.config_attr, 5)
        with self.assertRaises(NotFittedError):
            ttg.transform(self.df, "b")

        ttg.fit(self.df, "a")
        self.assertTrue(ttg.config_attr, 5)

        test = ttg.transform(self.df, "b")
        self.assertTrue("b" in test.columns)
        self.assertTrue("c" in test.columns)
        self.assertTrue("a" not in test.columns)

        test = ttg.fit_transform(self.df, "c")
        self.assertTrue("c" not in test.columns)
        self.assertTrue("a" in test.columns)
        self.assertTrue("b" in test.columns)

        with self.assertRaises(TypeError):
            TestTransformerBad()

    def test_DFTransformer_BaseEstimator_behavior(self):
        ttg = TestTransformerGood(5)
        ttg_nested = TestTransformerGood(ttg)

        self.assertEqual(ttg.get_params()["config_attr"], 5)
        self.assertEqual(ttg_nested.get_params()["config_attr__config_attr"], 5)

        ttg.set_params(config_attr=6)
        self.assertEqual(ttg.get_params()["config_attr"], 6)
        self.assertEqual(ttg_nested.get_params()["config_attr__config_attr"], 6)

        ttg_nested.set_params(config_attr__config_attr=7)
        self.assertEqual(ttg.get_params()["config_attr"], 7)
        self.assertEqual(ttg_nested.get_params()["config_attr__config_attr"], 7)

if __name__ == "__main__":
    unittest.main()


import gmpy2
import math
from multiprocessing import Pool
from tqdm import tqdm

import sys
sys.setrecursionlimit(5000)

def gen_next_gen(test):
    left, right, length = test
    merge = 10 ** length

    new = []
    for x in range(0, 10):
        l_temp = (left * 10 + x)
        join = l_temp * 10 * merge
        for y in range(0, 10):
            r_temp = right + y * merge
            temp = join + r_temp
            if gmpy2.is_prime(temp):
                new.append((l_temp, r_temp, length + 1))
    return new

total_count = 0
global_best = 30
def recursive(test):
    global global_best, total_count

    total_count += 1
    if total_count % 1000000 == 0:
        print("\t", total_count)

    left, right, merge, bitset = test

    if left >= global_best:
        global_best = left
        pairs = bin(bitset).count("1")
        print (2 * pairs, pairs, left, right)

    next_merge = 10 * merge

    new = []
    for x in range(0, 10):
        l_temp = (left * 10 + x)
        join = l_temp * 10 * merge

        for y in range(0, 10):
            new_bitset = bitset | (1 << (10*x+y))
            if bitset == new_bitset:
                continue

            r_temp = right + y * merge
            temp = join + r_temp

            if gmpy2.is_prime(temp):
                r_temp = right + y * merge
                recursive((l_temp, r_temp, next_merge, new_bitset))

# Takes ~200-300 minutes on 10 cores
def puzz_954():
    with Pool() as p:
        gen = [(p//10, p % 10, 1) for p in range(10, 100) if gmpy2.is_prime(p)]
        assert len(gen) == 21, gen

        while gen[0][2] <= 3:
            next_gen = [a for l in p.map(gen_next_gen, gen) for a in l]

            print(gen[0][2], len(next_gen))
            gen = next_gen

#    gen.sort(reverse=True)
    import random
    random.shuffle(gen)

    for test in tqdm(gen):
        # verify pairs are unique here and generate bitset
        left, right, length = test
        strL = str(left)
        strR = str(right).rjust(length, "0")
        assert len(strL) == len(strR)

        pairs = [int(strL[i] + strR[length-i-1]) for i in range(length)]
        if len(pairs) != len(set(pairs)):
            print ("\tskipping", strL + strR, pairs)
            continue

        bitset = 0
        for p in pairs:
            bitset |= 1 << p

        recursive((left, right, 10 ** length, bitset))

puzz_954()


from __future__ import print_function
from collections import OrderedDict
import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
from torch.autograd import Variable
from torchvision.utils import make_grid
from models.model import D_NET_Multi, SingleGenerator, Encoder, weights_init
from util.loss import GANLoss, KL_loss
from util.util import tensor2im
import numpy as np

################## SingleGAN #############################
class SingleGAN():
    def name(self):
        return 'SingleGAN'

    def initialize(self, opt):
        torch.cuda.set_device(opt.gpu)
        cudnn.benchmark = True
        self.opt = opt
        self.build_models()

    def build_models(self):
        ################### generator #########################################
        self.G = SingleGenerator(input_nc=self.opt.input_nc, output_nc=self.opt.input_nc, ngf=self.opt.ngf, nc=self.opt.c_num+self.opt.d_num, e_blocks=self.opt.e_blocks, norm_type=self.opt.norm)
        ################### encoder ###########################################
        self.E =None
        if self.opt.mode == 'multimodal':
            self.E = Encoder(input_nc=self.opt.input_nc, output_nc=self.opt.c_num, nef=self.opt.nef, nd=self.opt.d_num, n_blocks=4, norm_type=self.opt.norm)
        if self.opt.isTrain:
            ################### discriminators #####################################
            self.Ds = []
            for i in range(self.opt.d_num):
                self.Ds.append(D_NET_Multi(input_nc=self.opt.output_nc, ndf=self.opt.ndf, block_num=3,norm_type=self.opt.norm))
            ################### init_weights ########################################
            if self.opt.continue_train:
                self.G.load_state_dict(torch.load('{}/G_{}.pth'.format(self.opt.model_dir, self.opt.which_epoch)))
                if self.E is not None:
                    self.E.load_state_dict(torch.load('{}/E_{}.pth'.format(self.opt.model_dir, self.opt.which_epoch)))
                for i in range(self.opt.d_num):
                    self.Ds[i].load_state_dict(torch.load('{}/D_{}_{}.pth'.format(self.opt.model_dir, i, self.opt.which_epoch)))
            else:
                self.G.apply(weights_init(self.opt.init_type))
                if self.E is not None:
                    self.E.apply(weights_init(self.opt.init_type))
                for i in range(self.opt.d_num):
                    self.Ds[i].apply(weights_init(self.opt.init_type))
            ################### use GPU #############################################
            self.G.cuda()
            if self.E is not None:
                self.E.cuda()
            for i in range(self.opt.d_num):
                self.Ds[i].cuda()
            ################### set criterion ########################################
            self.criterionGAN = GANLoss(mse_loss=(self.opt.c_gan_mode == 'lsgan'))
            ################## define optimizers #####################################
            self.define_optimizers()
        else:
            self.G.load_state_dict(torch.load('{}/G_{}.pth'.format(self.opt.model_dir, self.opt.which_epoch)))
            self.G.cuda()
            self.G.eval()
            if self.E is not None:
                self.E.load_state_dict(torch.load('{}/E_{}.pth'.format(self.opt.model_dir, self.opt.which_epoch)))
                self.E.cuda()
                self.E.eval()

    def sample_latent_code(self, size):
        c = torch.cuda.FloatTensor(size).normal_()
        return Variable(c)

    def get_domain_code(self, domainLable):
        domainCode = torch.zeros([len(domainLable),self.opt.d_num])
        domainIndex_cache = [[] for i in range(self.opt.d_num)]
        for index in range(len(domainLable)):
            domainCode[index, domainLable[index]] = 1
            domainIndex_cache[domainLable[index]].append(index)
        domainIndex = []
        for index in domainIndex_cache:
            domainIndex.append(Variable(torch.LongTensor(index)).cuda())
        return Variable(domainCode).cuda(), domainIndex

    def define_optimizer(self, Net):
        return optim.Adam(Net.parameters(),
                                    lr=self.opt.lr,
                                    betas=(0.5, 0.999))
    def define_optimizers(self):
        self.G_opt = self.define_optimizer(self.G)
        self.E_opt = None
        if self.E is not None:
            self.E_opt = self.define_optimizer(self.E)
        self.Ds_opt = []
        for i in range(self.opt.d_num):
            self.Ds_opt.append(self.define_optimizer(self.Ds[i]))

    def update_lr(self, lr):
        for param_group in self.G_opt.param_groups:
            param_group['lr'] = lr
        if self.E_opt is not None:
            for param_group in self.E_opt.param_groups:
                param_group['lr'] = lr
        for i in range(self.opt.d_num):
            for param_group in self.Ds_opt[i].param_groups:
                param_group['lr'] = lr

    def save(self, name):
        torch.save(self.G.state_dict(), '{}/G_{}.pth'.format(self.opt.model_dir, name))
        if self.E_opt is not None:
            torch.save(self.E.state_dict(), '{}/E_{}.pth'.format(self.opt.model_dir, name))
        for i in range(self.opt.d_num):
            torch.save(self.Ds[i].state_dict(), '{}/D_{}_{}.pth'.format(self.opt.model_dir, i, name))

    def prepare_image(self, data):
        img, sourceD, targetD = data
        return Variable(torch.cat(img,0)).cuda(), torch.cat(sourceD,0), torch.cat(targetD,0)

    def translation(self, data):
        input, sourceD, targetD = self.prepare_image(data)
        sourceDC, sourceIndex = self.get_domain_code(sourceD)
        targetDC, targetIndex = self.get_domain_code(targetD)

        images, names =[], []
        for i in range(self.opt.d_num):
            images.append([tensor2im(input.index_select(0,sourceIndex[i])[0].data)])
            names.append(['D{}'.format(i)])

        if self.opt.mode == 'multimodal':
            for i in range(self.opt.n_samples):
                c_rand = self.sample_latent_code(torch.Size([input.size(0),self.opt.c_num]))
                targetC = torch.cat([targetDC, c_rand],1)
                output = self.G(input,targetC)
                for j in range(output.size(0)):
                    images[sourceD[j]].append(tensor2im(output[j].data))
                    names[sourceD[j]].append('{}to{}_{}'.format(sourceD[j],targetD[j],i))
        else:
            output = self.G(input,targetDC)
            for i in range(output.size(0)):
                images[sourceD[i]].append(tensor2im(output[i].data))
                names[sourceD[i]].append('{}to{}'.format(sourceD[i],targetD[i]))

        return  images, names

    def get_current_errors(self):
        dict = []
        for i in range(self.opt.d_num):
            dict += [('D_{}'.format(i), self.errDs[i].data.item())]
            dict += [('G_{}'.format(i), self.errGs[i].data.item())]
        dict += [('errCyc', self.errCyc.data.item())]
        if self.opt.lambda_ide > 0:
            dict += [('errIde', self.errIde.data.item())]
        if self.E is not None:
            dict += [('errKl', self.errKL.data.item())]
            dict += [('errCode', self.errCode.data.item())]
        return OrderedDict(dict)

    def get_current_visuals(self):
        real = make_grid(self.real.data,nrow=self.real.size(0),padding=0)
        fake = make_grid(self.fake.data,nrow=self.real.size(0),padding=0)
        cyc = make_grid(self.cyc.data,nrow=self.real.size(0),padding=0)
        img = [real,fake,cyc]
        name = 'rsal,fake,cyc'
        if self.opt.lambda_ide > 0:
            ide = make_grid(self.ide.data,nrow=self.real.size(0),padding=0)
            img.append(ide)
            name +=',ide'
        img = torch.cat(img,1)
        return OrderedDict([(name,tensor2im(img))])

    def update_D(self, D, D_opt, real, fake):
        D.zero_grad()
        pred_fake = D(fake.detach())
        pred_real = D(real)
        errD = self.criterionGAN(pred_fake,False) + self.criterionGAN(pred_real,True)
        errD.backward()
        D_opt.step()
        return errD

    def calculate_G(self, D, fake):
        pred_fake = D(fake)
        errG = self.criterionGAN(pred_fake,True)
        return errG

    def update_model(self,data):
        ### prepare data ###
        self.real, sourceD, targetD = self.prepare_image(data)
        sourceDC, self.sourceIndex = self.get_domain_code(sourceD)
        targetDC, self.targetIndex = self.get_domain_code(targetD)
        sourceC, targetC = sourceDC, targetDC
        ### generate image ###
        if self.E is not None:
            c_enc, mu, logvar = self.E(self.real,sourceDC)
            c_rand = self.sample_latent_code(c_enc.size())
            sourceC = torch.cat([sourceDC, c_enc],1)
            targetC = torch.cat([targetDC, c_rand],1)
        self.fake = self.G(self.real,targetC)
        self.cyc = self.G(self.fake,sourceC)
        if self.E is not None:
            _, mu_enc, _ = self.E(self.fake,targetDC)
        if self.opt.lambda_ide > 0:
            self.ide = self.G(self.real,sourceC)
        ### update D ###
        self.errDs = []
        for i in range(self.opt.d_num):
            errD = self.update_D(self.Ds[i], self.Ds_opt[i], self.real.index_select(0,self.sourceIndex[i]), self.fake.index_select(0,self.targetIndex[i]))
            self.errDs.append(errD)
        ### update G ###
        self.errGs, self.errKl, self.errCode, errG_total = [], 0, 0, 0
        self.G.zero_grad()
        for i in range(self.opt.d_num):
            errG = self.calculate_G(self.Ds[i], self.fake.index_select(0,self.targetIndex[i]))
            errG_total += errG
            self.errGs.append(errG)
        self.errCyc = torch.mean(torch.abs(self.cyc-self.real)) *  self.opt.lambda_cyc
        errG_total += self.errCyc
        if self.opt.lambda_ide > 0:
            self.errIde = torch.mean(torch.abs(self.ide-self.real)) *  self.opt.lambda_ide
            errG_total += self.errIde
        if self.E is not None:
            self.E.zero_grad()
            self.errKL = KL_loss(mu,logvar) * self.opt.lambda_kl
            errG_total += self.errKL
            errG_total.backward(retain_graph=True)
            self.G_opt.step()
            self.E_opt.step()
            self.G.zero_grad()
            self.E.zero_grad()
            self.errCode = torch.mean(torch.abs(mu_enc - c_rand)) * self.opt.lambda_c
            self.errCode.backward()
            self.G_opt.step()
        else:
            errG_total.backward()
            self.G_opt.step()


# -*- coding: utf-8 -*-
"""
Created on Fri Sep  9 12:13:58 2022

@author: SharwinA
"""

import os
import pandas as pd
import matplotlib.pyplot as plt

#Dataset Loading
os.chdir('S:\LAB FILES\Internship')
data = pd.read_csv('Iris(1).csv')
print(data.head(10))
print(data.describe())

data = data.drop(columns = 'Id')

#x = data.iloc[:, [0,1,2,3]].values
#y = data.iloc[:, 4].values
x = data.drop("Species", axis=1)
y= data["Species"]

print(x)
print(y)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)

from sklearn.tree import DecisionTreeClassifier
model= DecisionTreeClassifier()
model.fit(x,y)

y_pred = model.predict(X_test)
print(y_pred)

from sklearn import tree
#feature = data.drop(columns = 'Species')

classname = ['Iris-setosa', 'Iris-virginica', 'Iris-versicolor']
# Visualize the graph
fig = plt.figure(figsize=(25,20))
fig_ = tree.plot_tree(model, feature_names=x.columns,
                      class_names=classname,
                      filled=True)

#Checking the accuracy of the model by accuracy score on X_test data

from sklearn.metrics import accuracy_score
a1 = accuracy_score(y_test, y_pred)
print("Accuracy score : {:.2f}%".format(a1*100))

#Testing on random data points except from dataset
Testing = [[6.4,3.0,4.5,1.5],
           [6.5,2.8,4.6,1.3],
           [5.0,2.5,-0.5,1.4],
           [6.0,2.2,5.1,1.1],
           [6.0,3.0,1.5,-2.8]]

print(model.predict(Testing))

Prediction using Decision Tree/Internship decision tree.py


import json
import os
import sys

with open("ips.txt") as f: ips = (f.read()).split("\n")
with open("censysTruth.json") as f: censys = json.loads(f.read())
with open("nmapTruth.json") as f: nmap = json.loads(f.read())

def trim(tag):
	tag = tag.split(",")[0].lower()
	if tag == 'network':
		tag = 'router'
	if 'router' in tag:
		tag = 'router'
	return tag

devices = {}
vendors = {}
products= {}
tuples  = {}

total = 0

for ip in ips:
	d = ""
	v = ""
	p = ""

	if 'deviceType' in censys[ip]:
		device = censys[ip]['deviceType']
		device = trim(device)
		if device in devices:
			devices[device] += 1
		else:
			devices[device] = 1
		d = device

	if 'manufacturer' in censys[ip]:
		vendor = censys[ip]['manufacturer']
		vendor = trim(vendor)
		if vendor in vendors:
			vendors[vendor] += 1
		else:
			vendors[vendor] = 1
		v = vendor

	if 'product' in censys[ip]:
		product = censys[ip]['product']
		product = trim(product)
		if product in products:
			products[product] += 1
		else:
			products[product] = 1
		p = product

	tup = (d, v, p)

	if tup in tuples:
		tuples[tup] += 1
		total += 1
	else:
		tuples[tup] = 1
		total += 1

print(len(vendors))

# for ele in products:
# 	print(products[ele])


import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda, Compose
import torch.nn.utils.prune as prune
import prune_method as customized_prune

batch_size = 128
epoches = 100

def load_data(task_type):
    if task_type == "train":
        train = True
    elif task_type == "test":
        train = False
    else:
        raise ValueError
    data = datasets.CIFAR10(root='./data', train=train, download=True, transform=ToTensor(),)
    dataloader = DataLoader(data, batch_size=batch_size)
    return dataloader

def load_model():
    model = torch.hub.load('pytorch/vision:v0.10.0', 'vgg19', pretrained=False, num_classes=10)
    return model

def get_optimizer(model):
    optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, nesterov=True)
    torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epoches)
    return optimizer

accuracy_loss = nn.CrossEntropyLoss()

def save_model(model, epoch):
    print('save model for epoch {}'.format(epoch))
    torch.save(model, "./models/epoch_{}".format(epoch))

def train(dataloader, model, loss_fn, optimizer, regularization=True, lambda1 = 1e-7):
    model.train()
    for epoch in range(epoches):
        print("epoch {} begins:".format(epoch))
        save_model(model, 0)
        for batch, (X,y) in enumerate(dataloader):
            pred = model(X)
            loss = loss_fn(pred, y)
            if regularization:
                l1_params = torch.cat([x.view(-1) for x in model.parameters()])
                loss += lambda1 * torch.norm(l1_params, 1)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (batch % 10 ==0):
                print("pass {}, loss is {}".format(batch, loss.item()))
        if epoch == epoches/2 :
            for name, module in model.named_modules():
                if isinstance(module, torch.nn.Conv2d):
                    prune.l1_unstructured(module, name='weight', amount=0.3)
                    customized_prune.group_prune_structured(module, name='weight')
                    # trivial_prune_structured(module, name='weight')
        if (epoch + 1) % 10 == 0:
            save_model(model, epoch)

def test(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).item()
    test_loss /= size
    correct /= size
    print("test loss is {}".format(test_loss))
    print("correct rate is {}".format(correct))

train_dataloader = load_data("train")
model = load_model()
optimizer = get_optimizer(model)

train(train_dataloader, model, accuracy_loss, \
        optimizer=optimizer)
#test(test_dataloader, model, accuracy_loss)


from enum import Enum
from typing import List

class PieceColor(Enum):
    black = "BLACK"
    white = "WHITE"

class Board:
    def __init__(self, positions: 'Position', chess_set: 'ChessSet'):
        self._positions = positions
        self._set = chess_set

class ChessSet:
    def __init__(self, pieces: List['Piece'], board: Board):
        self._pieces = pieces
        self._board = board

    # pieces: List['Piece']
    # board: Board

def create_positions():
    h_p = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
    v_p = ['8', '7', '6', '5', '4', '3', '2', '1']
    row = []
    rows = []
    for i in v_p:
        for j in h_p:
            row.append(i + j)
        rows.append(row)
        row = []

    return rows

class Position:
    """Square of the board"""
    def __init__(self, chess_board, x, y):
        self._x = x
        self._y = y
        self._board = chess_board

class Player:
    """Player of the game"""
    pass

#     should initialize with chess set
#     can move piece when its turn

class Piece:
    color: PieceColor
    chess_set: ChessSet


import codecs
import os
import re
import sys

# Common distribution data
name = 'snapcraft'
version = 'devel'
description = 'Easily craft snaps from multiple sources'
author_email = 'snapcraft@lists.snapcraft.io'
url = 'https://github.com/snapcore/snapcraft'
packages = [
    'snapcraft',
    'snapcraft.cli',
    'snapcraft.extractors',
    'snapcraft.integrations',
    'snapcraft.internal',
    'snapcraft.internal.cache',
    'snapcraft.internal.deltas',
    'snapcraft.internal.lifecycle',
    'snapcraft.internal.lxd',
    'snapcraft.internal.meta',
    'snapcraft.internal.pluginhandler',
    'snapcraft.internal.project_loader',
    'snapcraft.internal.project_loader.grammar',
    'snapcraft.internal.project_loader.grammar_processing',
    'snapcraft.internal.repo',
    'snapcraft.internal.sources',
    'snapcraft.internal.states',
    'snapcraft.plugins',
    'snapcraft.plugins._ros',
    'snapcraft.plugins._python',
    'snapcraft.storeapi'
]
package_data = {'snapcraft.internal.repo': ['manifest.txt']}
license = 'GPL v3'
classifiers = (
    'Development Status :: 4 - Beta',
    'Environment :: Console',
    'Intended Audience :: Developers',
    'Intended Audience :: System Administrators',
    'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
    'Natural Language :: English',
    'Programming Language :: Python',
    'Programming Language :: Python :: 3',
    'Programming Language :: Python :: 3.5',
    'Topic :: Software Development :: Build Tools',
    'Topic :: System :: Software Distribution',
)

# look/set what version we have
changelog = 'debian/changelog'
if os.path.exists(changelog):
    head = codecs.open(changelog, encoding='utf-8').readline()
    match = re.compile('.*\((.*)\).*').match(head)
    if match:
        version = match.group(1)

# If on Windows, construct an exe distribution
if sys.platform == 'win32':
    from cx_Freeze import setup, Executable

    # cx_Freeze relevant options
    build_exe_options = {
        # Explicitly add any missed packages that are not found at runtime
        'packages': [
            'pkg_resources',
            'pymacaroons',
            'click',
            'responses',
            'configparser',
            'docopt',
            'cffi',
        ],
        # Explicit inclusion data, which is then clobbered.
        'include_files': [
            ('libraries', os.path.join('share', 'snapcraft', 'libraries')),
            ('schema', os.path.join('share', 'snapcraft', 'schema')),
        ],
    }

    exe = Executable(
        script='bin/snapcraft',
        base=None  # console subsystem
    )

    setup(
        name=name,
        version=version,
        description=description,
        author_email=author_email,
        url=url,
        packages=packages,
        package_data=package_data,
        license=license,
        classifiers=classifiers,
        # cx_Freeze-specific arguments
        options={'build_exe': build_exe_options},
        executables=[exe],
    )

# On other platforms, continue as normal
else:
    from setuptools import setup

    setup(
        name=name,
        version=version,
        description=description,
        author_email=author_email,
        url=url,
        packages=packages,
        package_data=package_data,
        license=license,
        classifiers=classifiers,
        # non-cx_Freeze arguments
        entry_points={
            'console_scripts': [
                'snapcraft = snapcraft.cli.__main__:run',
                'snapcraft-parser = snapcraft.internal.parser:main',
            ],
        },
        data_files=[
            ('share/snapcraft/schema',
             ['schema/' + x for x in os.listdir('schema')]),
            ('share/snapcraft/libraries',
             ['libraries/' + x for x in os.listdir('libraries')]),
        ],
        install_requires=[
            'pysha3',
            'pyxdg',
            'requests',
            'libarchive-c',
        ],
        test_suite='snapcraft.tests.unit',
    )