# Databricks notebook source
# MAGIC %run ./sampling/cdf_sampling

# COMMAND ----------

from functools import partial, reduce
from typing import Iterable, Callable, List

from pyspark.sql import DataFrame, functions as F

# Prevent spark from auto broadcasting the metadata
spark.conf.set("spark.sql.autoBroadcastJoinThreshold", -1)

# COMMAND ----------

union_dfs = partial(reduce, DataFrame.unionByName)

def get_pivot_field_values(df: DataFrame, pivot_field: str) -> List[str]:
  """Get a list of the distinct values in a field to be used in the values argument of a
  call to DataFrame.pivot. These values will be the columns in the pivoted dataframe.

  Args:
      df (pyspark.sql.DataFrame): DataFrame to be pivoted
      pivot_field (str): Field to pivot by. Distinct values will be the columns of the pivoted dataframe

  Returns:
      List[str]: Distinct values in the pivot field
  """
  pivot_field_values = (
    df
    .select(pivot_field)
    .distinct()
    .collect()
  )
  pivot_field_values = [x[pivot_field] for x in pivot_field_values]
  return pivot_field_values

def generate_empty_samples(
  metadata_df,
  sample_id_df,
  variable_cols=("TABLE_NAME", "FIELD_NAME"),
) -> DataFrame:
  """ Cross-joins the variable names in the metadata with the rows of sample_id_df,
  so that each row in sample_id_df is associated with each distinct combination
  of variables in the metadata.

  This doesnt actually do the sampling, it simply builds a tidy dataframe that
  the samples are generated onto (see the example).

  Args:
      metadata_df (pyspark.sql.DataFrame): Contains metadata
      sample_id_df (pyspark.sql.DataFrame): Contains indexes to generate samples for

  Returns:
      pyspark.sql.DataFrame:  Cariable names in the metadata cross-joined with the rows of sample_id_df

  Example:
      >> metadata_df
      | TABLE_NAME | FIELD_NAME | ... |
      | ---------- | ---------- | --- |
      |  table1    |   field1   | ... |
      |  table1    |   field2   | ... |
      |  table2    |   field1   | ... |

      >> sample_id_df
      | SAMPLE_ID |
      | --------- |
      |     1     |
      |     2     |

      >> generate_empty_samples(metadata_df, sample_id_df)
      | SAMPLE_ID | TABLE_NAME | FIELD_NAME |
      | --------- | ---------- | ---------- |
      |     1     |  table1    |   field1   |
      |     1     |  table1    |   field2   |
      |     1     |  table2    |   field1   |
      |     2     |  table1    |   field1   |
      |     2     |  table1    |   field2   |
      |     2     |  table2    |   field1   |

  """
  variable_df = (
    metadata_df
    .select(*variable_cols)
    .distinct()
  )

  sample_df = sample_id_df.crossJoin(variable_df.hint("broadcast"))

  return sample_df

def continuous_field_generator(meta_df: DataFrame, base_df: DataFrame, variable_fields: Iterable[str]) -> DataFrame:
  """Generate a dataframe of (long-formatted) values for fields with VALUE_TYPE='CONTINUOUS' in the metadata.
  This is done by uniform sampling from bins defining the percentiles of a continuous distribution (i.e.
  first we pick a bin with 1% width, then we sample a value from within that bin).

  Args:
      meta_df (pyspark.sql.DataFrame): Metadata for the dataset
      base_df (pyspark.sql.DataFrame): Base data to sample values for, indexed over patients / events.
      variable_fields (Iterable[str]): Names of metadata fields that define the fields to samples (e.g.
        we generally want to sample per combination of 'TABLE_NAME' and 'FIELD_NAME')

  Returns:
      DataFrame: DataFrame of generated data (long-formatted), with a single value per
        unique combination of variable_fields.
  """
  filtered_meta_df = meta_df.filter(F.col("VALUE_TYPE") == "CONTINUOUS")
  sample_df = generate_empty_samples(filtered_meta_df, base_df, variable_cols=variable_fields)

  # Sampling from percentile distributions
  bin_bounds_col = F.expr("transform(slice(VALUE_NUMERIC_ARRAY, 1, size(VALUE_NUMERIC_ARRAY)-1), (x, i) -> struct(x AS LOWER, VALUE_NUMERIC_ARRAY[i+1] AS UPPER))")
  sampled_bin_col = F.element_at(F.shuffle(bin_bounds_col), 1)
  sampled_value_col = F.col("SAMPLED_BIN.LOWER") + F.rand() * (F.col("SAMPLED_BIN.UPPER") - F.col("SAMPLED_BIN.LOWER"))

  generated_df = (
    filtered_meta_df.filter("VALUE_TYPE == 'CONTINUOUS'")
    .join(sample_df, how="inner", on=variable_fields)
    .withColumn("SAMPLED_BIN", sampled_bin_col)  # This needs to be attached using withColumn before sampled_value_col. Trying to do in a single step gives strange results!
    .withColumn("VALUE_NUMERIC", sampled_value_col)
  )

  generated_df = generated_df.select(*sample_df.columns, "VALUE_NUMERIC")

  return generated_df

def create_frequency_field_generator(
  value_type: str,
  *value_fields: Iterable[str]
) -> Callable[[DataFrame, DataFrame, Iterable[str]], DataFrame]:
  """Creates a function that generates a dataframe of (long-formatted) values for fields by sampling from a
  frequency distribution (i.e. of value, weight pairs).

  Args:
      value_type (str): Defines filter on the VALUE_TYPE column in the metadata, which specifies which type of
        metadata this generator operates on (e.g. 'CONTINUOUS', 'DATE', 'DISCRETE')
      *value_fields (Iterable[str]): Names of columns in the metadata which contain the values that the frequency
        distribution is over (each set of values is paired with a weight which defines its frequency)

  Returns:
      Callable[[DataFrame, DataFrame, Iterable[str]], DataFrame]: Generates a dataframe of values by sampling from a
        frequency distribution
  """
  value_type_condition_col = F.col("VALUE_TYPE") == value_type
  value_struct_col = F.struct(*value_fields)

  def _field_generator(meta_df: DataFrame, base_df: DataFrame, variable_fields: Iterable[str]) -> DataFrame:
    """Generates a dataframe of values for fields by sampling from a frequency distribution (i.e. value, weight pairs).

    Args:
        meta_df (pyspark.sql.DataFrame): Metadata for the dataset
        base_df (pyspark.sql.DataFrame): Base data to sample values for, indexed over patients / events.
        variable_fields (Iterable[str]): Names of metadata fields that define the fields to samples (e.g.
          we generally want to sample per combination of 'TABLE_NAME' and 'FIELD_NAME')

    Returns:
        DataFrame: DataFrame of generated data (long-formatted), with a single value per
          unique combination of variable_fields.
    """
    filtered_meta_df = meta_df.filter(value_type_condition_col)
    sample_df = generate_empty_samples(filtered_meta_df, base_df, variable_cols=variable_fields)

    filtered_meta_df = filtered_meta_df.withColumn("VALUE_STRUCT", value_struct_col)

    generated_df = sample_cdf(
      filtered_meta_df,
      sample_df,
      value_field="VALUE_STRUCT",
      frequency_field="WEIGHT",
      variable_fields=variable_fields
    )

    # Select the sampled values, explode the struct after sampling
    generated_df = generated_df.selectExpr(*sample_df.columns, "VALUE_STRUCT.*")

    return generated_df

  return _field_generator

categorical_field_generator = create_frequency_field_generator("CATEGORICAL", "VALUE_STRING")
date_field_generator        = create_frequency_field_generator("DATE", "VALUE_DATE")
discrete_field_generator    = create_frequency_field_generator("DISCRETE", "VALUE_NUMERIC")

def field_generator(
  meta_df: DataFrame,
  base_df: DataFrame,
  pivot_field: str,
  other_variable_fields: Iterable[str]=tuple()
) -> DataFrame:
  """Generates a dataframe of (wide-formatted) values for fields by sampling from metadata.

  Args:
      meta_df (pyspark.sql.DataFrame): Metadata for the dataset
      base_df (pyspark.sql.DataFrame): Base data to sample values for, indexed over patients / events.
      pivot_field (str): Name of the field in the metadata that we want to pivot the resulting generated
        dataframe by. Generally the values in this field are the names of the fields in the dataset
        the artificial data represents
      other_variable_fields (Iterable[str], optional): Fields that, in addition to the pivot field, define
        the unique fields in the metadata. Defaults to tuple().

  Returns:
      DataFrame: DataFrame of generated data (wide-formatted), with a single value per
        unique combination of pivot_field and other_variable_fields.
  """
  filtered_meta_df = meta_df.filter(F.col("VALUE_TYPE").isin(["CATEGORICAL", "DISCRETE", "DATE", "CONTINUOUS"]))

  variable_fields = [pivot_field, *other_variable_fields]
  generator_args = (filtered_meta_df, base_df, variable_fields)
  generated_long_df = union_dfs([
    categorical_field_generator(*generator_args).select(*variable_fields, *base_df.columns, F.col("VALUE_STRING") .alias("VALUE")),
    date_field_generator(       *generator_args).select(*variable_fields, *base_df.columns, F.col("VALUE_DATE")   .alias("VALUE")),
    discrete_field_generator(   *generator_args).select(*variable_fields, *base_df.columns, F.col("VALUE_NUMERIC").alias("VALUE")),
    continuous_field_generator( *generator_args).select(*variable_fields, *base_df.columns, F.col("VALUE_NUMERIC").alias("VALUE")),
  ])

  # Pivot to wide format
  pivot_field_values = get_pivot_field_values(filtered_meta_df, pivot_field)
  generated_wide_df = (
    generated_long_df
    .groupBy(*base_df.columns)
    .pivot(pivot_field, pivot_field_values)
    .agg(F.first("VALUE"))
  )

  return generated_wide_df


from flask import Flask, request, abort, render_template, jsonify

import os
import json
import hashlib
import hmac

import git

import openai
from openai.error import RateLimitError

app = Flask(__name__)
w_secret = str(os.environ.get('WEBHOOK_SECRET'))
openai.api_key = str(os.getenv("OPENAI_API_KEY"))

@app.route("/")
def hello_world():
    return "
henLO world! gpt api
"

# url_for('static', filename='style.css')

@app.route("/pong")
def pong():
    return render_template('pong.html')

@app.route("/henlo")
def henlo():
    return "
henLO
"

@app.route('/gpt4', methods=['GET', 'POST'])
def gpt4():
    user_input = request.args.get(
        'user_input') if request.method == 'GET' else request.form['user_input']
    messages = [{"role": "user", "content": user_input}]

    try:
        response = openai.ChatCompletion.create(
            model="gpt-4",
            messages=messages
        )
        content = response.choices[0].message["content"]
    except RateLimitError:
        content = "The server is experiencing a high volume of requests. Please try again later."

    return jsonify(content=content)

def is_valid_signature(x_hub_signature, data, private_key):
    hash_algorithm, github_signature = x_hub_signature.split('=', 1)
    algorithm = hashlib.__dict__.get(hash_algorithm)
    encoded_key = bytes(private_key, 'latin-1')
    mac = hmac.new(encoded_key, msg=data, digestmod=algorithm)
    return hmac.compare_digest(mac.hexdigest(), github_signature)

@app.route('/update_server', methods=['POST'])
def webhook():
    if request.method != 'POST':
        return 'OK'
    else:
        abort_code = 418
        # Do initial validations on required headers
        if 'X-Github-Event' not in request.headers:
            abort(abort_code)
        if 'X-Github-Delivery' not in request.headers:
            abort(abort_code)
        if 'X-Hub-Signature' not in request.headers:
            abort(abort_code)
        if not request.is_json:
            abort(abort_code)
        if 'User-Agent' not in request.headers:
            abort(abort_code)
        ua = request.headers.get('User-Agent')
        if not ua.startswith('GitHub-Hookshot/'):
            abort(abort_code)

        event = request.headers.get('X-GitHub-Event')
        if event == "ping":
            return json.dumps({'msg': 'Hi!'})
        if event != "push":
            return json.dumps({'msg': "Wrong event type"})

        x_hub_signature = request.headers.get('X-Hub-Signature')
        # webhook content type should be application/json for request.data to have the payload
        # request.data is empty in case of x-www-form-urlencoded
        if not is_valid_signature(x_hub_signature, request.data, w_secret):
            print('Deploy signature failed: {sig}'.format(sig=x_hub_signature))
            abort(abort_code)

        payload = request.get_json()
        if payload is None:
            print('Deploy payload is empty: {payload}'.format(
                payload=payload))
            abort(abort_code)

        # if payload['ref'] != 'refs/heads/master':
        #    return json.dumps({'msg': 'Not master; ignoring'})

        repo = git.Repo('~/pythonanywhere')
        origin = repo.remotes.origin

        pull_info = origin.pull()

        if len(pull_info) == 0:
            return json.dumps({'msg': "Didn't pull any information from remote!"})
        if pull_info[0].flags > 128:
            return json.dumps({'msg': "Didn't pull any information from remote!"})

        commit_hash = pull_info[0].commit.hexsha
        build_commit = f'build_commit = "{commit_hash}"'
        print(f'{build_commit}')
        return 'Updated PythonAnywhere server to commit {commit}'.format(commit=commit_hash)

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8000, debug=True)


# critter detector - polling solution

from gpiozero import DistanceSensor, Button, LED
from time import sleep

dsense = DistanceSensor(echo = 24, trigger = 18)
button = Button(16)
theLED = LED(12)

def IntruderAlert(device):
    if (device == "DistanceSensor"):
        theLED.blink(on_time = .25, off_time = .75, n = 10, background = True)
        print("Proximity: Intruder Alert!")
    else:
        print("Button: Intruder Alert!")
        theLED.blink(on_time = .75, off_time = .25, n = 10, background = True)

try:
    while True:
        sleep(3)
        intruderDistance = dsense.distance
        print(intruderDistance)
        if (intruderDistance < .3):
            IntruderAlert("DistanceSensor")
        if (button.is_pressed):
            IntruderAlert("Button")

except KeyboardInterrupt:
    print("cleanup")

Raspberry Pi Weekly/2020 - q1/05_08 polling interrupt/polling_solution.py


import argparse
import sys

from kedehub.services.identities_updater import identity_merge_users, identity_merge_users_with_human_help
from kedehub.services.project_service import load_all_project_names

def identity_merge_project(options):
    verbose = options.verbose

    projects_to_update = options.project
    if not projects_to_update:
        projects_to_update = load_all_project_names()

    number_or_projects_to_update = len(projects_to_update)

    print("First pass...")
    len_users, lean_authors, len_matched_users_by_email, len_matched_users_by_email_and_name, \
    authors_updated, users_inserted, users_merged_count, authors_merged_count = identity_merge_users(projects_to_update, verbose)
    print('Successfully merged {} into {} users with {} authors for {} projects. Created {} new users. '.format(len_users, len_matched_users_by_email_and_name, authors_updated, number_or_projects_to_update, users_inserted))

    print("Second pass...")
    len_users, lean_authors, len_matched_users_by_email, len_matched_users_by_email_and_name, \
    authors_updated, users_inserted, users_merged_count, authors_merged_count = identity_merge_users_with_human_help(projects_to_update, verbose)
    print(
        'Successfully merged {} into {} users with {} authors for {} projects. Created {} new users. '.format(len_users,
                                                                                len_matched_users_by_email_and_name,
                                                                                authors_updated,
                                                                                number_or_projects_to_update,
                                                                                users_inserted))

def parse_args(args):
    parser = argparse.ArgumentParser(prog='kedehub',
                                 description='Extract KEDE statistics from Git repositories')
    command_parsers = parser.add_subparsers(dest='subparser_name')

    create_identity_merge_parser(command_parsers)

    return parser.parse_args(args)

def create_identity_merge_parser(command_parsers):
    identity_merge_parser = command_parsers.add_parser('identity-merge', help='Merge identities of existing projects')
    identity_merge_parser.add_argument('-p', '--project', nargs='+',
                                       help='Name of the project which identities to merge')
    identity_merge_parser.add_argument('--verbose', dest='verbose', action='store_true')
    identity_merge_parser.add_argument('--no-verbose', dest='verbose', action='store_false')
    identity_merge_parser.set_defaults(verbose=False)
    identity_merge_parser.set_defaults(func=identity_merge_project)

def Main() -> None:
    # parse the args
    parsed_args = parse_args(sys.argv[1:])
    # call whatever function was selected
    parsed_args.func(parsed_args)

if __name__ == '__main__':
  Main()


from . import Atomic
from . import util
import argparse
import json as Json
import tty
import sys
import termios
import select
from os import isatty
from operator import itemgetter
import requests
from Atomic.backends._docker_errors import NoDockerDaemon

def check_negative(value):
    ivalue = int(value)
    if ivalue < 1:
        raise argparse.ArgumentTypeError("%s must be a positive integer value greater than 0." % value)
    return ivalue

def cli(subparser):
    # atomic top
    topp = subparser.add_parser("top",
                                help=_("Show top-like stats about processes running in containers"))
    topp.set_defaults(_class=Top, func='atomic_top')
    topp.add_argument("-d", type=int, default=1, help=_("Interval (secs) to refresh process information"))
    topp.add_argument("-o", "--optional", help=_("Additional fields to display"), nargs='?', action='append',
                      choices=['time', 'stime', 'ppid', 'uid', 'gid', 'user', 'group'])
    topp.add_argument("-n", help=_("Number of iterations"), type=check_negative)
    topp.add_argument("containers", nargs="*", help=_("list of containers to monitor, leave blank for all"))

class Top(Atomic):
    """
    Class to support atomic top; based on the Atomic class
    """
    # Set to true to output debug information

    def __init__(self):
        super(Top, self).__init__()
        self.input_var = None
        self._sort = 'CID'
        self.name_id = {}
        self.optional = None
        self.titles = None
        self.debug = False
        # To add a new column to the output, create a new dict inside self.headers
        # The fields are as follows:
        # shortname: <str> a unique key used to describe the dict
        # column: <str> column title, use parans to define a sort character
        # character: <str> a single character the user presses to sort on that column
        # sort: <bool> to determine IF the column can be sorted upon
        # index: <int> order the columns display left to right. Do not change 0-5
        # _field: <str> If the width of the column needs to be dynamically sized, put
        #               the formatting in _field, set field to None, and be sure to
        #               add a key for _min_width
        # field: <str> the preferred format for the column
        # active: if the column should be active by default
        # ps_opt: <str> the descriptor for ps to get the column's data (ps -o)
        # sort_order: <bool> True reverses the sort order from ascending to descending
        # _min_width: <int> Minimum field width if you are using dynamic fields
        self.headers = [
            {'shortname': '%CPU', 'column': '(C)PU', 'character': 'c', 'sort': True, 'index': 3,
             'field': '{:6}', 'active': True, 'ps_opt': 'pcpu', 'sort_order': True},
            {'shortname': 'CID', 'column': 'CONTA(I)NER', 'character': 'i', 'sort': True, 'index': 0,
             'field': '{:12.12}', 'active': True, 'ps_opt': None, 'sort_order': False},
            {'shortname': 'NAME', 'column': '(N)AME', 'character': 'n', 'sort': True, 'index': 1,
             '_field': '{:WIDTH}', 'field': None, 'active': True, 'ps_opt': None, 'sort_order': False,
             '_min_width': 10},
            {'shortname': 'PID', 'column': '(P)ID', 'character': 'p', 'sort': True, 'index': 2,
             'field': '{:<10}', 'active': True, 'ps_opt': 'pid', 'sort_order': False},
            {'shortname': '%MEM', 'column': '(M)EM', 'character': 'm', 'sort': True, 'index': 4,
             'field': '{:6}', 'active': True, 'ps_opt': 'pmem', 'sort_order': True},
            {'shortname': 'CMD', 'column': 'CMD', 'character': None, 'sort': False, 'index': 99,
             '_field': '{:WIDTH}', 'field': None, 'active': True, 'ps_opt': 'cmd', '_min_width': 10},
            {'shortname': 'TIME', 'column': '(T)IME', 'character': 't', 'sort': True, 'index': 6,
             'field': '{:10}', 'active': False, 'ps_opt': 'time', 'sort_order': True},
            {'shortname': 'STIME', 'column': '(S)TIME', 'character': 's', 'sort': True, 'index': 7,
             'field': '{:10}', 'active': False, 'ps_opt': 'stime', 'sort_order': True},
            {'shortname': 'PPID', 'column': 'PPI(D)', 'character': 'd', 'sort': True, 'index': 8,
             'field': '{:10}', 'active': False, 'ps_opt': 'ppid', 'sort_order': False},
            {'shortname': 'UID', 'column': '(U)ID', 'character': 'u', 'sort': True, 'index': 9,
             'field': '{:6}', 'active': True, 'ps_opt': 'uid', 'sort_order': True},
            {'shortname': 'GID', 'column': '(G)ID', 'character': 'g', 'sort': True, 'index': 10,
             'field': '{:6}', 'active': True, 'ps_opt': 'gid', 'sort_order': True},
            {'shortname': 'USER', 'column': 'USER', 'character': None, 'sort': False, 'index': 11,
             'field': '{:10}', 'active': False, 'ps_opt': 'user'},
            {'shortname': 'GROUP', 'column': 'GROUP', 'character': None, 'sort': False, 'index': 12,
             'field': '{:10}', 'active': False, 'ps_opt': 'group'}
        ]

    def _activate_optionals(self):
        """
        Sets the active bool to True for any optional  ps
        arguments.
        :return: None
        """
        if self.args.optional:
            for option in self.args.optional:
                for header in self.headers:
                    if header['ps_opt'] == option:
                        header['active'] = True

    def _set_dynamic_column_widths(self, ps_info):
        for header in self.headers:
            if '_field' in header:
                max_widths = [len(x[header['shortname']]) for x in ps_info]
                max_width = header['_min_width'] if not max_widths else max(max_widths)
                header['field'] = header['_field'].replace('WIDTH', str(max_width))

    def json(self):
        """
        Main sub-function for top
        :return: None
        """
        # Make sure the docker daemon is running
        self.ping()
        # Activate optional columns
        self._activate_optionals()
        proc_info = []
        if len(self.args.containers) < 1:
            try:
                con_ids = [x['Id'] for x in self.get_active_containers(refresh=True)]
            except requests.exceptions.ConnectionError:
                raise NoDockerDaemon()
        else:
            con_ids = []
            self.get_active_containers(refresh=True)
            # verify the inputs are valid
            for user_input in self.args.containers:
                con_ids.append(self.get_input_id(user_input))
        for cid in con_ids:
            proc_info += self.get_pids_by_container(cid)

        return Json.dumps(self.reformat_ps_info(proc_info))

    def atomic_top(self):
        """
        Main sub-function for top
        :return: None
        """
        # Make sure the docker daemon is running
        self.ping()
        # Set debug bool
        self.set_debug()
        # Activate optional columns
        self._activate_optionals()
        # Do we have a tty?
        # Can run ./atomic top <&- to replicate no tty
        has_tty = isatty(0)
        # Set up terminal, input handling
        if has_tty:
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
        sort_vals = [x['character'].upper() for x in self.headers if x['active'] and x['sort']]
        counter = 0
        while True:
            proc_info = []
            if len(self.args.containers) < 1:
                try:
                    con_ids = [x['Id'] for x in self.get_active_containers(refresh=True)]
                except requests.exceptions.ConnectionError:
                    raise NoDockerDaemon()
            else:
                con_ids = []
                self.get_active_containers(refresh=True)
                # verify the inputs are valid
                for user_input in self.args.containers:
                    con_ids.append(self.get_input_id(user_input))
            for cid in con_ids:
                proc_info += self.get_pids_by_container(cid)
            # Reset screen
            if not self.debug and has_tty:
                util.write_out("\033c")
            sorted_info = self.reformat_ps_info(proc_info)
            self._set_dynamic_column_widths(sorted_info)
            self.output_top(sorted_info)
            if has_tty:
                tty.setraw(sys.stdin.fileno())
            i, _, _ = select.select([sys.stdin], [], [], self.args.d)
            if i and has_tty:
                ch = sys.stdin.read(1)
                # Detect 'q' or Cntrl-c
                if ch.upper() == 'q'.upper() or ord(ch) == 3:
                    # reset terminal
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
                    raise SystemExit
                # Detect of the character pushed is one in self.headers
                elif ch.upper() in sort_vals:
                    self._sort = next((header['shortname'] for header in self.headers if header['character'] is not
                                       None and header['character'].upper() == ch.upper()), False)

            # reset terminal
            if has_tty:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
            counter += 1
            if counter == self.args.n:
                raise SystemExit

    def get_pids_by_container(self, con_id):
        """
        Call the custom API for docker python
        :param con_id: id of the container
        :return: dict
        """

        if len(self.active_containers) == 0:
            raise ValueError("No containers running")
        f_procs = []
        # Get the name of the container by ID
        con_name = str(next((l for l in self.active_containers if l['Id'].startswith(con_id)), None)['Names'][0])
        if con_name.startswith("/"):
            con_name = con_name.replace("/", "")
        # Assemble the ps args
        ps_args = [header['ps_opt']for header in sorted(self.headers, key=itemgetter('index')) if header['ps_opt']
                   is not None and header['active']]
        con_procs = self.d.top(con_id, ps_args="-eo {}".format(",".join(ps_args)))
        # Set the column header titles one-time
        if self.titles is None:
            self.titles = con_procs['Titles']

        if con_procs['Processes'] != None:
            # Massage the information into a dict
            for proc in con_procs['Processes']:
                t_dict = {'CID': con_id,
                          'NAME': con_name}
                for place in range(0, len(proc)):
                    t_dict[self.titles[place]] = proc[place]
                f_procs.append(t_dict)
        return f_procs

    def output_top(self, sorted_info):
        """
        Primary function for output to stdout
        :param sorted_info: list of dicts containing process information
        :return: None
        """
        cols = [col for col in sorted(self.headers, key=itemgetter('index')) if col['active']]
        active_column_names = [x['shortname'] for x in cols]
        out_format = " ".join([x['field'] for x in cols])
        col_headers = []

        # Add a '*' to the column header name currently being
        # sorted on
        for col in cols:
            title = col['column']
            if col['shortname'] == self._sort:
                title += "*"
            col_headers.append(title)
        formatted_col_headers = "\033[7m" + out_format.format(*col_headers) + "\033[0m"
        # output ATOMIC TOP title
        almost_center = len(formatted_col_headers)
        center_col = '{:^%WIDTH%}'.replace("%WIDTH%", str(almost_center))
        util.write_out(center_col.format("ATOMIC TOP\n"))
        # Output the headers
        util.write_out(formatted_col_headers)
        for ps in sorted_info:
            line_out = []
            for val in active_column_names:
                line_out.append(ps[val])
            # Output the ps information
            util.write_out(out_format.format(*line_out))

    def reformat_ps_info(self, proc_info):
        """
        Takes a structure of process information and re-organizes it into
        a list of dictionaries, where each dict is a process.
        :param proc_info:
        :return: a list of dicts
        """
        # Determine if the sort field needs to reverse the order of the sort
        _reverse = next((header['sort_order'] for header in self.headers if header['shortname'] == self._sort), False)
        if self.debug:
            util.write_out("sorting on {0} and reverse is {1}".format(self._sort, _reverse))
        return sorted(proc_info, key=itemgetter(self._sort), reverse=_reverse)


from operator import itemgetter
from anno1800.anno1800 import anno1800
import pyautogui as pag
import multiprocessing as mltprocess
import keyboard
import time
import os
import argparse
from logger import log

# from anno1800.store import store
# pyautogui.move(200, 200, 2)

class event:
    def __init__(self, flag=False):
        self.flag = flag

    def set(self, flag):
        self.flag = flag

    def get(self):
        return self.flag

    def toggle(self):
        if self.flag:
            self.flag = False
        else:
            self.flag = True

def auto_buy(items, type_store, skip_trade_button=True):
    from anno1800.anno1800 import anno1800
    from anno1800.detect import detect
    work_done = event()
    log.debug('items=[' + ','.join(items) + '] | type_store=' + type_store)

    anno1800 = anno1800()

    list_items = anno1800.format_items(items, type_store)

    det_items = detect(list_items
                    , anno1800.stores[type_store].region['list_items'], threshold_conf=0.90, delay=0.2)
    det_store = detect(anno1800.stores[type_store].image
                    , anno1800.stores[type_store].region['id'])
    det_state_no_inventory = detect(anno1800.stores[type_store].image_states['no_inventory']
                                , anno1800.stores[type_store].region['no_inventory'])
    det_state_out_of_stock = detect(anno1800.stores[type_store].image_states['out_of_stock']
                                , anno1800.stores[type_store].region['out_of_stock'])
    det_ship_trade_button = detect(anno1800.facilities['ship'].image_states['trade']
                                , anno1800.facilities['ship'].region['trade'])

    while(True):
        if work_done.get():
            break
        if det_ship_trade_button.find() or skip_trade_button:
            log.debug('detected the button of the ship trade')
            time.sleep(0.2)
            if not skip_trade_button:
                anno1800.windows.click_button('ship', 'trade')
            anno1800.windows.click_button(type_store, 'tab_trade_item')
            log.debug('trying to detect...')
            while(True):
                if not det_state_no_inventory.find():
                    if not det_state_out_of_stock.find():
                        dets = det_items.get_coords()
                        if len(dets) > 0:
                            dets.sort(key=itemgetter(2, 1), reverse=True)
                            for det in dets:
                                log.debug(det)
                                anno1800.windows.click(det[1:], delay=0.0)
                                anno1800.windows.click_button(type_store, 'trade_done', delay=0.1)
                            dets = []
                        anno1800.windows.click_button(type_store, 'dice')
                    else:
                        log.info('detected out of stock')
                        anno1800.windows.click_button(type_store, 'exit')
                        work_done.set(True)
                        break
                else:
                    log.info('detected no inventory')
                    anno1800.windows.click_button(type_store, 'exit')
                    work_done.set(True)
                    break
        else:
            log.debug('waiting for the button of the ship trade...')
            time.sleep(1)
    log.debug('process done')
    return

class multiprocess:
    def __init__(self, args, daemon=True):
        self.flag_started = False
        self.flag_dead = False
        self.name = args[0]
        self.args = args[1:]
        self.process = mltprocess.Process(target=auto_buy
                                        , name=self.name
                                        , args=self.args
                                        , daemon=daemon)
        log.debug('created the process: Process=' + self.name)

    def is_alive_c(self):
        return self.process.is_alive()

    def is_dead(self):
        return self.flag_dead

    def is_started(self):
        return self.flag_started

    def start(self):
        if not self.process.is_alive():
            log.debug('starting the process: Process=' + self.name)
            self.process.start()
            self.flag_started = True

    def kill(self):
        if self.process.is_alive():
            log.debug('stopping the process: Process=' + self.name)
            self.process.kill()
            self.process.join()
            self.flag_dead = True

if __name__ == '__main__':
    # initializing a program
    parser = argparse.ArgumentParser()
    parser.add_argument('--debug', action='store_true', help='set the level of logging', default=False)
    opt = parser.parse_args()
    log.setLevel(10 if opt.debug else 20) # 10:debug | 20:info | 30:warning | 40:error | 50:critical
    if opt.debug == 1:
        print(opt, '\n')
    log.info('initialized the program')

    escape = event()
    # user interfaces
    while True:
        escape.set(False)
        # input operation
        print('\nSyntax: {item1},{item2}... <blank> {target} [exit:Exit program]')
        operation = input('> input the operation: ')

        # processing exit operation
        if operation == 'exit':
            log.info('Program will exit')
            break

        # processing operation
        list_operation = operation.split(' ')
        if isinstance(list_operation, list) and len(list_operation) >= 2:
            items = list_operation[0].split(',')
            target = list_operation[1]
            log.debug('operation: items=[' + list_operation[0] + '] | target=' + list_operation[1])
        else:
            log.error('invalid syntax for operation: Operation=' + operation)
            continue

        # check all symbol in item operation
        # check valid item with registered items as anno1800()
        if not items[0] == '*':
            for item in items:
                if not anno1800.is_item(None, item):
                    log.error('invalid the item defined as operation: item=' + item)
                    continue

        # check valid store with registered stores as anno1800()
        # TO-DO: must be separated a store and a target
        if not anno1800.is_store(None, target):
            log.error('invalid the store defined as operation: store=' + target)
            continue

        while True:
            if escape.get():
                process.kill()
                log.info('Terminated the operation')
                break
            # create a process and register a hotkey
            process = multiprocess(['auto_buy', items, target])
            keyboard.add_hotkey('f1', lambda: process.start())
            keyboard.add_hotkey('f5', lambda: process.kill())
            keyboard.add_hotkey('f9', lambda: escape.set(True))
            log.info('Press the key: [F1]:Start | [F5]:Stop | [F9]:Exit')
            time.sleep(2)
            while True:
                try:
                    if escape.get():
                        break
                    if (process.is_alive_c() == False) and process.is_started():
                        try:
                            log.debug('try to kill the process: is_alive_c=' + process.is_alive_c() + " | is_started=" + process.is_started())
                            process.kill()
                        except:
                            log.warning('cannot terminate the process: is_alive_c=' + process.is_alive_c() + " | is_started=" + process.is_started())

                        log.info('Work done')
                        break
                except:
                    log.warning('Work done with exception')
                    break

            # t4_guard,t4_propeller equipments
            # t4_attack_abraheem,t4_attack_roo,t4_harbor_speed,t4_investor people


#! /usr/bin/python3
import os
import json
import praw
import time
import tweepy
import configparser

# Load custom settings from PRAW official file
if os.path.exists('./praw.ini'):
    praw_config = configparser.RawConfigParser()
    praw_config.read_file(open('./praw.ini'))

def save_tweets(posts):
    with open('posts.json', 'w') as outfile:
        print("\nPost saved")
        json.dump(posts, outfile, indent=4)

reddit = praw.Reddit(str(praw_config.get('user', 'nickname')))

print("Logged in as: " + str(reddit.user.me()))
print()

user = reddit.redditor(str(praw_config.get('user', 'nickname')))
posts = {}

# Load settings for tweepy
if os.path.exists('./tweepy.ini'):
    tweepy_config = configparser.RawConfigParser()
    tweepy_config.read_file(open('./tweepy.ini'))
else:
    print('tweepy.ini missing')

# Authenticate to Twitter
auth = tweepy.OAuthHandler(str(tweepy_config.get('tweepy', 'consumer_key')), str(tweepy_config.get('tweepy', 'consumer_secret')))
auth.set_access_token(str(tweepy_config.get('tweepy', 'access_token')), str(tweepy_config.get('tweepy', 'access_secret')))

# Create API object
api = tweepy.API(auth)

if os.path.exists('./posts.json'):
    print("File posts found")
    with open('./posts.json') as json_file:
        posts = json.load(json_file)
        print("Posts item: " + str(len(posts)))

excludes = str(praw_config.get('user', 'exclude'))
excludes = excludes.split(',')

for post in user.upvoted(limit=100):
    if post.id not in posts:
        tweet_it = True
        print("Missing " + post.id + " post")

        for exclude in excludes:
            if exclude.lower() == str(post.subreddit).lower():
                tweet_it = False
                break

        if 'crosspost_parent_list' in vars(post):
            if str(post.author.name).lower() == praw_config.get('user', 'nickname'):
                print("Crosspost of the user that is also the author, avoid publishing - ignored")
                continue

        if 'u_' + str(praw_config.get('user', 'nickname')) == str(post.subreddit).lower():
            continue

        if tweet_it:
            posts[post.id] = {'title': post.title, 'subreddit': str(post.subreddit).lower(), 'permalink': post.permalink}
            # Create a tweet
            print('Posting new tweet')
            save_tweets(posts)
            api.update_status(post.title + ' via /r/' + str(post.subreddit) + "\nhttps://www.reddit.com" + post.permalink)
            # Wait 10 minutes before tweet it
            time.sleep(600)


# -*- coding:utf8 -*-

'''
Single model may achieve LB scores at around 0.29+ ~ 0.30+
Average ensembles can easily get 0.28+ or less
Don't need to be an expert of feature engineering
All you need is a GPU!!!!!!!

The code is tested on Keras 2.0.0 using Tensorflow backend, and Python 2.7
'''

########################################
## import packages
########################################
import os
import re
import csv
import codecs
import numpy as np
import pandas as pd

from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from string import punctuation
from gensim.models import KeyedVectors
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model
from keras.callbacks import EarlyStopping, ModelCheckpoint

from config import BASE_DIR, EMBEDDING_FILE, TRAIN_DATA_FILE, TEST_DATA_FILE, MAX_SEQUENCE_LENGTH, MAX_NB_WORDS, EMBEDDING_DIM, VALIDATION_SPLIT, STAMP
from config import num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act, re_weight
from config import word2vec
from config import text_to_wordlist
from model import *

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

if __name__ == '__main__':
    texts_1 = []
    texts_2 = []
    labels = []
    with codecs.open(TRAIN_DATA_FILE, encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=',')
        header = next(reader)
        for values in reader:
            texts_1.append(text_to_wordlist(values[3], False, False))
            texts_2.append(text_to_wordlist(values[4], False, False))
            labels.append(int(values[5]))
            # break
    print('Found %s texts in train.csv' % len(texts_1))

    test_texts_1 = []
    test_texts_2 = []
    test_ids = []
    with codecs.open(TEST_DATA_FILE, encoding='utf-8') as f:
        reader = csv.reader(f, delimiter=',')
        header = next(reader)
        for values in reader:
            test_texts_1.append(text_to_wordlist(values[1], False, False))
            test_texts_2.append(text_to_wordlist(values[2], False, False))
            test_ids.append(values[0])
            # break
    print('Found %s texts in test.csv' % len(test_texts_1))

    tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
    tokenizer.fit_on_texts(texts_1 + texts_2 + test_texts_1 + test_texts_2)

    sequences_1 = tokenizer.texts_to_sequences(texts_1)
    sequences_2 = tokenizer.texts_to_sequences(texts_2)
    test_sequences_1 = tokenizer.texts_to_sequences(test_texts_1)
    test_sequences_2 = tokenizer.texts_to_sequences(test_texts_2)

    word_index = tokenizer.word_index
    print('Found %s unique tokens' % len(word_index))

    data_1 = pad_sequences(sequences_1, maxlen=MAX_SEQUENCE_LENGTH)
    data_2 = pad_sequences(sequences_2, maxlen=MAX_SEQUENCE_LENGTH)
    labels = np.array(labels)
    print('Shape of data tensor:', data_1.shape)
    print('Shape of label tensor:', labels.shape)

    ########################################
    ## prepare embeddings
    ########################################
    print('Preparing embedding matrix')

    nb_words = min(MAX_NB_WORDS, len(word_index))+1

    embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))
    for word, i in word_index.items():
        if word in word2vec.vocab:
            embedding_matrix[i] = word2vec.word_vec(word)
    print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))

    ########################################
    ## sample train/validation data
    ########################################
    #np.random.seed(1234)
    perm = np.random.permutation(len(data_1))
    idx_train = perm[:int(len(data_1)*(1-VALIDATION_SPLIT))]
    idx_val = perm[int(len(data_1)*(1-VALIDATION_SPLIT)):]

    data_1_train = np.vstack((data_1[idx_train], data_2[idx_train]))
    data_2_train = np.vstack((data_2[idx_train], data_1[idx_train]))
    labels_train = np.concatenate((labels[idx_train], labels[idx_train]))

    data_1_val = np.vstack((data_1[idx_val], data_2[idx_val]))
    data_2_val = np.vstack((data_2[idx_val], data_1[idx_val]))
    labels_val = np.concatenate((labels[idx_val], labels[idx_val]))

    weight_val = np.ones(len(labels_val))
    if re_weight:
        weight_val *= 0.472001959
        weight_val[labels_val==0] = 1.309028344

    ########################################
    ## define the model structure
    ########################################
    # model = lystdo(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'lystdo'
    # model = lstm_add(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'lstm_add'
    # model = lstm_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'lstm_multiply'
    # model = bilstm_concat(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bilstm_concat'
    # model = bigru_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bigru_multiply'

    # model = gru_concat(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'gru_concat'
    # model = gru_add(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'gru_add'
    # model = gru_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'gru_multiply'
    # model = gru_add_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'gru_add_multiply'
    # model = bigru_concat(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bigru_concat'
    # model = bigru_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bigru_multiply'
    # model = bi2gru_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bi2gru_multiply'
    # model = bigru_multiply_no_dense(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bigru_multiply_no_dense'
    # model = bigru_add_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bigru_add_multiply'

    model = cnn_multiply(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    model_name = 'cnn_multiply'

    # model = bilstm_distance_angle(nb_words, EMBEDDING_DIM, embedding_matrix, MAX_SEQUENCE_LENGTH, num_lstm, num_dense, rate_drop_lstm, rate_drop_dense, act)
    # model_name = 'bilstm_distance_angle'

    early_stopping =EarlyStopping(monitor='val_loss', patience=3)
    bst_model_path = './models/' + model_name + STAMP + '.h5'
    model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)

    ########################################
    ## add class weight
    ########################################
    if re_weight:
        class_weight = {0: 1.309028344, 1: 0.472001959}
    else:
        class_weight = None

    hist = model.fit([data_1_train, data_2_train], labels_train,
                     validation_data=([data_1_val, data_2_val], labels_val, weight_val),
                     epochs=200, batch_size=512, shuffle=True,
                     class_weight=class_weight, callbacks=[early_stopping, model_checkpoint])

    print(bst_model_path)
    # model.load_weights(bst_model_path)


# -*- coding: utf-8 -*-
"""
Let's fit a Gaussian process without the explicit solution'
"""

import numpy as np
from scipy.stats import multivariate_normal
import matplotlib.pyplot as plt
import math

def target_func(x):
    omega = 1
    return np.sin(2*np.pi*omega*x)

def make_data(train_points,noise_param):
    return target_func(train_points)+np.random.normal(loc=0,scale=noise_param,size=len(train_points))

def kernel(x,y):
    eps = 10**(-12)
    return np.exp(-abs(x-y)**2/(2*0.2**2))+eps*(abs(x-y)<0.01)

def compute_prior_covariance(points):
    N = len(points)
    covar = np.zeros((N,N))
    for i in range(N):
        for j in range(N):
            covar[i][j] = kernel(points[i],points[j])
    return covar

def set_up_prior(test_points):
    mean = np.array([0]*len(test_points))
    covar = np.array(compute_prior_covariance(test_points))
    return mean,covar

def matching_squared_sum(u,y,mesh_size):
    total = 0
    for x in y.keys():
        total += (y[x]-u[int(x/mesh_size)])**2
    return total

def unnormalized_posterior(u,y,prior_mean,prior_covar,noise_param,mesh_size):
    prior_eval = multivariate_normal.pdf(u,mean = prior_mean,cov=prior_covar)
    conditional_eval = np.exp((-len(y.keys())/(2*noise_param**2))*matching_squared_sum(u, y, mesh_size))
    return prior_eval*conditional_eval

#def gaussian_kernel_sample(x):
 #   z = np.random.normal(loc = x,scale = 0.1)
  #  return z

def crank_nicholson(x,prior_covar):
    beta = 0.12
    return np.sqrt(1-beta**2)*x+beta*np.random.multivariate_normal([0]*len(x),prior_covar)

def rho(x,z,y,prior_mean,prior_covar,noise_param,mesh_size):
    #z is the new proposal, x is the current state, y is the data
    proposal_score = np.exp((1/(2*noise_param**2))*(matching_squared_sum(x, y,mesh_size)-matching_squared_sum(z, y, mesh_size)))
    return min(proposal_score,1)

def metropolis_hastings_sample(N,y,prior_mean,prior_covar,noise_param,mesh_size,mh_kernel):
    accepted = 0
    rejected = 0
    test_points = np.linspace(0,1,21)
    x = prior_mean
    samples = []
    i = 0
    while i < N:
        z = mh_kernel(x,prior_covar)
        if rho(x,z,y,prior_mean,prior_covar,noise_param,mesh_size) > np.random.random():
            x = z
            samples.append(x)
            accepted += 1
        else:
            samples.append(x)
            rejected += 1
        i += 1
    return samples

def gaussian_kernel(x,y):
    eps = 0 #10**(-11)
    return 0.5*np.exp(-(x-y)**2/(2*0.3**2))+eps*(abs(x-y)<0.01)

def analytic_fitting(data,test_points,noise_param):
    def K(A,B):
        kernel_matrix = list(np.zeros((len(A),len(B))))
        for i in range(len(A)):
            for j in range(len(B)):
                kernel_matrix[i][j] = kernel(A[i],B[j])
        return np.array(kernel_matrix)
    y = [data[key] for key in data.keys()]
    noise_matrix = noise_param**2*np.identity(6)
    X = np.linspace(0,1,6)
    return K(test_points,X)@np.linalg.inv((K(X,X)+noise_matrix))@y

def mse(x,y):
    return np.sum((x-y)**2)/len(x)

def GR(chains):
    N = chains.shape[1]
    chain_vars = np.var(chains,axis=1,ddof=1)
    avg_vars = np.mean(chain_vars,axis=0)
    chain_means = np.mean(chains,axis=1)
    B_n = np.var(chain_means,axis=0,ddof=1)
    sig_sqr = ((N-1)/N)*avg_vars+B_n
    R = np.sqrt(np.divide(sig_sqr,avg_vars))
    return np.mean(R)

def main():

    mesh_size = 0.05
    train_point_count = 6
    noise_param = 0.25
    train_points = np.linspace(0,1,train_point_count)
    y =  dict(zip(train_points,make_data(train_points, noise_param)))
    samples_per_chain = 2000
    burn_in = 1000
    n_chains = 10

    test_point_count = int(1/mesh_size+1)
    test_points = np.linspace(0,1,test_point_count)
    prior_mean, prior_covar = set_up_prior(test_points)

    chains = []
    for i in range(n_chains):
        chain=metropolis_hastings_sample(samples_per_chain,y,prior_mean,prior_covar,noise_param,mesh_size,crank_nicholson)[burn_in:]
        chains.append(chain)
    chains = np.array(chains)

    y_keys = y.keys()
    true_y = [target_func(x) for x in test_points]
    plt.scatter(test_points,true_y,label = "True function, y = sin(2\u03C0x)",marker="o",s=20)
    plt.scatter(y_keys,[y[key] for key in y_keys],label="Data",marker="x",s=20)

    #plt.scatter(test_points,prior_mean)
    #print(chains.shape)
    posterior_experimental_mean = np.mean(chains,axis=(0,1))
    #print(posterior_experimental_mean)

    #plt.scatter(test_points,posterior_samples[-1],label="Posterior sample")
    plt.scatter(test_points,posterior_experimental_mean,label="Posterior MH mean",marker="s",s=20)

    analytic_mean = analytic_fitting(y,test_points,noise_param)
    #print(np.linalg.norm(posterior_experimental_mean-analytic_mean))
    plt.scatter(test_points,analytic_mean,label="Analytic mean",marker="*",s=20)
    plt.legend()
    plt.xlabel("x")
    plt.ylabel("y")
    plt.title("Metropolis-Hastings Mean GP")

    print(GR(chains))
    #analytic_sample_errors = [mse(analytic_mean,posterior_sample) for posterior_sample in posterior_samples]

    #plt.scatter([i for i in range(len(analytic_sample_errors))],analytic_sample_errors)

main()


from DbHelper import DbHelper
from pydantic import BaseModel
from ResponseModel import *

class ProductModel(BaseModel):
    productCode: str | None = None
    productName: str | None = None
    productLine: str | None = None
    productScale: str | None = None
    productVendor: str | None = None
    productDescription: str| None = None
    quantityInStock: int | None = None
    buyPrice: float | None = None
    MSRP: float| None = None

class DaoProducts:
    def __init__(self):
        self.db = DbHelper()

    def get_all_products(self):
        try:
            products = []
            query = "SELECT * FROM products;"
            rows = self.db.exe_query(query)
            for row in rows:
                products.append(ProductModel(productCode = row[0],
                                               productName=  row[1],
                                               productLine = row[2],
                                               productScale = row[3],
                                               productVendor = row[4],
                                               productDescription =  row[5],
                                               quantityInStock = row[6],
                                               buyPrice = row[7],
                                               MSRP = row[8]
                                               ))
            return response (
                message = 'OK',
                result = products
            )
        except Exception as error:
            return response (
                message = 'KO',
                result = error
            )

    def add_product(self, product):
        try:
            self.db.exe_query(f"INSERT INTO products (productCode, productName, productLine, productScale, productVendor, productDescription, quantityInStock , buyPrice, MSRP) VALUES ('{product.productCode}', '{product.productName}', '{product.productLine}', '{product.productScale}', '{product.productVendor}', '{product.productDescription}', '{product.quantityInStock }', '{product.buyPrice}', '{product.MSRP}');commit")
        except Exception as e:
            print('errore durante la connessione')
            print(e)


import pandas as pd
from scipy import optimize
import numpy_financial as npf
from datetime import date

def bondPrice(faceValue,coupon,daysUntilMaturity,daysBetweenCoupon,ytm):
    pv=(faceValue+coupon)/((1+ytm)**(daysUntilMaturity/365))
    daysUntilMaturity-=daysBetweenCoupon
    while daysUntilMaturity>0:
        pv+=coupon/((1+ytm)**(daysUntilMaturity/365))
        daysUntilMaturity-=daysBetweenCoupon
    return(pv)

file= pd.read_excel("Debt_TB2.xlsx", sheet_name=1)
todayDate=file.iloc[1,4]
debtPortfolio = file.iloc[4:,2:]
debtPortfolio.columns = ['S.No.','ISIN','SecurityName','InstrumentType','Maturity','Price','Yield','ModifiedDuration','Rating','RatingChange','ValuationTriggered','TriggerDate','FaceValue','MacaulayDuration','Period']
ytms=[]
for index, row in debtPortfolio.iterrows():
    price=row['Price']
    yld=row['Yield']
    period=row['Period']
    par=row['FaceValue']
    timeToMaturity=(row['Maturity']-todayDate).days
    print(timeToMaturity)
    get_yield= lambda int_rate: bondPrice(par,yld*par*period*30/365,timeToMaturity,period*30,int_rate) - price
    ytm=optimize.newton(get_yield, yld)
    ytms.append(ytm)
debtPortfolio['ytm']=ytms
print(debtPortfolio)
print(ytms)
debtPortfolio.to_csv("YTM.csv")


# import modules
from utils import decorator
from telegram.error import TelegramError
from errors.log import log
from config import log_channel
from telegram.constants import ParseMode

@decorator.general_admin
#@decorator.cancellacomandi
async def init(update, context):
    """
    This command is executed sending the command:
        /autoban reason

    where 'reason' is a message to explain to the user the reason of the ban

    The command must be used in reply to the user's message. The bot sends a message
    in the chat to the user providing the 'reason'. After 60 seconds the user is banned.
    """
    # decode command
    reason = update.message.text.replace('/autoban ', '').strip()
    # get username
    name = "@" + update.message.reply_to_message.from_user.username \
        if update.message.reply_to_message.from_user.username is not None \
        else update.message.reply_to_message.from_user.name

    # set and send message to the user
    msg = f"{name}, sarai bannato dal gruppo fra 60 secondi per questo motivo:\n\n{reason}"
    await update.message.reply_text(#chat_id=update.message.chat_id,
                             text=msg,)
                             #reply_to_message_id=update.message.reply_to_message.message_id)
    await context.bot.send_message(chat_id=log_channel ,
                                   text="ðŸ”´ Ban process #ban\n\nChat: {}\nChat_id: {}"
                                        "\nUser_id: {}"
                                        "\nName: {}"
                                        "\nUsername: @{}"
                                        "\n\nPerformed by admin: @{}"
                                        "\n\nBan reason: {}".format(update.message.chat.title,
                                                                           update.message.chat_id,
                                                                           update.message.reply_to_message.from_user.id,
                                                                           update.message.reply_to_message.from_user.first_name,
                                                                           update.message.reply_to_message.from_user.username,
                                                                           update.message.from_user.username,
                                                                           reason),
                                   parse_mode=ParseMode.HTML)
    await context.bot.delete_message(update.message.chat_id, update.message.message_id)

    # set delayed action
    async def delayed_ban(context, update=update):
        try:
            await context.bot.ban_chat_member(chat_id=update.message.chat.id,
                                        user_id=update.message.reply_to_message.from_user.id)

        except TelegramError:
            log("an error occurred [AUTOBAN] function")
            await update.message.reply_text(text="Error during autoban operation")

    # schedule action
    seconds = 60
    context.job_queue.run_once(delayed_ban, seconds)


from flask import Flask, render_template, request, json, redirect, session
from flask import Markup
import requests

app = Flask(__name__)
app.config["DEBUG"] = False

@app.route("/")
def home():
    return render_template('index.html')

@app.route("/chart")
def chart():
    r = requests.get('https://api.airtable.com/v0/appAIHCBijmOXqMWG/membership_points?api_key=keyHwnj1ElCshCfmN&sortField=_createdTime&sortDirection=desc')
    dict1 = r.json()
    dict2 = {}
    dataset = []
    name_list = []
    point_list = []
    for i in dict1['records']:
         dict2 = i['fields']
         dataset.append(dict2)
    for item in dataset:
        name_list.append(item.get('name'))
        point_list.append(item.get('point'))
    return render_template('chart.html', entries = zip(name_list, point_list))

if __name__ == '__main__':
   app.run(debug = True)


import json
import os
import re
import signal
import unicodedata
import urllib
from contextlib import contextmanager
import subprocess
from datetime import datetime, timedelta
from time import mktime, strptime
import time
import html2text as html2text
from bs4 import BeautifulSoup, UnicodeDammit
from dateutil import parser
import PyPDF2 as PyPDF2
import magic
from googlesearch import search
import logging, logging.handlers
import src.params as params
from src.evaluate_similarity import compute_best_doc

class Log(object):
    def __init__(self, dossier, nomFichier, niveau=logging.DEBUG):
        super(Log, self).__init__()

        self.__logger__ = logging.getLogger(nomFichier)
        self.__logger__.setLevel(niveau)

        format = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s', datefmt='%d-%m-%Y %H:%M:%S')
        fichierLog = logging.handlers.RotatingFileHandler("{0}{1}.log".format(dossier, nomFichier), 'a', 1000000, 1)

        fichierLog.setLevel(niveau)
        fichierLog.setFormatter(format)
        self.__logger__.addHandler(fichierLog)

        console = logging.StreamHandler()
        console.setLevel(niveau)
        self.__logger__.addHandler(console)

    def info(self, message):
        self.__logger__.info(message)

    def debug(self, message):
        self.__logger__.debug(message)

    def warning(self, message):
        self.__logger__.warning(message)

    def error(self, message):
        self.__logger__.error(message)

    def critical(self, message):
        self.__logger__.critical(message)

    def exception(self, message):
        self.__logger__.exception(message)

    def close(self):
        for handler in  self.__logger__.handlers[:] :
            handler.close()
            self.__logger__.removeHandler(handler)

class TextCleaner():
    """docstring for TextCleaner."""
    def __init__(self):
        """ Expressions régulières """
        self.match_alpha = re.compile(r'\w{3,}') # Existence d'un mot d'au moins 3 lettres
        self.match_formfeed = re.compile(r'\f') # Saut de page
        self.match_useless_line = re.compile(r'^[^\w•]+$|^[\d\. ]+$', re.MULTILINE) # Ligne ne contenant aucun caractère alphanumérique

        self.match_bad_nl = re.compile(r'([^\.?!\n:])\n+(?![IVX\d]+[\.\)]|ANNEXE)([\w\(«\"=<>])') # Mauvais saut de ligne
        self.match_bad_nl2 = re.compile(r'(\.{4,} ?\d+) (\w)') # Mauvais saut de ligne
        self.make_paragraph = re.compile(r'\.\n(?=\w)') # On sépare mieux les paragraphes
        self.match_toomuch_nl = re.compile(r'\n{3,}') # Nouvelles lignes surnuméraires
        self.match_begin_end_nl = re.compile(r'^\n+|\n+$') # Nouvelles lignes au début et à la fin

        self.match_begin_end_space = re.compile(r'^[ \t]+|[ \t]+$', re.MULTILINE) # Espace ou tabulation en début et fin de ligne
        self.match_toomuch_spaces = re.compile(r' {2,}|\t+') # Espaces surnuméraires et tabulations

        self.match_link = re.compile(r'!?\[.*?\]\(.*?\)|https?://[^ ]+', re.DOTALL) # Lien issu de la conversion depuis HTML
        self.match_cesure = re.compile(r'(\w)-(\w)') # Césure
        self.match_stuckwords = re.compile(r'(/\d{2,4}|[a-z])([A-ZÉÈÀÔ])') # Dates et mots collés

        self.match_odd = re.compile(r'[�●§\\\|]+|\(cid:\d+\)|(?<=- )W ') # caractère bizarre
        self.match_accent1 = re.compile(r'Ã©') # é
        self.match_accent2 = re.compile(r'Ã¨') # è
        self.match_accent3 = re.compile(r'Ã') # à
        self.match_puce1 = re.compile(r'')
        self.match_puce2 = re.compile(r'[]')
        self.match_diam = re.compile(r'diam\.')
        self.match_apostrophe = re.compile(r'’')

    def clean(self, text):
        # Remplacement des espaces insécables
        text = text.replace(u'\xa0', ' ')

        text = self.match_link.sub('', text)
        text = self.match_formfeed.sub('\n\n', text)
        text = self.match_useless_line.sub('\n', text)

        text = self.match_diam.sub('diamètre', text)
        text = self.match_accent1.sub('é', text)
        text = self.match_accent2.sub('è', text)
        text = self.match_accent3.sub('à', text)
        text = self.match_puce1.sub('*', text)
        text = self.match_puce2.sub('-', text)
        text = self.match_odd.sub('', text)
        text = self.match_apostrophe.sub('\'', text)

        text = self.match_begin_end_space.sub('', text)
        text = self.match_bad_nl.sub(r'\g<1> \g<2>', text)
        # On double la réparation des lignes, meilleurs résultats
        text = self.match_bad_nl.sub(r'\g<1> \g<2>', text)
        text = self.match_bad_nl2.sub(r'\g<1>\n\g<2>', text)
        text = self.make_paragraph.sub('.\n\n', text)
        text = self.match_stuckwords.sub(r'\g<1> \g<2>', text)
        text = self.match_toomuch_spaces.sub(' ', text)
        text = self.match_toomuch_nl.sub('\n\n', text)
        text = self.match_begin_end_nl.sub('', text)

        return text

    def exists_alpha(self, text):
        """ Contrôle l'existence de caractères alphanumériques """
        return self.match_alpha.search(text) is not None

def query_generator(city, motscles, logger, site):
    logger.info("Génération des requêtes")

    requetes_effectuees = []

    with open("{0}{1}/.sauvegarde.txt".format(params.CHEMIN_RESULTATS, city)) as file:
        to_be_queried = file.readlines()

    for i in range(0, len(motscles), 1):
        # motscles_couple = motscles[i].split("+")
        # Si aucun site n'est spécifié alors on n'exclut
        # seulement ceux définis dans les constants
        if site == "":
            site_or_not_sites = params.NOT_SITES
        # Sinon on cherche uniquement sur le site spécifié
        else:
            site_or_not_sites = "site:" + site
        query = "\"{0}\" AND {1}".format(city, motscles[i])
        if not any(query in s for s in to_be_queried):
            yield query

def format_word(mot):
    """
        Supprime les retour charriot et remplace les espace par un ?.

        @type  mot: String.
        @param mot: Mot à formaté.

        @rtype: String.
        @return: Mot formaté.
    """
    liste_caracteres = dict()

    liste_caracteres["\n"] = ""
    liste_caracteres[" "] = "?"

    for i in liste_caracteres :
        mot = mot.replace(i, liste_caracteres[i])

    return mot

class TimeoutException(Exception): pass

@contextmanager
def sleep_time(seconds):
    def signal_handler(signum, frame):
        raise TimeoutException
    signal.signal(signal.SIGALRM, signal_handler)
    signal.alarm(seconds)
    try:
        yield
    finally:
        signal.alarm(0)

def convert_pdf_to_txt(src_file_path):
    """
        Appel externe à pdftotext.

        -q : pas de message d'erreur dans la sortie.
         - : envoie la sortie dans la console au lieu d'un fichier texte.

        Capture de la sortie texte.

        @type  src_file_path: String.
        @param src_file_path: Chemin du fichier source.

        @rtype: String.
        @return: Texte brut.
    """
    completed_process = subprocess.run(["pdftotext", "-q", src_file_path, "-"], stdout=subprocess.PIPE)
    return completed_process.stdout.decode('utf-8')

def norm_string(string):
    """ On supprime les accents et on remplace les caractères spéciaux par un tiret bas """
    new_string = unicodedata.normalize('NFKD', string).encode('ascii', 'ignore').decode('utf-8')
    return re.sub(r'[^\w\[\]\./]+', '_', new_string)

def format_to_iso_date(pdfdate):
    datestring = ''
    if pdfdate !='':
        if len(pdfdate)==23:
            datestring = pdfdate[2:-7]
        elif len(pdfdate)==17:
            datestring = pdfdate[2:-1]
        elif len(pdfdate)==21:
            datestring = pdfdate[:-7]
        ts = strptime(datestring, "%Y%m%d%H%M%S")# "%Y-%m-%dT%H:%M:%S.%fZ" )#
        dt = datetime.fromtimestamp(mktime(ts))
        new_format = dt.isoformat()
    else:
        new_format = "no_date"
    return new_format

def get_pdf_info(f, logger):
    fd = PyPDF2.PdfReader(f, 'rb')
    doc_info = fd.metadata
    #print("doc_info :", doc_info)
    if '/ModDate' in doc_info:
        val_ = doc_info['/ModDate']
        try:
            val_ = format_to_iso_date(val_)
        except:
            logger.exception("cannot format the extracted date.\n")
    elif '/CreationDate' in doc_info:
        val_ = doc_info['/CreationDate']
        try:
            val_ = format_to_iso_date(val_)
        except:
            logger.exception("cannot format the extracted date.\n")
    else:
        val_ = None # 'no_date'

    if '/Title' in doc_info:
        title = doc_info['/Title']
    else:
        title = None # 'no_title'
    #print("title :", str(title))
    #print("date :", str(parser.parse(val_).date()))
    return str(title), str(parser.parse(val_).date())

def extract_date(text):
    # Motif : JJ/MM/AAAA
    match_date = re.search(r'(\d{2})/(\d{2})/(\d{4})', text)
    # Motif : JJ mois AAAA
    match_date2 = re.search(r'(\d{1,2}) (\w+) (\d{4})', text)

    if match_date:
        date = datetime(
            int(match_date[3]), # Année
            int(match_date[2]), # Mois
            int(match_date[1])  # Jour
        )

        return date

    elif match_date2:
        mapping_month = {
            'janvier': 1,
            'février': 2,
            'mars': 3,
            'avril': 4,
            'mai': 5,
            'juin': 6,
            'juillet': 7,
            'août': 8,
            'septembre': 9,
            'octobre': 10,
            'novembre': 11,
            'décembre': 12,
        }

        month = match_date2[2].lower()

        if month in mapping_month:
            date = datetime(
                int(match_date2[3]), # Année
                mapping_month[month], # Mois
                int(match_date2[1])  # Jour
            )

            return date

def less_html(html_doc):
    """ Prend du code HTML en entrée et retourne un code épuré de certaines balises """
    soup = BeautifulSoup(html_doc, 'html.parser')
    # Regex pour matcher les attributs contenant ces termes
    bad_classes = re.compile(r'menu|head|publici|share|social|button|alert|prev|next|foot|tags|label|sidebar|author|topics|contact|modal|nav|snippet|register|aside|logo|bandeau|immobilier', re.IGNORECASE)
    # Suppression des espaces ou des sauts de ligne au début et à la fin du titre
    title = re.sub(r'^\s|\s$', '', soup.find('title').text)

    # Dictionnaire des métadonnées
    metadata = {}
    metadata['title'] = title
    # Recherche d'une éventuelle balise dont la classe contient "date"
    # En principe la première date est la date de publication
    bloc_date = soup.find(class_=re.compile(r'date', re.IGNORECASE))

    if bloc_date:
        # Recherche du premier motif JJ/MM/AAAA,
        # en principe la date de publication
        metadata['post_date'] = ''
        date = extract_date(bloc_date.text)
        print(">>>> html_date :", str(date))
        if date: metadata['post_date'] = str(parser.parse(str(date)).date())# str(date)# html_date_to_isoformat(repr(date))
#         if date: metadata['post_date'] = html_date_to_isoformat(repr(date))

    for balise in soup.find_all():
        conditions = (
            balise.name == 'head',
            balise.name == 'nav',
            balise.name == 'footer',
            balise.name == 'aside',
            balise.name == 'script',
            balise.name == 'style',
            balise.name == 'a',
            balise.name == 'figure',
            balise.name == 'img',
            balise.name == 'svg',
            balise.name == 'noscript',
            balise.name == 'form',
            balise.name == 'button'
        )

        if any(conditions):
            balise.extract()
        # On ajoute un espace devant chaque span, pour éviter
        # parfois d'avoir des mots collés
        elif balise.name == 'span' and balise.string:
            balise.string = ' ' + balise.string

    for balise in soup.find_all(attrs={'class': bad_classes}):
        balise.decompose()

    for balise in soup.find_all(attrs={'id': bad_classes}):
        balise.decompose()

    for balise in soup.find_all():
        if balise.text == '': balise.extract()

    return metadata, str(soup)

def convert_html_to_txt(src_file_path):
    """
        Conversion à l'aide de html2text.
        Détection automatique de l'encodage (UnicodeDammit).
        On capture la sortie texte.
        @type  src_file_path: String.
        @param src_file_path: Chemin du fichier source.
        @rtype: String.
        @return: Texte brut.
    """
    html_file = open(src_file_path, 'rb').read()
    dammit = UnicodeDammit(html_file) # src_file_path
#     metadata, html_mini = less_html(html_file.decode(dammit.original_encoding))
    metadata, html_mini = less_html(html_file.decode(dammit.original_encoding))
    handler = html2text.HTML2Text()
    handler.ignore_links = True
    handler.ignore_emphasis = True
    text = handler.handle(html_mini)

    return metadata, text

# extract spatial named entities
def SNE_Extract(title_mtd):
    SNE = {}
    dc = params.nlp_model(title_mtd)
    i = 0
    for ent in dc.ents:
        if ent.label_ in ['LOC']:
            SNE['ent'+repr(i)] = repr(ent)
            i+=1
    return SNE

# ajout de meta données suplementaires pour l'intdexation spatiale et temporelle
def enrich_mtd(mtd):
    if 'title' in mtd:
        title_mtd = mtd['title']
        SNE = SNE_Extract(title_mtd)
        mtd['SNE'] = SNE #liste d'ENS

    mtd['TNE'] = {}
    #if 'post_date' in mtd and '$date' in mtd['post_date']:
    #    mtd['TNE']['date'] = mtd['post_date']['$date']
    if "post_date" in mtd:
        mtd['TNE']['date'] = []
        mtd['TNE']['date'] = mtd['post_date']
    #mtd = TNE_extract(mtd)
    return mtd

def sauvegarde_fichier_advanced(ville, url, logger, tc):
    """
		Enregistre un fichier quelqu'il soit.

		@type  ville: String.
		@param ville: Nom de la ville.

		@type  url: String.
		@param url: URL vers le fichier pdf.

		@type  log: Logger.
		@param log: Fichier de log.

		@type  tc: TextCleaner.
		@param tc: Nettoyeur de texte.
	"""
    # On prend l'URL de base pour désigner l'origine du fichier à laquelle on enlève le "www."
    origine = re.sub(r'^www\.', '', urllib.request.urlparse(url).netloc)

    url_split = url.split('/')

    # Extraction du nom du document
    if url_split[-1] == '':
        my_doc = norm_string(url_split[-2])
    else:
        my_doc = norm_string(url_split[-1])

    src_file_path = "{0}{1}/Documents_SRC/[{2}]{3}".format(params.CHEMIN_RESULTATS, ville, origine, my_doc)

#     print(">>>>>>>", url)
#     print("#####", src_file_path)

    if not os.path.isfile(src_file_path):
        req = urllib.request.Request(url, headers={'User-Agent': "Magic Browser"})
        response = urllib.request.urlopen(req)
        read_buffer = response.read()
        mime_type = magic.from_buffer(read_buffer, mime=True)

        source = {
            'file_link': src_file_path,
            'complete_url': url,
            'base_url': origine,
            'open_access': False,
        }

        document = {
            'name': my_doc,
            'mime_type': mime_type,
            'source': source,
            'manually_validated': False,
        }

        # Écriture du fichier téléchargé
        with open(src_file_path, "wb") as fichier:
            fichier.write(read_buffer)
            logger.info("Document sauvegardé.")

        # On détecte le type du document source pour utiliser la conversion adaptée
        # Si le document est un PDF
        if mime_type == 'application/pdf':
            try:
                # Conversion en texte brut
                raw_text = convert_pdf_to_txt(src_file_path)
                logger.info("Document PDF converti en texte brut.")
                title, p_d = get_pdf_info(src_file_path,logger)
                document['text'] = tc.clean(raw_text)
                document['title'] = title
                document['post_date'] = p_d
                logger.info("Texte nettoyé.\n")
                #print("title, p_d :", title, p_d)
            except:
                logger.exception("Erreur lors de la conversion en texte du pdf.\n")

        # Si le document est un HTML (ou autre fichier web)
        elif mime_type == 'text/html':
            try:
                # Conversion en texte brut
                metadata, raw_text = convert_html_to_txt(src_file_path)
                logger.info("Document web converti en texte brut.")
                document['text'] = tc.clean(raw_text)
                document.update(metadata)
                logger.info("Texte nettoyé.\n")

            except:
                logger.exception("Erreur lors de la conversion en texte de la page web.\n")

        else:
            logger.info("Autre type de document téléchargé.\n")

        document = enrich_mtd(document)

        return document

    else:
        logger.info("Le document existe déjà.\n")
        return None

### save database
def save_to_jsonl(mtd,spatial_extent):
#     with open('3m_db.jsonl', 'a+', encoding='utf8') as outfile:
    with open("{0}{1}/urbanisation_herelle.jsonl".format(params.CHEMIN_RESULTATS, spatial_extent), 'a+', encoding='utf8') as outfile:
        #for entry in JSON_file:
        json.dump(mtd, outfile, ensure_ascii=False)
        outfile.write('\n')

def pause(logger, minutes=0.5):
    """
		Effectue une pause.
		@type  log: Logger.
		@param log: Fichier de log.
		@type  minutes: Entier.
		@param minutes: Temps en minute à attendre.
	"""
    date = datetime.now().strftime('%d-%m-%Y:%Hh%M')
    current_time = datetime.now() + timedelta(minutes=minutes)

    logger.info("{0} : Nombre limite de requete atteint. Reprise du programme : {1}".format(date, current_time.strftime(
        '%d-%m-%Y:%Hh%M')))

    while datetime.now() < current_time:
        time.sleep(0.5)

### save database
def db_to_jsonl(mtd, thematic):
    head, tail = os.path.split(thematic)
    #print("tail :", tail)
    #     head,tail.split('.')[0]
    thematic = tail.split('.')[0]# keyword_file
    with open("./documents/" + str(thematic)+"_db.jsonl", 'a+', encoding='utf8') as outfile:
    #with open("{0}{1}/"+thematic+".jsonl".format(params.CHEMIN_RESULTATS, spatial_extent), 'a', encoding='utf8') as outfile:
        #for entry in JSON_file:
        json.dump(mtd, outfile, ensure_ascii=False)
        outfile.write('\n')

def get_doc(spatial_extent,voc_concept,motscles,thematic, logger, site):
    print("starting doc collector...")
    """
		Effecue la recherche.

		@type  ville: String.
		@param ville: Nom de la ville.

		@type  motscles: Liste de string.
		@param motscles: Liste des mots clés à rechercher.

		@type  logger: Logger.
		@param logger: Fichier de log.
	"""

    # Initialisation du nettoyeur de texte
    tc = TextCleaner()

    # Génération de la liste des requêtes
    queries_list = query_generator(format_word(spatial_extent), motscles, logger, site)
    #print('>>#', liste_requetes)
    for requete in queries_list:
        #print('###', requete)
        response = search(requete, lang='fr', stop=10) # stop=10
        # response = bing_search.search(requete)  # stop=10
        try:
            #liste_url = list(response)
            liste_url = set(response)
        except:
            liste_url = []
#         resume = 'resume'
        with (open("{0}{1}/resume.txt".format(params.CHEMIN_RESULTATS, spatial_extent), "a")) as res:
            logger.info("Nouvelle requête : {0}\n".format(requete))
            res.write("Requête : {0}\n".format(requete))
            res.write("Nombre de résultats affichés : {0}\n".format(len(liste_url)))
            res.write("\nListe des résultats\n")
            res.write("\n")
            for url in liste_url:
                if not url.endswith('xml.gz'):
                    logger.info("URL : {0}\n".format(url))
                    res.write("{0}\n".format(url))
                    try:
                        # On temporise chaque requête Google pour ne pas être bloqué
                        with sleep_time(30):
                            document = sauvegarde_fichier_advanced(spatial_extent, url, logger, tc)

                            # On s'assure que le document existe et qu'un texte y est associé
                            if document and 'text' in document:
                                #print("ok")
                                document = compute_best_doc(voc_concept, document)
                                document['source']['type'] = 'web_request'
                                document['source']['raw_request'] = requete
                                document['spatial_extent'] = spatial_extent
                                document['thematic'] = thematic
                                #print("thematic :", thematic)
                                # insert the doc into the Jsonl file
                                db_to_jsonl(document, thematic)
                                logger.info("Document inséré dans l'inventaire.\n")
                    except Exception as e:
                        print(e)
                        logger.info("Erreur pour l'URL : {0}\n".format(url))
                        res.write("Erreur pour l'URL : {0}\n".format(url))
            logger.info("***********************************************************************\n")
            res.write("***********************************************************************\n\n")
        with open("{0}{1}/.sauvegarde.txt".format(params.CHEMIN_RESULTATS, spatial_extent), "a") as f:
            f.write("{0}\n".format(requete))

        pause(logger)

def creation_dossier_resultat(chemin_resultats, ville, log, keyword_file) :
    """
        Création des dossiers de stockage et renvoi des mots clés.

        @type  ville: String.
        @param ville: Nom de la ville.

        @rtype: Liste de string.
        @return: Liste des mots clés à rechercher.
    """
    head, tail = os.path.split(keyword_file)
#     head,tail.split('.')[0]
    thematic = tail.split('.')[0]# keyword_file
    if not os.path.exists(chemin_resultats + ville):
        os.makedirs(chemin_resultats + ville)
        log.info("Création des dossier pour {0}".format(ville))
    if not os.path.exists("{0}{1}/Documents_SRC".format(chemin_resultats, ville)):
        os.makedirs("{0}{1}/Documents_SRC".format(chemin_resultats, ville))
        log.info("Création du dossier Documents_SRC\n")

    with open(keyword_file) as keywords_file:
        words = keywords_file.readlines()
        log.info("Lecture de keywords.txt ... \n")
    open("{0}{1}/.sauvegarde.txt".format(chemin_resultats, ville),"a").close()
    for i in range(0, len(words)):
        words[i] = format_word(words[i])
    return words, thematic

def scrapper(spatial_extent,voc_concept, site=''):
    logger = Log(params.CHEMIN_LOG, 'collecteDeDonnees_{0}'.format(datetime.today().strftime("%d-%m-%y")))
    spatial_extent = spatial_extent.title()
    motscles, thematic = creation_dossier_resultat(params.CHEMIN_RESULTATS, spatial_extent, logger, voc_concept)
    logger.info("Début de la recherche de document concernant la ville de {0}".format(spatial_extent))
    get_doc(spatial_extent,voc_concept, motscles,thematic, logger, site)


import math
import matplotlib.pyplot as plt
import os
import sys

kVals = [5, 10, 20, 50, 100]

def readQrel(qrelFile):
    relevant = {}
    print('reading qrel')

    with open(qrelFile, 'r') as qrel:
        line = qrel.readline()

        while line:
            split = line.strip().split()

            qID, docID, rel = split[0], split[2], split[3]

            if rel != '0':
                if qID in relevant:
                    relevant[qID].append(docID)
                else:
                    relevant[qID] = [docID]

            line = qrel.readline()

    qrel.close()

    return relevant

def readRes(resFile):
    results = {}
    print('reading res')

    with open(resFile, 'r') as res:
        line = res.readline()

        while line:
            split = line.strip().split()

            qID, docID = split[0], split[2]

            if qID in results:
                results[qID].append(docID)
            else:
                results[qID] = [docID]

            line = res.readline()

    res.close()

    return results

def calcNDCG(rel):
    i, dcg = 1, 0.0

    for score in rel:
        if i == 1:
            dcg += score
        else:
            dcg += score / math.log(i)

        i += 1

    return dcg

def calcCutoff(P, R, F1):
    tempP, tempR, tempF1 = {}, {}, {}

    for k in kVals:
        tempP[k] = math.fsum(P[k]) / len(P[k])
        tempR[k] = math.fsum(R[k]) / len(R[k])
        tempF1[k] = math.fsum(F1[k]) / len(F1[k])

    return tempP, tempR, tempF1

def printer(qID, ret, relDocs, relRet, avgPrec, rPrec, dcg, precCutoff, recCutoff, f1Cutoff):
    print('Query ID (num):', qID)
    print('\tRetrieved:', ret)
    print('\tRelevant:', relDocs)
    print('\tRelevant retrieved:', relRet)
    print('Avg prec (non-interpolated):', avgPrec)
    print('nDCG:', dcg)

    print('\n    k      Precision@k   Recall@k    F1-Measure@k')
    print('   ---     -----------   --------    ------------')
    for k in kVals:
        print('{0:4d} docs    {1:.4f}       {2:.4f}        {3:.4f}'.format(k, precCutoff[k], recCutoff[k], f1Cutoff[k]))
    print('R-Precision:', rPrec)

def graph(prec, rec, qID):
    print('graphing', qID)
    plt.plot(rec, prec)
    plt.title('Precision - Recall Curve for Query ' + qID)
    plt.xlabel('Precision')
    plt.ylabel('Recall')
    plt.savefig(os.getcwd() + '\\Graphs\\' + qID + '_all')
    plt.close()

def calc(option, relevant, results):
    totQuery, totRel, totRelRet, totRet = 0, 0, 0, 0
    avgPrec, rPrec, nDCG = [], [], []
    P, R, F1 = {}, {}, {}

    for qID in sorted(results.keys()):
        totQuery += 1
        relevanceScore = []
        totPrec, totRec = {}, {}
        sumPrec, ret, relRet, rp = 0, 0, 0, 0

        res = results[qID]

        if qID not in relevant:
            continue

        precCutQuery, recCutQuery, f1CutQuery = {}, {}, {}

        relResDocs = relevant[qID]
        numRel = len(relResDocs)
        totRel += numRel

        for doc in res:
            ret += 1
            isRelevant = False

            if doc in relResDocs:
                relRet += 1
                isRelevant = True

            precision = relRet / (ret)
            score = 0

            if isRelevant:
                sumPrec += precision
                score = 1

            relevanceScore.append(score)

            if numRel == 0:
                recall = 0
            else:
                recall = relRet / numRel

            if ret <= numRel:
                rp = relRet

            totPrec[ret] = precision
            totRec[ret] = recall

        count = ret
        finalRecall = relRet / numRel

        allPrec, allRec = [], []

        while count <= 1000:
            totPrec[count] = relRet / count
            totRec[count] = finalRecall
            allPrec.append(totPrec[count])
            allRec.append(totRec[count])
            count += 1

        precCutoff, recCutoff = [], []

        for k in kVals:
            tempP = totPrec[k]
            tempR = totRec[k]

            precCutoff.append(tempP)
            recCutoff.append(tempR)

            f1 = 0
            if tempP > 0 and tempR > 0:
                f1 = (2 * tempP * tempR) / (tempP + tempR)

            precCutQuery[k] = tempP
            recCutQuery[k] = tempR
            f1CutQuery[k] = f1

            if k in P:
                P[k].append(tempP)
            else:
                P[k] = [tempP]

            if k in R:
                R[k].append(tempR)
            else:
                R[k] = [tempR]

            if k in F1:
                F1[k].append(f1)
            else:
                F1[k] = [f1]

        avgP = 0.0

        if relRet != 0:
            avgP = sumPrec / numRel

        avgPrec.append(avgP)
        rPrec.append(rp / len(relResDocs))

        numDCG = calcNDCG(relevanceScore)
        relevanceScoreDesc = sorted(relevanceScore, reverse = True)
        dcgDesc = calcNDCG(relevanceScoreDesc)

        # graph(precCutoff, recCutoff, qID)

        tempNDCG = 0
        if dcgDesc != 0:
            tempNDCG = numDCG / dcgDesc

        nDCG.append(tempNDCG)
        totRelRet += relRet
        totRet += ret

        if option:
            printer(qID, ret, numRel, relRet, avgP, rp / len(relResDocs), tempNDCG, precCutQuery, recCutQuery, f1CutQuery)

    AvgPrec = math.fsum(avgPrec) / len(avgPrec)
    RPre = math.fsum(rPrec) / len(rPrec)
    NDCG = math.fsum(nDCG) / len(nDCG)

    finalP, finalR, finalF1 = calcCutoff(P, R, F1)

    printer(totQuery, totRet, totRel, totRelRet, AvgPrec, RPre, NDCG, finalP, finalR, finalF1)

def main():
    option = False
    qrelFile = None
    resFile = None

    if len(sys.argv) < 3 or len(sys.argv) > 4:
        print('Incorrect format')
        sys.exit(0)

    if len(sys.argv) == 3:
        qrelFile = sys.argv[1]
        resFile = sys.argv[2]
    else:
        option = True
        qrelFile = sys.argv[2]
        resFile = sys.argv[3]

    # print(option, qrelFile, resFile)

    relevant = readQrel(os.getcwd() + '\\Result Files\\' + qrelFile)
    results = readRes(os.getcwd() + '\\Result Files\\' + resFile)
    calc(option, relevant, results)

main()


import os
import json
from flask import Flask, render_template, jsonify, send_from_directory, request
from flask_swagger import swagger
from localstack.utils.aws.aws_stack import Environment
from localstack.utils import common
from localstack.dashboard import infra

root_path = os.path.dirname(os.path.realpath(__file__))
web_dir = root_path + '/web/'

app = Flask('app', template_folder=web_dir)
app.root_path = root_path

@app.route('/swagger.json')
def spec():
    swag = swagger(app)
    swag['info']['version'] = "0.1"
    swag['info']['title'] = "AWS Resources Dashboard"
    return jsonify(swag)

@app.route('/graph', methods=['POST'])
def get_graph():
    """ Get deployment graph
        ---
        operationId: 'getGraph'
        parameters:
            - name: request
              in: body
    """
    data = json.loads(request.data)
    env = Environment.from_string(data.get('awsEnvironment'))
    graph = infra.get_graph(name_filter=data['nameFilter'], env=env)
    return jsonify(graph)

@app.route('/kinesis/<streamName>/<shardId>/events/latest', methods=['POST'])
def get_kinesis_events(streamName, shardId):
    """ Get latest events from Kinesis.
        ---
        operationId: 'getKinesisEvents'
        parameters:
            - name: streamName
              in: path
            - name: shardId
              in: path
            - name: request
              in: body
    """
    data = json.loads(request.data)
    env = Environment.from_string(data.get('awsEnvironment'))
    result = infra.get_kinesis_events(stream_name=streamName, shard_id=shardId, env=env)
    return jsonify(result)

@app.route('/lambda/<functionName>/code', methods=['POST'])
def get_lambda_code(functionName):
    """ Get source code for Lambda function.
        ---
        operationId: 'getLambdaCode'
        parameters:
            - name: functionName
              in: path
            - name: request
              in: body
    """
    data = json.loads(request.data)
    env = Environment.from_string(data.get('awsEnvironment'))
    result = infra.get_lambda_code(func_name=functionName, env=env)
    return jsonify(result)

@app.route('/')
def hello():
    return render_template('index.html')

@app.route('/<path:path>')
def send_static(path):
    return send_from_directory(web_dir + '/', path)

def ensure_webapp_installed():
    web_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), 'web'))
    node_modules_dir = os.path.join(web_dir, 'node_modules', 'jquery')
    if not os.path.exists(node_modules_dir):
        print('Initializing installation of Web application (this could take long time, please be patient)')
        common.run('cd "%s"; npm install' % web_dir)

def serve(port):
    ensure_webapp_installed()
    app.run(port=int(port), debug=True, threaded=True, host='0.0.0.0')


from playwright.async_api import Page

class VeloCubeLocators:
    status = ".product_stock-status"
    CHOOSE_SIZE = ".variant_name"

class VeloCubePage:
    URL = "https://velocube.ru/products/merida-silex-400-2021"

    def __init__(self, page: Page):
        self.page = page

    async def open_page(self, url):
        await self.page.goto(url)

    async def get_all_elements(self):
        elements = await self.page.query_selector_all(VeloCubeLocators.CHOOSE_SIZE)
        return elements

    async def get_status_text(self) -> str:
        result = await self.page.inner_text(selector=VeloCubeLocators.status, timeout=50000)
        return result

    async def check_status(self) -> bool:
        status = await self.get_status_text()
        if status != "Ожидается" or "Нет в наличии":
            return False
        return True

    async def check_status_for_all_element(self):
        status = {}
        elements = await self.get_all_elements()
        for element in elements:
            size = await element.text_content()
            if "XS" and "LIME" in size:
                await element.click()
                result = await self.check_status()
                status.update({"lime": result})
            elif "XS" and "BLUE" in size:
                await element.click()
                result = await self.check_status()
                status.update({"blue": result})
        return [status]


from django import forms

class UploadFileForm(forms.Form):
    title = forms.CharField(max_length=50)
    file  = forms.FileField()

SOURCE_CHOICES = (
    ("Facebook",u'Facebook'),
    ("Email",u'Email'),
    ("From a friend",u'From a friend'),
    ("Notification on my phone",u'Notification on my phone'),
    ("Saw in the app",u'Saw in the app'),
    ("Found out at the venue",u'Found out at the venue'),
    ("Other",u'Other'),
    ("...",u"...")
)

FREQUENCY_CHOICES = (
    ("Never",u'Never'),
    ("A couple times a year",u'A couple times a year'),
    ("Once a month",u'Once a month'),
    ("A couple times a month",u'A couple times a month'),
    ("Once a week",u'Once a week'),
    ("...",u"...")
)

class EventFeedback(forms.Form):
    general = forms.CharField(required=False,
                              widget=forms.Textarea(attrs={'class':'form-control','placeholder':'feedback','rows':3}))
    source = forms.CharField(required=False,
                              widget=forms.Select(choices=SOURCE_CHOICES, attrs={'class':'form-control'}))
    frequency = forms.CharField(required=False,
                                 widget=forms.Select(choices=FREQUENCY_CHOICES, attrs={'class':'form-control'}))
    improvements = forms.CharField(required=False,
                                    widget=forms.Textarea(attrs={'class':'form-control','placeholder':'feedback','rows':3}))


import requests
import logging
from aws_requests_auth.aws_auth import AWSRequestsAuth

from base.config import config

logging.info('Connecting AWS ElasticSearch %s', config.ES_URI)
auth = AWSRequestsAuth(aws_access_key=config.AWS_ES_ACCESS_KEY,
                                  aws_secret_access_key=config.AWS_ES_SECRET_KEY,
                                  aws_host=config.ES_URI,
                                  aws_region=config.ES_REGION,
                                  aws_service='es')

def get_settings():
    url = "https://" + config.ES_URI + '/_cat/allocation?v'
    headers = {"Content-Type": "application/json"}
    response = requests.get(url, auth=auth, headers=headers)
    print(response.status_code)
    print(response.headers)
    return response.content

print(get_settings())


from ..helper import newer
from ._base import MapperTask

class Copy(MapperTask):
    """Copy files to a destination directory, Exclude standard
hidden files.
constructor arguments:
Copy(*files, dest=<destdir>, exclude=<defaults>)"""

    # pylint: disable=unused-argument
    def dojob(self, sname, dname, context):
        self.logger.debug('sname = %s; dname = %s', sname, dname)
        srcfile = self.join(sname)
        dstfile = self.join(dname)
        if srcfile.isdir:
            # remove whatever is there
            if dstfile.exists and not dstfile.isdir:
                dstfile.remove()
            # create the directory
            if not dstfile.exists:
                dstfile.mkdir()
        elif srcfile.isfile and newer(srcfile, dstfile, logger=self.logger):
            self.logger.debug('uptodate: %s', dstfile)
        else:
            if not dstfile.dirname.isdir:
                dstfile.dirname.mkdir()
            self.logger.info('copy2(%s,%s)', sname, dname)
            srcfile.copy(dstfile)

Copy.register()