import argparse
import re
import os
import sys

import_rx = re.compile("^@import url\\('(?P<file>[^']+)'\\);$")
opening_brace_rx = re.compile("^\\s*:root\s*{\\s*$")
closing_brace_rx = re.compile("^\\s*}\\s*$")
comment_rx = re.compile("^\\s*(/\\*.*\\*/)?\\s*$")
comment_start_rx = re.compile("^\\s*(/\\*.*)\\s*$")
comment_end_rx = re.compile("^\\s*(.*\\*/)\\s*$")
variable_declaration_rx = re.compile("^\\s*(?P<key>--[a-z-]+)\\s*:\\s*(?P<value>[^;]+)\\s*;\\s*(/\\*.*\\*/)?\\s*$")
variable_use_rx = re.compile("^(?P<before>.*)var\\((?P<key>--[a-z-]+)\\)(?P<after>.*)$")

def postprocess(files, process_imports, out_file):
    directory = os.path.dirname(files[0])

    if not out_file:
        basename, ext = os.path.splitext(files[0])
        out_file = basename + ".compiled" + ext

    variables = {}
    imported_files = []
    def parse(f):
        nonlocal variables, imported_files
        not_just_variable_declarations = False
        in_variable_declarations = False
        in_comment = False
        for line in f:
            # In comment and the comment is not ending yet, ignore
            if in_comment:
                if comment_end_rx.match(line):
                    in_comment = False
                continue

            # Import statement: add the file to additionally processed files
            # unless it's disabled
            match = import_rx.match(line)
            if match:
                if process_imports:
                    imported_files += [match.group('file')]
                continue

            # Opening brace of variable declaration block
            match = opening_brace_rx.match(line)
            if match:
                in_variable_declarations = True
                continue

            # Variable declaration
            match = variable_declaration_rx.match(line)
            if match and in_variable_declarations:
                # Define a variable to equal another
                key = match.group('key')
                value = match.group('value')
                match = variable_use_rx.match(value)
                if match and match.group('key') in variables:
                    value = match.group('before') + variables[match.group('key')]
                    if match.group('after').endswith('*/'):
                        value += match.group('after')[:match.group('after').rindex('/*')].rstrip()
                    else:
                        value += match.group('after')
                variables[key] = value
                continue

            # Comment or empty line, ignore
            if comment_rx.match(line):
                continue

            # Comment start line, ignore this and the next lines
            if comment_start_rx.match(line):
                in_comment = True
                continue

            # Closing brace of variable declaration block. If it was not just
            # variable declarations, put the closing brace to the output as
            # well.
            match = closing_brace_rx.match(line)
            if match and in_variable_declarations:
                if not_just_variable_declarations: out.write("}\n")
                in_variable_declarations = False
                continue

            # If inside variable declaration block, include also the opening
            # brace and remember to put the closing brace there as well
            if in_variable_declarations:
                out.write(":root {\n")
                not_just_variable_declarations = True

            # Variable use, replace with actual value
            # TODO: more variables on the same line?
            match = variable_use_rx.match(line)
            if match and match.group('key') in variables:
                out.write(match.group('before'))
                out.write(variables[match.group('key')])
                # Strip the trailing comment, if there, to save some bytes
                if match.group('after').endswith('*/'):
                    out.write(match.group('after')[:match.group('after').rindex('/*')].rstrip())
                else:
                    out.write(match.group('after'))
                out.write("\n")
                continue

            # Something else, copy verbatim to the output. Strip the trailing
            # comment, if there, to save some bytes.
            if line.rstrip().endswith('*/'):
                out.write(line[:line.rindex('/*')].rstrip() + '\n')
            else:
                out.write(line)

    with open(out_file, mode='w', encoding='utf8') as out:
        # Put a helper comment and a license blob on top
        out.write("""/* Generated using `./postprocess.py {}`. Do not edit. */

/*
    This file is part of m.css.

    Copyright © 2017, 2018, 2019, 2020, 2021, 2022, 2023
              Vladimír Vondruš <mosra@centrum.cz>

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/
""".format(' '.join(sys.argv[1:])))

        # Parse the top-level file
        with open(files[0], encoding='utf8') as f: parse(f)

        # Now open the imported files and parse them as well. Not doing any
        # recursive parsing.
        for i, file in enumerate(imported_files + files[1:]):
            if i: out.write('\n')

            with open(file, encoding='utf8') as f: parse(f)

    return 0

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=r"""
Postprocessor for removing @import statements and variables from CSS files.

Combines all files into a new *.compiled.css file. The basename is taken
implicitly from the first argument. The -o option can override the output
filename.""")
    parser.add_argument('files', nargs='+', help="input CSS file(s)")
    parser.add_argument('--no-import', help="ignore @import statements", action='store_true')
    parser.add_argument('-o', '--output', help="output file", default='')
    args = parser.parse_args()

    exit(postprocess(args.files, not args.no_import, args.output))


#!/usr/bin/env python3

import os
import logging
import subprocess
import pnrw

def cwd():
    directory = os.getcwd()
    logging.debug(directory)
    return directory

def get_env_variable(env, default):
    if not os.environ.get(env, default):
        return logging.warning("%s is not defined!" % env)
    else:
        logging.debug("%s properly set." % env)
        return os.environ.get(env, default)

abbreviation = "tst0"

def wallet_list():
    lines = []
    proc = subprocess.Popen(['%s/%s_node' % (cwd(), abbreviation), '--wallet_list'], stdout=subprocess.PIPE)
    while True:
        line = proc.stdout.readline().decode().replace('Wallet ID: ', '')
        if not line:
            break
        lines.append(str(line.rstrip()))

    return lines

def wallet_id_list():
    lines = []
    proc = subprocess.Popen(['./%s_node' % abbreviation, '--wallet_list'], stdout=subprocess.PIPE)
    while True:
        line = proc.stdout.readline().decode()
        if not line:
            break
        if "Wallet ID" in line:
            lines.append(str(line.rstrip()).replace('Wallet ID: ', ''))

    return lines

# pnrw
node_ip = get_env_variable('BOOTSTRAP_NODE_IP', '[::1]')
logging.debug(f" node_ip: {node_ip}")
node_port = get_env_variable('BOOTSTRAP_NODE_PORT', 7076)
node_url_suffix = get_env_variable('BOOTSTRAP_URL_SUFFIX', '/api')
node_url = f"{node_ip}:{node_port}{node_url_suffix}"
dont_use_https = get_env_variable('BOOTSTRAP_DONT_USE_HTTPS', 'True')
headers = get_env_variable('BOOTSTRAP_HEADERS', [])
banano = get_env_variable('BOOTSTRAP_BANANO', 'False')

# make node connection
try:
    node = pnrw.Node(node_url, dontUseHTTPS=dont_use_https, headers=headers, banano=banano)
    logging.info('Node connection established')
    logging.debug(f"node uptime: {node.uptime()}")
except pnrw.exceptions as node:
    logging.error(node)

max_supply = 340282366920938463463374607431768211455
divided_supply = max_supply / 7
print(f"divided_supply_raw: {divided_supply}")

def percentage(percent, whole):
  return (percent * whole) / 100.0

print(percentage(14, max_supply))


import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

#reading the dataset
def read_dataset():
    df=pd.read_csv("E:\\PycharmProjects\\tf edureka\\sonar.csv")
    ##here the DataFrame.values converts the pandas dataframe into
    ##a numpy array
    X=df[df.columns[0:60]].values
    y=df[df.columns[60]]
    ##encoding the labels
    encoder=LabelEncoder()
    encoder.fit(y)
    y=encoder.transform(y)
    Y=one_hot_encode(y)
    print(X.shape)
    return (X,Y)

##define the one hot encoder function
def one_hot_encode(labels):
    n_labels = len(labels)
    n_unique_labels = len(np.unique(labels))
    one_hot_encode =np.zeros((n_labels,n_unique_labels))
    one_hot_encode[np.arange(n_labels),labels]=1
    return one_hot_encode

# read the dataset using read_dataset function
X,Y = read_dataset()

#Shuffle the dataset to mix up the rows
X,Y = shuffle(X,Y, random_state=1)

#split the dataset into training and testing adatset using
# train_tset_split method in sklearn.model_selection
train_x , test_x , train_y , test_y = train_test_split(X, Y, test_size=0.20 ,random_state= 20)

#define the important parameters and variables to work with tensors
learning_rate = 0.3
training_epochs = 1000
cost_history = np.empty(shape=[1],dtype=float)
n_dim=X.shape[1]
print("n_dim",n_dim)
n_class=2
model_path="E:\\PycharmProjects\\tf edureka"

n_hidden_1=60
n_hidden_2=60
n_hidden_3=60
n_hidden_4=60

x=tf.placeholder(tf.float32,[None,n_dim])
W=tf.Variable(tf.zeros([n_dim, n_class]))
b=tf.Variable(tf.zeros([n_class]))
y_ = tf.placeholder(tf.float32,[None, n_class])

#define the model
def multilayer_perceptron(x, weights, biases):
    # hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x,weights['h1']), biases['b1'])
    layer_1 = tf.sigmoid(layer_1)

    layer_2 = tf.add(tf.matmul(layer_1,weights['h2']),biases['b2'])
    layer_2 = tf.sigmoid(layer_2)

    layer_3 = tf.add(tf.matmul(layer_2,weights['h3']),biases['b3'])
    layer_3 = tf.sigmoid(layer_3)

    layer_4 = tf.add(tf.matmul(layer_3,weights['h4']),biases['b4'])
    layer_4 = tf.sigmoid(layer_4)

    out_layer = tf.matmul(layer_4, weights['out']) + biases['out']
    return out_layer

weights = {
'h1': tf.Variable(tf.truncated_normal([n_dim, n_hidden_1])),
'h2': tf.Variable(tf.truncated_normal([n_hidden_1,n_hidden_2])),
'h3': tf.Variable(tf.truncated_normal([n_hidden_2, n_hidden_3])),
'h4': tf.Variable(tf.truncated_normal([n_hidden_3, n_hidden_4])),
'out': tf.Variable(tf.truncated_normal([n_hidden_4, n_class]))
}

biases={
'b1':tf.Variable(tf.truncated_normal([n_hidden_1])),
'b2':tf.Variable(tf.truncated_normal([n_hidden_2])),
'b3':tf.Variable(tf.truncated_normal([n_hidden_3])),
'b4':tf.Variable(tf.truncated_normal([n_hidden_4])),
'out':tf.Variable(tf.truncated_normal([n_class]))
}

# initialize all the variables

init = tf.global_variables_initializer()

saver = tf.train.Saver()

#define your model
y = multilayer_perceptron(x, weights, biases)

#define the cost function and optimizer
cost_function =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_))
training_step =  tf.train.GradientDescentOptimizer(learning_rate).minimize(cost_function)

sess = tf.Session()
sess.run(init)

#calculate the cost and accuracy for each epoch

mse_history = []
accuracy_history = []

for epoch in range(training_epochs):
    sess.run(training_step,feed_dict={x: train_x,y_: train_y})
    cost = sess.run(cost_function,feed_dict={x: train_x,y_: train_y})
    cost_history= np.append(cost_history,cost)
    correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
    # print("accuracy: " ,sess.run(accuracy,feed_dict={x: train_x,y_: train_y}))
    pred_y = sess.run(y,feed_dict={x: test_x})
    mse = tf.reduce_mean(tf.square(pred_y - test_y))
    mse_ = (sess.run(accuracy,feed_dict={x: train_x,y_: train_y}))
    mse_history.append(mse_)
    accuracy = (sess.run(accuracy,feed_dict={x: train_x,y_: train_y}))
    accuracy_history.append(accuracy)

    print('epoch : ',epoch , ' - ','cost: ',cost ,"-MSE: ",mse_,"-train_accuracy",accuracy)

save_path = saver.save(sess, model_path)
print("MOdel saved in file: %s" % save_path)

# plot mse and accuracy graph

plt.plot(mse_history, 'r')
plt.show()
plt.plot(accuracy_history)
plt.show()

# print the final accuracy

correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("Test Accuracy: ",(sess.run(accuracy, feed_dict={x: test_x,y_:test_y})))

#print final mean squared error

pred_y = sess.run(y,feed_dict={x: test_x})
mse = tf.reduce_mean(tf.square(pred_y-test_y))
print("MSE: %.4f"% sess.run(mse))


import boot_data
import common
import emu_target
import logging
import os
import re
import subprocess
import tempfile

_SSH_KEY_DIR = os.path.expanduser('~/.ssh')
_DEFAULT_SSH_PORT = 22
_DEVICE_PROTO_TEMPLATE = """
device_spec:  {{
  horizontal_resolution:  1024
  vertical_resolution:  600
  vm_heap:  192
  ram:  {ramsize}
  cache:  32
  screen_density:  240
}}
"""

def GetTargetType():
  return FvdlTarget

class EmulatorNetworkNotFoundError(Exception):
  """Raised when emulator's address cannot be found"""
  pass

class FvdlTarget(emu_target.EmuTarget):
  EMULATOR_NAME = 'aemu'
  _FVDL_PATH = os.path.join(common.SDK_ROOT, 'tools', 'x64', 'fvdl')

  def __init__(self, out_dir, target_cpu, system_log_file, require_kvm,
               enable_graphics, hardware_gpu, with_network, ram_size_mb):
    super(FvdlTarget, self).__init__(out_dir, target_cpu, system_log_file)
    self._require_kvm = require_kvm
    self._enable_graphics = enable_graphics
    self._hardware_gpu = hardware_gpu
    self._with_network = with_network
    self._ram_size_mb = ram_size_mb

    self._host = None
    self._pid = None

    # Use a temp file for vdl output.
    self._vdl_output_file = tempfile.NamedTemporaryFile()

    # Use a temp file for the device proto and write the ram size.
    self._device_proto_file = tempfile.NamedTemporaryFile()
    with open(self._device_proto_file.name, 'w') as file:
      file.write(_DEVICE_PROTO_TEMPLATE.format(ramsize=self._ram_size_mb))

  @staticmethod
  def CreateFromArgs(args):
    return FvdlTarget(args.out_dir, args.target_cpu, args.system_log_file,
                      args.require_kvm, args.enable_graphics, args.hardware_gpu,
                      args.with_network, args.ram_size_mb)

  @staticmethod
  def RegisterArgs(arg_parser):
    fvdl_args = arg_parser.add_argument_group('fvdl', 'FVDL arguments')
    fvdl_args.add_argument('--with-network',
                           action='store_true',
                           default=False,
                           help='Run emulator with emulated nic via tun/tap.')

  def _BuildCommand(self):
    boot_data.ProvisionSSH()
    self._host_ssh_port = common.GetAvailableTcpPort()
    kernel_image = common.EnsurePathExists(
        boot_data.GetTargetFile('qemu-kernel.kernel', self._GetTargetSdkArch(),
                                boot_data.TARGET_TYPE_QEMU))
    zbi_image = common.EnsurePathExists(
        boot_data.GetTargetFile('zircon-a.zbi', self._GetTargetSdkArch(),
                                boot_data.TARGET_TYPE_QEMU))
    fvm_image = common.EnsurePathExists(
        boot_data.GetTargetFile('storage-full.blk', self._GetTargetSdkArch(),
                                boot_data.TARGET_TYPE_QEMU))
    aemu_path = common.EnsurePathExists(
        os.path.join(common.GetEmuRootForPlatform(self.EMULATOR_NAME),
                     'emulator'))

    emu_command = [
        self._FVDL_PATH,
        '--sdk',
        'start',
        '--nopackageserver',
        '--nointeractive',

        # Host port mapping for user-networking mode.
        '--port-map',
        'hostfwd=tcp::{}-:22'.format(self._host_ssh_port),

        # no-interactive requires a --vdl-output flag to shutdown the emulator.
        '--vdl-output',
        self._vdl_output_file.name,

        # Use existing images instead of downloading new ones.
        '--kernel-image',
        kernel_image,
        '--zbi-image',
        zbi_image,
        '--fvm-image',
        fvm_image,
        '--image-architecture',
        self._target_cpu,

        # Use an existing emulator checked out by Chromium.
        '--aemu-path',
        aemu_path,

        # Use this flag and temp file to define ram size.
        '--device-proto',
        self._device_proto_file.name
    ]

    if not self._require_kvm:
      emu_command.append('--noacceleration')
    if not self._enable_graphics:
      emu_command.append('--headless')
    if self._hardware_gpu:
      emu_command.append('--host-gpu')
    if self._with_network:
      emu_command.append('-N')

    logging.info('FVDL command: ' + ' '.join(emu_command))

    return emu_command

  def _WaitUntilReady(self):
    # Indicates the FVDL command finished running.
    self._emu_process.communicate()
    super(FvdlTarget, self)._WaitUntilReady()

  def _IsEmuStillRunning(self):
    if not self._pid:
      try:
        with open(self._vdl_output_file.name) as vdl_file:
          for line in vdl_file:
            if 'pid' in line:
              match = re.match(r'.*pid:\s*(\d*).*', line)
              if match:
                self._pid = match.group(1)
      except IOError:
        logging.error('vdl_output file no longer found. '
                      'Cannot get emulator pid.')
        return False
    if subprocess.check_output(['ps', '-p', self._pid, 'o', 'comm=']):
      return True
    logging.error('Emulator pid no longer found. Emulator must be down.')
    return False

  def _GetEndpoint(self):
    if self._with_network:
      return self._GetNetworkAddress()
    return ('localhost', self._host_ssh_port)

  def _GetNetworkAddress(self):
    if self._host:
      return (self._host, _DEFAULT_SSH_PORT)
    try:
      with open(self._vdl_output_file.name) as vdl_file:
        for line in vdl_file:
          if 'network_address' in line:
            address = re.match(r'.*network_address:\s*"\[(.*)\]".*', line)
            if address:
              self._host = address.group(1)
              return (self._host, _DEFAULT_SSH_PORT)
        logging.error('Network address not found.')
        raise EmulatorNetworkNotFoundError()
    except IOError as e:
      logging.error('vdl_output file not found. Cannot get network address.')
      raise

  def Shutdown(self):
    if not self._emu_process:
      logging.error('%s did not start' % (self.EMULATOR_NAME))
      return
    femu_command = [
        self._FVDL_PATH, '--sdk', 'kill', '--launched-proto',
        self._vdl_output_file.name
    ]
    femu_process = subprocess.Popen(femu_command)
    returncode = femu_process.wait()
    if returncode == 0:
      logging.info('FVDL shutdown successfully')
    else:
      logging.info('FVDL kill returned error status {}'.format(returncode))
    emu_target.LogProcessStatistics('proc_stat_end_log')
    emu_target.LogSystemStatistics('system_statistics_end_log')
    self._vdl_output_file.close()
    self._device_proto_file.close()

  def _GetSshConfigPath(self):
    return boot_data.GetSSHConfigPath()


import os.path
import re
import time
from collections import namedtuple

import requests
from bs4 import BeautifulSoup

from lang_map import get_ext
from log import log
from moss_parser import problem_path, target_src_path, other_src_path
from settings import DMOJ_URL, DMOJ_API_KEY, DMOJ_REQUEST_DELAY, SAVED_DATA_DIR

Submission = namedtuple('Submission', 'sub_id user_id lang')
ProblemSubmissions = namedtuple('ProblemSubmissions', 'problem_id lang target_sub_id other_sub_ids sub_map')

dmoj_auth_header = {'Authorization': f'Bearer {DMOJ_API_KEY}'}

def get_solved_page(problem_id, page_no):
    """
    Parses a problem rank page and returns a list of accepted submissions along with the user that solved them
    :param problem_id: The problem id to parse
    :param page_no: The page to check
    :return:
    """

    def regex_find_href(tag, exp):
        return re.search(exp, tag.find('a', {'href': re.compile(exp)})['href'])

    res = requests.get(f'{DMOJ_URL}problem/{problem_id}/rank/{page_no}', headers=dmoj_auth_header)
    if res.status_code == 404:
        return []
    else:
        soup = BeautifulSoup(res.text, 'html.parser')
        subs = []
        for tag in soup.find_all('div', {'class': 'submission-row'}):
            if 'AC' in tag.find('div', {'class': 'sub-result'})['class']:  # Fake AC is not counted
                try:
                    user_id = regex_find_href(tag, r'/user/(\w+)').group(1)
                    sub_id = regex_find_href(tag, r'/submission/(\d+)').group(1)
                    lang = tag.find('span', {'class': 'language'}).text
                    subs.append(Submission(sub_id, user_id, lang))
                except Exception as e:
                    log.log(e)

        return subs  # Return nothing if subs is empty

def get_solved(problem_id):
    """
    Returns a list of submissions for a problem
    :param problem_id: The problem to check
    :return: A list of Submission objects
    """

    log.log(f'Getting best submissions for {problem_id}')

    subs = []
    page_no = 1
    cur_subs = get_solved_page(problem_id, page_no)
    time.sleep(DMOJ_REQUEST_DELAY)
    while cur_subs:
        subs.extend(cur_subs)
        log.log(f'Parsed page {page_no}, got {len(cur_subs)} submission ids')

        page_no += 1
        cur_subs = get_solved_page(problem_id, page_no)
        time.sleep(DMOJ_REQUEST_DELAY)

    return subs

def download_subs(target_handle, problem_id):
    """
    Downloads all accepted submissions for a problem that match the language used by a given target user.  Downloaded submissions will be in the folder ./SAVED_DATA_DIR/problem_id where SAVED_DATA_DIR is from configuration and `problem_id` is the parameter

    Additionally, this procedure assumes that user DMOJ_HANDLE has access to this problem's sources (either solved it already or is an Admin)
    :param target_handle: Only submissions with the same language as the target will be downloaded
    :param problem_id: The id of the problem to download submissions from
    :return: A ProblemSubmissions object, or None if an error occured
    """

    subs = get_solved(problem_id)
    target_sub_id = None
    target_lang = None

    # Make subs directory
    if not os.path.exists(SAVED_DATA_DIR):
        os.mkdir(SAVED_DATA_DIR)
    if not os.path.exists(problem_path(problem_id)):
        os.mkdir(problem_path(problem_id))

    # Submission map - maps submission id to handle
    sub_map = {}

    # Get target submission
    for sub_id, user_id, lang in subs:
        if user_id == target_handle:
            target_sub_id = sub_id
            target_lang = lang
            with open(target_src_path(problem_id, get_ext(lang)), 'w', encoding='utf8') as f:
                f.write(get_sub_src(sub_id))
            time.sleep(DMOJ_REQUEST_DELAY)

            log.log(f'Got submission source of target: {sub_id} in {lang}')
            sub_map[sub_id] = target_handle
            break

    if not target_lang:
        log.log(f'ERROR: Could not find Accepted submission for {problem_id}')

    # Get submission ids of other users
    other_ids = []
    target_lang_ext = get_ext(target_lang)
    if not target_lang_ext:
        log.log(f'ERROR: Language {target_lang} currently not supported')
        return None

    for sub_id, user_id, lang in subs:
        if user_id != target_handle and get_ext(lang) == target_lang_ext: # Use extension instead of DMOJ language so Java8/9/10/11 are grouped
            with open(other_src_path(problem_id, target_lang_ext, sub_id), 'w', encoding='utf8') as f:
                f.write(get_sub_src(sub_id))
            time.sleep(DMOJ_REQUEST_DELAY)
            other_ids.append(sub_id)

            log.log(f'Got accepted submission source: {sub_id} by {user_id}')
            sub_map[sub_id] = user_id

    return ProblemSubmissions(problem_id, target_lang, target_sub_id, other_ids, sub_map)

def get_user_subs(user_id):
    """
    Returns all submissions made by a user
    :param user_id: The DMOJ handle of the user
    :return: A list of submissions, in the DMOJ APIv2 format
    """

    return requests.get(f'{DMOJ_URL}api/v2/submissions?user={user_id}').json()

def get_sub_src(sub_id):
    """
    Returns the source code of a submission.  This assumes DMOJ_HANDLE has access to this submission already.
    :param sub_id: The id of the submission to download
    :return: Returns the source code of the submission
    """
    return requests.get(f'{DMOJ_URL}src/{sub_id}/raw', headers=dmoj_auth_header).text.replace('\r', '')


import json
import configparser
from configparser import SectionProxy
from azure.identity import DeviceCodeCredential, ClientSecretCredential
from msgraph.core import GraphClient

class Graph:
	settings: SectionProxy
	device_code_credential: DeviceCodeCredential
	user_client: GraphClient
	client_credential: ClientSecretCredential
	app_client: GraphClient

	def __init__(self, config: SectionProxy):
		self.settings = config
		client_id = self.settings['clientId']
		tenant_id = self.settings['authTenant']
		graph_scopes = self.settings['graphUserScopes'].split(' ')

		self.device_code_credential = DeviceCodeCredential(client_id, tenant_id = tenant_id)
		self.user_client = GraphClient(credential=self.device_code_credential, scopes=graph_scopes)

	def get_user_token(self):
		graph_scopes = self.settings['graphUserScopes']
		access_token = self.device_code_credential.get_token(graph_scopes)
		return access_token.token

	def get_user(self):
		endpoint = '/me'
		# Only request specific properties
		select = 'displayName,mail,userPrincipalName,id'
		request_url = f'{endpoint}?$select={select}'

		user_response = self.user_client.get(request_url)
		return user_response.json()

# def display_access_token(graph: Graph):
# 	token = graph.get_user_token()
# 	print('User token:', token, '\n')

# def greet_user(graph: Graph):
# 	user = graph.get_user()
# 	print('Hello,', user['displayName'])
# 	# For Work/school accounts, email is in mail property
# 	# Personal accounts, email is in userPrincipalName
# 	print('Email:', user['mail'] or user['userPrincipalName'], '\n')


import logging
import torch
from torch.distributions.categorical import Categorical
import numpy as np

def config_logger(log_dir):
    logger = logging.getLogger()
    logger.setLevel('INFO')
    basic_format = '%(message)s'
    formatter = logging.Formatter(basic_format)
    chlr = logging.StreamHandler()
    chlr.setFormatter(formatter)
    fhlr = logging.FileHandler(log_dir)
    logger.addHandler(chlr)
    logger.addHandler(fhlr)
    return logger

def select_actions(pi):
    actions = Categorical(pi).sample()
    return actions.detach().cpu().numpy().squeeze()

def evaluate_actions(pi, actions):
    distr = Categorical(pi)
    log_prob = distr.log_prob(actions).unsqueeze(-1)
    entropy = distr.entropy().mean()
    return log_prob, entropy


from setuptools import setup

APP = ['geekhub/geekhub.py']
DATA_FILES = [
    'Resources/alert.png',
    'Resources/notification.png',
]
OPTIONS = {
    'argv_emulation': True,
    'plist': {
        'LSUIElement': True,
    },
    'packages': ['rumps'],
    'iconfile': 'Resources/logo.icns',
}

VERSION = '0.1.3'

setup(
    name='geekhub',
    version=VERSION,
    description='A little statusbar for to geekhub',
    license='MIT',
    author='Leetao',
    author_email='leetao@gmail.com',
    packges=['geekhub'],
    app=APP,
    data_files=DATA_FILES,
    options={'py2app': OPTIONS},
    install_requires=['requests', 'BeautifulSoup4', 'rumps'],
    setup_requires=['py2app'],
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: MacOS X',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: Chinese',
        'Operating System :: MacOS :: MacOS X',
        'Programming Language :: Python :: 3.7',
    ]
)


from os.path import join

import numpy as np
import scipy.interpolate as spi
from pylab import matplotlib
from bokeh.io import output_file, export_png, export_svgs, show
from bokeh.transform import linear_cmap
from bokeh.plotting import figure
from bokeh.models import ColorBar, ColumnDataSource, Span, BoxAnnotation
from bokeh.layouts import gridplot
import bokeh.palettes
import colorcet as cc
import cmocean
import iqplot

from .constants import themes, banned_substrings
from .common import ban_columns_with_substrings, aggregate_trace
from .fourier import fft

def cmap2hexlist(cmap):

    """
    Converts a matplotlib.colors.LinearSegementedColormap object to
    a list of hex colours. Useful for converting attribute colour maps
    of cmocean.cm for use with bokeh.

    Parameters:
    -----------
    cmap: a matplotlib.colors.LinearSegmentedColormap object.
        Attributes of cmocean.cm are these objects and are colour maps.
        E.g. cmocean.cm.haline

    Returns:
    --------
    A list of hexadecimal colour codes.
    """

    hexcolours = []
    for i in range(cmap.N):
        # Returns rgba; take only first 3 to get rgb:
        rgb = cmap(i)[:3]
        hexcolours.append(matplotlib.colors.rgb2hex(rgb))

    return hexcolours

def get_all_cmocean_colours():

    """
    Returns a dictionary of the cmocean colour maps.
    Each key in the dictionary is the name of the colourmap and each value
    in the dictionary is a list of the colourmap's colours in hexadecimal.
    """

    cmocean_attrs = dir(cmocean.cm)

    hexcolourmaps_dict = {}
    for cmocean_attr in cmocean_attrs:

        attr_instance = getattr(cmocean.cm, cmocean_attr)
        attr_type = type(attr_instance)

        if attr_type is matplotlib.colors.LinearSegmentedColormap:

            # Convert val to hexlist and add key-val pair:
            hexcolourmaps_dict[cmocean_attr] = cmap2hexlist(attr_instance)

    return hexcolourmaps_dict

cm = get_all_cmocean_colours()

def load_plot_theme(p, theme=None, has_legend=False):

    """
    Load theme colours into a Bokeh plotting object.

    Parameters:
    -----------
    p: A Bokeh plotting object

    Returns:
    --------
    `p` with coloured attributes.
    """

    assert (theme in themes or theme is None), \
        f"{theme} is neither None nor a valid key in `themes`"

    if theme is not None:

        theme_colours = themes[theme]

        dark_hue = theme_colours["dark_hue"]
        light_hue = theme_colours["light_hue"]

        p.border_fill_color = light_hue
        p.xgrid.grid_line_color = dark_hue
        p.ygrid.grid_line_color = dark_hue
        p.background_fill_color = light_hue

        if has_legend == True:
            p.legend.background_fill_color = light_hue
        else:
            pass

        return p

    else:
        pass

def plot_power_spectra(df, val_cols, time_col,
                        is_evenly_sampled=False, window=np.hanning, pad=1,
                        cutoff_freq=None,
                        val_labels=None, time_label=None,
                        theme=None,
                        save_path_to=None, show_plots=True):

    """
    Perform a Fourier transform on FicTrac data for each ID. Plot
    power spectrum for each value in `val_cols`.
    Accepts one column from `df`, rather than a list of columns.

    Parameters:
    ------------
    df (DataFrame): Dataframe of FicTrac data generated from parse_dats()

    val_cols (list): List of column names from `df` to be Fourier-transformed.

    time_col (str): Column name in `df` that specifies time in SECONDS.

    is_evenly_sampled (bool): If False, will interpolate even sampling.

    cutoff_freq (float): x-intercept value for plotting a vertical line.
        To be used to visualize a candidate cut-off frequency. Default is None.

    val_labels (list): List of labels for the plot's y-axis. If None, will
        be a formatted version of val_cols. Default is None.

    time_label (str): Label for the plot's time-axis.

    theme (str or None): A pre-defined colour theme for plotting. If None,
        does not apply a theme. Default is None.

    save_path_to (str): Absolute path to which to save the plots as .png files.
        If None, will not save the plots. Default is None.

    show_plots (bool): If True, will show plots, but will not output Bokeh plotting
        objects. If False, will not show plots, but will output Bokeh plotting objects.
        If both save and show_plots are True, .html plots will be generated, in addition
        to the .png plots. Default is True.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    if not ('sec' in time_col or '(s)' in time_col):
        safe_secs = input(f"The substrings 'sec' or '(s)' was not detected in the 'time_col' variable, {time_col}. The units of the values in {time_col} MUST be in seconds. If the units are in seconds, please input 'y'. Otherwise input anything else to exit.")
        while True:
            if safe_secs.lower() == "y":
                break
            else:
                exit("Re-run this function with a 'time_col' whose units are secs.")

    assert (time_col in df), f"The column, {time_col}, is not in the input dataframe."

    time = list(df[str(time_col)])

    # Format axes labels:
    if time_label == None:
        time_label = time_col.replace("_", " ")
    if val_labels == None:
        val_labels = [val_col.replace("_", " ") for val_col in val_cols]

    def load_aesthetics(freq, amp):

        """
        Load aesthetics for plotting power spectra.
        """

        title = "Frequency domain"

        # Frequency domain:
        p1 = figure(
                title=title,
                width=1000,
                height=500,
                y_axis_label="power"
        )
        p1.line(
            x=freq,
            y=amp,
            color="darkgray"
        )
        p2 = figure(
                width=1000,
                height=500,
                x_axis_label="frequency (Hz)",
                y_axis_label="log power",
                y_axis_type="log"
        )
        p2.line(
            x=freq,
            y=amp,
            color="darkgray"
        )

        return p1, p2

    plots = []
    for i, val_col, in enumerate(val_cols):

        assert (len(df[time_col] == len(df[val_col]))), \
            "time and val are different lengths! They must be the same."
        assert (val_col in df), \
            f"The column, {val_col}, is not in the input dataframe."

        val = list(df[str(val_col)])

        # Fourier-transform:
        f = spi.interp1d(time, val)

        if is_evenly_sampled == False:
            time_interp = np.linspace(time[1], time[-1], len(time))
            val_interp = f(time_interp)
        else:
            time_interp = time
            val_interp = val

        amp, _, freq = fft( val_interp,
                            time_interp,
                            pad=1,
                            window=window,
                            post=True)

        # Plot:
        p1, p2 = load_aesthetics(freq, amp)

        p1.title.text = f"power spectrum of {val_labels[i]}"
        p1.title.text_font_size = "16pt"
        p1.yaxis.axis_label_text_font_size = "12pt"
        load_plot_theme(p1, theme=theme)

        p2.yaxis.axis_label_text_font_size = "12pt"
        p2.xaxis.axis_label_text_font_size = "12pt"
        load_plot_theme(p2, theme=theme)

        if cutoff_freq is not None:
            float(cutoff_freq)
            cutoff_line = Span(location=cutoff_freq,
                            dimension="height",
                            line_color="#775a42",
                            line_dash="dashed",
                            line_width=2)
            p1.add_layout(cutoff_line)
            p2.add_layout(cutoff_line)

        p = gridplot([p1, p2], ncols=1)

        # Output:
        if save_path_to is not None:
            filename = save_path_to + f"fictrac_freqs"

            # Bokeh does not atm support gridplot svg exports
            export_png(p, filename = filename + ".png")
            output_file(filename = filename + ".html",
                        title=f"fictrac_freqs")

        if show_plots == True:
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def plot_filtered(df, val_cols, time_col,
                  order, cutoff_freq,
                  val_labels=None, time_label=None,
                  view_perc=100,
                  theme=None,
                  save_path_to=None, show_plots=True):

    """
    Apply a low-pass Butterworth filter on offline FicTrac data.
    Plot filtered vs. non-filtered data.
    Purpose is to assess filter parameters on data.

    Parameters:
    -----------
    df (DataFrame): Filtered dataframe of FicTrac data generated from parse_dats().
        Should have columns with the "filtered_" prefix.

    val_cols (list): List of column names from `df` to be Fourier-transformed.

    time_col (str): Column name from `df` that specifies time.

    order (int): Order of the filter.

    cutoff_freq (float): The cutoff frequency for the filter in Hz.

    val_labels (list): List of labels for the plot's y-axis. If None, will
        be a formatted version of cmap_cols. Default is None.

    time_label (str): Label for the plot's time-axis.

    view_perc (float): Specifies how much of the data to plot as an initial
        percentage. Useful for assessing the effectieness of the filter over longer
        timecourses. Default is set to 1, i.e. plot the data over the entire
        timecourse. Must be a value between 0 and 1.

    theme (str or None): A pre-defined colour theme for plotting. If None,
        does not apply a theme. Default is None.

    save_path_to (str): Absolute path to which to save the plots as .png and .svg files.
        If None, will not save the plots. Default is None.

    show_plots (bool): If True, will show plots, but will not output Bokeh plotting
        objects. If False, will not show plots, but will output Bokeh plotting objects.
        If both save and show_plots are True, .html plots will be generated, in addition
        to the .png plots. Default is True.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    assert (0 <= view_perc <= 100), \
        f"The view percentage, {view_perc}, must be between 0 and 100."
    assert (len(df.filter(regex="^filtered_").columns) > 0), \
        "At least one column in the dataframe must begin with 'filtered_'"

    # Format axes labels:
    if time_label == None:
        time_label = time_col.replace("_", " ")
    if val_labels == None:
        val_labels = [val_col.replace("_", " ") for val_col in val_cols]

    # View the first _% of the data:
    domain = int(view_perc/100 * len(df[time_col]))

    plots = []
    for i, val_col in enumerate(val_cols):
        assert (len(df[time_col] == len(df[val_col]))), \
            "time and vals are different lengths! They must be the same."
        assert (time_col in df), \
            f"The column, {time_col}, is not in the input dataframe."
        assert (val_col in df), \
            f"The column, {val_col}, is not in the input dataframe."
        assert ("filtered_" in val_col), \
            f"The column, {val_col}, does not begin with the 'filtered_' prefix."

        # Plot:
        p = figure(
            width=1600,
            height=500,
            x_axis_label=time_label,
            y_axis_label=val_labels[i]
        )
        p.line(
            x=df[time_col][:domain],
            y=df[val_col.replace("filtered_","")][:domain],
            color=bokeh.palettes.brewer["Paired"][3][0],
            legend_label="raw"
        )
        p.line(
            x=df[time_col][:domain],
            y=df[val_col][:domain],
            color=bokeh.palettes.brewer["Paired"][3][1],
            legend_label="filtered"
        )
        p.title.text = f"first {view_perc}% with butterworth filter: cutoff = {cutoff_freq} Hz, order = {order}"
        p.title.text_font_size = "14pt"
        p.yaxis.axis_label_text_font_size = "12pt"
        p.yaxis.axis_label_text_font_size = "12pt"
        p.xaxis.axis_label_text_font_size = "12pt"
        load_plot_theme(p, theme=theme, has_legend=True)

        # Output:
        if save_path_to is not None:
            filename = join(save_path_to, val_col)
            p.output_backend = "svg"
            export_svgs(p, filename=filename + ".svg")
            export_png(p, filename=filename + ".png")
            output_file(filename=filename + ".html",
                        title=filename)

        if show_plots == True:
            # In case this script is run in Jupyter, change output_backend
            # back to "canvas" for faster performance:
            p.output_backend = "canvas"
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def plot_trajectory(df, cmap_cols, low=0, high_percentile=95, respective=False,
                    cmap_labels=None,
                    order=2, cutoff_freq=4,
                    palette=cm["thermal"], size=2.5, alpha=0.3,
                    theme=None,
                    show_start=False,
                    save_path_to=None, show_plots=True):

    """
    Plot XY coordinates of the individual with a linear colourmap for a each element
    in `cmap_cols`.

    Parameters:
    -----------
    df (DataFrame): Dataframe of FicTrac data generated from parse_dats()

    low (float): The minimum value of the colour map range. Any value below the set
        'low' value will be 'clamped', i.e. will appear as the same colour as
        the 'low' value. The 'low' value must be 0 or greater. Default value
        is 0.

    high_percentile (float): The max of the colour map range, as a percentile of the
        'cmap_col' variable's distribution. Any value above the 'high_percentile'
        will be clamped, i.e. will appear as the same colour as the
        'high_percentile' value. E.g. if set to 95, all values below the 95th
        percentile will be mapped to the colour map, and all values above the
        95th percentile will be clamped.

    respective (bool): If True, will re-scale colourmap for each individual to
        their respective 'high_percentile' cut-off value. If False, will use
        the 'high_percentile' value computed from the population, i.e. from `df`.
        Default is False.

    cmap_cols (list): List of column names from `df` to be colour-mapped.

    cmap_labels (list): List of labels for the plots' colourbars. If None, will
        be a formatted version of cmap_cols. Default is None.

    order (int): Order of the filter.

    cutoff_freq (float): The cutoff frequency for the filter in Hz.

    palette (list): A list of hexadecimal colours to be used for the colour map.

    size (float): The size of each datapoint specifying XY location.

    alpha(float): The transparency of each datapoint specifying XY location.
        Must be between 0 and 1.

    theme (str or None): A pre-defined colour theme for plotting. If None,
        does not apply a theme. Default is None.

    show_start (bool): If True, will plot a marking to explicitly denote the start
        site. Default is False.

    save_path_to (str): Absolute path to which to save the plots as .png and .svg files.
        If None, will not save the plots. Default is None.

    show_plots (bool): If True, will show plots, but will not output Bokeh plotting
        objects. If False, will not show plots, but will output Bokeh plotting objects.
        If both save and show_plots are True, .html plots will be generated, in addition
        to the .png plots. Default is True.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    assert (low >= 0), "The low end of the colour map range must be non-negative"
    assert ("X_mm" in df), "The column, 'X_mm', is not in the input dataframe."
    assert ("Y_mm" in df), "The column, 'Y_mm', is not in the input dataframe."

    # Format axes labels:
    if cmap_labels == None:
        cmap_labels = [cmap_col.replace("_", " ") for cmap_col in cmap_cols]

    plots = []
    for i, cmap_col in enumerate(cmap_cols):

        assert (cmap_col in df), \
            f"The column, {cmap_col}, is not in the input dataframe."
        assert (len(df["X_mm"] == len(df["Y_mm"]))), \
            "X_mm and Y_mm are different lengths! They must be the same."

        if respective == False:
            # Normalize colourmap range to population:
            high = np.percentile(df[cmap_col], high_percentile)
        elif respective == True:
            # Individual animal sets its own colourmap range:
            high = np.percentile(df[cmap_col], high_percentile)

        source = ColumnDataSource(df)

        mapper = linear_cmap(field_name=cmap_col,
                             palette=palette,
                             low=low,
                             high=high)

        p = figure(width=800,
                    height=800,
                    x_axis_label="X (mm)",
                    y_axis_label="Y (mm)")

        p.circle(source=source,
                 x="X_mm",
                 y="Y_mm",
                 color=mapper,
                 size=size,
                 alpha=alpha)

        if show_start == True:
            # Other options include .cross, .circle_x, and .hex:
            p.circle(x=df["X_mm"][0],
                     y=df["Y_mm"][0],
                     size=12,
                     color="darkgray",
                     fill_alpha=0.5)

        # TODO: also change colorbar labels so max has =< symbol
        # TODO: Change background colour of colour bar, according to theme
        color_bar = ColorBar(color_mapper=mapper['transform'],
                             title=cmap_labels[i],
                             title_text_font_size="7pt",
                             width=10,
                             background_fill_color="#f8f5f2",
                             location=(0,0))

        p.add_layout(color_bar, "right")
        p.title.text_font_size = "14pt"
        p.xaxis.axis_label_text_font_size = '10pt'
        p.yaxis.axis_label_text_font_size = '10pt'
        load_plot_theme(p, theme=theme)

        # Output:
        if save_path_to is not None:
            filename = save_path_to + f"fictrac_XY_{cmap_col}"
            p.output_backend = "svg"
            export_svgs(p, filename=filename + ".svg")
            export_png(p, filename=filename + ".png")
            output_file(filename=filename + ".html",
                        title=filename)

        if show_plots == True:
            # In case this script is run in Jupyter, change output_backend
            # back to "canvas" for faster performance:
            p.output_backend = "canvas"
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def plot_trajectories(df, cmap_cols, low=0, high_percentile=95, respective=False,
                      cmap_labels=None,
                      order=2, cutoff_freq=4,
                      palette=cm["thermal"],
                      other_palette=cm["ice"],
                      size=2.5, alpha=0.3, theme=None,
                      show_start=False,
                      save_path_to=None, show_plots=True):

    """
    Plot XY coordinates of two agents.
    Plots a linear colourmap for a each element in `cmap_cols`, for both agents.

    Parameters:
    ------------
    The parameters of this function are identical to `plot_trajectory`, but with the
    following addition:

    other_palette (list): A list of hexadecimal colour to be used for the other agent's colour map.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    assert ("other_X_mm" in df), "The column, 'other_X_mm' is not in the input dataframe"
    assert ("other_Y_mm" in df), "The column, 'other_Y_mm' is not in the input dataframe"

    # Format axes labels:
    if cmap_labels == None:
        cmap_labels = [cmap_col.replace("_", " ") for cmap_col in cmap_cols]

    plots = plot_trajectory(df, cmap_cols, low=low, high_percentile=high_percentile, respective=False,
                            cmap_labels=cmap_labels,
                            order=order, cutoff_freq=cutoff_freq,
                            palette=palette, size=size, alpha=alpha,
                            theme=theme,
                            show_start=show_start,
                            save_path_to=save_path_to, show_plots=False)

    source = ColumnDataSource(df)

    for i,(cmap_col, p) in enumerate(zip(cmap_cols, plots)):

        if respective is False:
            high = np.percentile(df[cmap_col], high_percentile)

        # Plot other agent:
        other_mapper = linear_cmap(field_name=cmap_col,
                        palette=other_palette,
                        low=low,
                        high=high)

        # TODO: also change colorbar labels so max has =< symbol
        # TODO: Change background colour of colour bar, according to theme
        other_color_bar = ColorBar(color_mapper=other_mapper['transform'],
                                   title="robot " + cmap_labels[i],
                                   title_text_font_size="7pt",
                                   background_fill_color="#f8f5f2",
                                   width=10,
                                   location=(0,0))

        p.add_layout(other_color_bar, "right")

        p.circle(source=source,
                 x="other_X_mm",
                 y="other_Y_mm",
                 color=other_mapper,
                 size=size,
                 alpha=alpha)

        # Output:
        if save_path_to is not None:
            filename = save_path_to + f"fictrac_XY_{cmap_col}"
            p.output_backend = "svg"
            export_svgs(p, filename=filename + ".svg")
            export_png(p, filename=filename + ".png")
            output_file(filename=filename + ".html",
                        title=filename)

        if show_plots == True:
            # In case this script is run in Jupyter, change output_backend
            # back to "canvas" for faster performance:
            p.output_backend = "canvas"
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def plot_histograms(df, cols=None, labels=None,
                    cutoff_percentile=95,
                    theme=None,
                    save_path_to=None, show_plots=True):

    """
    Generate histograms for multiple variables. Originally written for FicTrac data.

    Parameters:
    -----------
    df (DataFrame): Dataframe

    cols (list): List of strings specifying column names in `df`. If None,
        uses default columns that specify real-world and 'lab' kinematic measurements.
        See `banned_substrings`.
        Otherwise, will use both input arguments AND the default columns. Default is None.

    labels (list): List of strings specifying the labels for the histograms' x-axes.
        Its order must correspond to 'cols'.

    cutoff_percentile (float): Specifies the percentile of the AGGREGATE population data.
        Plots a line at this value. Default is 95th percentile.

    theme (str or None): A pre-defined colour theme for plotting. If None,
        does not apply a theme. Default is None.

    save_path_to (str): Absolute path to which to save the plots as .png and .svg files.
        If None, will not save the plots. Default is None.

    show_plots (bool): If True, will show plots, but will not output Bokeh plotting
        objects. If False, will not show plots, but will output Bokeh plotting objects.
        If both save and show_plots are True, .html plots will be generated, in addition
        to the .png plots. Default is True.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    assert ("ID" in df), "The column 'ID' is not in in the input dataframe."

    ok_cols = ban_columns_with_substrings(df)

    if cols == None:
        cols = ok_cols
    else:
        cols = ok_cols + cols
        for col in cols:
            assert (col in df), f"The column, {col}, is not in the input dataframe."

    if labels == None:
        labels = [col.replace("_", " ") for col in cols]

    plots = []
    for i, col in enumerate(cols):
        p = iqplot.histogram(data=df,
                             cats=['ID'],
                             val=col,
                             density=True,
                             width=1000,
                             height=500)

        cutoff_line = Span(location=np.percentile(df[col], cutoff_percentile),
                           dimension="height",
                           # line_color="#e41a1c",
                           line_color = "#775a42",
                           line_dash="dashed",
                           line_width=2)

        p.legend.location = "top_right"
        p.legend.title = "ID"
        load_plot_theme(p, theme=theme, has_legend=True)
        p.title.text = f" with aggregate {cutoff_percentile}% mark"
        p.xaxis.axis_label = labels[i]
        p.xaxis.axis_label_text_font_size = "12pt"
        p.yaxis.axis_label_text_font_size = "12pt"
        p.add_layout(cutoff_line)

        # Output:
        if save_path_to is not None:
            filename = join(save_path_to, f"fictrac_histogram_by_ID_{col}")
            p.output_backend = "svg"
            export_svgs(p, filename=filename + ".svg")
            export_png(p, filename=filename + ".png")
            output_file(filename=filename + ".html",
                        title=filename)

        if show_plots == True:
            # In case this script is run in Jupyter, change output_backend
            # back to "canvas" for faster performance:
            p.output_backend = "canvas"
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def plot_ecdfs(df, cols=None, labels=None,
               cutoff_percentile=95,
               theme=None,
               save_path_to=None, show_plots=True):

    """
    Generate ECDFs for multiple variables. Originally written for FicTrac data.

    Parameters:
    -----------
    df (DataFrame): Dataframe

    cols (list): List of strings specifying column names in 'df'. If None,
        uses default columns that specify real-world and 'lab' kinematic measurements.
        See `banned_substrings`
        Otherwise, will use both input arguments AND the default columns. Default is None.

    labels (list): List of strings specifying the labels for the ECDFs' x-axes.
        Its order must correspond to 'cols'.

    cutoff_percentile (float): Specifies the percentile of the AGGREGATE population data.
        Plots a line at this value. Default is 95th percentile.

    theme (str or None): A pre-defined colour theme for plotting. If None,
        does not apply a theme. Default is None.

    save_path_to (str): Absolute path to which to save the plots as .png and .svg files.
        If None, will not save the plots. Default is None.

    show_plots (bool): If True, will show plots, but will not output Bokeh plotting
        objects. If False, will not show plots, but will output Bokeh plotting objects.
        If both save and show_plots are True, .html plots will be generated, in addition
        to the .png plots. Default is True.

    Returns:
    ---------
    if show_plots is True: will show plots instead of outputting bokeh.plotting.figure object.
    if show_plots is False: will output bokeh.plotting.figure objects, instead of showing plots.
    if save_path_to is not None: will save .png plots to specified path.
    if save_path_to is None: will not save plots.
    """

    assert ("ID" in df), "The column 'ID' is not in in the input dataframe."

    ok_cols = ban_columns_with_substrings(df)

    if cols == None:
        cols = ok_cols
    else:
        cols = ok_cols + cols
        for col in cols:
            assert (col in df), f"The column, {col}, is not in the input dataframe."

    if labels == None:
        labels = [col.replace("_", " ") for col in cols]

    plots = []
    for i, col in enumerate(cols):
        p = iqplot.ecdf(data=df,
                        cats=["ID"],
                        val=col,
                        kind="colored",
                        width=1000,
                        height=500)

        cutoff_line = Span(location=np.percentile(df[col], cutoff_percentile),
                           dimension="height",
                           line_color="#775a42",
                           line_dash="dashed",
                           line_width=2)

        p.legend.location = 'top_right'
        p.legend.title = "ID"
        load_plot_theme(p, theme=theme, has_legend=True)
        p.title.text = f" with aggregate {cutoff_percentile}% mark"
        p.xaxis.axis_label = labels[i]
        p.xaxis.axis_label_text_font_size = "12pt"
        p.yaxis.axis_label_text_font_size = "12pt"
        p.add_layout(cutoff_line)

        # Output:
        if save_path_to is not None:
            filename = save_path_to + f"fictrac_ecdfs_{col}"

            p.output_backend = "svg"
            export_svgs(p, filename=filename + ".svg")
            export_png(p, filename=filename + ".png")
            output_file(filename=filename + ".html",
                        title=filename)

        if show_plots == True:
            # In case this script is run in Jupyter, change output_backend
            # back to "canvas" for faster performance:
            p.output_backend = "canvas"
            show(p)

        # In case show_plots is False:
        plots.append(p)

    if show_plots == False:
        return plots

def add_stimulus_annotation (p, style,
                             start, end, top, bottom,
                             alpha, level="underlay"):

    """

    Add a stimulus annotation to a Bokeh plotting object.

    Parameters:
    -----------
    p: A Bokeh plotting object
    style (str): One of three stimulus annotation styles: 1) "horiz_bar", which
        is a horizontal bar that denotes the duration of the stimulus,
        2) "background_box", which is an underlaid box that denotes the
        duration of the stimulus, or 3) "double_bars" which is a pair of
        vertical bars that denote the start and end of the stimulus.
    start (fl): Beginning of stimulus presentation in data coordinates.
    end (fl): End of stimulus presentation in data coordinates.
    top (fl): Applies only if `style`="horiz_bar". The top of the horizontal bar
        in data coordinates.
    bottom (fl): Applies only if `style`="horiz_bar". The bottom of the horizontal
        bar in data coordinates.
    alpha (fl): The transparency of the stimulus annotation. Must be between
        0 and 1.
    level (str): The display level of the stimulus annotation, relative to the plot.
        Default is "underlay". If `style`="background_box", can only be "underlay".

    Returns:
    --------
    `p` with stimulus annotation.

    """

    # TODO: Add None option to pass default colours, which an be overriden by arg
    # TODO: "background_box" requires top and bottom as args, even though it doesn't use them; incorporate None

    assert (style == "horiz_bar" or "background_box" or "double_bars"), \
        f"{style} is not a valid entry for `style`. Please input either \
        'horiz_bar', 'background_box', or 'double_bars'."

    if style == "horiz_bar":
        bar = BoxAnnotation(left=start,
                            right=end,
                            top=top,
                            bottom=bottom,
                            fill_alpha=alpha,
                            fill_color="#787878",
                            level=level)
        p.add_layout(bar)

    elif style == "background_box":
        box = BoxAnnotation(left=start,
                            right=end,
                            fill_alpha=alpha,
                            fill_color="#B6A290",
                            level="underlay")
        p.add_layout(box)

    elif style == "double_bars":
        line_colour = "#775a42"
        start_line = Span(location=start,
                          dimension="height",
                          line_color=line_colour,
                          line_dash="dotted",
                          line_width=2)
        end_line = Span(location=end,
                        dimension="height",
                        line_color=line_colour,
                        line_dash="dotted",
                        line_width=2)
        p.add_layout(start_line)
        p.add_layout(end_line)

    else:
        print()

    return p

def plot_aggregate_trace(df, group_by, val_col, time_col,
                         val_label, time_label,
                         palette,
                         aggregation_method="mean",
                         y_range=None,
                         legend_labels=None, theme=None,
                         mean_alpha=0.7, id_alpha=0.008,
                         line_width=5,
                         round_to=0, f_steps=1
                         ):

    """

    # TODO: ADD DOCS
    * N.B. Will ALWAYS make a legend.

    """

    assert ("ID" in df), "The column 'ID' is not in in the input dataframe."
    assert ("trial" in df), "The column 'trial' is not in in the input dataframe."

    # TODO: Add None option for args for val_label and time_label
    # TODO: Add show option and save option

    p = figure(width=1000,
               height=400,
               y_range=y_range,
               x_axis_label=time_label,
               y_axis_label=val_label)

    grouped = df.groupby(group_by)

    for i, ((name,group), hue) in enumerate(zip(grouped, palette)):

        # Make dataframes:
        agg_df = aggregate_trace(group,
                                 ["trial", time_col],
                                 method=aggregation_method,
                                 round_to=round_to, f_steps=f_steps
                                 )
        grouped_by_id = group.groupby(["ID"])

        assert len(palette)==len(grouped), \
            f"The lengths of `palette`, and `df.groupby({grouped})` must be equal."

        if legend_labels==None:
            legend_label = f"{name} | n={len(grouped_by_id)}"
        else:
            assert len(grouped)==len(legend_labels), \
                f"The lengths of `legend_labels`, `palette`, `df.groupby({grouped})` \
                must be equal."
            legend_label =  legend_labels[i]

        # Mean trace:
        p.line(x=agg_df[time_col], y=agg_df[val_col],
               legend_label=legend_label,
               color=hue,
               line_width=line_width,
               alpha=mean_alpha)

        # ID traces:
        for _, id_group in grouped_by_id:

            p.line(x=id_group[time_col], y=id_group[val_col],
                   color=hue,
                   line_width=1,
                   alpha=id_alpha,
                   legend_label=legend_label,
                   level="underlay")

    if theme==None:
        pass
    else:
        load_plot_theme(p, theme=theme, has_legend=True)

    return p


import json
import os
import pathlib
import pandas as pd
from glob import glob
import rouge
import nltk
import matplotlib.pyplot as plt
from scipy import stats

presumm_threshold = 0.2

def get_chapter_data(line):
	chapter = json.loads(line)
	return {
        'book_id': chapter['book_id'].replace(" ", "_").lower(),
        'source': chapter['source']
    }

def sort_by_score(x):
		return x["prediction"]

def get_text_and_scores(result_path):
	df = pd.read_csv(result_path, sep='\t', engine='python')
	text = df["text"].astype(str)
	predictions = df["score"].astype(float)
	assert len(predictions) == len(text)
	# print(predictions)
	# print(text)
	return {"predictions": list(predictions), "text": list(text), "filename": result_path}

def get_human_summary(summary_path):
    with open("../../booksum/scripts/" + summary_path, encoding='utf-8') as f:
        summary_json = json.load(f)
        return summary_json["summary"]

def generate_concat_summary(related_files, k):
	cat_related_files = []
	for related_file in related_files:
		cat_related_files.extend([{"prediction": related_file["predictions"][idx],\
			 "text": related_file["text"][idx]} for idx in range(len(related_file["predictions"]))])
	cat_related_files.sort(key=sort_by_score, reverse=True)
	return cat_related_files[:((k+1)*len(related_files))]

def generate_each_summary(related_files, k):
	generated_summary = []
	related_files_list = []
	for related_file in related_files:
		related_files_list.append([{"prediction": related_file["predictions"][idx],\
			 "text": related_file["text"][idx]} for idx in range(len(related_file["predictions"]))])
	for related_file in related_files_list:
		related_file.sort(key=sort_by_score, reverse=True)
		generated_summary.extend(related_file[:(k+1)])
	return generated_summary

def generate_threshold_summary(related_files, k):
	global presumm_threshold
	generated_summary = []
	for related_file in related_files:
		generated_summary.extend([{"prediction": related_file["predictions"][idx],\
			 "text": related_file["text"][idx]} for idx in range(len(related_file["predictions"])) \
				if related_file["predictions"][idx] > presumm_threshold])
	return generated_summary

def calculate_F1():
	global chapters_data
	max_k = 10
	used_files = []
	results= {
		"each": {
			"data": [], # Shape: [k, f1_score, summary_path]
			"means": [], # Shape: float

		},
		"concat": {
			"data": [], # Shape: [k, f1_score, summary_path]
			"means": [], # Shape: float

		},
		"threshold": {
			"data": [], # Shape: [k, f1_score, summary_path]
			"means": [], # Shape: float

		},
		"p_value": [],  # Shape: float
	}
	modes = ["each", "concat", "threshold"]
	threshold_f1_scores = []

	book_data = []
	f = open(pathlib.Path("../../booksum/alignments/book-level-summary-alignments/book_summaries_aligned_all.jsonl"), \
				encoding='utf-8')
	for line in f:
		content = json.loads(line)
		summary_path = content['summary_path']
		book_title = content['title']
		source = content['source']
		# Format title.
		book_title = book_title.replace(':', "").replace(" ", "_").lower()

		chapters = [x for x in chapters_data if x['book_id'].split('.')[0] == book_title and x['source'] == source]
		paths = [pathlib.Path("../../PreSumm/results/booksum_summaries/20000/").joinpath( \
			"ext_bertbooksum_summaries." + chapter['book_id'].strip() + "." + chapter['source'].strip() + '.tsv') for chapter in chapters]
		related_files = []
		for path in paths:
			try:
				related_files.append(get_text_and_scores(path))
			except:
				# print("Could not open {}.".format(path))
				pass

		book_data.append({
					"data": related_files,
					"summary_path": summary_path,
					"book_title": book_title,
					"source": source })
		used_files.extend(related_files)
	f.close()

	for k in range(max_k):

		for mode in modes:

			curr_data_list = []

			book_count = 0

			for book in book_data:
				summary_path = book['summary_path']
				book_title = book['book_title']
				source = book['source']

				try:
					human_summary = get_human_summary(summary_path)
					# print("Summary Length : {}".format(len(human_summary)))
				except:
					human_summary = None
					print("Could not find summary.")

				current_book_files = book['data']

				if len(chapters) == 0:
					print("No files found.")
					continue

				elif (human_summary is None):
					print("Files exist but no summary.")
					print(summary_path)

				if mode == "concat":
					generated_summary_sents = generate_concat_summary(current_book_files, k)
				elif mode == "each":
					generated_summary_sents = generate_each_summary(current_book_files, k)
				elif mode == "threshold":
					if k == 0:
						generated_summary_sents = generate_threshold_summary(current_book_files, k)
					else:
						curr_data_list.append([k, threshold_f1_scores[book_count], summary_path])
						book_count += 1
						continue
				else:
					print("Invalid mode '{}'".format(mode))
					return False

				#Concat summary sentences into one string.
				generated_summary = ""
				for sentence in generated_summary_sents:
					generated_summary = generated_summary + " " + sentence["text"]

				# Removes first " "
				generated_summary = generated_summary[1:]

				evaluator = rouge.Rouge(metrics=['rouge-n'],
							max_n=1,
							limit_length=False)
				scores = evaluator.get_scores(generated_summary, human_summary)

				if mode == "threshold":
					threshold_f1_scores.append(scores['rouge-1']['f'])

				# print(scores['rouge-1'])
				curr_data_list.append([ k, scores['rouge-1']['f'], summary_path])
				book_count += 1

			results[mode]["data"].append(curr_data_list)

			ROUGE_list = [curr_data_item[1] for curr_data_item in curr_data_list]
			ROUGE_mean = sum(ROUGE_list) / len(ROUGE_list)
			results[mode]["means"].append(ROUGE_mean)

			print("Mode: {}".format(mode))
			print("Books covered: {}".format(book_count))
			print("k: {}".format(k+1))
			print("Mean ROUGE-1 F1: {}".format(ROUGE_mean))
			print()

		each = [result_item[1] for result_item in results["each"]["data"][k]]
		concat = [result_item[1] for result_item in results["concat"]["data"][k]]
		paired_t_test = stats.ttest_rel(each, concat)
		results["p_value"].append(paired_t_test.pvalue)
		print("P Value: {} < 0.05 = {}".format(paired_t_test.pvalue, str(paired_t_test.pvalue < 0.05)))
		print()

	# Comment these out to avoid saving the csv files.
	# for mode in modes:
	# 	flattened_results = [item for sublist in results[mode]["data"] for item in sublist]
	# 	df = pd.DataFrame(flattened_results, columns=["k", "ROUGE-1 F1", "summary path"])
		# df.to_csv("../csv_results/booksum_summaries/booksumsumm-" + mode + "-results.csv") # Save file.

	# Create graph.
	x = [i + 1 for i in range(max_k)]
	for mode in modes:
		plt.plot(x, results[mode]["means"], marker='.')
	plt.title("Similarity between Generated Summary Approaches and Human Summary")
	plt.xlabel("Number of Sentences Per Chapter Summary (k)")
	plt.locator_params(axis="x", integer=True)
	plt.ylabel("Mean ROUGE-1 F1 Score")
	plt.axhline(0.475, color='gray', linestyle='--', label="Human Summary Mean")
	plt.legend(labels=["Top k Sentence(s) per Chapter Summary", "Top k*j Sentence(s)", "Saliency Thresholding"])
	# plt.savefig("booksum_summaries.pdf")
	plt.show(block=True)

chapters_test_split = open(pathlib.Path("../../booksum/alignments/chapter-level-summary-alignments/chapter_summary_aligned_all_split.jsonl"), \
     encoding='utf-8')
chapters_data = [get_chapter_data(line) for line in chapters_test_split]

def main():
	input_path_list = glob(os.path.normpath("../../PreSumm/results/booksum_summaries/20000/") + "/*")
	print("Evaluation results from {} files".format(len(input_path_list)))

	calculate_F1()

if __name__ == "__main__":
	nltk.download('punkt')

	main()


import os
import ui, events, language
import widgets as wg
import sb.widget.button as wb
import sb.widget.list
import sb.aodb as ao
import sb.profdb as pr
import widgets

MAPS = 'tab-maps'

# React to selection notifications.
listenSelections = True

# The map list box (has columns).
mapListBox = None

def init():
    # Create the Maps page.
    area = ui.createTab(MAPS)
    area.setBorderDirs(ui.BORDER_NOT_BOTTOM)
    area.setWeight(1)

    global mapListBox
    area.setBorderDirs(ui.BORDER_NOT_BOTTOM)
    mapListBox = area.createList('maps-list', sb.widget.list.List.STYLE_COLUMNS)

    # The columns.
    mapListBox.addColumn('maps-list-icon', 20)
    mapListBox.addColumn('maps-list-identifier', 180)
    mapListBox.addColumn('maps-list-count')

    # Prepare icons.
    global iconManager, mapIcons
    iconManager = widgets.IconManager(16, 16)
    mapIcons = [iconManager.get('unchecked'), iconManager.get('checked')]
    mapListBox.setImageList(iconManager.getImageList())

    # Some buttons in the bottom.
    area.setWeight(0)
    area.setBorderDirs(ui.BORDER_NOT_TOP)
    buttonArea = area.createArea(alignment=ui.ALIGN_HORIZONTAL, border=4)
    buttonArea.setBorderDirs(ui.BORDER_TOP | ui.BORDER_RIGHT)
    buttonArea.setWeight(0)
    clearButton = buttonArea.createButton('maps-list-clear', wb.Button.STYLE_MINI)
    clearButton.resizeToBestSize()
    refreshButton = buttonArea.createButton('refresh-addon-database', wb.Button.STYLE_MINI)
    refreshButton.resizeToBestSize()

    buttonArea.setWeight(1)
    buttonArea.addSpacer()
    buttonArea.setWeight(0)
    buttonArea.setBorderDirs(ui.BORDER_TOP | ui.BORDER_LEFT)
    buttonArea.createButton('show-addon-paths')

    # Listen to our commands.
    events.addCommandListener(handleCommand, ['maps-list-clear'])

    events.addNotifyListener(handleNotify, ['active-profile-changed',
                                            'maps-list-selected',
                                            'maps-list-deselected',
                                            'addon-attached',
                                            'addon-detached',
                                            'addon-database-reloaded',
                                            'language-changed'])

def handleCommand(event):
    """Handle a command event."""

    if event.hasId('maps-list-clear'):
        mapListBox.selectItem(None)

def handleNotify(event):
    """Handle a notification event."""

    global listenSelections

    if event.hasId('active-profile-changed'):
        refreshList()

    if event.hasId('maps-list-selected') and listenSelections:
        pr.getActive().useAddon(event.getSelection())

    if event.hasId('maps-list-deselected'):
        pr.getActive().dontUseAddon(event.getDeselection())

    if event.hasId('addon-attached'):
        listenSelections = False
        mapListBox.selectItem(event.getAddon())
        mapListBox.setItemImage(event.getAddon(), 1)
        listenSelections = True

    if event.hasId('addon-detached'):
        listenSelections = False
        mapListBox.deselectItem(event.getAddon())
        mapListBox.setItemImage(event.getAddon(), 0)
        listenSelections = True

    if event.hasId('addon-database-reloaded'):
        refreshList()

    if event.hasId('language-changed'):
        mapListBox.retranslateColumns()

def refreshList():
    """Fill the maps list with addons."""

    # Clear the list.
    mapListBox.clear()

    wads = [a for a in ao.getAvailableAddons(pr.getActive())
            if a.getType() == 'addon-type-wad' and a.isPWAD()]

    # Translate addon names.
    wads.sort(lambda a, b: cmp(a.getId(), b.getId()))

    for wad in wads:
        # TODO: More information titles.
        if not language.isDefined(wad.getId()):
            visibleName = language.translate(wad.getId())
        else:
            visibleName = os.path.basename(wad.getContentPath())
        mapListBox.addItemWithColumns(wad.getId(),
                                      0,
                                      visibleName,
                                      wad.getShortContentAnalysis())

    prof = pr.getActive()
    usedAddons = prof.getUsedAddons()
    theFirst = True

    # Select the addons currently attached to the profile.
    # Also make sure the first one is visible.
    for addonId in usedAddons:
        mapListBox.selectItem(addonId)
        mapListBox.setItemImage(addonId, 1)
        if theFirst:
            # Make sure it's visible.
            mapListBox.ensureVisible(addonId)
            theFirst = False


# ======================================================================
# Pyroclastic Flow
#   Advent of Code 2022 Day 17 -- Eric Wastl -- https://adventofcode.com
#
# Python implementation by Dr. Dean Earl Wright III
# ======================================================================

# ======================================================================
#                         s i m u l a t i o n . p y
# ======================================================================
"Simulation for the Advent of Code 2022 Day 17 puzzle"

# ----------------------------------------------------------------------
#                                                                 import
# ----------------------------------------------------------------------
from collections import namedtuple

import chamber
import rocks

# ----------------------------------------------------------------------
#                                                                  types
# ----------------------------------------------------------------------
State = namedtuple('State', 'rock, wind, heights')

# ----------------------------------------------------------------------
#                                                              constants
# ----------------------------------------------------------------------
DEQUE_SIZE = 30

# ======================================================================
#                                                             Simulation
# ======================================================================

class Simulation(object):   # pylint: disable=R0902, R0903, R0205
    "Object for Pyroclastic Flow"

    def __init__(self, text=None, part2=False):

        # 1. Set the initial values
        self.part2 = part2
        self.text = text
        self.chamber = None
        self.rocks = None
        self.winds = None
        self.next_wind = 0

        # 2. Process text (if any)
        if text is not None and len(text) > 0:
            self.winds = text[0]
            self.chamber = chamber.Chamber(text="7", part2=part2)
            self.rocks = rocks.Rocks(text=rocks.ROCKS, part2=part2)

    def one_step(self):
        "Execute one step of the simulation"

        # 0. I need something to work with
        if not self.chamber:
            return

        # 1. If there are no falling rocks, set one falling
        if self.chamber.falling is None:
            self.chamber.add_rock(self.rocks.next_rock())

        # 2. Push the rock sideways
        self.chamber.push_rock(self.winds[self.next_wind])
        self.next_wind = (self.next_wind + 1) % len(self.winds)

        # 3. Let the rock fall
        self.chamber.move_down()

    def one_rock(self):
        "Execure the simulation until a rock comes to rest"

        # 1. Get the current number of rocks
        num_rocks = self.chamber.rocks

        # 2. Loop until we the number of rocks changes
        while num_rocks == self.chamber.rocks:

            # 3. Execute a single step of the simulation
            self.one_step()

        # 4. Return the number of fallen rocks
        return self.chamber.rocks

    def run_for_n_rocks(self, number, delta=0, verbose=False):
        "Run the simulation until the specific number of rocks come to rest"

        # 0. I need something to work with
        if not self.chamber:
            return None

        # 1. Loop until we get enough rocks
        while self.chamber.rocks < number:

            # 2. Execute the simulation until the number of rocks increase
            rocks = self.one_rock()
            if verbose and 0 == rocks % 100:
                print(f"run_for_n_rocks, {rocks} down, {number - rocks} more to go")

        # 3. Return the height of the tower of rocks
        return self.chamber.get_highest(delta=delta)

    def calculate_n_rocks(self, number, verbose=False):
        "Run the simulation until the specific number of rocks come to rest"

        # 0. I need something to work with
        if not self.chamber:
            return None

        # 1. Set up for finding a cycle
        seen = {}
        last_height = 0

        # 2. Loop until we get enough rocks (or we find a cycle)
        while self.chamber.rocks < number:

            # 3. Execute a single simulation step (side/down)
            rocks = self.one_rock()
            if 0 == rocks % 100:
                print(f"look for cycle, {rocks} down, {number - rocks} more to go")

            # 4. Generate the current state
            heights = self.chamber.relative_heights()
            next_rock = self.rocks.next_rock
            next_wind = self.next_wind
            cur_rocks = self.chamber.rocks
            cur_height = self.chamber.get_highest()
            state = State(rock=next_rock, wind=next_wind, heights=heights)

            # 5. If this is a new state, add it and continue
            if state not in seen:
                seen[state] = (cur_rocks, cur_height)
                continue

            # 6. Found a cycle
            if verbose:
                print(f"found a cycle after {self.chamber.rocks} rocks")
            start_rocks, start_height = seen[state]
            cycle_rocks = cur_rocks - start_rocks
            cycle_height = cur_height - start_height
            if verbose:
                print(f"At start of cycle: rocks={start_rocks}, height = {start_height}")
                print(f"At end of cycle: rocks={cur_rocks}, height = {cur_height}")
                print(f"  difference in rocks={cycle_rocks}, height={cycle_height}")
            cycles_to_do = (number - cur_rocks) // cycle_rocks
            add_rocks = cycles_to_do * cycle_rocks
            add_height = cycles_to_do * cycle_height
            if verbose:
                print(f"Cycles left={cycles_to_do}")
                print(f"  additional rocks={add_rocks}, height={add_height}")

            # 7. Finish off the similation
            self.chamber.rocks += add_rocks
            return self.run_for_n_rocks(number, delta=add_height)

            #
        # 3. Return the height of the tower of rocks
        return self.chamber.get_highest()

# ----------------------------------------------------------------------
#                                                  module initialization
# ----------------------------------------------------------------------
if __name__ == '__main__':
    pass

# ======================================================================
# end                    s i m u l a t i o n . p y                   end
# ======================================================================


"""
    百度贴吧爬虫
        翻页获取标题及url

    ctrl+f 响应内容中搜索
    html中绿色字体表示被注释了，xpath不会识别:
        解决方法:
           1 需要修改user-Agent换低端一些的浏览器高端浏览器有时会将html信息注释，但是他
           靠引擎仍可以玩出信息，低端浏览器不行，所以不会注释，换个低端浏览器请求头就好了

           2 将data数据的注释符号去掉data.decode().replace("<--!","").replace("-->","")

    找翻页url时尽量不要使用索引，容易变化
"""

import requests
from lxml import etree

class Tieba:
    def __init__(self, name):
        self.url = "https://tieba.baidu.com/f?ie=utf-8&kw={}".format(name)
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKi'
            't/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'

            # 'User-Agent': "Mozilla/4.0 (compatible; MSIE 5.01; windows NT 5.0; DigExt)"
        }

    def get_data(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content

    def parse_data(self, data):
        # 创建elements对象
        data = data.decode().replace("<!--", "").replace("-->", "")
        html = etree.HTML(data)

        el_list = html.xpath('//li/div/div[2]/div[1]/div[1]/a')
        # print(len(el_list))

        data_list = []

        for el in el_list:
            temp = {}
            temp['title'] = el.xpath("./text()")[0]
            temp['link'] = 'http://tieba.baidu.com' + el.xpath("./@href")[0]
            data_list.append(temp)

        # 获取下一页url
        try:
            next_url = 'https:' + html.xpath('//a[contains(text(), "下一页>")]/@href')[0]
        except:
            next_url = None

        return data_list, next_url

    def save_data(self, data_list):
        # 假装保存
        for data in data_list:
            print(data)

    def run(self):
        # url
        # headers
        next_url = self.url
        while True:
            # 发送请求获取响应
            data = self.get_data(next_url)
            # 从响应中提取数据(数据和翻页用的url)
            data_list, next_url = self.parse_data(data)

            self.save_data(data_list)

            print(next_url)
            # 判断是否终结
            if next_url == None:
                break

if __name__ == '__main__':
    a = Tieba("星际争霸")
    a.run()

python/python学习小分队打卡/课程代码/爬虫/爬虫课程1/3/13 百度贴吧.py


from dagster import get_dagster_logger, job, op
from dagster.utils import file_relative_path
import config as cfg
from dagster_airbyte import airbyte_resource, airbyte_sync_op
from dagster_dbt import dbt_cli_resource, dbt_run_op
from dagster_graphql import DagsterGraphQLClient

sync_requestor = airbyte_sync_op.configured(
    {"connection_id": cfg.db_source_requestor["connection_id"]}, name="sync_requestor"
)

sync_provider1_base_consents = airbyte_sync_op.configured(
    {"connection_id": cfg.db_source_provider1["connection_id"]},
    name="sync_provider1_base_consents",
)

DBT_PROJECT_DIR = file_relative_path(__file__, cfg.dbt["project_dir"])

base_consents = dbt_run_op.alias(name="base_consents")

@op
def exec_provider1_base_consents_job(something):
    client = DagsterGraphQLClient(
        hostname=cfg.dagster_provider1["hostname"],
        port_number=cfg.dagster_provider1["port_number"],
    )
    client.submit_job_execution("crossing_datasets_job")

@job(
    resource_defs={
        "airbyte": airbyte_resource.configured(
            {
                "host": cfg.airbyte["host"],
                "port": cfg.airbyte["port"],
            }
        ),
        "dbt": dbt_cli_resource.configured(
            {
                "project_dir": DBT_PROJECT_DIR,
                "profiles_dir": DBT_PROJECT_DIR,
                "models": ["base_consents"],
            }
        ),
    }
)
def base_consents_job():
    logger = get_dagster_logger()
    logger.debug("Starting Job...")

    exec_provider1_base_consents_job(
        sync_provider1_base_consents(
            start_after=base_consents(start_after=sync_requestor())
        )
    )


#!/usr/bin/env python3
import dendropy
import argparse
import sys

def parse_args(args=None):
    parser = argparse.ArgumentParser(description='Load & prune trees.')
    parser.add_argument('File', metavar='FILE', help='File containing trees')
    parser.add_argument('Burnin', type=int, help='How many trees to discard as burnin')
    parser.add_argument('Outf', help='Output file name')
    return parser.parse_args(args)

def main():
    retain_list, tl = get_names(file)
    newtl = tree_ops(tl, burnin, retain_list)
    print("Writing out pruned trees")
    newtl.write_to_path(outf, schema='nexus')

def get_names(file):
    '''Opens trees using Dendropy, and determines which taxa are extinct'''
    print("Reading in trees")
    tl = dendropy.TreeList.get_from_path(file, schema="nexus", preserve_underscores=True)
    print(tl)
    retain_list = []
    print("Getting list of taxa to keep")
    for name in tl.taxon_namespace:
        if str(name).endswith('X\'', 2) is False:
            retain_list.append(name)
    print("To retain:", retain_list)
    return(retain_list, tl)

def tree_ops(tl, burnin, retain_list):
    '''Extract subtrees of extant taxa'''
    print("Disgaurding burnin")
    newtl = tl[burnin:]
    newTrees = dendropy.TreeList()
    print("Extracting good taxa")
    for t in newtl:
        t = t.extract_tree_with_taxa(retain_list)
        newTrees.append(t)
    return(newTrees)

if __name__ == "__main__":
    args = parse_args()
    file = args.File
    burnin = args.Burnin
    outf = args.Outf

    main()


from collections.abc import Iterator, Generator, Iterable
import unittest

def generator_count_down(num):
    print("Starting")
    while num > 0:
        yield num
        num -= 1

class test_iterator_generator(unittest.TestCase):

    def test_iterator_generator(self):
        ## generator is subclass of iterator, iterator is subclass of iterable
        self.assertTrue(issubclass(Generator, Iterator))
        self.assertFalse(issubclass(Iterator, Generator))
        self.assertTrue(issubclass(Generator, Iterable))

    def test_generator(self):
        ## both are ways to construct generators
        g = generator_count_down(5)
        self.assertTrue(isinstance(g, Generator))
        g2 = (x for x in range(5) if x % 2 == 0)
        self.assertTrue(isinstance(g2, Generator))

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

# In[4]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import csv

# In[5]:

pwd

# In[6]:

df = pd.read_csv('C:\\Users\\thinkpad\\desktop\\case_time_series.csv')
df.head()

# In[7]:

array_2d = np.linspace(1,5,12).reshape(4,3) # create numpy 2D array

print(array_2d) # print numpy array

# In[8]:

sns.heatmap(array_2d) # create heatmap

# In[14]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import csv

# In[47]:

# Load dataset

Case_time_series_df = pd.read_csv("C:\\Users\\thinkpad\\desktop\\Case_time_series.csv")
Case_time_series_df.head()

# In[75]:

# Load dataset
case_time_series_df = pd.read_csv("C:\\Users\\thinkpad\\desktop\\case_time_series.csv")
case_time_series_df.head()

# In[76]:

array_2d = np.linspace(1,5,12).reshape(4,3) # create numpy 2D array

print(array_2d) # print numpy array

# In[77]:

sns.heatmap(array_2d) # create heatmap

# In[78]:

# set country name as index and drop Country Code, Indicator Name and Indicator Code

case_time_series_df  = case_time_series_df.drop(columns=['Total Confirmed', 'Total Recovered', 'Total Deceased'], axis=1).set_index('Date')
case_time_series_df

# In[79]:

# Create heatmap

plt.figure(figsize=(16,9))
sns.heatmap(case_time_series_df)

# In[81]:

# vmin and vmax parameter set limit of colormap

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, vmin = 0, vmax = 21)

# In[83]:

# change heatmap color using cmap

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, cmap="coolwarm")

# In[84]:

# center parameter

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, cmap="coolwarm", center = 10.0)

# In[85]:

# robust

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, robust = True)

# In[86]:

# annot (annotate) parameter

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, annot = True)

# In[61]:

# pass 2D Numpy array to annot parameter

annot_arr = np.arange(1,13).reshape(4,3) # create 2D numpy array with 4 rows and 3 columns

sns.heatmap(array_2d, annot= annot_arr)

# In[62]:

# fmp parameter - add text on heatmap cell

annot_arr = np.array([['a00','a01','a02'],
                      ['a10','a11','a12'],
                      ['a20','a21','a22'],
                      ['a30','a31','a32']], dtype = str)

sns.heatmap(array_2d, annot = annot_arr, fmt="s") # s -string, d - decimal

# In[87]:

# annot_kws parameter

plt.figure(figsize=(16,9))

annot_kws={'fontsize':10,
           'fontstyle':'italic',
           'color':"k",
           'alpha':0.6,
           'rotation':"vertical",
           'verticalalignment':'center',
           'backgroundcolor':'w'}

sns.heatmap(case_time_series_df, annot = True, annot_kws= annot_kws)

# In[88]:

# linewidths parameter - divede each cell of heatmap

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, linewidths=4)

# In[89]:

# linecolor parameter - change the color of heatmap line

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, linewidths=4, linecolor="k")

# In[90]:

# hide color bar with cbar parameter

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df, cbar = False)

# In[91]:

# change style and format of color bar with cbar_kws parameter

plt.figure(figsize=(14,14))

cbar_kws = {"orientation":"horizontal",
            "shrink":1,
            'extend':'min',
            'extendfrac':0.1,
            "ticks":np.arange(0,22),
            "drawedges":True,
           }

sns.heatmap(case_time_series_df, cbar_kws=cbar_kws)

# In[92]:

# multiple heatmaps using subplots

plt.figure(figsize=(30,10))

plt.subplot(1,3,1) # first heatmap
sns.heatmap(case_time_series_df,  cbar=False, linecolor="w", linewidths=1)

plt.subplot(1,3,2) # second heatmap
sns.heatmap(case_time_series_df,  cbar=False, linecolor="k", linewidths=1)

plt.subplot(1,3,3) # third heatmap
sns.heatmap(case_time_series_df,  cbar=False, linecolor="y", linewidths=1)

plt.show()

# In[93]:

# set seaborn heatmap title, x-axis, y-axis label and font size

plt.figure(figsize=(16,9))

ax = sns.heatmap(case_time_series_df)

ax.set(title="Heatmap",
      xlabel="Condition",
      ylabel="Date",)

sns.set(font_scale=2) # set fontsize 2

# In[94]:

# Heatmap with keyword arguments (kwargs) parameter

plt.figure(figsize = (16,9))

kwargs = {'alpha':.9,'linewidth':10, 'linestyle':'--', 'rasterized':False, 'edgecolor':'w',  "capstyle":'projecting',}

ax = sns.heatmap(case_time_series_df,**kwargs)

# In[95]:

# sns heatmap correlation

plt.figure(figsize=(16,9))

sns.heatmap(case_time_series_df.corr(), annot = True)

# In[96]:

# Upper triangle heatmap

plt.figure(figsize=(16,9))

corr_mx = case_time_series_df.corr() # correlation matrix

matrix = np.tril(corr_mx) # take lower correlation matrix

sns.heatmap(corr_mx, mask=matrix)

# In[97]:

# Lower triangle heatmap

plt.figure(figsize=(16,9))

corr_mx = case_time_series_df.corr() # correlation matrix

matrix = np.triu(corr_mx) # take upper correlation matrix

sns.heatmap(corr_mx, mask=matrix)

# In[ ]: