# import whisper
import config
import whisper_timestamped as whisper

model = whisper.load_model(config.MODEL_WHISPER_SIZE, device=config.DEVICE)

# result = whisper_timestamped.transcribe(audio=audio_file, model=model)
def transcribe_timestamped(audio):
    audio = whisper.load_audio(audio)
    audio = whisper.pad_or_trim(audio)
    mel = whisper.log_mel_spectrogram(audio).to(model.device)
    _, probs = model.detect_language(mel)
    detected_language = max(probs, key=probs.get)
    result = model.transcribe(audio, language="en")
    text = result["text"]
    return text, result


import numpy as np
from scipy.spatial import Delaunay
from viz.state import WholeParser, InitialParser
from viz.rbf.shape import Sphere
from mayavi import mlab
import matplotlib.pyplot as plt

def periodize(x):
    return (x + 0.8) % 1.6 - 0.8

class StitchedEndo:
    def __init__(self, x, focus):
        self.update(x, focus)

    def _stitch(self, x):
        n, dims = x.shape
        dx = 1.6 * np.ones((n, 1)) @ np.array([[1.0, 0.0]])
        dz = 1.6 * np.ones((n, 1)) @ np.array([[0.0, 1.0]])
        return np.concatenate((x-dx-dz, x-dz, x-dx, x), axis=0)

    def _unstitch(self, k, x):
        n = x.shape[0]
        index = k % n
        grid = k // n
        xp = x[index, :]
        sx = grid % 2 - 1
        sz = grid // 2 - 1

        dx = 1.6 * sx.reshape((-1, 1)) @ np.array([[1.0, 0.0, 0.0]])
        dz = 1.6 * sz.reshape((-1, 1)) @ np.array([[0.0, 0.0, 1.0]])
        return xp + dx + dz

    def _triangulate(self, x):
        nrm = lambda x, y: np.sqrt(np.sum((x - y)**2, axis=1))
        d = Delaunay(x)
        s = d.simplices
        md = 0.05
        a, b, c = s[:, 0], s[:, 1], s[:, 2]
        u, v, w = x[a, :], x[b, :], x[c, :]
        r = (nrm(u, v) < md) * (nrm(u, w) < md) * (nrm(v, w) < md)
        return s[r, :]

    def _nearby(self, x, z):
        nrm = lambda x, y: np.max(np.abs(x-y), axis=1)
        n = x.shape[0]
        keep = nrm(x, z) < 0.8
        indices = np.arange(n)
        return indices[keep]

    def _update(self, x, z):
        zp = periodize(z)
        d = np.array([[1, 0], [0, 0], [0, 1]]) @ (zp - z)
        q = self._stitch(x[:, [0, 2]])
        k = self._nearby(q, zp)
        xp = self._unstitch(k, x)
        t = self._triangulate(xp[:, [0, 2]])
        return xp - d, t

    def update(self, x, focus):
        self.px, self.pt = self._update(x, focus[[0, 2]])
        self.x = x

    def __iter__(self):
        yield self.px, self.pt, {'color': (1, 1, 1)}

class Spheres:
    def __init__(self, x, f, options=None):
        n, m = x.shape
        self.nc = m // 3
        q = Sphere.sample(n)
        self.t = Sphere.triangulate(q)
        self.update(x, f)
        self.options = options

    def update(self, x, f):
        self.x = x
        self.f = f

    def __iter__(self):
        options = self.options
        for i in range(self.nc):
            xc = self.x[:, i::self.nc]
            mx = np.mean(xc, axis=0)
            d = mx[[0, 2]] - self.f[[0, 2]]
            dx = periodize(d) - d
            xc[:, [0, 2]] += dx
            yield xc, self.t, {} if options is None else options(i)

class Plotter:
    def __init__(self, focus):
        self.focus = focus
        fig = mlab.figure(size=(1080, 1080), bgcolor=(1, 1, 1))
        scene = fig.scene
        scene.renderer.set(use_depth_peeling=True)
        self.fig = fig

    def _update(self, center):
        # Gritty plot update details
        mlab.clf(figure=self.fig)  # Clear figure
        rbcs_done = False

        # Redraw objects
        for src in (self.rbcs, self.plts, self.endo):
            if src is None:  # For plots without RBCs, ignore that input
                rbcs_done = True
                continue
            for x, t, kwargs in src:
                if rbcs_done is False:
                    n = x.shape[0]//2
                mlab.triangular_mesh(x[:, 0], x[:, 1], x[:, 2], t, **kwargs)
            rbcs_done = True

        # Dark grey domain frame
        fx, fy, fz = center
        mlab.triangular_mesh(np.array([fx-0.8, fx-0.8, fx+0.8, fx+0.8]),
                             1 * np.array([1.2, 1.2, 1.2, 1.2]),
                             np.array([fz-0.8, fz+0.8, fz-0.8, fz+0.8]),
                             np.array([[0, 1, 2], [1, 3, 2]]),
                             color=(0.2, 0.2, 0.2))
        mlab.triangular_mesh(np.array([fx-0.8, fx-0.8, fx+0.8, fx+0.8]),
                             0 * np.array([1.2, 1.2, 1.2, 1.2]),
                             np.array([fz-0.8, fz+0.8, fz-0.8, fz+0.8]),
                             np.array([[0, 1, 2], [1, 3, 2]]),
                             color=(0.2, 0.2, 0.2))

    def update(self):
        rbc, plt, endo = (yield)
        center = self.focus(rbc, plt, endo)
        if rbc is not None:
            self.rbcs = Spheres(rbc, center, rbc_options)
        else:
            self.rbcs = None
        self.plts = Spheres(plt, center, plt_options)
        self.endo = StitchedEndo(endo, center)
        self._update(center)
        while True:
            rbc, plt, endo = (yield)
            center = self.focus(rbc, plt, endo)
            if rbc is not None:
                self.rbcs.update(rbc, center)
            self.plts.update(plt, center)
            self.endo.update(endo, center)
            self._update(center)

def plt_options(n):
    "Plotting options for platelets"
    c = 0.5 + 0.3 * n
    if n == 0:
        return {'color': (c, c, c), 'opacity': 0.8}
    return {'color': (c, c, c), 'opacity': 0.1}

def rbc_options(n):
    "Plotting options for RBCs"
    c = 0.4 + 0.6 * n / 7
    if n == 3:
        return {'color': (c, 0, 0), 'opacity': 0.8}
    return {'color': (c, 0, 0), 'opacity': 0.1}

def plt_focus(n):
    "Sets the camera focus on a certain platelet"
    def wrapped(rbc, plt, end):
        nc = plt.shape[1] // 3
        k = n % nc
        x = plt[:, k::nc]
        return np.mean(x, axis=0)
    return wrapped

def center(*args):
    return np.array([0.8, 0.6, 0.8])

def savefig(f, fig):
    map = mlab.screenshot(figure=fig, mode='rgba', antialiased=True)
    pfig = plt.figure(figsize=(9, 9))
    plt.imshow(map, zorder=4)
    plt.axis('off')
    plt.savefig(f, transparent=True, bbox_inches='tight')
    plt.close(pfig)

def main(filename):
    parser = WholeParser(sys.argv[1])
    focus = plt_focus(0)
    plotter = Plotter(focus)
    updater = plotter.update()
    updater.send(None)
    oriented = False
    rbc = None
    for i, frame in enumerate(parser):
        t, u, p, rbc, plt, endo = frame
        f0 = focus(rbc, plt, endo)
        focal = np.array([f0[0], 0.6, f0[2]])
        updater.send((rbc, plt, endo))
        mlab.view(azimuth=-15, elevation=-45, distance=5.6,
                  focalpoint=focal, roll=0)
        if not oriented:
            input(f"{t:4.2f}ms")
            oriented = True
        else:
            print(f"{t:4.2f}ms")
        savefig(f"frames/flyover{i:03d}.png", plotter.fig)

if __name__ == '__main__':
    import sys
    main(sys.argv[1])


import os
from flask import Flask
from  sqlalchemy.sql.expression import func
from models import Workouts

def gen_workout(muscle_group, experience_level, equipment):

    data = []
    workouts = Workouts.query.filter(Workouts.bodyPart==muscle_group, Workouts.level==experience_level, Workouts.equipment==equipment).order_by(func.random()).limit(5)

    for w in workouts:
        d = []
        d.append(w.title)
        d.append(w.bodyPart)
        d.append(w.level)

        data.append(d)

    return data


import cv2

cap = cv2.VideoCapture(0)

while(cap.isOpened()):
    ret, frame = cap.read()

    cv2.imshow('Video streaming', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

print('Close')
cap.release()
cv2.destroyAllWindows()


import grpc
import dash_pb2
import dash_pb2_grpc

class DashClient:
	def __init__(self, config):
		self.config = config
		self.host_string = '{0}:{1}'.format(self.config['host'], self.config['port'])

	def UpdateDashData(self, dashData):
		with grpc.insecure_channel(self.host_string) as channel:
			stub = dash_pb2_grpc.DashControlStub(channel)
			response = stub.UpdateDashData(dashData)
			# print(response.status)

	def UpdateDashExtendedData(self, dashExtendedData):
		with grpc.insecure_channel(self.host_string) as channel:
			stub = dash_pb2_grpc.DashControlStub(channel)
			response = stub.UpdateDashExtendedData(dashExtendedData)
			# print(response.status)


import sys
from collections import deque

input = sys.stdin.readline

n = int(input())
shift = list(map(int, input().split()))
balloons = deque()

for i in range(n):
    balloons.append(i+1)

pointer = 0

while balloons:
    balloon = balloons.popleft()
    print(balloon,end=' ')
    paper = shift[balloon-1]
    if paper>=0:
        balloons.rotate(-paper+1)
    else:
        balloons.rotate(-paper)


# %load_ext autoreload
# %autoreload 2

import numpy as np
import rdkit
from rdkit import Chem

import h5py, ast, pickle
from ddc_pub import ddc_v3 as ddc
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="2"
from ddc_pub import ddc_v3 as ddc

# Import existing (trained) model
# Ignore any warning(s) about training configuration or non-seriazable keyword arguments
model_name = "models/heteroencoder_model"
model = ddc.DDC(model_name=model_name)

######################################################################
# Input SMILES to auto-encode
# Input SMILES to auto-encode
# with open('datasets/ChEMBL_validation_set', "r") as f:
#     smiles_in = f.read().splitlines()

smiles_in = ['Cc1cccn2c(CN(C)C3CCCc4ccccc43)c(C(=O)N3CCOCC3)nc12',
             'COC(=O)NN=C(c1ccc(O)cc1)C1C(=O)N(C)C(=O)N(C)C1=O',
             'CCc1cc(CC)nc(OCCCn2c3c(c4cc(-c5nc(C)no5)ccc42)CC(F)(F)CC3)n1',
             'Cc1ccc2c(C(=O)Nc3ccccc3)c(SSc3c(C(=O)Nc4ccccc4)c4ccc(C)cc4n3C)n(C)c2c1',
             'Cc1cccc(-c2ccccc2)c1Oc1nc(O)nc(NCc2ccc3occc3c2)n1',
             'Cn1nnnc1SCC(=O)NN=Cc1ccc(Cl)cc1',
             'COc1cccc(NS(=O)(=O)c2ccc(OC)c(OC)c2)c1',
             'COc1ccc(OC)c(S(=O)(=O)n2nc(C)cc2C)c1',
             'NCCCn1cc(C2=C(c3ccncc3)C(=O)NC2=O)c2ccccc21',
             'CN(C)C(=O)N1CCN(C(c2ccc(Cl)cc2)c2cccnc2)CC1']

# MUST convert SMILES to binary mols for the model to accept them (it re-converts them to SMILES internally)
mols_in = [Chem.rdchem.Mol.ToBinary(Chem.MolFromSmiles(smiles)) for smiles in smiles_in]
# Encode the binary mols into their latent representations
latent = model.transform(model.vectorize(mols_in))

# with h5py.File('datasets/output_smiles_list.hdf5', 'r') as f:
#    data = f['smiles'][()]
# len(data)
# latent = data[:10]
# data.shape
# Convert back to SMILES
smiles_out = []
for lat in latent:
    smiles, _ = model.predict(lat, temp=0)
    smiles_out.append(smiles)

for idx, smiles in enumerate(smiles_out):
    mol = Chem.MolFromSmiles(smiles)
    if mol:
        smiles_out[idx] = Chem.MolToSmiles(mol, canonical=True)
    else:
        smiles_out[idx] = "INVALID"


from pyspark.sql import functions as F

def int_fun(x):
    try:
        x = int(x)
    except (TypeError, ValueError):
        x = 0
    return x

def reduce_fun(x, y):
    x['total_views'] += y['total_views']
    x["video_ids"] += y["video_ids"]
    return x

def task_4(spark, input_file):
    df = spark.read.csv(input_file, header=True)
    df = df.filter("char_length(video_id) == 11 and video_id != 'ABOUT WIRED'")
    ch = df.select('channel_title', 'views', 'trending_date', 'video_id') \
        .groupBy('channel_title', 'video_id') \
        .agg(F.max(df.trending_date).alias('MTD'))
    views = df.select('views', 'trending_date', 'video_id') \
        .withColumnRenamed('trending_date', 'td') \
        .withColumnRenamed('video_id', 'id')
    ch_join = ch.join(views, [ch.MTD == views.td, ch.video_id == views.id]) \
        .select('channel_title', 'video_id', 'MTD', 'views')
    ch_join = ch_join.rdd
    ch_join = ch_join.map(lambda x: (x['channel_title'], {"total_views": int_fun(x[3]),
                                                          "end_date": x[2],
                                                          "video_ids": [{
                                                           "video_id": x[1],
                                                           "video_view": x[3]
                                                          }]})) \
        .reduceByKey(reduce_fun) \
        .sortBy(lambda x: -x[1]["total_views"]) \
        .take(20)
    res = list(map(lambda x: {"channel_title": x[0],
                              "total_views": x[1]["total_views"],
                              "end_date": x[1]["end_date"],
                              "video_ids": x[1]["video_ids"]}, ch_join))
    res = {"channels": res}
    return res


"""
author: Antoine Spahr

date : 05.11.2020

----------

TO DO :
"""
import os
import sys
sys.path.append('../../')
import click

from src.postprocessing.update_pred import update_Kfold_folder, update_anomaly_pred_folder

@click.command()
@click.argument("exp_folder", type=click.Path(exists=True))
@click.argument("brain_mask_folder", type=click.Path(exists=True))
@click.argument("data_path", type=click.Path(exists=True))
@click.option("--save_path", type=click.Path(exists=False), default='None', help="Where to save the adjusted experiment. Default: exp_folder_BrainOnly")
@click.option("--brain_nii/--brain_bmp", default=True, help="Whether the brain mask are provided as nifti or bmp. Defualt : as nifti.")
@click.option("--rot/--no_rot", default=True, help="Whether brain mask are rotated 90° counterclockwise. Default True.")
@click.option("--kfold/--anomaly", default=True, help="Whether to process a KFold experiment or a Single prediction. Default : Kfold.")
def main(exp_folder, brain_mask_folder, data_path, save_path, brain_nii, rot, kfold):
    """
    Post-Process a Cross-Validated or Single prediction ICH segmentation experiment by removing prediction outside the brain.
    exp_folder --> path to the experiment to convert
    brain_mask_folder --> path to the brain_mask for each volumes (as folder of .bmp or as Nifti).
    data_path --> path to the ICH dataset.
    """
    if save_path == 'None':
        save_path = '/'.join(exp_folder.split('/')[:-1]) + '_BrainOnly/'

    if kfold:
        update_Kfold_folder(exp_folder, save_path, brain_mask_folder, data_path, rot=rot, print_progress=True, verbose=True)
    else:
        update_anomaly_pred_folder(os.path.join(exp_folder, 'pred'), save_path, brain_mask_folder, brain_nii, data_path,
                                   rot=rot, print_progress=True)

if __name__ == '__main__':
    main()


#! /usr/bin/env python
"""Create train, val, and test dataframes for object detection/localization"""

import argparse
import os

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

from utils import dev_env

DIRPATH_DATA = dev_env.get('imagenet', 'dirpath_data')
DIRPATH_METADATA_LISTS = os.path.join(
    DIRPATH_DATA, 'from_urls', 'metadata_lists'
)
DIRPATH_SYNSET_LISTS = os.path.join(DIRPATH_DATA, 'synset_lists')
DIRPATH_XMLS = os.path.join(
    DIRPATH_DATA, 'from_urls', 'bbox_xmls'
)

FPATH_DF_FPATHS_IMAGES_FILTERED = os.path.join(
    DIRPATH_METADATA_LISTS, 'df_fpaths_images_filtered.csv'
)
FPATH_SYNSET_WORDS = os.path.join(
    DIRPATH_DATA, 'synset_lists', 'synset_words.txt'
)

def add_fpath_xml_column(df_fpaths_images):
    """Add a column holding the XML file holding bbox coordinates

    If the given image doesn't have a corresponding XML file, an `np.nan` will
    be used as a placeholder; these rows should be filtered later, since they
    won't be useful during training.

    :param df_fpaths_images: holds the filepaths for the imagenet images
    :type df_fpaths_images: pandas.DataFrame
    :return: df_fpaths_images with 'fpath_xml' column added
    :rtype: pandas.DataFrame
    """

    def _get_fpath_xml(fpath_image):
        """Return the `fpath_xml` for the corresponding image

        This returns `np.nan` if there is no corresponding XML.

        :param fpath_image: filepath to the imagenet image
        :type fpath_image: str
        :return: filepath to the XML, or null if there isn't one
        :rtype: str or `np.nan`
        """

        # this takes [outer directory path]/n02093056/n02093056_16088 =>
        # n02093056, which is the wordnet synset ID; the XMLs are stored by the
        # wordnet synset ID
        dirname_xml = os.path.basename(os.path.dirname(fpath_image))
        fname = os.path.basename(fpath_image)
        fname_xml = fname + '.xml'

        fpath_xml = os.path.join(DIRPATH_XMLS, dirname_xml, fname_xml)
        if os.path.exists(fpath_xml):
            return fpath_xml
        else:
            return np.nan

    df_fpaths_images['fpath_xml'] = (
        df_fpaths_images['fpath_image'].apply(_get_fpath_xml)
    )
    return df_fpaths_images

def add_label_column(df_fpaths_images):
    """Add the corresponding 'label' for each 'synset' in `df_fpaths_images`

    This translates the 'synset' column to an integer label for use in
    training, i.e. n01580077 => 17.

    :param df_fpaths_images: holds the filepaths for the imagenet images
    :type df_fpaths_images: pandas.DataFrame
    :return: df_fpaths_images with 'label' column added
    :rtype: pandas.DataFrame
    """

    df_synsets_text = pd.read_table(FPATH_SYNSET_WORDS, names=['text'])
    synsets, _ = df_synsets_text['text'].str.split(' ', 1).str
    df_synsets = pd.DataFrame(synsets)
    df_synsets.rename(columns={'text': 'synset'}, inplace=True)
    df_synsets['label'] = df_synsets.index

    df_fpaths_images = pd.merge(df_fpaths_images, df_synsets, on='synset')
    return df_fpaths_images

def parse_args():
    """Parse command line arguments

    :return: name space holding the command line arguments
    :rtype: argparse.Namespace
    """

    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--task', required=True, choices=['localization', 'detection'],
        help='Whether to use synsets from the localization or detection task.'
    )

    args = parser.parse_args()
    return args

def main():
    """Main"""

    args = parse_args()

    df_fpaths_images = pd.read_csv(FPATH_DF_FPATHS_IMAGES_FILTERED)

    df_fpaths = add_fpath_xml_column(df_fpaths_images)
    df_fpaths = add_label_column(df_fpaths)

    if args.task == 'localization':
        fpath_synsets = os.path.join(DIRPATH_SYNSET_LISTS, 'synset_words.csv')
    else:
        fpath_synsets = os.path.join(
            DIRPATH_SYNSET_LISTS, 'det_synset_words.csv'
        )
    df_synsets = pd.read_csv(fpath_synsets)
    idx_keep = df_fpaths['synset'].isin(df_synsets['synset'])
    df_fpaths = df_fpaths[idx_keep]

    # factor out null XMLs *post-splitting*, because otherwise there are
    # typically not enough in some classes to split on; note this assumes that
    # the null XMLs will be randomly distributed with respect to the seed we
    # choose
    df_train, df_test = train_test_split(
        df_fpaths, train_size=0.80, test_size=0.20,
        random_state=529, stratify=df_fpaths['synset']
    )
    df_val, df_test = train_test_split(
        df_test, train_size=0.50, test_size=0.50, random_state=529,
        stratify=df_test['synset']
    )

    idx_null_xmls_test = pd.isnull(df_test['fpath_xml'])
    df_test = df_test[~idx_null_xmls_test]
    idx_null_xmls_val = pd.isnull(df_val['fpath_xml'])
    df_val = df_val[~idx_null_xmls_val]
    idx_null_xmls_train = pd.isnull(df_train['fpath_xml'])
    df_train = df_train[~idx_null_xmls_train]

    fpath_train_set = os.path.join(
        DIRPATH_METADATA_LISTS, 'df_{}_train_set.csv'.format(args.task)
    )
    fpath_val_set = os.path.join(
        DIRPATH_METADATA_LISTS, 'df_{}_val_set.csv'.format(args.task)
    )
    fpath_test_set = os.path.join(
        DIRPATH_METADATA_LISTS, 'df_{}_test_set.csv'.format(args.task)
    )

    df_train.to_csv(fpath_train_set, index=False)
    df_val.to_csv(fpath_val_set, index=False)
    df_test.to_csv(fpath_test_set, index=False)

if __name__ == '__main__':
    main()


import argparse
import yaml
import pandas as pd

parser = argparse.ArgumentParser()
parser.add_argument('--chunk', help='chunk of data')
parser.add_argument('--params', help='path to params.yaml')
args = parser.parse_args()

def main():
    params = load_params(args.params)
    process_data(params['base']['data_dir'], args.chunk)

def process_data(data_dir: str, chunk: str):
    df = pd.read_csv(f'{data_dir}/raw/data.csv', header=None)
    for index, row in df.iterrows():
        if not any(chunk in str(cell) for cell in row):
            df.drop(index, inplace=True)
    df.to_csv(f'{data_dir}/processed/data.csv', header=False, index=False)

def load_params(filename: str):
    with open(filename, 'r') as f:
        params = yaml.safe_load(f)
        return params

if __name__ == '__main__':
    main()


# implement refernece:
# https://medium.com/@eyyu/coding-ppo-from-scratch-with-pytorch-part-2-4-f9d8b8aa938a
from collections import namedtuple, deque
import torch
import random
import numpy as np
from PTorchEnv.Typechecker import TensorTypecheck
from PTorchEnv.Typechecker import TensorTypecheck
Transition = namedtuple('Transition',
                        ('state', 'action', 'next_state', 'reward'))

class PPO_ReplayMemory(object):

    def __init__(self, capacity):
        self.recordinglength=0
        self.batch_lastobs = []             # batch observations
        self.batch_obs = []             # batch observations
        self.batch_acts = []            # batch actions
        self.batch_log_probs = []       # log probs of each action
        self.batch_rews = []            # batch rewards
        self.batch_rtgs = []            # batch rewards-to-go
        self.batch_lens = []            # episodic lengths in batch
        self.ep_rews = []
        self.steps_count=0
        self.total_timestep=0
    def ResetNotify(self):
        self.batch_rews.append(self.ep_rews)
        self.ep_rews = []  #清空所有奖励，因为这是一个on-policy的算法
        #                     可能有bug，清空后检查一下batch_rews
        self.batch_lens.append(self.steps_count + 1) #需要知道这一次reset总共走了多少步
        self.steps_count=0
        pass
    def appendnew(self,lastobs,act,state,reward):
        # 需要在此处获知episode是否终结的消息
        # 只要进行重置就是终结
        # 重置的条件是terminated或者truncated
        # 这样的话，self.max_timesteps_per_episode就必须大于
        # 游戏规则的truncated极限
        self.total_timestep+=1 #计算到update策略之前的总步数，这个值只能被优化器设0
        self.steps_count+=1
        self.recordinglength+=1
        self.batch_lastobs.append(TensorTypecheck(lastobs))
        self.batch_obs.append(state)
        self.batch_acts.append(act[0])  #神经网络输出的act值
        self.batch_log_probs.append(act[1])  #神经网络输出的act对应log值
        self.ep_rews.append(reward)
    def clearbuffer(self):
        self.batch_lastobs = []             # batch observations
        self.batch_obs = []             # batch observations
        self.batch_acts = []            # batch actions
        self.batch_log_probs = []       # log probs of each action
        self.batch_rews = []            # batch rewards
        self.batch_rtgs = []            # batch rewards-to-go
        self.batch_lens = []            # episodic lengths in batch
        self.ep_rews = []
        self.steps_count=0
        self.total_timestep=0


import logging
from urllib.parse import urljoin
import typing as t

from ..lib.externaldata import Checker, ExternalData, ExternalFile
from ..lib.utils import filter_versions

log = logging.getLogger(__name__)

GNOME_MIRROR = "https://download.gnome.org/"

def _parse_checksums(text: str) -> t.Dict[str, str]:
    result = {}
    for line in text.splitlines():
        digest, filename = line.strip().split(maxsplit=1)
        result[filename] = digest
    return result

def _is_stable(version: str) -> bool:
    major, minor = version.split(".")[:2]
    if int(major) >= 40:
        return minor not in ["alpha", "beta", "rc"]
    return (int(minor) % 2) == 0

class GNOMEChecker(Checker):
    CHECKER_DATA_TYPE = "gnome"

    async def check(self, external_data):
        external_data: ExternalData
        project_name = external_data.checker_data["name"]
        stable_only = external_data.checker_data.get("stable-only", True)
        constraints = external_data.checker_data.get("versions", {}).items()
        assert isinstance(project_name, str)

        proj_url = urljoin(GNOME_MIRROR, f"sources/{project_name}/")
        async with self.session.get(urljoin(proj_url, "cache.json")) as cache_resp:
            cache_json = await cache_resp.json()
        _, downloads, versions, _ = cache_json

        filtered_versions = versions[project_name]
        if constraints:
            filtered_versions = filter_versions(filtered_versions, constraints)

        if stable_only:
            try:
                latest_version = list(filter(_is_stable, filtered_versions))[-1]
            except IndexError:
                latest_version = filtered_versions[-1]
                log.warning(
                    "Couldn't find any stable version for %s, selecting latest %s",
                    project_name,
                    latest_version,
                )
        else:
            latest_version = filtered_versions[-1]

        proj_files = downloads[project_name][latest_version]

        tarball = next(
            proj_files[prop]
            for prop in ["tar.xz", "tar.bz2", "tar.gz"]
            if prop in proj_files
        )
        async with self.session.get(
            urljoin(proj_url, proj_files["sha256sum"])
        ) as cs_resp:
            checksums = _parse_checksums(await cs_resp.text())
        checksum = checksums[tarball.split("/")[-1]]

        new_version = ExternalFile(
            url=urljoin(proj_url, tarball),
            checksum=checksum,
            size=None,
            version=latest_version,
            timestamp=None,
        )

        external_data.set_new_version(new_version)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='index'),
    path('create_profile/', views.create_profile, name='create_profile'),
    path('profile/', views.profile, name='profile'),
    path('update_profile/<int:id>', views.update_profile, name='update_profile'),
    path('upload/project/', views.upload, name="upload"),
    path('accounts/profile/', views.home, name='profile'),
    path('project/<int:project_id>', views.project, name='project'),
    path('search/', views.search_project, name='search_project'),
    path('rate/<int:id>', views.rate, name='rate'),
    path('api/projects/', views.ProjectList.as_view()),
    path('api/profiles/', views.ProfileList.as_view()),
]


from collections import ChainMap
from copy import deepcopy
from functools import partial
from itertools import chain, product
from tempfile import NamedTemporaryFile as ntf
from xml.dom import minidom
import os
import subprocess

import networkx as nx
import spacy

class NotProcessedException(Exception):
    """The text is not processed yet. Process it before to continue."""
    pass

def entity2dict(e):
    attr = e.attributes
    return {
        attr.get('id').value: {
            'name': attr.get('name', attr.get('text', '')).value,
            'semtypes': attr.get('semtypes', '').value.split(',')
        }
    }

class SemRepWrapper:
    _doc = None
    _dom = None

    def __init__(self, text, lang='en'):
        nlp = spacy.load(lang)
        self._doc = nlp(text)

    @property
    def doc(self):
        return self._doc

    @property
    def dom(self):
        if not self._dom:
            self._dom = minidom.parseString(self._raw_processed)
        return self._dom

    def _process_semrep(self, resolved_text):
        cmd = '/opt/public_semrep/bin/'
        cmd += 'semrep.v1.8 -L 2018 -Z 2018AA -X {in_} {out}'

        with ntf(mode='w') as in_, ntf('r+') as out:
            in_.write(resolved_text)
            in_.seek(0)
            cmd = cmd.format(in_=in_.name, out=out.name)
            subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                           shell=True)
            foo = '</SemRepAnnotation>'
            self._raw_processed = out.read()
            self._raw_processed += (
                foo if foo not in self._raw_processed else ''
            )

        return

    def process(self):
        return self._process_semrep(self._doc.text)

    @classmethod
    def load(cls, path):
        """Intence an object from the a semrep output file."""
        with open(path, 'r') as f_in:
            obj = cls('')
            obj._raw_processed = f_in.read()
        return obj

    @property
    def _utterances(self):
        for u in self.dom.getElementsByTagName('Utterance'):
            yield u

    @property
    def utterances(self):
        for u in self._utterances:
            yield u

    @property
    def _entities(self):
        for u in self._utterances:
            yield u.getElementsByTagName('Entity')

    @property
    def entities(self):
        for u in self._entities:
            yield dict(ChainMap(*[entity2dict(e) for e in u]))

    @property
    def _predications(self):
        for u in self._utterances:
            yield u.getElementsByTagName('Predication')

    @property
    def predications(self):
        def _entities_map(u):
            return dict(ChainMap(*[entity2dict(e)
                                   for e in u.getElementsByTagName('Entity')]))

        for u in self._utterances:
            predications = list()
            ents = _entities_map(u)

            for pr in u.getElementsByTagName('Predication'):
                s = ents[pr.getElementsByTagName('Subject')[0].attributes.get(
                    'entityID').value]
                p = pr.getElementsByTagName('Predicate')[0].attributes.get(
                    'type').value
                o = ents[pr.getElementsByTagName('Object')[0].attributes.get(
                        'entityID').value]
                predications.append({
                    'subject': s,
                    'predicate': p,
                    'object': o
                })
            if not predications:
                continue
            yield predications

def ent2node(G, e):
    aux = dict(e)
    name = aux.pop('name')
    G.add_node(name, **aux)
    return name

def _set_graph_attributes(G, edge_attrs={}):
    for k, v in edge_attrs.items():
        nx.set_edge_attributes(G, v, k)

class SemRepGraph(SemRepWrapper):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._G_e = nx.MultiDiGraph()
        self._G_p = nx.MultiDiGraph()

    def process(self):
        super().process()
        self.process_graphs()

    def _process_entities_graph(self):
        ents = chain.from_iterable(u.values() for u in self.entities)
        for s, t in product(ents, repeat=2):
            if s == t:
                continue
            self._G_e.add_edge(
                ent2node(self._G_e, s),
                ent2node(self._G_e, t)
            )

    def _process_predications_graph(self):
        for p in chain.from_iterable(self.predications):
            self._G_p.add_edge(
                ent2node(self._G_p, p['subject']),
                ent2node(self._G_p, p['object']),
                type=p['predicate']
            )

    def process_graphs(self):
        self._process_entities_graph()
        self._process_predications_graph()

    def G_entities(self, edge_attrs={}):
        aux = deepcopy(self._G_e)
        _set_graph_attributes(aux, edge_attrs)
        return aux

    def G_predications(self, edge_attrs={}):
        aux = deepcopy(self._G_p)
        _set_graph_attributes(aux, edge_attrs)
        return aux


"""Provide functions for evaluation."""

from sparse import sparse, sunparse, sprint
from SEnvironment import SEnvironment
from SException import SException

def seval(string, env=None):
    """Evaluate a string and return the result."""
    if env is None:
        env = SEnvironment()
    return seval_tree(sparse(string), env)

def seval_strip(string, env=None):
    return _strip_annotations(seval(string, env))

def _strip_annotations(tree):
    if tree.type == 'list':
        return tuple(_strip_annotations(e) for e in tree.value)
    return tree.value

def seval_tree(tree, env):
    """Evaluate the given parse tree."""
    if tree.type == 'id':
        return env.lookup(tree.value)
    if tree.type == 'list':
        if not tree.value:
            raise SException("Attempting to evaluate empty list.")
        if tree.value[0].type == 'sf':
            if len(tree.value) == 1:
                raise SException("Trying to evaluate pure magic.")
            args = tree.value[2:]
            func = seval_tree(tree.value[1], env)
        else:
            args = [seval_tree(subtree, env) for subtree in tree.value[1:]]
            func = seval_tree(tree.value[0], env)
        if func.type != 'function':
            raise SException("Attempting to call non-function.")
        return func.value(env, *args)
    else:
        return tree


from unicodedata import name
import undetected_chromedriver.v2 as uc
import time

path = "./driver/chromedriver.exe"

options = uc.ChromeOptions()
options.binary_location = "C:/Program Files/Google/Chrome/Application/chrome.exe"
options.add_argument('--headless')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--ignore-ssl-errors')
options.add_argument('--disable-gpu')

def main():
    print("screen_shot program.")

    url = input("input url >>")
    picture = input("input screen_shot name >> ")

    #open_browser
    driver = uc.Chrome(path,options=options)
    driver.get(url)
    print("open broser")

    #set_screen
    set_screen(driver)

    #screen_shot
    #driver.save_screenshot(f"./screen_shot/{picture}.png")
    driver.save_screenshot("{picture}.png")
    print("screen shot")

    #close_broser
    driver.close()

def set_screen(driver):
    page_width = driver.execute_script('return document.body.scrollWidth')
    page_height = driver.execute_script('return document.body.scrollHeight')
    driver.set_window_size(page_width, page_height)
    print("set screen")
    time.sleep(5)

if __name__ == '__main__' :
    main()


import json
import discord
from discord.ext import tasks
from discord.ext import commands
import server
import asyncio
import os
import storage.s3 as storage

intents = discord.Intents.default()
intents.message_content = True
intents.members = True
intents.presences = True
bot = commands.Bot(command_prefix='.o ', activity=discord.Activity(name=".o help",type=discord.ActivityType.listening), intents=intents)

s3 = storage.S3(bucket='valheim-worlds-config')
worlds_json = json.loads(s3.get('worlds.json'))

# iterate over the worlds and instantiate Valheim for each
valheim_worlds = {}
for k, v in worlds_json.items():
    cluster_instance = server.Valheim(cluster=v["clustername"])
    valheim_worlds.update({k: cluster_instance})

@bot.event
async def on_ready():
    cleanup_loop.start()
    print('Bot is ready')

@bot.command(name="storage-status", brief="Return storage usage %", help="It runs a df -h inside the container, showing the usage % of every mount")
async def storage_status(ctx):
    v = valheim_worlds[ctx.guild.name].get_storage_details()
    print(v)
    await ctx.send(v)

@bot.command(name="server-status", brief="Return server status", help="Server status: if server is fully loaded, the status will be LOADED")
async def server_status(ctx):
    try:
        sv_status = valheim_worlds[ctx.guild.name].status
        await ctx.send(f"Server {worlds_json[ctx.guild.name]['worldname']} is {sv_status}")
    except Exception as e:
        print(e)
        await ctx.send(f"Error getting status")

@bot.command(name="server-start", brief="Start Valheim", help="It goes thru the following steps: 1- Scale up VM, 2- Scale up Valheim service, 3- Wait for the 'Game connected' on logs to report LOADED state")
async def server_start(ctx):
    valheim_worlds[ctx.guild.name].guild = ctx.guild
    server_loaded = False
    task = valheim_worlds[ctx.guild.name].task_status()
    cur_task_status = task[0]
    print(f"current task status is: {cur_task_status}")
    if valheim_worlds[ctx.guild.name].status == 'LOADED':
        await ctx.send("Server already up and running, enjoy!")
        server_loaded = True

    if not server_loaded:
        valheim_worlds[ctx.guild.name].status = 'STARTING'
        await ctx.send(f"Server {worlds_json[ctx.guild.name]['worldname']} is starting, I'll send updates on every status change")
        async with ctx.typing():
            await valheim_worlds[ctx.guild.name].start()

        last_task_status = None
        while cur_task_status != 'RUNNING':
            task = valheim_worlds[ctx.guild.name].task_status()
            cur_task_status = task[0]
            async with ctx.typing():
                while cur_task_status == last_task_status:
                    task = valheim_worlds[ctx.guild.name].task_status()
                    cur_task_status = task[0]
                    print(f"[{worlds_json[ctx.guild.name]['worldname']}] current task {task[1]} status is: {cur_task_status}")
                    await asyncio.sleep(2)
            await ctx.send(f"Server is starting, status: {cur_task_status}")
            last_task_status = cur_task_status

        await ctx.send(f"Server is running, now loading world, I'll let you know when it's loaded")
        valheim_worlds[ctx.guild.name].status = 'LOADING'
        cur_gameserver_status = valheim_worlds[ctx.guild.name].gameserver_status(task[1])
        print(f"[{worlds_json[ctx.guild.name]['worldname']}] current gameserver status is: {cur_gameserver_status}")
        async with ctx.typing():
            while cur_gameserver_status  != 'LOADED':
                cur_gameserver_status = valheim_worlds[ctx.guild.name].gameserver_status(task[1])
                print(f"current gameserver status is: {cur_gameserver_status}")
                await asyncio.sleep(2)

        await ctx.send("Server is up and running, enjoy!")
        valheim_worlds[ctx.guild.name].status = 'LOADED'
        server_loaded = True

@bot.command(name="server-stop", brief="Stop server", help="It goes thru the following steps: 1- stops Valheim process, 2- Make Valheim bkp, 3- Scale down VM")
async def server_stop(ctx):
    if valheim_worlds[ctx.guild.name].status != 'STOPPED':
        valheim_worlds[ctx.guild.name].stop_valheim_process()
        if valheim_worlds[ctx.guild.name].status == 'LOADED':
            await ctx.send("Making backup before shutdown...")
            async with ctx.typing():
                    await bot.loop.run_in_executor(None, valheim_worlds[ctx.guild.name].make_valheim_bkp)
            valheim_worlds[ctx.guild.name].remove_infra()
            valheim_worlds[ctx.guild.name].status = 'STOPPED'
            await ctx.channel.send(f"server {worlds_json[ctx.guild.name]['worldname']} down, hope you had a great time!")
        else:
            await ctx.channel.send(f"server {worlds_json[ctx.guild.name]['worldname']} is already stopped")

@tasks.loop(seconds=30*60)
async def cleanup_loop():
    for guild in worlds_json:
        if valheim_worlds[guild].status == 'LOADED':
            print(f'{worlds_json[guild]["worldname"]} is loaded, cleaning up old valheim days')
            valheim_worlds[guild].cleanup_old_days(4)
        else:
            print(f'cleanup skipped because {worlds_json[guild]["worldname"]} is {valheim_worlds[guild].status}')

bot.run(os.getenv('ODIN_BOT_TOKEN'))


"""

    preprocess_tweets.py

    Cleans and preprocesses hydrated tweets (from AWS S3 bucket), uploads back to AWS.

"""
import os
import ast
import pandas as pd
import emoji
import re
import nltk
import datetime
from nltk.corpus import stopwords
import aws_helpers
from aws_helpers import save_to_AWS, load_from_AWS

def parse_location(location_dict):
    """
    Looks at location information and determines (1) if the location is in the USA or not and (2) if it
    is in the USA, what state is it in?

    Arg:
        location_dict: value from "place" field from tweet data (dict)
    Returns:
        is_USA: is the location in the USA? (bool)
        country_location: country (str)
        state: if it is in the USA, which state is it in? (str)

    """

    if location_dict is None:
        return ("N/A", "N/A", "N/A")
    elif location_dict["country_code"] != "US":
        return (False, location_dict["country_code"], "N/A")
    else:
        try:
            # e.g., "Los Angeles, CA" --> "CA"
            state = location_dict["full_name"].split(",")[1].strip()
        except Exception as e:
            # sometimes it just denotes the full name as "United States"
            state = location_dict["full_name"].strip()
        finally:
            return (True, "US", state)

def parse_dates(timestamp):
    """
    Processes the date information and extracts date and time information
    Arg:
        timestamp: timestamp of date + time (str)
    Returns:
        date: human-readable date (str) (e.g., "2021-02-09")
        year: calendar year (int)
        month: calendar month (str, "00" to "12")
        day: calendar day (int)
        hour: hour of day (int)

    """
    hour = pd.to_datetime(timestamp).hour
    dt_obj = pd.to_datetime(timestamp).date()
    year = dt_obj.year
    month = dt_obj.month
    day = dt_obj.day

    if month < 10:
        month = f"0{month}"

    month = str(month)

    date = f"{year}-{month}-{day}"

    return (date, year, month, day, hour)

def remove_emoji(text):
    """
    Removes emojis from a text

    Args:
        text: tweet/text to clean (str)
    Returns:
        clean_text: tweet/text without emojis (str)
    """

    text = text.encode("utf-8")
    allchars = [str for str in text.decode('utf-8')]
    emoji_list = [c for c in allchars if c in emoji.UNICODE_EMOJI]
    clean_text = ' '.join([str for str in text.decode(
        'utf-8').split() if not any(i in str for i in emoji_list)])
    return clean_text

def clean_text(text):
    """
    Removes punctuation, does string split (tokenization) and removes links
    Args:
        text: tweet/text to clean (str)
    Returns:
        tokenized_text_arr: tokenized and cleaned text (list of strings)
    """

    PUNCTUATION = '''!()-[]{};:'"\,<>./?@$%^&*_~'''  # keep hashtags
    STOPWORDS = stopwords.words("english")

    # remove punctuation
    text_no_punctuation = ""

    for char in text:
        if char not in PUNCTUATION:
            text_no_punctuation = text_no_punctuation + char

    # remove emojis
    text_no_punctuation = remove_emoji(text_no_punctuation)

    # remove \n and \t
    text_no_punctuation = re.sub(r'\n', '', text_no_punctuation)
    text_no_punctuation = re.sub(r'\t', '', text_no_punctuation)

    # remove escape sequences
    text_no_escape = ""

    for char in text_no_punctuation:
        try:
            char.encode('ascii')
            # this'll catch chars that don't have an ascii equivalent (e.g., emojis)
            text_no_escape = text_no_escape + char
        except:
            pass

    # add space between # and another char before it (e.g., split yes#baseball into yes #baseball)
    text_no_escape = re.sub(r"([a-zA-Z0-9]){1}#", r"\1 #", text_no_escape)

    # string split
    text_arr = text_no_escape.split(' ')

    tokenized_text_arr = []

    for word in text_arr:

        word = word.lower()

        if "http" not in word and word.strip() != '' and word not in STOPWORDS:
            tokenized_text_arr.append(word)

    return tokenized_text_arr

def parse_hashtags(text):
    """
    Extracts both (1) the hashtags in a piece of text as well as
    (2) the parts of the text that don't have a hashtag

    Args:
        text: tweet/piece of text (str)
    Return:
        hashtag_arr: array of hashtags (list of strings)
        non_hashtag_arr: array of non-hashtag words (list of strings)
    """
    hashtag_arr = []
    non_hashtag_arr = []

    tokenized_text_arr = text.split(' ')

    for word in tokenized_text_arr:
        if '#' in word:
            # how many hashtags are in the word?
            num_hashtags = 0
            for char in word:
                if char == '#':
                    num_hashtags += 1

            # if multiple hashtag words are squished together, split them on '#' and return the resulting words
            if num_hashtags > 1:
                word_arr = word.split('#')
                for inner_word in word_arr:
                    if inner_word != '#':
                        hashtag_arr.append("#" + inner_word)
            # else, if only one hashtag, add as is
            else:
                hashtag_arr.append(word)

        else:
            non_hashtag_arr.append(word)

    return (hashtag_arr, non_hashtag_arr)

def count_hashtags(text):
    """
    Counts the number of hashtags in a piece of text
    Args:
        text: tweet/piece of text (str)
    Return:
        count: number of hashtags (#s) in text
    """
    text_arr = text.split(' ')
    count = 0

    for word in text_arr:
        if '#' in word:
            count += 1

    return count

def try_literal_eval(text):
    """
    Tries to use ast's literal_eval function to
    evaluate a string into its literal representation (e.g., 'dict')

    Arg:
        text: string representation
    Returns:
        eval: literal representation of parsed string
    """
    try:
        eval = ast.literal_eval(text)
    except Exception as e:
        eval = None

    return eval

if __name__ == "__main__":

    # change dir to this file's directory
    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    # get env vars
    AWS_BUCKET = os.environ["AWS_BUCKET"]
    AWS_ACCESS = os.environ["AWS_ACCESS"]
    AWS_SECRET = os.environ["AWS_SECRET"]

    # initialize paths
    AWS_TWEET_DIR = "tweet_scrapes/"
    HYDRATED_TWEETS_PATH = "./../../tweets/hydrated_tweets/"
    PREPROCESSED_TWEETS_PATH = "./../../tweets/preprocessed_tweets/"

    # full paths of files
    HYDRATED_TWEETS_FILENAME = "hydrated_tweets_2020-03-20_2021-02-09.csv"
    PREPROCESSED_TWEETS_FILENAME = "preprocessed_tweets_2020-03-20_2021-02-09.csv"
    LOCAL_HYDRATED_TWEETS_PATH = HYDRATED_TWEETS_PATH + HYDRATED_TWEETS_FILENAME
    LOCAL_PREPROCESSED_TWEETS_PATH = PREPROCESSED_TWEETS_PATH + \
        PREPROCESSED_TWEETS_FILENAME

    AWS_HYDRATED_TWEETS_PATH = AWS_TWEET_DIR + \
        "hydrated_tweets/" + HYDRATED_TWEETS_FILENAME
    AWS_PREPROCESSED_TWEETS_PATH = AWS_TWEET_DIR + \
        "preprocessed_tweets/" + PREPROCESSED_TWEETS_FILENAME

    # load tweets from AWS, use subset of cols
    # for now, can choose to use local version of file, rather than manually load it

    use_local = True
    if use_local:
        tweets_df = pd.read_csv(LOCAL_HYDRATED_TWEETS_PATH)
    else:
        tweets_df = pd.read_csv(AWS_HYDRATED_TWEETS_PATH)

    tweets_df = tweets_df[["user", "created_at", "id",
                           "full_text", "geo", "coordinates",
                           "place", "retweet_count", "favorite_count"]]

    print(f"Parsing {tweets_df.shape[0]} tweets today...")

    # change place col to be read as dict, not str
    tweets_df["place"] = tweets_df["place"].apply(
        lambda x: try_literal_eval(x))

    # initialize lists to hold information
    is_USA_list = []
    country_location_list = []
    state_location_list = []
    dates_list = []
    year_list = []
    month_list = []
    day_list = []
    hour_list = []
    tokenized_text_list = []
    hashtags_list = []
    non_hashtags_list = []
    hashtag_counts_list = []

    # loop through the location column, get location information
    for idx, location_dict in enumerate(tweets_df["place"]):

        try:
            is_USA, country, state = parse_location(location_dict)

            is_USA_list.append(is_USA)
            country_location_list.append(country)
            state_location_list.append(state)
        except Exception as e:
            print("Error in looping through locations column and getting locations")
            print(f"Error happened at index {idx}\n")
            print(f"The location was: {location_dict}\n")
            print(f"The error was: {e}\n")
            if type(location_dict) == int:
                is_USA_list.append("N/A")
                country_location_list.append("N/A")
                state_location_list.append("N/A")
                continue
            else:
                raise ValueError(
                    "Please address error in looping through locations column")

    # loop through the dates column, get date info
    for idx, timestamp in enumerate(tweets_df["created_at"]):

        try:
            date, year, month, day, hour = parse_dates(timestamp)

            dates_list.append(date)
            year_list.append(year)
            month_list.append(month)
            day_list.append(day)
            hour_list.append(hour)
        except Exception as e:
            print("Error in looping through dates column and getting dates")
            print(f"Error happened at index {idx}\n")
            print(f"The timestamp was: {timestamp}\n")
            print(f"The error was: {e}\n")

            dates_list.append("N/A")
            year_list.append("N/A")
            month_list.append("N/A")
            day_list.append("N/A")
            hour_list.append("N/A")

    # loop through the text column, get parsed text
    for idx, text in enumerate(tweets_df["full_text"]):

        try:
            # cleaned, tokenized text
            tokenized_text_arr = clean_text(text)
            tokenized_text_list.append(tokenized_text_arr)

            # get hashtags
            hashtag_arr, non_hashtag_arr = parse_hashtags(text)
            hashtags_list.append(hashtag_arr)
            non_hashtags_list.append(non_hashtag_arr)

            # get hashtag counts
            hashtag_count = count_hashtags(text)
            hashtag_counts_list.append(hashtag_count)

        except Exception as e:
            print("Error in looping through text column and getting parsed text")
            print(f"Error happened at index {idx}\n")
            print(f"The text was: {text}\n")
            print(f"The error was: {e}\n")
            raise ValueError(
                "Please address error in looping through text column")

    # add lists as columns to our df
    tweets_df["is_USA"] = is_USA_list
    tweets_df["country"] = country_location_list
    tweets_df["state"] = state_location_list
    tweets_df["date"] = dates_list
    tweets_df["year"] = year_list
    tweets_df["month"] = month_list
    tweets_df["day"] = day_list
    tweets_df["hour"] = hour_list
    tweets_df["tokenized_text"] = tokenized_text_list
    tweets_df["hashtags_list"] = hashtags_list
    tweets_df["non_hashtags_list"] = non_hashtags_list
    tweets_df["hashtag_count"] = hashtag_counts_list

    # export df as local .csv file
    try:
        tweets_df.to_csv(LOCAL_PREPROCESSED_TWEETS_PATH)
    except Exception as e:
        print("Problem with exporting tweets df as local file")
        print(e)
        raise ValueError(
            "Please address issue with exporting tweets df as local file")

    # export df to AWS bucket
    try:
        # save to AWS
        save_to_AWS(LOCAL_PREPROCESSED_TWEETS_PATH,
                    AWS_PREPROCESSED_TWEETS_PATH,
                    "s3",
                    AWS_BUCKET,
                    AWS_ACCESS,
                    AWS_SECRET)

        # remove local version
        os.remove(LOCAL_PREPROCESSED_TWEETS_PATH)
    except Exception as e:
        print("Error in exporting the local files to AWS")
        print(e)
        print("------------------")
        raise ValueError(
            "Please fix the error in exporting the local files to AWS")
    finally:
        print(
            f"Finished with the execution of 'preprocess_tweets.py' at (in UTC time): {datetime.datetime.utcnow()}")


import argparse
import sys
import time
from datetime import datetime

import cnn.model_single as model_single
from cnn.constants import *
from cnn.data import *
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

from cnn import model_branch
from utils import logger, angular_error_scalar

FLAGS = None

def training(images, labels):
    assert (len(images) == len(labels))
    run_name = datetime.now().strftime("%I:%M%p on %B %d, %Y")

    train_X, test_X, train_y, test_y = \
        train_test_split(images, labels, test_size=FLAGS.test_percent, random_state=0)
    train_X, train_y = split_to_patches(train_X, train_y)
    train_X, val_X, train_y, val_y = \
        train_test_split(train_X, train_y, test_size=FLAGS.valid_percent, random_state=0)

    learning_rate = FLAGS.learning_rate
    logger.info("learning rate: {0}".format(learning_rate))
    # Prepare for training
    num_batches_per_epoch = len(train_X) // FLAGS.batch_size

    with tf.Graph().as_default():

        # Define the input and label placeholders that will be used by
        # either single branch or multi branch model.
        image_placeholder = \
            tf.placeholder(tf.float32, (None, PATCH_SIZE[0], PATCH_SIZE[1],
                                        PATCH_SIZE[2]), name="Images")
        label_placeholder = tf.placeholder(tf.float32, (None, 2), name="Labels")

        # Single branch model
        if FLAGS.model == 'single':
            output = model_single.hyp_net_inference(image_placeholder)
            loss = model_single.hyp_net_loss(output, label_placeholder)
            train_op = model_single.hyp_net_training(loss, learning_rate)
            output_patch = model_single.hyp_net_inference(image_placeholder)
            init = tf.global_variables_initializer()
            saver = tf.train.Saver()

            # Summaries
            tf.summary.scalar("Loss", loss)

            with tf.Session() as sess:
                sess.run(init)
                summary_writer = tf.summary.FileWriter(os.path.join(log_dir, run_name), sess.graph)
                merged = tf.summary.merge_all()

                logger.info('Training...')
                non_improve_count = 0
                min_error = sys.maxsize
                for i in range(FLAGS.epochs):
                    train_X, train_y = shuffle(train_X, train_y)
                    for j in range(num_batches_per_epoch):
                        offset = j * FLAGS.batch_size
                        batch_X = train_X[offset:offset + FLAGS.batch_size]
                        batch_y = train_y[offset:offset + FLAGS.batch_size]
                        _, loss_value = sess.run([train_op, loss], feed_dict={image_placeholder: batch_X, label_placeholder: batch_y})
                        # loss_value = loss_value.item()
                        # logger.info("Training loss: %f" % loss_value)

                    loss_value, summary = sess.run([loss, merged], feed_dict={image_placeholder: val_X, label_placeholder: val_y})
                    loss_value = loss_value.item()  # convert to float
                    logger.info("Epoch: %d, Validation loss: %f"
                                % (i + 1, loss_value))
                    summary_writer.add_summary(summary, i)

                    # Early stop: If validation loss doesn't decrease for more
                    # than "--early_stop_threshold", then stop the training
                    if loss_value < min_error:
                        min_error = loss_value
                    else:
                        non_improve_count += 1
                    if non_improve_count > FLAGS.early_stop_threshold:
                        break

                saver.save(sess, check_ptr_dir)

                angular_errors = []
                for i in range(len(test_X)):
                    patches, labels = split_to_patches(np.asarray([test_X[i]]), np.asarray([test_y[i]]))
                    local_estimation = sess.run(output_patch, feed_dict={image_placeholder: patches, label_placeholder: labels})
                    angular_loss = angular_error_scalar(local_estimation, labels)
                    angular_errors.append(angular_loss)
                mean_error = np.mean(angular_errors)
                median_error = np.median(angular_errors)
                logger.info("########## Comparison by angular error: Mean is %s, Median is %s" % (mean_error, median_error))
                summary_writer.close()

        elif FLAGS.model == 'multiple':
            ground_truth_score = tf.placeholder(tf.float32, (None, 2), name="Ground_Truth_Score")

            outputA, outputB = model_branch.hyp_net_inference(image_placeholder)
            hyp_loss = model_branch.hyp_net_loss(outputA, outputB, label_placeholder)
            hyp_train_op = model_branch.hyp_net_training(hyp_loss, learning_rate)
            hyp_eval_correct = model_branch.hyp_net_evaluation(outputA, outputB, label_placeholder)
            calc_gt_score = model_branch.calc_ground_truth_score(outputA, outputB, label_placeholder)

            output_sel = model_branch.sel_net_inference(image_placeholder)
            sel_loss = model_branch.sel_net_loss(output_sel, ground_truth_score)
            sel_train_op = model_branch.sel_net_training(sel_loss, learning_rate)
            sel_eval_correct = model_branch.sel_net_loss(output_sel, ground_truth_score)
            output_patch = model_branch.inference(outputA, outputB, output_sel)
            init = tf.global_variables_initializer()
            saver = tf.train.Saver()

            # Summaries
            tf.summary.scalar("Hyp_Training_Loss", hyp_loss)
            tf.summary.scalar("Sel_Training_Loss", sel_loss)

            with tf.Session() as sess:
                sess.run(init)
                summary_writer = tf.summary.FileWriter(os.path.join(log_dir, run_name), sess.graph)
                merged = tf.summary.merge_all()

                logger.info('Training HypNet...')
                non_improve_count = 0
                min_error = sys.maxsize
                hyp_train_epoch = 0
                for i in range(FLAGS.epochs):
                    train_X, train_y = shuffle(train_X, train_y)
                    for j in range(num_batches_per_epoch):
                        offset = j * FLAGS.batch_size
                        batch_X = train_X[offset:offset + FLAGS.batch_size]
                        batch_y = train_y[offset:offset + FLAGS.batch_size]
                        _, summary = sess.run([hyp_train_op, merged],
                                              feed_dict={image_placeholder: batch_X, label_placeholder: batch_y, ground_truth_score: batch_y})
                    loss_value = sess.run(hyp_eval_correct, feed_dict={image_placeholder: val_X, label_placeholder: val_y})
                    loss_value = loss_value.item()  # convert to float
                    logger.info("Epoch: %d, Hyp Validation loss: %f" % (i + 1, loss_value))

                    if loss_value < min_error:
                        min_error = loss_value
                    else:
                        non_improve_count += 1
                    if non_improve_count > FLAGS.early_stop_threshold:
                        break

                    hyp_train_epoch += 1
                    summary_writer.add_summary(summary, hyp_train_epoch)
                a, b = sess.run([outputA, outputB], feed_dict={image_placeholder: val_X, label_placeholder: val_y})
                logger.info("%s \n%s" % (a, b))

                logger.info("Finish training HypNet. Test set loss: %s" % loss_value)

                logger.info('Training SelNet...')
                non_improve_count = 0
                min_error = sys.maxsize
                gt_score_train = sess.run(calc_gt_score, feed_dict={image_placeholder: train_X, label_placeholder: train_y})
                gt_score_val = sess.run(calc_gt_score, feed_dict={image_placeholder: val_X, label_placeholder: val_y})
                for i in range(FLAGS.epochs):
                    train_X, gt_score_train = shuffle(train_X, gt_score_train)
                    for j in range(num_batches_per_epoch):
                        offset = j * FLAGS.batch_size
                        start_time = time.time()
                        batch_X, batch_gt_score = train_X[offset:offset + FLAGS.batch_size], \
                                                  gt_score_train[offset:offset + FLAGS.batch_size]
                        _, summary = sess.run([sel_train_op, merged],
                                              feed_dict={image_placeholder: batch_X, label_placeholder: batch_gt_score, ground_truth_score: batch_gt_score})
                        duration = time.time() - start_time
                    loss_value = sess.run(sel_eval_correct,
                                          feed_dict={image_placeholder: val_X, label_placeholder: gt_score_val, ground_truth_score: gt_score_val})
                    loss_value = loss_value.item()  # convert to float
                    logger.info("Epoch: %d, Sel Validation loss: %f" % (i + 1, loss_value))

                    if loss_value < min_error:
                        min_error = loss_value
                    else:
                        non_improve_count += 1
                    if non_improve_count > FLAGS.early_stop_threshold:
                        break

                    summary_writer.add_summary(summary, i + hyp_train_epoch)
                saver.save(sess, check_ptr_dir)

                angular_errors = []
                for i in range(len(test_X)):
                    patches, labels = split_to_patches(np.asarray([test_X[i]]), np.asarray([test_y[i]]))
                    local_estimation = sess.run(output_patch, feed_dict={image_placeholder: patches, label_placeholder: labels})
                    angular_loss = angular_error_scalar(local_estimation, labels)
                    angular_errors.append(angular_loss)
                mean_error = np.mean(angular_errors)
                median_error = np.median(angular_errors)
                logger.info("########## Comparison by angular error: Mean is %s, Median is %s" % (mean_error, median_error))
                summary_writer.close()

def main(_):
    data_set_name = 'Canon5D'
    images, labels = load_data(data_set_name, debug=FLAGS.debug)
    training(images, labels)

if __name__ == "__main__":
    # Sequence of training between the two networks?
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--debug',
        action='store_true',
        default=False,
        help='Debug mode.')

    parser.add_argument(
        '--learning_rate',
        type=float,
        default=0.01,
        help='Initial learning rate.'
    )

    parser.add_argument(
        '--epochs',
        type=int,
        default=10,
        help='Number of steps to run trainer.'
    )

    parser.add_argument(
        '--batch_size',
        type=int,
        default=64,
        help='Number of data item in a batch.'
    )

    parser.add_argument(
        '--early_stop_threshold',
        type=int,
        default=5,
        help='Number of epoch without improvement to early stop'
    )

    parser.add_argument(
        '--test_percent',
        type=int,
        default=0.2,
        help='Number of steps to run trainer.'
    )

    parser.add_argument(
        '--valid_percent',
        type=int,
        default=0.2,
        help='Number of steps to run trainer.'
    )

    parser.add_argument(
        '--model',
        choices=['single', 'multiple'],
        default='multiple',
        help='Model of the network')

    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


import time
import cv2
import numpy as np
import onnxruntime
from .blazeFaceUtils import gen_anchors, AnchorsOptions
import json
import time
import os

class BlazeFaceDetector:

    def __init__(self, typ = "front", scoreThreshold = 0.7, iouThreshold = 0.3):
        BASE_DIR = os.path.dirname(__file__)
        self.type = typ
        self.scoreThreshold = scoreThreshold
        self.iouThreshold = iouThreshold
        self.sigmoidScoreThreshold = np.log(self.scoreThreshold/(1-self.scoreThreshold))
        self.initModel(typ,BASE_DIR)

        self.generateAnchorBoxes(typ)

    def initModel(self, typ, BASE_DIR):
        """
        Read and initialize the model
        """
        front_model_path = os.path.join(BASE_DIR , "face_detection_front_128x128.onnx")
        back_model_path = os.path.join(BASE_DIR , "face_detection_back_256x256.onnx")
        if typ == "front":
            self.session = onnxruntime.InferenceSession(front_model_path, None)
        elif typ =="back":
            self.session = onnxruntime.InferenceSession(back_model_path, None)

        self.getModelInputDetails()
        self.getModelOutputDetails()

    def detectFaces(self, image, maxFaces = 2):
        """
        This function aims to detect faces in image
        input: [image] a numpy cv2 array (i.e. BGR image)
        returns:
        """
        preprocessed_image = self.detectionPreprocessing(image)
        result = self.inference(preprocessed_image)

        scores, goodDetections = self.filterDetections(result[1])

        faces_locations = self.extractDetections(result[0], goodDetections)
        faces_locations = self.nms(faces_locations, scores, maxFaces)
        return faces_locations

    def __call__(self, image, maxFaces=2):
        return self.detectFaces(image, maxFaces)

    def getModelInputDetails(self):
        """
        This function aims to get inlut node details for onnx inference
        """
        self.input_details = self.session.get_inputs()[0]
        self.input_name = self.input_details.name
        input_shape = self.input_details.shape
        self.inputHeight = input_shape[1]
        self.inputWidth = input_shape[2]
        self.channels = input_shape[3]

    def getModelOutputDetails(self):
        """
        this funtion aims to get the output nodes details (mainly we need the names only) for onnx inference
        """
        self.output_details = self.session.get_outputs()

    def generateAnchorBoxes(self, typ):
        """
        This function aims to generate anchor boxes depends on the type of the model (front model or back model)

        """
        if typ == "front":
		# Options to generate anchors for SSD object detection models.
            anchors_options = AnchorsOptions(input_size_width=128, input_size_height=128, min_scale=0.1484375, max_scale=0.75
                , anchor_offset_x=0.5, anchor_offset_y=0.5, num_layers=4
                , feature_map_width=[], feature_map_height=[]
                , strides=[8, 16, 16, 16], aspect_ratios=[1.0]
                , reduce_boxes_in_lowest_layer=False, interpolated_scale_aspect_ratio=1.0
                , fixed_anchor_size=True)

        elif typ == "back":
            # Options to generate anchors for SSD object detection models.
            anchors_options = AnchorsOptions(input_size_width=256, input_size_height=256, min_scale=0.15625, max_scale=0.75
                    , anchor_offset_x=0.5, anchor_offset_y=0.5, num_layers=4
                    , feature_map_width=[], feature_map_height=[]
                    , strides=[16, 32, 32, 32], aspect_ratios=[1.0]
                    , reduce_boxes_in_lowest_layer=False, interpolated_scale_aspect_ratio=1.0
                    , fixed_anchor_size=True)
        self.anchors = gen_anchors(anchors_options)

    def detectionPreprocessing(self, image):
        """
        This function aims to preproccess the image before feeding it to the onnx model
        input: [image] image loaded with cv2.imread (i.e. BGR image)
        returns: [data] processed image for onnx model feeding
        """
        img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        self.img_height, self.img_width, self.img_channels = img.shape
        # Input values should be from -1 to 1 with a size of 128 x 128 pixels for the fornt model
        # and 256 x 256 pixels for the back model
        img = img / 255.0
        """ img_resized = tf.image.resize(img, [self.inputHeight,self.inputWidth],
                                    method='bicubic', preserve_aspect_ratio=False)
         """
        img_resized = cv2.resize(img, (self.inputHeight,self.inputWidth), interpolation=cv2.INTER_CUBIC)
        img_input = (img_resized - 0.5) / 0.5

        # Adjust matrix dimenstions
        reshape_img = img_input.reshape(1,self.inputHeight,self.inputWidth,self.channels)

        data = reshape_img.astype('float32')

        return data

    def inference(self, data):
        """
        This function takes the data from detectionPreprocessing function and passes it to BlazeFace model using onnx
        input: [data] data we got from detectionPreprocessing which is image as a numpy array
        return: [boxes] bounding boxes (including face landmarks)
                [scores] confidence of each output [boxes]
        """

        result = self.session.run([self.output_details[0].name,self.output_details[1].name,self.output_details[2].name,self.output_details[3].name], {self.input_name: data})

        scores = np.squeeze(np.concatenate((result[0], result[1]), axis = 1), axis =0)
        boxes = np.squeeze(np.concatenate((result[2], result[3]), axis = 1), axis = 0)

        return boxes, scores

    def filterDetections(self, scores):
        """
        This function aims to filter the bounding boxes by there confidence level
        input: [scores] numpy array with shape (num_boxes, 1) holds the confidence level of each box
        returns: numpy array scores after being filtered, and indices of those scores
        """
		# Filter based on the score threshold before applying sigmoid function
        goodDetections = np.where(scores > self.sigmoidScoreThreshold)[0]

		# Convert scores back from sigmoid values
        scores = 1.0 /(1.0 + np.exp(-scores[goodDetections]))

        return scores, goodDetections

    def extractDetections(self, predections, selectedIndices):
        """
        This function aims to extract the outputs from the anchor boxes
        inputs: [predections] numpy array holds the predicted outputs (boxes and keypoints), but we will use here the boxes only shape (num_of_boxes, 16)
                [selectedIndices] numpy array holds the indices of selected boxes
        returns: numpy array of boxes extracted from generated anchors shape (num_of_boxes, 4)
        """

        numGoodDetections = selectedIndices.shape[0]

        boxes = np.zeros((numGoodDetections, 4))
        for idx, detectionIdx in enumerate(selectedIndices):
            anchor = self.anchors[detectionIdx]

            sx = predections[detectionIdx, 0]
            sy = predections[detectionIdx, 1]
            w = predections[detectionIdx, 2]
            h = predections[detectionIdx, 3]

            cx = sx + anchor.x_center * self.inputWidth
            cy = sy + anchor.y_center * self.inputHeight

            cx /= self.inputWidth
            cy /= self.inputHeight
            w /= self.inputWidth
            h /= self.inputHeight

            # The next section is for facial keypoints detected from BlazeFace model
            # commented it for speed concerns

            """ for j in range(KEY_POINT_SIZE):
                lx = output0[detectionIdx, 4 + (2 * j) + 0]
                ly = output0[detectionIdx, 4 + (2 * j) + 1]
                lx += anchor.x_center * self.inputWidth
                ly += anchor.y_center * self.inputHeight
                lx /= self.inputWidth
                ly /= self.inputHeight
                keypoints[idx,j,:] = np.array([lx, ly])
 """
            boxes[idx,:] = np.array([cx - w * 0.5, cy - h * 0.5, cx + w * 0.5, cy + h * 0.5])

        return boxes

    def nms(self, boxes, probs=None, max_num_outputs = 1, iouThreshold=0.3):
        """
        This a fast non max suppression (NMS) implementation
        inputs: [boxes] numpy array conatins bounding boxes in format of [x1, y1, x2, y2] with shape (num_boxes, 4)
                , [probs] numpy array holds the confidence level of each box from 0-1 with shape (num_boxes, 1)
                , [max_num_outputs] an intger holds the maximum number of ouputs needed
                , [iouThrshold] an integer from 0-1 represents the intersection over union threshold
        returns: numpy array of boxes after applying NMS
        """
	# if there are no boxes, return an empty list
        if len(boxes) == 0:
            return []

        # if the bounding boxes are integers, convert them to floats -- this
        # is important since we'll be doing a bunch of divisions
        if boxes.dtype.kind == "i":
            boxes = boxes.astype("float")

        # initialize the list of picked indexes
        pick = []

        # grab the coordinates of the bounding boxes
        x1 = boxes[:, 0]
        y1 = boxes[:, 1]
        x2 = boxes[:, 2]
        y2 = boxes[:, 3]

        # compute the area of the bounding boxes and grab the indexes to sort
        # (in the case that no probabilities are provided, simply sort on the
        # bottom-left y-coordinate)
        area = (x2 - x1 + 1) * (y2 - y1 + 1)
        idxs = y2

        # if probabilities are provided, sort on them instead
        if probs is not None:
            idxs = probs.squeeze()

        # sort the indexes
        idxs = np.argsort(idxs)
        face_counter = 0
        # keep looping while some indexes still remain in the indexes list
        while len(idxs) > 0:
            # grab the last index in the indexes list and add the index value
            # to the list of picked indexes
            if face_counter == max_num_outputs:
                break
            face_counter +=1
            last = len(idxs) - 1
            i = idxs[last]

            pick.append(i)

            # find the largest (x, y) coordinates for the start of the bounding
            # box and the smallest (x, y) coordinates for the end of the bounding
            # box
            xx1 = np.maximum(x1[i], x1[idxs[:last]])
            yy1 = np.maximum(y1[i], y1[idxs[:last]])
            xx2 = np.minimum(x2[i], x2[idxs[:last]])
            yy2 = np.minimum(y2[i], y2[idxs[:last]])

            # compute the width and height of the bounding box
            w = np.maximum(0, xx2 - xx1 + 1)
            h = np.maximum(0, yy2 - yy1 + 1)

            # compute the ratio of overlap
            overlap = (w * h) / area[idxs[:last]]

            # delete all indexes from the index list that have overlap greater
            # than the provided overlap threshold
            idxs = np.delete(idxs, np.concatenate(([last],
                np.where(overlap > iouThreshold)[0])))
        # return only the bounding boxes that were picked

        boxes = boxes[pick]
        boxes[:,0] = self.img_width * boxes[:,0]
        boxes[:,2] = self.img_width * boxes[:,2]
        boxes[:,1] = self.img_height * boxes[:,1]
        boxes[:,3] = self.img_height * boxes[:,3]

        # convert from X1y1x2y2 to xywh
        boxes[:,2] = boxes[:,2] - boxes[:,0]
        boxes[:,3] = boxes[:,3] - boxes[:,1]

        return boxes.astype("int")


import pkg_resources
import six

# Use collections on py2, collections.abc on py3
if six.PY2:
    import collections as abc
else:
    from collections import abc

__all__ = ['EntrypointList', 'EntrypointDict', 'eps']

class AttrGroup(object):
    """
    Allows attribute-type access to sub-groups.
    """

    def __init__(self, prefix):
        """
        Initialize an ``AttrGroup`` instance.

        :param str prefix: The group name prefix.  This will be
                           prepended to the group name provided to
                           subordinate ``EntrypointDict`` objects.
        """

        # Save the prefix
        self._prefix = prefix

        # Initialize the attribute cache
        self._attr_cache = {}

    def __getattr__(self, name):
        """
        Get a subordinate ``EntrypointDict`` object.

        :param str name: The name of the subordinate
                         ``EntrypointDict`` object.  This name may not
                         begin with '_', but any other characters,
                         including '.', are permitted.

        :returns: A subordinate ``EntrypointDict`` object.
        :rtype: ``EntrypointDict``

        :raises AttributeError:
            An attempt was made to access an internal attribute.
        """

        # Prohibit internal names
        if name.startswith('_'):
            raise AttributeError(
                "'%s' object has no attribute '%s'" %
                (self.__class__.__name__, name)
            )

        # If '.' is in the name, treat it specially
        if '.' in name:
            parts = name.split('.')
            obj = self
            for part in parts:
                obj = getattr(obj, part)
            return obj

        # Dynamically create an EntrypointDict as needed
        if name not in self._attr_cache:
            self._attr_cache[name] = EntrypointDict(self._prefix + name)

        return self._attr_cache[name]

@six.python_2_unicode_compatible
class EntrypointList(abc.Sequence):
    """
    A lazy list of entrypoints.  This ``list`` class is designed to
    only query as many entrypoints as are necessary to accomplish the
    required task.  For instance, asking for ``obj[0]`` will find and
    return only the first loadable entrypoint, without exploring any
    others, yet a subsequent ``obj[1]`` will load the rest of the
    entrypoints.
    """

    def __init__(self, group, name):
        """
        Initialize an ``EntrypointList`` instance.

        :param str group: The name of the entrypoint group.
        :param str name: The name of the entrypoint itself.
        """

        # Save the entrypoint group and name
        self._group = group
        self._name = name

        # List of all found entrypoints
        self._eps = []

        # Have we found 1?
        self._found = False

        # Have we found all of them?
        self._explored = False

        # Set of EntryPoint object IDs that we can skip
        self._entries = set()

    def __str__(self):
        """
        Return a string representation of this object.

        :returns: A string containing a representation of the object.
        :rtype: ``str``
        """

        return '[%s]' % ', '.join(str(item) for item in self)

    def __repr__(self):
        """
        Return a representation of this object.  This variant ensures that
        no entrypoints are explored.

        :returns: A string containing a representation of the object.
        :rtype: ``str``
        """

        return '<%s.%s object at 0x%x>' % (
            self.__class__.__module__, self.__class__.__name__, id(self)
        )

    def __len__(self):
        """
        Determine the number of entrypoints available.

        :returns: The number of loadable entrypoints.
        :rtype: ``int``
        """

        # Make sure we've fully explored the space
        self._find()

        return len(self._eps)

    def __getitem__(self, idx):
        """
        Obtain the specified entrypoint.

        :param idx: An integer or ``slice`` that specifies the
                    entrypoint or entrypoints to return.

        :returns: The designated entrypoint.

        :raises TypeError:
            The ``idx`` is not an integer or slice.

        :raises IndexError:
            The ``idx`` does not exist.
        """

        # Sanity-check the index value
        if not isinstance(idx, (six.integer_types, slice)):
            raise TypeError('list indices must be integers, not str')

        # If it's identically 0, only look for one entry
        if idx == 0:
            self._find(True)
            return self._eps[idx]  # will raise IndexError if empty

        # OK, have to make sure we have the full space
        self._find()
        return self._eps[idx]

    def __bool__(self):
        """
        Determine if the entrypoint list is empty.

        :returns: A ``False`` value if the entrypoint list is empty,
                  ``True`` otherwise.
        :rtype: ``bool``
        """

        # Only look for 1
        self._find(True)

        return bool(self._eps)

    # Alias for Python 2 functionality
    __nonzero__ = __bool__

    def _find(self, one=False):
        """
        Explore the entrypoint space.  This fills ``self._eps`` with the
        entrypoints that can be loaded.

        :param bool one: If ``True``, exploration stops after the
                         first loadable entrypoint is found.  Defaults
                         to ``False``.
        """

        # If we've found what's required, we're done
        if self._explored or (one and self._found):
            return

        # Look up entrypoints
        for ep in pkg_resources.iter_entry_points(self._group, self._name):
            # Skip entrypoints we know about
            if id(ep) in self._entries:
                continue

            # OK, we don't know about it; mark it so we don't try again
            self._entries.add(id(ep))

            # See if we can load it
            try:
                obj = ep.load()
            except (ImportError, AttributeError, pkg_resources.UnknownExtra):
                continue

            # OK, we can load it
            self._eps.append(obj)
            self._found = True

            # Was that all?
            if one:
                return

        # We've fully explored the entrypoint space
        self._found = True
        self._explored = True

    @property
    def group(self):
        """
        Retrieve the entrypoint group.
        """

        return self._group

    @property
    def name(self):
        """
        Retrieve the name of the entrypoint.
        """

        return self._name

@six.python_2_unicode_compatible
class EntrypointDict(AttrGroup, abc.Mapping):
    """
    A lazy dictionary of entrypoints.  This ``dict`` class is designed
    to only query as many entrypoints as are necessary to accomplish
    the required task.  For instance, asking for ``obj['name']`` will
    find and return only the first loadable entrypoint having the name
    ``'name'``, without exploring any others, yet a ``len(obj)`` will
    explore the entire entrypoint space and return the total number of
    unique entrypoint names within the entrypoint group.
    """

    def __init__(self, group):
        """
        Initialize an ``EntrypointDict`` instance.

        :param str group: The name of the entrypoint group.
        """

        # Save the group name
        self._group = group

        # Cache of the entrypoints we've found
        self._entries = {}

        # Have we explored the whole space?
        self._explored = False

        # Initialize the attribute group
        super(EntrypointDict, self).__init__(group + '.')

    def __str__(self):
        """
        Return a string representation of this object.

        :returns: A string containing a representation of the object.
        :rtype: ``str``
        """

        return '{%s}' % ', '.join('%r: %s' % (k, v)
                                  for k, v in self._entries.items())

    def __repr__(self):
        """
        Return a representation of this object.  This variant ensures that
        no entrypoints are explored.

        :returns: A string containing a representation of the object.
        :rtype: ``str``
        """

        return '<%s.%s object at 0x%x>' % (
            self.__class__.__module__, self.__class__.__name__, id(self)
        )

    def __getitem__(self, key):
        """
        Retrieve an entrypoint.

        :param str key: The name of the entrypoint.

        :returns: The requested entrypoint.

        :raises KeyError:
            The designated entrypoint does not exist.
        """

        # Skip if it's not a string
        if not isinstance(key, six.string_types):
            raise KeyError(key)

        # Look up the entrypoint if we don't have it in the cache
        if key not in self._entries:
            self._entries[key] = EntrypointList(self._group, key)

        # We've looked it up but couldn't find one
        if not self._entries[key]:
            raise KeyError(key)

        # Return the first entrypoint
        return self._entries[key][0]

    def __len__(self):
        """
        Determine the number of entrypoints in this entrypoint group.

        :returns: The number of unique entrypoint names in this
                  entrypoint group.
        :rtype: ``int``
        """

        # Make sure we've explored the entrypoint space
        self._find()

        # Subtract out the empty entries
        return (
            len(self._entries) -
            sum(1 for v in self._entries.values() if not v)
        )

    def __iter__(self):
        """
        Iterate through all the entrypoints in this entrypoint group.

        :returns: An iterator that yields entrypoint names.
        """

        # Make sure we've explored the entrypoint space
        self._find()

        for key, value in self._entries.items():
            # Skip entries that weren't found
            if not value:
                continue

            yield key

    def _find(self):
        """
        Explore the entrypoint space.  This fills ``self._entries`` with
        the entrypoints that can be loaded.  Note that this will
        explore the whole space, and update the entrypoint lists to
        reflect this.
        """

        # If we've already explored the space, we're done
        if self._explored:
            return

        # Iterate through the entrypoints
        encountered = set()
        for ep in pkg_resources.iter_entry_points(self._group):
            # We'll want to set the caching attributes later
            encountered.add(ep.name)

            # Is it an entrypoint we've seen already?
            if (ep.name in self._entries and
                    id(ep) in self._entries[ep.name]._entries):
                continue

            # Mark it so we don't try again
            if ep.name not in self._entries:
                self._entries[ep.name] = EntrypointList(self._group, ep.name)
            self._entries[ep.name]._entries.add(id(ep))

            # See if we can load it
            try:
                obj = ep.load()
            except (ImportError, AttributeError, pkg_resources.UnknownExtra):
                continue

            # OK, we can load it
            self._entries[ep.name]._eps.append(obj)

        # Mark that we've fully explored the entrypoint space
        self._explored = True
        for epname in encountered:
            self._entries[epname]._found = True
            self._entries[epname]._explored = True

    # Alias keys_all() so it exists
    keys_all = __iter__

    def items_all(self):
        """
        Like ``items()``, but returns the full list of entrypoints instead
        of just the first entrypoint.

        :returns: An iterator yielding tuples of the entrypoint name
                  and the list of entrypoints with that name.
        """

        # Make sure we've explored the entrypoint space
        self._find()

        for key, value in self._entries.items():
            # Skip entries that weren't found
            if not value:
                continue

            yield (key, value)

    def values_all(self):
        """
        Like ``values()``, but returns the full list of entrypoints instead
        of just the first entrypoint.

        :returns: An iterator yielding the list of entrypoints with a
                  given name.
        """

        # Make sure we've explored the entrypoint space
        self._find()

        for key, value in self._entries.items():
            # Skip entries that weren't found
            if not value:
                continue

            yield value

    def get_all(self, key, default=None):
        """
        Like ``get()``, but returns the full list of entrypoints instead
        of just the first entrypoint.

        :param str key: The name of the entrypoint.
        :param default: A value to return if no entrypoints with the
                        specified name exist.  Defaults to ``None``.

        :returns: A list of entrypoints, or the specified ``default``.
        :rtype: ``EntrypointList``
        """

        # Skip if it's not a string
        if not isinstance(key, six.string_types):
            return default

        # Look up the entrypoint if we don't have it in the cache
        if key not in self._entries:
            self._entries[key] = EntrypointList(self._group, key)

        # We've looked it up but couldn't find one
        if not self._entries[key]:
            return default

        # Return all the entrypoints
        return self._entries[key]

    @property
    def group(self):
        """
        Retrieve the entrypoint group.
        """

        return self._group

# A place to get all the endpoints
eps = AttrGroup('')