from __future__ import division
import os
from scipy import io as scio
import copy
from PIL.Image import Resampling
import sys
import random
import cv2
import torch
import numpy as np
from PIL import Image
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from torchvision.transforms import Compose, ColorJitter, ToTensor, Normalize

def get_citypersons(root_dir='data/cityperson', mode='train'):
    all_img_path = os.path.join(root_dir, 'images')
    all_anno_path = os.path.join(root_dir, 'annotations')
    rows, cols = 1024, 2048

    anno_path = os.path.join(all_anno_path, 'anno_' + mode + '.mat')
    image_data = []
    annos = scio.loadmat(anno_path)
    index = 'anno_' + mode + '_aligned'
    valid_count = 0
    iggt_count = 0
    box_count = 0

    for l in range(len(annos[index][0])):
        anno = annos[index][0][l]
        cityname = anno[0][0][0][0]
        imgname = anno[0][0][1][0]
        gts = anno[0][0][2]
        img_path = os.path.join(all_img_path, mode + '/' + cityname + '/' + imgname)
        boxes = []
        ig_boxes = []
        vis_boxes = []
        for i in range(len(gts)):
            label, x1, y1, w, h = gts[i, :5]
            x1, y1 = max(int(x1), 0), max(int(y1), 0)
            w, h = min(int(w), cols - x1 - 1), min(int(h), rows - y1 - 1)
            xv1, yv1, wv, hv = gts[i, 6:]
            xv1, yv1 = max(int(xv1), 0), max(int(yv1), 0)
            wv, hv = min(int(wv), cols - xv1 - 1), min(int(hv), rows - yv1 - 1)

            if label == 1 and h >= 50:
                box = np.array([int(x1), int(y1), int(x1) + int(w), int(y1) + int(h)])
                boxes.append(box)
                vis_box = np.array([int(xv1), int(yv1), int(xv1) + int(wv), int(yv1) + int(hv)])
                vis_boxes.append(vis_box)
            else:
                ig_box = np.array([int(x1), int(y1), int(x1) + int(w), int(y1) + int(h)])
                ig_boxes.append(ig_box)
        boxes = np.array(boxes)
        vis_boxes = np.array(vis_boxes)
        ig_boxes = np.array(ig_boxes)

        if len(boxes) > 0:
            valid_count += 1
        annotation = {}
        annotation['filepath'] = img_path
        box_count += len(boxes)
        iggt_count += len(ig_boxes)
        annotation['bboxes'] = boxes
        annotation['vis_bboxes'] = vis_boxes
        annotation['ignoreareas'] = ig_boxes
        image_data.append(annotation)

    return image_data

def _brightness(image, min=0.5, max=2.0):
    '''
    Randomly change the brightness of the input image.
    Protected against overflow.
    '''
    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)

    random_br = np.random.uniform(min, max)

    # To protect against overflow: Calculate a mask for all pixels
    # where adjustment of the brightness would exceed the maximum
    # brightness value and set the value to the maximum at those pixels.
    mask = hsv[:, :, 2] * random_br > 255
    v_channel = np.where(mask, 255, hsv[:, :, 2] * random_br)
    hsv[:, :, 2] = v_channel

    return cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)

def resize_image(image, gts, igs, scale=(0.4, 1.5)):
    height, width = image.shape[0:2]
    ratio = np.random.uniform(scale[0], scale[1])
    # if len(gts)>0 and np.max(gts[:,3]-gts[:,1])>300:
    #     ratio = np.random.uniform(scale[0], 1.0)
    new_height, new_width = int(ratio * height), int(ratio * width)
    image = cv2.resize(image, (new_width, new_height))
    if len(gts) > 0:
        gts = np.asarray(gts, dtype=float)
        gts[:, 0:4:2] *= ratio
        gts[:, 1:4:2] *= ratio

    if len(igs) > 0:
        igs = np.asarray(igs, dtype=float)
        igs[:, 0:4:2] *= ratio
        igs[:, 1:4:2] *= ratio

    return image, gts, igs

def random_crop(image, gts, igs, crop_size, limit=8):
    img_height, img_width = image.shape[0:2]
    crop_h, crop_w = crop_size

    if len(gts) > 0:
        sel_id = np.random.randint(0, len(gts))
        sel_center_x = int((gts[sel_id, 0] + gts[sel_id, 2]) / 2.0)
        sel_center_y = int((gts[sel_id, 1] + gts[sel_id, 3]) / 2.0)
    else:
        sel_center_x = int(np.random.randint(0, img_width - crop_w + 1) + crop_w * 0.5)
        sel_center_y = int(np.random.randint(0, img_height - crop_h + 1) + crop_h * 0.5)

    crop_x1 = max(sel_center_x - int(crop_w * 0.5), int(0))
    crop_y1 = max(sel_center_y - int(crop_h * 0.5), int(0))
    diff_x = max(crop_x1 + crop_w - img_width, int(0))
    crop_x1 -= diff_x
    diff_y = max(crop_y1 + crop_h - img_height, int(0))
    crop_y1 -= diff_y
    cropped_image = np.copy(image[crop_y1:crop_y1 + crop_h, crop_x1:crop_x1 + crop_w])
    # crop detections
    if len(igs) > 0:
        igs[:, 0:4:2] -= crop_x1
        igs[:, 1:4:2] -= crop_y1
        igs[:, 0:4:2] = np.clip(igs[:, 0:4:2], 0, crop_w)
        igs[:, 1:4:2] = np.clip(igs[:, 1:4:2], 0, crop_h)
        keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & \
                    ((igs[:, 3] - igs[:, 1]) >= 8)
        igs = igs[keep_inds]
    if len(gts) > 0:
        ori_gts = np.copy(gts)
        gts[:, 0:4:2] -= crop_x1
        gts[:, 1:4:2] -= crop_y1
        gts[:, 0:4:2] = np.clip(gts[:, 0:4:2], 0, crop_w)
        gts[:, 1:4:2] = np.clip(gts[:, 1:4:2], 0, crop_h)

        before_area = (ori_gts[:, 2] - ori_gts[:, 0]) * (ori_gts[:, 3] - ori_gts[:, 1])
        after_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])

        keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit) & \
                    (after_area >= 0.5 * before_area)
        gts = gts[keep_inds]

    return cropped_image, gts, igs

def random_pave(image, gts, igs, pave_size, limit=8):
    img_height, img_width = image.shape[0:2]
    pave_h, pave_w = pave_size
    # paved_image = np.zeros((pave_h, pave_w, 3), dtype=image.dtype)
    paved_image = np.ones((pave_h, pave_w, 3), dtype=image.dtype) * np.mean(image, dtype=int)
    pave_x = int(np.random.randint(0, pave_w - img_width + 1))
    pave_y = int(np.random.randint(0, pave_h - img_height + 1))
    paved_image[pave_y:pave_y + img_height, pave_x:pave_x + img_width] = image
    # pave detections
    if len(igs) > 0:
        igs[:, 0:4:2] += pave_x
        igs[:, 1:4:2] += pave_y
        keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & \
                    ((igs[:, 3] - igs[:, 1]) >= 8)
        igs = igs[keep_inds]

    if len(gts) > 0:
        gts[:, 0:4:2] += pave_x
        gts[:, 1:4:2] += pave_y
        keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit)
        gts = gts[keep_inds]

    return paved_image, gts, igs

def augment(img_data, c, img):
    assert 'filepath' in img_data
    assert 'bboxes' in img_data
    img_data_aug = copy.deepcopy(img_data)
    if img is None:
        img = cv2.imread(img_data_aug['filepath'])
    img_height, img_width = img.shape[:2]

    # random brightness
    if c.brightness and np.random.randint(0, 2) == 0:
        img = _brightness(img, min=c.brightness[0], max=c.brightness[1])
    # random horizontal flip
    if c.use_horizontal_flips and np.random.randint(0, 2) == 0:
        img = cv2.flip(img, 1)
        if len(img_data_aug['bboxes']) > 0:
            img_data_aug['bboxes'][:, [0, 2]] = img_width - img_data_aug['bboxes'][:, [2, 0]]
        if len(img_data_aug['ignoreareas']) > 0:
            img_data_aug['ignoreareas'][:, [0, 2]] = img_width - img_data_aug['ignoreareas'][:, [2, 0]]

    gts = np.copy(img_data_aug['bboxes'])
    igs = np.copy(img_data_aug['ignoreareas'])

    img, gts, igs = resize_image(img, gts, igs, scale=(0.4, 1.5))
    if img.shape[0] >= c.size_train[0]:
        img, gts, igs = random_crop(img, gts, igs, c.size_train, limit=16)
    else:
        img, gts, igs = random_pave(img, gts, igs, c.size_train, limit=16)

    img_data_aug['bboxes'] = gts
    img_data_aug['ignoreareas'] = igs

    img_data_aug['width'] = c.size_train[1]
    img_data_aug['height'] = c.size_train[0]

    return img_data_aug, img

class CityPersons(Dataset):
    def __init__(self, path, mode, config, transform=None):
        if transform is None:
            transform = transforms.Compose(
                [transforms.ColorJitter(brightness=0.5), transforms.ToTensor(),
                 transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
        self.dataset = get_citypersons(root_dir=path, mode=mode)
        self.dataset_len = len(self.dataset)
        self.mode = mode

        if self.mode == 'train' and config.train_random:
            random.shuffle(self.dataset)
        self.config = config
        self.transform = transform

        if self.mode == 'train':
            self.preprocess = RandomResizeFix(size=config.size_train, scale=(0.4, 1.5))
        else:
            self.preprocess = None

    def __getitem__(self, item):

        # input is RGB order, and normalized
        img_data = self.dataset[item]
        img = Image.open(img_data['filepath'])

        if self.mode == 'train':
            gts = img_data['bboxes'].copy()
            igs = img_data['ignoreareas'].copy()

            x_img, gts, igs = self.preprocess(img, gts, igs)

            y_center, y_height, y_offset = self.calc_gt_center(gts, igs, radius=2, stride=self.config.down)

            if self.transform is not None:
                x_img = self.transform(x_img)

            return x_img, [y_center, y_height, y_offset]

        else:
            if self.transform is not None:
                x_img = self.transform(img)
            else:
                x_img = img

            return x_img

    def __len__(self):
        return self.dataset_len

    def calc_gt_center(self, gts, igs, radius=2, stride=4):

        def gaussian(kernel):
            sigma = ((kernel - 1) * 0.5 - 1) * 0.3 + 0.8
            s = 2 * (sigma ** 2)
            dx = np.exp(-np.square(np.arange(kernel) - int(kernel / 2)) / s)
            return np.reshape(dx, (-1, 1))

        scale_map = np.zeros((2, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        offset_map = np.zeros((3, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        pos_map = np.zeros((3, int(self.config.size_train[0] / stride), int(self.config.size_train[1] / stride)))
        pos_map[1, :, :, ] = 1  # channel 1: 1-value mask, ignore area will be set to 0

        if len(igs) > 0:
            igs = igs / stride
            for ind in range(len(igs)):
                x1, y1, x2, y2 = int(igs[ind, 0]), int(igs[ind, 1]), int(np.ceil(igs[ind, 2])), int(
                    np.ceil(igs[ind, 3]))
                pos_map[1, y1:y2, x1:x2] = 0

        if len(gts) > 0:
            gts = gts / stride
            for ind in range(len(gts)):
                x1, y1, x2, y2 = int(np.ceil(gts[ind, 0])), int(np.ceil(gts[ind, 1])), int(gts[ind, 2]), int(
                    gts[ind, 3])
                c_x, c_y = int((gts[ind, 0] + gts[ind, 2]) / 2), int((gts[ind, 1] + gts[ind, 3]) / 2)

                dx = gaussian(x2 - x1)
                dy = gaussian(y2 - y1)
                gau_map = np.multiply(dy, np.transpose(dx))

                pos_map[0, y1:y2, x1:x2] = np.maximum(pos_map[0, y1:y2, x1:x2], gau_map)  # gauss map
                pos_map[1, y1:y2, x1:x2] = 1  # 1-mask map
                pos_map[2, c_y, c_x] = 1  # center map

                scale_map[0, c_y - radius:c_y + radius + 1, c_x - radius:c_x + radius + 1] = np.log(
                    gts[ind, 3] - gts[ind, 1])  # log value of height
                scale_map[1, c_y - radius:c_y + radius + 1, c_x - radius:c_x + radius + 1] = 1  # 1-mask

                offset_map[0, c_y, c_x] = (gts[ind, 1] + gts[ind, 3]) / 2 - c_y - 0.5  # height-Y offset
                offset_map[1, c_y, c_x] = (gts[ind, 0] + gts[ind, 2]) / 2 - c_x - 0.5  # width-X offset
                offset_map[2, c_y, c_x] = 1  # 1-mask

        return pos_map, scale_map, offset_map

class RandomResizeFix(object):
    """
    Args:
        size: expected output size of each edge
        scale: scale factor
        interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, scale=(0.4, 1.5), interpolation=Resampling.BILINEAR):
        self.size = size
        self.interpolation = interpolation
        self.scale = scale

    def __call__(self, img, gts, igs):
        # resize image
        w, h = img.size
        ratio = np.random.uniform(self.scale[0], self.scale[1])
        n_w, n_h = int(ratio * w), int(ratio * h)
        img = img.resize((n_w, n_h), self.interpolation)
        gts = gts.copy()
        igs = igs.copy()

        # resize label
        if len(gts) > 0:
            gts = np.asarray(gts, dtype=float)
            gts *= ratio

        if len(igs) > 0:
            igs = np.asarray(igs, dtype=float)
            igs *= ratio

        # random flip
        w, h = img.size
        if np.random.randint(0, 2) == 0:
            img = img.transpose(Image.FLIP_LEFT_RIGHT)
            if len(gts) > 0:
                gts[:, [0, 2]] = w - gts[:, [2, 0]]
            if len(igs) > 0:
                igs[:, [0, 2]] = w - igs[:, [2, 0]]

        if h >= self.size[0]:
            # random crop
            img, gts, igs = self.random_crop(img, gts, igs, self.size, limit=16)
        else:
            # random pad
            img, gts, igs = self.random_pave(img, gts, igs, self.size, limit=16)

        return img, gts, igs

    @staticmethod
    def random_crop(img, gts, igs, size, limit=8):
        w, h = img.size
        crop_h, crop_w = size

        if len(gts) > 0:
            sel_id = np.random.randint(0, len(gts))
            sel_center_x = int((gts[sel_id, 0] + gts[sel_id, 2]) / 2.0)
            sel_center_y = int((gts[sel_id, 1] + gts[sel_id, 3]) / 2.0)
        else:
            sel_center_x = int(np.random.randint(0, w - crop_w + 1) + crop_w * 0.5)
            sel_center_y = int(np.random.randint(0, h - crop_h + 1) + crop_h * 0.5)

        crop_x1 = max(sel_center_x - int(crop_w * 0.5), int(0))
        crop_y1 = max(sel_center_y - int(crop_h * 0.5), int(0))
        diff_x = max(crop_x1 + crop_w - w, int(0))
        crop_x1 -= diff_x
        diff_y = max(crop_y1 + crop_h - h, int(0))
        crop_y1 -= diff_y
        cropped_img = img.crop((crop_x1, crop_y1, crop_x1 + crop_w, crop_y1 + crop_h))

        # crop detections
        if len(igs) > 0:
            igs[:, 0:4:2] -= crop_x1
            igs[:, 1:4:2] -= crop_y1
            igs[:, 0:4:2] = np.clip(igs[:, 0:4:2], 0, crop_w)
            igs[:, 1:4:2] = np.clip(igs[:, 1:4:2], 0, crop_h)
            keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & ((igs[:, 3] - igs[:, 1]) >= 8)
            igs = igs[keep_inds]

        if len(gts) > 0:
            before_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])
            gts[:, 0:4:2] -= crop_x1
            gts[:, 1:4:2] -= crop_y1
            gts[:, 0:4:2] = np.clip(gts[:, 0:4:2], 0, crop_w)
            gts[:, 1:4:2] = np.clip(gts[:, 1:4:2], 0, crop_h)

            after_area = (gts[:, 2] - gts[:, 0]) * (gts[:, 3] - gts[:, 1])

            keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit) & (after_area >= 0.5 * before_area)
            gts = gts[keep_inds]

        return cropped_img, gts, igs

    @staticmethod
    def random_pave(img, gts, igs, size, limit=8):
        img = np.asarray(img)
        h, w = img.shape[0:2]
        pave_h, pave_w = size
        # paved_image = np.zeros((pave_h, pave_w, 3), dtype=image.dtype)
        paved_image = np.ones((pave_h, pave_w, 3), dtype=img.dtype) * np.mean(img, dtype=int)
        pave_x = int(np.random.randint(0, pave_w - w + 1))
        pave_y = int(np.random.randint(0, pave_h - h + 1))
        paved_image[pave_y:pave_y + h, pave_x:pave_x + w] = img
        # pave detections
        if len(igs) > 0:
            igs[:, 0:4:2] += pave_x
            igs[:, 1:4:2] += pave_y
            keep_inds = ((igs[:, 2] - igs[:, 0]) >= 8) & ((igs[:, 3] - igs[:, 1]) >= 8)
            igs = igs[keep_inds]

        if len(gts) > 0:
            gts[:, 0:4:2] += pave_x
            gts[:, 1:4:2] += pave_y
            keep_inds = ((gts[:, 2] - gts[:, 0]) >= limit)
            gts = gts[keep_inds]

        return Image.fromarray(paved_image), gts, igs

class Config(object):
    def __init__(self):
        self.gpu_ids = [0, 1]
        self.onegpu = 4
        self.num_epochs = 150
        self.add_epoch = 0
        self.iter_per_epoch = 2000
        self.init_lr = 2e-4
        self.alpha = 0.999

        # dataset
        self.train_path = './data/citypersons'
        self.train_random = True

        # setting for network architechture
        self.network = 'resnet50'  # or 'mobilenet'
        self.point = 'center'  # or 'top', 'bottom
        self.scale = 'h'  # or 'w', 'hw'
        self.num_scale = 1  # 1 for height (or width) prediction, 2 for height+width prediction
        self.offset = True  # append offset prediction or not
        self.down = 4  # downsampling rate of the feature map for detection
        self.radius = 2  # surrounding areas of positives for the scale map

        # setting for data augmentation
        self.use_horizontal_flips = True
        self.brightness = (0.5, 2, 0.5)
        self.size_train = (640, 1280)
        self.size_test = (1024, 2048)

        # image channel-wise mean to subtract, the order is BGR
        self.img_channel_mean = [103.939, 116.779, 123.68]

        # use teacher
        self.teacher = True

        self.test_path = './data/citypersons'

        # whether or not to do validation during training
        self.val = True
        self.val_frequency = 1


from includes.RootLogger import rootLogger
import traceback
import datetime
import concurrent.futures
import time
import threading

from includes.SMTPSocks import MYSMTP, MYSMTP_SSL
from includes.SMTPThread import SMTPThread

# campaign state constants
CS_CREATED = 0
CS_PROCESSING = 1
CS_PAUSED = 2
CS_HIBERNATED = 3
CS_COMPLETED = 4
CS_DELETED = 5

# campaign REQ_MARK behaviours
FB_SET_FAIL = 0
FB_SET_NULL = 1

# every processed task behaviour
RS_NEW = 0
RS_PROCESSING = 1
RS_SUCCESS = 2
RS_FAILED = 3
RS_SKIPPED = 4
RS_BLACKLIST = 5

class CampaignThread(threading.Thread):
    def __init__(self, campaign, socks=[], settings=None, my_redis=None, threadLock=None, attachements=[], macro_data=[], accounts=[]):
        self.campaign = campaign
        # self.prefix = prefix
        self.started_at = None
        self.socks = socks
        self.settings = settings
        self.my_redis = my_redis
        self.status = CS_PROCESSING
        self.accounts = accounts
        self.threads = []
        self.threadLock = threadLock
        self.attachements = attachements
        self.macro_data = macro_data

        self.prefix = f"Campaign[{self.campaign.id}]:"

        self.result = {"_id": campaign.id, "status": self.status, "updated_at": None, "finnished_at": None}
        # startt regular
        threading.Thread.__init__(self)

    def run(self):
        self.started_at = time.time()
        rootLogger.info(f"{self.prefix} Begin campaign")
        # generate as many workers as nneded

        # run untill it's either ends or just paused, thou if paused, we don't give a fuck for that
        while True:
            # remove dead threads from list
            live_threads = [thread for thread in self.threads if thread.isAlive()]
            dead_threads = [thread for thread in self.threads if not thread.isAlive()]

            # for dead threads, extract sock and apply to others...

            self.threads = live_threads

            # generate missing threads for campaign
            self.appendWorkers()

            # check whether it's time to stop or not
            if len(self.threads) < 1:
                rootLogger.info(f"{self.prefix} all threads are finnished, marking as done and exit")
                self.status = CS_COMPLETED
                break
            elif self.status != CS_PROCESSING:
                # mark all threads as done and exit
                rootLogger.info(f"{self.prefix} marked as done from Main or UI, stop smtp-threads and exit")
                for thread in self.threads:
                    # this one if necessearly only because of tests
                    stop = getattr(thread, "stop", None)
                    if callable(stop):
                        thread.stop()
                        # stop(thread)

                break

            time.sleep(5)
            rootLogger.info(f"{self.prefix} work in progress ({len(self.threads)} active workers)")

            # # FIXME: remove after tests
            # if time.time() - self.started_at > 15*60:
            #     self.threads = []
            #     rootLogger.info(f"{self.prefix} simulate all threads are finnished")

        self.result.update({"status": CS_COMPLETED, "updated_at": datetime.datetime.utcnow(), "finnished_at": datetime.datetime.utcnow()})

    # controlling from outside
    def stop(self, force=False):
        self.status = CS_PAUSED if force else CS_COMPLETED

    def appendWorkers(self):
        # if amount of thread is lower than

        while len(self.threads) < int(self.settings['workers_per_campaign']):
            try:
                account = next(self.accounts)

                rootLogger.info(f"{self.prefix} got new account from self.acocunts:{account.id}")
                # create new smtp-sender thread
                new_thread = SMTPThread(
                    campaign = self.campaign,
                    account=account,
                    socks = self.socks,
                    settings = self.settings,
                    my_redis = self.my_redis,
                    threadLock = self.threadLock,
                    attachements = self.attachements,
                    macro_tpls = self.macro_data)

                new_thread.daemon = True
                new_thread.start()

                self.threads.append( new_thread )

            except Exception as e:
                rootLogger.error(f"{self.prefix}[run][appendWorkers]:{e}")
                # raise e
                break

        # if amount of thread is higher than settings, just finish first few
        if len(self.threads) > int(self.settings['workers_per_campaign']):
            kill_amount = len(self.threads) - int(self.settings['workers_per_campaign'])
            kill_threads = self.threads[0:kill_amount]

            for thread in self.threads:
                # this one if necessearly only because of tests
                stop = getattr(thread, "stop", None)
                if callable(stop):
                    thread.stop()

    def updateSettings(self, _new_settings):
        # make copy of settings
        self.settings = dict(_new_settings)
        for thread  in self.threads:
            if thread.isAlive():
                thread.updateSettings( _new_settings )

    def updateSocks(self, _new_socks):
        # commonly i hope just to overwrite here as all child threads uses same list

        current_socks_ids = [item['id'] for item in self.socks]
        # append items
        append_new = [item for item in _new_socks if item['id'] not in current_socks_ids]

        # remove items
        new_socks_ids = [item['id'] for item in _new_socks]
        removal_ids = [item['id'] for item in self.socks if item['id'] not in new_socks_ids]

        self.threadLock.acquire(blocking=True, timeout=300)
        try:

            # remove bad items
            for item in self.socks:
                if item['id'] in removal_ids:
                    self.socks.remove( item )

            # append new items
            self.socks.extend( append_socks )

        except Exception as e:
            rootLogger.error(f"{self.prefix}[updateSocks]:{e}")
        finally:
            if self.threadLock.locked():
                self.threadLock.release()


""" Usage:
    analyze --in=INPUT_FILE --out=OUTPUT_FILE

    Analyze question distribution in QA-SRL input file. Store the output in the given output file.
"""
from collections import defaultdict
from docopt import docopt
import logging
logging.basicConfig(level = logging.DEBUG)
from nltk.stem import WordNetLemmatizer

from oie_readers.extraction import QUESTION_OBJ2_INDEX, QUESTION_PP_INDEX, QUESTION_TRG_INDEX,\
    generalize_question

from qa_to_oie import Qa2OIE

class Analyzer:
    """
    Container for analysis functions
    """
    def __init__(self):
        """
        Intialize memebers:
        question_dist - generalized-question distribution of the assigned extraction
                        location.
        """
        self.question_dist = defaultdict(lambda : defaultdict(lambda : 0))
        self.lmtzr = WordNetLemmatizer()

    def analyze_question_patterns(self, extractions):
        """
        Calcualtes a per generalized-question distribution of the assigned location in
        the tuple. Stores the output in the question_distribution memeber variable.
        """
        for ex in extractions:
            # Count the observed location for each generalized question
            for ind, gen_question in enumerate([generalize_question(question)
                                                for (_, question) in ex.getSortedArgs()]):
                self.question_dist[gen_question][ind] += 1

    def get_dist(self):
        """
        Get simple dict representation of the distribtion
        """
        return dict([(k, v)
                     for k, v in self.question_dist.items()])

    def get_sorted_dist(self):
        """
        Get the question distribution sorted by occurences
        """
        return sorted(iter(self.get_dist().items()),
                      key = lambda q_dist: sum(q_dist[1].values()),
                      reverse = True)

    def output_dist_to_file(self, json_fn):
        """
        Write this distribution (as a sorted list, based on occurences)
        to an output json file
        """
        import json
        with open(json_fn, 'w') as fout:
            json.dump(self.get_dist(),
                      fout)

if __name__ == "__main__":
    # Parse arguments
    args = docopt(__doc__)
    logging.debug(args)
    fin = args['--in']
    fout = args['--out']

    # Get extractions
    logging.info("Reading QA-SRL from: {}".format(fin))
    q = Qa2OIE(fin)
    extractions = [ex
                   for cur_extractions in list(q.dic.values())
                   for ex in cur_extractions]

    # Analyze frequency
    logging.info("Analyzing frquency")
    analyzer = Analyzer()
    analyzer.analyze_question_patterns(extractions)
    d = analyzer.get_sorted_dist()

    # Write to file
    logging.info("Writing output to file: {}".format(fout))
    analyzer.output_dist_to_file(fout)

    logging.info("DONE!")


import matplotlib.pyplot as plt
import numpy as np
from collections import deque

class Plotter(object):
    def __init__(self, fig_num=0, capacity=20000, save_path='./figure/v1',
                 show_figure=False, interval=100, episodes=100):
        self.fig_num = fig_num
        self.x = deque(iterable=[], maxlen=capacity)
        self.t = deque(iterable=[], maxlen=capacity)
        self.show_figure = show_figure
        self.t_now = 0
        self.interval = interval
        self.episodes = episodes
        self.title_text = 'Average Reward on {interval} Steps'.format(interval=self.episodes)
        self.save_path = save_path
        return

    def plot(self, x):
        plt.figure(self.fig_num)
        self.x.append(x)
        self.t.append(self.t_now)
        self.t_now += self.interval
        line_1, = plt.plot(self.t, self.x, label='line_0')
        plt.legend(handles=[line_1])
        plt.xlabel('Step')
        plt.ylabel('Reward')
        plt.title(self.title_text)
        plt.savefig(self.save_path)
        plt.close(self.fig_num)
        return


"""
https://adventofcode.com/2018/day/7
"""
import re
from collections import namedtuple

TEST_INPUT = """Step C must be finished before step A can begin.
Step C must be finished before step F can begin.
Step A must be finished before step B can begin.
Step A must be finished before step D can begin.
Step B must be finished before step E can begin.
Step D must be finished before step E can begin.
Step F must be finished before step E can begin."""

rgx = r"Step ([A-Z]) must be finished before step ([A-Z]) can begin."

def parse_requirement(line):
    tail, head = re.match(rgx, line).groups()
    return (tail, head)

REQUIREMENTS = [parse_requirement(line) for line in TEST_INPUT.split('\n')]

def get_steps(reqs):
    steps = set()
    steps = steps.union([i for (i,_) in reqs])
    steps = steps.union([j for (_,j) in reqs])
    return steps

def make_dependencies(reqs):
    depends_on = dict()
    steps = get_steps(reqs)
    heads = {req[0] for req in reqs}
    tails = {req[1] for req in reqs}
    initials = {req[0] for req in reqs if req[0] not in tails}
    terminals = {req[1] for req in reqs if req[1] not in heads}
    return terminals, initials

def order(reqs):
    """lets try a naive approach"""
    requirements = reqs[:]
    lis = []
    terminals, initials = make_dependencies(reqs)
    while initials:
        next_step = min(initials)
        lis.append(next_step)
        requirements = [req for req in requirements if req[0] != next_step]
        _, initials = make_dependencies(requirements)
    lis.append(min(terminals))
    return lis
    # does not work for multiple terminals?
    #completed = set()
    #for step in get_steps(deps):
assert ''.join(order(REQUIREMENTS)) == 'CABDFE'

with open('data/day07_input.txt') as f:
    lines = [line.strip() for line in f]
    requirements = [parse_requirement(line) for line in lines]

# Part 1:
#print(''.join(order(requirements)))

#class Task:
#    def __init__(self, label, worked_on, remaining_dur):
#        self.label = label
#        self.worked_on = worked_on
#        self.remaining_dur = remaining_dur

Task = namedtuple('Task', ['label', 'worked_on', 'remaining_dur'])
State = namedtuple('State', ['actives', 'availables', 'elves', 'reqs'])
#class State:
#    def __init__(self, actives, availables, elves, reqs):
#        self.actives = actives,
#        self.availables = availables,
#        self.elves = elves,
#        self.reqs = reqs
#    def is_valid(self):
#        return (not bool(availables)) or (not bool(elves))

def extra_dur(c):
    return ord(c) - 64

def make_dependencies_tasks(reqs, base_dur):
    depends_on = dict()
    steps = get_steps(reqs)
    heads = {req[0] for req in reqs}
    tails = {req[1] for req in reqs}
    initials  = {req[0] for req in reqs if req[0] not in tails}
    terminals = {req[1] for req in reqs if req[1] not in heads}
    initial_tasks  = {Task(i, False, extra_dur(i) + base_dur)
                      for i in initials}
    terminal_tasks = {Task(i, False, extra_dur(i) + base_dur)
                      for i in terminals}
    return terminal_tasks, initial_tasks

def initial_state(reqs, elves, base_dur):
    terminals, initials = make_dependencies_tasks(reqs, base_dur)
    active_tasks = set()
    while elves and initials:
        new = min(initials, key = lambda x: x.label)
        active_tasks.add(new._replace(worked_on = True))
        initials.remove(new)
        elves -= 1
    return active_tasks

def time_step(tasks, elves):
    pass

def parallel_time(reqs, elves, base_dur):
    requirements = reqs[:]
    free_elves = elves
    _, inits = make_dependencies(requirements)
    duration = 0
    completed, actives = set(), set()
    while free_elves:
        if inits:
            for init in inits:
                actives.add(Task(min(inits), True, extra_dur(init) + base_dur))
                elves -= 1
                inits.remove(init)
        else: break


import re
import yaml
from shutil import copyfile
from loguru import logger

import revive.computation.operators as opt

OPERATORS_NAME = sorted(opt.__all__, reverse=False)

def get_nodes(yaml_file_path,):
    with open(yaml_file_path, 'r', encoding='UTF-8') as f:
        raw_config = yaml.load(f, Loader=yaml.FullLoader)

    data_config = raw_config['metadata']['columns']
    nodes_name = set([list(d.values())[0]['dim'] for d in data_config])
    nodes = { node_name : [list(d.keys())[0]  for d in data_config if list(d.values())[0]['dim'] == node_name] for node_name in nodes_name}

    for node in raw_config['metadata']['graph'].keys():
        if "next_" in node and node[5:] in nodes.keys():
            nodes[node] = nodes[node[5:]]

    return nodes

def matching_bracket(string, idx, brackets=[]):
    a = {
        "(": ")",
        "[": "]",
        "{": "}",
        "<": ">"
    }
    if brackets:
        a = { k:v for k in a.items() for k in brackets}

    close_bracket = a[string[idx]]

    if idx != string.rindex(string[idx]):
        b = string.rindex(string[idx])
        c = string[b:].index(close_bracket)
        str_list = list(string)
        str_list[b], str_list[b+c] = ".", "."
        return matching_bracket(''.join(str_list), idx)
    else:
        d = string[idx:].index(close_bracket)
        return idx + d

def strip(s):
    return re.sub("\s|\t|\n", "", s)

def find_all(s, sub_s):
    return [m.start() for m in re.finditer(sub_s, s)]

def find_node(line):
    nodes = {}
    for node_name in sorted(NODES.keys(), reverse=True):
        res = find_all(line, node_name)
        for index in res:
            bracket_start_index = index+len(node_name)
            if bracket_start_index==len(line):
                continue
            if line[bracket_start_index] != "[" or line[bracket_start_index+1:bracket_start_index+5] == "...,":
                continue
            bracket_stop_index = matching_bracket(line, bracket_start_index) + 1
            nodes[bracket_start_index] = bracket_stop_index

    nodes = [[index, nodes[index]] for index in sorted(nodes.keys())]
    return nodes

def find_column(line):
    columns = {}
    for node_name in NODES.keys():
        for column_name in NODES[node_name]:
            res = find_all(line, column_name)
            for index in res:
                if line[index-1:index+len(column_name)+1] == f'"{column_name}"' or \
                    line[index-1:index+len(column_name)+1] == f"'{column_name}'":
                    columns[index] = [column_name,node_name]

    columns = [[index, columns[index]] for index in sorted(columns.keys())]
    return columns

def find_operator(line):
    operators = {}
    for operator_name in OPERATORS_NAME:
        res = find_all(line, operator_name)
        for index in res:
            if line[index-1] in [" ", "=", "(", ","] and not bool(re.match('[a-zA-Z0-9_]+', line[index+len(operator_name)])):
                operators[index] = operator_name

    operators = [[index, operators[index]] for index in sorted(operators.keys())]

    return operators

def convert_operator(oral_operator_name):
    return "opt." + oral_operator_name

def convert_column(oral_column,flag):
    column_name, node_name = oral_column
    return f"{NODES[node_name].index(column_name)}"

def convert_node(bracket_start_index):
    return f"{NODES[node_name].index(column_name)}"

def convert_operators(line):
    """ Convert operator

    Example: add -> opt.add
    """
    operators = find_operator(line)
    if operators:
        first_operator_index, first_operator_name = operators[0]
        line = line[:first_operator_index] \
               + convert_operator(first_operator_name) \
               + line[first_operator_index+len(first_operator_name):]
        return convert_operators(line)
    else:
        return line

def convert_columns(line):
    """ Convert column name to column index

    Example: "obs_1" -> 1
    """
    columns = find_column(line)
    if columns:
        first_column_index, first_column_name = columns[0]
        if "[" == line[first_column_index-2]:
            flag = "L"
        elif "]" == line[first_column_index+len(first_column_name[0])+1]:
            flag = "R"
        else:
            flag = "C"

        line = line[:first_column_index-1] \
               + convert_column(first_column_name,flag) \
               + line[first_column_index+len(first_column_name[0])+1:]
        return convert_columns(line)
    else:
        return line

def convert_nodes(line):
    """ Convert node to tensor

    Example: obs[1,2] -> obs[...,[1,2]]
    """
    nodes = find_node(line)
    if nodes:
        bracket_start_index, bracket_stop_index = nodes[0]
        if ":" in line[bracket_start_index:bracket_stop_index]:
            line = line[:bracket_start_index] \
                   + "[...," \
                   + line[bracket_start_index+1:bracket_stop_index-1] \
                   + "]" \
                   + line[bracket_stop_index:]
        else:
            line = line[:bracket_start_index] \
                   + "[...," \
                   + line[bracket_start_index:bracket_stop_index] \
                   + "]" \
                   + line[bracket_stop_index:]
        return convert_nodes(line)
    else:
        return line

def convert_line(line):
    line = strip(line)
    line = convert_operators(line)
    line = convert_columns(line)
    line = convert_nodes(line)
    return line

def checkt_convert(line):
    comvert_line_copy = line
    for fn in [convert_operators,convert_columns,convert_nodes]:
        comvert_line_copy = fn(comvert_line_copy)
    if comvert_line_copy == line:
        return False
    return True

def convert_fn_def(origin_code : list):
    ''' find the intent of original code '''
    codes = []
    index = 0
    for i, code in enumerate(origin_code):
        if code.startswith('def '):
            bracket_start_index = code.index("(")
            codes.append(code[:bracket_start_index]+"(data: Dict[str, torch.Tensor]) -> torch.Tensor:\n")
            index = i
        if ":" in code:
            index = i
            break

    args = strip("".join(origin_code[:index+1]))
    bracket_start_index = args.index("(")
    bracket_stop_index = matching_bracket(args,bracket_start_index)
    args = args[bracket_start_index+1:bracket_stop_index].split(",")
    for arg in args:
        codes.append(f'    {arg}=data["{arg}"]\n')

    other_codes = origin_code[index+1:]

    return codes, other_codes

def get_fn_list(origin_code_list):
    fn_start_index = []
    fn_stop_index = []
    for i,code in enumerate(origin_code_list):
        if code.startswith('def '):
            if fn_start_index:
                fn_stop_index.append(i)
            fn_start_index.append(i)
    fn_stop_index.append(i+1)

    return [origin_code_list[i:j] for i,j in zip(fn_start_index,fn_stop_index)]

def parser(input_file : str,
           output_file : str,
           yaml_file : str):

    global NODES
    NODES = get_nodes(yaml_file)

    with open(input_file, 'r') as f:
        origin_code_list = f.readlines()

    for code in origin_code_list:
        if code.startswith("import torch"):
            logger.info(f'Not parser function in {input_file}')
            copyfile(input_file, output_file)
            return

    logger.info(f'Parser function in {input_file}')

    output_codes = []
    output_codes.append("import torch\n")
    output_codes.append("from typing import Dict\n")
    output_codes.append("\n")
    output_codes.append("import revive.computation.operators as opt\n")
    output_codes.append("\n")

    fn_list = get_fn_list(origin_code_list)

    for fn in fn_list:
        start_codes, other_codes = convert_fn_def(fn)
        output_codes += start_codes

        for code in other_codes:
            if not checkt_convert(code):
                output_codes.append(code)
                continue
            if " return " in code:
                if "(" not in code and "[" not in code:
                    output_codes.append(code)
                else:
                    sub_code = convert_line("return="+code[code.index("return")+len("return"):])
                    sub_code = sub_code.replace("return=", "return ")
                    output_codes.append(" "*(len(code) - len(code.lstrip()))+sub_code+"\n")
            else:
                output_codes.append(" "*(len(code) - len(code.lstrip()))+convert_line(code)+"\n")

    with open(output_file, 'w') as f:
        f.writelines(output_codes)


import torch
from torch.utils.data import Dataset, DataLoader
from transformers import BertTokenizer, BertForMaskedLM, AdamW, DataCollatorForLanguageModeling
import time

device = "mps" if getattr(torch,'has_mps',False) \
    else "gpu" if torch.cuda.is_available() else "cpu"
# 1. Load a pre-trained BERT model and tokenizer
model_name = "hfl/chinese-bert-wwm"
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForMaskedLM.from_pretrained(model_name).to(device)

# 2. Prepare your dataset of Chinese poems
# poems = ["床前明月光", "疑是地上霜", "举头望明月", "低头思故乡"]  # Replace with your own dataset

file_path = "../Raw_data/chinese_poems.txt"

with open(file_path, "r", encoding="utf-8") as f:
    poems = [line.strip() for line in f.readlines()]
# 3. Tokenize your dataset using the tokenizer
class PoemDataset(Dataset):
    def __init__(self, poems, tokenizer):
        self.poems = poems
        self.tokenizer = tokenizer

    def __len__(self):
        return len(self.poems)

    def __getitem__(self, idx):
        poem = self.poems[idx]
        encoding = tokenizer.encode_plus(
            poem,
            add_special_tokens=True,
            return_tensors="pt",
            padding="max_length",
            truncation=True,
            max_length=128
        )
        return {key: tensor.squeeze(0) for key, tensor in encoding.items()}

poem_dataset = PoemDataset(poems, tokenizer)
dataloader = DataLoader(poem_dataset, batch_size=256)  # Adjust batch_size based on your available resources

# 4. Fine-tune the BERT model on the MLM task
data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=True, mlm_probability=0.15)

epochs = 3
optimizer = AdamW(model.parameters(), lr=2e-5)

for epoch in range(epochs):
    model.train()
    epoch_start_time = time.time()
    total_loss = 0
    num_batches = 0

    for batch in dataloader:
        batch_start_time = time.time()

        input_ids = batch["input_ids"].to(device)
        attention_mask = batch["attention_mask"].to(device)

        examples = [{k: v[i] for k, v in batch.items()} for i in range(batch["input_ids"].shape[0])]
        batch = data_collator(examples)

        labels = batch["labels"].to(device)

        optimizer.zero_grad()
        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
        loss = outputs.loss
        loss.backward()
        optimizer.step()

        batch_time = time.time() - batch_start_time
        total_loss += loss.item()
        num_batches += 1

        print(f"Epoch {epoch + 1}/{epochs}, Batch {num_batches}: Loss = {loss.item()}, Time = {batch_time:.2f}s")

    epoch_time = time.time() - epoch_start_time
    avg_loss = total_loss / num_batches
    fine_tuned_model_path = "fine_tuned_chinese_bert" + str(epoch)
    tokenizer.save_pretrained(fine_tuned_model_path)
    model.save_pretrained(fine_tuned_model_path)
    print(f"Epoch {epoch + 1}/{epochs}: Average Loss = {avg_loss}, Time = {epoch_time:.2f}s")

# 5. Save the fine-tuned model and tokenizer
fine_tuned_model_path = "fine_tuned_chinese_bert"
tokenizer.save_pretrained(fine_tuned_model_path)
model.save_pretrained(fine_tuned_model_path)


import math
from datetime import datetime
import time
import location
from gtfsstatic import GtfsStatic

class GtfsFunctions:

    def __init__(self, data_directory, gtfs_directory, position: dict = None):
        self.__data_directory = data_directory
        self.__gtfs_directory = gtfs_directory
        self.gtfs = GtfsStatic(data_directory, gtfs_directory).gtfs
        if position is None:
            self.position = self.get_location()
        else:
            self.position = position

    # get current location coordinates from the IOS location module
    def get_location(self):
    	location.start_updates()
    	time.sleep(1)
    	loc = location.get_location()
    	location.stop_updates()
    	if loc:
    		return {'lat': loc['latitude'], 'lon': loc['longitude']}

    # Haversine formula
    def distance(self, lat1, lon1, lat2, lon2):  # generally used geo measurement function
        R = 6378.137; # Radius of earth in KM
        dLat = lat2 * math.pi / 180 - lat1 * math.pi / 180
        dLon = lon2 * math.pi / 180 - lon1 * math.pi / 180
        a = (math.sin(dLat/2) * math.sin(dLat/2) +
            math.cos(lat1 * math.pi / 180) * math.cos(lat2 * math.pi / 180) *
            math.sin(dLon/2) * math.sin(dLon/2))
        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
        d = R * c
        return d * 1000; # meters

    def get_time(self):
        now = datetime.now()
        cur_time = now.strftime("%H:%M:%S")
        if cur_time[0] is '0':
            cur_time = cur_time[1:]
        print(cur_time)
        return cur_time

    # return 1 (Monday) - 7 (Sunday)
    def get_weekday(self):
        now = datetime.now()
        return now.strftime("%A").lower()

    def get_nearest_stops(self, count: int = 1):
        nearest = [None] * count
        stops = self.gtfs['stops'].get_inorder()
        for i in range(count):
            nearest_gps = self.distance(self.position['lat'], self.position['lon'], float(stops[0]['stop_lat']), float(stops[0]['stop_lon']))
            for stop in stops:
                dist = self.distance(self.position['lat'], self.position['lon'], float(stop['stop_lat']), float(stop['stop_lon']))
                if dist < nearest_gps:
                    nearest[i] = stop
                    nearest_gps = dist
            stops.remove(nearest[i])
        return nearest

    def get_service_id_by_weekday(self, weekday: str):
        for day in self.gtfs['calendar']:
            if day[weekday] == '1':
                return day['service_id']

    def get_trip_id_by_service_id(self, service_id):
        trip_ids = []
        for trip in self.gtfs['trips'].values():
            if trip['service_id'] == service_id:
                trip_ids.append(trip['trip_id'])
        return trip_ids

    def get_all_buses_at_stop(self, stop_id):
        return [stop for stop in self.gtfs['stop_times'] if stop_id in stop['stop_id']]

    def get_next_bus_at_stop(self, stop_id, trip_ids, cur_time: str, count: int = 1):
        next_buses = []
        last_buses = []

        # use i to track last stops
        for stop_time, i in zip(self.gtfs['stop_times'],range(len(self.gtfs['stop_times']),-1,-1)):
            if stop_id in stop_time['stop_id'] and stop_time['arrival_time'] >= cur_time and stop_time['trip_id'] in trip_ids and stop_time not in next_buses:
                next_buses.append(stop_time)

        # sort incoming buses by arrival time
        next_buses = sorted(next_buses, key = lambda x: x['arrival_time'])

        count = len(next_buses) if count > len(next_buses) else count
        return next_buses[:count]

    def print_stops(self, nearest, count=1):
        for i in range(count):
            print('[' + str(i) + '] ' + str(round(self.distance(self.position['lat'], self.position['lon'], float(nearest[i]['stop_lat']), float(nearest[i]['stop_lon'])))) + 'm | '+ nearest[i]['stop_name'] + ' (' + nearest[i]['stop_code'] + ') ' + nearest[i]['stop_lat'] + ',' + nearest[i]['stop_lon'])

    def print_stop_times(self, stop_times, count=1):
        trips = self.gtfs['trips']
        stop_time_trips = []
        for stop_time in stop_times:
            stop_time_trips.append(trips[stop_time['trip_id']])

        count = len(stop_times) if count > len(stop_times) else count
        for i in range(count):
            print(stop_times[i]['arrival_time'][:len(stop_times[i]['arrival_time'])-3] + ' | ' + stop_time_trips[i]['trip_headsign'])

    def indices(self, lst, item):
        return [i for i, x in enumerate(lst) if x == item]

    def direction(self, destination):
        GtfsFunctions.dest_gtfsf = GtfsFunctions(self.__data_directory, self.__gtfs_directory, destination)
        dest_gtfsf = GtfsFunctions.dest_gtfsf
        weekday = self.get_weekday()
        time = self.get_time()
        today_service_id = self.get_service_id_by_weekday(weekday)
        today_trip_ids = self.get_trip_id_by_service_id(today_service_id)

        count = 6

        origin_nearest = self.get_nearest_stops(count)
        dest_nearest = dest_gtfsf.get_nearest_stops(count)

        next_bus_at_origin = self.get_next_bus_at_stop(origin_nearest[0]['stop_id'], today_trip_ids, time)[0]

        print(time)

        matching_stop_times = []
        matching_stop_ids = []

        start_stop = []

        for stop_time in self.gtfs['stop_times']:
            if (stop_time['stop_id'] == origin_nearest[0]['stop_id'] or stop_time['stop_id'] == dest_nearest[0]['stop_id']) and stop_time['trip_id'] in today_trip_ids and stop_time['trip_id'] in next_bus_at_origin['trip_id']:
                if stop_time['trip_id'] in matching_stop_ids and stop_time['trip_id'] in next_bus_at_origin['trip_id']:
                    matching_stop_ids.append(stop_time['trip_id'])
                    matching_stop_times.append(stop_time)

                    dup_indices = self.indices(matching_stop_ids, matching_stop_ids[-1])
                    start_stop.append((matching_stop_times[dup_indices[0]], matching_stop_times[dup_indices[1]]))

                    matching_stop_times.clear()
                    matching_stop_ids.clear()

                    break

                matching_stop_ids.append(stop_time['trip_id'])
                matching_stop_times.append(stop_time)

        print(start_stop)

        '''
        duplicate = []
        temp = []
        for matching_id in matching_stop_ids:
            for temp_trip in temp:
                if matching_id is not temp_trip and matching_id['trip_id'] == temp_trip['trip_id']:
                    duplicate.append(matching_id)
                    duplicate.append(temp_trip)
            temp.append(matching_id)

        print(duplicate[:2])
        '''


from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import VALID_NICKNAME, irc_lower

class NickCommand(Command):
    def onUse(self, user, data):
        if user.registered == 0:
            user.nick(data["nick"])
        else:
            if not user.nickname:
                user.registered -= 1
            user.nickname = str(data["nick"])
            if user.registered == 0:
                user.register()

    def processParams(self, user, params):
        if not params:
            user.sendMessage(irc.ERR_NONICKNAMEGIVEN, ":No nickname given")
            return {}
        if not params[0]:
            user.sendMessage(irc.ERR_ERRONEUSNICKNAME, "*", ":Erroneous nickname")
            return {}
        if not VALID_NICKNAME.match(params[0]):
            user.sendMessage(irc.ERR_ERRONEUSNICKNAME, params[0], ":Erroneous nickname")
            return {}
        if params[0] == user.nickname:
            return {}
        if params[0] in self.ircd.users and (not user.nickname or irc_lower(params[0]) != irc_lower(user.nickname)):
            user.sendMessage(irc.ERR_NICKNAMEINUSE, self.ircd.users[params[0]].nickname, ":Nickname is already in use")
            return {}
        return {
            "user": user,
            "nick": params[0]
        }

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd

    def spawn(self):
        return {
            "commands": {
                "NICK": NickCommand()
            }
        }


import re
from requests import get
from helpers.command import Command

def get_quote(symbol):
    params = {'q': "select BidRealtime,Bid,Name,ChangeinPercent,YearRange from yahoo.finance.quotes WHERE symbol='%s'" % symbol,
              'format': 'json', 'env': 'store://datatables.org/alltableswithkeys'}
    data = get("http://query.yahooapis.com/v1/public/yql", params=params).json()
    return data['query']['results']

def gen_stock(msg):
    quote = None
    num = 0
    while quote is None:
        if num > 5:
            break
        num += 1
        quote = get_quote(msg)
    if quote is not None:
        quote = quote['quote']
    else:
        return "No Results"
    if quote['Name'] is None:
        return "Invalid Symbol."
    else:
        if quote['BidRealtime']:
            val = quote['BidRealtime']
        elif quote['Bid']:
            val = quote['Bid']
        else:
            val = "No Data"
        return "%s (%s) -- %s %s 52wk: %s" % (quote['Name'], msg, val, quote['ChangeinPercent'], quote['YearRange'])

def random_stock():
    html = get('http://www.openicon.com/rsp/rsp_n100.php').text
    return re.search('\((.*)\)', html).group(1)

@Command('stock')
def cmd(send, msg, args):
    """Gets a stock quote.
    Syntax: !stock <symbol>
    """
    if not msg:
        msg = random_stock()
    send(gen_stock(msg))


from urllib import request
from urllib import parse
from bs4 import BeautifulSoup
from urllib.request import urlopen

import time
import random
import json

class tencent(object):
	def __init__(self):
		self.switch = True
		self.page = 1
		self.url = "https://hr.tencent.com/"
		self.headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36"}
		self.counter = 1

	def pause(self):
		second = random.randint(1, 3)

		time.sleep(second)

	def write2json(self, list):
		print("Writing to file")

		print("\n")

		data = json.dumps(list, ensure_ascii=False)

		with open("tencent_jobs.json", "a") as f:
			f.write(data)

		print("Write complished")

		print("\n")

	def parseHtml(self, html):
		soup = BeautifulSoup(html, "lxml")

		list = soup.select("tr[class='even']")

		list = list + soup.select("tr[class='odd']")

		job_list = []

		for each in list:
			print("Process on job %d"%self.counter)

			print("\n")

			job = {}

			positionName = each.select("td a")[0].get_text()

			category = each.select("td")[1].get_text()

			number = each.select("td")[2].get_text()

			location = each.select("td")[3].get_text()

			publishDate = each.select("td")[4].get_text()

			link = self.url + each.select("td a")[0].attrs['href']

			job["positionName"] = positionName

			job["category"] = category

			job["number"] = number

			job["location"] = location

			job["publishDate"] = publishDate

			job["link"] = link

			job_list.append(job)

			print("Job %d was completed"%self.counter)

			self.counter += 1

			print("\n")

		self.write2json(job_list)

	def loadPage(self, page):
		self.pause()
		# https://hr.tencent.com/position.php?keywords=&start=30#a
		num = (page-1) * 10

		start = {"start" : str(num)}

		start = parse.urlencode(start)

		url = self.url + "position.php?keywords=&" + start + "#a"

		req = request.Request(url, headers=self.headers)

		html = urlopen(req).read().decode("utf-8")

		# print(html)
		# with open("tencent.html", "w") as f:
		# 	f.write(html)

		self.parseHtml(html)

	def go(self):
		input("Push the enter to crawl the jobs information")

		while self.switch:
			try:
				self.loadPage(self.page)
			except request.URLError as e:
				print(e.message)
				continue

			haha = input("Push the enter to crawl next page, or type quit to exit")

			print("\n")

			self.page += 1

			if haha == 'quit':
				self.switch = False

				print("%d jobs was crawled. Thanks for use, have a good day, bye!"%self.counter)

				break

if __name__ == "__main__":
	spider = tencent()
	spider.go()


from django.core.urlresolvers import reverse
from django.conf import settings
from rest_framework.test import APITestCase
from rest_framework import status
from .factories import set_up_single, ArtistFactory, FestivalFactory
from stagegage.models import Artist, Festival
import json

class ArtistTests(APITestCase):
    """Integration tests for the artist API"""

    def setUp(self):
        """Set up a single instance of each model and assign to self."""
        dct = set_up_single()
        for k, v in dct.items():
            setattr(self, k, v)

        self.test_dict = {'id': self.artist.id,
                          'created': format_date(self.artist.created),
                          'name': self.artist.name,
                          'score': self.artist.score,
                          'genres': [self.genre.genre],
                          'review': self.review.text[:100],
                          'festivals': [
                              {'id': self.festival.id,
                               'name': self.festival.name,
                               'start_date': self.festival.start_date,
                               'created': format_date(self.festival.created)}]}
        self.client.force_authenticate(user=self.user)

    def test_list_view(self):
        """Test that get /artist returns list."""
        response = self.client.get(reverse('artists-list'))
        response_dict = json.loads(response.content)['results']
        self.assertEqual(response_dict, [self.test_dict])

    def test_create_single_artist(self):
        """Test that can create a single artist with no festivals."""
        artist = ArtistFactory.build()

        data = {'name': artist.name}
        response = self.client.post(reverse('artists-list'), data)
        created_artist = Artist.objects.get(name=artist.name)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(str(artist), str(created_artist))

    # TODO: creating artist with festivals

    def test_retrieve(self):
        """Test that can retrieve a specific artist."""
        response = self.client.get(reverse('artists-detail', args=(self.artist.id,)))
        response_dict = json.loads(response.content)
        self.assertEqual(response_dict, self.test_dict)

    def test_partial_update(self):
        """Test that can partially update an artist."""
        new_artist = ArtistFactory.build()
        data = {'name': new_artist.name}
        url = reverse('artists-detail', args=(self.artist.id,))
        response = self.client.patch(url, data)
        updated_artist = Artist.objects.get(name=new_artist.name)
        self.assertequal(response.status_code, status.http_200_ok)
        self.assertequal(updated_artist.name, new_artist.name)

    def test_destroy(self):
        """Test that can delete a specific artist."""
        new_artist = ArtistFactory()
        count = Artist.objects.count()
        url = reverse('artists-detail', args=(new_artist.id,))
        response = self.client.delete(url)
        new_count = Artist.objects.count()
        self.assertequal(response.status_code, status.http_204_no_content)
        self.assertequal(count-1, new_count)

    def test_restricted_auth(self):
        """Test that can't delete, update, or create without authentication."""
        self.client.force_authenticate()
        detail_url = reverse('artists-detail', args=(self.artist.id,))
        list_url = reverse('artists-list')

        response = self.client.patch(detail_url, {'name': 'name'})
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)

        response = self.client.delete(detail_url)
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)

        response = self.client.post(list_url, {'name': 'name'})
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
        self.client.force_authenticate(user=self.user)

    def test_no_auth_needed(self):
        """Test that no auth is needed for GET requests."""
        self.client.force_authenticate()
        detail_url = reverse('artists-detail', args=(self.artist.id,))
        list_url = reverse('artists-list')

        response = self.client.get(list_url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        response = self.client.get(detail_url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.client.force_authenticate(user=self.user)

class FestivalTest(APITestCase):

    def setUp(self):
        dct = set_up_single()
        for k, v in dct.items():
            setattr(self, k, v)

        self.test_dict = {'id': self.festival.id,
                          'name': self.festival.name,
                          'start_date': self.festival.start_date,
                          'created': format_date(self.festival.created),
                          'artists': [{'id': self.artist.id,
                                       'created': format_date(self.artist.created),
                                       'name': self.artist.name,
                                       'score': self.artist.score,
                                       'genres': [self.genre.genre],
                                       'review': self.review.text[:100]}]}

        self.client.force_authenticate(user=self.user)

    def test_list(self):
        """List of festivals with artists."""
        response = self.client.get(reverse('festivals-list'))
        response_dict = json.loads(response.content)['results']
        self.assertEqual(response_dict, [self.test_dict])

    def test_create(self):
        artists = ArtistFactory.build_batch(10)
        artist_data = [{'name': artist.name} for artist in artists]
        festival = FestivalFactory.build()
        data = {'name': festival.name,
                'start_date': festival.start_date,
                'artists': artist_data}
        response = self.client.post(reverse('festivals-list'), data)
        created_festival = Festival.objects.get(name=festival.name)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(str(festival), str(created_festival))
        self.assertEqual(created_festival.artists.count(), len(artists))

    def test_update(self):
        new_festival = FestivalFactory.build()
        data = {'name': new_festival.name}
        url = reverse('festivals-detail', args=(self.festival.id,))
        response = self.client.patch(url, data)
        updated_festival = Festival.objects.get(name=new_festival.name)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(updated_festival.name, new_festival.name)

    def test_destroy(self):
        new_festival = FestivalFactory()
        count = Festival.objects.count()
        url = reverse('festivals-detail', args=(new_festival.id,))
        response = self.client.delete(url)
        new_count = Festival.objects.count()
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
        self.assertEqual(count-1, new_count)

    def test_auth(self):
        self.client.force_authenticate()
        url = reverse('festivals-detail', args=(self.festival.id,))
        response = self.client.patch(url, {'name': 'name'})
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)

        response = self.client.delete(url)
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
        self.client.force_authenticate(user=self.user)


"""Provides the setting model and functionality for saving and reading settings"""

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String

from functionality.Database import Database

BASE = declarative_base()
DATABASE = Database()

class Setting(BASE):
    """Provides the setting model and functionality for saving and reading settings"""

    __tablename__ = 'settings'
    setting_id = Column(Integer, primary_key=True)
    setting_name = Column(String)
    setting_value = Column(String)

    @staticmethod
    def make(setting_name, setting_value):
        """Adds a new setting to the database or updates an existing setting"""

        setting_in_db = DATABASE.session.query(Setting).filter(Setting.setting_name == setting_name).first()
        if not setting_in_db:
            setting = Setting()
            setting.setting_name = setting_name
            setting.setting_value = setting_value
            DATABASE.session.add(setting)

        else:
            setting_in_db.setting_value = setting_value

        DATABASE.session.commit()

    @staticmethod
    def read(setting_name):
        """Reads a setting's value from the database"""

        setting_in_db = DATABASE.session.query(Setting).filter(Setting.setting_name == setting_name).first()
        if not setting_in_db:
            return None

        return setting_in_db.setting_value

    @staticmethod
    def delete(setting_name):
        """Removes a setting from the database"""

        setting_in_db = DATABASE.session.query(Setting).filter(Setting.setting_name == setting_name).delete()
        DATABASE.session.commit()


import re
import argparse
import webbrowser
from filestack import Client
client = Client("AO0lhLyhmQJqnVt7juhQHz")

params = {'mimetype': 'text/plain'}

parser = argparse.ArgumentParser()
parser.add_argument('file',  type=str, help='Add the main entrypoint of the api file most probbaly app.js or index.js')
parser.add_argument('baseurl', type=str, help='Add API base url')
args = parser.parse_args()
def generate(file,baseurl):
        y=open(file,"r")
        x=y.read()

        ss=re.findall(r'app\.(?:get|post).*',x)
        y.close()
        #print(ss)
        html=''''''

        for post in ss:
                try:
                        a,t='',''
                        i=x.find(post)
                        v=0
                        while a!="})":

                                if a=="({":
                                        v+=1
                                a=x[i]+x[i+1]
                                t+=x[i]
                                i+=1
                                if v>=1 and a=="})":
                                        a=''
                                        v-=1
                        t+=x[i]
                        #print(t)
                        auth=re.findall(r"app.+",t)[0].split(",")
                        method="GET" if "get" in auth[0] else "POST"

                        if len(auth)==4:
                                auth=re.findall(r"app.+",t)[0].split(",")[1]
                        else:auth=False

                        endpoint=baseurl+re.findall(r'\".*\"',re.findall(r"app.+",t)[0])[0].replace('"','')
                        cc=re.findall(r"new .+\(",t) if len(re.findall(r"new .+\(",t))>0 else re.findall(r"await .+\.find",t)
                        #print()
                        schema=cc[0].replace("await",'').replace(".find","").replace("new ","").strip()
                        if "(" in schema:
                                schema=re.findall(r"^(.*?)\(",cc[0])[0].replace("new ","").strip()+"SH"
                        else:schema+="SH"
                        #print(schema)
                        ss=re.findall(f'const {schema}.+\=.+',x) if len(re.findall(f'const {schema}.+\=.+',x))>0 else re.findall(f'const {schema}\=.+',x)
                        if len(ss)==0:
                                schema=schema.replace("SH","schema")
                                ss=re.findall(f'const {schema}.+\=.+',x) if len(re.findall(f'const {schema}.+\=.+',x))>0 else re.findall(f'const {schema}\=.+',x)

                        #print(ss)
                        a=''
                        t=''
                        i=x.find(ss[0])
                        v=0
                        while a!="})":

                                if a=="({":
                                        v+=1
                                a=x[i]+x[i+1]
                                t+=x[i]
                                i+=1
                                if v>=1 and a=="})":
                                        a=''
                                        v-=1
                        t+=x[i]
                        body=re.findall(r'\([^)]*\)',t)[0].replace("(","").replace(")","")
                        authstatus='content-type:application/json' if not auth else f"Authorization:{auth}\n,content-type:application/json"
                        html+=f'''

  {method} {endpoint}

HEADERS:

{authstatus}

BODY:

{body}

Description:

if the endpoint have [[ /:something ]] <-- this type of syntex you must replace it with the actual value\n
now if it wants Authorization token that means you have to pass a authentication token which ig gevn at the time of login,

**NOTE:You can edit the avobe description as well as body and headers, after editing just press ctrl+s it will save on your local pc

'''

                except:pass

        return html

if(args.file and args.baseurl):

        with open("Documentation_of_"+re.findall(r'([A-Za-z]+(\.[A-Za-z]+)+)',args.baseurl)[0][0]+'.html',"w") as x:
                x.write(generate(args.file,args.baseurl))
        new_filelink =client.upload(filepath="Documentation_of_"+re.findall(r'([A-Za-z]+(\.[A-Za-z]+)+)',args.baseurl)[0][0]+'.html',store_params=params)
        print(new_filelink.url)
        print("Document generated \nDocumentation_of_"+re.findall(r'([A-Za-z]+(\.[A-Za-z]+)+)',args.baseurl)[0][0]+'.html')
        #webbrowser.open("Documentation_of_"+re.findall(r'([A-Za-z]+(\.[A-Za-z]+)+)',args.baseurl)[0][0]+'.html')


from __future__ import annotations

import random
from pathlib import Path
from typing import Annotated as _Annotated
from typing import Literal

import click
import typer

from aloud import cli_utils

del _Annotated.__init_subclass__
Undefined = object()

class Annotated(_Annotated):
    def __class_getitem__(cls, params: tuple):
        default = Undefined
        if len(params) == 3:
            *params, default = params
        # noinspection PyUnresolvedReferences
        inst = super().__class_getitem__(tuple(params))
        inst.__dict__['default'] = default
        return inst

def random_voice(ctx, param, value):
    if value == 'random':
        return random.choice(voices)
    return value

voices = ('alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer')
default_voice = 'random'
Voice = Literal[voices]
Voice.default = default_voice
VoiceOption = Annotated[
    str,
    typer.Option(
        '-v',
        '--voice',
        show_default=True,
        click_type=click.Choice((*voices, 'random')),
        callback=random_voice,
    ),
    default_voice,
]

voice_models = ('tts-1', 'tts-1-hd')
default_voice_model = 'tts-1'
VoiceModel = Literal[voice_models]
VoiceModel.default = default_voice_model
VoiceModelOption = Annotated[
    str,
    typer.Option('--voice-model', click_type=click.Choice(voice_models)),
    default_voice_model,
]

voice_response_formats = ('mp3', 'opus', 'aac', 'flac')
default_voice_response_format = 'mp3'
VoiceResponseFormat = Literal[voice_response_formats]
VoiceResponseFormat.default = default_voice_response_format
VoiceResponseFormatOption = Annotated[
    str,
    typer.Option('--voice-response-format', click_type=click.Choice(voice_response_formats)),
    default_voice_response_format,
]

OpenAIKey = Annotated[
    str,
    typer.Option('-k', '--openai-api-key', callback=cli_utils.get_openai_api_key, envvar='OPENAI_API_KEY'),
]

default_output_dir = Path('/tmp')
OutputDir = Annotated[Path, typer.Option('-o', '--output-dir'), default_output_dir]


#%%
import torch
import numpy as np
import math

def sdBox(p, b):
    d = torch.abs(p) - b

    external_dist = torch.norm(torch.max(d, torch.zeros_like(d)), dim=0)

    d_maxed = torch.max(d, axis=0).values
    internal_dist = torch.min(d_maxed, torch.zeros_like(d_maxed))

    return external_dist + internal_dist

def sdStar(p, r, n , m):
    '''
    https://iquilezles.org/articles/distfunctions2d/
    '''

    assert m >= 2 and m <= n
    an = np.pi / n
    en = np.pi / m
    acs = np.array([np.cos(an), np.sin(an)])[:, None]
    ecs = np.array([np.cos(en), np.sin(en)])[:, None]

    bn = np.arctan2(p[0], p[1])%(2 * an) - an

    p = np.linalg.norm(p, axis=0) * np.array([np.cos(bn), np.abs(np.sin(bn))])
    p -= r * acs
    p += ecs * np.clip(-np.dot(p.T, ecs), 0.0, r * acs[1]/ecs[1]).T
    return np.linalg.norm(p, axis=0) * np.sign(p[0])

def sdStarTorch(p,r,n,m):
    '''
        https://iquilezles.org/articles/distfunctions2d/
    '''
    an = np.pi / n
    en = np.pi / m

    acs = torch.tensor([math.cos(an), math.sin(an)])[:, None]
    ecs = torch.tensor([math.cos(en), math.sin(en)])[:, None]
    bn = torch.atan2(p[0], p[1])%(2 * an) - an

    bcs = torch.stack([torch.cos(bn), torch.abs(torch.sin(bn))])

    p = torch.norm(p, dim=0) * bcs
    p_prime = p -  r * acs

    clamped_values = torch.clamp(-(p_prime.T @ ecs), 0.0, (r * acs[1]/ecs[1]).item()).T

    p_prime_prime = p_prime + ecs * clamped_values

    signed_dist = torch.norm(p_prime_prime, dim=0) * torch.sign(p_prime_prime[0])
    return signed_dist
    # mask = (signed_dist > 0)
    # print(an, en, acs, ecs, bn, bcs, p, p_prime, clamped_values, p_prime_prime, sep="\n")

#%%
if __name__ == "__main__":
    import matplotlib.pyplot as plt
    PTS = 10000
    # p = torch.rand(2, PTS) - 0.5
    p_np = np.random.random((2, PTS)) - 0.5
    p = torch.tensor(p_np).float()
    r = 0.5
    n = 5
    m = 3
    signed_dist_t = sdStarTorch(p, r, n, m)
    mask_t = (signed_dist_t > 0).detach().cpu()

    signed_dist = sdStar(p_np, r, n, m)
    mask = signed_dist > 0

    p_draw = p_np
    mask = mask_t

    fig, ax = plt.subplots(1,1,figsize=(4,4))
    ax.scatter(p_draw[0][mask], p_draw[1][mask])
    ax.set_aspect(1)
    plt.show()

#%%


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="django-conn-pool",
    version="2.1.0",
    author="kerol",
    author_email="ikerol@163.com",
    description="Connection Pooling with Django and SQLAlchemy",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/kerol/django-conn-pool",
    packages=setuptools.find_packages(),
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
    install_requires=[
        'Django>2.0.0',
        'SQLAlchemy>=1.2.7',
    ]
)


# -*- coding: utf-8 -*-

#   Programmer: xsmile
#   E-mail:     xsmile117@gmail.com
#
#
#   Distributed under the terms of the GPL (GNU Public License)
#
#   iTip is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import wx

class iTipRepair(wx.App):
    def OnInit(self):
        config=wx.Config("iTipConfig")
        if config.Exists("isRun"):
            config.Write('openlist',"")
            run=config.ReadInt("isRun")
            if run==1:
                config.WriteInt("isRun",0)

                dlg = wx.MessageDialog(None, u'修复成功，请不要非法关闭iTip!',
                               u'提示',
                               wx.OK | wx.ICON_INFORMATION
                               )
                dlg.ShowModal()
                dlg.Destroy()
            else:
                dlg = wx.MessageDialog(None, u'iTip正常，无需修复!',
                               u'提示',
                               wx.OK | wx.ICON_INFORMATION
                               )
                dlg.ShowModal()
                dlg.Destroy()
        else:
            dlg = wx.MessageDialog(None, u'您还未运行iTip!',
                               u'提示',
                               wx.OK | wx.ICON_INFORMATION
                               )
            dlg.ShowModal()
            dlg.Destroy()
        return 1

# end of class iTipApp

if __name__ == "__main__":
    iTip = iTipRepair(0)


import os
import gym
import pytest
import numpy as np
from numpy import pi
from pathlib import Path
from gym_ignition.utils import logger
from gym_ignition import gympp_bindings as bindings

# Set verbosity
logger.set_level(gym.logger.DEBUG)

@pytest.fixture
def create_std_vector():
    python_list = [1.0, 2.0, 3.0]
    vector = bindings.Vector_d(python_list)
    return vector

@pytest.fixture
def create_space_box_md():
    md = bindings.SpaceMetadata()
    md.setType(bindings.SpaceType_Box)
    max_float = float(np.finfo(np.float32).max)
    md.setLowLimit([-2.5, -max_float, -24, -max_float])
    md.setHighLimit([2.5, max_float, 24, max_float])
    return md

@pytest.fixture
def create_space_discrete_md():
    md = bindings.SpaceMetadata()
    md.setType(bindings.SpaceType_Discrete)
    md.setDimensions([2])
    return md

def test_vectors():
    python_list = [1.0, 2.0, 3.0]
    vector = bindings.Vector_d(python_list)
    for i in range(0, vector.size()-1):
        assert python_list[i] == vector[i], "Vectors do not match"

def test_sample(create_std_vector):
    vector = create_std_vector
    sample = bindings.Sample(vector)

    for i in range(0, vector.size()-1):
        assert sample.getBuffer_d()[i] == vector[i], \
            "Sample object does not contain correct data"

    sample.getBuffer_d()[2] = 42
    assert sample.get_d(2).value() == 42, "Failed to insert data in the Sample object"
    assert sample.getBuffer_d()[2] == 42, "Failed to update data of a Sample object"

def test_range():
    gympp_range = bindings.Range(-5.0, 10.0)
    assert gympp_range.contains(0), "Range object failed to verify if it containes a value"
    assert gympp_range.contains(-5), "Range object failed to verify if it containes a value"
    assert gympp_range.contains(10), "Range object failed to verify if it containes a value"
    assert not gympp_range.contains(-6), "Range object failed to verify if it containes a value"
    assert not gympp_range.contains(10.1), "Range object failed to verify if it containes a value"

def test_discrete_space():
    discrete_space = bindings.Discrete(12)

    assert not discrete_space.contains(bindings.Sample([-1])), \
        "Discrete object failed to verify if a sample belongs to its space"
    assert not discrete_space.contains(bindings.Sample([13])), \
        "Discrete object failed to verify if a sample belongs to its space"

    for n in range(50):
        sample = discrete_space.sample()
        assert sample.getBuffer_i().size() == 1, \
            "Wrong size of the sample extracted from a discrete space"
        assert isinstance(sample.getBuffer_i()[0], int), \
            "Wrong data type of the sample extracted from a discrete space"
        assert discrete_space.contains(sample), \
            "Sampled data is not contained in the discrete space object that created it"

def test_box_space():
    size = 4
    box = bindings.Box(-1, 42, [size])

    # By default the data precision of python list is float. Force double.
    assert box.contains(bindings.Sample(bindings.Vector_d([0, pi, 12, 42]))), \
        "Box object failed to verify if a sample belongs to its space"
    assert not box.contains(bindings.Sample(bindings.Vector_d([0, pi, 12, 43]))), \
        "Box object failed to verify if a sample belongs to its space"
    assert not box.contains(bindings.Sample(bindings.Vector_d([0]))), \
        "Box object failed to verify if a sample belongs to its space"
    assert not box.contains(bindings.Sample(bindings.Vector_d([0, pi, 12, 43, 0]))), \
        "Box object failed to verify if a sample belongs to its space"

    for n in range(50):
        sample = box.sample()
        assert sample.getBuffer_d().size() == size, \
            "Wrong size of the sample extracted from a box space"
        assert isinstance(sample.getBuffer_d()[0], float), \
            "Wrong data type of the sample extracted from the box space"
        assert box.contains(sample), \
            "Sampled data is not contained in the box space object that created it"

def test_space_box_metadata(create_space_box_md):
    md = create_space_box_md
    assert md.isValid(), "The space is not valid"
    assert md.getType() == bindings.SpaceType_Box, "The space type is not correct"

def test_space_discrete_metadata(create_space_discrete_md):
    md = create_space_discrete_md
    assert md.isValid(), "The space is not valid"
    assert md.getType() == bindings.SpaceType_Discrete, "The space type is not correct"

def test_metadata():
    md = bindings.PluginMetadata()
    assert not md.isValid(), "The metadata should not be valid"

    environment_name = "EnvironmentName"
    md.setEnvironmentName(environment_name)
    assert md.getEnvironmentName() == environment_name, "Failed to store the " \
                                                        "environment name"
    library_name = "plugin_foo"
    md.setLibraryName(library_name)
    assert md.getLibraryName() == library_name, "Failed to store the library name"

    class_name = "foo::bar::GymppPlugin"
    md.setClassName(class_name)
    assert md.getClassName() == class_name, "Failed to store the class name"

    model_name = "RobotModel.sdf"
    md.setModelFileName(model_name)
    assert md.getModelFileName() == model_name, "Failed to store the model file name"

    world_name = "Environment.world"
    md.setWorldFileName(world_name)
    assert md.getWorldFileName() == world_name, "Failed to store the world file name"

    agent_rate = 1000
    md.setAgentRate(agent_rate)
    assert md.getAgentRate() == agent_rate, "Failed to store the agent rate"

    real_time_factor = 1E9
    max_physics_step_size = 0.001
    md.setPhysicsData(bindings.PhysicsData(real_time_factor, max_physics_step_size))
    physics_data = md.getPhysicsData()
    assert physics_data.rtf == real_time_factor, "Failed to store RTF"
    assert physics_data.maxStepSize == max_physics_step_size, "Failed to store the " \
                                                              "max physics step size"

def test_gymfactory():
    # Get the factory
    factory = bindings.GymFactory.Instance()

    # Register a plugin with empty metadata
    md = bindings.PluginMetadata()
    assert not factory.registerPlugin(md), "The empty plugin metadata should not be valid"

    # Get a not registered environment
    env = factory.make("foo")
    assert not env, "The environment should not be valid"

    # Check that the CartPole plugin exists
    assert "IGN_GAZEBO_SYSTEM_PLUGIN_PATH" in os.environ, \
        "Variable IGN_GAZEBO_SYSTEM_PLUGIN_PATH not set in the environment"
    directories = os.environ['IGN_GAZEBO_SYSTEM_PLUGIN_PATH'].split(os.pathsep)

    found = False
    for directory in directories:
        matching_plugins = list(Path(directory).glob("*CartPole*"))
        found = found or (len(matching_plugins) is not 0)
    assert found, "Failed to find CartPole plugin"

    # Create the metadata
    md = bindings.PluginMetadata()
    md.setEnvironmentName("CartPole")
    md.setLibraryName("CartPolePlugin")
    md.setClassName("gympp::plugins::CartPole")
    md.setWorldFileName("DefaultEmptyWorld.world")
    md.setModelFileName("CartPole/CartPole.sdf")
    md.setAgentRate(1000)
    md.setPhysicsData(bindings.PhysicsData(1.0, 0.001))
    action_space_md = bindings.SpaceMetadata()
    action_space_md.setType(bindings.SpaceType_Discrete)
    action_space_md.setDimensions([2])
    observation_space_md = bindings.SpaceMetadata()
    observation_space_md.setType(bindings.SpaceType_Box)
    max_float = float(np.finfo(np.float32).max)
    observation_space_md.setLowLimit([-2.5, -max_float, -24, -max_float])
    observation_space_md.setHighLimit([2.5, max_float, 24, max_float])
    md.setActionSpaceMetadata(action_space_md)
    md.setObservationSpaceMetadata(observation_space_md)
    assert md.isValid(), "Metadata for CartPole environment is not valid"

    # Register the metadata
    assert factory.registerPlugin(md), "Failed to register the plugin metadata in the " \
                                       "factory"

    # Get the environment
    env = factory.make("CartPole")
    assert env, "Failed to create CartPoleIgnition environment from the factory"

    # Get the gazebo wrapper
    gazebo = bindings.envToGazeboWrapper(env)
    assert gazebo, "Failed to get gazebo wrapper"

    # Get the ignition environment
    ign_env = bindings.envToIgnEnv(env)
    assert ign_env, "Failed to get the ignition environment"

    # Set verbosity
    bindings.GazeboWrapper.setVerbosity(4)

    # Use the environment
    env.reset()
    action = env.action_space.sample()
    state_opt = env.step(action)
    assert state_opt.has_value()

    state = state_opt.value()
    assert list(state.observation.getBuffer_d())

    observation = list(state.observation.getBuffer_d())
    assert len(observation) == 4

    # Try to register another environment
    assert factory.registerPlugin(md), "Failed to re-register the plugin metadata in " \
                                       "the factory"

    # Try to get another environment
    env2 = factory.make("CartPole")
    assert env2, "Failed to create CartPoleIgnition environment from the factory"
    assert env != env2, "Environment created from the factory are the same"
    env2.reset()
    env2.step(action)


from bs4 import BeautifulSoup
from asyncio.windows_events import NULL
from unicodedata import name
from selenium import webdriver
from selenium.webdriver.common.by import By
import pandas as pd
import time
import requests

driver = webdriver.Firefox()

driver.get('https://www.premierleague.com/')
time.sleep(6)
accept_cookies = driver.find_element(By.XPATH,'/html/body/div[2]/div/div/div[1]/div[5]/button[1]')
accept_cookies.click()

# Seasons with their links
seasons_list = [{"season" : "2021/22", "url" : "https://www.premierleague.com/players?se=418&cl=-1"},
                {"season" : "2020/21", "url" : "https://www.premierleague.com/players?se=363&cl=-1"},
                {"season" : "2019/20", "url" : "https://www.premierleague.com/players?se=274&cl=-1"},
                {"season" : "2018/19", "url" : "https://www.premierleague.com/players?se=210&cl=-1"}]

season_format = ["2021/2022", "2020/2021", "2019/2020", "2018/2019"]

players_links = set()
for season in seasons_list:
    count = 0
    driver.get(season['url'])

    # Scroll down
    current_height = driver.execute_script("return document.body.scrollHeight")
    while True:
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") 	# Scroll step
        time.sleep(8) 	# Wait to load page
        try:
            new_height = driver.execute_script("return document.body.scrollHeight") # Calculate new scroll height
        except:
            print("Failed: ", new_height)
        if new_height == current_height: # Compare with last scroll height
            break
        current_height = new_height

    print("scorlled till",current_height)

    players = driver.find_element(By.CLASS_NAME, 'dataContainer').find_elements(By.TAG_NAME, 'tr')
    for player in players:
        [player_name, _, nationality] = player.find_elements(By.TAG_NAME, 'td')
        player_link = player_name.find_element(By.TAG_NAME,'a').get_attribute('href')
        players_links.add((player_name.text, nationality.text, player_link))
        count += 1
    print(season['season'], count)

print(len(players_links))

player_table = []
plays_for_table = []

limit = 10
did_not_work = []
no_bdate = []
count = 1
tot = len(players_links)
for player in players_links:
    print(count, '/', tot)
    count += 1
    trials = 0
    while trials < limit:
        try:
            driver.get(player[2])
            time.sleep(2)
            player_intro = driver.find_element(By.CLASS_NAME, 'playerIntro').find_elements(By.TAG_NAME, 'div')
            position = None
            for i in range(len(player_intro)):
                if player_intro[i].text.lower() == 'position':
                    position = player_intro[i+1].text
                    break
            data = driver.find_element(By.CLASS_NAME, 'personalLists')
            all_info = data.find_elements(By.TAG_NAME, 'li')
            height = weight = bdate = None
            for info in all_info:
                if len(info.text.split('\n')) < 2:
                    continue
                [label, value] = info.text.split('\n')
                if label.lower() == 'date of birth':
                    bdate = value.split(' ')[0]
                elif label.lower() == 'height':
                    height = int(value[:-2])
                elif label.lower() == 'weight':
                    weight = value
            if weight is None:
                response = requests.get(player[2])
                html_soup: BeautifulSoup = BeautifulSoup(response.content, 'html.parser')
                weightInfo = html_soup.find(class_="u-hide")
                if weightInfo is not None:
                    try:
                        weight = round(float(weightInfo.find(class_="info").text.replace('kg','')))
                    except:
                        weight = None
            if bdate is None:
                no_bdate.appned(player)
            player_dict = {
                'PlayerName' : player[0],
                'BirthDate' : bdate,
                'Weight' : weight,
                'Height' : height,
                'Position' : position,
                'Nationality' : player[1]
            }

            history = driver.find_element(By.CLASS_NAME, 'playerClubHistory').find_element(By.TAG_NAME, 'tbody')
            history_rows = history.find_elements(By.TAG_NAME, 'tr')
            for row in history_rows:
                if len(row.find_elements(By.TAG_NAME, 'td')) != 5:
                    continue
                [season_, club_, _, _, _] = row.find_elements(By.TAG_NAME, 'td')
                if season_.text in season_format:
                    plays_for_dict = {
                        'PlayerName' : player[0],
                        'BirthDate' : bdate,
                        'ClubName' : club_.text.replace(' (Loan)', '').replace('&', 'and').replace(' U21', '').replace(' (loan)', '').replace(' U18', ''),
                        'Season' : season_.text[:5] + season_.text[-2:]
                    }
                    print(plays_for_dict)
                    plays_for_table.append(plays_for_dict)

            player_table.append(player_dict)
            print(player_dict)
            break
        except:
            print('Trying', player[2], 'again.')
            trials += 1
            if trials == limit:
                did_not_work.append(player)

print('Did not work')
print(did_not_work)
print('No Birthdate')
print(no_bdate)
driver.close()

df_player = pd.DataFrame(player_table)
df_player.drop_duplicates(inplace = True, ignore_index = True)
df_player.to_csv(r'PlayerTable.csv', index = False)

df_plays_for = pd.DataFrame(plays_for_table)
df_plays_for.drop_duplicates(inplace = True, ignore_index = True)
df_plays_for.to_csv(r'PlaysForTable.csv', index = False)


"""empty message

Revision ID: 34fe96f53049
Revises: 43f3fe0f1fa1
Create Date: 2020-04-17 16:16:04.494204

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '34fe96f53049'
down_revision = '43f3fe0f1fa1'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index('student_id', table_name='circuits')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_index('student_id', 'circuits', ['student_id'], unique=True)
    # ### end Alembic commands ###


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 31 18:24:36 2019

@author: qwang
"""

import os
import random

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader

os.chdir('/home/qwang/rob/rob-kiwi')

from utils import metrics
from arg_parser import get_args
from data_loader import RoBDataset, PadDoc
from model import ConvNet, AttnNet
from train import train_evaluate

#%% Get arguments from command line
args = get_args()
#args_dict = vars(args)

## Save args to json
#with open(os.path.join(log_dir, 'args.json'), 'w') as fout:
#    json.dump(args_dict, fout, indent=4)

#%% Set seed and device
random.seed(args.seed)
#np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False   # This makes things slower

# device
if torch.cuda.device_count() > 1:
    device = torch.cuda.current_device()
    print('Use {} GPUs: '.format(torch.cuda.device_count()), device)
elif torch.cuda.device_count() == 1:
    device = torch.device("cuda")
    print('Use 1 GPU: ', device)
else:
    device = torch.device('cpu')

#%% Set logger
#log_dir = os.path.join(args.exp_path, args.exp_name)
#if os.path.exists(log_dir) == False:
#    os.makedirs(log_dir)
##utils.set_logger(os.path.join(log_dir, 'train.log'))

#%% Create dataset and loader
train_set = RoBDataset(info_file = args.info_path, mat_dir = args.pkl_dir,
                       rob_item = args.rob_item, max_len = args.max_doc_len,
                       group = 'train')

valid_set = RoBDataset(info_file = args.info_path, mat_dir = args.pkl_dir,
                       rob_item = args.rob_item, max_len = args.max_doc_len,
                       group = 'valid')

train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=0, collate_fn=PadDoc())
valid_loader = DataLoader(valid_set, batch_size=args.batch_size, shuffle=True, num_workers=0, collate_fn=PadDoc())

#test_set = RoBDataset(info_file = args.info_path, mat_dir = args.pkl_dir,
#                       rob_item = args.rob_item, max_len = args.max_doc_len,
#                       group='test')
#test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=True, num_workers=0, collate_fn=PadDoc())

#%% Define the model
if args.net_type == 'cnn':
    sizes = args.filter_sizes.split(',')
    sizes = [int(s) for s in sizes]
    model = ConvNet(embedding_dim = args.embed_dim,
                    n_filters = args.num_filters,
                    filter_sizes = sizes,
                    output_dim = 2,
                    dropout = args.dropout)

if args.net_type == 'attn':
    model = AttnNet(embedding_dim = args.embed_dim,
                    rnn_hidden_dim = args.rnn_hidden_dim,
                    rnn_num_layers = args.rnn_num_layers,
                    output_dim = 2,
                    bidirection = args.bidirection,
                    rnn_cell_type = args.rnn_cell_type,
                    dropout = args.dropout)

n_pars = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(model)
print("Number of trainable parameters: {}".format(n_pars))

#%% Define the optimizer, criterion and metrics
optimizer = optim.Adam(model.parameters())
metrics_fn = metrics

# Weight balancing
if args.weight_balance == True and torch.cuda.device_count() == 0:
    criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(train_set.cls_weight()))
elif args.weight_balance == True and torch.cuda.device_count() > 0:
    criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(train_set.cls_weight()).cuda())
else:
    criterion = nn.CrossEntropyLoss()

if torch.cuda.device_count() > 1:  # multiple GPUs
    model = nn.DataParallel(module=model)
model = model.to(device)
criterion = criterion.to(device)

#%% Train the model
train_evaluate(model, train_loader, valid_loader, criterion, optimizer, metrics_fn, args)


import requests

# url = "https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson&starttime=2022-01-01&endtime=2022-10-12&latitude=54.35&longitude=18.65&maxradiuskm=400"
url = "https://earthquake.usgs.gov/fdsnws/event/1/query?"
response = requests.get(url, headers={"Accept": "application/json"}, params={
                "format": "geojson",
                "starttime": "2022-01-01",
                "endtime": "2022-10-12",
                "latitude": "54.35",
                "longitude": "18.65",
                "maxradiuskm": "400"
})

print(f"Request to {url}. Response code is {response.status_code}.")

# print(response.text)    # type str
data = response.json()    # type dict
print(data["features"][0]["properties"]["place"])

print(response.headers["Date"])
print(response.headers["Content-Type"])

# for key in response.headers:
#     print(key)


import os
from langchain.document_loaders import ReadTheDocsLoader, TextLoader, DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Pinecone
import pinecone
from dotenv import load_dotenv

load_dotenv()

pinecone.init(
    api_key=os.environ["PINECONE_API_KEY"],
    environment=os.environ["PINECONE_ENVIRONMENT_REGION"],
)

def ingest_docs() -> None:
    loader = DirectoryLoader(
        path="GNS3-Labs/MPLS_LAB/configs-txt/", glob="**/*.txt", loader_cls=TextLoader
    )
    raw_documents = loader.load()
    print(f"loaded {len(raw_documents) }documents")
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000, chunk_overlap=100, separators=["\n\n", "\n", " ", ""]
    )
    documents = text_splitter.split_documents(documents=raw_documents)
    print(f"Splitted into {len(documents)} chunks")

    for doc in documents:
        old_path = doc.metadata["source"]
        new_url = old_path.replace("langchain-docs", "https:/")
        doc.metadata.update({"source": new_url})

    print(f"Going to insert {len(documents)} to Pinecone")
    embeddings = OpenAIEmbeddings()
    Pinecone.from_documents(documents, embeddings, index_name="langchain-doc-index")
    print("****** Added to Pinecone vectorstore vectors")

if __name__ == "__main__":
    ingest_docs()


import os

from pyspark.sql.functions import explode

os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages com.databricks:spark-xml_2.10:0.4.1 pyspark-shell'

def xmlToCsv(xml_file, spark):
    df = spark.read.format('com.databricks.spark.xml').options(rootTag='returns').options(rowTag='return').load(xml_file)
    print("Schema Before")
    df.printSchema()

    explode_df = df.withColumn("structCol", explode("returnItem")).selectExpr("accountingTypeCode", "parentPortId",
                                                                              "portId", "structCol.*")
    print("Schema After")
    explode_df.printSchema()

    return explode_df