import paho.mqtt.client as mqtt
import time
import json
import sys
import os

# set options
roomName = "Gallery/"
languages = ["EN", "NL"]
nrPlayers = [2, 3, 4, 5, 6, 7]
programs = [{"name": "60 minutes", "file": "new_program.json"}, {"name": "60 minutes old", "file": "program.json"}]
debug = True
# end of options

# standard settings
selectedProgram = programs[0]['file']
players = 4
lang = 'NL'

currentProgram = {}
status = "Aborted"
rem = 0
totalTime = 0
startTime = 0
path = os.path.realpath('.')
devicesList = os.path.join(path, 'devices.json')
devices = []

try:
    with open(devicesList) as devicesFile:
        devices = json.load(devicesFile)
except FileNotFoundError:
    print("ERROR: Unable to load devices file")
except json.decoder.JSONDecodeError:
    print("ERROR: Fault in devices JSON file")

client = None

def signal_handler():
    print("\nprogram exiting nice")
    sys.exit(0)

def run_program(prog):
    global currentProgram
    global client
    try:
        for item in currentProgram[prog]["set"]:
            if item['topic'] == "status" or item['topic'] == "time":
                # always true?
                client.publish(roomName + item['topic'], item['action'], 1, True)
            else:
                client.publish(roomName + item['topic'], item['action'], 1, False)
        currentProgram[prog]['played'] = True
        client.publish(roomName + "message", prog, 1, False)
    except KeyError:
        print("unable to set settings for program: " + prog)
        client.publish(roomName + "message", "ERROR: Unable to run program: " + prog, 1, False)
    try:
        # Queue programs which are triggered by this program and cancel programs which are cancelled by this program
        for key, value in currentProgram.items():
            if currentProgram[key].get('start') == prog:
                if not currentProgram[key].get('played'):
                    print(key + " triggered at " + str(rem - int(currentProgram[key]['after'])))
                    currentProgram[key]['time'] = (rem - int(currentProgram[key]['after']))
            if currentProgram[key].get('cancel') == prog:
                currentProgram[key]['played'] = True

    except KeyError:
        print("error in program")
        client.publish(roomName + "message", "ERROR: Unable to queue or cancel after: " + prog, 1, False)
    client.publish(roomName + "currentProgram", json.dumps(currentProgram), 1, True)

def pause_program(prog):
    global currentProgram
    try:
        currentProgram[prog]["played"] = True
        client.publish(roomName + "currentProgram", json.dumps(currentProgram), 1, True)
    except KeyError:
        print("unable to pause program: " + prog)

def main():
    global client
    global status
    global rem
    global currentProgram

    last_rem = 0

    # connect to MQTT broker
    client = mqtt.Client()
    client.on_connect = on_connect
    client.on_message = on_message
    client.connect("localhost", 1883, 60)
    client.loop_start()

    while True:
        if status == "Started":
            rem = int(totalTime - (time.time() - startTime))
            if rem != last_rem and rem > -1:
                client.publish(roomName + "time", rem, 1, True)
                last_rem = rem
            if rem < -20:
                status = "Aborted"
                run_program("end")
            for key, value in currentProgram.items():
                if currentProgram[key].get('time') is not None and currentProgram[key].get('time') >= rem and \
                        not currentProgram[key].get('played'):
                    run_program(key)
        time.sleep(0.1)

def on_connect(client, userdata, flags, rc):

    if debug:
        print("Connected with result code "+str(rc))

    client.publish(roomName + "programs", json.dumps(programs), 1, True)
    client.publish(roomName + "selectedProgram", selectedProgram, 1, True)
    client.publish(roomName + "players", players, 1, True)
    client.publish(roomName + "languages", ",".join(languages), 1, True)
    client.publish(roomName + "lang", lang, 1, True)
    client.publish(roomName + "status", status, 1, True)
    client.publish(roomName + "devices", json.dumps(devices), 1, True)
    client.subscribe(roomName + "#")

def on_message(client, userdata, msg):
    global currentProgram
    global selectedProgram
    global programs
    global rem
    global status
    global totalTime
    global startTime

    pay = msg.payload.decode("utf-8")

    if debug and msg.topic != roomName + "time" and msg.topic != roomName + "currentProgram":
        print("Received: " + msg.topic + " " + pay)

    if msg.topic[:len(roomName) + 10] == roomName + "runProgram":
        run_program(pay)
    elif msg.topic[:len(roomName) + 12] == roomName + "pauseProgram":
        pause_program(pay)
    elif msg.topic == roomName + "status":
        status = pay
        if pay == "Prepared":
            rem = totalTime
            client.publish(roomName + "selectedProgram", selectedProgram, 1, True)
        elif pay == "Started":
            startTime = time.time()
            totalTime = rem
    elif msg.topic == roomName + "totalTime":
        try:
            totalTime = int(pay)
            rem = totalTime
        except ValueError:
            print("ERROR: unable to set totalTime")
    elif msg.topic == roomName + "selectedProgram":
        prev_program = selectedProgram
        selectedProgram = pay
        # print(traceback.format_exc())
        try:
            program_file_path = os.path.join(path, selectedProgram)
            with open(program_file_path) as program_file:
                currentProgram = json.load(program_file)
                client.publish(roomName + "currentProgram", json.dumps(currentProgram), 1, True)
        except FileNotFoundError:
            client.publish(roomName + "message", "ERROR: Program file not found", 1, False)
            selectedProgram = prev_program
            print("ERROR: Program file not found")
        except json.decoder.JSONDecodeError:
            client.publish(roomName + "message", "ERROR: Fault in program json file", 1, False)
            selectedProgram = prev_program
            print("ERROR: Fault in Program JSON file")
    elif pay == "Disconnected":
        # Handle disconnect of devices
        try:
            for area in devices['areas']:
                for device in area['devices']:
                    if device['device'] == msg.topic:
                        client.publish(device['topic'], "", 1, True)
        except KeyError:
            print("ERROR: Unable to clean disconnect")

if __name__ == '__main__':
    main()


import librosa as lib
import librosa.display
import matplotlib.pyplot as plt
import numpy as np

y, sr = lib.load("aefe.wav")
S = np.abs(lib.stft(y))

stream = lib.stream("aefe.wav", block_length=256,
frame_length=4096, hop_length=1024)

for y_block in stream:
    m_block = lib.feature.melspectrogram(y_block, sr=sr,
    n_fft=2048, hop_length=2048, center=False)

fig, ax = plt.subplots()
S_dB = lib.power_to_db(m_block, ref=np.max)

img = lib.display.specshow(S_dB, x_axis='time',
y_axis='mel', sr=sr, fmax=10000, ax=ax)

fig.colorbar(img,ax=ax,format="%+2.0f dB")
ax.set_title('Mel-Freq Spectrogram')
plt.show()


# Open Close Principle
"""
open for extension (inherit and do modification)
closed for modification
"""

from enum import Enum

class Color(Enum):
  RED = 1
  GREEN = 2
  BLUE = 3

class Size(Enum):
  SMALL = 1
  MEDIUM = 2
  LARGE = 3

class Weight(Enum):
  LITE = 1
  NORMAL = 2
  HEAVY = 3

class Product:
  def __init__(self, name, color, size, weight):
    self.name = name
    self.color = color
    self.size = size
    self.weight = weight

# This approach does not scale is added more criterias
# for example we add weight also
class ProductFilter:
  def filter_by_color(self, products, color):
    for product in products:
      if product.color == color:
        yield product

  def filter_by_size(self, products, size):
    for product in products:
      if product.size == size:
        yield product

  def filter_by_size_and_color(self, products, size, color):
    for product in products:
      if product.size == size and product.color == color:
        yield product

# Specification
class Specification:
  def is_satisfied(self, item):
    pass

  def __and__(self, other):
    return AndSpecification(self, other)

class Filter:
  def filter(self, items, spec):
    pass

class ColorSpeccification(Specification):
  def __init__(self, color):
    self.color = color

  def is_satisfied(self, item):
    return item.color == self.color

class SizeSpeccification(Specification):
  def __init__(self, size):
    self.size = size

  def is_satisfied(self, item):
      return item.size == self.size

class WeightSpecification(Specification):
  def __init__(self, weight):
    self.weight = weight

  def is_satisfied(self, item):
    return self.weight == item.weight

class AndSpecification(Specification):
  def __init__(self, *args):
    self.args = args

  def is_satisfied(self, item):
    return all(map(lambda spec: spec.is_satisfied(item), self.args))

class BetterFilter(Filter):
  def filter(self, items, spec):
    for item in items:
      if spec.is_satisfied(item):
        yield item

if __name__ == '__main__':
  apple = Product('Apple', Color.GREEN, Size.SMALL, Weight.LITE)
  tree = Product('Tree', Color.GREEN, Size.LARGE, Weight.NORMAL)
  house = Product('House', Color.BLUE, Size.LARGE, Weight.HEAVY)

  products = [apple, tree, house]
  pf = ProductFilter()
  print('Green Products (old):')
  for product in pf.filter_by_color(products=products, color=Color.GREEN):
    print(f' - {product.name} is green')

  bf = BetterFilter()
  print('Green Products (new):')
  green = ColorSpeccification(Color.GREEN)
  for product in bf.filter(items=products, spec=green):
    print(f' - {product.name} is green')

  print('Large Products (new):')
  large = SizeSpeccification(Size.LARGE)
  for product in bf.filter(items=products, spec=large):
    print(f' - {product.name} is large')

  print('Large and Blue Products (new):')
  large_blue = large & ColorSpeccification(Color.BLUE)
  for product in bf.filter(items=products, spec=large_blue):
    print(f' - {product.name} is large and blue')

  print('Large and Blue and Heavy Products (new):')
  large_blue_heavy = large & ColorSpeccification(Color.BLUE) & WeightSpecification(Weight.HEAVY)
  for product in bf.filter(items=products, spec=large_blue_heavy):
    print(f' - {product.name} is large and blue and heavy')

Solid Design Principles/Open Close Principle.py


from mv3d import config
from mv3d.dsets import scenelists
from mv3d.dsets import frameselector
from mv3d.dsets import dataset
from mv3d.eval import tsdf_atlas
import os
import numpy as np
import glob
import tqdm
import json
import cv2
import torch
from scipy.spatial.transform.rotation import Rotation
import open3d as o3d

K = np.array([[525.0, 0.0, 320.0],
              [0.0, 525.0, 240.0],
              [0.0, 0.0, 1.0]])
IMG_BATCH = 20

# Volumetric-based 3D options
VOX_RES = 0.02
VOL_PRCNT = .995
VOL_MARGIN = 1.5
TRUNC_RATIO = 3

def get_projection_matrices(K, poses):
    n_imgs = poses.shape[0]
    zeros = torch.zeros((n_imgs, 3, 1)).type_as(K)  # append a column of 0s to camera intrinsics
    K = torch.cat((K, zeros), dim=2)
    P = torch.bmm(K, poses)
    return P

def depth_projection_batched(depth, P):
    h, w = depth.shape[-2:]
    n_imgs = depth.shape[0]

    pts_y = torch.arange(h).type_as(depth)
    pts_x = torch.arange(w).type_as(depth)
    pts_yy, pts_xx = torch.meshgrid(pts_y, pts_x)
    pts_zz = torch.ones_like(pts_xx)

    pts_H = torch.stack((pts_xx, pts_yy, pts_zz), dim=0)
    pts_H = pts_H.unsqueeze(0).repeat(n_imgs, 1, 1, 1)

    pts_ww = 1. / depth.unsqueeze(1)
    pts_H = torch.cat((pts_H, pts_ww), dim=1)

    # add [0, 0, 0, 1] row to projection matrix to make 4x4
    eye_row = torch.tensor([[0, 0, 0, 1]]).unsqueeze(0).repeat(n_imgs, 1, 1).type_as(P)
    P = torch.cat((P, eye_row), dim=1)
    P_inv = P.inverse()

    pts_H = torch.bmm(P_inv, pts_H.view(n_imgs, 4, h*w))
    pts = pts_H[:, :3] / pts_H[:, 3:]

    return pts.transpose(2, 1)

def generate_gt_mesh(scene, max_depth=None):
    # get frames for trimming mesh
    selector = frameselector.EveryNthSelector(1)
    dset = dataset.Dataset([scene, ], selector, IMG_BATCH, (480, 640), (480, 640), False, scale_rgb=1.,
                           mean_rgb=[0., 0., 0.], std_rgb=[1., 1., 1.], crop=False, n_src_on_either_side=0)
    n_imgs = len(json.load(open(os.path.join(scene, 'info.json'), 'r'))['frames'])
    n_batches = (n_imgs - 1) // IMG_BATCH + 1

    # first, run through to get vol bounds
    origin = None
    vol_max = None
    for i in range(n_batches):
        batch, _, _, img_idx = dset.get(0, seed_idx=i * IMG_BATCH, return_verbose=True)
        poses = torch.cat((batch.rotmats, batch.tvecs.unsqueeze(2)), dim=2)
        eye_row = torch.tensor([[[0., 0., 0., 1.]]]).repeat(poses.shape[0], 1, 1)
        poses = torch.cat((poses, eye_row), dim=1)

        # mask out large depth values if necessary
        if max_depth is not None:
            batch.depth_images[batch.depth_images > 4.] = 0.

        P = get_projection_matrices(batch.K, poses).cuda()
        pts = depth_projection_batched(batch.depth_images.cuda(), P)
        pts = pts.reshape(-1, 3)
        pts = pts[~torch.any(torch.isnan(pts), dim=1)]  # avoid any div/0s in projection step
        pts = pts.cpu().numpy()
        if pts.shape[0] == 0:  # avoid any fully 0 depth maps from cut off reconstruction
            continue

        # use top and bottom vol_prcnt of points plus vol_margin
        origin_batch = torch.as_tensor(np.quantile(pts, 1 - VOL_PRCNT, axis=0) - VOL_MARGIN).float()
        vol_max_batch = torch.as_tensor(np.quantile(pts, VOL_PRCNT, axis=0) + VOL_MARGIN).float()

        if origin is None:
            origin = origin_batch
        else:
            origin = torch.min(torch.stack((origin, origin_batch), dim=0), dim=0)[0]

        if vol_max is None:
            vol_max = vol_max_batch
        else:
            vol_max = torch.max(torch.stack((vol_max, vol_max_batch), dim=0), dim=0)[0]
    vol_dim = ((vol_max - origin) / VOX_RES).int().tolist()

    # re-run through dset for tsdf fusion step
    tsdf_fusion = tsdf_atlas.TSDFFusion(vol_dim, VOX_RES, origin, TRUNC_RATIO, torch.device('cuda'),
                                        label=False)
    for i in range(n_batches):
        batch, _, _, img_idx = dset.get(0, seed_idx=i * IMG_BATCH, return_verbose=True)

        poses = torch.cat((batch.rotmats, batch.tvecs.unsqueeze(2)), dim=2)
        eye_row = torch.tensor([[[0., 0., 0., 1.]]]).repeat(poses.shape[0], 1, 1)
        poses = torch.cat((poses, eye_row), dim=1)

        P = get_projection_matrices(batch.K, poses).cuda()
        for i in range(batch.depth_images.shape[0]):
            projection = P[i]
            image = batch.images[i].cuda()
            depth = batch.depth_images[i]
            tsdf_fusion.integrate(projection, depth, image)

    tsdf = tsdf_fusion.get_tsdf()
    trimmed_mesh = tsdf.get_mesh()
    return trimmed_mesh

def get_closest_index(target_timestamp, other_timestamps):
    differences = np.abs(other_timestamps - target_timestamp)
    return np.argmin(differences)

def process_scene(scene, overwrite_depth=False, overwrite_mesh=False):
    depth_out_dir = os.path.join(scene, 'depth_processed')
    if not os.path.exists(depth_out_dir):
        os.makedirs(depth_out_dir)
    scene_name = os.path.basename(scene)
    gt_mesh_file = os.path.join(scene, 'gt_mesh.ply')

    data = {
        'scene': scene_name,
        'path': scene,
        'intrinsics': K.tolist(),
        'gt_mesh': gt_mesh_file,
        'frames': []
    }

    image_filenames = sorted(glob.glob(os.path.join(scene, 'rgb', '*.png')))
    image_timestamps = np.loadtxt(os.path.join(scene, 'rgb.txt'), usecols=0)

    depth_filenames = sorted(glob.glob(os.path.join(scene, 'depth', '*.png')))
    depth_timestamps = np.loadtxt(os.path.join(scene, 'depth.txt'), usecols=0)

    poses_with_quat = np.loadtxt(os.path.join(scene, 'groundtruth.txt'))
    pose_timestamps = poses_with_quat[:, 0]
    pose_locations = poses_with_quat[:, 1:4]
    pose_quaternions = poses_with_quat[:, 4:]

    for i in range(len(depth_filenames)):
        depth_timestamp = depth_timestamps[i]

        pose_index = get_closest_index(depth_timestamp, pose_timestamps)
        image_index = get_closest_index(depth_timestamp, image_timestamps)

        depth_filename_src = depth_filenames[i]
        depth_filename_dst = os.path.join(depth_out_dir, os.path.basename(depth_filename_src))
        image_filename = image_filenames[image_index]
        pose_location = pose_locations[pose_index]
        pose_quaternion = pose_quaternions[pose_index]
        rot = Rotation.from_quat(pose_quaternion).as_matrix()
        pose = np.eye(4)
        pose[0:3, 0:3] = rot
        pose[0:3, 3] = pose_location

        if not os.path.exists(depth_filename_dst) or overwrite_depth:
            depth = cv2.imread(depth_filename_src, -1).astype(np.float32) / 5000.
            depth[np.isnan(depth)] = 0.
            depth[np.isinf(depth)] = 0.
            depth = (depth * 1000.).astype(np.uint16)
            cv2.imwrite(depth_filename_dst, depth)

        frame = {
            'filename_color': image_filename,
            'filename_depth': depth_filename_dst,
            'pose': pose.tolist()
        }
        data['frames'].append(frame)
    json.dump(data, open(os.path.join(scene, 'info.json'), 'w'))

    if not os.path.exists(gt_mesh_file) or overwrite_mesh:
        gt_mesh = generate_gt_mesh(scene)
        o3d.io.write_triangle_mesh(gt_mesh_file, gt_mesh)
    return

if __name__ == '__main__':
    scenes = scenelists.get_scenes_tum_rgbd(config.TUM_RGBD_DIR)
    for scene in tqdm.tqdm(scenes):
        process_scene(scene)


import logging
from cv2 import VideoCapture, VideoWriter, VideoWriter_fourcc, imshow
from lowpolyfy.resources.utils.video_utils import video_exists, get_video_parameters, initialize_views_directory
from lowpolyfy.resources.videocube.VideoCube import VideoCube
from lowpolyfy.resources.pointplacement.PointPlacer import PointPlacer

logger = logging.getLogger(__name__)

class LowPolyfy():
    def _initialize_point_placer(self, algorithm, video_cube, num_points, video):
        # Create the point placer
        logger.info("Creating the point placer object.")
        placer = PointPlacer(video)

        # Set the point placement algorithm
        logger.info("Setting the placement algorithm to be {}.".format(algorithm))
        algorithm_set = placer.set_algorithm(algorithm)
        if not algorithm_set:
            # Signify initialization failed
            return False

        # Place points within the video cube
        placer.place_points(video_cube, num_points)
        return True

    def approximate(self, source_path, algorithm, num_points):
        # Check to ensure that a file exists at the specified path
        if (not video_exists(source_path)):
            logger.error("Failed to find a video file at the specified path.")
            return

        # Initialize the views directory
        VIEWS_DIRECTORY = "views"
        initialize_views_directory(VIEWS_DIRECTORY)

        # Setting up video reader
        video = VideoCapture(source_path)

        # Find the dimensions of the video to define the video cube
        num_frames, video_width, video_height, fps = get_video_parameters(video)
        vc = VideoCube(num_frames, video_height, video_width)

        # Setting up video writer
        fourcc = VideoWriter_fourcc(*'mp4v')
        video_out = VideoWriter(VIEWS_DIRECTORY + "/output.mp4", fourcc, fps, (video_height, video_width))
        video_out_lines = VideoWriter(VIEWS_DIRECTORY + "/output_view_lines.mp4", fourcc, fps, (video_height, video_width))
        video_out_corners = VideoWriter(VIEWS_DIRECTORY + "/output_view_corners.mp4", fourcc, fps, (video_height, video_width))
        video_out_keypoints = VideoWriter(VIEWS_DIRECTORY + "/output_view_keypoints.mp4", fourcc, fps, (video_height, video_width))

        # Initialize the video cube according to the point initialization algorithm
        # Exit if points failed to be placed
        if not self._initialize_point_placer(algorithm, vc, num_points, video):
            logger.error("Point placement failed to initialize.")
            return

        # Tetrahedralize the video cube
        vc.tetrahedralize()

        # Loop through the
        frame_number = 0
        while video.isOpened():
            # Read a frame of the video
            frames_remain, frame = video.read()

            # Stop reading if we reach the end of the video
            if not frames_remain:
                break

            # Slice the video cube at this frame and create a low poly frame
            frame_lp, frame_lp_lines, frame_lp_corners, lp_frame_keypoints = vc.slice_cube(frame, frame_number)
            frame_number += 1

            # Write the low poly frame
            video_out.write(frame_lp)
            video_out_lines.write(frame_lp_lines)
            video_out_corners.write(frame_lp_corners)
            video_out_keypoints.write(lp_frame_keypoints)

        # Release video reader and writer
        video_out.release()
        video_out_lines.release()
        video_out_corners.release()
        video_out_keypoints.release()
        video.release()


import os
import sys
import requests
import geopandas as gpd
import pandas as pd
import numpy as np
import calendar
from sklearn import linear_model

import os
import sys
import requests

class data_acquisition:

    def __init__(self, working_dir, plot=False):

        self.plot = plot
        self.working_dir = working_dir

        try:
            raw_shape_ger = gpd.read_file("../"+self.working_dir+"/DEU_adm1.shp")
            raw_shape_ger_eez = gpd.read_file("../"+self.working_dir+"/eez.shp")
        except RuntimeError:
            print(sys.exc_info())
            sys.exit()

        raw_shape_ger = raw_shape_ger[:].to_crs('EPSG:4326')
        raw_shape_ger_eez = raw_shape_ger_eez.to_crs('EPSG:4326')

        # Drop not needed rows, merge both geodataframes, plot a map and save as shape file
        shape_ger = raw_shape_ger[['NAME_1', 'geometry']].rename(columns = {'NAME_1':'geoname'})
        shape_ger_eez = raw_shape_ger_eez[['geoname', 'geometry']]

        self._shape_ger = pd.concat([shape_ger, shape_ger_eez]).reset_index(drop=True)
        self._shape_ger.to_file("../"+self.working_dir+"/shape_ger.shp")

        if self.plot == True:
            ax = self._shape_ger.plot(color="none", edgecolor='gainsboro')

    def fetch_data(self, years = [2019,2020,2021,2022], weather_parameters = 'T2M,T2MDEW,T2MWET,TS,WS50M,WD50M,ALLSKY_SFC_SW_DWN,CLRSKY_SFC_SW_DWN,ALLSKY_SFC_LW_DWN,ALLSKY_SFC_UVA,ALLSKY_SFC_UVB'):

        self.years = years
        self.weather_parameters = weather_parameters

        self._fetch_weather_data()
        self._reformat_weather_data()
        self._prepare_electricity_data()

    def _fetch_weather_data(self):

        # download the weather data
        # this is split into east and west, because the API only allows 10° LAT and LON
        # and into years, because the API only allows 366 days in one download

        for year in self.years:
            if year == 2022:
                mmdd = '0401'
            else:
                mmdd = '1231'

            exists = os.path.isfile('../'+self.working_dir+'/weather_map_raw_ger_west_'+str(year)+'.csv')
            if not exists:
                url = 'https://power.larc.nasa.gov/api/temporal/daily/regional?parameters='+self.weather_parameters+'&start='+str(year)+'0101&end='+str(year)+mmdd+'&community=RE&format=CSV&latitude-min=47.25&latitude-max=56.25&longitude-min=3.25&longitude-max=10.25'
                data = requests.get(url, allow_redirects=True)
                open('../'+self.working_dir+'/weather_map_raw_ger_west_'+str(year)+'.csv', 'wb').write(data.content)

            exists = os.path.isfile('../'+self.working_dir+'/weather_map_raw_ger_east_'+str(year)+'.csv')
            if not exists:
                url = 'https://power.larc.nasa.gov/api/temporal/daily/regional?parameters='+self.weather_parameters+'&start='+str(year)+'0101&end='+str(year)+mmdd+'&community=RE&format=CSV&latitude-min=47.25&latitude-max=56.25&longitude-min=10.75&longitude-max=15.25'
                data = requests.get(url, allow_redirects=True)
                open('../'+self.working_dir+'/weather_map_raw_ger_east_'+str(year)+'.csv', 'wb').write(data.content)

        # merge east and west and all years to one file

        weather_raw_ger = {}

        for year in self.years:
            weather_raw_ger_east = pd.read_csv('../'+self.working_dir+'/weather_map_raw_ger_east_'+str(year)+'.csv',
                                                   header=19)
            weather_raw_ger_west = pd.read_csv('../'+self.working_dir+'/weather_map_raw_ger_west_'+str(year)+'.csv',
                                                   header=19)

            weather_raw_ger[year] = pd.concat([weather_raw_ger_east,
                                               weather_raw_ger_west])

        weather_raw_ger = pd.concat([weather_raw_ger[year] for year in self.years]).reset_index(drop=True)

        # infer the date to a pandas datetime (therefore rename stuff..)
        weather_raw_ger.rename(columns = {'MO':'month', 'DY':'day'}, inplace=True)
        weather_raw_ger['date'] = pd.to_datetime(weather_raw_ger[['YEAR', 'month', 'day']])
        weather_raw_ger.drop(['YEAR', 'month', 'day'], axis=1, inplace=True)

        # convert weather data to a geodataframe (with LON and LAT) so its easy to inspect
        # plot the shape file and the weather data locations (only for one datetime, locations are the same for all datetimes)
        weather_raw_ger_geo = gpd.GeoDataFrame(weather_raw_ger,
                                               geometry = gpd.points_from_xy(weather_raw_ger.LON, weather_raw_ger.LAT),
                                               crs="EPSG:4326")

        if self.plot == True:
            ax = self._shape_ger.plot(color="none", edgecolor='gainsboro')
            weather_raw_ger_geo[weather_raw_ger['date'] == pd.Timestamp('2019-01-01')].plot(color="k", markersize=1, ax=ax)

        self.weather_ger_geo = self._mask_weather_data(weather_raw_ger, weather_raw_ger_geo)

        if self.plot == True:
            ax = self._shape_ger.plot(color="none", edgecolor='gainsboro')
            self.weather_ger_geo[self.weather_ger_geo['date'] == pd.Timestamp('2019-07-04')].plot(color="k", markersize=1, ax=ax)

    def _reformat_weather_data(self):

        # Energy generation data will only be available for whole germany, therefore each row has to contain all LAT LON information.
        # Generate feature map by translating LAT LON from columns to single row:

        weather_ger_df = pd.DataFrame(self.weather_ger_geo.drop(columns='geometry'))#[self.weather_ger_geo.date.dt.year != 2022])

        lat_lon = weather_ger_df[['LAT', 'LON']].drop_duplicates()
        column_names = []
        for i in weather_ger_df.columns[2:-1]:
            for index, row in lat_lon.iterrows():
                column_names.append(i+' '+str(row['LAT'])+' '+str(row['LON']))
        column_names.append('date')

        # A bit hacky, but correct and fast:
        self._weather_ger_reshape = pd.DataFrame(columns = column_names, index=np.arange(len(pd.unique(weather_ger_df.date))))
        for j in range(len(self._weather_ger_reshape)):
            for i in range(len(weather_ger_df.columns)-3):
                self._weather_ger_reshape.loc[j][i*len(lat_lon):(i+1)*len(lat_lon)] = weather_ger_df.loc[j*len(lat_lon):(j+1)*len(lat_lon)-1][weather_ger_df.columns[i+2]]
            self._weather_ger_reshape.loc[j][-1] = weather_ger_df.loc[j*len(lat_lon)].date
        self._weather_ger_reshape.set_index('date', inplace=True)

    def _mask_weather_data(self, weather_raw_ger, weather_raw_ger_geo):

        # Sort the data for a computationally efficent application of the mask
        weather_raw_ger = weather_raw_ger.sort_values(by=['date', 'LAT', 'LON']).reset_index(drop=True)
        weather_raw_ger_geo = weather_raw_ger_geo.sort_values(by=['date', 'LAT', 'LON']).reset_index(drop=True)

        # for one specific datetime generate a mask of weather locations that are within germany and eez
        mask_individual_shape = [weather_raw_ger_geo[weather_raw_ger['date'] == pd.Timestamp('2019-01-01')].within(self._shape_ger.loc[i, 'geometry']) for i in range(len(self._shape_ger))]
        mask_individual_time = pd.Series([any([mask_individual_shape[i].iloc[j] for i in range(len(mask_individual_shape))]) for j in range(len(mask_individual_shape[0]))])

        # expand this mask by concatinating copys of it
        mask = mask_individual_time.copy()
        mask_t = mask_individual_time.copy()
        for i in range(int(len(weather_raw_ger)/len(mask_individual_time)-1)):
            mask_t.index += len(mask_individual_time)
            mask = pd.concat([mask, mask_t])

        # Apply the mask and plot the result for a single datetime
        return weather_raw_ger_geo[mask].reset_index(drop=True)

    def _power_to_energy(self, power_single_day):
        measurements_per_hour = len(power_single_day)/24
        energy = np.sum(power_single_day) / measurements_per_hour
        return energy

    def _prepare_electricity_data(self):

        electricity_raw = {}

        for year in self.years:
            if year <= 2019:
                electricity_raw[year] = pd.read_csv('../'+self.working_dir+'/energy-charts_Electricity_production_in_Germany_in_'+str(year)+'.csv')
            else:
                electricity_raw[year] = pd.read_csv('../'+self.working_dir+'/energy-charts_Electricity_generation_in_Germany_in_'+str(year)+'.csv')

        electricity_raw = pd.concat([electricity_raw[year] for year in self.years]).reset_index(drop=True)
        electricity_raw['Date (UTC)'] = pd.to_datetime(electricity_raw['Date (UTC)'], utc=True)
        electricity_raw.set_index(['Date (UTC)'], inplace = True)
        electricity_raw.index = electricity_raw.index.tz_convert(None)

        electricity_raw_daybinned = electricity_raw.resample('D').apply(self._power_to_energy)

        self._electricity_daybinned = pd.DataFrame(index = electricity_raw_daybinned.index)
        self._electricity_daybinned['Wind'] = electricity_raw_daybinned[electricity_raw_daybinned.columns[0:6]].sum(axis=1)
        self._electricity_daybinned['Solar'] = electricity_raw_daybinned[electricity_raw_daybinned.columns[6:-1]].sum(axis=1)

        self._electricity_daybinned.index.names = ['date']

        self.weather_electricity_data = pd.merge(left = self._weather_ger_reshape, right = self._electricity_daybinned, how='left', on = 'date')
        self.weather_electricity_data

    def save_data(self, file_name = 'weather_electricity_data.csv'):
        # save data as da pandas dataframe
        self.weather_electricity_data.to_csv('../'+self.working_dir+'/'+file_name)

class data_conditioning:

    def __init__(self, X_train, X_test, y_train, y_test):

        self.y_train_stationary, self.y_test_stationary = self._make_stationary(X_train, X_test, y_train, y_test)

    def _make_stationary(self, X_train, X_test, y_train, y_test):

        # average data for each year in the training data set
        y_yearly = y_train.resample('Y').mean()[:-1]
        days_from_start_yearly = np.cumsum(np.array([self._nr_days(year) for year in y_yearly.index.year])) - self._nr_days(y_yearly.index.year[0])
        X_yearly = days_from_start_yearly.reshape(-1, 1)

        # fit linear model to the yearly averages
        self._model = linear_model.LinearRegression().fit(X_yearly, y_yearly.values)

        # subtract linear slope from train and test data
        train_days_from_start = np.array([(date - X_train.index[0]).days for date in X_train.index])
        y_train_stationary = y_train.values - train_days_from_start * self._model.coef_[0]
        y_train_stationary = pd.Series(data=y_train_stationary, index=X_train.index)

        test_days_from_start = np.array([(date - X_train.index[0]).days for date in X_test.index])
        y_test_stationary = y_test.values - test_days_from_start * self._model.coef_[0]
        y_test_stationary = pd.Series(data=y_test_stationary, index=X_test.index)

        self._test_days_from_start = test_days_from_start

        return y_train_stationary, y_test_stationary

    def reverse_stationary(self, prediction):

        # add linear slope from train and test data
        prediction_reversed = prediction + self._test_days_from_start * self._model.coef_[0]

        return prediction_reversed

    def _nr_days(self, year):
        return 366 if calendar.isleap(year) else 365


from siteinterface.mod_calendar import bp_calendar
from siteinterface import app
from flask import request, jsonify
from logic.StandardPointName import *
from openpyxl import Workbook
import os
from datetime import datetime
import traceback
from .. BEOPMySqlDBContainer import BEOPMySqlDBContainer
from siteinterface.BEOPSqliteAccess import *
from siteinterface.BEOPDataAccess import BEOPDataAccess
import logging
import xlrd
from siteinterface.utils import send_alert_to_cloud_center, get_proj_name

def isValidDate(strDate, strFormat):
    res = False
    try:
        if not isinstance(strDate, str):
            return res

        tTime = datetime.strptime(strDate, strFormat)
        res = True
    except:
        pass
    finally:
        return res

def get_last_day_of_this_month(curDate):
    res = None
    try:
        if not isinstance(curDate, datetime):
            return None

        tNextMonth = curDate.replace(day=28) + timedelta(days=4)
        firstDayOfNextMonth = tNextMonth.replace(day=1)
        res = firstDayOfNextMonth - timedelta(days=1)
    except:
        pass
    finally:
        return res

def modeIdExistsInMode(modeId):
    """
    检查modeId是否已存在于mode表中
    """
    res = False
    try:
        rvConduct = BEOPDataAccess.getInstance().modeIdExistsInMode(modeId)
        if rvConduct is None:
            return None

        if rvConduct and len(rvConduct):
            if len(rvConduct[0]):
                if rvConduct[0][0] > 0:
                    res = True

    except Exception as e:
        strLog = "ERROR in modeIdExistsInMode: %s" % e.__str__()
        logging.error(strLog)

    finally:
        return res

@bp_calendar.route("/getCalendarWithWeather", methods=["POST"])
def get_calendar_with_weather():
    try:
        rcv = request.get_json()
        year = rcv.get("year") if rcv.get("year") else None
        month = rcv.get("month") if rcv.get("month") else None

        if not year:
            return jsonify(dict(err=1, msg="年不能为空", data=[]))

        if not month:
            return jsonify(dict(err=1, msg="月不能为空", data=[]))

        timeFormat = "%Y-%m-%d"

        strTimeStart = "%s-%02d-01" % (year, month)

        tTimeStart = datetime.strptime("%s-%02d-01" % (year, month), timeFormat)

        tTimeEnd = get_last_day_of_this_month(datetime.strptime("%s-%02d-01" % (year, month), timeFormat))

        strTimeEnd = tTimeEnd.strftime(timeFormat)

        weatherDict = BEOPDataAccess.getInstance().getCalendarWithWeather(strTimeStart, strTimeEnd)

        if weatherDict is None:
            return jsonify(dict(err=1, msg="数据库繁忙，请重试", data=[]))

        if not weatherDict:
            return jsonify(dict(err=1, msg="无日历数据", data=[]))

        dataList = list()
        curDate = tTimeStart
        while curDate <= tTimeEnd:

            strCurDate = curDate.strftime("%Y-%m-%d")

            weatherInfo = weatherDict.get(strCurDate) if weatherDict.get(strCurDate) else {}

            dataList.append({
                "day": curDate.day,
                "weather": weatherInfo
            })

            curDate = curDate + timedelta(days=1)

        return jsonify(dict(err=0, data=dataList, msg="获取成功"))

    except Exception as e:
        strLog = "ERROR in /calendar/getMonthWeatherData: %s" % e.__str__()
        logging.error(strLog)

        return jsonify(dict(err=1, msg="获取失败: %s" % e.__str__(), data=[]))

@bp_calendar.route("/addModeToCalendar", methods=["POST"])
def add_mode_to_calendar():
    try:
        rcv = request.get_json()
        date = rcv.get("date") if rcv.get("date") else None
        modeId = rcv.get("modeId") if rcv.get("modeId") is not None else None
        type = rcv.get("type") if rcv.get("type") is not None else None
        creator = rcv.get("creator") if rcv.get("creator") else None
        now = datetime.now()
        dateToday = datetime.strptime(now.strftime("%Y-%m-%d"), "%Y-%m-%d")

        strSource = rcv.get("source", "")
        strIP = request.remote_addr
        strLog = '[IMPORTANT]addModeToCalendar: 来自IP:%s, 用户:%s, date:%s, type:%s  modeId:%s' % (
        strIP, strSource, date, type, modeId)
        logging.error(strLog)

        if strSource:
            bReadOnly = BEOPDataAccess.getInstance().is_user_read_only(strSource)
            if bReadOnly:
                return json.dumps(dict(err=1, msg='user has no write priority', data={}), ensure_ascii=False)

        if date is None:
            return jsonify(dict(err=1, msg="日期不能为空"))

        if not isValidDate(date, "%Y-%m-%d"):
            return jsonify(dict(err=1, msg="日期格式有误"))

        tDate = datetime.strptime(date, "%Y-%m-%d")

        if tDate < dateToday:
            return jsonify(dict(err=1, msg="今日之前的模式不可编辑"))

        if modeId is None:
            return jsonify(dict(err=1, msg="模式ID不能为空"))

        if not isinstance(modeId, int):
            return jsonify(dict(err=1, msg="模式ID必须为整数"))

        if not modeIdExistsInMode(modeId):
            return jsonify(dict(err=1, msg="模式ID不存在"))

        if type is None:
            return jsonify(dict(err=1, msg="模式类型不能为空"))

        if not isinstance(type, int):
            return jsonify(err=1, msg="模式类型必须为整数")

        if not creator:
            return jsonify(dict(err=1, msg="创建者不能为空"))
        if not isinstance(creator, str):
            return jsonify(err=1, msg="创建者名称必须为字符串")

        rvConduct = BEOPDataAccess.getInstance().getModeById(modeId)
        if rvConduct is None:
            return jsonify(dict(err=1, msg="数据库繁忙，请重试"))

        if not len(rvConduct[1]):
            strMsg = "IP为{ip}的用户正在执行模式绑定(项目:{proj},模式ID:{modeId})，但该模式内容为空".format(ip=strIP,
                                                                                proj=get_proj_name(), modeId=modeId)
            send_alert_to_cloud_center(get_proj_name(), strMsg)

        result = BEOPDataAccess.getInstance().insertIntoModeCalendar(date, modeId, creator, type)
        try:
            BEOPDataAccess.getInstance().appendOperationLog(strSource, '从地址%s发起模式修改'%(strIP))
        except:
            pass

        if result.get("code") > 0:
            return jsonify(dict(err=1, msg=result.get("msg"), data=result.get("data")))
        else:
            return jsonify(dict(err=0, msg=result.get("msg"), data=result.get("data")))

    except Exception as e:
        strLog = "ERROR in /calendar/addModeToCalendar: %s" % e.__str__()
        logging.error(strLog)

        return jsonify(dict(err=1, msg="保存失败: %s" % e.__str__(), data=False))

@bp_calendar.route("/batchAddModeToCalendar", methods=["POST"])
def batch_add_mode_to_calendar():
    try:
        rcv = request.get_json()
        year = rcv.get("year") if rcv.get("year") is not None else None
        month = rcv.get("month") if rcv.get("month") is not None else None
        modeId = rcv.get("modeId") if rcv.get("modeId") is not None else None
        type = rcv.get("type") if rcv.get("type") is not None else None
        creator = rcv.get("creator") if rcv.get("creator") else None
        strSource = rcv.get("source", "")
        nWeekdayList = rcv.get('weekdayList', [])
        if len(nWeekdayList)==0:
            nWeekdayList = [0,1,2,3,4,5,6]
        strIP = request.remote_addr

        strLog = '[IMPORTANT]batchAddModeToCalendar: 来自IP:%s, 用户:%s, year:%s,  month:%s,  modeId:%s'%(strIP, strSource, year, month, modeId)
        logging.error(strLog)

        if strSource:
            bReadOnly = BEOPDataAccess.getInstance().is_user_read_only(strSource)
            if bReadOnly:
                return json.dumps(dict(err=1, msg='user has no write priority', data={}), ensure_ascii=False)

        now = datetime.now()
        today = now.day
        thisYear = now.year
        thisMonth = now.month

        if year is None:
            return jsonify(dict(err=1, msg="年不能为空"))

        if not isinstance(year, int):
            return jsonify(dict(err=1, msg="年必须为整数"))

        if month is None:
            return jsonify(dict(err=1, msg="月不能为空"))

        if not isinstance(month, int):
            return jsonify(dict(err=1, msg="月必须为整数"))

        if year < thisYear:
            return jsonify(dict(err=1, msg="无法编辑去年的日程"))

        if year == thisYear and month < thisMonth:
            return jsonify(dict(err=1, msg="无法编辑本月之前的日程"))

        if modeId is None:
            return jsonify(dict(err=1, msg="模式ID不能为空"))

        if not isinstance(modeId, int):
            return jsonify(dict(err=1, msg="模式ID必须为整数"))

        if not modeIdExistsInMode(modeId):
            return jsonify(dict(err=1, msg="模式ID不存在"))

        if type is None:
            return jsonify(dict(err=1, msg="模式类型不能为空"))

        if not isinstance(type, int):
            return jsonify(err=1, msg="模式ID必须为整数")

        if not creator:
            return jsonify(dict(err=1, msg="创建者不能为空"))

        if not isinstance(creator, str):
            return jsonify(err=1, msg="创建者名称必须为字符串")

        fromDay = 1
        if year == thisYear and month == thisMonth:
            fromDay = today

        lastDate = get_last_day_of_this_month(datetime.strptime("%s-%02d-01" % (year, month), "%Y-%m-%d"))
        toDay = lastDate.day

        rvConduct = BEOPDataAccess.getInstance().getModeById(modeId)
        if rvConduct is None:
            return jsonify(dict(err=1, msg="数据库繁忙，请重试"))

        if not len(rvConduct[1]):
            strMsg = "IP为{ip}的用户正在执行批量模式绑定(项目:{proj},模式ID:{modeId})，但该模式内容为空".format(ip=strIP,
                                                                                proj=get_proj_name(), modeId=modeId)
            send_alert_to_cloud_center(get_proj_name(), strMsg)

        result = BEOPDataAccess.getInstance().batchInsertIntoModeCalendar(year, month, fromDay, toDay, modeId, creator, type, nWeekdayList)

        try:
            BEOPDataAccess.getInstance().appendOperationLog(strSource, '从地址%s发起模式批量绑定'%(strIP))
        except:
            pass

        if result.get("code") > 0:
            return jsonify(dict(err=1, msg=result.get("msg"), data=result.get("data")))

        return jsonify(dict(err=0, msg=result.get("msg"), data=result.get("data")))

    except Exception as e:
        strLog = "ERROR in /calendar/batchAddModeToCalendar: %s" % e.__str__()
        logging.error(strLog)

        return jsonify(dict(err=1, msg="保存失败: %s" % e.__str__(), data=[]))

@bp_calendar.route("/getCalendarWithMode", methods=["POST"])
def get_calendar_with_mode():
    try:
        rcv = request.get_json()
        year = rcv.get("year") if rcv.get("year") else None
        month = rcv.get("month") if rcv.get("month") else None
        type = rcv.get("type") if rcv.get("type") is not None else None

        if type is not None:
            if not isinstance(type, int):
                return jsonify(dict(err=1, msg="type必须是一个整数"))

        if not year:
            return jsonify(dict(err=1, msg="年不能为空", data=[]))

        if not month:
            return jsonify(dict(err=1, msg="月不能为空", data=[]))

        strTimeStart = "%s-%02d-01" % (year, month)

        tTimeStart = datetime.strptime("%s-%02d-01" % (year, month), "%Y-%m-%d")

        tTimeEnd = get_last_day_of_this_month(datetime.strptime("{0}-{1}-01".format(year, month), "%Y-%m-%d"))

        strTimeEnd = tTimeEnd.strftime("%Y-%m-%d")

        result = BEOPDataAccess.getInstance().getCalendarWithMode(strTimeStart, strTimeEnd, type)

        if result["code"] == 1:
            return jsonify(dict(err=1, msg=result["msg"], data=result.get("data")))

        if result["code"] == 2:
            return jsonify(dict(err=0, msg=result["msg"], data=result.get("data")))

        dataList = []
        curDate = tTimeStart
        dataDict = result.get("data")[0]
        weatherDict = result.get("data")[1]
        while curDate <= tTimeEnd:
            strCurDate = curDate.strftime("%Y-%m-%d")

            lInfo = dataDict.get(strCurDate) if dataDict.get(strCurDate) else []
            weather = weatherDict.get(strCurDate) if weatherDict.get(strCurDate) else {}
            dataList.append({
                "day": curDate.day,
                "mode": lInfo,
                "weather": weather
            })

            curDate = curDate + timedelta(days=1)

        return jsonify(dict(err=0, msg="获取成功", data=dataList))

    except Exception as e:
        strLog = "ERROR in /calendar/getCalendarWithMode: %s" % e.__str__()
        logging.error(strLog)

        return jsonify(dict(err=1, msg="保存失败: %s" % e.__str__(), data=[]))

@bp_calendar.route("/removeModeFromCalendar", methods=["POST"])
def remove_mode_from_calendar():
    try:
        rcv = request.get_json()
        modeId = rcv.get("modeId") if rcv.get("modeId") is not None else None
        date = rcv.get("date") if rcv.get("date") else None
        strSource = rcv.get("source", "")
        strIP = request.remote_addr

        strLog = '[IMPORTANT]removeModeFromCalendar: 来自IP:%s, 用户:%s, date:%s, modeId:%s'%(strIP, strSource, date, modeId)
        logging.error(strLog)

        if strSource:
            bReadOnly = BEOPDataAccess.getInstance().is_user_read_only(strSource)
            if bReadOnly:
                return json.dumps(dict(err=1, msg='user has no write priority', data={}), ensure_ascii=False)

        if modeId is None:
            return jsonify(dict(err=1, msg="模式ID不能为空"))

        if not isinstance(modeId, int):
            return jsonify(dict(err=1, msg="模式ID必须为整数"))

        if not modeIdExistsInMode(modeId):
            return jsonify(dict(err=1, msg="模式ID不存在"))

        if not isValidDate(date, "%Y-%m-%d"):
            return jsonify(dict(err=1, msg="日期格式有误"))

        result = BEOPDataAccess.getInstance().removeModeFromCalendar(modeId, date)

        try:
            BEOPDataAccess.getInstance().appendOperationLog(strSource, '从地址%s发起模式解除'%(strIP))
        except:
            pass

        if result.get("code") > 0:
            return jsonify(dict(err=1, msg=result.get("msg"), data=False))
        else:
            return jsonify(dict(err=0, msg=result.get("msg"), data=True))

    except Exception as e:
        strLog = "ERROR in /calendar/removeModeFromCalendar: %s" % e.__str__()
        logging.error(strLog)

        return jsonify(dict(err=1, msg="删除失败: %s" % e.__str__(), data=False))

@bp_calendar.route("/batchRemoveModeFromCalendar", methods=["POST"])
def batch_remove_mode_from_calendar():
    try:
        rcv = request.get_json()
        modeId = rcv.get("modeId", None)
        strFrom = rcv.get("from", "")
        strTo = rcv.get("to", "")
        strSource = rcv.get("source", "")
        strIP = request.remote_addr

        strLog = '[IMPORTANT]batchRemoveModeFromCalendar: 来自IP:%s, 用户:%s, date:%s, modeId:%s' % (strIP, strSource, date, modeId)
        logging.error(strLog)

        if strSource:
            bReadOnly = BEOPDataAccess.getInstance().is_user_read_only(strSource)
            if bReadOnly:
                return json.dumps(dict(err=1, msg='user has no write priority', data={}), ensure_ascii=False)

        if modeId is None:
            return jsonify(dict(err=1, msg="模式ID不能为空", data=False))

        if not isinstance(modeId, int):
            return jsonify(dict(err=1, msg="模式ID必须为整数", data=False))

        if not modeIdExistsInMode(modeId):
            return jsonify(dict(err=1, msg="模式ID不存在", data=False))

        if not len(strFrom) or not len(strTo):
            return jsonify(dict(err=1, msg="开始、结束日期不能为空", data=False))

        if not isValidDate(strFrom, "%Y-%m-%d"):
            return jsonify(dict(err=1, msg="开始日期格式有误", data=False))

        if not isValidDate(strTo, "%Y-%m-%d"):
            return jsonify(dict(err=1, msg="结束日期格式有误", data=False))

        tFrom = datetime.strptime(strFrom, "%Y-%m-%d")
        tTo = datetime.strptime(strTo, "%Y-%m-%d")
        if tFrom < datetime.now().replace(hour=0, minute=0, second=0, microsecond=0):
            return jsonify(dict(err=1, msg="今天之前的模式无法解绑", data=False))

        if tFrom > tTo:
            return jsonify(dict(err=1, msg="开始日期不能大于结束日期", data=False))

        result = BEOPDataAccess.getInstance().batchRemoveModeFromCalendar(modeId, tFrom, tTo)

        try:
            BEOPDataAccess.getInstance().appendOperationLog(strSource, '从地址%s发起模式批量解除'%(strIP))
        except:
            pass

        if result.get("code") > 0:
            return jsonify(dict(err=1, msg=result.get("msg"), data=False))
        return jsonify(dict(err=0, msg=result.get("msg"), data=True))

    except Exception as e:
        strLog = "ERROR in /calendar/batchRemoveModeFromCalendar: %s" % e.__str__()
        logging.error(strLog)
        return jsonify(dict(err=1, msg="解绑失败: %s" % e.__str__(), data=False))

@bp_calendar.route("/batchRemoveModeFromCalendarThisMonth", methods=["POST"])
def batch_remove_mode_from_calendar_this_month():
    try:
        rcv = request.get_json()
        nType = rcv.get("type", None)
        strDate = rcv.get("date", "")

        strSource = rcv.get("source", "")
        strIP = request.remote_addr

        strLog = '[IMPORTANT]batchRemoveModeFromCalendarThisMonth: 来自IP:%s, 用户:%s, date:%s, modeId:%s' % (strIP, strSource, strDate, nType)
        logging.error(strLog)

        if strSource:
            bReadOnly = BEOPDataAccess.getInstance().is_user_read_only(strSource)
            if bReadOnly:
                return json.dumps(dict(err=1, msg='user has no write priority', data={}), ensure_ascii=False)

        # if nType is None:
        #     return jsonify(dict(err=1, msg="类型不能为空", data=False))

        if nType is not None:
            if not isinstance(nType, int):
                return jsonify(dict(err=1, msg="类型必须为整数", data=False))

        if not len(strDate):
            return jsonify(dict(err=1, msg="日期不能为空", data=False))

        if not isValidDate(strDate, "%Y-%m"):
            return jsonify(dict(err=1, msg="日期格式有误", data=False))

        tDate = datetime.strptime(strDate, "%Y-%m")
        tFrom = tDate.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
        tTo = (time_get_first_day_of_next_month(tDate) - timedelta(days=1)).replace(hour=23, minute=59, second=59, microsecond=0)
        if tFrom < datetime.now():
            tFrom = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)

        result = BEOPDataAccess.getInstance().batchRemoveModeFromCalendarThisMonth(nType, tFrom, tTo)

        try:
            BEOPDataAccess.getInstance().appendOperationLog(strSource, '从地址%s发起模式批量解除'%(strIP))
        except:
            pass

        if result.get("code") > 0:
            return jsonify(dict(err=1, msg=result.get("msg"), data=False))
        return jsonify(dict(err=0, msg=result.get("msg"), data=True))

    except Exception as e:
        strLog = "ERROR in /calendar/batchRemoveModeFromCalendarThisMonth: %s" % e.__str__()
        logging.error(strLog)
        return jsonify(dict(err=1, msg="解绑失败: %s" % e.__str__(), data=False))

@bp_calendar.route("/getModeOfTypeOnCertainDate", methods=["POST"])
def get_mode_of_type_on_certain_date():
    rcv = request.get_json()
    nType = rcv.get("type", None)
    date = rcv.get("date", "")

    if nType is None:
        return jsonify(dict(err=1, msg="类型不能为空", data={}))
    if not isinstance(nType, int):
        return jsonify(dict(err=1, msg="类型必须为整数", data={}))
    if not date:
        return jsonify(dict(err=1, msg="日期不能为空", data={}))
    if not isinstance(date, str):
        return jsonify(dict(err=1, msg="日期必须为字符串", data={}))
    if not isValidDate(date, "%Y-%m-%d"):
        return jsonify(dict(err=1, msg="日期格式有误", data={}))

    res = BEOPDataAccess.getInstance().getModeOfTypeOnCertainDate(nType, date)
    if res.get("code") > 0:
        return jsonify(dict(err=1, msg=res.get("msg"), data={}))
    return jsonify(dict(err=0, msg="获取成功", data=res.get("data")))


import argparse
import sys

import pandas as pd

def make_cli():
    cli = argparse.ArgumentParser()

    cli.add_argument(
        "tsvs",
        nargs="+",
        type=str,
        help="Path to tsvs to be summarized.",
    )

    return cli

def main():
    args = vars(make_cli().parse_args())

    files = args["tsvs"]

    df = pd.concat((pd.read_table(path) for path in files), axis="index")
    df.to_csv(sys.stdout, sep="\t", index=False)

if __name__ == "__main__":
    main()


import pulumi
from pulumi import export, Output, ResourceOptions
from prometheus import PrometheusArgs, Prometheus

config = pulumi.Config()
service_name = config.get('service_name') or 'example'
networkingStack = pulumi.StackReference(config.require('NetworkingStack'))

# for this to work, two docker images have to be built:
# 1) ecs_sd for automatic service discovery and writing of the /etc/config/ecs_auto_sd.yaml
# 2) prometheus for the custom prom image with the prometheus.yaml config file bind mounted to a VOLUME
#    that will end up in the ecs-task-def
prom = Prometheus(
    f'{service_name}-prometheus-ecs',
    PrometheusArgs(
        vpc_id=networkingStack.get_output("vpc_id"),
        subnet_ids=[networkingStack.get_output("private_subnet1"),
                    networkingStack.get_output("private_subnet2")
        ],
        sg_groups=[networkingStack.get_output("prom_server_sg"), # allows prom server to send metrics to timescaledb for long-term storage
                   networkingStack.get_output("grafana_sg") # allows grafana dashboard access
        ],
        ecs_cluster_arn=networkingStack.get_output("ecs_cluster_arn"),
    )
)


import bpy
from typing import Optional, Type

class TabbedPanelHelper:
    _tab_panels: dict[str, bpy.types.Panel] = {}
    _panel_toggle_ops: dict[str, bpy.types.Operator] = {}

    use_grid: bool = True
    default_tab: Optional[str] = None

    def draw(self, context: bpy.types.Context):
        self.draw_before(context)
        self._draw_toggle_buttons(context)
        self.draw_after(context)

    def draw_before(self, context: bpy.types.Context):
        """Draw before buttons"""
        ...

    def draw_after(self, context: bpy.types.Context):
        """Draw after buttons"""
        ...

    def _draw_toggle_buttons(self, context: bpy.types.Context):
        layout = self.layout.grid_flow(
            align=True, row_major=True, columns=4) if self.use_grid else self.layout.row(align=True)

        for tab_id, op_cls in self._panel_toggle_ops.items():
            if tab_id not in self._tab_panels:
                continue

            tab_panel = self._tab_panels[tab_id]

            if not tab_panel.poll_tab(context):
                continue

            is_active = tab_panel._get_active_tab() == op_cls.tab_id

            layout.operator(op_cls.bl_idname, text=tab_panel.bl_label,
                            icon=tab_panel.icon, depress=is_active)

    @classmethod
    def register(cls):
        cls._tab_panels = {}
        cls._panel_toggle_ops = {}

class TabPanel:
    parent_tab_panel: Type[TabbedPanelHelper]
    icon: str = "NONE"

    bl_options = {"HIDE_HEADER"}

    @classmethod
    def poll(cls, context: bpy.types.Context):
        return cls._is_active and cls.poll_tab(context)

    @classmethod
    def poll_tab(cls, context: bpy.types.Context):
        """Poll method for tabs (do not override poll)"""
        return True

    @classmethod
    def _get_active_tab_propname(cls):
        return f"{cls.bl_parent_id}_active"

    @classmethod
    def _get_active_tab(cls) -> str:
        tabbed_panel = cls.parent_tab_panel
        active_tab = getattr(bpy.context.scene, cls._get_active_tab_propname())

        if active_tab == "NONE":
            return ""

        if active_tab:
            return active_tab

        if tabbed_panel.default_tab is not None and tabbed_panel.default_tab in tabbed_panel._tab_panels:
            return tabbed_panel.default_tab

    @classmethod
    def _set_active_tab(cls, tab_id: str):
        setattr(bpy.context.scene, cls._get_active_tab_propname(), tab_id)

    @classmethod
    @property
    def _is_active(cls):
        return cls._get_active_tab() == cls.bl_idname

    @classmethod
    def register(cls):
        if not hasattr(cls, "parent_tab_panel"):
            raise NotImplementedError(
                f"Panel tab '{cls.__name__}' has no ``parent_tab_panel`` defined!")

        tabbed_panel = cls.parent_tab_panel
        active_tab_prop_name = cls._get_active_tab_propname()

        if not hasattr(bpy.types.Scene, active_tab_prop_name):
            setattr(
                bpy.types.Scene,
                active_tab_prop_name,
                bpy.props.StringProperty()
            )

        toggle_op = cls._tab_toggle_op_factory(cls.bl_idname)
        tabbed_panel._panel_toggle_ops[cls.bl_idname] = toggle_op

        bpy.utils.register_class(toggle_op)

        tabbed_panel._tab_panels[cls.bl_idname] = cls

    @classmethod
    def unregister(cls):
        active_tab_prop_name = cls._get_active_tab_propname()
        if hasattr(bpy.types.Scene, active_tab_prop_name):
            delattr(bpy.types.Scene, active_tab_prop_name)

    @classmethod
    def _tab_toggle_op_factory(cls, _tab_id: str):
        """Dynamically create a tab toggle operator class"""
        class SOLLUMZ_OT_toggle_tab(bpy.types.Operator):
            bl_idname = f"sollumz.toggletab_{_tab_id.lower()}"
            bl_description = f"Toggle the {cls.bl_label} tab"
            bl_label = _tab_id
            tab_id = _tab_id

            def execute(self, context: bpy.types.Context):
                active_tab = cls._get_active_tab()

                if active_tab == self.tab_id:
                    cls._set_active_tab("NONE")
                else:
                    cls._set_active_tab(self.tab_id)

                return {"FINISHED"}

        return SOLLUMZ_OT_toggle_tab


# this is the program API, designed with the following capabilities in mind:
# + evaluate a single face against the model (easy),
# + add a new face to the model (intermediate),
# + render the facial landmark and HOG results to the GUI for extra points
#   (hard).
# thus it will host three main function calls.
import logging
import docker
# import re
import os
import yaml
with open("config.yaml") as f:
    conf = yaml.load(f.read())

client = docker.from_env()
logger = logging.getLogger(__name__)

# absolute path of program in host os
dir_ls = os.getcwd().split('facial_recog')[0] + 'facial_recog/'

def save_config(directory):
    """
    saves the config when it is changed during the program.
    """

    with open("config.yaml", 'r+') as f:
        conf = yaml.load(f.read())
        conf["classifier_path"] = ''.join([directory, "classifier.pkl"])
        conf["embeddings_path"] = ''.join([directory, "embeddings.npy"])
        f.truncate()
        f.flush()
        f.write(yaml.dump(conf, default_flow_style=False))

# formats the directory nicely for Docker
def docker_dir(abspath):
    dir_list = abspath.split('facial_recog')[0].split('/')[1:]
    return '/'.join(['', 'facial_recog'] + [x for x in dir_list] + [''])

def evaluate_face(input_dir=dir_ls):
    """
    function that recognizes the face on a picture.
    input: location of the picture. (str)
    output: predicted label of the picture. (str)
    errors:
    + picture_name does not exist/is not and image - throw type error
    + no positive match to any of the labels - throw nonexistent error
    """
    # print(input_dir)

    # directory = re.split("facialReco", input_dir)[1]

    command_string = ' '.join([
        "python3 /facial_recog/facenet_reco"
        "/train_classifier.py --input-dir", "/facial_recog/temp/",
        "--model-path /facial_recog/etc/20170511-185253/"
        "20170511-185253.pb --classifier-path "
        "/facial_recog/output/classifier.pkl "
        "--num-threads 16 --num-epochs 5 "
        "--min-num-images-per-class 1"
    ])

    return (client.containers.run(
        "colemurray/medium-facenet-tutorial",
        command_string,
        environment=["PYTHONPATH=$PYTHONPATH:/facial_recog"],
        volumes={
            dir_ls: {
                'bind': '/facial_recog',
                'mode': 'rw'
            }
        }))

def add_face(input_dir, existing_subjects):
    """
    function that retrains the model to add a new face.
    input: location of the images. (str)
    output: error code, 0 otherwise. (exception, none)
    errors:
    + pictures_folder is not a folder - throw type error
    + pictures_folder contains a non-image - fail silently and continue
      for the rest
    + pictures_folder does not have [enough] pictures - throw exception and
      exit
    + pictures_folder contains pictures that are not from the same people -
      this kills the model.
    """

    # dir_list = input_dir.split('facial_recog')[1].split('/')
    # directory = '/'.join(['', 'facial_recog]'] + [x for x in dir_list])

    command_string = ' '.join([
        "python3 /facial_recog/facenet_reco"
        "/train_classifier.py --input-dir", docker_dir(input_dir),
        "--model-path /facial_recog/etc/20170511-185253/"
        "20170511-185253.pb --classifier-path", conf["classifier_path"],
        "--num-threads 16 --num-epochs 5 "
        "--existing-subjects", existing_subjects,
        "--min-num-images-per-class 10 --is-train --is-retrain"
    ])
    # save_config(docker_dir(input_dir))

    return(client.containers.run(
        "colemurray/medium-facenet-tutorial",
        command_string,
        environment=["PYTHONPATH=$PYTHONPATH:/facial_recog"],
        volumes={
            dir_ls: {
                'bind': '/facial_recog',
                'mode': 'rw'
            }
        }))

def process_image(picture):
    """function that takes the pictures from the main folder and preprocesses
    them.
    """

    print(docker_dir(picture))

    command_string = ' '.join([
        "python3 /facial_recog/facenet_reco/preprocess.py "
        "--input-dir", docker_dir(picture),
        "--output-dir", conf["prov_img"],
        "--crop-dim 180"
    ])

    return (client.containers.run(
        "colemurray/medium-facenet-tutorial",
        command_string,
        environment=["PYTHONPATH=$PYTHONPATH:/facial_recog"],
        volumes={
            dir_ls: {
                'bind': '/facial_recog',
                'mode': 'rw'
            }
        }))

def adjust_algo(direc):
    """
    function that trains a new model from the pictures in the selected folder.
    """

    command_string = ' '.join([
        "python3 /facial_recog/facenet_reco"
        "/train_classifier.py --input-dir", docker_dir(direc),
        "--model-path /facial_recog/etc/20170511-185253/"
        "20170511-185253.pb --classifier-path", conf["classifier_path"],
        "--num-threads 16 --num-epochs 5 "
        "--min-num-images-per-class 10 --is-train"
    ])
    # save_config(docker_dir(direc))

    return(client.containers.run(
        "colemurray/medium-facenet-tutorial",
        command_string,
        environment=["PYTHONPATH=$PYTHONPATH:/facial_recog"],
        volumes={
            dir_ls: {
                'bind': '/facial_recog',
                'mode': 'rw'
            }
        }))


from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from test_resources.test_variables import Locators
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
import time

class Keywords:
    """This class contains various utility methods for interacting with a web application."""

    def __init__(self, driver, logger):
        """
        Initializes the Keywords class with a Selenium WebDriver instance and a logger.

        Parameters:
            - driver (WebDriver): Selenium WebDriver instance.
            - logger (Logger): Logging object for reporting.
        """
        self.driver = driver
        self.logger = logger

    def open_site(self):
        """Opens the website and maximizes the window."""
        self.driver.get(Locators.url)
        self.driver.maximize_window()
        self.logger.info(f"Opened {Locators.url}")

    def click_element(self, element):
        """Clicks an element given a xpath"""
        WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, element))).click()

    def element_is_displayed(self, element):
        """Element is displayed given a xpath. Returns True if displayed, otherwise False."""
        try:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
            return True
        except TimeoutException:
            return False

    def element_input_text(self, element, input):
        """Inputs text given a xpath element"""
        WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element))).send_keys(input)

    def element_text(self, element):
        """Retrieves the text attribute of an element"""
        return WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element))).text

    def navigate_to_page(self, page):
        """
        Navigates to the specified page on the website.

        Parameters:
            - page (str): The name of the page to navigate to.
        """
        url_slug = Locators.page_urls[page]
        self.driver.get(f"{Locators.url}/{url_slug}")
        self.logger.info(f"Navigated to page{page}")

    def check_page_title(self, title):
        """
        Checks if the current page title matches the expected title.

        Parameters:
            - title (str): The expected title of the page.
        """
        time.sleep(1)
        assert title == self.driver.title, f"{title} does not match {self.driver.title}"
        self.logger.info(f"Title {title} matches")

    def page_contains_upper_and_lower_buttons(self):
        """Verifies that the page contains the expected upper and lower navigation buttons."""
        elements = [
            Locators.home_upper_button_xpath,
            Locators.about_me_upper_button_xpath,
            Locators.cv_upper_button_xpath,
            Locators.contact_upper_button_xpath,
            Locators.home_lower_button_xpath,
            Locators.about_me_lower_button_xpath,
            Locators.cv_lower_button_xpath,
            Locators.contact_lower_button_xpath,
            Locators.tg_main_button_xpath,
            Locators.tg_secondary_button_xpath
        ]
        for element in elements:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
        self.logger.info("Page contains upper and lower buttons")

    def home_page_contains_inner_elements(self):
        """Verifies that the home page contains all the expected inner elements."""
        elements = [
            Locators.lets_talk_button_xpath,
            Locators.contact_me_button_xpath,
            Locators.main_skills_xpath
        ]
        for element in elements:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
        self.logger.info("Home page contains inner elements")

    def about_me_page_contains_inner_elements(self):
        """Verifies that the About Me page contains all the expected inner elements."""
        elements = [Locators.contact_me_button_xpath]
        for element in elements:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
        self.logger.info("About me page contains inner elements")

    def cv_page_contains_inner_elements(self):
        """Verifies that the CV page contains all the expected inner elements."""
        elements = [
            Locators.contact_me_button_xpath,
            Locators.anritsu_xpath,
            Locators.continental_xpath,
            Locators.ness_xpath,
            Locators.barra_xpath,
            Locators.dima_xpath,
            Locators.etti_xpath,
            Locators.negruzzi_xpath,
            Locators.download_cv_xpath
        ]
        for element in elements:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
        self.logger.info("CV page contains inner elements")

    def contact_page_contains_inner_elements(self):
        """Verifies that the Contact page contains all the expected inner elements."""
        elements = [
            Locators.linkedin_xpath,
            Locators.whatsapp_xpath,
            Locators.github_xpath,
            Locators.facebook_xpath,
            Locators.instagram_xpath,
            Locators.twitter_xpath,
            Locators.submit_button_xpath
        ]
        for element in elements:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.XPATH, element)))
        self.logger.info("Contact page contains inner elements")

    def user_is_on_home_page(self):
        """Verifies that the user is currently on the home page by checking the title and essential elements."""
        time.sleep(1)
        self.check_page_title(Locators.home_title)
        self.page_contains_upper_and_lower_buttons()
        self.home_page_contains_inner_elements()
        self.logger.info("User is on Home page")

    def user_is_on_about_me_page(self):
        """Verifies that the user is currently on the About Me page by checking the title and essential elements."""
        time.sleep(1)
        self.check_page_title(Locators.about_me_title)
        self.page_contains_upper_and_lower_buttons()
        self.about_me_page_contains_inner_elements()
        self.logger.info("User is on About me page")

    def user_is_on_cv_page(self):
        """Verifies that the user is currently on the CV page by checking the title and essential elements."""
        time.sleep(1)
        self.check_page_title(Locators.cv_title)
        self.page_contains_upper_and_lower_buttons()
        self.cv_page_contains_inner_elements()
        self.logger.info("User is on CV page")

    def user_is_on_contact_page(self):
        """Verifies that the user is currently on the Contact page by checking the title and essential elements."""
        time.sleep(1)
        self.check_page_title(Locators.contact_title)
        self.page_contains_upper_and_lower_buttons()
        self.contact_page_contains_inner_elements()
        self.logger.info("User is on Contact page")

    def switch_to_new_tab(self):
        """Switches the driver's context to the last opened tab."""

        handles = self.driver.window_handles
        self.driver.switch_to.window(handles[-1])
        self.logger.info("Switched to new tab")

    def verify_pdf_cv(self, expected_url=Locators.cv_pdf_url):
        """Verifies that a PDF has the correct link when the user clicks the 'Download my PDF Curriculum Vitae' link."""

        element = WebDriverWait(self.driver, 10).until(EC.presence_of_element_located(
            (By.XPATH, Locators.download_cv_xpath)))
        self.driver.execute_script("arguments[0].scrollIntoView(true);", element)
        actual_url = element.get_attribute('href')
        print("Actual URL: ", actual_url)
        print("Expected URL: ", expected_url)
        assert actual_url == expected_url, "URLs do not match"
        self.logger.info(f"CV element {element} has the correct URL: {expected_url}")

        target_value = element.get_attribute('target')
        assert target_value == "_blank", "Link does not open in new tab"

    def verify_social_media_link_opens_in_new_tab(self, element_locator, expected_url):
        """
        Verifies that clicking on a social media link opens the correct URL in a new tab.

        Parameters:
            - element_locator (str): The XPATH locator for the social media element.
            - expected_url (str): The expected URL that should be opened in a new tab.
        """

        element = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, element_locator)))
        actual_url = element.get_attribute('href')
        assert actual_url == expected_url, "URLs do not match"
        self.logger.info(f"Social Medial element {element_locator} has the correct URL: {expected_url}")

        target_value = element.get_attribute('target')
        assert target_value == "_blank", "Link does not open in new tab"

        element.click()
        self.logger.info(f"Clicked on {element_locator}")

        all_handles = self.driver.window_handles
        new_handle = all_handles[-1]
        self.driver.switch_to.window(new_handle)
        WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
        self.logger.info("Successfully switched to new tab")
        self.driver.close()
        self.logger.info("Closed the new tab")

        original_handle = all_handles[0]
        self.driver.switch_to.window(original_handle)
        self.logger.info("Switched back to the original tab")

    def verify_email_or_phone_link(self, element_locator, expected_url):
        """
        Verifies that the email or phone link is correct.

        Parameters:
            - element_locator (str): The XPATH locator for the email or phone element.
            - expected_url (str): The expected 'mailto' or 'tel' URL.
        """

        element = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, element_locator)))
        actual_url = element.get_attribute('href')
        assert actual_url == expected_url, "URLs do not match"
        self.logger.info(f"Email/Phone element {element_locator} has the correct URL: {expected_url}")


from rest_framework.decorators import api_view, authentication_classes
from rest_framework import status
from rest_framework.response import Response
from core.models import User
from core.authentication import MyAuthentication
from core.serializers import UserSerializer
from utils.helper import log
from utils.exceptions import ResponseException
from core.serializers import UserSerializer

@api_view(http_method_names=['get'])
@authentication_classes([MyAuthentication])
def following(request):
    """ gets all the users that are followed by the current user """

    try:
        current_user: User = request.user
        log('CURRENT USER', current_user.username)

        if request.method == 'GET':
            queryset = current_user.following.all()
            serializer = UserSerializer(queryset, many=True)
            log('DATA', serializer.data)
            return Response(serializer.data, status=status.HTTP_200_OK)

    except Exception as e:
        return ResponseException(msg=e)

@api_view(http_method_names=['post'])
@authentication_classes([MyAuthentication])
def get_user(request):
    """ get the user details by taking 'username' as input """

    try:
        data = request.data
        user = User.objects.get(username=data['username'])
        return Response(UserSerializer(user).data, status=status.HTTP_200_OK)
    except Exception as e:
        return ResponseException(msg=e)

@api_view(http_method_names=['delete', 'post'])
@authentication_classes([MyAuthentication])
def follow_or_unfollow(request, user_id):
    """ follow or unfollow the user using their 'user_id' """
    try:
        current_user: User = request.user
        user_following = current_user.following

        if request.method == 'POST':
            current_user.following.add(user_id)
            return Response({'follow_status': 'followed successfully'}, status=status.HTTP_201_CREATED)

        if request.method == 'DELETE':
            log('FOLLOWING USER', current_user.username)
            user_following.remove(user_id)
            return Response({'follow_status': 'unfollowed successfully'}, status=status.HTTP_202_ACCEPTED)

    except Exception as e:
        return ResponseException(msg=e)


'''
Bohrn effective charge tensor
'''
import numpy
import jax
from pyscfad import gto, scf
from pyscfad.lib import numpy as jnp

mol = gto.Mole()
mol.atom = '''H  ,  0.   0.   0.
              F  ,  0.   0.   .917'''
mol.basis = '631g'
mol.build(trace_exp=False, trace_ctr_coeff=False)

def dip_moment(mol):
    mf = scf.RHF(mol)
    ao_dip = mol.intor_symmetric('int1e_r', comp=3)
    h1 = mf.get_hcore()
    E = numpy.zeros((3))
    mf.get_hcore = lambda *args, **kwargs: h1 + jnp.einsum('x,xij->ij', E, ao_dip)
    mf.kernel()
    return mf.dip_moment(mol, mf.make_rdm1(), unit='AU', verbose=0)

z = jax.jacrev(dip_moment)(mol).coords
print(z)


from pathlib import Path
import sys

file = Path(__file__).resolve()
parent, root = file.parent, file.parents[2]
sys.path.append(str(root))

from fastapi import APIRouter
from fastapi.requests import Request

from IO_handler.webtoken import return_id_if_is_valid
from database.operations import get_user_tables, get_user_information
from IO_handler.responses import arrange_table_response, assembly_tables_dict
from IO_handler.responses import arrange_user_information, create_tables_response

router = APIRouter()

@router.post("/internal-page-validation")
async def return_page(request: Request):
    data = await request.json()
    user_token = data["token"]
    jwt_result = return_id_if_is_valid(user_token)
    jwt_is_valid = jwt_result["valid"]
    if jwt_is_valid:
        user_id = jwt_result["id"]
        folders_tables = get_user_tables(user_id, "folders")
        completeds_tables = get_user_tables(user_id, "completeds")
        deleteds_tables = get_user_tables(user_id, "deleteds")
        folders_list = arrange_table_response(folders_tables)
        completeds_list = arrange_table_response(completeds_tables)
        deleteds_list = arrange_table_response(deleteds_tables)
        tables_dict = assembly_tables_dict(folders_list, completeds_list, deleteds_list)
        user_raw_informations = get_user_information(user_id)
        user_arranged_informations = arrange_user_information(user_raw_informations)
        return create_tables_response(True, tables_dict, user_arranged_informations)
    else:
        return create_tables_response(False, "", "")


import os
import time
import glob
import pandas as pd
from tqdm import tqdm
from nltk.tokenize import TweetTokenizer
from joblib import Parallel, delayed
tqdm.pandas()

def make_data(data: list):
    cols = ["tweetid",
            "user_screen_name",
            "tweet_language",
            "tweet_text",
            "tweet_time",
            "is_retweet",
            "in_reply_to_tweetid",
            "hashtags",
            "urls",
            "user_mentions"]

    df = pd.DataFrame()

    for d in data:
        print(f"LOADING {d}")
        for chunk in pd.read_csv(d, chunksize=250000, low_memory=False, usecols=cols):
            df = pd.concat([df, chunk])

    # drop any possible duplicates:
    og_len = len(df)
    df.drop_duplicates(subset="tweetid", inplace=True)
    deduped = len(df)

    print(f"Found {og_len - deduped} duplicates")

    # extract english tweets:
    print("Extracting English tweets")
    df = df[df.tweet_language == "en"]

    # remove retweets:
    df = df[df.is_retweet == False]

    # remove excessively short tweets:
    print("Counting tokens")
    tokenizer = TweetTokenizer()
    df["tokens"] = Parallel(n_jobs=7)(delayed(tokenizer.tokenize)(line) for line in tqdm(df.tweet_text))
    df["n_tokens"] = df.tokens.apply(lambda row: len(row))
    df = df[df.n_tokens >= 10]

    print(f"Final sample size: {len(df)}")

    # save:
    print("Saving")
    df.reset_index(inplace=True, drop=True)
    df.to_feather(os.path.join("data", "processed", "ru_en_tweets.feather"))

if __name__ == "__main__":
    start = time.time()

    files = glob.glob(os.path.join("data", "raw-twitter", "*.csv"))
    make_data(files)

    end = time.time()
    timer = round((end - start) / 60, 3)

    print("Done!")
    print(f"Run time: {timer} minutes")


from pathlib import Path

from allennlp.common.testing import AllenNlpTestCase
from allennlp.common.util import ensure_list

from nbadd.dataset_reader.aoc_reader import AOCDatasetReader

class TestAOCDatasetReader(AllenNlpTestCase):

    def test_read_from_file(self):
        reader = AOCDatasetReader()
        test_fp = Path(__file__, '..', '..', 'test_data', 'aoc_test_data.json')

        instance1 = {"text": ["الف", "مليون", "مبارك", "لفريق", "الوحدات"],
                     "label": "msa"}
        instance2 = {"text": ["ف", "بلاش", "التعميم", "وتشويه", "صوره"],
                     "label": "egyptian"}
        instance3 = {"text": ["ارامكو", "تقول", "انوو", "7000", "مهوب", "6000"],
                     "label": "gulf"}
        instance4 = {"text": ["اكيد", "الطفل", "ورد", "في", "قلوب"],
                     "label": "levantine"}

        instances = ensure_list(reader.read(str(test_fp.resolve())))

        assert len(instances) == 4

        fields = instances[0].fields
        assert [t.text for t in fields["text"].tokens][:5] == instance1["text"]
        assert fields["label"].label == instance1["label"]
        fields = instances[1].fields
        assert [t.text for t in fields["text"].tokens][:5] == instance2["text"]
        assert fields["label"].label == instance2["label"]
        fields = instances[2].fields
        assert [t.text for t in fields["text"].tokens] == instance3["text"]
        assert fields["label"].label == instance3["label"]
        fields = instances[3].fields
        assert [t.text for t in fields["text"].tokens][:5] == instance4["text"]
        assert fields["label"].label == instance4["label"]


import json
import os
import shutil
import subprocess

from textwrap import dedent
from jinja2 import Environment, FileSystemLoader

VARS = dict(
    init_app=True,
    custom_box=False,
    templates_dir=os.path.join(os.path.dirname(__file__), 'templates')
)

def falstart_print(message, error=False, prefix='[falstart] >'):
    """ Display given message with prefix"""
    bold, red, end = '\033[1m', '\033[31m', '\033[0m'
    print(''.join((
        red if error else '',
        bold, prefix, end,
        red if error else '',
        ' ', message, end,
    )).strip())

def run(command):
    """ wrapper of subprocess check_call fun """
    falstart_print('run: "{}"'.format(command))
    subprocess.check_call(command, shell=True)

def put(src, dst):
    falstart_print('copy: "{}" > "{}"'.format(src, dst))
    if os.path.exists(dst):
        shutil.rmtree(dst)
    shutil.copytree(src, dst)

def mkdir(path):
    falstart_print('make dir: "{}"'.format(path))
    try:
        os.makedirs(path)
    except OSError:
        pass

def render_template(template_name, remote_name):
    """ Render wrapper for simplify touch rendered template into target """
    if os.path.dirname(remote_name):
        mkdir(os.path.dirname(remote_name))
    # load jinja template
    jinja_env = Environment(loader=FileSystemLoader(VARS['templates_dir']))
    template = jinja_env.get_template(template_name)
    # write to remote file
    falstart_print('render "{}" > "{}"'.format(template_name, remote_name))
    with open(remote_name, 'w') as target_file:
        target_file.write(template.render(**VARS))

def common(taskname, root_dir, config=None):
    """ Run all common tasks """
    root_dir = os.path.join(os.getcwd(), root_dir)
    mkdir(root_dir)
    os.chdir(root_dir)
    # update configs
    if config is None:
        VARS.update(json.load(open('.falstart.json', 'r')))
    else:
        json.dump(config, open('.falstart.json', 'w'))
        VARS.update(config)
    VARS['root_dir'] = root_dir
    globals()[taskname]()

def start_box():
    """ Create Vagrant file and up virtual machine """
    # render templates for vagrant up
    render_template('Vagrantfile.j2', 'Vagrantfile')
    render_template('Makefile.j2', 'Makefile')
    run('chmod +x Makefile')
    # make provisioning folder
    mkdir('provision')
    render_template('provision_fabfile.j2', 'provision/fabric_provisioner.py')
    render_template('requirements.j2', 'requirements.txt')
    render_template('requirements.j2', 'requirements-remote.txt')
    render_template('lintrc.j2', '.lintrc')
    render_template('coveragerc.j2', '.coveragerc')
    render_template('py_codes/settings_local.j2', '{proj_name}/settings_local.py.example'.format(**VARS))
    if VARS.get('CELERY'):
        render_template('py_codes/celery.j2', '{proj_name}/celery.py'.format(**VARS))
    # copy templates for vagrant fabric render
    put(os.path.join(VARS['templates_dir'], 'vagrant_templates'), 'provision/templates')
    try:
        env_vars = ('NO_NFS=1 ', '')[bool(VARS.get('VAGRANT_NFS'))]
        run('{}vagrant up'.format(env_vars))
    except:
        falstart_print('Error occurred when up the vagrant machine.', error=True)
        falstart_print('You can try to manually start the Vagrant.', error=True)

    # replace template
    VARS['init_app'] = False
    render_template('provision_fabfile.j2', 'provision/fabric_provisioner.py')
    # commit changes
    falstart_commit()

def falstart_commit():
    """ Try to commit after box start """
    render_template('gitignore.j2', '.gitignore')
    extra_ignore = dedent('''
        # custom ignore'
        settings_local.py
        .vagrant
        static/
    ''')

    falstart_print('Update .gitignore')
    with open('.gitignore', 'a') as gitignore:
        gitignore.write(extra_ignore)

    for attempt in range(2):
        try:
            run('git add . && git commit -m ":rocket: falstart init commit"')
            break
        except:
            falstart_print('not a git repository', error=True)
            run('git init')

def make_custom_box():
    VARS['custom_box'] = True
    run('vagrant destroy -f')
    # render templates to no provide app and syncfolder
    render_template('Vagrantfile.j2', 'Vagrantfile')
    run('vagrant up')
    for cmd in (
            'sudo dd if=/dev/zero of=/EMPTY bs=1M', 'sudo rm -f /EMPTY',
            'cat /dev/null > ~/.bash_history && history -c'):
        run('vagrant ssh -c "{}"'.format(cmd))
    # make custom box
    run('vagrant package --output ~/{proj_name}.box'.format(**VARS))
    # return template to back
    VARS['custom_box'] = False
    render_template('Vagrantfile.j2', 'Vagrantfile')

def rmproj():
    """ Remove project """
    run('cd {root_dir} && vagrant destroy -f'.format(**VARS))
    run('rm {root_dir} -rf'.format(**VARS))


import  requests
from bs4 import BeautifulSoup

def getPageString(url):
    data = requests.get(url)
    bsObj = BeautifulSoup(data.content, "html.parser")

    print(bsObj)

    return data.content

url = "https://www.coupang.com/np/categories/186764"

print(getPageString(url))


import sys

import pygame
from fangkuai import Fang

def run_game():
    pygame.init()
    screen = pygame.display.set_mode((800, 600))#窗口大小
    pygame.display.set_caption("Moving Game")#窗口名字

    bg_color = (200, 200, 200)
    fang = Fang(screen)
    #开始游戏
    while True:
        for event in pygame.event.get():    #检测鼠标和键盘事件
            if event.type == pygame.QUIT:   #如果按下关闭
                sys.exit()                  #执行退出窗口

        fang.draw_fang()
        pygame.display.flip()               #让最近绘制的窗口可见

run_game()


import os

from utils import list_tracks, exclude_tags, match_tags, \
                  match_any_tag

# plists = {'all_shuffle' : {'mode' : 'exclude',
#                            'tags' : ['protected']}}

class Plists_Generator():

    def __init__(self, audio_dirs, plist_dir,
                 plists, extensions=['.mp3', '.flac']):

        self.audio_dirs = audio_dirs
        self.plist_dir = plist_dir
        self.plists = plists
        self.extensions = extensions

    def _add_output_array(self):
        for fname in self.plists.keys():
            self.plists[fname]['tracks']=[]

    def _match_track_to_plist(self, track, fname):

        if self.plists[fname]['mode'] == 'match':
            if match_tags(track, self.plists[fname]['tags']):
                self.plists[fname]['tracks'].append(track)

        elif self.plists[fname]['mode'] == 'any':
            if match_any_tag(track, self.plists[fname]['tags']):
                self.plists[fname]['tracks'].append(track)

        elif self.plists[fname]['mode'] == 'exclude':
            if not exclude_tags(track, self.plists[fname]['tags']):
                self.plists[fname]['tracks'].append(track)

    def scan_audio_dirs_and_add_tracks(self):

        for folder in self.audio_dirs:

            for track in list_tracks(folder, self.extensions):

                for fname in self.plists.keys():
                    # add_to_plist(track, fname)
                    # self.plists[fname]['tracks'].append(track)
                    self._match_track_to_plist(track, fname)

    def save_plists(self):

        for fname in self.plists.keys():

            fpath = os.path.join(self.plist_dir, fname + '.m3u8')

            tracks = [os.path.relpath(track, self.plist_dir)
                        for track in self.plists[fname]['tracks']]

            tracks = map(lambda x: x + '\n', tracks)

            with open(fpath, 'w') as file_m3u8:
                file_m3u8.writelines(tracks)

    def generate(self):

        self._add_output_array()
        self.scan_audio_dirs_and_add_tracks()

        # for fname in self.plists.keys():
        #     print(self.plists[fname]['tracks'])

        # print(self.plists)

        self.save_plists()