import os
from typing_extensions import SupportsIndex

import Bio.Align
import pkg_resources

import pandas as pd
import nglview as nv
import seaborn as sns

from typing import Dict, Tuple, List, Any
from ipywidgets import Box

from Bio import AlignIO
from Bio.PDB import PDBParser
from Bio.PDB.Structure import Structure

from melodia.geometryparser import GeometryParser

from sklearn.preprocessing import StandardScaler
from sklearn.cluster import AgglomerativeClustering

def geometry_from_structure_file(file_name: str) -> pd.DataFrame:
    """
         Function used to compute the geometric properties around residues
         for (BioPython compatible) structures.
         It computes curvature, torsion, arc-length and writhing number
         :param file_name: Protein file name
         :type file_name: String
         :return: Pandas dataframe
         :rtype: pd.DataFrame
    """
    parser = PDBParser()
    name, ext = os.path.splitext(file_name)
    structure = parser.get_structure(name, file_name)

    base = proc_chains(structure)

    return base

def geometry_from_structure(structure: Structure) -> pd.DataFrame:
    """
         Function used to compute the geometric properties around residues
         for (BioPython compatible) structures.
         It computes curvature, torsion, arc-length and writhing number
         :param structure: BioPython PDB Structure class
         :type structure: Structure
         :return: Panda dataframe
         :rtype: pd.DataFrame
    """
    base = proc_chains(structure)

    return base

def proc_chains(structure: Structure) -> pd.DataFrame:
    """
         Function used to compute the geometric properties around residues
         for (BioPython compatible) structures.
         It computes curvature, torsion, arc-length and writhing number
         :param structure: BioPython PDB Structure class
         :type structure: Structure
         :return: Panda dataframe
         :rtype: pd.DataFrame
    """
    pdb_code = structure.id.upper()
    chains = {}
    for model in structure:
        for chain in model:
            chain_id = chain.id
            key = f'{pdb_code}:{model.id}:{chain_id}'
            chains[key] = GeometryParser(chain)

    # Generate a Pandas Dataframe
    identity = []
    model = []
    chain = []
    order = []
    name = []
    curvature = []
    torsion = []
    arc_length = []
    writhing = []
    angle_phi = []
    angle_psi = []
    for key, value in chains.items():
        # print(key)
        tokens = key.split(':')

        for res_id in value.residues:
            model.append(int(tokens[1]))
            chain.append(tokens[2])

            identity.append(value.residues[res_id].res_num)
            order.append(value.residues[res_id].res_order)
            name.append(value.residues[res_id].name)
            curvature.append(value.residues[res_id].curvature)
            torsion.append(value.residues[res_id].torsion)
            arc_length.append(value.residues[res_id].arc_len)
            writhing.append(value.residues[res_id].writhing)
            angle_phi.append(value.residues[res_id].phi)
            angle_psi.append(value.residues[res_id].psi)

    base = {'id': identity,
            'model': model,
            'code': pdb_code,
            'chain': chain,
            'order': order,
            'name': name,
            'curvature': curvature,
            'torsion': torsion,
            'arc_length': arc_length,
            'writhing': writhing,
            'phi': angle_phi,
            'psi': angle_psi}

    return pd.DataFrame(base)

def geometry_dict_from_structure(structure: Structure) -> Dict[str, GeometryParser]:
    """
         Function used to compute the geometric properties around residues
         for (BioPython compatible) structures.
         It computes curvature, torsion, arc-length and writhing number
         :param structure: BioPython PDB Structure class
         :type structure: Structure
         :return: Geometry Dictionary
         :rtype: Dict[str, GeometryParser]
    """
    chains = {}
    for model in structure:
        for chain in model:
            chain_id = chain.id
            key = f'{model.id}:{chain_id}'
            residues = [res.id[1] for res in list(chain.get_residues()) if res.id[0] == ' ']
            if residues:
                chains[key] = GeometryParser(chain)

    return chains

def bfactor_from_geo(structure: Structure, attribute: str, geo: Dict[str, GeometryParser] = None) -> None:
    """
        Set the PDB bfactor to geometric values
         :param structure: BioPython PDB Structure class
         :type structure: Structure
         :param attribute: geometric attribute
         :type attribute: str
        :param geo: Geometry Dictionary
        :type: Dict[str, GeometryParser]

    """
    if geo is None:
        geo = geometry_dict_from_structure(structure)

    for model in structure.get_list():
        for chain in model.get_list():
            for atom in chain.get_atoms():
                het_flag, sequence_id, insertion_code = atom.get_parent().id
                if het_flag[0] == " ":
                    key = f'{model.id}:{chain.id}'
                    res = geo[key].residues_map[sequence_id]

                    if attribute == 'curvature':
                        value = geo[key].residues[res].curvature
                    elif attribute == 'torsion':
                        value = geo[key].residues[res].torsion
                    else:
                        value = 0.0

                    atom.set_bfactor(value)

def view_putty(structure: Structure, radius_scale=1.0, width=1200, height=600) -> Box:
    """
        Display PDB structure as a putty model
        :param structure: BioPython PDB Structure class
        :type structure: Structure
        :param radius_scale: Radius Scale
        :type width: float
        :param width: Widget width
        :type width: int
        :param height: Widget height
        :type height: int
    """
    view = nv.show_biopython(structure)
    view.representations = [
        {"type": "tube",
         "params": {
             "sele": "protein",
             "radius": "bfactor",
             "radiusScale": radius_scale,
             "color": "bfactor",
             "colorScale": "RdYlBu"
         }}
    ]
    view.layout.width = 'auto'
    view.layout.height = 'auto'

    box = Box([view])
    box.layout.width = f'{width}px'
    box.layout.height = f'{height}px'

    return box

def view_cartoon(structure: Structure, width=1200, height=600) -> Box:
    """
        Display PDB structure as a cartoon model
        :param structure: BioPython PDB Structure class
        :type structure: Structure
        :param width: Widget width
        :type width: int
        :param height: Widget height
        :type height: int
    """
    view = nv.show_biopython(structure)
    view.representations = [
        {"type": "cartoon",
         "params": {
             "sele": "protein",
             "color": "bfactor",
             "colorScale": "RdYlBu"
         }}
    ]
    view.layout.width = 'auto'
    view.layout.height = 'auto'

    box = Box([view])
    box.layout.width = f'{width}px'
    box.layout.height = f'{height}px'

    return box

def view_tube(structure: Structure, width=1200, height=600) -> Box:
    """
        Display PDB structure as a tube model
        :param structure: BioPython PDB Structure class
        :type structure: Structure
        :param width: Widget width
        :type width: int
        :param height: Widget height
        :type height: int
    """
    view = nv.show_biopython(structure)
    view.representations = [
        {"type": "tube",
         "params": {
             "sele": "protein",
             "color": "bfactor",
             "colorScale": "RdYlBu"
         }}
    ]
    view.layout.width = 'auto'
    view.layout.height = 'auto'

    box = Box([view])
    box.layout.width = f'{width}px'
    box.layout.height = f'{height}px'

    return box

def parser_pir_file(pir_file: str) -> Bio.Align.MultipleSeqAlignment:
    """
        Parser for PIR alignment file and associated PDB structures
        :param pir_file: BioPython PDB Structure class
        :type pir_file: str
        :return: Protein sequence alignment
        :rtype: Bio.Align.MultipleSeqAlignment
    """
    c321 = {'CYS': 'C', 'ASP': 'D', 'SER': 'S', 'GLN': 'Q', 'LYS': 'K',
            'ILE': 'I', 'PRO': 'P', 'THR': 'T', 'PHE': 'F', 'ASN': 'N',
            'GLY': 'G', 'HIS': 'H', 'LEU': 'L', 'ARG': 'R', 'TRP': 'W',
            'ALA': 'A', 'VAL': 'V', 'GLU': 'E', 'TYR': 'Y', 'MET': 'M'}

    align = AlignIO.read(pir_file, 'pir')

    parser = PDBParser()
    for record in align:
        # Only select structures
        if record.description[:9] != 'structure':
            continue

        file_name = f'{record.id}.pdb'

        structure = parser.get_structure(record.id, file_name)

        # Renumber the residues to start with 1
        residue_number = 1
        for model in structure:
            for chain in model:
                for residue in chain:
                    # print(residue.id)
                    residue.id = (residue.id[0], residue_number, 'Z')
                    residue_number += 1
                for residue in chain:
                    residue.id = (residue.id[0], residue.id[1], ' ')
                    # print('----', residue.id)

        geo = geometry_dict_from_structure(structure)

        curvature = []
        torsion = []
        arc_length = []
        writhing = []
        phi = []
        psi = []

        idx = []
        for key in geo:
            for res in geo[key].residues:
                idx.append((key, res))
        j = 0
        for i, letter in enumerate(record.seq):
            if letter != '-' and letter != '/':
                try:
                    curr_chain = idx[j][0]
                    curr_residue = idx[j][1]
                    res_code = c321[geo[curr_chain].residues[curr_residue].name]
                except KeyError:
                    raise NameError(f'Alignment error: {record.id} {curr_chain} residue {curr_residue + 1} {letter}')
                if letter != res_code:
                    raise NameError(f'Alignment error: {record.id} res={i + 1} seq={letter} pdb={res_code}')
                curvature.append(geo[curr_chain].residues[curr_residue].curvature)
                torsion.append(geo[curr_chain].residues[curr_residue].torsion)
                arc_length.append(geo[curr_chain].residues[curr_residue].arc_len)
                writhing.append(geo[curr_chain].residues[curr_residue].writhing)
                phi.append(geo[curr_chain].residues[curr_residue].phi)
                psi.append(geo[curr_chain].residues[curr_residue].psi)
                j += 1
            else:
                curvature.append(0.0)
                torsion.append(0.0)
                arc_length.append(0.0)
                writhing.append(0.0)
                phi.append(0.0)
                psi.append(0.0)

        record.letter_annotations['curvature'] = curvature
        record.letter_annotations['torsion'] = torsion
        record.letter_annotations['arc_length'] = arc_length
        record.letter_annotations['writhing'] = writhing
        record.letter_annotations['phi'] = phi
        record.letter_annotations['psi'] = psi

    return align

def dataframe_from_alignment(align: Bio.Align.MultipleSeqAlignment, keys=None) -> pd.DataFrame:
    """
         Create a Pandas Dataframe from an annotated alignment
         :param align: Protein alignment
         :type align: Bio.Align.MultipleSeqAlignment
         :param keys: keys for the annotations
         :type keys: list
         :return: Pandas dataframe
         :rtype: pd.DataFrame
    """
    data = {}

    for record in align:
        if record.description[:9] == 'structure':
            data[f'seq_{record.id}'] = list(record.seq)
            if keys is None:
                items = record.letter_annotations.keys()
            else:
                items = keys
            for key in items:
                data[f'{key}_{record.id}'] = record.letter_annotations[key]

    return pd.DataFrame.from_dict(data)

class PropensityTable:
    """
    Class for the Protein Propensity Table data
    """
    __slots__ = ('__ini', '__end', '__lines', '__data')

    def __init__(self):
        """
        Load the Propensity Table data
        """

        def find_block():
            found = False
            while self.__ini < self.__end:
                if self.__lines[self.__ini][0] == '>':
                    found = True
                    break
                self.__ini += 1
            return found

        def load_block():
            tag = self.__lines[self.__ini].split()
            self.__ini += 1
            rows = []
            while self.__lines[self.__ini][0] != 'U':
                rows.append(self.__lines[self.__ini])
                self.__ini += 1
            rows.append(self.__lines[self.__ini])
            return int(tag[1]), rows

        stream = pkg_resources.resource_stream(__name__, 'data/luthier.dat')
        lines = stream.readlines()

        # Remove newlines
        for i, line in enumerate(lines):
            lines[i] = lines[i].decode('utf-8')
            if lines[i][-1] == '\n':
                lines[i] = lines[i][:-1]

        # TODO: trim data after use
        self.__ini = 0
        self.__end = len(lines)
        self.__lines = lines
        self.__data = {}

        while find_block():
            key, table = load_block()

            dct = {}
            head = table[0].split()
            for i in range(1, len(table)):
                row = table[i].split()
                key1 = row[0]
                for j in range(1, len(row)):
                    key2 = head[j]
                    key12 = f'{key1},{key2}'
                    value = int(row[j])
                    dct[key12] = value

            self.__data[key] = dct

    def get_score(self, target: str, residue: str, phi: float, psi: float) -> int:
        """
        Find anomalies in the protein's chain
        :param target: Target residue 1 letter code
        :type target: str
        :param residue: Residue 1 letter code
        :type residue: str
        :param phi: phi angle (degrees)
        :type phi: float
        :param psi: psi angle (degrees)
        :type psi: float
        :return: Score
        :rtype: int
        """
        max_phi = [0.0, 0.0, 0.0, -110.0, -110.0, 0.0, 140.0, 180.0, 180.0]
        min_phi = [-180.0, -110.0, -110.0, -180.0, -180.0, -180.0, 20.0, 0.0, 0.0]
        max_psi = [45.0, 180.0, -90.0, 180.0, -90.0, 100.0, 80.0, -40.0, 180.0]
        min_psi = [-90.0, 100.0, -180.0, 100.0, -180.0, 45.0, -40.0, -180.0, 80.0]

        tab_map = [0, 1, 1, 2, 2, 3, 4, 5, 5]

        score = 0
        for i in range(len(tab_map)):
            if (min_phi[i] <= phi < max_phi[i]) and (min_psi[i] <= psi < max_psi[i]):
                key = f'{target.upper()},{residue.upper()}'
                score = self.__data[3][key]
                break

        return score

def cluster_alignment(align: Bio.Align.MultipleSeqAlignment, threshold=0.7, long=False) -> None:
    """
         Cluster the alignments by structural similarity
         :param align: Protein alignment
         :type align: Bio.Align.MultipleSeqAlignment
         :param threshold: similarity threshold
         :type threshold: float
         :param long: output only long cluster (length > 3)
         :type long: bool
    """
    data = []
    id2pos = {}
    for position, record in enumerate(align):
        if 'structure' in record.description:
            id2pos[record.id] = position
            for pair in zip(record.letter_annotations['curvature'], record.letter_annotations['torsion']):
                data.append(list(pair))
            record.letter_annotations['cluster'] = [0 for _ in record.seq]

    scaler = StandardScaler()
    scaler.fit(data)

    clustering = AgglomerativeClustering(distance_threshold=threshold, n_clusters=None)
    for i in range(align.get_alignment_length()):
        xy = []
        tags = []
        for identity, position in id2pos.items():
            record = align[position]
            if record.seq[i] != '-':
                xy.append([record.letter_annotations['curvature'][i], record.letter_annotations['torsion'][i]])
                tags.append(identity)

        xy = scaler.transform(xy)

        clusters = clustering.fit_predict(xy)

        map_of_clusters = {pair[0]: pair[1] for pair in zip(tags, clusters)}

        for identity, position in id2pos.items():
            record = align[position]
            if identity in map_of_clusters:
                record.letter_annotations['cluster'][i] = map_of_clusters[identity]
                # print(id, record.letter_annotations['cluster'][i])

    last_cluster = 0
    for record in align:
        if 'structure' not in record.description:
            continue
        max_cluster = max(record.letter_annotations['cluster'])
        if max_cluster > last_cluster:
            last_cluster = max_cluster

    for i in range(align.get_alignment_length() - 1):
        j = i + 1
        left = {}
        right = {}
        for k, record in enumerate(align):
            if 'structure' not in record.description:
                continue

            ca = record.letter_annotations['cluster'][i]
            cb = record.letter_annotations['cluster'][j]

            if ca not in left:
                left[ca] = [k]
            else:
                left[ca].append(k)

            if cb not in right:
                right[cb] = [k]
            else:
                right[cb].append(k)

        for key, value in left.items():
            left[key] = set(value)

        for key, value in right.items():
            right[key] = set(value)

        for right_key in right:
            found_key = None
            for left_key in left:
                if right[right_key].symmetric_difference(left[left_key]) == set():
                    found_key = left_key

            if found_key is None:
                last_cluster += 1
                for k in right[right_key]:
                    record = align[k]
                    record.letter_annotations['cluster'][j] = last_cluster
            else:
                for k in right[right_key]:
                    record = align[k]
                    record.letter_annotations['cluster'][j] = found_key

    data, idx = get_idx(align)

    if long:
        last_cluster = 0
        for j in idx:
            cluster, ini, end, size = data[j]
            if size < 3:
                for i in range(ini, end + 1):
                    for record in align:
                        if 'structure' not in record.description:
                            continue
                        if record.letter_annotations['cluster'][i] == cluster:
                            record.letter_annotations['cluster'][i] = -1
                data[j] = (-1, ini, end, size)
            else:
                # print(f'{ini}-{end} {cluster}->{p}')
                for i in range(ini, end + 1):
                    for record in align:
                        if 'structure' not in record.description:
                            continue
                        if record.letter_annotations['cluster'][i] == cluster:
                            record.letter_annotations['cluster'][i] = last_cluster

                data[j] = (last_cluster, ini, end, size)
                last_cluster += 1
    return

def save_pymol_script(align: Bio.Align.MultipleSeqAlignment, pml_file: str, palette='Dark2', colors=7) -> None:
    """
         Cluster the alignments by structural similarity
         :param align: Protein alignment
         :type align: Bio.Align.MultipleSeqAlignment
         :param pml_file: Pymol script file name (pml)
         :type pml_file: str
         :param palette: colour palette
         :type palette: str
         :param colors: number of colours in the palette
         :type colors: int
    """
    data, idx = get_idx(align)

    tags = []
    pal = sns.color_palette(palette, colors).as_hex()
    with open(f'{pml_file}.pml', 'w') as f:
        f.write('# Script generated by Melodia\n\n')
        f.write('# load structures\n')
        for record in align:
            if 'structure' not in record.description:
                continue
            tags.append(record.id)
            f.write(f'load {record.id}.pdb\n')
        f.write('\n')

        f.write('# superimpose structures\n')
        for i in range(1, len(tags)):
            f.write(f'super {tags[i]}, {tags[0]}\n')
        f.write('\n')

        f.write('# non-conserved cluster color\n')
        f.write(f'color gray40\n\n')
        f.write('# cluster colors\n')
        for i in idx:
            positions = data[i]
            cluster, ini, end, size = positions
            if cluster >= 0:
                for record in align:
                    if 'structure' not in record.description:
                        continue
                    if record.letter_annotations['cluster'][ini] == cluster:
                        colour = f'0x{pal[cluster % colors][1:]}'
                        f.write(f'color {colour}, {record.id} and resi {ini + 1}-{end + 1}\n')
                f.write('\n')
    return

def get_idx(align: Bio.Align.MultipleSeqAlignment) -> Tuple[List[Tuple[Any, int, int, int]], List[SupportsIndex]]:
    """
    Internal function for clustering
    :param align: Protein alignment
    :return: Indexed clusters
    """
    clusters = {}
    for i, record in enumerate(align):
        if 'structure' not in record.description:
            continue
        for j, cluster in enumerate(record.letter_annotations['cluster']):
            if cluster not in clusters:
                clusters[cluster] = [j]
            else:
                clusters[cluster].append(j)
    data = []
    block_init = []
    for key, value in clusters.items():
        data.append((key, min(value), max(value), max(value) - min(value) + 1))
        block_init.append(min(value))
    idx = sorted(range(len(block_init)), key=block_init.__getitem__)
    return data, idx

def save_align_to_ps(align: Bio.Align.MultipleSeqAlignment, ps_file: str, palette='Dark2', colors=7) -> None:
    """
         Cluster the alignments by structural similarity
         :param align: Protein alignment
         :type align: Bio.Align.MultipleSeqAlignment
         :param ps_file: post-script file name
         :type ps_file: str
         :param palette: colour palette
         :type palette: str
         :param colors: number of colours in the palette
         :type colors: int
    """

    pal = sns.color_palette(palette, colors)
    rgb = [f'{c[0]:4.2f} {c[1]:4.2f} {c[2]:4.2f}' for c in pal]

    black = '0.00 0.00 0.00'
    grey = '0.50 0.50 0.50'

    out_file = f'{ps_file}.ps'
    with open(out_file, 'w') as ps:
        ps.write('%%!PS-Adobe-3.0\n')
        ps.write('%%%%Pages: 1\n')
        ps.write('%%%%Creator: Choral 1.0\n')
        ps.write('%%%%CreationDate:\n')
        ps.write('%%%%EndComments\n')
        ps.write('%%%%Page: 1 1\n')
        ps.write('/Courier-Regular findfont  16.0 scalefont  setfont\n')
        ps.write('0.00 0.00 0.83 setrgbcolor\n')
        ps.write(f'72.0 735.0 moveto ({ps_file}) show\n')

        length = align.get_alignment_length()
        count = len(align)
        total = int(length / 50)
        block = count + 4
        blocks_per_page = int(76 / block) + 1

        page = 1
        line = 705.0
        blocks = 0
        position = 10

        for j in range(total + 1):
            ini = 0 + 50 * j
            end = 49 + 50 * j

            if end >= length:
                end = length - 1

            column = 203.0

            ps.write('/Courier-Regular findfont  8.0 scalefont  setfont\n')
            ps.write('0.00 0.00 0.83 setrgbcolor\n')

            for k in range(1, 6):
                if position < 100:
                    ps.write('%5.1f %5.1f moveto (%d) show\n' % (column, line, position))
                else:
                    ps.write('%5.1f %5.1f moveto (%d) show\n' % (column - 2.0, line, position))

                position += 10
                column += 80.0

                if position > length:
                    break

            i = 1
            for record in align:
                column = 52.0
                line -= 10.0
                ps.write('/Courier-Regular findfont  10.0 scalefont  setfont\n')
                ps.write('0.00 0.00 0.00 setrgbcolor\n')
                ps.write('%5.1f %5.1f moveto (%3d) show\n' % (column, line, i))

                column += 20.0

                ps.write('%5.1f %5.1f moveto (%s) show\n' % (column, line, record.id))

                column = 131.1
                bold = False
                last_color = black
                for cur_res in range(ini, end + 1):
                    if record.seq[cur_res] == '-':
                        if bold:
                            bold = False
                            ps.write('/Courier-Regular findfont  10.0 scalefont  setfont\n')
                        if last_color != black:
                            last_color = black
                            ps.write(f'{last_color} setrgbcolor\n')
                    else:
                        if 'structure' in record.description:
                            k = record.letter_annotations['cluster'][cur_res]
                            if k >= 0:
                                c = k % colors
                                if not bold:
                                    bold = True
                                    ps.write('/Courier-Bold findfont  10.0 scalefont  setfont\n')
                                if last_color != rgb[c]:
                                    last_color = rgb[c]
                                    ps.write(f'{last_color} setrgbcolor\n')
                            else:
                                if bold:
                                    bold = False
                                    ps.write('/Courier-Regular findfont  10.0 scalefont  setfont\n')
                                if last_color != grey:
                                    last_color = grey
                                    ps.write(f'{last_color} setrgbcolor\n')
                        else:
                            if bold:
                                bold = False
                                ps.write('/Courier-Regular findfont  10.0 scalefont  setfont\n')
                            if last_color != black:
                                last_color = black
                                ps.write(f'{last_color} setrgbcolor\n')
                    if 'structure' in record.description:
                        if record.letter_annotations['cluster'][cur_res] < 0:
                            ps.write(f'%5.1f %5.1f moveto (%c) show\n' % (column, line, record.seq[cur_res].lower()))
                        else:
                            ps.write(f'%5.1f %5.1f moveto (%c) show\n' % (column, line, record.seq[cur_res]))
                    else:
                        ps.write(f'%5.1f %5.1f moveto (%c) show\n' % (column, line, record.seq[cur_res].lower()))
                    column += 8.0
                i += 1

            line -= 20.0
            blocks += 1

            if blocks == blocks_per_page and j != total:
                blocks = 0
                line = 705.0
                page += 1

                ps.write('showpage\n')
                ps.write(f'%%%%Page: {page} {page}\n')
                ps.write('/Courier-Regular findfont  16.0 scalefont  setfont\n')
                ps.write('0.00 0.00 0.83 setrgbcolor\n')
                ps.write(f'72.0 735.0 moveto (out_file) show\n')

        ps.write('showpage\n')

    return


# Author: Anthony Kai Kwang Ma
# Email: anthonyma27@gmail.com, akma327@stanford.edu
# Date: 05/14/17
# gpcr_atomic_contact_fingerprint.py

import os
import sys
import json
import csv
import numpy as np
import re
from utils import *

USAGE_STR = """

# Purpose
# Input: GPCR-Contacts at atomic resolution for a set of receptors.
# Output: Fingerprint Flareplot format json for visualization.

# Usage
# python gpcr_contact_fingerprint.py <OUTPUT_DIR> <INPUT_FILE_1> <INPUT_FILE_2> ...

# Arguments
# <OUTPUT_DIR> Output directory for <interaction_type>.json
# <INPUT_DIR_1> Input contacts table.txt file for a particular receptor
# ...
# <INPUT_DIR_N>

# Example
OUTPUT_DIR="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-fingerprint/opioid"
INPUT_FILE_1="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRM_MOUSE_4DKL_A/OPRM_MOUSE_4DKL_A_table.txt"
INPUT_FILE_2="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRD_HUMAN_4N6H_A/OPRD_HUMAN_4N6H_A_table.txt"
INPUT_FILE_3="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRK_HUMAN_4DJH_A/OPRK_HUMAN_4DJH_A_table.txt"
INPUT_FILE_4="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/OPRX_HUMAN_4EA3_A/OPRX_HUMAN_4EA3_A_table.txt"
cd /scratch/PI/rondror/akma327/GPCRContacts/src/fingerprint
python gpcr_atomic_contact_fingerprint.py $OUTPUT_DIR $INPUT_FILE_1 $INPUT_FILE_2 $INPUT_FILE_3 $INPUT_FILE_4

OUTPUT_DIR="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-fingerprint/muscarinic"
INPUT_FILE_1="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM2_HUMAN_3UON_A/ACM2_HUMAN_3UON_A_table.txt"
INPUT_FILE_2="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM3_RAT_4DAJ_A/ACM3_RAT_4DAJ_A_table.txt"
INPUT_FILE_3="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM1_HUMAN_5CXV_A/ACM1_HUMAN_5CXV_A_table.txt"
INPUT_FILE_4="/scratch/PI/rondror/akma327/GPCRContacts/data/atomic-static-contacts/classA-gpcr-pdbs/ACM4_HUMAN_5DSG_A/ACM4_HUMAN_5DSG_A_table.txt"
cd /scratch/PI/rondror/akma327/GPCRContacts/src/fingerprint
python gpcr_atomic_contact_fingerprint.py $OUTPUT_DIR $INPUT_FILE_1 $INPUT_FILE_2 $INPUT_FILE_3 $INPUT_FILE_4

"""

K_MIN_ARG = 3

def sort_paths(INPUT_FILES):
	path_comp = [p.split("/") for p in INPUT_FILES]
	sorted_input_files = ["/".join(sp) for sp in sorted(path_comp, key=lambda x: x[-1])]
	labels = [sp[-1].strip("_table.txt") for sp in sorted(path_comp, key = lambda x: x[-1])]
	return sorted_input_files, labels

def get_edge_and_frameDict(sorted_input_files, labels):
	"""
		sorted_input_files: Tables containing the gpcrdb pairs and atomic resolution interaction data
		labels: GPCR Receptor name
	"""
	frameDict = {}
	edge_dict = {}

	# residue_to_atomic_contacts = {itype: {gpcrdb1:gpcrdb2: {receptor_index1: [list of atomic resolution contacts], ...}}}
	# {"hbbb": {3x53:3x54: {0:[ALA168-N:VAL169-N, ALA168-O:VAL169-N,...], 1: []}}}
	residue_to_atomic_contacts = {}
	for i in range(len(sorted_input_files)):
		p, label = sorted_input_files[i], labels[i]
		frameDict[i] = label

		f = open(p, 'r')
		for line in f:
			gpcrdb1, gpcrdb2, atom_list, itype = line.strip().split("\t")
			if(itype not in edge_dict):
				edge_dict[itype] = {}
			if((gpcrdb1, gpcrdb2) not in edge_dict[itype]):
				edge_dict[itype][(gpcrdb1, gpcrdb2)] = set()
			edge_dict[itype][(gpcrdb1, gpcrdb2)].add(i)

			### Atomic level contacts encoding into flareplot json
			gpcrdb_pair = gpcrdb1 + ":" + gpcrdb2
			if(itype not in residue_to_atomic_contacts):
				residue_to_atomic_contacts[itype] = {}
			if(gpcrdb_pair not in residue_to_atomic_contacts[itype]):
				residue_to_atomic_contacts[itype][gpcrdb_pair] = {}
			if(atom_list not in residue_to_atomic_contacts[itype][gpcrdb_pair]):
				residue_to_atomic_contacts[itype][gpcrdb_pair][i] = []
			residue_to_atomic_contacts[itype][gpcrdb_pair][i].append(atom_list)

	return edge_dict, frameDict, residue_to_atomic_contacts

def fingerprint(OUTPUT_DIR, INPUT_FILES):
	if not os.path.exists(OUTPUT_DIR):
		os.makedirs(OUTPUT_DIR)

	sorted_input_files, labels = sort_paths(INPUT_FILES)
	edge_dict, frameDict, residue_to_atomic_contacts = get_edge_and_frameDict(sorted_input_files, labels)
	for itype in edge_dict:
		out_path = OUTPUT_DIR + "/" + itype + ".json"
		edges = []
		for edge in edge_dict[itype]:
			name1, name2 = edge
			frames = sorted(list(edge_dict[itype][edge]))
			edges.append({"name1": name1, "name2": name2, "frames": frames})

		json_dict = partial_input
		json_dict["edges"] = edges
		json_dict["frameDict"] = frameDict

		### Atomic level contacts encoding into flareplot json
		json_dict['residue_to_atomic_contacts'] = residue_to_atomic_contacts[itype]

		with open(out_path, 'w') as outfile:
			json.dump(json_dict, outfile)

if __name__ == "__main__":
	if(len(sys.argv) < K_MIN_ARG):
		print(USAGE_STR)
		exit(0)
	OUTPUT_DIR = sys.argv[1]
	INPUT_FILES = sys.argv[2:]
	fingerprint(OUTPUT_DIR, INPUT_FILES)


import pandas as pd
from datetime import datetime
from sklearn.metrics import mean_absolute_error
from sklearn.linear_model import LinearRegression

stock = pd.read_csv("sphist.csv")
stock["Date"] = pd.to_datetime(stock["Date"])
boolean = stock["Date"] > datetime(year=2015, month=4, day=1)
stock = stock.sort_values(by="Date", ascending=True)
# print(stock.columns)
stock["day_5_price"] = stock.rolling(window=5,
                                     on="Date").mean().loc[:,"Close"]
stock["day_30_price"] = stock.rolling(window=30,
                                      on="Date").mean().loc[:,"Close"]
stock["day_365_price"] = stock.rolling(window=365,
                                       on="Date").mean().loc[:,"Close"]
stock["mean_ratio"] = stock["day_5_price"] / stock["day_365_price"]
stock["day_5_std"] = stock.rolling(window=5,
                                   on="Date").std().loc[:,"Close"]
stock["day_365_std"] = stock.rolling(window=5,
                                     on="Date").std().loc[:,"Close"]
stock["std_ratio"] = stock["day_5_std"] / stock["day_365_std"]

# print(stock["day_5"].head(7))
stock = stock.shift(periods=1)
# print(stock.head(7))
stock = stock[stock["Date"] > datetime(year=1951, month=1, day=3)]
stock = stock.dropna(axis=0).copy()
# print(stock['Close'].isnull().sum())
train = stock[stock["Date"] < datetime(year=2013, month=1, day=1)]
test = stock[stock["Date"] >= datetime(year=2013, month=1, day=1)]
# print(len(train), len(test))
model = LinearRegression()
features = train.drop(['Close', 'High', 'Low', 'Open', 'Volume',
                    'Adj Close', 'Date'], axis=1).copy()
target = train["Close"]
model.fit(features, target)
test_features = test.drop(['Close', 'High', 'Low', 'Open', 'Volume',
                    'Adj Close', 'Date'], axis=1).copy()
predictions = model.predict(test_features)
mae = mean_absolute_error(test["Close"], predictions)
print(len(test['Close']))
print(len(predictions))
print(test.columns)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Feb 15 10:23:37 2019

@author: thomas
"""

import numpy as np
import matplotlib.pyplot as plt

# function to create the uniform distribution
def uniform(x,a,b):

    f=(x>=a) * (x<=b)
    f=1.0/(b-a)*f
    return(f)

def gauss(x,mu,s):

    f=1.0/np.sqrt(2*np.pi)/s*np.exp(- np.power(x-mu,2)/2/np.power(s,2) )
    return(f)

x = np.arange(-5, 6, 0.01)
a=-2.0
b=+2.0
mu=1.0
s=2.0

fg=gauss(x,mu,s)
fs=uniform(x,a,b)

font = {'family' : 'serif',
        'weight' : 'normal',
        'size'   : 16}
plt.rc('font', **font)

# Plot in frequency domain
plt.figure(1)
plt.xlabel('$x$',**font)
plt.ylabel('Distribution')
plt.plot(x,abs(fg),label='Gaussian')
plt.plot(x,abs(fs),label='uniform')
plt.xlim(-5.0,6.0)
plt.legend(bbox_to_anchor=(1.3, 1.05))
plt.tight_layout()
plt.savefig('pdfs.png')


import wikipedia
from collections import Counter
from nltk import word_tokenize
from nltk import sent_tokenize
from nltk import pos_tag
from nltk.corpus import wordnet
from nltk.wsd import lesk

def noun_finder(content):
    '''
    This function finds all the nouns from a Wikipedia page and returns them
    in a list along with the corresponding sentence.
    '''
    sentence_list = []
    noun_list = []
    sent_tokens = sent_tokenize(content)
    for elem in sent_tokens:
        sentence_list.append(word_tokenize(elem))
    for sent in sentence_list:
        tagged_sent = pos_tag(sent)
        for word in tagged_sent:
            if word[1][:2] == 'NN':
                noun_list.append([word[0], sent])
    return noun_list

def synsets(noun_list):
    '''
    Finds all corresponding synsets and their senses to the nouns, tracks the amount of nouns and
    whether they are unisemous or polysemous.
    '''
    noun_count = 0
    ps_words = 0
    us_words = 0
    total_ps_senses = 0
    words_10 = 0
    nouns_list = []
    full_nouns_list = []
    synset_list, sense_list, diff_senses = [], [], []
    for noun in noun_list:
        synsets = wordnet.synsets(noun[0], 'n')
        synset_list.append(synsets)
        if len(synsets) == 1:
            sense_list.append(len(synsets))
            noun_count += 1
            us_words += 1
        elif len(synsets) > 1:
            sense_list.append(len(synsets))
            noun_count += 1
            total_ps_senses += len(synsets)
            ps_words += 1
            if words_10 < 10:
                nouns_list.append(noun)
                words_10 += 1
            if words_10 != 'loop':
                full_nouns_list.append(noun)
    return synset_list, nouns_list, noun_count, us_words, ps_words, total_ps_senses, sense_list, full_nouns_list

def count_nouns(full_nouns_list):
    noun_definitions = {}
    for noun in full_nouns_list:
        if noun[0] in noun_definitions.keys():
            noun_definitions[noun[0]].append(noun[1])
        else:
            noun_definitions[noun[0]] = [noun[1]]
    for key, values in noun_definitions.items():
        if len(values) > 1:
            for value in values:
                print(key, " : ", lesk(value, key).definition())
            print("\n")

def main():
    wiki = wikipedia.page('Colonel Sanders')
    content = wiki.content
    noun_list = noun_finder(content)
    synset_list, first_nouns, noun_count, us_words, ps_words, total_ps_senses, sense_list, full_nouns_list = synsets(noun_list)
    print('1.\nThere were a total of {0} nouns in the text. {1} of those were unisemous and {2} were polysemous. '\
          'The proportion of polysemous nouns is {3:.2f}.\n'.format(noun_count, us_words, ps_words, ps_words/noun_count))
    if ps_words == 1:
        print('2.\nThis page contains a polysemous word.\n')
    elif ps_words == 0:
        print('2.\nThis page does not contain any polysemous words.\n')
    elif ps_words > 1:
        print('2.\nThis page contains multiple polysemous words.\n')
    print('3.\nThe average number of senses per polysemous words is {0:.2f}.\n'.format(total_ps_senses/ps_words))
    print('4.\nCount for different number of senses (format: nr of senses:count):\n')
    print(Counter(sense_list))
    print('\n5.')
    for noun in first_nouns:
        print(noun[0], " : ", lesk(noun[1], noun[0]).definition())
    print("\nIn this case it get about 2/3 words right. Especially words that are very context specific like chain and David"\
    " seem to be difficult, however it does get some tricky cases right like the word chicken. In this case it was"\
    " indeed meant as the food chicken and not the animal itself.\n")
    print("6.\nNo, this did not happen in question 5. However, it can happen, depending on the sentence that belongs to the word. If the word around the main word"\
          " differs too much in context, the wrong definition will be found. If you look at the same words over the entire Wikipedia page,"\
          " there are a lot of words that do differ in definition. The function count_nouns can be used to look at all the nouns that have"\
          " mulitple occurances and their definitions. Some of these definitions differ.\n")
    print("7.\nI did expect that some words would be correct, when they were not. For instance 'David' is placed in the correct context"\
          " and was still given a wrong definition.\n")
    print("8.\nDisambiguation can help assign the right meaning of a word to that word. This can help link to the right Wikipedia"\
    " page so that for example the sentence \"I am going to school\" will link to the institute and not to the page related to fish.")

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn import tree
from sklearn.datasets import load_digits

dataDg = load_digits()
# print(dir(dataDg))
# print(len(dataDg['target']))

dfDg = pd.DataFrame(
    dataDg['data'],
    columns=np.arange(len(dataDg['data'][0]))
)
dfDg['target']=dataDg['target']
# print(dfDg)

letarget = LabelEncoder()
dfDg['letarget'] = letarget.fit_transform(dfDg['target'])

x = dfDg.drop(['target','letarget'],axis=1)
y = dfDg['letarget']
model = tree.DecisionTreeClassifier()
model.fit(x,y)

# print(x.iloc[0])
# print(y.iloc[0])
# tree.export_graphviz(
#     model.fit(x,y),
#     out_file='digits.dot',
#     feature_names=x.iloc[0],
#     class_names=y.iloc[0]
# )


import cv2 as cv
import numpy as np
import math

def find_contours(img_frame):
    """
    find and draw contours on the main frame
    :param img_frame: a frame from the video
    :return: image with contours and the contours matrix
    """
    img = cv.cvtColor(img_frame, cv.COLOR_BGR2GRAY)
    frame_thresh = cv.adaptiveThreshold(img,255,cv.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv.THRESH_BINARY,11,2)
    contours, _ = cv.findContours(frame_thresh, cv.RETR_TREE, cv.CHAIN_APPROX_SIMPLE)
    return contours

def draw_grid(grid_shape, scale, color=(255, 255, 255), thickness=1):
    img = np.zeros((scale[1],scale[0], 1), dtype=np.uint8)
    h, w, _ = img.shape
    cols, rows = grid_shape
    dy, dx = h / rows, w / cols

    # draw vertical lines
    for x in np.linspace(start=dx, stop=w-dx, num=cols-1):
        x = int(round(x))
        cv.line(img, (x, 0), (x, h), color=color, thickness=thickness)

    # draw horizontal lines
    for y in np.linspace(start=dy, stop=h-dy, num=rows-1):
        y = int(round(y))
        cv.line(img, (0, y), (w, y), color=color, thickness=thickness)

    return img

def return_grid(grid = (2, 2), pos = (0, 0), res = (640, 640)):
    """ get grid in which (pos_x, pos_y) is in
    :param (cols, rows) number of grid cells (pos_x,pos_y) position to identify. (height, width) = resolution
    :return: matrix identifying the position of the point
    """

    w , h = res[1],res[0]
    cols, rows = grid[0],grid[1]
    x, y = pos[0],pos[1]

    cell_h, cell_w = h/rows, w/cols

    #grid_array = np.zeros((cols,rows), dtype=int)
    x, y = (int)(x/cell_w), (int)(y/cell_h)
    #grid_array[x, y] = 1
    return (x,y)

def distFromCell(pos, res, cell, grid):
    w , h = res[1],res[0]
    cols, rows = grid[0],grid[1]

    cell_h, cell_w = h/rows, w/cols
    cell_pos = (cell_w*cell[0], cell_h*cell[1])

    return (cell_pos[0] - pos[0],cell_pos[1] - pos[1])

def angle_of_vectors(vec1,vec2):
    a,b = vec1
    c,d = vec2
    dotProduct = a*c + b*d
    modOfVector1 = math.sqrt( a*a + b*b)*math.sqrt(c*c + d*d)
    angle = dotProduct/modOfVector1
    angleInDegree = math.degrees(math.acos(angle))*(vec1[1]/abs(vec1[1]))
    if (angle == 1.0):
        return 0
    return angleInDegree

def get_tag_orientation(img_frame):
    """ get orientation from the image frame
    :param img_frame: image frame from the video
    :return: orientation of the tag
    edges = [(corners[1][0]-corners[0][0], corners[1][1]-corners[0][1]),
            (corners[2][0]-corners[1][0], corners[2][1]-corners[1][1]),
            (corners[2][0]-corners[3][0], corners[2][1]-corners[3][1]),
            (corners[3][0]-corners[0][0], corners[3][1]-corners[0][1])]

    """
    corners = [(100, 100), (300, 100), (300, 300),  (100, 300)]
    # Check get_H_matrix function in superimpose for orientation notation
    orientations = {0: 0, 1: 0, 2: 0, 3: 0}
    # Orientation: North
    for i in range(100,300):
        for j in range(100,150):
            orientations[0] += img_frame[i, j]
    # Orientation: East
    for i in range(250,300):
        for j in range(100,300):
            orientations[1] += img_frame[i, j]
    # Orientation: South
    for i in range(100,300):
        for j in range(250, 300):
            orientations[2] += img_frame[i, j]
    # Orientation: West
    for i in range(100,150):
        for j in range(100,300):
            orientations[3] += img_frame[i, j]

    return min (orientations, key=orientations.get)

def get_tag_id(img_frame, orientation):
    """
    :param img_frame: current frame of the video
    :param orientation: orientation of the tag
    :return: tag ID
    """
    tag_id = ''
    keys = []
    # Check get_H_matrix function in superimpose.py for orientation notation
    if orientation == 0:
        keys = [1, 0, 2, 3]
    elif orientation == 1:
        keys = [3, 1, 0, 2]
    elif orientation == 2:
        keys = [2, 3, 1, 0]
    elif orientation == 3:
        keys = [0, 2, 3, 1]
    structure = {0: [200, 250, 200, 250], 1: [150, 200, 200, 250], 2: [200, 250, 150, 200], 3: [150, 200, 150, 200]}

    total = 0
    for key in keys:
        for i in range(structure[key][0], structure[key][1]):
            for j in range(structure[key][2], structure[key][3]):
                total += img_frame[i][j]

        if (total / 2500) > 220:
            tag_id += '1'
        else:
            tag_id += '0'
    return tag_id


from intermediateApi import ffi, lib

from c_types import SizeT, Void
from ua_types_parent import _ptr, _val, _is_null, _get_c_type, _is_ptr, UaType
from typing import Union, List

from ua_types_primitive import UaInt64, UaDouble, UaBoolean, UaString

class UaList(UaType):
    def __init__(self, val: Union[UaType, List] = None, size=None, ua_class=None, list_of_pointers=False):
        # TODO: implement default for list_of_pointers (e.g. primitives, NodesIds...)
        self._list_of_pointers = list_of_pointers
        c_type = ""
        if type(val) is list:
            if ua_class is None:
                if type(val[0]) is int:
                    c_type = "UA_Int64"
                    ua_class = UaInt64
                elif type(val[0]) is float:
                    c_type = "UA_Double"
                    ua_class = UaDouble
                elif type(val[0]) is str:
                    c_type = "UA_String"
                    ua_class = UaString
                    val = list(map(lambda string:
                                   UaString(string)._ptr if self._list_of_pointers else UaString(string)._val, val))
                elif type(val[0]) is bool:
                    c_type = "UA_Boolean"
                    ua_class = UaBoolean
            elif isinstance(ua_class(), UaType):
                c_type = _get_c_type(ua_class()._ptr)
            else:
                raise AttributeError("'ua_class' has to be None or a Subclass of UaType.")

            if self._list_of_pointers:
                c_type += "*"
            array = ffi.new(f"{c_type}[]", val)
            size = len(val)

        else:
            if isinstance(ua_class(), UaType):
                c_type = _get_c_type(ua_class()._ptr)
                if list_of_pointers:
                    c_type += "*"
            else:
                raise AttributeError("'ua_class' has do be None or a subclass of UaType.")

            if size is None:
                raise AttributeError("if 'val' is not a python list, 'size' has to be set.")
            elif isinstance(size, SizeT):
                size = size._val

            if isinstance(val, UaType):
                array = ffi.cast(f"{c_type}[{size}]", val._ptr)
            elif val is None:
                array = ffi.new(f"{c_type}[{size}]")
            else:
                array = ffi.cast(f"{c_type}[{size}]", val)

        super().__init__(val=array, is_pointer=True)
        self._size = size
        self._ua_type = ua_class
        self._c_type = c_type

    @property
    def _ptr(self):
        """Returns a pointer to the first array item. If list_of_pointers is True this is a **"""
        return self._value

    @property
    def ua_type(self):
        return self._ua_type

    @property
    def value(self):
        """Returns a python list of the array. If list_of_pointers is True this is a list of pointers."""
        return ffi.unpack(self._ptr, self._size)

    def __len__(self):
        return self._size

    def __setitem__(self, index: int, value):
        if 0 > index or index >= self._size:
            raise KeyError("index out of bound")
        if isinstance(value, UaType):
            self._ptr[index] = value._ptr if self._list_of_pointers else value._val

    def __getitem__(self, index):
        if isinstance(index, int):
            if 0 > index or index > self._size:
                raise KeyError("index out of bound")
            return self._ua_type(val=self.value[index], is_pointer=self._list_of_pointers)
        elif isinstance(index, slice):
            UaList(self.value[index], ua_class=self._ua_type)
        else:
            raise AttributeError("invalid indexing")

    def __str__(self, n=None):
        return str(self.value) + ("" if n is None else "\n")


"""
Comparison of virtual MWI observation with estimates based on a top-hat SRF
and selected frequencies.
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import xarray as xr
from string import ascii_lowercase as abc
import os
from helpers import mwi, wyo, colors
from dotenv import load_dotenv

load_dotenv()
plt.ion()

if __name__ == '__main__':

    # read data
    ds_com_rsd = xr.open_dataset(os.path.join(
        os.environ['PATH_BRT'],
        'TB_radiosondes_2019_MWI.nc'))

    ds_com_era = xr.open_dataset(os.path.join(
        os.environ['PATH_BRT'],
        'TB_era5_MWI.nc'))
    ds_com_era = ds_com_era.stack({'profile': ('grid_x', 'grid_y')})

    ds_com_erh = xr.open_dataset(os.path.join(
        os.environ['PATH_BRT'],
        'TB_era5_hyd_MWI.nc'))
    ds_com_erh = ds_com_erh.stack({'profile': ('grid_x', 'grid_y')})

    c_list = [colors.colors_rs[p] for p in ds_com_rsd.station.values]

    #%% print some statistics
    for channel in ds_com_rsd.channel:
        for est_type in ds_com_rsd.est_type:
            s = dict(channel=channel, est_type=est_type)

            dtb_max_rsd = np.round(np.abs(ds_com_rsd.dtb_mwi_est)
                                   .max('profile').sel(**s).item(), 1)

            dtb_max_era = np.round(np.abs(ds_com_era.dtb_mwi_est)
                                   .max('profile').sel(**s).item(), 1)

            dtb_max_erh = np.round(np.abs(ds_com_erh.dtb_mwi_est)
                                   .max('profile').sel(**s).item(), 1)

            print(f'channel {channel.item()}, est_type {est_type.item()}: '+
                  f'{dtb_max_rsd} | {dtb_max_era} | {dtb_max_erh}')

    print(np.abs(ds_com_rsd.dtb_mwi_est).max(['profile', 'channel']))
    print(np.abs(ds_com_era.dtb_mwi_est).max(['profile', 'channel']))
    print(np.abs(ds_com_erh.dtb_mwi_est).max(['profile', 'channel']))

    print(ds_com_rsd.dtb_mwi_est.min(['profile', 'channel', 'est_type']).item())
    print(ds_com_era.dtb_mwi_est.min(['profile', 'channel', 'est_type']).item())
    print(ds_com_erh.dtb_mwi_est.min(['profile', 'channel', 'est_type']).item())

    print(ds_com_rsd.dtb_mwi_est.max(['profile', 'channel', 'est_type']).item())
    print(ds_com_era.dtb_mwi_est.max(['profile', 'channel', 'est_type']).item())
    print(ds_com_erh.dtb_mwi_est.max(['profile', 'channel', 'est_type']).item())

    #%% scatter plot and histogram
    # x: TB obs or IWV
    # y: TB obs - TB mod

    # choose x axis style
    styles = ['tb_obs', 'iwv']
    style = styles[0]

    # choose if with or without era-5
    era5 = False  # this is just as a test to see if clear sky matches

    fig, axes = plt.subplots(5, 4, figsize=(6, 7), sharex=True, sharey=True,
                             constrained_layout=True)

    axes[-1, 0].set_ylabel('$\Delta$TB [K]')

    if style == 'tb_obs':
        axes[-1, 0].set_xlabel('$TB_{obs}$ [K]')
    elif style == 'iwv':
        axes[-1, 0].set_xlabel('IWV [kg m$^{-2}$]')

    kwargs = dict(xy=(0.5, 1), xycoords='axes fraction', ha='center',
                  va='bottom')
    axes[0, 0].annotate('center', **kwargs)
    axes[0, 1].annotate('cutoff', **kwargs)
    axes[0, 2].annotate('center+cutoff', **kwargs)
    axes[0, 3].annotate('top-hat', **kwargs)

    for i, ax in enumerate(axes.flatten('F')):

        ax.spines.top.set_visible(False)
        ax.spines.right.set_visible(False)

        ax.annotate(f'({abc[i]})', xy=(0.02, 0.98), xycoords='axes fraction',
                    ha='left', va='top')

        # x axis
        ax.set_xticks(ticks=np.arange(0, 400, 25), minor=False)
        ax.set_xticks(ticks=np.arange(0, 400, 5), minor=True)

        # y axis
        ax.set_yticks(ticks=np.arange(-2, 2, 0.5), minor=False)
        ax.set_yticks(ticks=np.arange(-2, 2, 0.1), minor=True)

        ax.tick_params(which='major')
        ax.tick_params(which='minor', length=1)

        ax.axhline(y=0, color='k', linewidth=1)

        ax.set_ylim([-0.9, 1.4])

        if style == 'tb_obs':
            ax.set_xlim([235, 290])
        elif style == 'iwv':
            ax.set_xlim([0, 75])

    # annotate channel name
    for i, ax in enumerate(axes[:, -1]):
        ax.annotate(text=mwi.freq_txt[i].split('\n')[0], xy=(1.1, 0.5),
                    xycoords='axes fraction', ha='left', va='center',
                    rotation=90)

    for i, channel in enumerate(mwi.channels_int):
        for j, est_type in enumerate(['freq_center', 'freq_bw',
                                      'freq_bw_center', 'top-hat']):

            if style == 'tb_obs':
                x = ds_com_rsd.tb_mwi_orig.sel(channel=channel)
            elif style == 'iwv':
                x = ds_com_rsd.iwv

            # radiosondes
            axes[i, j].scatter(x,
                               ds_com_rsd.dtb_mwi_est.sel(channel=channel,
                                                          est_type=est_type),
                               s=2, c=c_list, linewidths=0)

            # annotate mean bias
            mean = ds_com_rsd.dtb_mwi_est.sel(
                channel=channel,
                est_type=est_type).mean('profile').item()
            mean = np.round(mean, 2)
            axes[i, j].annotate(f'{mean} K', xy=(1, 0),
                                xycoords='axes fraction', ha='right',
                                va='bottom')

            if era5:
                if style == 'tb_obs':
                    x = ds_com_era.tb_mwi_orig.sel(channel=channel)
                elif style == 'iwv':
                    x = ds_com_era.iwv

                # era5 without hydrometeors
                axes[i, j].scatter(x,
                                   ds_com_era.dtb_mwi_est.sel(channel=channel,
                                                              est_type=est_type),
                                   s=2, c='k', linewidths=0)

                # era5 with hydrometeors
                axes[i, j].scatter(x,
                                   ds_com_erh.dtb_mwi_est.sel(channel=channel,
                                                              est_type=est_type),
                                   s=2, c='gray', linewidths=0, alpha=0.01)

    # legend below
    patches = []
    stations = wyo.station_id.keys()
    for station in stations:
        patches.append(mpatches.Patch(color=colors.colors_rs[station],
                                      label=station))
    axes[0, 0].legend(handles=patches[:2], frameon=False, fontsize=7)
    axes[0, 1].legend(handles=patches[2:], frameon=False, fontsize=7)

    if era5:
        ext = '_era5'
    else:
        ext = ''

    plt.savefig(os.path.join(
        os.environ['PATH_PLT'],
        'evaluation_dtb_mwi_est_vs_'+style+'_radiosondes'+ext+'.png'),
        dpi=400, bbox_inches='tight')

    plt.close('all')

    #%% 2d histogram iwv and tb dependence
    # x: TB MWI
    # y: IWV
    # color: dTb
    fig, axes = plt.subplots(5, 4, figsize=(7, 6), sharex=True, sharey=True,
                             constrained_layout=True)

    axes[-1, 0].set_ylabel('$\Delta$TB [K]')

    axes[-1, 0].set_ylabel('$TB_{obs}$ [K]')
    axes[-1, 0].set_xlabel('IWV [kg m$^{-2}$]')

    kwargs = dict(xy=(0.5, 1), xycoords='axes fraction', ha='center',
                  va='bottom')
    axes[0, 0].annotate('center', **kwargs)
    axes[0, 1].annotate('cutoff', **kwargs)
    axes[0, 2].annotate('center+cutoff', **kwargs)
    axes[0, 3].annotate('top-hat', **kwargs)

    for i, ax in enumerate(axes.flatten('F')):

        ax.spines.top.set_visible(False)
        ax.spines.right.set_visible(False)

        ax.annotate(f'({abc[i]})', xy=(0.02, 0.98), xycoords='axes fraction',
                    ha='left', va='top')

        # x axis
        ax.set_xticks(ticks=np.arange(0, 400, 25), minor=False)
        ax.set_xticks(ticks=np.arange(0, 400, 5), minor=True)

        # y axis
        ax.set_xticks(ticks=np.arange(0, 400, 25), minor=False)
        ax.set_xticks(ticks=np.arange(0, 400, 5), minor=True)

        ax.tick_params(which='major')
        ax.tick_params(which='minor', length=1)

        ax.axhline(y=0, color='k', linewidth=1)

        ax.set_ylim([235, 290])
        ax.set_xlim([0, 75])

    # annotate channel name
    for i, ax in enumerate(axes[:, -1]):
        ax.annotate(text=mwi.freq_txt[i].split('\n')[0], xy=(1.1, 0.5),
                    xycoords='axes fraction', ha='left', va='center',
                    rotation=90)

    for i, channel in enumerate(mwi.channels_int):
        for j, est_type in enumerate(['freq_center', 'freq_bw',
                                      'freq_bw_center', 'top-hat']):

            c_max = np.max(np.fabs(ds_com_rsd.dtb_mwi_est.sel(channel=channel,
                                         est_type=est_type)))

            im = axes[i, j].scatter(
                ds_com_rsd.iwv,
                ds_com_rsd.tb_mwi_orig.sel(channel=channel),
                c=ds_com_rsd.dtb_mwi_est.sel(channel=channel,
                                             est_type=est_type),
                               cmap='BrBG_r',
                               s=2, linewidths=0, vmin=-c_max, vmax=c_max)
            fig.colorbar(im, ax=axes[i, j], orientation='horizontal')

    plt.savefig(os.path.join(
        os.environ['PATH_PLT'],
        'evaluation_dtb_mwi_est_vs_tb_obs_vs_iwv_radiosondes.png'),
        dpi=400, bbox_inches='tight')

    plt.close('all')


#following this tutorial: https://pythonprogramming.net/pygame-drawing-shapes-objects/?completed=/adding-score-pygame-video-game/

#imports
import pygame

#initialize pygame
pygame.init()

#make rgb colors
white = (255,255,255)
black = (0,0,0)

red = (255,0,0)
green = (0,255,0)
blue = (0,0,255)

#set up display
gameDisplay = pygame.display.set_mode((800,600))
gameDisplay.fill(black)

#change a single pixel array (located at 10,20) to the color green
pixAr = pygame.PixelArray(gameDisplay)
pixAr[10][20] = green

#draw a blue line on game display starting at (100,200) and ending at (300,450) with a width of 5
pygame.draw.line(gameDisplay, blue, (100,200), (300,450), 5)

#draw a red filled in rectangle with the corner coordinates:
#400 (top right x)
#400 (top right y)
#50 (width)
#25 (height)
pygame.draw.rect(gameDisplay,red,(400,400,50,25))

#draw white circle with top right coordinates (150,150) and center point at 75
pygame.draw.circle(gameDisplay,white,(150,150),75)

#draw green polygon with specified points
pygame.draw.polygon(gameDisplay,green, ((25,75),(76,125),(250,375),(400,25),(60,540)))

#main loop
while True:
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			pygame.quit()
			quit()
	pygame.display.update()


from pyspark import SparkConf
from pyspark.sql import SparkSession, functions

from clustering import get_clusters_for_partition

base_data_dir = "/Users/timor/projects/data/amazonlab/nasa_fire_info/BOLIVIA_DL_FIRE_M6_5242/"
csvInPath = base_data_dir + "fire_archive_M6_5242.csv"
parquetOutPath = base_data_dir + "fire_archive_M6_5242_clustered_eps005_s4.parquet"

spark_conf = SparkConf()
spark_conf.set('spark.executor.memory', '4g')
spark_conf.setAppName("furry-lab")
spark_conf.setMaster("local[*]")
spark = SparkSession.builder.config(conf=spark_conf).getOrCreate()

csv_options = {"delimiter": ",", "header": "true", "inferSchema": "true"}
# add grouping week with year
ds = spark.read.format("csv").options(**csv_options).load(csvInPath) \
  .withColumn("week", functions.concat(functions.year("acq_date"), functions.lit("_"), functions.weekofyear("acq_date")))

week_ds = ds.groupBy("week").agg(
    functions.collect_list(functions.struct(
        "latitude",
        "longitude",
        "brightness",
        "acq_date",
        "confidence",
        "bright_t31",
        "week"
    )).alias("events")
).sort("week")

clustered_ds = week_ds.rdd.mapPartitions(get_clusters_for_partition).toDF().cache()
clustered_ds.count()
clustered_ds.coalesce(1).write.parquet(parquetOutPath)


# -*- coding:utf-8 -*-
# ！/usr/bin/env python
# @Time : 2023/5/26 10:24
# @Author : waxberry
# @File : dubbo_api.py
# @Software : PyCharm

import json
import telnetlib

from PyQt5.QtCore.QUrl import port

class Dubbo(telnetlib.Telnet):
    prompt = 'dubbo>'

    def __init__(self, host=None, port=0):
        super().__init__(host, port)
        self.write(b'\n')

    def command(self, flag, str_=''):
        data = self.read_until(flag.encode())
        self.write(str_.encode() + b'\n')
        return data

    def invoke(self, service_name, method_name):
        command_str = 'invoke {0}.{1}()'.format(service_name, method_name)
        self.command(Dubbo.prompt, command_str)
        data = self.command(Dubbo.prompt, '')
        data = json.loads(data.decode(Dubbo.coding, errors='ignore').split('\n')[0].strip())

if __name__ == '__main__':
    conn = Dubbo('serviceIp', port)
    result = conn.invoke(
        'serviceName',
        'serviceName.method'
    )
    print(result)


# Ke Yan, Imaging Biomarkers and Computer-Aided Diagnosis Laboratory,
# National Institutes of Health Clinical Center, July 2019
"""The truncated Densenet-121 with FPN and 3DCE"""
from collections import namedtuple

import torch
import torch.nn.functional as F
from torch import nn
import torchvision.models as models
from torchvision.models.densenet import _DenseBlock, _Transition, model_urls
import math
import torch.utils.model_zoo as model_zoo
import re
from collections import OrderedDict

from maskrcnn.config import cfg

class DenseNetCustomTrunc(nn.Module):
    """The truncated Densenet-121 with FPN and 3DCE"""
    # truncated since transition layer 3 since we find it works better in DeepLesion
    # We only keep the finest-level feature map after FPN
    def __init__(self):
        super(DenseNetCustomTrunc, self).__init__()
        name = cfg.MODEL.BACKBONE.CONV_BODY
        self.depth = int(name.split('-')[1])
        self.feature_upsample = cfg.MODEL.BACKBONE.FEATURE_UPSAMPLE

        assert self.depth in [121]
        if self.depth == 121:
            num_init_features = 64
            growth_rate = 32
            block_config = (6, 12, 24)
            self.in_dim = [64, 256, 512, 1024]
        bn_size = 4
        drop_rate = 0

        # First convolution
        self.conv0 = nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)
        self.norm0 = nn.BatchNorm2d(num_init_features)
        self.relu0 = nn.ReLU(inplace=True)
        self.pool0 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        # Each denseblock
        num_features = num_init_features
        for i, num_layers in enumerate(block_config):
            block = _DenseBlock(num_layers=num_layers, num_input_features=num_features,
                                bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate)
            self.add_module('denseblock%d' % (i + 1), block)
            num_features = num_features + num_layers * growth_rate
            if i != len(block_config) - 1:
                trans = _Transition(num_input_features=num_features, num_output_features=num_features // 2)
                self.add_module('transition%d' % (i + 1), trans)
                num_features = num_features // 2
        cfg.runtime_info.backbone_ft_dim = self.in_dim[-1]

        # Final batch norm
        # self.add_module('norm5', nn.BatchNorm2d(num_features))

        # Official init from torch repo.
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight.data)
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

        if self.feature_upsample:
            self.out_dim = cfg.MODEL.BACKBONE.OUT_CHANNELS
            self.fpn_finest_layer = cfg.MODEL.BACKBONE.FEATURE_UPSAMPLE_LEVEL-1
            for p in range(4, self.fpn_finest_layer - 1, -1):
                layer = nn.Conv2d(self.in_dim[p - 1], self.out_dim, 1)
                name = 'lateral%d' % p
                self.add_module(name, layer)

                nn.init.kaiming_uniform_(layer.weight, a=1)
                nn.init.constant_(layer.bias, 0)
            cfg.runtime_info.backbone_ft_dim = self.out_dim

        self.indim_ilf = [64, 128, 256]
        self.num_image = cfg.INPUT.NUM_IMAGES_3DCE
        self.feature_fusion_level_list = cfg.MODEL.BACKBONE.FEATURE_FUSION_LEVELS
        for p in range(len(self.feature_fusion_level_list)):
            if self.feature_fusion_level_list[p]:
                layer = nn.Conv2d(self.num_image * self.indim_ilf[p], self.indim_ilf[p], 1)
                self.add_module('conv_ilf%d'%p, layer)

    def inter_layer_fuse(self, ft, level):
        """Improved 3DCE, 3D feature fusion"""
        if self.feature_fusion_level_list[level]:
            ft_ilf = ft.reshape(-1, self.num_image * self.indim_ilf[level], ft.shape[2], ft.shape[3])
            ft_ilf = getattr(self, 'conv_ilf%d'%level)(ft_ilf)
            ft_out = ft.clone()
            ft_out[int(self.num_image/2)::self.num_image] = ft_ilf
            return ft_out
        else:
            return ft

    def forward(self, x):
        x = self.conv0(x)
        x = self.norm0(x)
        relu0 = self.relu0(x)
        pool0 = self.pool0(relu0)
        pool0_ilf = self.inter_layer_fuse(pool0, 0)

        db1 = self.denseblock1(pool0_ilf)
        ts1 = self.transition1(db1)
        ts1_ilf = self.inter_layer_fuse(ts1, 1)

        db2 = self.denseblock2(ts1_ilf)
        ts2 = self.transition2(db2)
        ts2_ilf = self.inter_layer_fuse(ts2, 2)

        db3 = self.denseblock3(ts2_ilf)

        # truncated since here since we find it works better in DeepLesion
        # ts3 = self.transition3(db3)
        # db4 = self.denseblock4(ts3)

        if self.feature_upsample:
            ftmaps = [relu0, db1, db2, db3]
            x = self.lateral4(ftmaps[-1])
            for p in range(3, self.fpn_finest_layer - 1, -1):
                x = F.interpolate(x, scale_factor=2, mode="nearest")
                y = ftmaps[p-1]
                lateral = getattr(self, 'lateral%d' % p)(y)
                x += lateral
            return [x]
        else:
            return [db3]

    def load_pretrained_weights(self):
        pattern = re.compile(
            r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$')
        state_dict = model_zoo.load_url(model_urls['densenet121'])
        for key in list(state_dict.keys()):
            res = pattern.match(key)
            if res:
                new_key = res.group(1) + res.group(2)
                state_dict[new_key] = state_dict[key]
                del state_dict[key]
        state_dict1 = {}
        for key in list(state_dict.keys()):
            new_key = key.replace('features.', '')
            state_dict1[new_key] = state_dict[key]

        self.load_state_dict(state_dict1, strict=False)

    def freeze(self):
        for name, param in self.named_parameters():
            print('freezing', name)
            param.requires_grad = False


import csv
from datetime import datetime

def main():
    try:
        current_date_and_time = datetime.now()
        products = read_products("products.csv")
        print("GIANT EAGLE")
        print()
        num_and_price = process_requests("request.csv", products)
        print()
        print(f"Number of Items: {num_and_price[0]}")
        print(f"Subtotal: {num_and_price[1]}")
        sales_tax = .06
        sales_tax = round(num_and_price[1]*sales_tax,2)
        print(f"Sales Tax: {sales_tax}")
        total= sales_tax+num_and_price[1]
        print(f"Total: {total}")
        print()
        print("Thank you, come again!")
        print(f"{current_date_and_time:%a %b %d %I:%M.%S %Y}")

    except FileNotFoundError as file_not_found_err:
        print(f"Error: cannot open file"
                "because it doesn't exist.")
    except PermissionError as perm_err:
        print(f"Error: cannot read from file"
                "because you don't have permission.")
    except KeyError as key_err:
        print(type(key_err).__name__, key_err)

def read_products(filename):
    prod_dict ={}
    PRODUCT_INDEX= 0
    NAME_INDEX= 1
    PRICE_INDEX=2
    with open(filename, "rt") as product_file:
        reader =csv.reader(product_file)
        next(reader)

        for row in reader:
            prod_num = row[PRODUCT_INDEX]
            prod_name = row[NAME_INDEX]
            prod_price = float(row[PRICE_INDEX])

            prod_dict[prod_num] = [prod_name, prod_price]

    return prod_dict

def process_requests(filename, products):
    with open(filename, "rt") as request_file:
        reader = csv.reader(request_file)
        next(reader)
        quantity=0
        total=0

        for row in reader:
            product = row[0]
            new_quantity = int(row[1])

            if product in products:
                value = products[product]
                name_of_product = value[0]
                price_of_product= float(value[1])
                print(f"{name_of_product}: {new_quantity} @ {price_of_product}")
                quantity += new_quantity
                new_total = new_quantity * price_of_product
                total += new_total
            else:
                print("User odered non-existent item.")

    return  quantity, total

if __name__ == "__main__":
    main()


import adal
from urllib.parse import urljoin
# AADTokenCredentials for multi-factor authentication
from msrestazure.azure_active_directory import AADTokenCredentials

class Profile:
    """
    A Class representing user profile.
    """

    def __init__(self, client_id, tenant, resource, authority_url):
        self.client_id = client_id
        self.tenant = tenant
        self.resource = resource
        self.authority_url = authority_url

    def _get_authentication_context(self):
        auth_url = urljoin(self.authority_url, self.tenant)

        return adal.AuthenticationContext(
            authority=auth_url,
            api_version=None)

    def authenticate_device_code(self):
        """
        Authenticate the end-user using device auth.
        """
        context = self._get_authentication_context()

        code = context.acquire_user_code(
            resource=self.resource,
            client_id=self.client_id)

        print(code['message'])

        mgmt_token = context.acquire_token_with_device_code(
            resource=self.resource,
            user_code_info=code,
            client_id=self.client_id)

        credentials = AADTokenCredentials(
            token=mgmt_token,
            client_id=self.client_id)

        return credentials.token


import os
import requests
from asana.util import get_value_in_data_file
from furl import furl

try:
    ASANA_TOKEN = os.environ['ASANA_TOKEN']
except:
    raise KeyError("Please set ASANA_TOKEN as an environment variable.")

URL = 'https://app.asana.com/api/1.0'
HEADERS = {
    'Accept': 'application/json',
    'Authorization': f'Bearer {ASANA_TOKEN}'}

class AsanaAPI:

    def __init__(self):
        self.url = URL
        self.headers = HEADERS
        self.workspace = get_value_in_data_file('workspace', 'gid')
        self.assignee = get_value_in_data_file('user', 'gid')

    def get_user_info(self):
        """
        Gets the current user of Asana's API.
        return dict: eg: {'gid': str, 'email': [users email]. 'name': [users first and last name]}
        """
        r = self._get('users/me')
        return r.json()['data']

    def get_user_workspaces(self):
        """
        Gets all workspaces visible to authorized user
        return [dict]: eg: {'gid': str, 'name': [name of workspace], 'resource_type': 'workspace'}
        """
        r = self._get('workspaces')
        return r.json()['data']

    def get_projects(self):
        """
        Gets all projects visible to authorized user
        return [dict]: eg: {'gid': str, 'name': [name of project], 'resource_type': 'project'}
        """
        r = self._get('projects')
        return r.json()['data']

    def get_tasks(self, workspace, **params):
        """
        Gets all tasks assigned to authorized user
        return [dict]: eg: {'gid': str, 'name': [name of task], 'resource_type': 'task'}
        """
        r = self._get(f'workspaces/{workspace}/tasks/search', **params)
        return r.json()['data']

    def get_project_tasks(self, project_gid):
        """
        Gets all tasks assigned to authorized user in given project
        return [dict]: eg: [{'gid': str, 'name': [name of task], 'resource_type': 'task'},]
        """
        r = self._get(f'projects/{project_gid}/tasks')
        return r.json()['data']

    def get_project_sections(self, project_gid):
        r = self._get(f'projects/{project_gid}/sections')
        return r.json()['data']

    def get_tasks_details(self, task_gid):
        """
        Gets additial detils on each task. ie: assignee, completed, etc.
        """
        r = self._get(f'tasks/{task_gid}')
        return r.json()['data']

    def update_task(self, task_gid, **kwargs):
        """
        Updates the given task from task_gid
        """
        r = self._put(f'tasks/{task_gid}', data=kwargs)
        return r.json()['data']

    def get_tasks_from_section(self, section_gid):
        """
        Gets all tasks from a given section
        :param section_gid: section id to get tasks in
        :return:
        """
        r = self._get(f'sections/{section_gid}/tasks')
        return r.json()['data']

    def add_task_to_section(self, section_gid, **kwargs):
        """
        Adds a task on the board in the specific section
        :param task_name: name of task
        :param section_gid: section id to add task in
        """
        r = self._post(f'/sections/{section_gid}/addTask', data=kwargs)
        return r.json()['data']

    def _get(self, endpoint: str, data: dict = None, **params):

        url = f'{self.url}/{endpoint}'
        if params:
            params = self.__manipulate_kwargs(params)
            url = self.__with_parameters(url, params)

        r = requests.get(
            url=url,
            headers=self.headers,
            data=data)
        if int(r.status_code) == 200 or int(r.status_code) == 201:
            return r
        # raise HTTPError(f'{r.status_code}: {r.content}')

    def _post(self, endpoint: str, data: dict):
        r = requests.post(
            url=f'{self.url}/{endpoint}',
            headers=HEADERS,
            data=data)
        if int(r.status_code) == 200 or int(r.status_code) == 201:
            return r
        # raise HTTPError(f'{r.status_code}: {r.content}')

    def _put(self, endpoint: str, data: dict):
        r = requests.put(
            url=f'{self.url}/{endpoint}',
            headers=HEADERS,
            data=data)
        if int(r.status_code) == 200 or int(r.status_code) == 201:
            return r
        # raise HTTPError(f'{r.status_code}: {r.content}')

    @staticmethod
    def __with_parameters(url, params):
        unfurled = furl(url)
        unfurled.add(query_params=params)
        return unfurled.url

    @staticmethod
    def __manipulate_kwargs(kwargs):
        if 'assignee' in kwargs.keys():
            kwargs['assignee.any'] = kwargs.pop('assignee')
        if 'sections' in kwargs.keys():
            kwargs['sections.any'] = kwargs.pop('sections')
        return kwargs


from pathlib import Path
from random import sample
from typing import List, Union
import shelve
import time
import wx
import wx.adv
import constants as C
import options_gui
from common import OptionsData, ResultsData
from data_gui import ResultsFrame

class GenLotteryMainFrame(options_gui.MainFrame):
    """ A Controller sub class of MainFrame
    """

    options_data = OptionsData()
    # This is the save file path name with no extension
    _saved_file: str = C.SAVE_FILE_DIR + options_data.get_lottery_name()

    def __init__(self, parent):
        options_gui.MainFrame.__init__(self, parent)
        self.update_status()

    def update_status(self, text=None) -> None:
        """Utility function used to update the status bar
        :param text: text to display
        :return: None
        """
        status_bar = self.GetStatusBar()
        if text is None:
            opt: OptionsData = self.options_data
            status_bar.SetStatusText(
                f"OPTIONS - Action: "
                f"{opt.get_option_name()}"
                f", Type: "
                f"{opt.get_lottery_name()}"
                f", Lines: "
                f"{opt.number_of_lines}"
            )
        else:
            status_bar.SetStatusText(text)

    def on_exit(self, event):
        """Close the frame and terminate the application."""
        self.Close(force=True)
        event.Skip()

    def on_about(self, _):
        about_info = wx.adv.AboutDialogInfo()
        about_info.SetName(C.PROGRAM)
        about_info.SetVersion(C.VERSION)
        about_info.SetDescription("Lottery numbers generator with wxPython")
        about_info.SetCopyright("(C) 2019-")
        about_info.AddDeveloper(C.AUTHOR)

        wx.adv.AboutBox(about_info)

    def on_type_combo(self, event):
        opt = self.options_data
        opt.lottery_type = event.GetSelection()
        opt.lottery_type_name = C.LOTTERY_CHOICES[opt.lottery_type]
        self._saved_file = C.SAVE_FILE_DIR + opt.lottery_type_name

        self.update_status()
        event.Skip()

    def on_line_control(self, event):
        opt = self.options_data
        opt.number_of_lines = event.GetValue()
        self.update_status()
        event.Skip()

    def on_radiobox(self, event):
        opt = self.options_data
        opt.option = event.GetSelection()
        opt.option_name = C.OPTIONS_CHOICES[opt.option]
        self.update_status()
        event.Skip()

    def on_cancel(self, event):
        self.on_exit(event)

    def on_ok(self, event):
        opt = self.options_data
        if opt.option == 3:
            self.delete_saved_file()
        elif opt.option == 2:
            self.show_saved()
        else:
            self.generate_numbers()
        event.Skip()

    def generate_numbers(self) -> None:
        """Generate several random numbers and optionally save them
        :return:
        """

        def add_leading_zero(values: []) -> []:
            """Add a leading zero to numbers in the list < 10
            :param values: list of numbers
            :return: array containing formatted numbers
            """
            return [f"{v:02d}" for v in values]

        def choose_numbers(maximum: int, quantity: int) -> []:
            """Generate the random numbers required
            :param maximum: the highest number to choose from
            :param quantity: the number of numbers to generate
            :return: a sorted list of generated numbers
            """
            valid_range: range = range(C.RULE_START, maximum)
            return add_leading_zero(sorted(sample(valid_range, quantity)))

        result = ResultsData
        result.clear_data()

        opt = self.options_data
        result.lottery_type_name = opt.get_lottery_name()
        result.number_of_lines = opt.number_of_lines

        main_max, main_qty,\
            extra_max, extra_qty = C.RULES[opt.get_lottery_name()]

        shelf = None
        result.generated = True
        if opt.option == 1:
            msg: str = "The generated numbers have not been saved"
            result.saved = False
            self.update_status(msg)
        else:
            result.saved = True
            directory = Path(C.SAVE_FILE_DIR)
            if not directory.exists():
                directory.mkdir(parents=True)
            shelf = shelve.open(filename=self._saved_file,
                                protocol=C.SHELF_PROTOCOL)
            shelf[C.SHELF_ARGS["DATE"]] = time.time()
            shelf[C.SHELF_ARGS["TYPE"]] = C.LOTTERY_CHOICES[opt.lottery_type]
            shelf[C.SHELF_ARGS["LINES"]] = opt.number_of_lines

        # count the actual number of lines generated
        count: int = 0
        # generate the required number of lines
        for _ in range(opt.number_of_lines):
            # x_1 holds the first group of numbers generated
            # main_max is the the highest number to generate plus 1
            # main_qty is the quantity of numbers to generate each line
            x_1: List[str] = choose_numbers(maximum=main_max,
                                            quantity=main_qty)

            # x_2 holds the secondary group of numbers generated if
            # any are required
            # extra_max is the the highest number to generate plus 1
            # extra_qty is the quantity of numbers to
            # generate in each line
            x_2: List[Union[str, None]] = [None]
            # only generate the second group of numbers if required
            # if extra_qty > 0
            if extra_qty:
                x_2 = choose_numbers(maximum=extra_max, quantity=extra_qty)
            # If shelf is None it means no_save was specified
            if shelf is not None:
                shelf[C.SHELF_ARGS["PART1"] + str(count)] = x_1
                # Save the extra numbers even if there are none to save
                # will return None on subsequent reading
                shelf[C.SHELF_ARGS["PART2"] + str(count)] = x_2
            count += 1
            if x_2 == [None]:
                result.data.append(str(x_1))
            else:
                result.data.append(str(x_1) + " - " + str(x_2))
        # close the save file if there is one
        if opt.get_option_name() != C.OPTIONS_CHOICES[1]:
            shelf.close()
        test: bool = opt.get_option_name() == C.OPTIONS_CHOICES[1]
        msg: str = (
            f"The numbers have{' not' if test else ''}"
            f" been saved and {count} lines were generated"
        )
        self.update_status(msg)
        frm = ResultsFrame(None, result)
        frm.Show()

    def delete_saved_file(self) -> None:
        """Delete a previously saved file
        :return: None
        """
        # add the filename extension
        file_name: str = self._saved_file + C.SAVE_FILE_TYPE
        path = Path(file_name)
        file_exists: bool = path.exists()
        if file_exists:
            path.unlink()
        msg: str = f"File: <{file_name}> was "\
            f"{'deleted' if file_exists else 'not found'}"
        self.update_status(msg)

    def show_saved(self) -> None:
        """Display a previously generated and saved batch of numbers
        :return: None
        """
        opt = self.options_data
        # add the filename extension
        path = Path(self._saved_file + C.SAVE_FILE_TYPE)
        if path.exists() and path.is_file():
            shelf = shelve.open(
                filename=self._saved_file,
                flag=C.SHELF_READONLY,
                protocol=C.SHELF_PROTOCOL,
            )
            save_time = time.localtime(shelf[C.SHELF_ARGS["DATE"]])
            result = ResultsData
            result.clear_data()

            result.saved = False
            result.generated = False
            result.lottery_type_name = opt.get_lottery_name()
            result.stored_date = time.strftime(C.DATE_FORMAT, save_time)
            count: int = 0
            for line in range(shelf[C.SHELF_ARGS["LINES"]]):
                # Don't display second group if none exist
                x_1: List[str] = shelf[C.SHELF_ARGS["PART1"] + str(line)]
                x_2: List[Union[str, None]] = shelf[C.SHELF_ARGS["PART2"]
                                                    + str(line)]
                if x_2 == [None]:
                    result.set_data_item(str(x_1))
                else:
                    result.set_data_item(str(x_1) + " - " + str(x_2))
                count += 1
            result.number_of_lines = count
            shelf.close()
            frm = ResultsFrame(None, result)
            frm.Show()
        else:
            msg: str = f"File <{path}> is missing"
            self.update_status(msg)


import pydicom as pyd
import copy
import cv2
import numpy as np
from PIL import Image
from pydicom.pixel_data_handlers import apply_rescale, apply_voi_lut

# def read_cxr_image(path):

def get_bits_stored(dcm: pyd.FileDataset):
    return dcm[0x0028, 0x0101].value

def get_bits_allocted(dcm: pyd.FileDataset):
    return dcm[0x0028, 0x0100].value

def invert_pixel_array(
    pixel_array: np.ndarray, dcm: pyd.FileDataset, use_dcm_header=False
):
    """
    invert dicom pixel array
    return : inverted pixel array
    """

    if use_dcm_header:
        bits_stored = get_bits_stored(dcm)
        max_value = 2 ** int(bits_stored) - 1
    else:
        max_value = np.max(pixel_array)

    return max_value - pixel_array

def save_image_uint16(image, path):
    image = image.astype(np.uint16)
    array_buffer = image.tobytes()
    output = Image.new("I", image.T.shape)
    output.frombytes(array_buffer, "raw", "I;16")
    output.save(path)

def get_image_from_dcm(
    dicom,
    output_path=None,
    use_dcm_header=False,
):
    """
    dcm : str or pyd.FileDataset
    return : np.array
    """
    if isinstance(dicom, str):
        dcm = pyd.dcmread(dicom)
    elif isinstance(dicom, pyd.FileDataset):
        dcm = copy.deepcopy(dicom)

    pixel_array = dcm.pixel_array
    pixel_array = apply_rescale(pixel_array, dcm)

    # check
    # print(dcm[0x0028, 0x0004].value)
    # origin = dcm[0x0028, 0x0004].value
    # converted = str(origin).upper().strip()
    # if origin != converted:
    #     print(origin, converted, dicom)
    #     dcm[0x0028, 0x0004].value = "MONOCHROME1"
    # dcm[0x0028, 0x0004].value = str(origin).upper().strip()

    if use_dcm_header:
        pixel_array = apply_voi_lut(pixel_array, dcm)

    if get_photometric_interpretation(dcm) == "MONOCHROME1":
        pixel_array = invert_pixel_array(pixel_array, dcm, use_dcm_header)

    # min max normalization
    max_value = np.max(pixel_array)
    min_value = np.min(pixel_array)
    pixel_array = (pixel_array - min_value) / (max_value - min_value)
    pixel_array *= 2**16 - 1

    assert np.max(pixel_array) == 65535 and np.min(pixel_array) == 0

    if output_path is not None:
        save_image_uint16(pixel_array, output_path)
    return pixel_array

def get_bitsstored_bitsallocated(dcm: pyd.FileDataset):
    bits_stored = get_bits_stored(dcm)
    bits_allocated = get_bits_allocted(dcm)
    return bits_stored, bits_allocated

def get_photometric_interpretation(dcm: pyd.FileDataset):
    """
    get photometric interpretation value from dicom

    return : 'MONOCHROME1' or 'MONOCHROME2'
    """
    photometric = str(dcm[0x0028, 0x0004].value)
    photometric = "MONOCHROME2" if photometric is None else photometric
    photometric = photometric.upper().strip()

    return photometric

def from_uint16_to_float32(pixel_array: np.ndarray, dcm: pyd.FileDataset):
    """ """
    bits_stored = get_bits_stored(dcm)
    max_value = (
        2 ** int(bits_stored) - 1 if bits_stored is not None else np.max(pixel_array)
    )
    return pixel_array.astype(np.float32) / max_value

def get_max_contour(contours):
    max_size = -1
    max_contour = None
    for contour in contours[0]:
        if cv2.contourArea(contour) > max_size:
            max_contour = contour
            max_size = cv2.contourArea(contour)
    return max_contour

def get_box_from_mask(path_mask, output_format="cxcywh"):
    # from torchvision.ops.boxes import box_convert

    mask: np.array = cv2.imread(path_mask, cv2.IMREAD_UNCHANGED).astype(np.uint8)
    height, width = mask.shape[:2]
    contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    if len(contours[0]) != 1:
        raise ValueError(f"{path_mask} has multiple contours")

    max_contour = get_max_contour(contours)
    x, y, w, h = cv2.boundingRect(max_contour)

    x = x / width
    w = w / width

    y = y / height
    h = h / height

    cx = x + w / 2
    cy = y + h / 2
    # output = box_convert(box, in_fmt="xywh", out_fmt=output_format)[0]
    return np.array([cx, cy, w, h])

# output_arr = utils_cxr.from_uint16_to_float32(pixel_array,dcm)
# output_arr = Image.fromarray(output_arr)
# output_path = path_dcm.replace(".dcm", ".tiff")
# output_path = output_path.replace("dicom", "png")
# output_arr.save(output_path)


from zrb.helper.typecheck import typechecked
import libcst as cst

@typechecked
class TaskTransformer(cst.CSTTransformer):
    def __init__(self, task_name: str, upstream_task_name: str):
        super().__init__()
        self.task_name: str = task_name
        self.upstream_task_name: str = upstream_task_name
        self._on_task_call = False
        self._upstream_added = False

    def _is_task(self, node: cst.Call) -> bool:
        if (
            not isinstance(node.func, cst.Name)
            or node.func.value != "Task"
        ):
            return False
        # Look for name keyword
        for name_arg in node.args:
            if (
                isinstance(name_arg, cst.Arg)
                and name_arg.keyword.value == "name"
                and name_arg.value.evaluated_value == self.task_name
            ):
                return True
        return False

    def visit_Call(self, node: cst.Call):
        # Visit all function call (or class initiation)
        # If it is the task we seek, flag it
        if self._is_task(node):
            self._on_task_call = True

    def leave_Call(
        self, original_node: cst.Call, updated_node: cst.Call
    ) -> cst.Call:
        # Remove flags.
        # If upstreams argument not found (see: self.leave_Arg), add it
        if not self._is_task(updated_node):
            return updated_node
        self._on_task_call = False
        if self._upstream_added:
            return updated_node
        new_args = [arg for arg in updated_node.args]
        new_args.append(cst.Arg(
            keyword=cst.parse_expression('upstreams'),
            value=cst.List([
                cst.Element(cst.parse_expression(self.upstream_task_name))
            ])
        ))
        self._upstream_added = True
        return updated_node.with_changes(
            args=new_args
        )

    def leave_Arg(self, orginnal_node: cst.Arg, updated_node: cst.Arg):
        # Once the flag was set (see: self.visit_Call), look for
        # `upstreams` argument and modify the value
        if (
            not self._on_task_call
            or updated_node.keyword is None
            or updated_node.keyword.value != 'upstreams'
        ):
            return updated_node
        new_elements = [
            old_element for old_element in updated_node.value.elements
        ]
        new_elements.append(cst.Element(
            value=cst.parse_expression(self.upstream_task_name)
        ))
        self._upstream_added = True
        return updated_node.with_changes(
            value=cst.List(new_elements)
        )

@typechecked
def add_upstream_to_task(
    code: str, task_name: str, upstream_task_name: str
) -> str:
    """
    Transformer to add an upstream task to a Task object.

    Args:
        code (str): The code to modify.
        task_name (str): Name of the task to modify.
        upstream_task_name (str): Name of the upstream task to add.

    Returns:
        A modified version of the input code with the upstream task added.
    """
    transformer: TaskTransformer = TaskTransformer(
        task_name, upstream_task_name
    )
    original_tree: cst.Module = cst.parse_module(code)
    modified_tree: cst.Module = original_tree.visit(transformer)
    return modified_tree.code


# This piece of source code has been inspired by Padmamala ma'am's notes and geeksforgeeks.org.
# Link: https://www.geeksforgeeks.org/strongly-connected-components/
# Status of this code: Rough; works on all inputs, but still need to clean it.
import collections

# Padmamala ma'am had taught us about post-order numbering where we assign a discovery number to a vertex at the very end,
# i.e., after it has been explored completely. And then, we should reverse the graph and start with the vertex with the highest
# post-order number. The same effect can be achieved using a stack. Although both approaches are equally good (probably the
# post-order numbering is slightly better as we don't have to create a stack and thus save space) I found the stack approach
# a little easier to imagine and code; hence this variant.
# This function creates a stack of vertices where each vertex is pushed after it has been completely explored. It achieves the
# same goal as post-order numbering. After reversing the graph, we keep popping off vertices to find out the SSCs.
# The graph, a source vertex and an empty stack are taken as input. The final stack is stored in the stack object sent as input
def createDFSStack(graph, v, stack):
    try:                                # Python does not support static variables as C does, this is a small bodge
                                        # to emulate the same effect
        createDFSStack.count += 1       # try incrementing the function counter variable..
    except AttributeError:              # if it has not yet been defined, then Python will throw an error..
        createDFSStack.count = 0        # initialise the counter. It is used to assign a discovery number to each vertex

    v.dfsNum = createDFSStack.count     # assign the next discovery number to the current vertex 'v'
    v.status = "Visited"                # Mark 'v' as Visited

    for neighbour in v.adjVertices:                     # for each neighbour of v, do the following..
        if neighbour.status != "Visited":               # if the neighbour is unvisited..
            createDFSStack(graph, neighbour, stack)     # call the same function on the neighbouring vertex

    stack.append(v)                                     # push the vertex into the stack at the very end

    if v.dfsNum == 0:                                   # If this is the first instance of the function..
        for vertex in graph.vertices:
            if vertex.status != "Visited":              # ..then check if any of the nodes have not yet been visited..
                createDFSStack(graph, vertex, stack)    # if so, run DFS on them..

# Performs a basic DFS on the graph object starting from vertex 'v' and writes the output to file object fout
def basicDFS(graph, v, fout):
    v.status = "Visited"                        # Mark vertex 'v' as visited
    fout.write(v.name + " ")
    for neighbour in v.adjVertices:             # for each neighbour..
        if neighbour.status != "Visited":       # if it has not yet been visited..
            basicDFS(graph, neighbour, fout)    # run DFS on them..

# Finds the strongly connected components. Input: the reversed graph, the stack containing vertices in order of
# exploration-completion time and the output file object
def findSSCs(graph, stack, fout):
    while len(stack) != 0:              # continue until the stack is empty
        v = stack.pop()                 # pop the vertex 'v' on top of the stack
        if v.status != "Visited":       # if it has not yet been visited..
            fout.write("{")             # then, it is a new component..
            basicDFS(graph, v, fout)    # run a basic DFS starting from that vertex
            fout.write("}\t")           # after the DFS call, we know all vertices of that component have been explored

def SSCDriver(graph, fout):                             # A driver function to find out the SSCs
    stack = collections.deque()                         # Create an empty stack
    createDFSStack(graph, graph.vertices[0], stack)     # create a stack of vertices which achieves the same goal as
    graph.revGraph()                                    # post-order numbering..and reverse the graph
    findSSCs(graph, stack, fout)                        # finally, find out the strongly connected components


import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import metrics

def wine_filter(inputf, t_size=0.2):
    dtset = pd.read_csv(inputf, delimiter=";", index_col=None)
    X = dtset.iloc[:, :-1].values
    Y = dtset.iloc[:, -1].values
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=t_size, random_state=3)
    sc_x = StandardScaler()
    X_train = sc_x.fit_transform(X_train)
    X_test = sc_x.transform(X_test)
    return X_train, X_test, Y_train, Y_test

X_train, X_test, Y_train, Y_test =  wine_filter('winequality-white.csv',0.2)
clf = RandomForestClassifier(n_estimators=1000,random_state=2000)
clf.fit(X_train, Y_train)
Y_pred=clf.predict(X_test)
print (metrics.accuracy_score(Y_test,Y_pred))