from mitmproxy import ctx, http
import logging
import time
import os
import html
import json
import pdf_printer

# Global variables for the request template and article list
articles = []
template_flow = None

def response(flow: http.HTTPFlow) -> None:
    if "https://mp.weixin.qq.com/mp/profile_ext?action=getmsg" in flow.request.pretty_url:
        global articles

        response_json = json.loads(flow.response.text)
        has_next = response_json['can_msg_continue']
        needs_replay = has_next or not flow.is_replay

        # Modify the query parameters for the next request
        query_params = dict(flow.request.query.fields)
        updated_params = query_params.copy()

        # template, no need to handle the response yet
        if not flow.is_replay:
            template_flow = flow.copy()
            updated_params['offset'] = 0

        # start adding new articles; modify parameters for next replay
        else:
            santinised_json_raw = response_json['general_msg_list'].replace("\/", "/")
            new_articles = json.loads(santinised_json_raw)['list']
            articles += flat_records_from(new_articles)

            offset = int(query_params.get('offset', '0'))
            count = int(query_params.get('count', '10'))
            updated_params['offset'] = str(offset + count)

        # Replay the request every 2 seconds if needed
        if needs_replay:
            replay_flow = flow.copy()
            replay_flow.request.query.fields = list(updated_params.items())
            time.sleep(2)
            ctx.master.commands.call("replay.client", [replay_flow])

        # Stop intercepting traffic once no longer needed
        else:
            with open('articles.json', 'w', encoding='utf8') as f:
                json.dump(articles, f, ensure_ascii=False)

            logging.info('Finished capturing articles. Saved to articles.json')
            ctx.master.shutdown()

def done():
    logging.info('Done')

def replace_uncommon_characters(input_str):
    for s in ['(',')','|',' ','/','*','\\','.','<','>','&','?',' ']:
        output_str = input_str.replace(s, '-')
    return output_str

def sub_dict(data, keys):
    return {k: html.unescape(data[k]) for k in data if k in keys}

def sanitised_record(msg_info, p_date):
    date_pretty = time.strftime('%Y%m%d',time.localtime(p_date))

    keys = ['title', 'author', 'content_url', 'digest', 'cover', 'source_url']
    data = sub_dict(msg_info, keys)
    title = replace_uncommon_characters(data['title'])
    record_item = {}
    record_item['title'] = title
    record_item['date'] = date_pretty
    record_item['url'] = data['content_url']

    # record = "{}_{} : {}\n".format(date_pretty, title, data['content_url'])
    return record_item

def flat_records_from(msg_list):
    records = []
    for msg in msg_list:
        p_date = msg.get('comm_msg_info').get('datetime')

        msg_info = msg.get("app_msg_ext_info")

        if msg_info:
            multi_msg_info = msg_info.get("multi_app_msg_item_list")
            if multi_msg_info:
                for multi_msg_item in multi_msg_info:
                    records.append(sanitised_record(multi_msg_item, p_date))
            else:
                records.append(sanitised_record(msg_info, p_date))
        else:
            ctx.log.info(u"Non articles message，data=%s" % json.dumps(msg.get("comm_msg_info")))
    return records


'''
Some functions that help to HT process.
'''
# standard
import os
import logging
import json
import pandas
import re

# third party
from Bio import Entrez, Medline
import multigenomic_api as mg_api
import pymongo

# local
from libs import constants as EC

def get_collection_name(collection_path):
    '''
    Checks collection path to determine the collection name.
    Param
        collection_path, String, Raw collection path.
    Returns
        collection_name, String, Final collection name.
    '''
    collection_name = collection_path
    if 'ChIP-exo' in collection_name:
        collection_name = EC.CHIP_EXO
    if 'ChIP-seq' in collection_name:
        collection_name = EC.CHIP_SEQ
    if 'TU' in collection_name:
        collection_name = EC.TUS
    if 'TSS' in collection_name:
        collection_name = EC.TSS
    if 'TTS' in collection_name:
        collection_name = EC.TTS
    if 'RNA-seq' in collection_name:
        collection_name = EC.RNA
    if 'gSELEX' in collection_name:
        collection_name = EC.GSELEX
    if 'DAP-seq' in collection_name:
        collection_name = EC.DAPS
    return collection_name

def get_collection_type(collection_path):
    '''
    Determines the Strategy which was used to process the collection.
    Param
        collection_path, String, Raw collection path.
    Returns
        collection_type, String, Final collection type.
    '''
    collection_type = collection_path
    if 'ChIP-exo' in collection_type:
        collection_type = 'CHIP_EXO_'
    if 'ChIP-seq' in collection_type:
        collection_type = ''
    if 'gSELEX' in collection_type:
        collection_type = 'GSELEX_'
    if 'DAP' in collection_type:
        collection_type = 'DAPS_'
    return collection_type

def set_json_object(filename, data_list, organism, sub_class_acronym, child_class_acronym):
    '''
    Sets the JSON output format of the collection..

    Param
        filename, String, the the output file name.
        data_list, List, the list with the collection data.
        organism, String, the organism name.

    Returns
        json_object, Dict, the dictionary with the final JSON file format
    '''
    json_object = {
        'collectionName': filename,
        'collectionData': data_list,
        'organism': organism,
        'subClassAcronym': sub_class_acronym,
        'classAcronym': organism,
        'childClassAcronym': child_class_acronym,
    }
    return json_object

def verify_bed_path(bed_path):
    '''
    This function filters BED files in the path and returns only correctly formatted path.

    Param
        bed_path, String, raw directory path.

    Returns
        bed_path, String, verified directory path.
    '''

    if os.path.isfile(bed_path) and bed_path.endswith('.bed'):
        logging.info(
            f'Reading dataset {bed_path}')
        return bed_path
    else:
        logging.warning(
            f'{bed_path} is not a valid BED file will be ignored')
        return None

def verify_txt_path(txt_path):
    '''
    This function filters TXT files in the path and returns only correctly formatted path.

    Param
        txt_path, String, raw directory path.

    Returns
        txt_path, String, verified directory path.
    '''

    if os.path.isfile(txt_path) and txt_path.endswith('.txt'):
        logging.info(
            f'Reading dataset {txt_path}')
        return txt_path
    else:
        logging.warning(
            f'{txt_path} is not a valid TXT file will be ignored')
        return None

def verify_tsv_path(tsv_path):
    '''
    This function filters TSV files in the path and returns only correctly formatted path.

    Param
        tsv_path, String, raw directory path.

    Returns
        tsv_path, String, verified directory path.
    '''

    if os.path.isfile(tsv_path) and tsv_path.endswith('.tsv'):
        logging.info(
            f'Reading dataset {tsv_path}')
        return tsv_path
    else:
        logging.warning(
            f'{tsv_path} is not a valid TSV file will be ignored')
        return None

def verify_csv_path(csv_path):
    '''
    This function filters CSV files in the path and returns only correctly formatted path.

    Param
        csv_path, String, raw directory path.

    Returns
        csv_path, String, verified directory path.
    '''

    if os.path.isfile(csv_path) and csv_path.endswith('.csv'):
        logging.info(
            f'Reading dataset {csv_path}')
        return csv_path
    else:
        logging.warning(
            f'{csv_path} is not a valid TSV file will be ignored')
        return None

def validate_directory(data_path):
    '''
    Verify if the output path directory exists.

    Param
        data_path, String, directory path.

    Return
        Boolean, boolean.
    '''
    if not os.path.isdir(data_path):
        logging.error(
            f'There is not {data_path} directory')
        return False
    else:
        return True

def validate_directories(data_path):
    '''
    Verify if the output path directories exists.

    Param
        data_path, String, directories path.

    Return
        Rise IOError if not valid directory
    '''
    if not os.path.isdir(data_path):
        raise IOError("Please, verify '{}' directory path".format(data_path))

def set_log(log_path, log_name, log_date):
    '''
    Initializes the execution log to examine any problems that arise during extraction.

    Param
        log_path, String, the execution log path.
    '''
    log_file_name = f'ht_etl_{log_name}_{log_date}.log'
    log_file_name = log_file_name.replace('/', '')
    log_file_name = log_file_name.replace('-', '_')
    print(log_file_name)
    validate_directories(log_path)
    logging.basicConfig(filename=os.path.join(log_path, log_file_name),
                        format='%(levelname)s - %(asctime)s - %(message)s', filemode='w', level=logging.INFO)

def create_json(objects, filename, output):
    '''
    Create and write the JSON file with the results.

    Param
        objects, Object, a Python serializable object that you want to convert to JSON format.
        filename, String, JSON file name.
        output, String, output path.
    '''
    filename = os.path.join(output, filename)
    with open(f'{filename}.json', 'w') as json_file:
        json.dump(objects, json_file, indent=4, sort_keys=True)

# TODO: Not used, must be deleted?

def list_to_dict(data):
    '''
    Turns a data List into a directory object.

    Param
        data, List, data list.

    Returns
        data_dict, Dict, data list converted to dictionary.
    '''
    data_dict = {}
    for object_dict in data:
        data_dict.update(object_dict)
    return data_dict

def get_data_frame(filename: str, load_sheet, rows_to_skip: int) -> pandas.DataFrame:
    '''
    Read and convert the Excel file to Panda DataFrame.

    Param
        filename, String, full XLSX file path.
        load_sheet, Integer, Excel sheet number that will be loaded.
        rows_to_skip, Integer, number of rows to skip.

    Returns
        dataset_df, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.
    '''
    dataset_df = pandas.read_excel(
        filename, sheet_name=load_sheet, skiprows=rows_to_skip)
    return dataset_df

def get_author_data_frame(filename: str, load_sheet, rows_to_skip: int) -> pandas.DataFrame:
    '''
    Read and convert the Excel file to Panda DataFrame.

    Param
        filename, String, full XLSX file path.
        load_sheet, Integer, Excel sheet number that will be loaded.
        rows_to_skip, Integer, number of rows to skip.

    Returns
        dataset_df, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.
    '''
    dataset_df = pandas.read_excel(
        filename, sheet_name=load_sheet, skiprows=rows_to_skip, index_col=0)
    nan_value = float("NaN")
    dataset_df.replace("", nan_value, inplace=True)
    dataset_df.dropna(how='all', axis=1, inplace=True)
    return dataset_df

def get_data_frame_tsv(filename: str) -> pandas.DataFrame:
    '''
    Read and convert the TSV file to Panda DataFrame.

    Param
        filename, String, full tsv file path.

    Returns
        dataset_df, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.
    '''
    dataset_df = pandas.read_csv(filename, sep='\t', header=0, index_col=False)
    return dataset_df

def get_author_data_frame_tsv(filename: str) -> pandas.DataFrame:
    '''
    Read and convert the TSV file to Panda DataFrame.

    Param
        filename, String, full tsv file path.

    Returns
        dataset_df, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.
    '''
    dataset_df = pandas.read_csv(filename, sep='\t',)
    return dataset_df

# TODO: Not used, must be deleted?
def get_data_frame_tsv_coma(filename: str) -> pandas.DataFrame:
    '''
    Read and convert the TSV file to Panda DataFrame.

    Param
        filename, String, full tsv file path.

    Returns
        dataset_df, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.
    '''
    dataset_df = pandas.read_csv(filename, sep=',', header=0)
    return dataset_df

def get_json_from_data_frame(data_frame: pandas.DataFrame) -> dict:
    '''
    Converts DataFrame into JSON format.

    Param
        data_frame, pandas.DataFrame, DataFrame with the Datasets Record Excel file data.

    Returns
        json_dict, Dict, JSON string converted  to a dictionary.
    '''
    string_json = data_frame.to_json(orient='records')
    string_json = re.sub(r'\([0-9]\)\s*', '', string_json)
    json_dict = json.loads(string_json)
    return json_dict

def get_excel_data(filename: str, load_sheet, rows_to_skip: int) -> dict:
    '''
    Process the XLSX file as a DataFrame and return it as a JSON object

    Param
        filename, String, Excel file name.

    Returns
        data_frame_json, Dict, json dictionary with the Excel data.
    '''
    data_frame = get_data_frame(filename, load_sheet, rows_to_skip)
    data_frame_json = get_json_from_data_frame(data_frame)
    return data_frame_json

def get_tsv_data(filename: str) -> dict:
    '''
    Process the tsv file as a DataFrame and return it as a JSON object

    Param
        filename, String, tsv file name.

    Returns
        data_frame_json, Dict, json dictionary with the tsv data.
    '''
    data_frame = get_data_frame_tsv(filename)
    data_frame_json = get_json_from_data_frame(data_frame)
    return data_frame_json

def to_camel_case(snake_str):
    '''
    Converts snake_case String into camelCase.
    Capitalize the first letter of each component except the first one with the 'title' method and join them together.

    Param
        snake_str, String, snake_case string.

    Returns
        camelStr, String, camelCase string.
    '''
    components = snake_str.split('_')
    camelStr = components[0] + ''.join(x.title() for x in components[1:])
    return camelStr

def get_pubmed_data(pmids, email):
    '''
    Connects to PUBMED database through Entrez API and gets the necessary publication data.
    The Entrez API returns a dictionary with the medline data, see also https://biopython.org/docs/1.75/api/Bio.Medline.html for more information about the keys obtained from this dictionary.

    Param
        pmid, Integer, PUBMED publication id.
        email, String, User email address to connect to PUBMED database.

    Returns
        publication, Dict, dictionary with the publication data.
    '''
    if not pmids:
        return None
    Entrez.email = email
    publications = []
    if isinstance(pmids, int) or isinstance(pmids, float):
        pmids = [pmids]
    elif isinstance(pmids, str):
        pmids = pmids.replace(' ', '')
        pmids = pmids.split(',')
    for pmid in pmids:
        if isinstance(pmid, float):
            pmid = int(pmid)
        # print(pmid)
        handle = Entrez.efetch(db='pubmed', id=pmid,
                               rettype='medline', retmode='text')
        publication = {}
        record = Medline.read(handle)
        pubmed_authors = record.get('AU')
        if isinstance(pubmed_authors, str):
            pubmed_authors = pubmed_authors.split(',')
        publication.setdefault('authors', pubmed_authors)
        publication.setdefault('abstract', record.get('AB'))
        publication.setdefault('date', record.get('DP'))
        publication.setdefault('pmcid', record.get('PMC'))
        publication.setdefault('pmid', int(record.get('PMID')))
        publication.setdefault('title', record.get('TI'))
        article_identifier = record.get('AID')
        for identifier in article_identifier:
            if ' [doi]' in identifier:
                publication.setdefault('doi', identifier.replace(' [doi]', ''))
        publication = {k: v for k, v in publication.items() if v}
        publications.append(publication)
    return publications

def format_cross_reference_url(url, object_id):
    '''
    Corrects the External Cross References URL removing '~A' characters and adding object_id at the end.

    Param
        url, String, External Cross References raw URL.
        object_id, String, External Cross References Object ID.

    Returns
        formated_url, String, External Cross References final URL.
    '''
    formated_url = f'{url.replace("~A", "")}{object_id}'
    return formated_url

def get_object_tested(protein_names, database, url):
    '''
    Gets TF data from the RegulonDBMultigenomic database and returns the object tested dictionary.

    Param
        protein_name, String, TF protein name.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.

    Returns
        object_tested, Dict, dictionary with the object tested data.
    '''
    # TODO: check unique objects
    mg_api.connect(database, url)
    objects_tested = []
    if protein_names:
        for protein_name in protein_names:
            object_tested = {}
            mg_tf = mg_api.transcription_factors.find_by_name(protein_name)

            client = pymongo.MongoClient(url)
            db = client[database]
            collection = db['transcriptionFactors']
            mg_tf = collection.find_one({'abbreviatedName': protein_name})
            if not mg_tf:
                print(protein_name, database)
                mg_tf = collection.find_one({'name': protein_name})

            if mg_tf:
                tf_id = mg_tf.get('_id')
                active_conformations = []
                external_cross_references = []
                for active_conf in mg_tf.get('activeConformations', []):
                    active_conformations.append(active_conf.get('_id'))
                for cross_ref in mg_tf.get('externalCrossReferences', []):
                    mg_cross_ref = mg_api.external_cross_references.find_by_id(
                        cross_ref.get('externalCrossReferences_id'))
                    external_cross_references.append(
                        {
                            'externalCrossReferenceId': cross_ref.get('externalCrossReferences_id'),
                            'objectId': cross_ref.get('objectId'),
                            'externalCrossReferenceName': mg_cross_ref.name,
                            'url': format_cross_reference_url(mg_cross_ref.url, cross_ref.get('objectId'))
                        }
                    )
                genes = []
                for product_id in mg_tf.get('products_ids'):
                    mg_product = mg_api.products.find_by_id(product_id)
                    mg_gene = mg_api.genes.find_by_id(mg_product.genes_id)
                    gene = {
                        '_id': mg_gene.id,
                        'name': mg_gene.name
                    }
                    genes.append(gene)

                object_tested = {
                    '_id': tf_id,
                    'name': mg_tf.get('name'),
                    'abbreviatedName': mg_tf.get('abbreviatedName'),
                    'synonyms': mg_tf.get('synonyms'),
                    'genes': genes,
                    'note': mg_tf.get('note'),
                    'activeConformations': active_conformations,
                    'externalCrossReferences': external_cross_references
                }
                object_tested = {k: v for k, v in object_tested.items() if v}
                objects_tested.append(object_tested)
            else:
                object_tested = {
                    '_id': None,
                    'name': protein_name,
                    'synonyms': [],
                    'genes': [],
                    'note': None,
                    'activeConformations': [],
                    'externalCrossReferences': [],
                }
                object_tested = {k: v for k, v in object_tested.items() if v}
                objects_tested.append(object_tested)
    mg_api.disconnect()
    return objects_tested

def get_center_pos(left_pos, right_pos):
    '''
    Calculates the center center position of the chromosome.

    Param
        left_pos, String, Start position in the sequence (it's converted to Integer).
        right_pos, String, End position in the sequence (it's converted to Integer).

    Returns
        center_pos, Float, Center position in the sequence.
    '''
    center_pos = int(right_pos) - int(left_pos)
    center_pos = (center_pos / 2) + int(left_pos)
    return center_pos

def set_genome_intervals():
    '''
    Set the genes ranges to calculate the closest_genes.

    Param

    Returns
        genes_ranges, List, Array of coordinate pairs of the calculated ranges.
    '''
    genome_length = EC.GENOME_LENGTH
    intervals = EC.INTERVALS
    intervals_length = int(genome_length / intervals)
    genes_ranges = []
    for interval in range(intervals):
        genes_ranges.append(
            [intervals_length + ((interval - 1) * intervals_length), intervals_length + (interval * intervals_length)])
    return genes_ranges

def find_closest_gene(left_pos, right_pos, database, url, genes_ranges):
    '''
    Calculates the center center position of the chromosome.

    Param
        left_pos, String, Start position in the sequence (it's converted to Integer).
        right_pos, String, End position in the sequence (it's converted to Integer).
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
        genes_ranges, List, Array of coordinate pairs of the calculated ranges.

    Returns
        closest_genes, List, Dict List with the verified closest genes.
    '''
    minimum_distance = EC.MINIMUM_DISTANCE
    genome_length = EC.GENOME_LENGTH
    intervals = EC.INTERVALS

    chromosome_center_pos = get_center_pos(int(left_pos), int(right_pos))

    found_genes_interval = genes_ranges[int(
        (chromosome_center_pos * intervals) / genome_length)]

    mg_api.connect(database, url)

    mg_genes = mg_api.genes.get_closest_genes_to_central_position(
        (found_genes_interval[0] - minimum_distance), (found_genes_interval[1] + minimum_distance), chromosome_center_pos, minimum_distance)
    closest_genes = []
    for gene in mg_genes:
        gene_strand = gene.strand
        gene_left_pos = gene.left_end_position
        gene_right_pos = gene.right_end_position
        gene_product_name = []
        tus_dict_list = []
        try:
            mg_tus = mg_api.transcription_units.find_by_gene_id(gene.id)
            for mg_tu in mg_tus:
                tu_dict = {}
                tu_dict.setdefault('_id', mg_tu. id)
                tu_dict.setdefault('name', mg_tu.name)
                tus_dict_list.append(tu_dict)
        except Exception:
            logging.error(f'Can not find TU in Gene {gene.id}')
        try:
            mg_products = mg_api.products.find_by_gene_id(gene.id)
            for product in mg_products:
                gene_product_name.append(product.name)
        except Exception:
            logging.error(f'Can not find Product Name in Gene {gene.id}')
        if gene_strand == 'forward':
            distance = float(gene_left_pos) - chromosome_center_pos
            closest_genes.append(
                {'_id': gene.id, 'name': gene.name, 'distanceTo': abs(distance), 'productName': gene_product_name, 'transcriptionUnits': tus_dict_list})
        elif gene_strand == 'reverse':
            distance = chromosome_center_pos - float(gene_right_pos)
            closest_genes.append(
                {'_id': gene.id, 'name': gene.name, 'distanceTo': abs(distance), 'productName': gene_product_name, 'transcriptionUnits': tus_dict_list})
    mg_api.disconnect()
    return closest_genes

def find_terminators(left_pos, right_pos, tts_id, database, url):
    '''
    Calculates the center center position of the chromosome.

    Param
        left_pos, String, Start position in the sequence (it's converted to Integer).
        right_pos, String, End position in the sequence (it's converted to Integer).
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
        genes_ranges, List, Array of coordinate pairs of the calculated ranges.

    Returns
        closest_genes, List, Dict List with the verified closest genes.
    '''
    mg_api.connect(database, url)

    terminators = []
    try:
        mg_terminators = mg_api.terminators.get_closer_terminators(
            (left_pos - 30), (right_pos + 30))
        for terminator in mg_terminators:
            terminator_dict = {}
            terminator_dict.setdefault('_id', terminator.id)
            mg_tus = mg_api.transcription_units.find_by_terminator_id(
                terminator.id)
            tus_dict_list = []
            if mg_tus:
                for tu in mg_tus:
                    tu_dict = {}
                    tu_dict.setdefault('_id', tu.id)
                    tu_dict.setdefault('name', tu.name)
                    mg_promoter = mg_api.promoters.find_by_id(tu.promoters_id)
                    promoter = {}
                    promoter.setdefault('_id', mg_promoter.id)
                    promoter.setdefault('name', mg_promoter.name)
                    promoter.setdefault('sequence', mg_promoter.sequence)
                    promoter.setdefault('leftEndPosition',
                                        mg_promoter.transcription_start_site.left_end_position)
                    promoter.setdefault('rightEndPosition',
                                        mg_promoter.transcription_start_site.right_end_position)
                    promoter.setdefault('strand', mg_promoter.strand)

                    tu_dict.setdefault('promoter', promoter)
                    tus_dict_list.append(tu_dict)

            terminator_dict.setdefault('transcriptionUnits', tus_dict_list)

            terminators.append(terminator_dict)
    except Exception:
        logging.error(f'Can not find Terminator Name in Gene {tts_id}')
    mg_api.disconnect()
    return terminators

def get_sites_ids_by_tf(tf_names, database, url):
    '''
    Uses MG API to get the sites IDs by TF name.

    Param
        tf_names, List, TF names list.
        database, String, RegulonDB Multigenomic database name
        url, String, URL to RegulonDB Multigenomic database.

    Returns
        sites_ids, List, List of sites IDs.
    '''
    sites_ids = []
    mg_api.connect(database, url)
    for tf_name in tf_names:
        try:
            # TODO: Find by short name
            #mg_tf = mg_api.transcription_factors.find_by_name(tf_name)
            #tf_id = mg_tf[0].id
            client = pymongo.MongoClient(url)
            db = client[database]
            collection = db['transcriptionFactors']
            mg_tf = collection.find_one({'abbreviatedName': tf_name})
            tf_id = mg_tf.get('_id')
            try:
                mg_sites = mg_api.regulatory_sites.get_tf_binding_sites(tf_id)
                for site in mg_sites:
                    sites_ids.append(site.id)
            except Exception:
                logging.error(f'Can not find Sites in TF {tf_id}')
        except IndexError:
            logging.error(f'Can not find Trasncription Factor {tf_name}')
    mg_api.disconnect()
    return sites_ids

def get_tf_sites_abs_pos(tf_id, database, url):
    '''
    Gets Regualtory Sites objects with absolutePosition.

    Param
        tf_id, String, TF ID.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.

    Returns
        site, Object, Site object with absolutePosition property.
    '''
    site = None
    mg_api.connect(database, url)
    try:
        mg_site = mg_api.regulatory_sites.find_by_id(tf_id)
        site = {
            '_id': tf_id,
            'absolutePosition': mg_site.absolute_position,
            'siteObject': mg_site
        }
    except Exception:
        logging.error(f'Can not find Sites in TF {tf_id}')
    mg_api.disconnect()
    return site

def get_citations(database, url, citations_obj_list):
    '''
    Uses Multigenomic API to get the formatted Citations list.

    Param
        citations_obj_list, List, Citations Object List.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.

    Returns
        citations, List, formatted Citations list.
    '''
    citations = []
    citation = {}
    mg_api.connect(database, url)
    for citation_obj in citations_obj_list:
        evidence_id = citation_obj.evidences_id
        publication_id = citation_obj.publications_id
        evidence = {}
        publication = {}
        try:
            mg_evidence = mg_api.evidences.find_by_id(evidence_id)
            evidence.setdefault('id', mg_evidence.id)
            evidence.setdefault('name', mg_evidence.name)
            evidence.setdefault('code', mg_evidence.code)
            ev_type = mg_evidence.type
            if ev_type == []:
                ev_type = None
            evidence.setdefault('type', mg_evidence.type)
            evidence = {k: v for k, v in evidence.items() if v}
        except Exception:
            logging.error(f'Can not find Evidence {evidence_id}')
        try:
            mg_publication = mg_api.publications.find_by_id(publication_id)
            publication.setdefault('id', mg_publication.id)
            publication.setdefault('authors', mg_publication.authors)
            publication.setdefault('citation', mg_publication.citation)
            publication.setdefault('pmid', mg_publication.pmid)
            publication.setdefault('title', mg_publication.title)
            publication.setdefault('url', mg_publication.url)
            publication.setdefault('year', mg_publication.year)
            publication = {k: v for k, v in publication.items() if v}
        except Exception:
            logging.error(f'Can not find Publication {publication_id}')
        citation = {
            'evidence': evidence,
            'publication': publication
        }
        citation = {k: v for k, v in citation.items() if v}
        citations.append(citation)
    mg_api.disconnect()
    return citations

def get_tss_distance(database, url, regulated_entity, strand, rend, lend):
    '''
    Calculates the distance between the given Regualtory Interaction and the closest Trasncription Start Site.

    Param
        regulated_entity, Object, Regualtory Interaction regulated entity object.
        strand, String, Regualtory Site strand forward or reverse ('-', '+').
        lend, String, Start position in the sequence.
        rend, String, End position in the sequence.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.

    Returns
        distance, Integer, Distance between the given Regualtory Interaction and the closest Trasncription Start Site.
    '''
    mg_api.connect(database, url)
    distance = None
    reg_entity_type = regulated_entity.type
    reg_entity_id = regulated_entity.id
    if reg_entity_type == 'gene':
        try:
            mg_tu = mg_api.transcription_units.find_by_gene_id(reg_entity_id)
            promoter_id = mg_tu[0].promoters_id
            mg_promoter = mg_api.promoters.find_by_id(promoter_id)
            tss = mg_promoter.transcription_start_site
            tss_rend = tss.right_end_position
            tss_lend = tss.left_end_position
            if strand == '-':
                distance = lend - tss_rend
            if strand == '+':
                distance = rend - tss_lend
        except Exception:
            logging.error(
                f'Can not find TU from {reg_entity_id}')
    if reg_entity_type == 'transcriptionUnit':
        try:
            mg_tu = mg_api.transcription_units.find_by_id(reg_entity_id)
            promoter_id = mg_tu.promoters_id
            mg_promoter = mg_api.promoters.find_by_id(promoter_id)
            tss = mg_promoter.transcription_start_site
            tss_rend = tss.right_end_position
            tss_lend = tss.left_end_position
            if strand == '-':
                distance = lend - tss_rend
            if strand == '+':
                distance = rend - tss_lend
        except Exception:
            logging.error(
                f'Can not find TU from {reg_entity_id}')
    if reg_entity_type == 'promoter':
        try:
            mg_promoter = mg_api.promoters.find_by_id(reg_entity_id)
            tss = mg_promoter.transcription_start_site
            tss_rend = tss.right_end_position
            tss_lend = tss.left_end_position
            if strand == '-':
                distance = lend - tss_rend
            if strand == '+':
                distance = rend - tss_lend
        except Exception:
            logging.error(
                f'Can not find Promoter from {reg_entity_id}')
    if distance:
        distance = abs(distance)
    return distance

def get_gene_distance(database, url, regulated_entity, strand, rend, lend):
    '''
    Calculates the distance between the given Regualtory Interaction and the closest Gene.

    Param
        regulated_entity, Object, Regualtory Interaction regulated entity object.
        strand, String, Regualtory Site strand forward or reverse ('-', '+').
        lend, String, Start position in the sequence.
        rend, String, End position in the sequence.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.

    Returns
        distance, Integer, Distance between the given Regualtory Interaction and the closest Gene.
    '''
    mg_api.connect(database, url)
    distance = None
    reg_entity_type = regulated_entity.type
    reg_entity_id = regulated_entity.id
    if reg_entity_type == 'gene':
        try:
            mg_gene = mg_api.genes.find_by_id(reg_entity_id)
            gene_rend = mg_gene.right_end_position
            gene_lend = mg_gene.left_end_position
            if strand == '-':
                distance = lend - gene_rend
            if strand == '+':
                distance = rend - gene_lend
        except Exception:
            logging.error(
                f'Can not find TU from {reg_entity_id}')
    if reg_entity_type == 'transcriptionUnit':
        try:
            mg_tu = mg_api.transcription_units.find_by_id(reg_entity_id)
            genes_ids = mg_tu.genes_ids
            temp_gene_distances = []
            for gene_id in genes_ids:
                mg_gene = mg_api.genes.find_by_id(gene_id)
                gene_rend = mg_gene.right_end_position
                gene_lend = mg_gene.left_end_position
                if strand == '-':
                    temp_gene_distances.append(abs(lend - gene_rend))
                if strand == '+':
                    temp_gene_distances.append(abs(rend - gene_lend))
            temp_gene_distances.sort()
            distance = temp_gene_distances[0]
        except Exception:
            logging.error(
                f'Can not find TU from {reg_entity_id}')
    if reg_entity_type == 'promoter':
        try:
            mg_tu = mg_api.transcription_units.find_by_promoter_id(
                reg_entity_id)
            genes_ids = mg_tu[0].genes_ids
            temp_gene_distances = []
            for gene_id in genes_ids:
                mg_gene = mg_api.genes.find_by_id(gene_id)
                gene_rend = mg_gene.right_end_position
                gene_lend = mg_gene.left_end_position
                if strand == '-':
                    temp_gene_distances.append(abs(lend - gene_rend))
                if strand == '+':
                    temp_gene_distances.append(abs(rend - gene_lend))
            temp_gene_distances.sort()
            distance = temp_gene_distances[0]
        except Exception:
            logging.error(
                f'Can not find TU from {reg_entity_id}')
    if distance:
        distance = abs(distance)
    return distance

def get_classic_ris(lend, rend, strand, tf_sites, database, url, origin):
    '''
    Gets Regualtory Interactions on RegulonDB Multigenomic database.

    Param
        lend, Float, RI's leftEndPosition.
        rend, Float, RI's rigthEndPosition.
        strand, Float, RI's strand.
        tf_sites, List, Sites in the dataset.

    Returns
        classic_ris, List, RIs found on RegulonDB List.
    '''
    center_pos = get_center_pos(lend, rend)
    classic_ris = []
    for site in tf_sites:
        tf_center = site.get('absolutePosition', None)
        site_object = site.get('siteObject', None)
        if tf_center and site_object:
            if tf_center == center_pos or tf_center == (center_pos + EC.PAIR_OF_BASES) or tf_center == (center_pos - EC.PAIR_OF_BASES):
                classic_ri = {}
                ri_regulated_entity = {}
                mg_api.connect(database, url)
                try:
                    mg_ri = mg_api.regulatory_interactions.find_by_reg_site(
                        site_object.id)
                    classic_ri.setdefault('_id', mg_ri[0].id)
                    ri_regulated_entity = mg_ri[0].regulated_entity
                except Exception:
                    logging.error(
                        f'Can not find RI from Site {site_object.id}')
                relative_tss_distance = get_tss_distance(
                    database, url, ri_regulated_entity, strand, site_object.right_end_position, site_object.left_end_position)
                classic_ri.setdefault(
                    'relativeTSSDistance', relative_tss_distance)
                relative_gene_distance = get_gene_distance(
                    database, url, ri_regulated_entity, strand, site_object.right_end_position, site_object.left_end_position)
                classic_ri.setdefault(
                    'relativeGeneDistance', relative_gene_distance)
                classic_ri.setdefault('tfbsLeftPosition',
                                      site_object.left_end_position)
                classic_ri.setdefault('tfbsRightPosition',
                                      site_object.right_end_position)
                classic_ri.setdefault('strand', strand)
                classic_ri.setdefault('sequence',
                                      site_object.sequence)
                citations = get_citations(database, url, site_object.citations)
                classic_ri.setdefault('citations', citations)
                classic_ri.setdefault('origin', origin)
                classic_ris.append(classic_ri)
    return classic_ris

def get_tu_by_gene_id(gene_id, database, url):
    '''
    Gets TU in the RegulonDB Multigenomic database by Gene ID.

    Param
        gene_id, String, Gene ID.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
    Returns
        tu, Object, Trasncription Unit object from RegulonDB Multigenomic database.
    '''
    tu = None
    mg_api.connect(database, url)
    try:
        tu = mg_api.transcription_units.find_by_gene_id(
            gene_id)
    except IndexError:
        # logging.error(f'Can not find TU from: {gene_id}')
        pass
    mg_api.disconnect()
    return tu

def get_promoter(lend, rend, database, url):
    '''
    Gets Closer Promoter in the RegulonDB Multigenomic database by left_end_position and right_end_position.

    Param
        lend, Float, dataset leftEndPosition.
        rend, Float, dataset rigthEndPosition.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
    Returns
        promoters, List, Closer Promoters to dataset in RegulonDB Multigenomic database.
    '''
    promoters = []
    mg_api.connect(database, url)
    try:
        promoters_objects = mg_api.promoters.get_closer_promoters(
            (lend - 5), (rend + 5))
        for promoter_obj in promoters_objects:
            binds_sigma_factor = promoter_obj.binds_sigma_factor
            sigma_factor_name = None
            if binds_sigma_factor:
                sigma_factor_id = binds_sigma_factor.sigma_factors_id
                sigma_factor = mg_api.sigma_factors.find_by_id(sigma_factor_id)
                if sigma_factor:
                    sigma_factor_name = sigma_factor.name
            promoter = {
                '_id': promoter_obj.id,
                'name': promoter_obj.name,
                'strand': promoter_obj.strand,
                'pos+1': promoter_obj.pos1,
                'sigma': sigma_factor_name,
                'confidenceLevel': promoter_obj.confidence_level,
            }
            promoters.append(promoter)
    except IndexError:
        logging.error(f'Can not find Promoter from: {lend}, {rend}')
    mg_api.disconnect()
    return promoters

def get_genes_by_bnumber(bnumbers, database, url):
    '''
    Gets Genes in the RegulonDB Multigenomic database by Gene BNumber.

    Param
        bnumbers, List, Bnumbers String Array.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
    Returns
        genes, List, Genes List from RegulonDB Multigenomic database.
    '''
    genes = []
    mg_genes = []
    mg_api.connect(database, url)
    try:
        for bnumber in bnumbers:
            mg_genes.append(mg_api.genes.find_by_bnumber(bnumber)[0])
        for gene in mg_genes:
            gene_dict = {}
            gene_dict.setdefault('_id', gene.id)
            gene_dict.setdefault('name', gene.name)
            gene_dict.setdefault('bnumber', gene.bnumber)
            genes.append(gene_dict)

    except IndexError:
        logging.error(f'Can not find Gene bnumbers: {bnumbers}')
    mg_api.disconnect()
    return genes

def get_gene_by_bnumber(bnumber, database, url):
    '''
    Gets Gene in the RegulonDB Multigenomic database associated to a Gene BNumber.

    Param
        bnumber, List, Bnumbers String Array.
        database, String, Multigenomic database to get external data.
        url, String, URL where database is located.
    Returns
        gene, Dict, Gene object from RegulonDB Multigenomic database.
    '''
    gene_dict = {}
    mg_api.connect(database, url)
    try:
        gene = mg_api.genes.find_by_bnumber(bnumber)[0]
        gene_dict.setdefault('_id', gene.id)
        gene_dict.setdefault('name', gene.name)
        gene_dict.setdefault('bnumber', gene.bnumber)
        gene_dict.setdefault('synonyms', gene.synonyms)
        gene_dict.setdefault('leftEndPosition', gene.left_end_position)
        gene_dict.setdefault("rightEndPosition", gene.right_end_position)

    except IndexError:
        logging.error(f'Can not find Gene bnumbers: {bnumber}')
    mg_api.disconnect()
    return gene_dict

def find_one_in_dict_list(dict_list, key_name, value):
    '''
    Finds dictionary in a dictionary List by certain key.

    Param
        dict_list, List, Dictionaries List.
        key_name, String, Key Name to search.
        value, String, Value to find the dictionary by key name.
    Returns
        found_dict, Dict, Dictionary that matches the search.
    '''
    found_dict = next(
        (item for item in dict_list if item[key_name] == value),
        None
    )
    return found_dict

def verify_json_path(json_path):
    '''
    This function reads JSON file in the path and returns an valid dir for use

    Param
        json_path, String, raw directory path.

    Returns
        txt_path, String, verified directory path.
    '''

    if os.path.isfile(json_path) and json_path.endswith('.json'):
        logging.info(
            f'Reading JSON file {json_path}')
        return json_path
    else:
        logging.warning(
            f'{json_path} is not a valid JSON file will be ignored')
        return None

def read_json_from_path(json_path):
    '''
    Opens a JSON file and returns JSON object.

    Param
        json_path, String, path to JSON.
    Returns
        Loaded JSON Object.
    '''
    json_file = open(json_path)
    return json.load(json_file)

def get_external_reference(external_ref):
    '''
    [Description]

    Param
        [Description]

    Returns
        [Description]
    '''
    external_references_list = []
    external_references = [external_ref]
    for external_reference in external_references:
        reference_name = external_reference.split(':')[0]
        reference_url = re.search(
            "(?P<url>https?://[^\s]+)", external_reference).group("url")
        external_reference_dict = {
            'name': reference_name,
            'url': reference_url,
            'description': '',
            'internalComment': '',
            'note': '',
        }
        external_reference_dict = {k: v for k,
                                   v in external_reference_dict.items() if v}
        external_references_list.append(external_reference_dict)

    return external_references_list


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2023/9/20 21:02
# @Author  : toby
# @File    : 39.协程-事件循环set_task_factory方法.py
# @Software: PyCharm
# @Desc:
import asyncio

def custom_task_factory(loop, coro):
    task = asyncio.Task(coro, loop=loop)
    task.my_custom_attribute = "This is a custom attribute"
    return task

async def my_coroutine():
    await asyncio.sleep(1)

async def main():
    loop = asyncio.get_event_loop()

    # 设置自定义任务工厂
    loop.set_task_factory(custom_task_factory)

    # 创建协程任务
    task = asyncio.create_task(my_coroutine())
    await task

    # 访问自定义属性
    print(task.my_custom_attribute)

asyncio.run(main())


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, Http404, HttpResponseRedirect
from django.urls import reverse
from django.utils import timezone

from .models import Todoitem

def index(request, **kwargs):
    context = {
        'list_header': 'Anything!',
        'comp_header': 'Look On Your Works',
        'todos': Todoitem.objects.filter(completion_status=False),
        'done_todos': Todoitem.objects.filter(completion_status=True),
    }
    return render(request, 'todo/index.html', context)

def new_todo(request, **kwargs):
    todo_txt = request.POST['todo_txt']
    todo = Todoitem(body=todo_txt, completion_status=False)
    todo.save()
    return HttpResponseRedirect(reverse('todo:index'))

def complete_todo(request):
    todo_id = request.POST['todo_id']
    item = get_object_or_404(Todoitem, pk=todo_id)
    item.completion_status = True
    item.complete_date = timezone.now()
    item.save()
    return HttpResponseRedirect(reverse('todo:index'))


from flask import Flask, render_template,request,flash,redirect

import tensorflow as tf
from colabcode import ColabCode
from fastapi import FastAPI
from pydantic import BaseModel
from fastapi.middleware.cors import CORSMiddleware
from keras.preprocessing.image import ImageDataGenerator
import uvicorn
import numpy as np

import keras.utils as image

class Image(BaseModel):
    imageName : str

train_datagen = ImageDataGenerator(rescale = 1./255,
                                   shear_range = 0.2,
                                   zoom_range = 0.2,
                                   horizontal_flip = True)
training_set = train_datagen.flow_from_directory('training_set',
                                                 target_size = (128, 128),
                                                 batch_size = 128,
                                                 class_mode = 'categorical',
                                                 shuffle=True)
#model loading
cnn=tf.keras.models.load_model("./myModel_v1")

app = FastAPI()

origins = ["*"]
app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)
app=Flask(__name__)

#-------------------------------------------------------------------------------------------------------#
# page d'accueil #
@app.route('/')
def accueil ():
    return render_template('index.html')
###########

# test #
@app.route('/test')
def get_image_category():
    return render_template('test.html')

from werkzeug.utils import secure_filename

import PIL.Image
import cv2

@app.route('/uploader', methods = ['GET', 'POST'])
def upload_file():
   if request.method == 'POST':
      f = request.files['file'].read()
      npimg = np.fromstring(f,np.uint8)
      img = cv2.imdecode(npimg,cv2.IMREAD_COLOR)
      print(img)
      test_image = image.img_to_array(img)
      test_image = np.expand_dims(test_image, axis = 0)
      print(test_image)
      result = cnn.predict(test_image)
      training_set.class_indices
      print(result)
      if result[0][0] == 0:
        prediction = 'Cancer'
      else:
        prediction = 'NORMAL'
      return {'classificaton_result': prediction}

#main#
if __name__=="__main__":
    app.run(debug=True)
#------------------------#


from PIL import Image, ImageFilter

print('''---------------------Pillow---------------------
''')

# PIL：Python Imaging Library，已经是Python平台事实上的图像处理标准库了。PIL功能非常强大，但API却非常简单易用。

# *********** 操作图像 ***********

# 来看看最常见的图像缩放操作，只需三四行代码：
#
# 打开一个jpg图像文件，注意是当前路径:
im = Image.open('test.jpg')
# 获得图像尺寸:
w, h = im.size
print('Original image size: %sx%s' % (w, h))
# 缩放到50%:
im.thumbnail((w // 2, h // 2))
print('Resize image to: %sx%s' % (w // 2, h // 2))
# 把缩放后的图像用jpeg格式保存:
im.save('thumbnail.jpg', 'jpeg')

# 应用模糊滤镜:
im2 = Image.open('test.jpg')
im3 = im2.filter(ImageFilter.BLUR)
im3.save('blur.jpg', 'jpeg')


from copy import copy
import sys
import threading
import wx

import Conversion

class VirtualListCtrl(wx.ListCtrl):
    def __init__(self, cols, dataFunc, parent, customDataToStringFuncs=[], rowIdCol=None, allowSort=True, defaultSortCol='', defaultSortDirection='ASC', id=-1,
                 pos=wx.DefaultPosition, size=wx.DefaultSize, style=wx.LC_REPORT | wx.LC_VIRTUAL | wx.LC_VRULES | wx.CLIP_CHILDREN, **kwargs):
        wx.ListCtrl.__init__(self, parent, id, pos, size, style, **kwargs)
        """
        cols:
            [(*columnName*, *dataKeyword*, *dataType*, *columnWidth*), ...]
        """
        self.dataFunc = dataFunc

        #get a copy of the converter dicts
        self.dataToStringFuncs = copy(Conversion.dataToStringFuncs)

        #add custom funcs
        for funcTup in customDataToStringFuncs:
            if not funcTup[0] in self.dataToStringFuncs:
                self.dataToStringFuncs[funcTup[0]]=funcTup[1]

        #image stuff
        self.imageList, self.images = self._createColSortImages()
        self.SetImageList(self.imageList, wx.IMAGE_LIST_SMALL)

        #columns
        self.rowIdCol = rowIdCol
        self.defaultSortCol = defaultSortCol
        self.defaultSortDirection = defaultSortDirection
        self.allowSort = allowSort

        self.colDefaults = cols #default column data (size and so on)
        self._initColumns(cols) #sets class vars!

        #lock
        self.lock = threading.RLock() #guess...

        #events
        if self.allowSort:
            self.Bind(wx.EVT_LIST_COL_CLICK, self.OnColClick, self) #sort on col click
        self.Bind(wx.EVT_LIST_COL_RIGHT_CLICK, self.OnColRightClick, self) #display options on right click on col
        self.Bind(wx.EVT_LIST_COL_END_DRAG, self.OnColResize, self) #update the size info for the resized column

        #tell ListCtrl how many items we currently have
        self.SetItemCount(0)

    ##internal functions - images

    def _createColSortImages(self):
        imageList = wx.ImageList(15, 15, mask=True)
        images = {}
        for imageName, imageIdent in (('arrowUp', wx.ART_GO_UP), ('arrowDown', wx.ART_GO_DOWN)):
            #add one image to the list
            orgImg = wx.ArtProvider_GetBitmap(imageIdent, wx.ART_OTHER, (14,14)).ConvertToImage()
            if (not isinstance(orgImg, type(wx.NullBitmap))) and sys.platform[:5] == 'linux' and orgImg.HasAlpha():
                #looks ok, init empty new image
                newImg = wx.EmptyImage(15, 15, clear=True)
                newImg.InitAlpha()
                for x in xrange(0, 15):
                    for y in xrange(0, 15):
                        newImg.SetAlpha(x, y, 0)

                #copy original image into new one
                newImg.Paste(orgImg.GetSubImage((1,0,12,12)), 1, 2)
                if orgImg.HasAlpha():
                    for x in xrange(0, 12):
                        for y in xrange(0, 12):
                            newImg.SetAlpha(x+1, y+2, orgImg.GetAlpha(x+1, y))

            else:
                #this looks borked (Windows?), draw bitmap from scratch
                orgImg = wx.EmptyBitmap(15, 15)
                memDc = wx.MemoryDC()
                memDc.SelectObject(orgImg)

                #draw background
                memDc.SetPen(wx.Pen('WHITE', 1))
                memDc.SetBrush(wx.Brush('WHITE', wx.SOLID))
                memDc.DrawRectangle(0, 0, 15, 15)

                #draw triangle
                memDc.SetPen(wx.Pen('BLACK', 1))
                memDc.SetBrush(wx.Brush('BLACK', wx.SOLID))
                if imageIdent is wx.ART_GO_UP:
                    memDc.DrawPolygon(((3,11), (11,11), (7,3)))
                elif imageIdent is wx.ART_GO_DOWN:
                    memDc.DrawPolygon(((3,3), (11,3), (7,11)))

                memDc.SelectObject(wx.NullBitmap)

                #convert to image, apply alpha
                newImg = orgImg.ConvertToImage()
                newImg.InitAlpha()
                for x in xrange(0, 15):
                    for y in xrange(0, 15):
                        if newImg.GetGreen(x, y) == 0:
                            #black
                            newImg.SetAlpha(x, y, 95)
                        else:
                            #white
                            newImg.SetAlpha(x, y, 0)

            #finally convert back to bitmap and store in dict
            images[imageName] = imageList.Add(newImg.ConvertToBitmap())

        return imageList, images

    def _applyColSortImages(self, sortColumn, sortDirection):
        self.ClearColumnImage(self.sortColumn)
        if sortDirection == 'ASC':
            self.SetColumnImage(sortColumn, self.images['arrowDown'])
        else:
            self.SetColumnImage(sortColumn, self.images['arrowUp'])

    ##internal functions - columns

    def _createColumnInfo(self, cols):
        #cols = [(*columnName*, *dataKeyword*, *dataType*, *columnWidth*, *active*), ...]

        colInfo = {}   #columns info stuff
        colMapper = [] #maps a column name to a position

        #build up mapper and info dict
        place = 0
        while place < len(cols):
            #process one col
            colSet = cols[place]
            colInfo[colSet[0]] = {'dataKeyword':colSet[1],
                                  'dataType':colSet[2],
                                  'columnWidth':colSet[3]}
            if colSet[4]:
                #active column
                colMapper.append(colSet[0])
            place += 1
        return colInfo, colMapper

    def _applyColumnInfo(self, colInfo, colMapper):
        #create the 'real' (=gui) columns
        for i in xrange(0, len(colMapper)):
            colName = colMapper[i]
            self.InsertColumn(i, colName)
            self.SetColumnWidth(i, colInfo[colName]['columnWidth'])

    def _createColumns(self, colInfo, colMapper):
        #create the columns itself
        self._applyColumnInfo(colInfo, colMapper)

        #set class vars
        self.colInfo = colInfo            #column info
        self.colMapper = colMapper        #maps a column name to a position
        self.colNum = len(self.colMapper) #number of columns
        self.colData = dict(((colName, []) for colName in self.colInfo)) #row data, per column one list

    def _initColumns(self, cols):
        #initialises everything columns related
        colInfo, colMapper = self._createColumnInfo(cols)
        self._createColumns(colInfo, colMapper)

        #index of the column, which is used for sorting
        if self.defaultSortCol in colMapper:
            self.sortColumn = colMapper.index(self.defaultSortCol)
        else:
            self.sortColumn = 0
        self.sortDirection = self.defaultSortDirection #sort direction, either ASC or DESC
        self._applyColSortImages(self.sortColumn, self.sortDirection)

    def _resetColumns(self):
        #reset everything back to the defaults
        self.DeleteAllColumns()                                         #remove all existing columns
        colInfo, colMapper = self._createColumnInfo(self.colDefaults)   #build the needed vars out of the defaults
        self._createColumns(colInfo, colMapper)                         #create new columns according to these default

        #index of the column, which is used for sorting
        if self.defaultSortCol in colMapper:
            self.sortColumn = colMapper.index(self.defaultSortCol)
        else:
            self.sortColumn = 0
        self.sortDirection = self.defaultSortDirection #sort direction, either ASC or DESC
        self._applyColSortImages(self.sortColumn, self.sortDirection)
        self.dataUpdate() #refill the columns

    ##internal functions - data

    def _dataUpdate(self, data):
        oldDataAmount = len(self.colData[self.colMapper[0]])

        #remember selected rows if possible
        if self.rowIdCol is not None:
            rowIdColData = self.colData[self.rowIdCol]
            selectedRows = self._getSelectedRows()
            selectedRowIds = [rowIdColData[rowIdx] for rowIdx in selectedRows if rowIdx < len(rowIdColData)]
            for rowIdx in selectedRows:
                self.Select(rowIdx, on=0)

        #create sort list
        colName = self.colMapper[self.sortColumn]
        dataKeyword = self.colInfo[colName]['dataKeyword']
        if type(dataKeyword)==str:
            #single item
            sortList = (row[dataKeyword] for row in data)
        else:
            #multiple items
            sortList = []
            for row in data:
                sortList.append([row[singleDataKeyword] for singleDataKeyword in dataKeyword])

        #add row index
        sortList = map(lambda tup: (tup[1], tup[0]), enumerate(sortList))

        #sort the list
        sortList.sort(reverse=(self.sortDirection == 'DESC'))

        #create new data list
        for colName in self.colData.keys():
            #fill one column
            colDataKeyword = self.colInfo[colName]['dataKeyword']
            if type(colDataKeyword)==str:
                #only a single data item
                self.colData[colName] = [data[rowNum][colDataKeyword] for rowData, rowNum in sortList]

            else:
                #multiple data items
                colData = []
                for rowData, rowNum in sortList:
                    colData.append([data[rowNum][singleDataKeyword] for singleDataKeyword in colDataKeyword])
                self.colData[colName] = colData

        #set new item length if rows got added or removed
        newDataAmount = len(sortList)
        if newDataAmount != oldDataAmount:
            self.SetItemCount(newDataAmount)

        #restore row selection if possible
        if self.rowIdCol is not None:
            rowIdColData = self.colData[self.rowIdCol]
            selectedRows = [rowIdColData.index(rowId) for rowId in selectedRowIds if rowId in rowIdColData]
            for rowIdx in sorted(selectedRows):
                self.Select(rowIdx, on=1)

        #refresh currently displayed items
        firstDisplayedRow = self.GetTopItem()
        lastDisplayedRow = min(firstDisplayedRow + self.GetCountPerPage(), newDataAmount)
        self.RefreshItems(firstDisplayedRow, lastDisplayedRow)

        #windows is a bit strange and requires this call (the "RefreshItems" call should do the same ...)
        if sys.platform[:3] == 'win':
            if newDataAmount >= oldDataAmount:
                self.Refresh(False)
            else:
                self.Refresh(True)

    ##internal functions - columns

    def _disableCol(self, colName):
        if self.colNum > 1 and not (self.allowSort == False and self.defaultSortCol == colName):
            #ok to disable column, its not the last one
            colPos = self.colMapper.index(colName)
            del self.colMapper[colPos]
            self.colNum -= 1
            self.DeleteColumn(colPos)

            #clean the whole sorting stuff up
            if self.sortColumn == colPos:
                #we deleted the row which is used for sorting, default back to the first col
                self.sortColumn = 0
                self.sortDirection = 'ASC'
                self._applyColSortImages(self.sortColumn, self.sortDirection)

            elif self.sortColumn > colPos:
                #we sort depending on a column which is after the deleted column
                self.sortColumn -= 1

        self.Refresh()

    def _enableCol(self, colName, colPos):
        if colPos is None:
            colPos = self.colNum

        #add col
        self.InsertColumn(colPos, colName)
        self.SetColumnWidth(colPos, self.colInfo[colName]['columnWidth'])
        self.colMapper.insert(colPos, colName)
        self.colNum += 1

        #cleanup sorting
        if self.sortColumn >= colPos:
            #we added a row before or at the position of the sorting column
            self.sortColumn += 1

        self.dataUpdate()

    def _moveCol(self, colName, direction):
        colPos = self.colMapper.index(colName)
        if (colPos + direction) >= 0 and (colPos + direction) < self.colNum:
            self.DeleteColumn(colPos)
            self.InsertColumn(colPos + direction, colName)
            self.SetColumnWidth(colPos + direction, self.colInfo[colName]['columnWidth'])
            del self.colMapper[colPos]
            self.colMapper.insert(colPos + direction, colName)

            #clean the whole sorting stuff up
            if direction < 0 and self.sortColumn < colPos and self.sortColumn >= colPos+direction:
                self.sortColumn += 1

            elif direction > 0 and self.sortColumn > colPos and self.sortColumn <= colPos+direction:
                self.sortColumn -= 1

            elif self.sortColumn == colPos:
                self.sortColumn += direction

            self._applyColSortImages(self.sortColumn, self.sortDirection)

        self.dataUpdate()

    ##internal functions - events

    def _onColClick(self, event):
        clickedCol = event.GetColumn()
        if not clickedCol == self.sortColumn:
            #use another column for sorting stuff
            self._applyColSortImages(clickedCol, 'ASC')
            self.sortColumn = clickedCol
            self.sortDirection = 'ASC'
        else:
            #just reverse sorting order
            if self.sortDirection == 'ASC':
                self._applyColSortImages(self.sortColumn, 'DESC')
                self.sortDirection = 'DESC'
            else:
                self._applyColSortImages(self.sortColumn, 'ASC')
                self.sortDirection = 'ASC'

        self.dataUpdate()

    def _onColRightClick(self, event):
        cols = []
        for colName in self.colData.iterkeys():
            active = (colName in self.colMapper)
            cols.append((colName, active))
        cols.sort()

        clickedCol = event.GetColumn()
        if clickedCol == -1:
            merk = ColumnSelectionPopup(cols, self, None, None, None)
        else:
            merk = ColumnSelectionPopup(cols, self, self.colMapper[clickedCol], clickedCol, len(self.colMapper))
        self.PopupMenu(merk)

    def _onColResize(self, event):
        resizedColNum = event.GetColumn()
        resizedColName = self.colMapper[resizedColNum]
        newSize = self.GetColumnWidth(resizedColNum)
        self.colInfo[resizedColName]['columnWidth'] = newSize

    ##internal functions - other

    def _getRowCount(self):
        return len(self.colData[self.colMapper[0]])

    def _getRawData(self, colName, rowIndex):
        return self.colData[colName][rowIndex]

    def _getSelectedRows(self):
        selectedRows = []
        selected = self.GetFirstSelected()
        while not selected==-1:
            selectedRows.append(selected)
            selected = self.GetNextSelected(selected)
        return selectedRows

    ##external functions - data

    def getRawData(self, colName, rowIndex):
        self.lock.acquire()
        data = self._getRawData(colName, rowIndex)
        self.lock.release()
        return data

    def getSelectedRows(self):
        self.lock.acquire()
        rows = self._getSelectedRows()
        self.lock.release()
        return rows

    def clear(self):
        self.lock.acquire()
        self._dataUpdate([])
        self.lock.release()

    def dataUpdate(self):
        self.lock.acquire()
        data = self.dataFunc()
        self._dataUpdate(data)
        self.lock.release()

    ##external functions - columns

    def disableCol(self, colName):
        self.lock.acquire()
        self._disableCol(colName)
        self.lock.release()

    def enableCol(self, colName, colPos=None):
        self.lock.acquire()
        self._enableCol(colName, colPos)
        self.lock.release()

    def moveCol(self, colName, direction):
        self.lock.acquire()
        self._moveCol(colName, direction)
        self.lock.release()

    def resetCols(self):
        self.lock.acquire()
        self._resetColumns()
        self.lock.release()

    ##external functions - events

    def OnColClick(self, event):
        self.lock.acquire()
        self._onColClick(event)
        self.lock.release()

    def OnColRightClick(self, event):
        self.lock.acquire()
        self._onColRightClick(event)
        self.lock.release()

    def OnColResize(self, event):
        self.lock.acquire()
        self._onColResize(event)
        self.lock.release()

    def OnGetItemText(self, rowIndex, columnIndex):
        self.lock.acquire()
        if columnIndex >= self.colNum or rowIndex >= len(self.colData[self.colMapper[0]]):
            data = ''
        else:
            colName = self.colMapper[columnIndex]
            data = self.colData[colName][rowIndex]
            if data is None:
                data = ''
            else:
                data = self.dataToStringFuncs[self.colInfo[colName]['dataType']](data)
        self.lock.release()
        return data

class ColumnSelectionPopup(wx.Menu):
    def __init__(self, cols, parentDiag, clickedCol, clickedColNum, maxColNum, *args, **kwargs):
        wx.Menu.__init__(self, *args, **kwargs)
        #static
        self.parentDiag = parentDiag
        self.clickedCol = clickedCol
        self.clickedColNum = clickedColNum
        self.maxColNum = maxColNum

        id = wx.NewId()
        self.Append(id, 'Reset to defaults', 'Restore the default column ordering and size')
        self.Bind(wx.EVT_MENU, self.OnColumnReset, id=id)
        self.AppendSeparator()

        if clickedCol is not None:
            #move funcs
            id = wx.NewId()
            self.Append(id, 'Move to left', 'Moves column one step to the left')
            self.Bind(wx.EVT_MENU, self.OnMoveToLeft, id=id)

            id = wx.NewId()
            self.Append(id, 'Move x steps', 'Moves column x steps')
            self.Bind(wx.EVT_MENU, self.OnMoveToX, id=id)

            id = wx.NewId()
            self.Append(id, 'Move to right', 'Moves column one step to the right')
            self.Bind(wx.EVT_MENU, self.OnMoveToRight, id=id)

            self.AppendSeparator()

        #enable/disable columns
        self.idMapper = {}
        for tup in cols:
            id = wx.NewId()
            m = self.AppendCheckItem(id, tup[0], 'Activates or deactivates this column')
            #m.SetCheckable(True)
            if tup[1]==True:
                m.Toggle()
            self.idMapper[id]=tup[0]

            #event
            self.Bind(wx.EVT_MENU, self.OnClick, id=id)

    def OnClick(self, event):
        colName = self.idMapper[event.GetId()]
        if event.IsChecked() == True:
            if self.clickedColNum is not None:
                self.parentDiag.enableCol(colName, colPos=self.clickedColNum + 1)
            else:
                self.parentDiag.enableCol(colName, colPos=None)
        else:
            self.parentDiag.disableCol(colName)
        #event.Skip(False)

    def OnMoveToLeft(self, event):
        self.parentDiag.moveCol(self.clickedCol, -1)

    def OnMoveToX(self, event):
        min = self.clickedColNum * -1
        max = self.maxColNum - self.clickedColNum - 1
        diag = wx.NumberEntryDialog(self.parentDiag, 'How many steps should the column be moved?', 'Steps (left = -, right = +):', 'Move Column', 0, min, max)
        if diag.ShowModal() == wx.ID_OK:
            self.parentDiag.moveCol(self.clickedCol, diag.GetValue())

    def OnMoveToRight(self, event):
        self.parentDiag.moveCol(self.clickedCol, 1)

    def OnColumnReset(self, event):
        self.parentDiag.resetCols()

class PersistentVirtualListCtrl(VirtualListCtrl):
    def __init__(self, persister, persistPrefix, updateFunc, currentVersion, cols, dataFunc, parent, customDataToStringFuncs=[], rowIdCol=None, allowSort=True, defaultSortCol='', defaultSortDirection='ASC',
                 id=-1, pos=wx.DefaultPosition, size=wx.DefaultSize, style=wx.LC_REPORT | wx.LC_VIRTUAL | wx.LC_VRULES, **kwargs):
        self.persister = persister
        self.persistKey = persistPrefix + 'columnInfo'
        self.updateFunc = updateFunc
        self.currentVersion = currentVersion

        VirtualListCtrl.__init__(self, cols, dataFunc, parent, customDataToStringFuncs, rowIdCol, allowSort=allowSort, defaultSortCol=defaultSortCol, defaultSortDirection=defaultSortDirection, id=id, pos=pos, size=size, style=style, **kwargs)

    ##internal funcs - persisting

    def _initColumns(self, cols):
        #initialises everything columns related
        persColData = self.persister.get(self.persistKey, None)
        if persColData is None:
            #nothing persisted exists
            colInfo, colMapper = self._createColumnInfo(cols)
            self.sortDirection = self.defaultSortDirection
            if self.defaultSortCol in colMapper:
                self.sortColumn = colMapper.index(self.defaultSortCol)
            else:
                self.sortColumn = 0
        else:
            #managed to get persisted data, update
            persColData = self.updateFunc(persColData, self.currentVersion)

            #restore
            colInfo = persColData['colInfo']
            colMapper = persColData['colMapper']
            self.sortColumn = persColData['sortColumn']
            self.sortDirection = persColData['sortDirection']

            #check for new and changed cols
            for colSet in cols:
                if colSet[0] not in colInfo:
                    #new, add it
                    colInfo[colSet[0]] = {'dataKeyword':colSet[1],
                                          'dataType':colSet[2],
                                          'columnWidth':colSet[3]}

                else:
                    #old, update dataKeyword and dataType in case anything changed
                    colInfo[colSet[0]]['dataKeyword'] = colSet[1]
                    colInfo[colSet[0]]['dataType'] = colSet[2]

        self._createColumns(colInfo, colMapper)
        self._applyColSortImages(self.sortColumn, self.sortDirection)

    def _persist(self):
        persColData = {'colInfo':self.colInfo,
                       'colMapper':self.colMapper,
                       'sortColumn':self.sortColumn,
                       'sortDirection':self.sortDirection,
                       'version':self.currentVersion}
        self.persister.store(self.persistKey, persColData)

    ##internal funcs - other

    def _resetColumns(self):
        VirtualListCtrl._resetColumns(self)
        self._persist()

    def _onColClick(self, event):
        VirtualListCtrl._onColClick(self, event)
        self._persist()

    def _onColResize(self, event):
        VirtualListCtrl._onColResize(self, event)
        self._persist()

    def _disableCol(self, colName):
        VirtualListCtrl._disableCol(self, colName)
        self._persist()

    def _enableCol(self, colName, colPos):
        VirtualListCtrl._enableCol(self, colName, colPos)
        self._persist()

    def _moveCol(self, colName, direction):
        VirtualListCtrl._moveCol(self, colName, direction)
        self._persist()


import datamanage
import datetime

if __name__=='__main__':
    dm = datamanage.datamanage(startdate=(datetime.datetime.today()-datetime.timedelta(5)).strftime('%Y%m%d'))
    dm.getstocksdaily()
    dm.getbasicdaily()
    dm.getindexdaily()
    dm.getadjfactor()
    dm.getfundsdaily()


import json
import requests
from django.shortcuts import render,reverse
from django.views.decorators.csrf import csrf_exempt
from django.http import *
from web.forms.file import FileFolderModelForm, FileModelForm
from django.http import JsonResponse
from web.models import *
from utils.tencent.cos import *

def file(request, project_id):
    '''文件列表'''
    # 区分父级目录
    parent_object = None
    folder_id = request.GET.get('folder', "")
    if folder_id.isdecimal():
        parent_object = FileRepository.objects.filter(id=int(folder_id), file_type=2,
                                                      project=request.saas.project).first()
    # 查看页面`
    if request.method == 'GET':
        # 导航条
        breadcrumb_list = []
        parent = parent_object
        while parent:
            breadcrumb_list.insert(0, {'id': parent.id, 'name': parent.name})
            parent = parent.parent

        # 当前目录下的所有文件 $ 文件夹获取到即可
        queryset = FileRepository.objects.filter(project=request.saas.project)
        if parent_object:
            # 进入了某个目录
            file_object_list = queryset.filter(parent=parent_object).order_by('-file_type')  # 倒序,文件夹在上文件在下
        else:
            # 根目录
            file_object_list = queryset.filter(parent__isnull=True).order_by('-file_type')
        form = FileFolderModelForm(request, parent_object)
        context = {
            'form': form,
            "file_object_list": file_object_list,
            "breadcrumb_list": breadcrumb_list,
            'folder_object': parent_object
        }
        return render(request, 'file.html', context)
    # 文件夹添加 & 修改
    fid = request.POST.get('fid', '')
    edit_object = None
    if fid.isdecimal():
        # 修改
        edit_object = FileRepository.objects.filter(id=int(fid), file_type=2,
                                                    project=request.saas.project).first()
    if edit_object:
        form = FileFolderModelForm(request, parent_object, data=request.POST, instance=edit_object)
    else:
        form = FileFolderModelForm(request, parent_object, data=request.POST)
    if form.is_valid():
        form.instance.project = request.saas.project
        form.instance.file_type = 2
        form.instance.update_user = request.saas.user
        form.instance.parent = parent_object
        form.save()
        return JsonResponse({'status': True})

    return JsonResponse({'status': False, 'error': form.errors})

def file_delete(request, project_id):
    """删除文件"""
    fid = request.GET.get('fid')
    # 删除数据库文件和文件夹(级联删除)
    delete_object = FileRepository.objects.filter(id=fid, project=request.saas.project).first()
    if delete_object.file_type == 1:
        # 删除文件(数据库文件/cos文件),并且把空间还回去
        # 删除文件,将容量还回去
        request.saas.project.use_space -= delete_object.file_size
        request.saas.project.save()
        # 去cos中删除文件
        delete_file(request.saas.project.bucket, request.saas.project.region, delete_object.key)
        delete_object.delete()
        return JsonResponse({'status': True})
    else:
        # 删除文件夹(找到文件夹下面的所有文件,删除并且把空间还回去)
        total_size = 0
        key_list = []
        folder_list = [delete_object, ]
        for folder in folder_list:
            child_list = FileRepository.objects.filter(project=request.saas.project, parent=folder).order_by(
                '-file_type')
            for child in child_list:
                if child.file_type == 2:
                    folder_list.append(child)
                else:
                    # 文件大小汇总
                    total_size += child.file_type

                    # 删除文件
                    key_list.append({"Key": child.key})

        # cos 批量删除文件
        if key_list:
            delete_file_list(request.saas.project.bucket, request.saas.project.region, key_list)
        # 归还容量
        if total_size:
            request.saas.project.use_space -= total_size
            request.saas.project.save()

        delete_object.delete()
        return JsonResponse({'status': True})

@csrf_exempt
def cos_credential(request, project_id):
    '''获取上传临时凭证'''
    per_file_limit = request.saas.price_policy.per_file_size * 1024 * 1024  # 限制 单个文件
    total_file_limit = request.saas.price_policy.per_file_size * 1024 * 1024 * 1024  # 限制   单个项目
    file_list = json.loads(request.body.decode('utf-8'))
    total_size = 0
    # 单个容量限制
    for item in file_list:
        # 文件字节大小 item['size]=B  单文件限制大小 M
        if item['size'] > per_file_limit:
            # 超出限制
            msg = '单文件超出限制(最大{}M),文件:{},请升级套餐'.format(request.saas.price_policy.per_file_size, item['name'])
            return JsonResponse({'status': False, 'error': msg})
    # 总容量限制
    # request.saas.price_policy.project_space #项目允许使用的空间
    # request.saas.project.use_space #项目已使用的空间
    if request.saas.project.use_space + total_size > total_file_limit:
        # 超出限制
        return JsonResponse({'status': False, 'error': '容量超过限制,请升级套餐'})
    data_dict = credential(request.saas.project.bucket, request.saas.project.region)

    return JsonResponse({'status': True, 'data': data_dict})

@csrf_exempt
def file_post(request, project_id):
    """ 已上传成功的文件写入到数据 """
    """
    name: fileName,
    key: key,
    file_size: fileSize,
    parent: CURRENT_FOLDER_ID,
    # etag: data.ETag,
    file_path: data.Location
    """

    # 根据key再去cos获取文件Etag和"db7c0d83e50474f934fd4ddf059406e5"

    print(request.POST)
    # 把获取到的数据写入数据库即可
    form = FileModelForm(request, data=request.POST)
    if form.is_valid():
        # 校验通过,写入数据库
        # form.instance.file_type = 1
        # form._update_user = request.saas.user
        data_dict = form.cleaned_data
        data_dict.pop('etag')
        data_dict.update({'project': request.saas.project, 'file_type': 1, 'update_user': request.saas.user})
        instance = FileRepository.objects.create(**data_dict)
        #项目的已使用空间
        request.saas.project.use_space += data_dict['file_size']
        request.saas.project.save()

        result = {
            'id': instance.id,
            'name': instance.name,
            'file_size': instance.file_size,
            'username': instance.update_user.name,
            'datetime': instance.update_datetime.strftime('%Y年-%m月-%d日 %H:%M'),
            # 'file_type':instance.get_file_type_display()
            'download_url':reverse('file_download',kwargs = {'project_id':project_id,'file_id':instance.id})
        }
        return JsonResponse({'status': True, 'data': result})
    return JsonResponse({'status': False, 'data': '文件错误'})

def file_download(request, project_id,file_id):
    '''下载文件'''
    file_object = FileRepository.objects.filter(id=file_id, project_id=project_id).first()
    res = requests.get(file_object.file_path)

    # 文件分块处理（适用于大文件）     @孙歆尧
    data = res.iter_content()

    # 设置content_type=application/octet-stream 用于提示下载框        @孙歆尧
    response = HttpResponse(data, content_type="application/octet-stream")
    from django.utils.encoding import escape_uri_path

    # 设置响应头：中文件文件名转义      @王洋
    response['Content-Disposition'] = "attachment; filename={};".format(escape_uri_path(file_object.name))
    return response


from darkflow.net.build import TFNet
import cv2
import os
import json

options = {"model":"./cfg/yolo.cfg","load": "./yolo.weights", "threshold": 0.25,"gpu":1}
tfnet = TFNet(options)

def car_detection(input_image,output_image):

  filename = os.path.basename(input_image)
  output_dict = {"filename":filename,
                 "label":0,
                 "width":0,
                 "height":0,
                 "num_of_points":0}

  imgcv = cv2.imread('%s' % input_image)

  output_dict["height"] = imgcv.shape[0]
  output_dict["width"] = imgcv.shape[1]

  result = tfnet.return_predict(imgcv)

  #result = tfnet.return_predict(frame)

  num_of_points = 0
  kigou = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'

  for item in result:
    tlx = item['topleft']['x']
    tly = item['topleft']['y']
    brx = item['bottomright']['x']
    bry = item['bottomright']['y']
    label = item['label']
    conf = item['confidence']

    center_x = int(tlx + (brx - tlx)/2)
    center_y = int(tly + (bry - tly)/2)

    if label == 'car' or label == 'bus' or label == 'truck':
      num_of_points += 1

      newdict = {"%s" % kigou[num_of_points-1]:{"x":center_x,"y":center_y}}
      output_dict["num_of_points"] = num_of_points
      output_dict.update(newdict)

      cv2.rectangle(imgcv, (tlx, tly), (brx, bry), (0,0,255), 5)
      text = label + " " + ('%.2f' % conf)
      cv2.rectangle(imgcv, (tlx, tly - 15), (tlx + 100, tly + 5), (0,0,255), -1)
      cv2.putText(imgcv, text, (tlx, tly), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,0,0), 1)
      cv2.circle(imgcv, (center_x, center_y), 15, (255, 255, 0), -1)
    else:
      pass

  cv2.imwrite('%s' % output_image, imgcv)

  f = open('./images_testing/images_testing.json','a')
  f.write(',\n')
  json.dump(output_dict, f, indent=2, separators=(',',':'))
  f.close()

  print('%s' % output_image)
  #print(output_dict)

def resize(input_image,output_image):
  imgcv = cv2.imread('%s' % input_image)
  #hight = imgcv.shape[0]
  #width = imgcv.shape[1]

  image_resized = cv2.resize(imgcv, (1080, 1440))
  cv2.imwrite('%s' % output_image, image_resized)

  print('%s' % output_image)

if __name__ == '__main__':
  car_detection('./image04.jpg','./output/04/output01.jpg')


# << 뮤직카우의 곡별 정보 크롤링 코드 >>
# 작성자 : 정예원
#
# 코드설명
# [[ 뮤직카우 정보란에 '1,2차 옥션결과' 와 '저작권 정보' 를 모으는 코드 입니다 ]]
# 1. 곡 페이지 링크, 1,2 차 옥션 결과 , 공표일자, 저작권료 지분  을 크롤링 합니다.
# 2. 크롤링된 데이터는 musicInfo 명의 콜렉션으로 디비에 저장됩니다.
# 3. 한번 크롤링된 데이터는 변동되지 않으므로 다시 크롤링 할 필요가 없기 때문에,
#    musicCowData에 있는, 그러나 musicInfo에 없는 곡들에 대해서만 크롤링을 합니다.
# 4. musicInfo 에 저장되는 song_title과 song_artist는 따로 split 을 거치지 않은 뮤직카우 에 등록된 곡명과 가수명 입니다.
# 5. 크롤링 완료 후 다시 코드를 돌리면 "이미 musicInfo에 모든 데이터가 들어있습니다" 메시지를 print 합니다.
#
# 코드 수행시간 : 5분

import requests
import re
from bs4 import BeautifulSoup
from pymongo import MongoClient
import numpy as np

def musicInfoCrawler(col1, col4):
    musicInfoSongNumListCurrent = col4.find({}, {'num': {"$slice": [1, 1]}})  # 현재 musicInfoData 디비에 있는 곡 번호 리스트
    musicCowSongNumListCurrent = col1.find({}, {'num': {"$slice": [1, 1]}})
    # 현재 musicCowData <-> musicInfoData 디비 비교했을 때, "musicCowData에 있는, 그러나 musicInfo에 없는 곡"들을 정확히 찾아내기 위해
    # 정확히 지금 현재 시점에서 musicCowData 에 있는 곡들을 다시 한번 탐색하여 self.musicCowSongNumCurrent 변수에 할당.

    songList = []  # musicCowData에 있는, 그러나 musicInfo에 없는 곡
    for x in musicCowSongNumListCurrent:
        songList.append(x['num'])
    for x in musicInfoSongNumListCurrent:
        if x['num'] in songList:
            songList.remove(x['num'])

    if len(songList) == 0:
        print("이미 musicInfo에 모든 데이터가 들어있습니다")
    else:
        for x in songList:
            page = "https://www.musicow.com/song/{0}?tab=info".format(x)
            url = requests.get(page)
            html = url.text
            soup = BeautifulSoup(html, 'html.parser')

            song_title = str(soup.select('div.song_header > div.song_header--bar.song_header--bar_1 > div.song_header--title.txt_ellipsis > strong'))
            song_title = re.sub('\<.+?>|\[|\]', '', song_title, 0).replace('&', '&').strip()

            #print("{0}번 곡 확인 중".format(x))

            crawl_link(x, col4, soup, page, song_title)

        print("========= << 곡 information 크롤링을 마쳤습니다 >> ========== ")

def crawl_link(x, col4, soup, page, song_title):
    print("{0}번 곡 뮤직카우 크롤링 시작".format(x))

    song_artist = str(soup.select('div.card_body > div > dl > dd:nth-child(4)'))
    song_artist = re.sub('\<.+?>|\[|\'|\]', '', song_artist, 0).replace('&', '&').strip()

    auc_date_1 = str(soup.select('div:nth-child(1) > h2 > small'))
    auc_date_1 = re.sub('\<.+?>|\[|\'|\]', '', auc_date_1, 0).strip()
    auc_date_1_start = auc_date_1.split('~')[0].strip()
    auc_date_1_end = auc_date_1.split('~')[1].strip()

    auc_stock_1 = str(soup.select('div.card_body > div > div:nth-child(1) > dl > dd:nth-child(2)'))
    auc_stock_1 = re.sub('\<.+?>|\[|\'|\]|\,', '', auc_stock_1, 0).replace('주','').strip()
    auc_price_1 = str(soup.select('div.card_body > div > div:nth-child(1) > dl > dd:nth-child(8)'))
    auc_price_1 = re.sub('\<.+?>|\[|\'|\]|\,', '', auc_price_1, 0).replace('캐쉬','').strip()

    auc_date_2 = str(soup.select('div:nth-child(2) > h2 > small'))
    auc_date_2 = re.sub('\<.+?>|\[|\'|\]', '', auc_date_2, 0).strip()
    if auc_date_2[0:-1] == '':
        # None 말고 NaN과 0 이라는 값 넣은 이유는 string과 int로 type 통일하기 위해서
        auc_date_2_start = np.nan; auc_date_2_end = np.nan
        auc_stock_2 = np.nan; auc_price_2 = np.nan
    else:
        auc_date_2_start = auc_date_2.split('~')[0].strip()
        auc_date_2_end = auc_date_2.split('~')[1].strip()

    auc_stock_2 = str(soup.select('div.card_body > div > div:nth-child(2) > dl > dd:nth-child(2)'))
    auc_stock_2 = re.sub('\<.+?>|\[|\'|\]|\,', '', auc_stock_2, 0).replace('주','').strip()
    auc_price_2 = str(soup.select('div.card_body > div > div:nth-child(2) > dl > dd:nth-child(8)'))
    auc_price_2 = re.sub('\<.+?>|\[|\'|\]|\,', '', auc_price_2, 0).replace('캐쉬','').strip()

    song_release_date = str(soup.select('div.card_body > div > dl > dd:nth-child(2)'))
    song_release_date = re.sub('\<.+?>|\[|\'|\]', '', song_release_date, 0).strip()

    # stock_num 앞뒤 공백 제거 후 숫자만 추출
    stock_num = str(soup.select('div.lst_copy_info dd p')).split('2차적')[0]
    stock_num = re.sub('\<.+?>|\[|\'|\]|\t|\n|\,', '', stock_num, 0).replace('1/','').strip()

    print("{0}번 곡 DB 입력 중".format(x))

    list_music = {
        'num': x,
        'song_title': song_title,
        'song_artist': song_artist,
        'page': page,
        'auc1_info': {
            'auc_start_date': auc_date_1_start, 'auc_end_date': auc_date_1_end,
            'auc_stock': auc_stock_1, 'auc_price': auc_price_1},
        'auc2_info': {
            'auc_start_date': auc_date_2_start, 'auc_end_date': auc_date_2_end,
            'auc_stock': auc_stock_2, 'auc_price': auc_price_2},
        'song_release_date': song_release_date,
        'stock_num': stock_num
    }

    col4.insert_one(list_music).inserted_id


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from telegram.constants import PARSEMODE_HTML

def send_message(update, context, msg, parse_HTML = False) -> None:
    if parse_HTML:
        context.bot.send_message(chat_id=update.effective_chat.id, text = msg, parse_mode = PARSEMODE_HTML)
    else:
        context.bot.send_message(chat_id=update.effective_chat.id, text = msg)

def c_bot(update, context) -> None:
    # /bot
    send_message(update, context, 'Hello there! I\'m JarvisBot, a bot for group management.\n\nFor a list of commands, type /commands')

def c_commands(update, context) -> None:
    # /commands
    send_message(update, context, 'Command list for JarvisBot:\n/bot - Display general info about the bot\n/commands - Display a list of commands\n/lyrics - Display the lyrics of a song')

def c_kick(update, context) -> None:
    chat_id = update.message.chat_id
    prev_msg = update.message.reply_to_message

    if not prev_msg:
        msg = 'Usage: reply to a message of the user you want to kick with /kick'
    else:
        user_id = prev_msg.from_user.id
        bot = context.bot
        code = kick_member(bot, chat_id, user_id)

        if code == 1:
            msg = 'User kicked succesfully. Good bye, bitch!'
        elif code == -1:
            msg = 'Couldn\'t kick the user'
        elif code == -2:
            msg = 'Not enough rights to kick chat member'
        elif code == -3:
            msg = 'User is an administrator of the chat'
        else:
            msg = 'Unknown error'

    send_message(update, context, msg)

def kick_member(bot, chat_id, user_id) -> int:
    return_code = 0
    try:
        user_data = bot.getChatMember(chat_id, user_id)
        if (user_data['status'] != "left") and (user_data['status'] != "kicked"):
            bot.kickChatMember(chat_id, user_id)
            bot.unbanChatMember(chat_id, user_id)
            return_code = 1
        else:
            return_code = -1
    except Exception as e:
        if str(e) == "Not enough rights to restrict/unrestrict chat member":
            return_code = -2
        elif str(e) == "User is an administrator of the chat":
            return_code = -3

    return return_code


import json
import pprint
import urllib
from PIL import Image, ImageDraw, ImageFilter, ImageOps
import discord
import requests
from discord.ext import commands
from discord.ext.commands import bot

class Acciones(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    def respuestaMensaje(accion):
        global url
        apikey = "Z20FPG65VAWS"
        lmt = 1
        r = requests.get(
            "https://api.tenor.com/v1/random?q=%s&key=%s&limit=%s" % (accion, apikey, lmt))

        if r.status_code == 200:
            pp = pprint.PrettyPrinter(indent=4)
            gif = json.loads(r.content)
            for i in range(len(gif['results'])):
                url = gif['results'][i]['media'][0]['gif']['url']

    @commands.command(name='hug',
                      brief='Abraza a tu compaÃ±ero uwu')
    async def hug(self, ctx, user):
        """Abraza a tu compaÃ±ero uwu."""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        Acciones.respuestaMensaje("hug")
        embed.set_image(url=url)
        try:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** abrazÃ³ a **{user}**. uwu""")
        except discord.ext.commands.errors.MissingRequiredArgument:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se abrazÃ³ a sÃ mismo, esto es un poco triste...""")

        await ctx.send(embed=embed)

    @commands.command(name='hostia',
                      brief='Dale una hostia a alguien xd')
    async def hostia(self, ctx, user):
        """Dale una hostia a alguien xd"""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        Acciones.respuestaMensaje("slap")
        embed.set_image(url=url)
        try:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** le diÃ³ una hostia a **{user}**. ðŸ˜ˆ""")
        except discord.ext.commands.errors.MissingRequiredArgument:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se diÃ³ una hostia a sÃ mismo... Â¿EstÃ¡s bien?""")

        await ctx.send(embed=embed)

    @commands.command(name='kill',
                      brief='Literalmente mata a alguien xddd')
    async def kill(self, ctx, user):
        """Literalmente mata a alguien xddd"""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        print(user)
        print(ctx.author.id)
        try:
            if user != ("<@741962880097845340>" and "<@!741962880097845340>") :
                embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** puto matÃ³ a **{user}**. ðŸ˜Žâœ¨""")
                Acciones.respuestaMensaje("kill you")
                embed.set_image(url=url)
                await ctx.send(embed=embed)
            else:
                async with ctx.channel.typing():
                    bg = Image.open('cogs/img/CallAnAmbulance....png').convert("RGBA")
                    width, height = bg.size
                    userIcon = ctx.author.avatar_url
                    img_data = requests.get(userIcon).content
                    with open('cogs/img/userIcon.png', 'wb') as handler:
                        handler.write(img_data)
                    userIcon = Image.open('cogs/img/userIcon.png').convert("RGBA")
                    mask = Image.open("cogs/img/circuloMask.png").convert("L")
                    userIcon = userIcon.resize((100, 100), Image.ANTIALIAS)
                    userIcon = ImageOps.fit(userIcon, mask.size, centering=(0.5, 0.5))
                    userIcon.putalpha(mask)
                    userIcon = userIcon.resize((100, 100), Image.ANTIALIAS)

                    text_img = Image.new('RGBA', (width, height), (0, 0, 0, 0))
                    text_img.paste(bg, (0, 0))
                    text_img.paste(userIcon, (90, -5), mask=userIcon)
                    text_img.save("cogs/img/killDyson.png", format="png")
                    file = discord.File("cogs/img/killDyson.png", filename="image.png")
                    embed.set_image(url="attachment://image.png")
                    embed.add_field(name="â€Ž", value="...")
                    await ctx.send(file=file, embed=embed)

        except discord.ext.commands.errors.MissingRequiredArgument:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se puto matÃ³. ðŸ˜Žâœ¨""")
            Acciones.respuestaMensaje("kms")
            embed.set_image(url=url)
            await ctx.send(embed=embed)

    @commands.command(name='kiko',
                      brief='Te puto matas. Momento xd.')
    async def kiko(self, ctx):
        """Te puto matas. Momento xd."""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se puto matÃ³. ðŸ˜Žâœ¨""")
        Acciones.respuestaMensaje("kms")
        embed.set_image(url=url)
        await ctx.send(embed=embed)

    @commands.command(name='cry',
                      brief='WAAAH WAAAH',
                      alias="llorar")
    async def cry(self, ctx):
        """WAAAH WAAAH"""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se puso a llorar. ðŸ˜""")
        Acciones.respuestaMensaje("cry")
        embed.set_image(url=url)
        await ctx.send(embed=embed)

    @commands.command(name='cat',
                      brief='Miau xd',
                      alias="gato")
    async def cat(self, ctx):
        """WAAAH WAAAH"""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** ahora tiene un gatito. ðŸ˜¼""")
        Acciones.respuestaMensaje("cat")
        embed.set_image(url=url)
        await ctx.send(embed=embed)

    @commands.command(name='heal',
                      brief='Curas a quien quieras.')
    async def heal(self, ctx, user):
        """Curas a quien quieras"""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))

        try:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** curÃ³ a **{user}** ðŸ’‰""")
            Acciones.respuestaMensaje("heal")
        except discord.ext.commands.errors.MissingRequiredArgument:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** se curÃ³ ðŸ’‰""")
            Acciones.respuestaMensaje("heal")

        embed.set_image(url=url)
        await ctx.send(embed=embed)

    @commands.command(name='horny',
                      brief='Mandas a la horny jail a alguien horny.')
    async def horny(self, ctx, user):
        """Mandas a la horny jail a alguien horny."""
        embed = discord.Embed(colour=discord.Colour(0x9013fe))

        try:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** mandÃ³ a **{user}** a la Horny Jail""")
        except discord.ext.commands.errors.MissingRequiredArgument:
            embed.add_field(name="â€Ž", value=f"""**<@!{ctx.author.id}>** estÃ¡ horny. Tened cuidado :0""")

        embed.set_image(url="https://media1.tenor.com/images/062ed2c1ab2756d774367ef0f8a1f53b/tenor.gif?itemid=17582752")
        await ctx.send(embed=embed)

    @commands.command(name='bonk',
                      brief='bonk.')
    async def bonk(self, ctx, user):
        """bonk."""
        actionUser = ctx.message.content.replace("d.bonk ", "")
        embed = discord.Embed(colour=discord.Colour(0x9013fe))
        embed.add_field(name="â€Ž", value=f"""Bonk a  **{actionUser}** de parte de **<@!{ctx.author.id}>**""")
        embed.set_image(
            url="https://media1.tenor.com/images/ae34b2d6cbac150bfddf05133a0d8337/tenor.gif?itemid=14889944")
        await ctx.send(embed=embed)
        #bonk.

def setup(bot):
    bot.add_cog(Acciones(bot))


# weather_service.py
import requests
import sys
sys.path.append("..")
from my_lib import helper_funcs

config = helper_funcs.read_config_file()

API_HOST = config['ROUTING_SVC']['API_HOST']
CLIENT_ID_HEADER = config['ROUTING_SVC']['CLIENT_ID_HEADER']
CLIENT_SECRET_HEADER = config['ROUTING_SVC']['CLIENT_SECRET_HEADER']
CLIENT_ID = config['ROUTING_SVC']['CLIENT_ID']
CLIENT_SECRET = config['ROUTING_SVC']['CLIENT_SECRET']

def get_route(start, goal):
    start_geo = ",".join(start)
    goal_geo = ",".join(goal)
    routing_query = "?start="+start_geo+"&goal="+goal_geo+"&option=trafast"

    uri = API_HOST + routing_query
    headers = {
            CLIENT_ID_HEADER: CLIENT_ID,
            CLIENT_SECRET_HEADER: CLIENT_SECRET
            }

    with requests.get(uri, headers=headers) as resp:
        result = resp.json()

    return result


from flask import Flask, render_template, request
import numpy as np
from joblib import load
from sklearn.preprocessing import StandardScaler

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def house_price_pred():
    request_type_str = request.method
    if request_type_str == 'GET':
        return render_template('index.html')
    else:
        int_features = np.array([float(x) for x in request.form.values() if is_float(x)])
        if int_features.shape[0] == 15:
            int_features_final = int_features.reshape(1,-1)
            model = load('model.joblib')
            scalery = load(open('scaler_y.pkl', 'rb'))
            scalerX = load(open('scaler_X.pkl', 'rb'))
            results = make_pred(model, scalerX, scalery, int_features_final)
        else:
            results = "Non-float value(s) inserted! Please insert values as per insructions."
        return render_template('index.html', prediction_text=results)

def make_pred(model, scaler_X, scaler_y, new_array):
    scaled_X = scaler_X.transform(new_array)
    y_pred = model.predict(scaled_X)
    y_pred_inv_scaled = scaler_y.inverse_transform(y_pred.reshape(-1,1))
    return y_pred_inv_scaled

def is_float(s):
    try:
        float(s)
        return True
    except:
        return False

if __name__ == "__main__":
    app.run(debug=True)