#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 20 21:28:06 2022

@author: Maurice
"""
import os
import sys
import csv
import subprocess
from Bio import SeqIO

def check_dependancy(programs=[], rScript='', logfile=None):
    log = open(logfile, 'a')
    # the argument programs is a list of dependancies
    check = 'Checking dependancies:\n'
    log.write(check +'\n')
    print(check)
    missingPrograms = []

    # Check Biopython
    if 'Biopython' in programs:
        try:
            from Bio import SeqIO
            check = f' Biopython: OK ---'
            log.write(check +'\n')
            print(check)
        except ModuleNotFoundError:
            missingPrograms.append('Biopython')
            check =' Biopython: Module not found'
            log.write(check +'\n')
            print(check)

    # Check for system programs
    for program in programs:
        if not program == 'Biopython':
            track = subprocess.run(['which', program], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            #path = track.stdout.strip().decode('utf-8')
            if track.returncode != 0:
                check = f' {program}: Executable not found'
                log.write(check +'\n')
                print(check)
                missingPrograms.append(program)
            else:
                check = f' {program}: OK ---'
                log.write(check +'\n')
                print(check)

    # check for R libraries
    if not rScript == '':
        results = subprocess.run(['Rscript', rScript], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        for line in results.stdout.decode('utf-8').strip().split('\n'):
            log.write(line + '\n')
            print(line)
            if 'Library not found' in line:
                lib = line.split(':')[0]
                missingPrograms.append(lib)
    log.close()
    return missingPrograms

class genViz:
    def __init__(self, outDir='None', outFileBase='None', fastq1='None',
                 fastq2='None', unpairedFastq='None', ref='None', gff='None', cpus=2, zs='None', ze='None', logfile='None'):
        self.outDir = outDir
        self.outFileBase = outFileBase
        self.fastq1 = fastq1
        self.fastq2 = fastq2
        self.unpairedFastq = unpairedFastq
        self.ref = ref
        self.gff = gff
        self.cpus = cpus
        self.zs = zs
        self.ze = ze
        seqRecord = SeqIO.read(self.ref, 'fasta')
        self.refLen = len(seqRecord.seq)
        self.log = logfile

    def __str__(self):

        # params = [
        # f'\nParameters to be used:\n',
        # f' Basename: {self.outFileBase}',
        # f' OutDir: {os.path.abspath(self.outDir)}',
        # f' First read file: {os.path.abspath(self.fastq1)}',
        # f' Second read file: {os.path.abspath(self.fastq2)}',
        # f' Unpared reads file: {os.path.abspath(self.unpairedFastq)}',
        # f' Reference genome: {os.path.abspath(self.ref)}',
        # f' Interval file: {os.path.abspath(self.gff)}',
        # f' Zoom will start at: {self.zs}',
        # f' Zoom will end at: {self.ze}',
        # f' Threads to be used: {self.cpus}'
        # ]

        params = [
        f'\nParameters to be used:\n',
        f' Basename: {self.outFileBase}',
        f' OutDir: {os.path.abspath(self.outDir)}',
        f' First read file: {self.fastq1}',
        f' Second read file: {self.fastq2}',
        f' Unpared reads file: {self.unpairedFastq}',
        f' Reference genome: {os.path.abspath(self.ref)}',
        f' Interval file: {os.path.abspath(self.gff)}',
        f' Zoom will start at: {self.zs}',
        f' Zoom will end at: {self.ze}',
        f' Threads to be used: {self.cpus}'
        ]

        with open (self.log, 'a') as log:
            for param in params:
                log.write(param + '\n')
                print(param)

        return ''

    def trimmer(self):
        with open (self.log, 'a') as log:
            message = '\n====== Trimming raw reads for quality control ======\n'
            log.write(message +'\n')
            print(message)
            newReadsFolder = os.path.join(self.outDir, 'trimmed_reads')
            if not os.path.exists(newReadsFolder):
                os.makedirs(newReadsFolder)

            if self.unpairedFastq != 'None':
                base = os.path.basename(self.unpairedFastq)
                newRead = os.path.join(newReadsFolder, f'trimmed_{base}')
                cmd = f'fastp -i {self.unpairedFastq} -o {newRead} -w {self.cpus} -j /dev/null -h /dev/null -q 20 -u 40 -l 15 -z 4'
                newRead1 = newRead
                newRead2 = 'None'
            else:
                base1 = os.path.basename(self.fastq1)
                base2 = os.path.basename(self.fastq2)
                newRead1 = os.path.join(newReadsFolder, f'trimmed_{base1}')
                newRead2 = os.path.join(newReadsFolder, f'trimmed_{base2}')
                cmd = f'fastp -i {self.fastq1} -I {self.fastq2} -o {newRead1} -O {newRead2} -w {self.cpus} -j /dev/null -h /dev/null -q 20 -u 40 -l 15 -z 4'
            if not os.path.exists(newRead1) and not os.path.exists(newRead2):
                execute = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                for line in execute.stdout.decode('utf-8').strip().split('\n'):
                    log.write(line +'\n')

                if execute.returncode != 0:
                    print("\nFailed to trim reads for quality with fastp. Check the log file for more details.\n")
                    sys.exit()

            else:
                message = '\n=============== TRIMMED READS EXIST ALREADY ========== SKIPPING!!!'
                log.write(message +'\n')
                print(message)
            outmessage = [
            '\nDone Trimming:\n',
            f'\t---> Trimmed read1: {newRead1}',
            f'\t---> Trimmed read2: {newRead2}\n',
            ]
            for message in outmessage:
                log.write(message + '\n')
                print(message)
            return [newRead1, newRead2]

    def mapper(self, read1=None, read2=None, mapper='bwa', rmdup=True):
        with open (self.log, 'a') as log:
            message1 = '\n====== Aligning reads to the reference genome ======\n'
            message2 = f'\t---> Reference length: {self.refLen:,}'
            log.write(message1 + '\n')
            print(message1)
            log.write(message2 + '\n')
            print(message2)
            rawBam = os.path.join(self.outDir, self.outFileBase + '_raw.bam')
            outBam = os.path.join(self.outDir, self.outFileBase + '_dedupped.bam')  # PCR duplicates removed

            if mapper == 'bwa':
                cmd1 = f'bwa index {self.ref}'
                if self.unpairedFastq != 'None':
                    cmd2 = f'bwa mem -t {self.cpus} {self.ref} {read1} | samtools view -b | samtools sort -@{self.cpus} > {rawBam}'
                else:
                    cmd2 = f'bwa mem -t {self.cpus} {self.ref} {read1} {read2} | samtools view -b | samtools sort -@{self.cpus} > {rawBam}'

                if not os.path.exists(self.ref + '.pac'):
                    message = '\t---> Indexing the reference genome'
                    log.write(message + '\n')
                    print(message)

                    execute = subprocess.run(cmd1.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    for line in execute.stdout.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute.returncode != 0:
                        print("\nFailed to index the reference. Check the log file for more details.\n")
                        sys.exit()

            elif mapper == 'bowtie2':
                cmd1 = f'bowtie2-build {self.ref} {self.ref}'
                if self.unpairedFastq != 'None':
                    cmd2 = f'bowtie2 -p {self.cpus} -x {self.ref} -U {read1} | samtools view -b | samtools sort -@{self.cpus} > {rawBam}'
                else:
                    cmd2 = f'bowtie2 -p {self.cpus} -x {self.ref} -1 {read1} -2 {read2} | samtools view -b | samtools sort -@{self.cpus} > {rawBam}'

                if not os.path.exists(self.ref + '.1.bt2'):
                    message = '\t---> Indexing the reference genome'
                    log.write(message + '\n')
                    print(message)

                    execute = subprocess.run(cmd1.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    for line in execute.stdout.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute.returncode != 0:
                        print("\nFailed to index the reference. Check the log file for more details.\n")
                        sys.exit()

            elif mapper == 'bbmap.sh':
                rawSam = os.path.join(self.outDir, self.outFileBase + '_raw.sam')
                if self.unpairedFastq != None:
                    cmd2a = f'bbmap.sh threads={self.cpus} ref={self.ref} in={read1} out={rawSam} nodisk'
                else:
                    cmd2a = f'bbmap.sh threads={self.cpus} ref={self.ref} in={read1} in2={read2} out={rawSam} nodisk'
                cmd2b = f'samtools view -b {rawSam} | samtools sort -@{self.cpus} > {rawBam}'

            cmd3 = f'samtools rmdup {rawBam} {outBam}'

            if not os.path.exists(outBam):
                message1 = f'\t---> Aligning reads with {mapper}'
                log.write(message1 + '\n')
                print(message1)

                if mapper == 'bbmap.sh':

                    execute2a = subprocess.run(cmd2a, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    for line in execute2a.stdout.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute2a.returncode != 0:
                        print("\nbbmap failed. Check the log file for more details.\n")
                        sys.exit()

                    execute2b = subprocess.run(cmd2b, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    for line in execute2b.stdout.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute2b.returncode != 0:
                        print("\nSamtools failed. Check the log file for more details.\n")
                        sys.exit()
                    os.remove(rawSam)
                else:

                    execute2 = subprocess.run(cmd2, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    for line in execute2.stdout.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute2.returncode != 0:
                        print(f"\n{mapper} failed. Check the log file for more details.\n")
                        sys.exit()

                message2 = '\t---> Removing PCR and optical duplicates'
                log.write(message2 + '\n')
                print(message2)

                execute3 = subprocess.run(cmd3.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                for line in execute3.stdout.decode('utf-8').strip().split('\n'):
                    log.write(line + '\n')

                if execute3.returncode != 0:
                    print("\nSamtools failed. Check the log file for more details.\n")
                    sys.exit()

            else:
                message1 = '\n====== ALIGNMENT FILE EXISTS ALREADY ======== SKIPPING!!!'
                log.write(message1 + '\n')
                print(message1)

            if rmdup == False:
                dupInfo = '\nThe option to remove duplicates was turned off. The raw alignment file will be used. \n'
                log.write(dupInfo + '\n')
                print(dupInfo)

                outBam = rawBam

            outinfo = ['\nDone with mapping:\n', f'\t---> Alignment file: {outBam}']

            for info in outinfo:
                log.write(info + '\n')
                print(info)

        return outBam

    def makeBed(self, inBam):
        with open (self.log, 'a') as log:
            inmessage = '\n====== Making a coverage bedGraph file from the alignment file ======\n'
            log.write(inmessage + '\n')
            print(inmessage)

            bedGraph = os.path.join(self.outDir, f'{self.outFileBase}.bedGraph')
            if not os.path.exists(bedGraph):
                cmd = f'bedtools genomecov -ibam {inBam} -bga'
                with open(bedGraph, 'w') as bed:
                    execute = subprocess.run(cmd.split(), stdout=bed, stderr=subprocess.PIPE)
                    for line in execute.stderr.decode('utf-8').strip().split('\n'):
                        log.write(line + '\n')

                    if execute.returncode != 0:
                        print("\nFailed to make the coverage file from thre bam file. Check the log file for more details.\n")
                        sys.exit()
            else:
                message = '\n====== BEDGRAPH FILE EXISTS ALREADY ======== SKIPPING!!!\n'
                log.write(message + '\n')
                print(message)

            outmessage = f'\t---> bedGraph file: {bedGraph}'
            log.write(outmessage + '\n')
            print(outmessage)

            with open(bedGraph, 'r') as bed:
                chrom = bed.readline().split('\t')[0]
            return [chrom, os.path.abspath(bedGraph)]

    def collectVarNames(self, values):  # values is a list of values
        outFile = os.path.join(self.outDir, self.outFileBase + '_values.csv')
        with open(outFile, 'w') as vals:
            writer = csv.writer(vals, dialect=csv.excel_tab)
            writer.writerow(['sample', 'out_dir', 'virus', 're_fasta', 'gff', 'zoomStart', 'zoomEnd', 'seqEnd', 'bam', 'bedfile', 'chromosome'])
            writer.writerow(values)
        return(outFile)

    def generateGraphs(self, rScript, valueFile):
        with open (self.log, 'a') as log:
            inmessage = '\n====== Generating coverage graphs ======\n'
            log.write(inmessage + '\n')
            print(inmessage)

            execute = subprocess.run(['Rscript', rScript, valueFile], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

            for line in execute.stdout.decode('utf-8').strip().split('\n'):
                log.write(line + '\n')

            if execute.returncode != 0:
                print("\nFailed to produce the coverage images. Check the log file for more details.\n")
                sys.exit()
            else:
                message = 'The following alignment graphs were generated:\n'
                log.write(message + '\n')
                print(message)

                file1 = os.path.join(self.outDir, f'graphs/{self.outFileBase}_full_coverage.png')
                # file2 = os.path.join(self.outDir, f'graphs/{self.outFileBase}_zoom_{self.zs}-{self.ze}_coverage.png')
                # file3 = os.path.join(self.outDir, f'graphs/{self.outFileBase}_full_alignment.png')
                # file4 = os.path.join(self.outDir, f'graphs/{self.outFileBase}_zoom_{self.zs}-{self.ze}_alignment.png')
                for file in [file1]:#, file2]:#, file4]:
                    if os.path.exists(file):
                        outmessage = f'\t---> {file}'
                        log.write(outmessage + '\n')
                        print(outmessage)
                    else:
                        log.write(f'Graph {os.path.basename(file)} was not generated! Something is wrong with input files\n')
                        print(f'Graph {os.path.basename(file)} was not generated! Check log file for more details')
                print('\n')


import keras

def LSTM( units, gpu=False, **kwargs ):
  '''
  Uses optimized CuDNNLSTM if gpu is activated
  '''
  if gpu:
    layer = keras.layers.CuDNNLSTM( units=units, **kwargs )
  else:
    # Recurrent activation should default to sigmoid so that it
    # behaves similar to CuDNNLSTM. The normal default is hard
    # sigmoid
    if not 'recurrent_activation' in kwargs:
      kwargs['recurrent_activation'] = 'sigmoid'
    layer = keras.layers.LSTM( units=units, **kwargs )

  return layer


import pickle
from rdkit import Chem
import torch
from rdkit.Chem import AllChem
from rdkit.Chem import Descriptors
class smi_inform(object):
    def __init__(self,radius=2, nBits=1024):
        self.radius = radius
        self.nBits = nBits
    def get_mol(self,smi):
        mol = Chem.MolFromSmiles(smi)
        return mol
    def get_prop(self,mol):
        fp1 = AllChem.GetMorganFingerprintAsBitVect(mol,self.radius,self.nBits,useChirality=True)
        self.fp = torch.Tensor(fp1)
        self.TPSA = Descriptors.TPSA(mol)
        self.logp = Descriptors.MolLogP(mol)
        self.rings = Descriptors.RingCount(mol)
        return self


"""Open Api Specs request decorator."""
import copy
from functools import partial
from typing import Any

# locations supported by openapi
VALID_SCHEMA_LOCATIONS = (
    "cookies",
    "files",
    "form",
    "headers",
    "json",
    "match_info",
    "path",
    "query",
    "querystring",
)

def request_schema(
    schema: Any,
    location: str = "json",
    put_into: Any = None,
    example: Any = None,
    add_to_refs: bool = False,
    **kwargs: Any,
) -> Any:
    """Request info for Open API docs.

    Add request info into the swagger spec.
    put_into (Any): name of the key in Request object where validated
    data will be placed. If None (by default) default key will be used

    add_to_refs (bool): Working only if example not None, if True,
    add example for ref schema, otherwise add example to endpoint.

    Parameters:
        schema (Any): `Schema <marshmallow.Schema>` class or instance
        location (str): Default request locations to parse
        put_into (Any): name of the key in Request object
        example (Any): Adding example for current schema
        add_to_refs (bool): Default False
        **kwargs (Any): extra arguments

    Raises:
        ValueError: if invalid location found or if multiple location found

    Returns:
        function of annotation
    """  # noqa: RST210
    if location not in VALID_SCHEMA_LOCATIONS:
        raise ValueError(f"Invalid location argument: {location}")

    if callable(schema):
        schema = schema()

    options = {"required": kwargs.pop("required", False)}

    def wrapper(func: Any) -> Any:
        if not hasattr(func, "__apispec__"):
            func.__apispec__ = {
                "schemas": [],
                "responses": {},
                "parameters": [],
            }
            func.__schemas__ = []

        _example = copy.copy(example) or {}
        if _example:
            _example["add_to_refs"] = add_to_refs
        func.__apispec__["schemas"].append(
            {
                "schema": schema,
                "location": location,
                "options": options,
                "example": _example,
            }
        )

        # "body" location was replaced by "json" location
        if location == "json" and any(
            func_schema["location"] == "json"
            for func_schema in func.__schemas__
        ):
            raise RuntimeError("Multiple json locations are not allowed")

        func.__schemas__.append(
            {"schema": schema, "location": location, "put_into": put_into}
        )

        return func

    return wrapper

# For backward compatibility
use_kwargs = request_schema

# Decorators for specific request data validations (shortenings)
match_info_schema = partial(
    request_schema, location="match_info", put_into="match_info"
)
querystring_schema = partial(
    request_schema, location="querystring", put_into="querystring"
)
form_schema = partial(request_schema, location="form", put_into="form")
json_schema = partial(request_schema, location="json", put_into="json")
headers_schema = partial(
    request_schema, location="headers", put_into="headers"
)
cookies_schema = partial(
    request_schema, location="cookies", put_into="cookies"
)


import cv2
import PIL
import numpy as np
import colorgram
from PIL import Image
from matplotlib import pyplot as plt

hijau = 0
hitam = 0

def treshold(img1, hijau, hitam):
		row, col = img1.shape
		treshold = np.zeros((row,col,1), np.uint8)
		for i in range(0, row):
			for j in range(0, col):
				val = img1[i,j]

				if(val > 15 and val < 65):
					val = 0
					hijau = hijau + 1
				elif (val > 65): val = 255
				else: hitam = hitam + 1

				treshold.itemset((i, j, 0), val)
		return treshold, hijau, hitam

img = cv2.imread("DSC_0322.jpg", 0)
plt.hist(img.ravel(),256,[0,256])
#plt.show()

row, col = img.shape
byka= row*col

img2, hijau, hitam = treshold(img, hijau, hitam)
byk = float(hijau+hitam)/1.0
prop = float(hijau)/byk
hit = float(hitam)/byk
print("hitam: ",float(hit),"prop: ",float(prop),"hijau: ",hijau,"byk: ",float(byk/byka))

# cv2.imshow("img 1", img)
# cv2.imshow("img", img2)

# img = cv2.medianBlur(img, 5)
# th3 = cv2.adaptiveThreshold(img,70,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)
# cv2.imshow(th3)
# color = ('b', 'g', 'r')
# for i, col in enumerate(color):
#     histr = cv2.calcHist([img],[i],None,[256],[0,256])
#     plt.plot(histr, color = col)
#     plt.xlim([0,256])
# plt.show()
#
# print(histr)

# # img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# colors = colorgram.extract('mangga.jpg', 6)
#
# def hitung(image):
#     hijau, hitam = 0
#     row, col, ch= image.shape
#     canvas = np.zeros((row, col, 3), np.uint8)
#     for m in range(0, row):
#         for n in range(0, col):
#             h, s, v = image[m, n]
#            # gray = (red * 0.299 + green * 0.587 + blue * 0.114)
#            # canvas.itemset((m,n,0),gray)
#     return canvas

# print(colors)

cv2.waitKey(0)
cv2.destroyAllWindows()


import json
import os
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup
from tqdm import tqdm

headers = {
    'Content-Type': 'application/x-www-form-urlencoded',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Sec-Fetch-Site': 'same-origin',
    'Accept-Language': 'zh-TW,zh-Hant;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Sec-Fetch-Mode': 'navigate',
    'Host': 'exam.naer.edu.tw',
    'Origin': 'https://exam.naer.edu.tw',
    'Referer': 'https://exam.naer.edu.tw/searchResult.php',
    'Connection': 'keep-alive',
    'Sec-Fetch-Dest': 'document',
}

total_pages = 4767
data_file = "./data.json"

if os.path.exists(data_file):
    with open(data_file, 'r', encoding='utf-8') as f:
        all_data = json.load(f)
    if all_data:
        last_record = all_data[-1]
        last_page = int(last_record['page'])
        start_page = last_page + 1
    else:
        start_page = 1
else:
    all_data = []
    start_page = 1

for page in tqdm(range(start_page, total_pages + 1)):
    data = {
        'page': page,
        'orderBy': 'lastest',
        'selCategory': '0',
        'selTech': '0',
        'chkClass[]': ['15', '16', '17'],
        'keyword': '',
        'selCountry': '',
        'selYear': '',
        'selTerm': '',
        'selType': '',
        'selPublisher': '',
    }

    response = requests.post('https://exam.naer.edu.tw/searchResult.php', headers=headers, data=data)
    soup = BeautifulSoup(response.text, 'html.parser')
    rows = soup.find_all('tr')
    for row in rows[1:]:
        try:
            cols = row.find_all('td')
            if len(cols) < 11:
                continue
            # Check if the "縣市" column contains invalid data
            city = cols[0].text.strip()
            if "回首頁" in city or "版權所有" in city:
                continue
            test_paper_link = cols[9].find('a')['href'] if cols[9].find('a') else None
            answer_paper_link = cols[10].find('a')['href'] if cols[10].find('a') else None
            if test_paper_link is None and answer_paper_link is None:
                continue
            record = {
                "縣市": cols[0].text.strip(),
                "學校名稱": cols[1].text.strip(),
                "年級": cols[2].text.strip(),
                "學年度": cols[3].text.strip(),
                "領域/群科": cols[4].text.strip(),
                "科目": cols[5].text.strip(),
                "種類": cols[6].text.strip(),
                "版本": cols[7].text.strip(),
                "點閱率": cols[8].text.strip(),
                "下載試卷": urljoin('https://exam.naer.edu.tw', test_paper_link),
                "下載答案": urljoin('https://exam.naer.edu.tw', answer_paper_link),
                "page": page,
            }
            all_data.append(record)
        except:
            pass

    with open(data_file, 'w', encoding='utf-8') as f:
        json.dump(all_data, f, ensure_ascii=False, indent=2)


# -*- coding: utf-8 -*-
from lxml import etree
import sys
import os

def seed_target_from_sbml(draft,biomass_name,repository) :

        if not os.path.isdir(repository) :
            os.makedirs(repository)

        seeds = set()
        bio_reac = set()
        BDD_species = {}

        draft_parse = etree.parse(draft)
        sbml = draft_parse.find('sbml')
        root = "}".join(draft_parse.getroot().tag.split("}")[:-1])+"}"

        listOfSpecies = draft_parse.find(root+'model/'+root+'listOfSpecies')
        for specie in listOfSpecies :
                BDD_species.update({specie.attrib["id"] : specie})
                if specie.attrib["boundaryCondition"] == "true" :
                        seeds.add(specie)

        listOfReactions = draft_parse.find(root+'model/'+root+'listOfReactions')
        for reaction in listOfReactions :
                try :
                        id_reac = reaction.attrib["id"]
                except KeyError :
                        id_reac = ""
                try :
                        id_name_reac = "_".join(id_reac.split("_")[1:])
                except KeyError :
                        id_name_reac = ""
                try :
                        name_reac = reaction.attrib["name"]
                except KeyError :
                        name_reac = ""
                if (biomass_name in id_reac) or (biomass_name in id_name_reac) or (biomass_name in name_reac) :
                        Reactants = reaction.find(root+'listOfReactants')
                        for specie in Reactants :
                                bio_reac.add(BDD_species[specie.attrib["species"]])

        with open(draft,"r") as file :
                line0 = file.readlines()[0]
        ################################################################ seed

        seed_doc = draft_parse
        root = "}".join(seed_doc.getroot().tag.split("}")[:-1])+"}"
        model = seed_doc.find(root+'model')
        listR = model.find(root+"listOfReactions")
        model.remove(listR)

        listS = model.find(root+"listOfSpecies")
        listS.clear()
        for element in seeds :
                listS.append(element)
        outFile=open(repository+'/seeds_from_sbml.sbml','wb')
        outFile.write(line0.encode('utf-8'))
        seed_doc.write(outFile, pretty_print=True)

        ################################################################ target

        target_doc = draft_parse
        root = "}".join(target_doc.getroot().tag.split("}")[:-1])+"}"
        model = target_doc.find(root+'model')
        try :
                listR = model.find(root+"listOfReactions")
                model.remove(listR)
        except TypeError :
                pass
        listS = model.find(root+"listOfSpecies")
        listS.clear()
        for element in bio_reac :
                listS.append(element)
        outFile=open(repository+'/targets_from_sbml.sbml','wb')
        outFile.write(line0.encode('utf-8'))
        target_doc.write(outFile, pretty_print=True)

if __name__ == '__main__':

        draft = sys.argv[1]
        biomass_name = sys.argv[2]
        try :
                repository = sys.argv[3]
        except IndexError :
                repository = "/".join(file_read.split("/")[:-1])

        seed_target_from_sbml(draft,biomass_name,repository)


import colored
import os
import requests
import threading
from bs4 import BeautifulSoup as bs

play_list_url = "https://www.youtube.com/user/TheViperAOC/playlists"

def get_list():
    list_page = requests.get(play_list_url).text
    list_page_soup = bs(list_page, "lxml")
    play_list = list_page_soup.find_all("h3", {"class": "yt-lockup-title"})
    number = 0
    info_list = []
    print("All play list is : ")
    for item in play_list:
        list_title = item.a.get_text()
        list_url = "https://www.youtube.com" + item.a["href"]
        print("+------------------------------------------------------------------------+")
        print("{}Number : {}".format(colored.fg(5), colored.attr(0)) + str(number))
        print("{}List Title : {}".format(colored.fg(11), colored.attr(0)) + list_title)
        print("{}List URL : {}".format(colored.fg(1), colored.attr(0)) + list_url)
        list_info = (number, list_title, list_url)
        number += 1
        info_list.append(list_info)
    return info_list

def get_video(selected_info):
    list_page_url = selected_info[2]
    list_page = requests.get(list_page_url).text
    list_soup = bs(list_page, "lxml")
    video_list = list_soup.find_all("tr")
    number = 0
    info_list = []
    for item in video_list:
        info = item.find("a", {"class": "pl-video-title-link yt-uix-tile-link yt-uix-sessionlink spf-link"})
        video_url = "https://www.youtube.com" + info["href"]
        video_name = info.get_text()
        video_name = video_name.replace("\n", "")
        video_name = video_name.replace(" ", "")
        list_info = (number, video_name, video_url)
        number += 1
        info_list.append(list_info)
    return info_list

def download_subtitle(url):
    os.system("youtube-dl.exe --no-playlist --proxy \"http://127.0.0.1:1099\" --write-auto-sub --skip-download \"%s\"" % url)

def download_video(url):
    os.system("youtube-dl.exe --no-playlist --proxy \"http://127.0.0.1:1099\" -f \"(mp4)[width>=1920][height>=1080][fps>=60]\" \"%s\"" % url)

def download_audio(url):
    os.system("youtube-dl.exe --no-playlist --proxy \"http://127.0.0.1:1099\" -f \"(m4a)[asr=44100][abr<=320]\" \"%s\"" % url)

def threading_download(info):
    while True:
        if_only_sub_select = input("Would you like to Download Video+Audio+Subtitle or just Subtitle ? VAS/S").upper()
        if if_only_sub_select == "VAS":
            print("Downloading Video+Audio+Subtitle")
            for video in info:
                url = str(video[2])
                threading.Thread(target=download_audio, args=(url,)).start()
                threading.Thread(target=download_video, args=(url,)).start()
                threading.Thread(target=download_subtitle, args=(url,)).start()
            break
        elif if_only_sub_select == "S":
            print("Downloading only the Subtitle")
            for video in info:
                url = str(video[2])
                threading.Thread(target=download_subtitle, args=(url,)).start()
            break
        else:
            print("Please enter 'VAS'(for Video+Audio+Subtitle) or 'S'(only Subtitle)")
            continue

if __name__ == '__main__':
    print("Launching, please wait...")
    play_list_info_list = get_list()
    selected_list = None
    while True:
        try:
            list_selection = int(input("Which one would you like to use ? "))
        except TypeError:
            print("Wrong Input, Try again!")
            continue
        if list_selection > len(play_list_info_list):
            print("Too large number, Try again!")
            continue
        else:
            print("\nYour selection is :")
            print("+------------------------------------------------------------------------+")
            print("{}Number : {}".format(colored.fg(5), colored.attr(0)) + str(play_list_info_list[list_selection][0]))
            print("{}List Title : {}".format(colored.fg(11), colored.attr(0)) + play_list_info_list[list_selection][1])
            print("{}List URL : {}".format(colored.fg(1), colored.attr(0)) + play_list_info_list[list_selection][2])
            while True:
                try:
                    list_confirm = input("Is that correct ? Y/N").upper()
                except TypeError:
                    print("Please input 'Y' or 'N', Try again!")
                    continue
                if list_confirm == "Y":
                    print("Processing...")
                    selected_list = play_list_info_list[list_selection]
                    break
                elif list_confirm == "N":
                    print("Please try again...")
                    continue
                else:
                    print("Wrong input, please try again!")
                    continue
            break
    print("Entering List...")
    video_list_info = get_video(selected_list)
    threading_download(video_list_info)


from flask import Flask, render_template, jsonify, request, send_from_directory
from random import *
from threading import Thread
from flask_cors import CORS
from wattpad2epub import get_book
import time
import os
import platform

def creation_date(path_to_file):
    """
    Try to get the date that a file was created, falling back to when it was
    last modified if that isn't possible.
    See http://stackoverflow.com/a/39501288/1709587 for explanation.
    """
    if platform.system() == 'Windows':
        return os.path.getctime(path_to_file)
    else:
        stat = os.stat(path_to_file)
        try:
            return stat.st_birthtime
        except AttributeError:
            # We're probably on Linux. No easy way to get creation dates here,
            # so we'll settle for when its content was last modified.
            return stat.st_mtime

def cleanupFiles():
  while (1 == 1):
    current_time = time.time()
    for f in os.listdir("./books"):
      creation_time = os.path.getctime(os.path.join("./books", f))
      if (current_time - 7 * 24 * 60 * 60 > creation_time):
        os.unlink(os.path.join("./books", f))
        print('{} removed'.format(f))
    time.sleep(10)

app = Flask(__name__,
            static_folder = "./dist/",
            template_folder= "./dist/")

cors = CORS(app, resources={r"/api/*": {"origins": "*"}})

@app.route('/cdn/<path:filename>')
def custom_static(filename):
    return send_from_directory('./', filename)

@app.route('/<path:filename>')
def stat(filename):
    return send_from_directory('./dist', filename)

@app.route('/api', methods=["POST"])
def random_number():
    return get_book(request.get_json()["url"])

@app.route('/')
def catch_all():
    return render_template("index.html")

port = int(os.environ.get("PORT", 5000))
app.run(host='0.0.0.0', port=port)

cleanup = Thread(target=cleanupFiles)
cleanup.start()


"""Site Views

(Views that do not belong in any app but are needed by site.)

I'm breaking the rules here for simplicity: this should not
be in config, as this is not configuration for the project.
However, introducing an app specifically for a single view
would overcomplicate the work we're doing, so it's going
here instead.
"""

from django.http import HttpResponse
from rest_framework.response import Response
from rest_framework.reverse import reverse
from rest_framework.status import HTTP_200_OK
from rest_framework.views import APIView

from oauth2_provider.contrib.rest_framework.permissions import (
    IsAuthenticatedOrTokenHasScope,
)

from .celery import debug_task

def test_celery(request):
    """Use Celery debug task"""
    debug_task.delay("Hello from Celery")
    return HttpResponse("Hello from Django")

class RootApiView(APIView):
    """Direct users to other API endpoints"""

    permission_classes = [IsAuthenticatedOrTokenHasScope]

    def get(self, request, *args, **kwargs):
        """Build & display links to other endpoints"""
        api_endpoints = [
            # (name, url_name),
            ("tag", "api-tag-list"),
            ("startup", "api-startup-list"),
            ("newslink", "api-newslink-list"),
            ("blog", "api-post-list"),
        ]
        data = {
            name: reverse(
                url_name,
                request=request,
                format=kwargs.get("format", None),
            )
            for (name, url_name) in api_endpoints
        }
        return Response(data=data, status=HTTP_200_OK)


import numpy as np
import bpy  # noqa
import gpu  # noqa
import bgl  # noqa
from OpenGL.GL import glGetTexImage

from .camera import Camera
from .utils import find_first_view3d

class OffScreenRenderer:
    """Provides offscreen scene rendering using Eevee.

    Rendering reusing the first found 3D Space View in Blender. The way this view is configured also defines how the resulting image looks like. Use the helper method `set_render_style` to adjust the appearance from within Python.

    This class' `render` method is expected to be called from a `POST_PIXEL` callback, which `AnimationController` takes care of. That is, invoking `render()` from  withing `post_frame` is considered save.

    Note, color images are in linear color spaces and may appear darker than
    expected when visualized. Use `btt.colors.gamma` to gamma encode before
    visualizing.

    Params
    ------
    camera: btb.Camera, None
        Camera view to be rendered. When None, default camera is used.
    origin: str
        When 'upper-left' flip the rendered data to match OpenCV image coordinate
        system. When 'lower-left' image is created using OpenGL coordinate system. Defaults to 'upper-left'.
    mode: str
        Defines the number of color channels. Either 'RGBA' or 'RGB'
    """

    def __init__(self, camera=None, mode="rgba", origin="upper-left"):
        assert mode in ["rgba", "rgb"]
        assert origin in ["upper-left", "lower-left"]
        self.camera = camera or Camera()
        self.offscreen = gpu.types.GPUOffScreen(self.shape[1], self.shape[0])
        self.area, self.space, self.region = find_first_view3d()
        self.handle = None
        self.origin = origin
        channels = 4 if mode == "rgba" else 3
        self.buffer = np.zeros((self.shape[0], self.shape[1], channels), dtype=np.uint8)
        self.mode = bgl.GL_RGBA if mode == "rgba" else bgl.GL_RGB

    @property
    def shape(self):
        return self.camera.shape

    def render(self):
        """Render the scene and return image as buffer.

        Returns
        -------
        image: HxWxD array
            where D is 4 when `mode=='RGBA'` else 3.
        """
        with self.offscreen.bind():
            self.offscreen.draw_view3d(
                bpy.context.scene,
                bpy.context.view_layer,
                self.space,  # bpy.context.space_data
                self.region,  # bpy.context.region
                self.camera.view_matrix,
                self.camera.proj_matrix,
            )

            bgl.glActiveTexture(bgl.GL_TEXTURE0)
            bgl.glBindTexture(bgl.GL_TEXTURE_2D, self.offscreen.color_texture)

            # np.asarray seems slow, because bgl.buffer does not support the python buffer protocol
            # bgl.glGetTexImage(bgl.GL_TEXTURE_2D, 0, bgl.GL_RGB, bgl.GL_UNSIGNED_BYTE, self.buffer)
            # https://docs.blender.org/api/blender2.8/gpu.html
            # That's why we use PyOpenGL at this point instead.
            glGetTexImage(
                bgl.GL_TEXTURE_2D, 0, self.mode, bgl.GL_UNSIGNED_BYTE, self.buffer
            )

        buffer = self.buffer
        if self.origin == "upper-left":
            buffer = np.flipud(buffer)
        return buffer

    def set_render_style(self, shading="RENDERED", overlays=False):
        self.space.shading.type = shading
        self.space.overlay.show_overlays = overlays


# -*- coding: utf-8 -*-
"""
Created on Tue Aug  1 19:11:11 2023

@author: hp
"""

import os
import pandas as pd
import numpy as np
from plotnine import *
from siuba import *

#%%
ruta = "https://raw.githubusercontent.com/scidatmath2020/ML_Py_23/main/data/datos_regresion.csv"
mi_tabla = pd.read_csv(ruta)

mi_tabla

(ggplot(data = mi_tabla) +
 geom_point(mapping=aes(x="caracteristica_1",y="valor_real"),color="red")
)

#%%

variables_independientes = mi_tabla >> select(_.caracteristica_1)
objetivo = mi_tabla >> select(_.valor_real)

#%%

from sklearn.linear_model import LinearRegression
modelo = LinearRegression()

#%%

modelo.fit(X=variables_independientes,y=objetivo) # Aquí se calcula la regresión
modelo.intercept_ #esta es la alpha de la regresión
modelo.coef_ # estas son las beta's de la regresión

#%%

mi_tabla["predicciones"] = modelo.predict(variables_independientes)

#%%

(ggplot(data = mi_tabla) +
 geom_point(mapping=aes(x="caracteristica_1",y="valor_real"),color="blue") +
 geom_point(mapping=aes(x="caracteristica_1",y="predicciones"),color="red") +
 geom_abline(slope=1.85,intercept=5.711) +
 geom_smooth(mapping=aes(x="caracteristica_1",y="valor_real"),color="green")
)

modelo.coef_
modelo.intercept_

#%%

from sklearn import metrics

metrics.mean_absolute_error(mi_tabla["valor_real"],mi_tabla["predicciones"])
metrics.mean_squared_error(mi_tabla["valor_real"],mi_tabla["predicciones"])
np.sqrt(metrics.mean_squared_error(mi_tabla["valor_real"],mi_tabla["predicciones"]))

mi_tabla >> mutate(error = _.valor_real-_.predicciones)

R2 = metrics.r2_score(mi_tabla["valor_real"],mi_tabla["predicciones"])

1-(1-R2)*(50-1)/(50-1-1)

#%%

n = mi_tabla.shape[0]
k = variables_independientes.shape[1]

1-(1-R2)*(n-1)/(n-k-1)


from pyspark.sql import SparkSession
import pyspark.sql.functions as f
from pyspark.sql.types import ArrayType, StringType, StructType, StructField

BOOTSTRAP_SERVERS = "34.213.179.50:9092,34.220.78.168:9092,34.210.71.28:9092"
S3_CHECKPOINT_LOCATION = "s3n://tweets1/checkpoints/"
S3_PARQUET_SINK = "s3n://tweets1/tweet/"
# Schema of Tweet and Tweet Metadata from Twitter API
json_schema =StructType([
    StructField("data",
        StructType([
                    StructField( "created_at", StringType(),  nullable = False )
                    , StructField( "id", StringType(),  nullable = False)
                    , StructField("text", StringType(),  nullable = False)
               ])
        , nullable = False)
    , StructField( "matching_rules",
             ArrayType(StructType([
                    StructField("id", StringType(), nullable = False)
                , StructField("tag", StringType(), nullable= False)
                ])
                )
            )
    ])

spark = SparkSession \
    .builder \
    .appName("StructuredNetworkWordCount") \
    .getOrCreate()

# Start Spark Structured Streaming
df = spark \
  .readStream \
  .format("kafka") \
  .option("kafka.bootstrap.servers", BOOTSTRAP_SERVERS) \
  .option("subscribe", "twitter_on_stocks") \
  .option("failOnDataLoss", "false") \
  .load()

tweets = df.selectExpr("CAST(value AS STRING)")

tweets = tweets.select( f.from_json("value", json_schema).alias("value")  )

tweets = tweets.selectExpr( "CAST(value.data.id as BIGINT) AS id", "value.data.created_at AS created_at", "CAST(value.data.text as STRING) AS text", "LOWER(CAST(value.matching_rules.tag[0] AS STRING)) as company"   )

# Count words in the tweet
tweets = tweets.withColumn( "word_count",  f.size(f.split(f.col('text'), ' '))  )

# Save set of data pulled from Kafka every 15 seconds
query = tweets.writeStream.format("parquet"). \
    option("checkpointLocation", S3_CHECKPOINT_LOCATION). \
    option("path", S3_PARQUET_SINK). \
    outputMode('append'). \
    trigger(processingTime='15 seconds'). \
    partitionBy('company').start()

query.awaitTermination()


import numpy as np
from torch.nn import init

def init_weights(net, init_type='normal'):
    if init_type == 'kaiming':
        net.apply(weights_init_kaiming)
    else:
        raise NotImplementedError('Initialization method [%s] is not implemented.' % init_type)

def weights_init_kaiming(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
    elif classname.find('Linear') != -1:
        init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
    elif classname.find('BatchNorm') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)

def count_param(model):
    param_count = 0
    for param in model.parameters():
        param_count += param.view(-1).size()[0]
    return param_count

def loc2bbox(src_bbox, loc):
    if src_bbox.shape[0] == 0:
        return np.zeros((0, 4), dtype=loc.dtype)

    src_bbox = src_bbox.astype(src_bbox.dtype, copy=False)

    src_height = src_bbox[:, 2] - src_bbox[:, 0]
    src_width = src_bbox[:, 3] - src_bbox[:, 1]
    src_ctr_y = src_bbox[:, 0] + 0.5 * src_height
    src_ctr_x = src_bbox[:, 1] + 0.5 * src_width
    dy = loc[:, 0::4]
    dx = loc[:, 1::4]
    dh = loc[:, 2::4]
    dw = loc[:, 3::4]
    ctr_y = dy * src_height[:, np.newaxis] + src_ctr_y[:, np.newaxis]
    ctr_x = dx * src_width[:, np.newaxis] + src_ctr_x[:, np.newaxis]
    print(ctr_x.shape, ctr_y.shape)
    h = np.exp(dh) * src_height[:, np.newaxis]
    w = np.exp(dw) * src_width[:, np.newaxis]

    dst_bbox = np.zeros(loc.shape, dtype=loc.dtype)
    dst_bbox[:, 0::4] = ctr_y - 0.5 * h
    dst_bbox[:, 1::4] = ctr_x - 0.5 * w
    dst_bbox[:, 2::4] = ctr_y + 0.5 * h
    dst_bbox[:, 3::4] = ctr_x + 0.5 * w

    return dst_bbox

def bbox2loc(src_bbox, dst_bbox):
    height = src_bbox[:, 2] - src_bbox[:, 0]
    width = src_bbox[:, 3] - src_bbox[:, 1]
    ctr_y = src_bbox[:, 0] + 0.5 * height
    ctr_x = src_bbox[:, 1] + 0.5 * width

    base_height = dst_bbox[:, 2] - dst_bbox[:, 0]
    base_width = dst_bbox[:, 3] - dst_bbox[:, 1]
    base_ctr_y = dst_bbox[:, 0] + 0.5 * base_height
    base_ctr_x = dst_bbox[:, 1] + 0.5 * base_width

    eps = np.finfo(height.dtype).eps
    height = np.maximum(height, eps)
    width = np.maximum(width, eps)

    dy = (base_ctr_y - ctr_y) / height
    dx = (base_ctr_x - ctr_x) / width
    dh = np.log(base_height / height)
    dw = np.log(base_width / width)

    loc = np.vstack((dy, dx, dh, dw)).transpose()
    return loc


import re
from itertools import groupby
import os
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import gensim.downloader as api

cleanup = re.compile(r'[^a-z0-9]')

def word_prob(word):
	return dictionary.get(word, 0) / total

def entry(line):
    w, c = line.split("\t", 2)
    return (w, int(c))

def segment(text):
    text = re.sub(cleanup, ' ', text)
    probs, lasts = [1.0], [0]
    for i in range(1,len(text) + 1):
        prob_k, k = max((probs[j] * word_prob(text[j:i]), j)for j in range(max(0, i - max_word_length), i))
        probs.append(prob_k)
        lasts.append(k)
    words = []
    i = len(text)
    while i > 0:
        words.append(text[lasts[i]:i])
        i = lasts[i]
    words.reverse()
    return words, probs[-1]

dict_path = "dict.txt"
dictionary = dict(entry(line) for line in open(dict_path))
max_word_length = max(map(len, dictionary))
total = float(sum(dictionary.values()))

def words(text): return re.findall(r'\w+', text.lower())

WORDS = dictionary

def P(word, N=sum(WORDS.values())):
    return WORDS[word] / N

def correction(word):
    return max(candidates(word), key=P)

def candidates(word):
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])

def known(words):
    return set(w for w in words if w in WORDS)

def edits1(word):
    letters    = 'abcdefghijklmnopqrstuvwxyz'
    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]
    deletes    = [L + R[1:]               for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]
    inserts    = [L + c + R               for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)

def edits2(word):
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))

x = y = 'bodyshaming' #enter any n-gram with correct spelling or unigram with correct/wrong spelling

#print(candidates(x))
try:
    x = correction(x)
except:
    x = x
x = ' '.join(segment(x)[0])
y = ' '.join(segment(y)[0])
print(x)
print(y)

model = api.load('fasttext-wiki-news-subwords-300')

def sentence_vec(sentence, model, emb_size, vocab):
    nwords = 0
    sen_emb = np.zeros((emb_size), dtype = "float32")
    for word in sentence:
        if word in vocab:
            nwords = nwords+1
            sen_emb = np.add(sen_emb, model[word])
    if nwords > 0:
        sen_emb = np.divide(sen_emb, nwords)
    return sen_emb

sentence_1 = y
sentence_1_avg_vector = sentence_vec(sentence_1.split(), model=model, emb_size=300, vocab = model.vocab)

sentence_2 = 'body issues'
sentence_2_avg_vector = sentence_vec(sentence_2.split(), model=model, emb_size=300, vocab = model.vocab)

sen1_sen2_similarity =  cosine_similarity(sentence_1_avg_vector.reshape(1,-1),sentence_2_avg_vector.reshape(1,-1))
print(sen1_sen2_similarity)

keyword='sad'
url='https://www.instagram.com/web/search/topsearch/?context=blended&query={0}&__a=1'.format(keyword)
r=requests.get(url)
data=json.loads(r.text)
hashtags=[data['hashtags'][i]['hashtag']['name'] for i in range(len(data['hashtags']))]
#with open('hashtags.json','w') as outfile:
#    json.dump(hashtags,outfile)

for tags in hashtags:
    print(tags)
sentence_1 = keyword
sentence_1_avg_vector = sentence_vec(sentence_1.split(), model=model, emb_size=300, vocab = model.vocab)

for tag in hashtags:
    x = y = tag
    try:
        x = correction(x)
    except:
        x = x
    x = segment(x)[0]
    y = segment(y)[0]
    if len(x) < len(y):
        #print(' '.join(x))
        sentence_2 = ' '.join(x)
    else:
        #print(' '.join(y))
        sentence_2 = ' '.join(y)

    sentence_2_avg_vector = sentence_vec(sentence_2.split(), model=model, emb_size=300, vocab = model.vocab)
    sen1_sen2_similarity =  cosine_similarity(sentence_1_avg_vector.reshape(1,-1),sentence_2_avg_vector.reshape(1,-1))
    if(sen1_sen2_similarity > 0.5):
        #print(sentence_2)
        print(''.join(sentence_2.split(' ')))


import fractions
import math

def gcd(a,b):
    return fractions.gcd(a, b)

# def gcd(a,b):
#     return math.gcd(a,b)

def calculate(arr):
    # L length (N)
    L = [arr[0]]*len(arr)

    # R length (N)
    R = [arr[-1]]*len(arr)

    for i in range(len(arr)):
        if i == 0:
            continue

        L[i] = gcd(L[i-1],arr[i])

        R[len(arr) - (i + 1)] = gcd(R[len(arr) - i],arr[len(arr) - (i + 1)])

    max = 0
    for i in range(len(arr)):

        if i == 0:
            max = R[i+1]

        elif i == len(arr) - 1:
            if max < L[i-1]:
                max = L[i-1]
        else:
            if max < gcd(L[i-1],R[i+1]):
                max = gcd(L[i-1],R[i+1])

    print(max)

N = int(input())
arr = [int(s) for s in input().split(" ")]
calculate(arr)


from ofxstatement.plugin import Plugin
from ofxstatement.parser import StatementParser
from ofxstatement.statement import Statement, StatementLine

import sys
from xml.etree import ElementTree
import hashlib

class PostbankXMLPlugin(Plugin):
    def get_parser(self, filename):
        encoding = self.settings.get('charset', 'iso-8859-1')
        file_obj = open(filename, 'r', encoding=encoding)
        parser = PostbankXMLParser(file_obj)
        parser.statement.account_id = self.settings['account']
        #parser.statement.bank_id = self.settings['bank']
        parser.statement.currency = self.settings.get('currency', 'EUR')
        return parser

class PostbankXMLParser(StatementParser):
    date_format = "%Y-%m-%d"

    def __init__(self, file_obj):
        super(PostbankXMLParser,self).__init__()
        self.__data = file_obj.read()
        self.statement = Statement()

    def P(self,*names):
        res = []
        for name in names:
            res.append("{urn:iso:std:iso:20022:tech:xsd:camt.052.001.03}" + name)
        return '/'.join(res)

    def split_records(self):
        root = ElementTree.fromstring(self.__data)
        return root.findall(self.P('BkToCstmrAcctRpt','Rpt','Ntry'))

    def parse_record(self, record):
        P = self.P

        sl = StatementLine()

        sl.date = self.parse_datetime(record.findall(P('ValDt','Dt'))[0].text)
        sl.amount = float(record.findall(P('Amt'))[0].text)
        sl.trntype = 'DEBIT' if (record.findall(P('CdtDbtInd'))[0].text == "DBIT") else 'CREDIT'
        if sl.trntype == 'DEBIT' and sl.amount > 0:
            sl.amount = -sl.amount
        sl.payee = record.findall(P('NtryDtls','TxDtls','RltdPties','Cdtr','Nm'))[0].text
        sl.memo = ' '.join([e.text for e in record.findall(P('NtryDtls','TxDtls','RmtInf','Ustrd'))])

        # generate unique id
        h = hashlib.sha256()
        h.update(str(sl.date).encode('utf-8'))
        h.update(str(sl.amount).encode('utf-8'))
        h.update(str(sl.trntype).encode('utf-8'))
        h.update(str(sl.payee).encode('utf-8'))
        h.update(str(sl.memo).encode('utf-8'))

        sl.id = h.hexdigest()

        return sl


import torch
import gym
import numpy as np
from torch import nn
from torch.utils.tensorboard import SummaryWriter
from ..memory import Memory
from tqdm.auto import tqdm
import pickle

class BaseAgent(nn.Module):
    def __init__(self, save_path='evals/auto'):
        super().__init__()
        self.memory = Memory()
        self.eval_memory = Memory()
        self.env_steps = nn.Parameter(torch.zeros(1)[0], requires_grad=False)
        self.env_episodes = nn.Parameter(torch.zeros(1)[0], requires_grad=False)
        self.eval_env_steps = nn.Parameter(torch.zeros(1)[0], requires_grad=False)
        self.eval_env_episodes = nn.Parameter(torch.zeros(1)[0], requires_grad=False)
        self.writer = None
        self.save_path = save_path
        self.write_tensorboard_files = False     # is set to True when train is called, so tensorboard file is not written when only evaluation is done

        # if save_path is not None:
        #     self.load(save_path)

    def load(self):
        self.writer = SummaryWriter(self.save_path, flush_secs=10)
        pass

    def write_scalar(self, tag, value, step=None):
        if self.write_tensorboard_files:
            if self.writer is None:
                self.load()
            step = self.env_steps if step is None else step
            self.writer.add_scalar(tag, value, step)

    def write_scalars(self, main_tag, tag_scalar_dict, step=None):
        if self.write_tensorboard_files:
            if self.writer is None:
                self.load()
            step = self.env_steps if step is None else step
            self.writer.add_scalars(main_tag, tag_scalar_dict, step)

    def store_agent(self, store_additional=False):
        if self.save_path is not None:
            save_file = self.save_path + '/agent'
            with open(save_file, 'wb') as f:
                pickle.dump(self.state_dict(), f)
            if store_additional:
                save_file2 = self.save_path + '/agent{}'.format(self.env_steps)
                with open(save_file2, 'wb') as f:
                    pickle.dump(self.state_dict(), f)

    def load_agent_params(self):
        if self.save_path is not None:
            save_file = self.save_path + '/agent'
            with open(save_file, 'rb') as f:
                state_dict = pickle.load(f)
            self.load_state_dict(state_dict)

    def train(self, env, episodes=1000, eval_freq=None, eval_env=None, store_agent_every=100000, **kwargs):
        self.write_tensorboard_files = True
        store_additional = False
        for episode in tqdm(range(episodes)):
            done = False
            state = env.reset()
            while not done:
                action, action_info = self.get_action(state, False, kwargs)
                next_state, reward, done, _ = env.step(action)
                self.memory.append(state=state, action=action, next_state=next_state, reward=reward, done=done, **action_info)
                state = next_state
                done = done.max() == 1
                self.env_steps += 1
                if self.env_steps % store_agent_every == 0:
                    store_additional = True
                self.after_step(done, False, kwargs)

            self.env_episodes += 1

            if eval_freq is not None and self.env_episodes % eval_freq == 0 and ("warmup_steps" not in kwargs or len(self.memory) > kwargs["warmup_steps"]):
                self.store_agent(store_additional=store_additional)
                store_additional = False
                self.eval(env if eval_env is None else eval_env, episodes=1, **kwargs)

        self.store_agent()   # store at the end, so it is stored also when eval is off

    def eval(self, env, eval_episodes=10, render=False, **kwargs):
        for episode in range(eval_episodes):
            done = False
            state = env.reset()
            while not done:
                if render:
                    env.render()
                action, action_info = self.get_action(state, True, kwargs)
                next_state, reward, done, _ = env.step(action)
                self.eval_memory.append(state=state, action=action, next_state=next_state, reward=reward, done=done, **action_info)
                state = next_state
                done = done.max() == 1
                self.eval_env_steps += 1
                self.after_step(done, True, kwargs)

            self.eval_env_episodes += 1

    def get_action(self, state, eval, args):
        raise NotImplementedError()

    def after_step(self, done, eval, args):
        raise NotImplementedError()


from datetime import datetime

import MySQLdb
from openpyxl import Workbook, load_workbook
from openpyxl.drawing.image import Image
from openpyxl.styles import Font, colors

class ExcelUtils(object):
    """
    Excel文件工具类
    """
    def __init__(self):
        self.wb = Workbook()
        self.ws = self.wb.active
        self.ws_two = self.wb.create_sheet('我的表单')
        self.ws_three = self.wb.create_sheet()

    def do_sth(self):
        self.ws['A1'] = 66
        self.ws['A2'] = 'Hello'
        self.ws['A3'] = datetime.now()

        for row in self.ws_two['A1:E5']:
            for cell in row:
                cell.value = 2

        # 求和
        self.ws_two['G1'] = '=SUM(A1:E1)'

        # 插入图片
        img = Image('./static/test.jpg')
        self.ws.add_image(img, 'K25')

        # 合并单元格
        self.ws.merge_cells('A4:E5')
        self.ws.unmerge_cells('A4:E5')

        # 设置文字
        font = Font(sz=18, color=colors.RED)
        self.ws['A2'].font = font
        self.wb.save('./static/test.xlsx')

    def read_xls(self):
        """
        读取excel文件
        """
        ws = load_workbook('./static/score.xlsx')
        names = ws.get_sheet_names()
        print(names)
        # wb = ws.active
        wb = ws[names[0]]

        for (i, row) in enumerate(wb.rows):
            if i < 2:
                continue
            year = wb['A{0}'.format(i + 1)].value
            ma = wb['B{0}'.format(i + 1)].value
            avg = wb['C{0}'.format(i + 1)].value

            if year is None:
                continue

            conn = self.get_conn()
            cursor = conn.cursor()
            sql = 'INSERT INTO `score` (`year`, `max`, `avg`) VALUES({year}, {max}, {avg})'.format(year=year, max=ma, avg=avg)
            print(sql)
            cursor.execute(sql)
            conn.autocommit(True)

    def get_conn(self):
        """获取mysql链接"""
        try:
            conn = MySQLdb.connect(
            db='user_grade',
            host='localhost',
            user='root',
            password='',
            charset='utf8',
            )
        except:
            pass
        return conn

    def export_xls(self):
        """导出数据库数据到excel"""
        conn = self.get_conn()
        cursor = conn.cursor()
        #查询语句
        sql = 'SELECT `year`, `max`, `avg` FROM `score`'
        #数据量大则分页查询
        # sql = 'SELECT `year`, `max`, `avg` FROM `score` limit 0, 100'
        cursor.execute(sql)
        rows = cursor.fetchall()

        #写入数据
        wb = Workbook()
        ws = wb.active
        for (i, row) in enumerate(rows):
            print(row)
            (ws['A{0}'.format(i + 1)],
            ws['B{0}'.format(i + 1)],
            ws['C{0}'.format(i + 1)]) = row

        # 保存excel
        wb.save('./static/export.xlsx')

if __name__ == "__main__":
    client = ExcelUtils()
    # client.do_sth()
    # client.read_xls()
    client.export_xls()


import streamlit as st
import pandas as pd
import numpy as np
import altair as alt
import matplotlib.pyplot as plt
import graphviz as graphviz
import seaborn as sns

#Defining Columns
c1, c2, c3 = st.columns(3)
# Defining Metrics

c1.metric("Rainfall", "100 cm", "10 cm")
c2.metric(label="Population", value="123 Billions", delta="1 Billions", delta_color="inverse")
c3.metric(label="Customers", value=100, delta=10, delta_color="off")

st.metric(label="Speed", value=None, delta=0)
st.metric("Trees", "91456", "-1132649")

df = pd.DataFrame(
     np.random.randn(30, 10),
     columns=('col_no %d' % i for i in range(10)))
# defining multiple arguments in write function
st.write('Here is our Data', df, 'Data is in dataframe format.\n', "\nWrite is Super function")

df = pd.DataFrame(
     np.random.randn(10, 2),
     columns=['a', 'b'])
# Defining Chart
chart = alt.Chart(df).mark_bar().encode(
     x='a', y='b',  tooltip=['a', 'b'])
# Defining Chart in write() function
st.write(chart)

s = np.random.logistic(10, 5, size=5)
chart, ax = plt.subplots()
ax.hist(s, bins=15)
st.write(chart)

df = pd.DataFrame(
    np.random.randn(40, 4),
    columns=["C1", "C2", "C3", "C4"])
# Bar Chart
st.bar_chart(df)

df = pd.DataFrame(
    np.random.randn(40, 4),
    columns=["C1", "C2", "C3", "C4"])
st.line_chart(df)

df = pd.DataFrame(
    np.random.randn(40, 4),
    columns=["C1", "C2", "C3", "C4"])
st.area_chart(df)

locate_map = pd.DataFrame(
  np.random.randn(50, 2)/[10,10] + [15.4589, 75.0078],
  columns = ['latitude', 'longitude'])
# Map Function
st.map(locate_map)

st.graphviz_chart('''
    digraph {
        "Training Data" -> "ML Algorithm"
        "ML Algorithm" -> "Model"
        "Model" -> "Result Forecasting"
        "New Data" -> "Model"
    }
''')


#!/usr/local/bin/python3

# Setup import
import argparse, yaml , logging , sys , os

#!/usr/local/bin/python3
def create_arguments():
    text = '''
    This application is the middleware application for the CEG4912/3 Group 2 Capstone Project

    For more information, email rconr060@uottawa.ca
    '''
    parser = argparse.ArgumentParser(description=text)
    parser.add_argument("--base_url", "-u", help="set the value of the base_url to a different value (default https://localhost:8080)")
    parser.add_argument("-M", "--mock", help="Run the application using mock sensor data ", action="store_true")
    parser.add_argument("-O", "--offline", help="Run the system offline (API Calls print to sysout)", action="store_true")
    return parser.parse_args()

def parse_config():
    with open('hardware/src/config.yaml') as f:
        return yaml.load(f, Loader=yaml.FullLoader)

def setup_logging(flag = logging.INFO , log_file ="debug.log" , sys_out=sys.stdout ):
    logging.basicConfig(
        level=flag,
        format="%(asctime)s [%(levelname)s] %(message)s",
        handlers=[
            logging.FileHandler(log_file),
            logging.StreamHandler(sys_out)
        ]
    )


import matplotlib.pyplot as plt
import numpy as np
from evaluate_point_triangle import evaluate_point_triangle, evaluate_point_triangle_matrix, evaluate_point_triangle_transform

# from https://stackoverflow.com/questions/10481990/matplotlib-axis-with-two-scales-shared-origin
def align_yaxis(ax1, ax2):
    _, y1 = ax1.transData.transform((0, 0))
    _, y2 = ax2.transData.transform((0, 0))
    inv = ax2.transData.inverted()
    _, dy = inv.transform((0, 0)) - inv.transform((0, y1-y2))
    miny, maxy = ax2.get_ylim()
    ax2.set_ylim(miny+dy, maxy+dy)

def get_surface_points(controls):
    p200, p020, p002, p110, p101, p011 = controls
    points = []
    u_s = np.linspace(0., 1., 200)
    for u_ in u_s:
        for v_ in u_s:
            if u_ + v_ <= 1:
                w_ = 1-(u_+v_)
                p = p200*(u_**2) + p020*(v_**2) + p002*(w_**2) + \
                    2*p110*u_*v_ + 2*p101*u_*w_ + 2*p011*v_*w_
                points.append(p)
    return np.array(points)

def plot_evaluations(controls):
    origin = np.array([0., 0., 0.])
    u_ = .33
    v_ = .33
    w_ = 1-(u_+v_)
    addends = [0, 1e4, 6.5e4, 1e6, 1e13]
    figure, axis = plt.subplots(2, 3)
    plt.axis('off')
    for position in ['top','bottom','left','right']:
        axis[1,2].spines[position].set_color('gray')
    for i in range(len(addends)):
        addend = addends[i]
        control_points = controls + addend
        p200, p020, p002, p110, p101, p011 = control_points
        surface_point = p200*(u_**2) + p020*(v_**2) + p002 * \
            (w_**2) + 2*p110*u_*v_ + 2*p101*u_*w_ + 2*p011*v_*w_
        ray = surface_point - origin
        ray /= np.linalg.norm(ray)
        results = []
        transform_results = []
        for t in np.arange(-1, 1.01, 0.01):
            p = surface_point + t*ray
            result = evaluate_point_triangle(
                p, p200, p020, p002, p110, p101, p011)
            transform_result = evaluate_point_triangle_transform(
                p, p200, p020, p002, p110, p101, p011)
            results.append((t, result))
            transform_results.append((t, transform_result))
        determinants = []
        for t in np.arange(-1, 1, .1):
            p = surface_point + t*ray
            det = evaluate_point_triangle_matrix(
                p, p200, p020, p002, p110, p101, p011)
            determinants.append((t, det))
        patch_planar_center = (p200+p020+p002)/3
        patch_radius = np.max(np.linalg.norm(
            np.array([p200, p020, p002])-patch_planar_center, axis=1))
        t = [entry[0] for entry in results]
        result = [entry[1] for entry in results]
        transform_result = [entry[1] for entry in transform_results]
        determinant_t = [entry[0] for entry in determinants]
        determinant = [entry[1] for entry in determinants]
        j = i // 3
        k = i % 3
        axis[j, k].plot(t, result, zorder=-1, color='C0', linewidth=2, label='Implicit')
        axis[j, k].scatter(determinant_t, determinant, color='C1', s=24,
                           zorder=1, label='Numerical det')
        ax2 = axis[j, k].twinx()
        ax2.plot(t, transform_result, zorder=0, color='C2', dashes=[3, 3],
                 linewidth=3, label='Transformed implicit')
        axis_lines, axis_labels = axis[j, k].get_legend_handles_labels()
        ax2_lines, ax2_labels = ax2.get_legend_handles_labels()
        axis[j, k].set_xlabel('t', fontsize=12)
        plt.xticks(fontsize=12)
        axis[j,k].set_ylabel('Value', fontsize=12)
        ax2.set_ylabel('Value', fontsize=12, color='C2')
        plt.yticks(fontsize=12)
        ax2.tick_params(labelsize=10)
        if j==0 and k==0:
            axis[j, k].set_title("Offset: 0", fontsize=12)
        elif j==0 and k==2:
            axis[j, k].set_title("Offset: {:.1E}".format(addend), fontsize=12)
        else:
            axis[j, k].set_title("Offset: {:.0E}".format(int(addend)), fontsize=12)
        plt.axhline(0, color='black', linestyle=':')
        plt.axvline(0, color='black', linestyle=':')
        axis[j, k].spines['top'].set_visible(False)
        axis[j, k].spines['right'].set_visible(False)
        axis[j, k].spines['bottom'].set_visible(False)
        axis[j, k].spines['left'].set_visible(False)
        align_yaxis(ax2, axis[j, k])
    last_lines, last_labels = axis[1,2].get_legend_handles_labels()
    axis[1,2].legend(axis_lines + ax2_lines + last_lines, axis_labels +
                  ax2_labels + last_labels, fontsize=12, borderaxespad=0, borderpad=2, frameon=True, loc='center')
    plt.subplots_adjust(wspace=.4, hspace=.4)
    plt.show()

control_points = np.array([
    [1.0, 0.0, 0.0],
    [0.0, 1.0, 0.0],
    [0.0, 0.0, 1.0],
    [.65, .65, 0.0],
    [.65, 0.0, .65],
    [0.0, .65, .65]
])
plot_evaluations(control_points)


# ClearML - Example of manual model reporting
from clearml import Task, OutputModel
from clearml.utilities.plotly_reporter import SeriesInfo
import pandas as pd
import numpy as np

# Connecting ClearML with the current process,
task = Task.init(project_name="examples", task_name="Model reporting plots example")

# Create output model and connect it to the task
output_model = OutputModel(task=task)

# Optional: add labels to the model, so we do not forget
labels = {"background": 0, "cat": 1, "dog": 2}
output_model.update_labels(labels)

# Register an already existing Model file somewhere
model_url = "https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5x6.pt"
output_model.update_weights(register_uri=model_url)

output_model.report_scalar("Reported Metrics", "Val - mAP@50", 72.7, 0)
output_model.report_single_value("Total epochs", 8)

df = pd.DataFrame(
    {
        "Model": ["YOLOv5n", "YOLOv5s", "YOLOv5m6", "YOLOv5x6"],
        "size (pixels)": [640, 640, 1280, 1280],
        "Val - mAP@50-95": [28.0, 37.4, 51.3, 55.0],
        "CPU Speed b1 (ms)": [45, 98, 887, 3136],
    }
)
output_model.report_table(
    title="Summary Table", series="Comparison", iteration=0, table_plot=df
)

output_model.report_line_plot(
    title="Accuracy",
    series=[
        SeriesInfo(
            name="Validation",
            data=np.array(
                [
                    [0, 0.3],
                    [1, 0.55],
                    [2, 0.7],
                    [3, 0.77],
                    [4, 0.8],
                    [5, 0.816],
                    [6, 0.822],
                    [7, 0.829],
                ]
            ),
        )
    ],
    xaxis="Iteration",
    yaxis="Validation Accuracy",
)

# Or upload a local model file to be later used
# output_model.update_weights(weights_filename="/path/to/file.onnx")

print("Model registration completed")


"""Posetta, translating between different position formats

Posetta is a command line and GUI utility for translating between different
file formats used for representing points and possibly velocities.

See README.md and __main__.py for more information.
"""

# Standard library imports
from datetime import date as _date
from collections import namedtuple as _namedtuple

# Version of Posetta
#
# This is automatically set using the bumpversion script
__version__ = "0.0.1"

# Start of development
_birthday = _date(2018, 4, 20)

# Maintainers of Posetta
_Author = _namedtuple("_Author", ["name", "email", "start", "end"])
_AUTHORS = [
    _Author("Geir Arne Hjelle", "geir.arne.hjelle@kartverket.no", _birthday, _date.max),
    _Author("Kristian Evers", "kbevers@sdfe.dk", _birthday, _date.max),
]

__author__ = ", ".join(a.name for a in _AUTHORS if a.start < _date.today() < a.end)
__contact__ = ", ".join(a.email for a in _AUTHORS if a.start < _date.today() < a.end)
__url__ = "https://github.com/NordicGeodesy/posetta"

# Copyleft of the library
__copyright__ = f"{_birthday.year} - {_date.today().year} Nordic Geodetic Commision"


import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix,classification_report
import pickle

# 加载数据
data = pd.read_csv('./data/data_new.csv',index_col=0)
# 求皮尔相关系数（找重要特征）
corr = data.corr()
# 以0.08为阈值进行筛选
feature_index = corr['IS_LOST'].drop('IS_LOST').abs()>0.08
# 筛选出出特征的名字
feature_name = feature_index.loc[feature_index].index
# ['INNET_MONTH', 'CREDIT_LEVEL', 'NO_ROAM_CDR_NUM', 'CALL_DAYS',
#       'CALLING_DAYS', 'CALLED_DAYS', 'CALL_RING', 'CALLED_RING']

#自变量【影响用户流失的关键属性】
X = data.loc[:,feature_name]
y = data.loc[:,'IS_LOST'] # 预测目标

# 正样本索引（1） - 已经流失的用户
index_positive = y.index[y==1]
# 负样本索引（0） - 留下的客户 当前比例是1:30
index_negative = np.random.choice(y.index[y==0].tolist(),y.value_counts()[1])
# 取出流失用户与留下用户的特征数据
X_positive = X.loc[index_positive,:]
X_negative = X.loc[index_negative,:]
# 取出流失用户和留下用户的结果 0/1
y_positive = y.loc[index_positive]
y_negative = y.loc[index_negative]
# 数据拼接
X = pd.concat([X_positive,X_negative],axis=0)
y = pd.concat([y_positive,y_negative],axis=0)

#------------------------
# 划分数据集
X_train,X_test,y_tran,y_test=train_test_split(X,y,test_size=0.2,stratify=y)
# 构建模型1：集成学习
# 随机森林：n_estimators默认训练100次
rfc = RandomForestClassifier(n_estimators=500)
# 导入训练数据
rfc.fit(X_train,y_tran)
# 调用模型预测结果
y_pre = rfc.predict(X_test)
# 模型评估
score = rfc.score(X_test,y_test)
print(score) #79
# 混淆矩阵
confusion_matrix(y_test,y_pre)
# -----------------------------------
# 保存模型

with open('./data/yjw_Model.pkl','wb') as f:
    pickle.dump(rfc,f)
    print('模型保存成功')
    print(classification_report(y_test,y_pre))


from bs4 import BeautifulSoup
import requests
import re
import lxml
import csv

website = "https://order.wingstop.com/"
fieldnames = ['location name', 'street', 'city', 'state', 'zip','phone']

html = requests.get("https://order.wingstop.com/locations")
soup = BeautifulSoup(html.text, 'lxml')

lists = soup.findAll('a')
urls = list()

for url in lists:
	if(re.search('locations/',url['href'])):
		urls.append(website + url['href'])

with open('wingstop 2.csv', 'w', newline='') as csvfile:
	writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
	writer.writeheader()
	for link in urls:
		response = requests.get(link)
		source = BeautifulSoup(response.text, 'lxml')
		locations = source.findAll('li',class_='vcard')

		for location in locations:
			if location.a is not None:
				html = requests.get(location.a['href'])
				locData = BeautifulSoup(html.text, 'lxml')
			else:
				print(location)

			name = locData.h1.text.strip()
			street = locData.find('span', class_='street-address').text.strip()
			locality = locData.find('span',class_='locality').text.strip()
			region = locData.find('span',class_='region').text.strip()
			postal_code = locData.find('span', class_='postal-code').text.strip()
			telephon = locData.find('span', class_='tel').text.strip()

			writer.writerow({'location name':name, 'street':street, 'city':locality, 'state':region, 'zip':postal_code,'phone':telephon})


from typing import overload, Union, Text, BinaryIO, Optional, Any, List, Callable
import sys
import os

if sys.version_info >= (3, 6):
    _File = Union[Text, os.PathLike[Text], BinaryIO]
else:
    _File = Union[Text, BinaryIO]

@overload
def what(file: _File) -> Optional[str]: ...
@overload
def what(file: Any, h: bytes) -> Optional[str]: ...
tests: List[Callable[[bytes, BinaryIO], Optional[str]]]


import gym
import random
import numpy as np
import argparse
from actorcitic4 import Actor, act
import torch
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Normal
import os
import random
import torch.nn as nn

SAVEPATH1 = os.getcwd() + '/train/actor_params.pth'

env = gym.make("FetchPickAndPlace-v1")
env2 = gym.wrappers.FlattenDictWrapper(env, dict_keys=['observation', 'desired_goal'])

parser = argparse.ArgumentParser(description='A3C')
parser.add_argument('--use-cuda',default=True,
                    help='run on gpu.')
parser.add_argument('--save-path1',default=SAVEPATH1,
                    help='model save interval (default: {})'.format(SAVEPATH1))
parser.add_argument('--max-grad-norm', type=float, default=250,
                    help='value loss coefficient (default: 50)')
parser.add_argument('--entropy-coef', type=float, default=0.01,
                    help='entropy term coefficient (default: 0.01)')
parser.add_argument('--value-loss-coef', type=float, default=0.5,
                    help='value loss coefficient (default: 0.5)')
parser.add_argument('--gamma', type=float, default=0.9,
                    help='discount factor for rewards (default: 0.9)')
parser.add_argument('--tau', type=float, default=1.00,
                    help='parameter for GAE (default: 1.00)')
args = parser.parse_args()

model = Actor()

if args.use_cuda:
    model.cuda()

torch.cuda.manual_seed_all(25)
optimizer = optim.Adam(model.parameters(), lr=0.0001)

if os.path.isfile(args.save_path1):
    print('Loading A3C parametets ...')
    model.load_state_dict(torch.load(args.save_path1))

for p in model.fc1.parameters():
    p.requires_grad = False
for p in model.fc2.parameters():
    p.requires_grad = False

FloatTensor = torch.cuda.FloatTensor if args.use_cuda else torch.FloatTensor

model.eval()

max_eps = 200000
max_steps = 50
ep_numb = 0
done = True
success = 0

while ep_numb < max_eps:
    ep_numb +=1
    lastObs = env.reset()
    goal = lastObs['desired_goal']
    objectPos = lastObs['observation'][3:6]
    object_rel_pos = lastObs['observation'][6:9]
    object_oriented_goal = object_rel_pos.copy()
    object_oriented_goal[2] += 0.03 # first make the gripper go slightly above the object
    timeStep = 0 #count the total number of timesteps
    state_inp = torch.from_numpy(env2.observation(lastObs)).type(FloatTensor)
    Ratio=[]
    while np.linalg.norm(object_oriented_goal) >= 0.015 and timeStep <= env._max_episode_steps:
        env.render()
        action = [0, 0, 0, 0, 0, 0]
        act_tensor, ratio = act(state_inp, model, True, False)
        #print(act_tensor)

        Ratio.append(ratio.cpu().detach().numpy())
        for i in range(len(object_oriented_goal)):
            action[i] = act_tensor[i].cpu().detach().numpy()

        object_oriented_goal = object_rel_pos.copy()
        object_oriented_goal[2] += 0.03

        action[3] = 0.05
        obsDataNew, reward, done, info = env.step(action)
        timeStep += 1
        objectPos = obsDataNew['observation'][3:6]
        object_rel_pos = obsDataNew['observation'][6:9]
        state_inp = torch.from_numpy(env2.observation(obsDataNew)).type(FloatTensor)
        if timeStep >= env._max_episode_steps: break

    while np.linalg.norm(object_rel_pos) >= 0.005 and timeStep <= env._max_episode_steps :
        env.render()
        action = [0, 0, 0, 0, 0, 0]
        act_tensor, ratio = act(state_inp, model, False, False)
        Ratio.append(ratio.cpu().detach().numpy())
        for i in range(len(object_oriented_goal)):
            action[i] = act_tensor[i].cpu().detach().numpy()

        action[3]= -0.01
        action[5] = act_tensor[3].cpu().detach().numpy()
        obsDataNew, reward, done, info = env.step(action)
        timeStep += 1

        objectPos = obsDataNew['observation'][3:6]
        object_rel_pos = obsDataNew['observation'][6:9]
        state_inp = torch.from_numpy(env2.observation(obsDataNew)).type(FloatTensor)
        if timeStep >= env._max_episode_steps: break

    while np.linalg.norm(goal - objectPos) >= 0.01 and timeStep <= env._max_episode_steps :

        env.render()
        action = [0, 0, 0, 0, 0, 0]
        act_tensor, ratio = act(state_inp, model, False, True)
        Ratio.append(ratio.cpu().detach().numpy())

        for i in range(len(goal - objectPos)):
            action[i] = act_tensor[i].cpu().detach().numpy()

        action[3] = -0.01
        obsDataNew, reward, done, info = env.step(action)
        timeStep += 1
        state_inp = torch.from_numpy(env2.observation(obsDataNew)).type(FloatTensor)
        objectPos = obsDataNew['observation'][3:6]
        object_rel_pos = obsDataNew['observation'][6:9]
        if timeStep >= env._max_episode_steps: break

    while True: #limit the number of timesteps in the episode to a fixed duration
        env.render()
        action = [0, 0, 0, 0, 0, 0]
        action[3] = -0.01 # keep the gripper closed

        obsDataNew, reward, done, info = env.step(action)
        timeStep += 1

        objectPos = obsDataNew['observation'][3:6]
        object_rel_pos = obsDataNew['observation'][6:9]

        if timeStep >= env._max_episode_steps: break

    if info['is_success'] == 1.0:
        success +=1
    if done:
        if ep_numb % 100==0:
            print("num episodes {}, success {}".format(ep_numb, success))


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import cv2
# import seaborn as sns
# import matplotlib.pyplot as plt
import torch
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils, models
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from sklearn.metrics import confusion_matrix
import pickle
import time
import argparse
from scipy import fft

def change_output_layers(model):
    # model.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False) # change 3 chanels to 1 chanel for gray scale
    model.fc = nn.Linear(2048, classes, bias=True)
    return model

def load_pkl_data(dataset_dst):
    with open(dataset_dst, "rb") as the_file:
        return pickle.load(the_file)
        the_file.close()

def pytorch_rolling_window(x, window_size, step_size=1):
    # unfold dimension to make our rolling window
    return x.unfold(0,window_size,step_size)

def preprocess(mode ='train', rows = 60000):

    axes = ['x','y','z']
    pass_dir = r"D:\Side Work Data\PHM\Demo_box\demo_box_0518\wav\pass"
    ng1_dir = r"D:\Side Work Data\PHM\Demo_box\demo_box_0518\wav\ng1"
    ng2_dir = r"D:\Side Work Data\PHM\Demo_box\demo_box_0518\wav\ng2"
    all_data = np.array([])
    all_label_data = np.array([])
    window_size = 51200
    step_size = 512

    for ax in axes:

        pass_filename = ""
        ng1_filename = ""
        ng2_filename = ""

        # there are three files
        if mode == 'train':
            pass_filename = pass_dir + '/' + ax + '/' + '20210518112208.csv'
            ng1_filename = ng1_dir + '/' + ax + '/' + '20210518114530.csv'
            ng2_filename = ng2_dir + '/' + ax + '/' + '20210518134847.csv'
        elif mode == 'test1':
            pass_filename = pass_dir + '/' + ax + '/' + '20210518112338.csv'
            ng1_filename = ng1_dir + '/' + ax + '/' + '20210518114602.csv'
            ng2_filename = ng2_dir + '/' + ax + '/' + '20210518134922.csv'
            step_size = window_size #no augmentation for test
        elif mode == 'test2':
            pass_filename = pass_dir + '/' + ax + '/' + '20210518112416.csv'
            ng1_filename = ng1_dir + '/' + ax + '/' + '20210518114635.csv'
            ng2_filename = ng2_dir + '/' + ax + '/' + '20210518134957.csv'
            step_size = window_size #no augmentation for test

        _temp_data = np.array([])
        _temp_lable_data = np.array([])

        for filename, label in zip([pass_filename, ng1_filename, ng2_filename],[0,1,2]):
            print('processing for file {}'.format(filename))

            data = pd.read_csv(filename).values[:rows]
            data = torch.tensor(data, dtype=torch.float)
            data = pytorch_rolling_window(data, window_size = window_size, step_size = step_size)
            data = data.numpy().reshape((-1,window_size)) #important: you must reshape it!
            n_samples = len(data)
            label_data = np.array([label]*n_samples).reshape((-1,1))

            if label == 0:
                _temp_data = data
                _temp_lable_data = label_data
            else:
                _temp_data = np.vstack((_temp_data,data))
                _temp_lable_data = np.vstack((_temp_lable_data,label_data))

        if ax == 'x':
            all_data = _temp_data
            all_label_data = _temp_lable_data
        else:
            all_data = np.hstack((all_data, _temp_data))

    # print(all_data.shape)
    # print(all_label_data.shape)
    return np.hstack((all_data,all_label_data))

def merge_testdata(all_data1,all_data2):
    data1, label1 = all_data1[:,:-1], all_data1[:,-1].reshape((-1,1))
    data2, label2 = all_data2[:,:-1], all_data2[:,-1].reshape((-1,1))

    data, label = np.vstack((data1,data2)), np.vstack((label1,label2))

    return np.hstack((data,label))

class PHM_dataset(Dataset):
    def __init__(self, all_data):

        self.rows = len(all_data)
        self.imgnp = all_data[:self.rows, :-1]
        self.labels = all_data[:self.rows, -1]

    def __len__(self):
        return self.rows

    def __getitem__(self, idx):
        xyz_data = self.imgnp[idx]
        fft_data = self.do_fft(xyz_data)
        image = torch.tensor(fft_data, dtype=torch.float)
        image = image.view(3, 160, 160)  # (channel, height, width)
        label = torch.tensor(self.labels[idx], dtype = torch.long)
        return (image, label)

    def do_fft(self, xyz_data):

        n = len(xyz_data)
        x_data = fft(xyz_data[:int(n/3)])
        y_data = fft(xyz_data[int(n/3):int(2*n/3)])
        z_data = fft(xyz_data[int(2*n/3):])
        N = int(len(x_data)/2)
        x_data = np.abs(x_data[range(N)])
        y_data = np.abs(y_data[range(N)])
        z_data = np.abs(z_data[range(N)])

        res = np.array([])
        for _data in [x_data,y_data,z_data]:
            res = np.append(res,_data)

        return res

if __name__ == "__main__":

    parser=argparse.ArgumentParser()
    help_="train the model"
    parser.add_argument("-t","--train",help=help_,action="store_true")
    help_="verify the model"
    parser.add_argument("-v","--verify",help=help_,action="store_true")
    help_="load the previous weights"
    parser.add_argument("-w","--weights",help=help_)
    args=parser.parse_args()

    lr = 0.0001
    batch_size = 4
    epochs = 50
    classes = 3
    display_frequency = 7

    model = models.resnet50(pretrained=True)
    model = change_output_layers(model)
    print(model)

    if(torch.cuda.is_available()):
        model = model.cuda()
        torch.backends.cudnn.benchmark=True

    criterion = nn.CrossEntropyLoss()
    # optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
    optimizer = torch.optim.Adam(model.parameters(), lr = lr)

    _train_data = preprocess('train')
    train_dataset = PHM_dataset(_train_data)
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
    train_loader_no_shuffle = DataLoader(train_dataset, batch_size=batch_size, shuffle=False)

    _test_data1 = preprocess('test1')
    _test_data2 = preprocess('test2')
    _test_data = merge_testdata(_test_data1, _test_data2)

    test_dataset = PHM_dataset(_test_data)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

    if args.train:

        last_loss = float('Inf')
        current_loss = 0

        print("len of train_loader = %d" %len(train_loader))
        print("len of test_loader = %d" %len(test_loader))
        print("len of train data = %d" %len(train_dataset))
        print("len of test data = %d" %len(test_dataset))

        print('Start Training....')
        train_losses, val_losses = [], []
        train_accu, val_accu = [], []

        start = time.time()

        if args.weights:
            model.load_state_dict(torch.load(args.weights))

        for epoch in range(epochs):
            epoch_start_time=time.time()
            running_loss=0
            sub_total = 0
            display_loss=0
            for i, data in enumerate(train_loader, 1):
                images, labels = data

                if(torch.cuda.is_available()):
                    images = images.cuda()
                    labels = labels.cuda()

                optimizer.zero_grad()
                outputs = model(images)

                _, predicted = torch.max(outputs.data, 1)
                sub_total += labels.size(0)
                loss = criterion(outputs, labels)
                running_loss+=loss.item()
                display_loss+=loss.item()

                loss.backward()
                optimizer.step()

                if(i%display_frequency == (display_frequency-1)):
                    print('Epoch: {} Batch: {}/{} loss: {:.6f}'.format(epoch+1, i, len(train_loader), display_loss/display_frequency))
                    display_loss = 0

            # saving best model according to loss
            current_loss = running_loss/len(train_loader)
            if current_loss < last_loss:
                print("loss improved from {:.6f} to {:.6f}...saving best model".format(last_loss,current_loss))
                torch.save(model.state_dict(), 'best_weights.pkl')
                last_loss = current_loss
            else:
                print("loss did not improve from {:.6f} to {:.6f}...".format(last_loss,current_loss))

            ##################################### evaluate the model####################################
            accuracy=0
            sub_total = 0
            with torch.no_grad():
                for j, data in enumerate(train_loader_no_shuffle,1):
                    images, labels = data

                    if(torch.cuda.is_available()):
                        images = images.cuda()
                        labels = labels.cuda()

                    outputs = model(images)

                    _, predicted = torch.max(outputs.data, 1)
                    sub_total += labels.size(0)
                    accuracy += (predicted == labels).sum().item()

            train_losses.append(current_loss)
            train_accu.append((accuracy/sub_total)*100)

            val_loss=0
            accuracy=0
            sub_total = 0
            with torch.no_grad():
                for j, data in enumerate(test_loader,1):
                    images, labels = data

                    if(torch.cuda.is_available()):
                        images = images.cuda()
                        labels = labels.cuda()

                    outputs = model(images)

                    _, predicted = torch.max(outputs.data, 1)
                    sub_total += labels.size(0)
                    accuracy += (predicted == labels).sum().item()
                    val_loss += criterion(outputs, labels).item()

            val_losses.append(val_loss/len(test_loader))
            val_accu.append((accuracy/sub_total)*100)

            print("Summary at Epoch: {}/{}..".format(epoch+1,epochs),
            "Time: {:.2f}s..".format(time.time()-epoch_start_time),
            "Training_Loss: {:.3f}..".format(train_losses[-1]),
            "Training_Accu: {:.3f}%..".format(train_accu[-1]),
            "Val_Loss: {:.3f}..".format(val_losses[-1]),
            "Val_Accu: {:.3f}%".format(val_accu[-1]))
            ##################################### evaluate the model####################################

        torch.save(model.state_dict(), 'resnet50_last_weights.pkl')
        # torch.save(model, 'resnet50.pkl')
        print('Training Completed in: {} secs'.format(time.time()-start))

    if args.weights and args.verify:

        model.load_state_dict(torch.load(args.weights))

        print("len of train_loader = %d" %len(train_loader_no_shuffle))
        print("len of test_loader = %d" %len(test_loader))
        print("len of train data = %d" %len(train_dataset))
        print("len of test data = %d" %len(test_dataset))

        #------------------------confusion matrix--------------------------#
        print('Calculate confusion matrix of testing data....')
        start = time.time()

        predictions = torch.LongTensor()
        for i, data in enumerate(test_loader, 1):
            images, labels = data
            images = images.cuda()

            outputs = model(images)

            pred = outputs.cpu().data.max(1, keepdim=True)[1]
            predictions = torch.cat((predictions, pred), dim=0)

        print('Test Completed in {} secs'.format(time.time() - start))
        test_matrix = confusion_matrix(test_dataset.labels , predictions.numpy())

        print('Calculate confusion matrix of training data....')
        start = time.time()

        predictions = torch.LongTensor()
        for i, data in enumerate(train_loader_no_shuffle, 1):
            images, labels = data
            images = images.cuda()

            outputs = model(images)

            pred = outputs.cpu().data.max(1, keepdim=True)[1]
            predictions = torch.cat((predictions, pred), dim=0)

        print('Train Completed in {} secs'.format(time.time() - start))
        train_matrix = confusion_matrix(train_dataset.labels , predictions.numpy())

        for matrix , filename in zip([test_matrix,train_matrix],["test_matrix.csv", "train_matrix.csv"]):
            df_mat = pd.DataFrame(matrix)
            df_mat.to_csv(filename)

        print("test matrix:\n",test_matrix)
        print("train matrix:\n",train_matrix)

        #------------------------confusion matrix--------------------------#