from tqdm.auto import tqdm
from Person import Person
from Eye import Eye
from EyeImage import EyeImage
from IOP import IOP
from Refractive_Defect import Refractive_Defect
import pandas as pd

class Database:

    def __init__(self,
                 od_Spreadsheet_filePath,
                 os_Spreadsheet_filePath,
                 odfundusImages,
                 osfundusImages,
                 odimagesWithContours,
                 osimagesWithContours):
        pd.set_option('display.max_colwidth', 500)
        df_od = pd.read_excel(od_Spreadsheet_filePath, header=2)
        df_od.columns=[ 'id'\
                        , 'age'\
                        , 'gender'\
                        , 'od_eye_diagnosis'\
                        , 'od_eye_refractive_defect_dioptre_1'\
                        , 'od_eye_refractive_defect_dioptre_2'\
                        , 'od_eye_refractive_defect_astigmatism'\
                        , 'od_eye_phakic_Pseudophakic'\
                        , 'od_eye_iop_pneumatic'\
                        , 'od_eye_iop_perkins'\
                        , 'od_eye_pachymetry'\
                        , 'od_eye_axialLength'\
                        , 'od_eye_vf_md']
        df_od['id'] = df_od['id'].str[-3:]

        df_os = pd.read_excel(os_Spreadsheet_filePath, header=2)
        df_os.columns=['id'\
                        , 'age'\
                        , 'gender'\
                        , 'os_eye_diagnosis'\
                        , 'os_eye_refractive_defect_dioptre_1'\
                        , 'os_eye_refractive_defect_dioptre_2'\
                        , 'os_eye_refractive_defect_astigmatism'\
                        , 'os_eye_phakic_Pseudophakic'\
                        , 'os_eye_iop_pneumatic'\
                        , 'os_eye_iop_perkins'\
                        , 'os_eye_pachymetry'\
                        , 'os_eye_axialLength'\
                        , 'os_eye_vf_md']
        df_os['id'] = df_os['id'].str[-3:]

        df_os=df_os.drop(columns=['age', 'gender'])

        df = pd.merge(df_od, df_os, on='id', how='inner')
        df['id'] = df['id'].astype(int)

        df_osfundusImages = pd.DataFrame(data=osfundusImages, columns=['id'\
                                                                        , 'os_eye_fundus_image_fileName'\
                                                                        , 'os_eye_fundus_image_format'\
                                                                        , 'os_eye_fundus_image_width'\
                                                                        , 'os_eye_fundus_image_height'\
                                                                        , 'os_eye_fundus_image_path'])
        df_odfundusImages = pd.DataFrame(data=odfundusImages, columns=['id'\
                                                                        , 'od_eye_fundus_image_fileName'\
                                                                        , 'od_eye_fundus_image_format'\
                                                                        , 'od_eye_fundus_image_width'\
                                                                        , 'od_eye_fundus_image_height'\
                                                                        , 'od_eye_fundus_image_path'])

        df_fundusImages = pd.merge(df_osfundusImages, df_odfundusImages, on = 'id', how='inner')

        df_osimagesWithContours = pd.DataFrame(data=osimagesWithContours, columns=['id'\
                                                                        , 'os_eye_image_with_contour_fileName'\
                                                                        , 'os_eye_image_with_contour_format'\
                                                                        , 'os_eye_image_with_contour_width'\
                                                                        , 'os_eye_image_with_contour_height'\
                                                                        , 'os_eye_image_with_contour_path'])
        df_odimagesWithContours = pd.DataFrame(data=odimagesWithContours, columns=['id'\
                                                                        , 'od_eye_image_with_contour_fileName'\
                                                                        , 'od_eye_image_with_contour_format'\
                                                                        , 'od_eye_image_with_contour_width'\
                                                                        , 'od_eye_image_with_contour_height'\
                                                                        , 'od_eye_image_with_contour_path'])

        df_odimagesWithContours = pd.merge(df_osimagesWithContours, df_odimagesWithContours, on = 'id', how='inner')

        df_images = pd.merge(df_fundusImages, df_odimagesWithContours, on = 'id', how='inner')

        self._datasource = pd.merge(df, df_images)
        self._datasource['os_eye_fundus_image_fileName'] = self._datasource['os_eye_fundus_image_fileName'].astype('string')
        self._datasource['os_eye_fundus_image_path'] = self._datasource['os_eye_fundus_image_path'].astype('string')
        self._datasource['od_eye_fundus_image_fileName'] = self._datasource['od_eye_fundus_image_fileName'].astype('string')
        self._datasource['od_eye_fundus_image_path'] = self._datasource['od_eye_fundus_image_path'].astype('string')
        self._datasource['os_eye_image_with_contour_fileName'] = self._datasource['os_eye_image_with_contour_fileName'].astype('string')
        self._datasource['os_eye_image_with_contour_path'] = self._datasource['os_eye_image_with_contour_path'].astype('string')
        self._datasource['od_eye_image_with_contour_fileName'] = self._datasource['od_eye_image_with_contour_fileName'].astype('string')
        self._datasource['od_eye_image_with_contour_path'] = self._datasource['od_eye_image_with_contour_path'].astype('string')

        self._datasource.set_index('id')

    def insert(self, person: Person) -> int:
        id = int(self._datasource['id'].max()) + 1
        item = {
            'id' : id\
            , 'age': person.age\
            , 'gender': person.gender\
           , 'od_eye_diagnosis': person.odEye.diagnosis\
            , 'od_eye_refractive_defect_dioptre_1': person.odEye.refractiveDefect.dioptre1\
            , 'od_eye_refractive_defect_dioptre_2': person.odEye.refractiveDefect.dioptre2\
            , 'od_eye_refractive_defect_astigmatism': person.odEye.refractiveDefect.astigmatism\
            , 'od_eye_phakic_Pseudophakic': person.odEye.phakic_Pseudophakic\
            , 'od_eye_iop_pneumatic': person.odEye.iop.pneumatic\
            , 'od_eye_iop_perkins': person.odEye.iop.perkins\
            , 'od_eye_pachymetry': person.odEye.pachymetry\
            , 'od_eye_axialLength': person.odEye.axialLength\
            , 'od_eye_vf_md': person.odEye.vf_md\
            , 'os_eye_diagnosis': person.osEye.diagnosis\
            , 'os_eye_refractive_defect_dioptre_1': person.osEye.refractiveDefect.dioptre1\
            , 'os_eye_refractive_defect_dioptre_2': person.osEye.refractiveDefect.dioptre2\
            , 'os_eye_refractive_defect_astigmatism': person.osEye.refractiveDefect.astigmatism\
            , 'os_eye_phakic_Pseudophakic': person.osEye.phakic_Pseudophakic\
            , 'os_eye_iop_pneumatic': person.osEye.iop.pneumatic\
            , 'os_eye_iop_perkins': person.osEye.iop.perkins\
            , 'os_eye_pachymetry': person.osEye.pachymetry\
            , 'os_eye_axialLength': person.osEye.axialLength\
            , 'os_eye_vf_md': person.osEye.vf_md\
            , 'os_eye_fundus_image_fileName': person.osEye.fundusImage.fileName\
            , 'os_eye_fundus_image_format': person.osEye.fundusImage.format\
            , 'os_eye_fundus_image_width': person.osEye.fundusImage.width\
            , 'os_eye_fundus_image_height': person.osEye.fundusImage.height\
            , 'os_eye_fundus_image_path': person.osEye.fundusImage.path\
            , 'od_eye_fundus_image_fileName': person.odEye.fundusImage.fileName\
            , 'od_eye_fundus_image_format': person.odEye.fundusImage.format\
            , 'od_eye_fundus_image_width': person.odEye.fundusImage.width\
            , 'od_eye_fundus_image_height': person.odEye.fundusImage.height\
            , 'od_eye_fundus_image_path': person.odEye.fundusImage.path\
            , 'os_eye_image_with_contour_fileName': person.osEye.imageWithContour.fileName\
            , 'os_eye_image_with_contour_format': person.osEye.imageWithContour.format\
            , 'os_eye_image_with_contour_width': person.osEye.imageWithContour.width\
            , 'os_eye_image_with_contour_height': person.osEye.imageWithContour.height\
            , 'os_eye_image_with_contour_path': person.osEye.imageWithContour.path\
            , 'od_eye_image_with_contour_fileName': person.odEye.imageWithContour.fileName\
            , 'od_eye_image_with_contour_format': person.odEye.imageWithContour.format\
            , 'od_eye_image_with_contour_width': person.odEye.imageWithContour.width\
            , 'od_eye_image_with_contour_height': person.odEye.imageWithContour.height\
            , 'od_eye_image_with_contour_path': person.odEye.imageWithContour.path\
        }
        self._datasource = self._datasource.append(item, ignore_index=True)
        return id

    def select(self, top=None) -> pd.DataFrame:
        if (top is None):
            return self._datasource
        else:
            return self._datasource.head(top)

    def delete(self, index: str):
        self._datasource = self._datasource.drop(index)
        return self

    def get(self, index) -> Person:

        for i, samble in tqdm(self._datasource.iterrows()):
          if self._datasource.at[i, 'id'] == index:
            od_eye = Eye(self._datasource.at[i,'od_eye_diagnosis']\
                      , Refractive_Defect(self._datasource.at[i,'od_eye_refractive_defect_dioptre_1']\
                                          , self._datasource.at[i,'od_eye_refractive_defect_dioptre_2']\
                                          , self._datasource.at[i,'od_eye_refractive_defect_astigmatism'])\
                      , self._datasource.at[i,'od_eye_phakic_Pseudophakic']\
                      , IOP( self._datasource.at[i,'od_eye_iop_pneumatic']\
                              , self._datasource.at[i,'od_eye_iop_perkins'])
                      , self._datasource.at[i,'od_eye_pachymetry']\
                      , self._datasource.at[i,'od_eye_axialLength']\
                      , self._datasource.at[i,'od_eye_vf_md']
                      , EyeImage(self._datasource.at[i,'id']\
                                          , self._datasource.at[i,'od_eye_fundus_image_fileName']\
                                          , self._datasource.at[i,'od_eye_fundus_image_format']\
                                          , self._datasource.at[i,'od_eye_fundus_image_width']\
                                          , self._datasource.at[i,'od_eye_fundus_image_height']\
                                          , self._datasource.at[i,'od_eye_fundus_image_path'])\
                      , EyeImage(self._datasource.at[i,'id']\
                                          , self._datasource.at[i,'od_eye_image_with_contour_fileName']\
                                          , self._datasource.at[i,'od_eye_image_with_contour_format']\
                                          , self._datasource.at[i,'od_eye_image_with_contour_width']\
                                          , self._datasource.at[i,'od_eye_image_with_contour_height']\
                                          , self._datasource.at[i,'od_eye_image_with_contour_path']))
            os_eye = Eye(self._datasource.at[i,'os_eye_diagnosis']\
                      , Refractive_Defect(self._datasource.at[i,'os_eye_refractive_defect_dioptre_1']\
                                          , self._datasource.at[i,'os_eye_refractive_defect_dioptre_2']\
                                          , self._datasource.at[i,'os_eye_refractive_defect_astigmatism'])\
                      , self._datasource.at[i,'os_eye_phakic_Pseudophakic']\
                      , IOP( self._datasource.at[i,'os_eye_iop_pneumatic']\
                            , self._datasource.at[i,'os_eye_iop_perkins'])\
                      , self._datasource.at[i,'os_eye_pachymetry']\
                      , self._datasource.at[i,'os_eye_axialLength']\
                      , self._datasource.at[i,'os_eye_vf_md']\
                      , EyeImage(self._datasource.at[i,'id']\
                                          , self._datasource.at[i,'os_eye_fundus_image_fileName']\
                                          , self._datasource.at[i,'os_eye_fundus_image_format']\
                                          , self._datasource.at[i,'os_eye_fundus_image_width']\
                                          , self._datasource.at[i,'os_eye_fundus_image_height']\
                                          , self._datasource.at[i,'os_eye_fundus_image_path'])\
                      , EyeImage(self._datasource.at[i,'id']\
                                          , self._datasource.at[i,'os_eye_image_with_contour_fileName']\
                                          , self._datasource.at[i,'os_eye_image_with_contour_format']\
                                          , self._datasource.at[i,'os_eye_image_with_contour_width']\
                                          , self._datasource.at[i,'os_eye_image_with_contour_height']\
                                          , self._datasource.at[i,'os_eye_image_with_contour_path']))
            return Person(self._datasource.at[i,'id'], self._datasource.at[i,'age'], self._datasource.at[i,'gender'], od_eye, os_eye )
        return None


from PIL import Image
# PIL 패키지의 Image 모듈

img = Image.open('sample_image.jpg')
# open 함수 사용, 모드는 기본적으로 'r'(read)
print(img)
# mode=RGB, size=1984x1220 등 정보들이 기록된 객체가 출력된다

def show_image():
    input('Press enter to show image')
    print(img.format)
    # 이미지의 확장자
    print(img.size)
    # 사이즈 : (width, height) 형태의 튜플
    print(img.mode)
    # 이미지의 모드(RGB, RGBA 등)
    img.show()
    # show() 메소드를 통해 로컬의 기본 사진 앱을 통해 사진을 띄워준다

def crop_image():
    # 이미지 잘라내기
    input('Press enter to crop image')
    img_width, img_height = img.size
    # 튜플은 이렇게 예쁘게 리턴받을 수 있다
    box = (80, 500, img_width, img_height)
    # 좌측 상단을 (0, 0)으로 두고, 우측 하단 방향으로 값이 상승한다고 치면
    # start_x, start_y, end_x, end_y 순서
    img.crop(box).show()
    # crop된 새로운 이미지를 반환

def combine_image():
    input('Press enter to combine images')
    img_to_combine = Image.open('image_to_combine.png')
    # combine을 위한 이미지 open
    comb_width, comb_height = img_to_combine.size
    comb_box = (300, 300, comb_width + 300, comb_height + 300)
    # 여기도 start_x, start_y, end_x, end_y 순서
    img.paste(img_to_combine, comb_box)
    # paste 함수를 통해 결합, 반환은 없음(이미지 객체에 덮어씌움)
    img.show()

show_image()
crop_image()
combine_image()

000. External Packages/064. [pillow] Load, Crop, Combine/main.py


from langchain.llms import LlamaCpp
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler

from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, MessagesPlaceholder, HumanMessagePromptTemplate
from langchain.memory import ConversationBufferMemory
from typing import Optional
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.document_loaders import TextLoader

from fastapi import FastAPI
from langchain.chat_models import ChatOpenAI
from langchain.output_parsers.pydantic import PydanticOutputParser
from pydantic import BaseModel, Field, validator
from datetime import datetime
from typing import List
from langchain.document_loaders import DirectoryLoader
from langchain.text_splitter import CharacterTextSplitter

from translatepy import Translator

class PlaceOrder(BaseModel):
    skus: List[str] = Field(description="products, that user asked for")
    address: str = Field(description="delivery address")
    deliverytime : datetime = Field(description="delivery datetime")

parser = PydanticOutputParser(pydantic_object=PlaceOrder)

callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])

llm = LlamaCpp(
      model_path="llama-2-7b-chat.Q4_K_M.gguf",
      temperature=0.25,
      max_tokens=2000,
      top_p=1,
      callback_manager=callback_manager,
      verbose=True, # Verbose is required to pass to the callback manager
      n_ctx=1200
)
app = FastAPI()
redis_backed_dict = {}

prompt = ChatPromptTemplate(
    messages=[
        SystemMessagePromptTemplate.from_template(
            "You are a robot operator of Tinkoff Bank. You must answer customer questions only on Russian language. After the answer one question wait for the next question from the customer.",
        ),
        MessagesPlaceholder(variable_name="chat_history"),
        HumanMessagePromptTemplate.from_template("{question}")
    ],
#    partial_variables={
#        "menu": 'get_from_backend()',
#        "format_instructions": parser.get_format_instructions()
#        },
    output_language="Russian"
)

translator = Translator()

loader = DirectoryLoader('datas/', glob="*")
text_splitter = CharacterTextSplitter(
    separator = "\n\n",
    chunk_size = 500,
    chunk_overlap = 50,
    length_function = len,
    is_separator_regex = False,
)
loader = DirectoryLoader('datas/', glob="*")
all_text = loader.load()

chunks = text_splitter.split_documents(all_text)#[0].page_content+uisd[1].page_content
embeddings_model = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")

db = Chroma.from_documents(chunks, embeddings_model)#

@app.post("/message")
def message(user_id: str, message: str):
    memory = redis_backed_dict.get(
        user_id,
        ConversationBufferMemory(memory_key="chat_history", return_messages=True)
    )
    conversation = LLMChain(
        llm=llm,
        prompt=prompt,
        memory=memory
    )
    question_en = translator.translate(message, "English").result
    docs = db.similarity_search(question_en)
    context = 'Для ответа на вопрос используй эту информацию:'
    for frag in docs[:5]:
        context += frag.page_content

    ai_message = conversation.run({"question": question_en+context})
    return {"message": translator.translate(ai_message, "Russian").result}


'''
Created on Apr 28, 2017

@author: Owen
'''

import cv2

camera = cv2.VideoCapture(0)
retval, image = camera.read()

# Create the haar cascade
faceCascade = cv2.CascadeClassifier('C:/Users/Owen/Documents/GitHub/opencv/data/haarcascades/haarcascade_frontalface_default.xml')

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# Detect faces in the image
faces = faceCascade.detectMultiScale(gray, 1.3, 5)

print("Found {0} faces!".format(len(faces)))

# Draw a rectangle around the faces
for (x, y, w, h) in faces:
    cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)

cv2.imshow("Faces found" ,image)
cv2.waitKey(0)


import pytest
from pyscfad import gto

@pytest.fixture(scope="module")
def get_mol():
    mol = gto.Mole()
    mol.atom = 'H 0. 0. 0.; F 0. 0. 1.1'
    mol.basis = '631g'
    mol.verbose = 0
    mol.incore_anyway = True
    mol.build(trace_exp=False, trace_ctr_coeff=False)
    return mol


import pandas as pd
import os
import dataframe_image as dfi

# input
coins = ['ARB', 'ETH'] # , 'GMX', 'MCB', 'STG'
sortby_vars = ['closetime_APR', 'pnl_closetime_usd_value']

# set paths
data_dir = '../data'
png_dir = '../png'
os.makedirs(png_dir, exist_ok=True)

for sortby_var in sortby_vars:
    for coin in coins:
        # read data
        lst = []
        for fname in os.listdir(data_dir):
            if fname.endswith(f'{coin}-pairs.csv') and fname.startswith('cleaned-arbi-univ3lp'):
                fpath = os.path.join(data_dir, fname)
                lst.append(
                    pd.read_csv(fpath,
                                parse_dates=['open_datetime', 'close_datetime'],
                                infer_datetime_format=True)
                    )
        df = pd.concat(lst).sort_values(sortby_var, ascending=False)

        # find top 10 positions by closetime APR with >=$10 PnL at closing time
        # cond = (df.columns.str.startswith('sent_') |
        #         df.columns.str.startswith('withdrawn_'))
        # cols = df.columns[cond].to_list()
        cols = ['investment_opentime_usd_value',
                'duration_days',
                'closetime_APR',
                'current_APR',
                'investment_closetime_usd_value',
                'investment_current_usd_value',
                'pnl_closetime_usd_value',
                'pnl_current_usd_value',]
        df_top10 = df.loc[df['pnl_closetime_usd_value']>=10, cols]\
            .head(10).reset_index(drop=True)

        # print out
        format_dict = {
            'investment_opentime_usd_value': '${:,.0f}',
            'duration_days': '{:,.3f}',
            'closetime_APR': '{:.0%}',
            'current_APR': '{:.0%}',
            'investment_closetime_usd_value': '${:,.0f}',
            'investment_current_usd_value': '${:,.0f}',
            'pnl_closetime_usd_value': '${:,.2f}',
            'pnl_current_usd_value': '${:,.2f}',
            }
        df_styled = df_top10.style.format(format_dict)
        tmp_var = sortby_var.replace('_usd_value', '').replace('_', '-')
        fname = f'top10-by-{tmp_var}-{coin}-pairs.png'
        dfi.export(df_styled, os.path.join(png_dir, fname))


import random
import re
import statistics
from graph import PercentPerGeneration
import sys
import argparse
import os

# Constants
ALPHABET = "abcdefghijklmnopqrstuvwxyz"
CIPHER_TEXT_FILE = "enc.txt"
PLAIN_TEXT_FILE = "plain.txt"
PERMUTATION_FILE = "perm.txt"
COMMON_WORD = "dict.txt"
FREQ_FILE = "Letter_Freq.txt"

# Genetic Algorithm Parameters
POPULATION_SIZE = 100
MUTATION_RATE = 0.05
MAX_GENERATIONS = 2000
# How much to take from the best solutions
SELECTION_BIASED = POPULATION_SIZE * 0.5
SELECTION_BIASED_PART2 = POPULATION_SIZE * 0.1
# How much to take from the best solutions for crossover
SELECTION_BIASED_CROSSOVER = POPULATION_SIZE * 0.8
CHANGE_LAMARCK = 0.5
COUNTER_CONVERGENCE = 200

def process_arguments():
    parser = argparse.ArgumentParser(description='Example script with command-line arguments')
    parser.add_argument('-a', '--algorithem', type=str,
                        help='algorithem to run, by defult run the main algorithem, enter 2 for darwin 3 for lamark')
    parser.add_argument('-e', '--enc', type=str, help='path to ciphper text')
    parser.add_argument('-f', '--freq', type=str, help='path to Letter_Freq')
    parser.add_argument('-c', '--common', type=str, help='path to common word text')
    parser.add_argument('-r', '--result', type=str, help='path to translatation text')
    parser.add_argument('-d', '--dict', type=str, help='path to translatation dict')
    args = parser.parse_args()
    return args

# return dict key=letter, val=num apper
def statistic_letter(text):
    conter = 0
    letter_count = {}
    for letter in text:
        if letter.isalpha():
            letter = letter.lower()
            letter_count[letter] = letter_count.get(letter, 0) + 1
            conter += 1
    for key, value in letter_count.items():
        letter_count[key] = value / conter
    return (letter_count)

def file_to_arr(file_path):
    data_dict = set()
    with open(file_path, "r") as file:
        for line in file:
            data_dict.add(line.strip())
    return data_dict

# for each line the first word is key the secound value
def file_to_dict(file_path):
    data_dict = {}
    with open(file_path, "r") as file:
        for line in file:
            words = line.lower().strip().split()
            if len(words) >= 2:
                key = words[1]
                value = words[0]
                data_dict[key] = float(value)
    return data_dict

def remove_special_characters(input_string):
    # Regular expression pattern to match special characters
    pattern = r'[^a-zA-Z0-9\s]'  # Matches any character that is not a letter, digit, or whitespace
    output_string = re.sub(pattern, '', input_string)
    return output_string

# delete duplicates and check that all the letters are found, if not add them at the end in random order
def missing_letters(letters):
    missing_letters = []
    combined_list_set = list(set(letters))
    # Sort the combined list
    combined_list_set.sort()
    # Check if all letters are present
    if len(combined_list_set) != 26:
        # Create a list of missing letters
        missing_letters = [chr(i) for i in range(ord('a'), ord('z') + 1) if chr(i) not in combined_list_set]
    return missing_letters

# Generate initial population
def generate_population():
    population = []
    for _ in range(POPULATION_SIZE):
        permutation = list(ALPHABET)
        random.shuffle(permutation)
        population.append(permutation)
    return population

# Calculate fitness score
def calculate_fitness(permutation):
    decrypted_text = cipher_text.translate(str.maketrans(ALPHABET, "".join(permutation)))
    decrypted_text_arr = remove_special_characters(decrypted_text).lower().strip().split()
    # Checks how many common words appear in the decoded text
    set1 = set(decrypted_text_arr)
    set2 = set(common_word)
    intersection = set1.intersection(set2)
    res_word = len(intersection)
    res_letter = calculate_statisc_letter(permutation)
    return res_word * 3 + res_letter * 0.0001

def calculate_statisc_letter(permutation):
    res = 0
    for key, value in sulotionFreq.items():
        letter = permutation[ALPHABET.index(key)]
        try:
            add = 1 / abs(value - real_freq[letter])
            res += 100000000 if add > 100000000 else add
        except:
            res += 100000000
    return res

# Perform single-point crossover
def crossover(parent1, parent2):
    crossover_point = random.randint(0, 26)
    child = fix_cross(parent1[:crossover_point], parent2[crossover_point:])
    return child

def fix_cross(a, b):
    combined_list = list(set(a + b))
    missing_letter = missing_letters(combined_list)

    # Add missing letters randomly to the combined list
    random.shuffle(missing_letter)
    combined_list += missing_letter

    # Return the combination
    return combined_list

# Perform mutation
def mutate(permutation):
    for i in range(int(len(ALPHABET) * MUTATION_RATE)):
        j = random.randint(0, len(ALPHABET) - 1)
        z = random.randint(0, len(ALPHABET) - 1)
        permutation[z], permutation[j] = permutation[j], permutation[z]
    return permutation

def sort_by_grade(objects, grades):
    def myFunc(x):
        return x[1]

    # Combine objects and grades into pairs
    pairs = zip(objects, grades)

    # Sort the pairs based on the grade value
    sorted_pairs = sorted(pairs, key=lambda x: x[1], reverse=True)

    # Create a new array with pairs of object and grade
    result = [(obj, grade) for obj, grade in sorted_pairs]
    return result

# Main genetic algorithm loop
def genetic_algorithm():
    best_results = []
    worst_results = []
    average_results = []
    population = generate_population()
    generations = 0
    best_individual = None
    best_ans_for_generations = ""
    counter_convergence = COUNTER_CONVERGENCE
    while generations < MAX_GENERATIONS and counter_convergence:

        # Calculate fitness for each individual
        fitness_scores = [calculate_fitness(individual) for individual in population]
        # Find best individual
        best_individual = sort_by_grade(population, fitness_scores)
        x = best_individual

        final_ans = best_individual[0]

        print(final_ans)
        print("generations: ", generations)
        print("counter_convergence: ", counter_convergence)

        best_results.append(best_individual[0][1])
        worst_results.append(best_individual[-1][1])
        average_results.append(statistics.mean(fitness_scores))

        if best_ans_for_generations != "" and best_ans_for_generations == final_ans[1]:
            counter_convergence -= 1
        else:
            counter_convergence = COUNTER_CONVERGENCE

        best_ans_for_generations = final_ans[1]
        # Create new population through selection, crossover, and mutation
        new_population = create_population_part1(best_individual)

        population = new_population.copy()
        generations += 1

    writeRes(final_ans[0])
    graph_results(best_results, worst_results, average_results)
    return generations

def create_population_part1(best_individual):
    new_population = []
    size = 10
    size2 = 20
    for i in range(size):
        new_population.append(best_individual[0][0].copy())

    for i in range(size):
        new_population.append(mutate(best_individual[0][0].copy()))
    population = int(SELECTION_BIASED)
    if population > (POPULATION_SIZE - size) - size2:
        population = (POPULATION_SIZE - size) - size2
    new_population.extend([mutate(best[0]) for best in best_individual[:population]])

    for_crossover = best_individual[:int(SELECTION_BIASED_CROSSOVER)]
    while len(new_population) < POPULATION_SIZE:
        parent1 = random.choice(for_crossover)[0]
        parent2 = random.choice(for_crossover)[0]
        child = crossover(parent1, parent2)
        if len(child) != 26:
            print("error")
        child = mutate(child)
        if len(child) != 26:
            print("error 2")
        new_population.append(child)
    # new_population.append(list('yxintozjcebldukmsvpqrhwgaf'))
    return new_population

def lamarck_algorithm():
    # for graph - each generation information
    best_results = []
    worst_results = []
    average_results = []
    population = generate_population()  # generate random population
    generations = 0
    best_individual = None
    best_ans_for_generations = ""
    counter_convergence = COUNTER_CONVERGENCE
    while generations < MAX_GENERATIONS and counter_convergence:
        # Create new population through selection, crossover, and mutation
        [mutate(individual) for individual in population]

        if best_ans_for_generations != "":
            add = best_individual[0][0].copy()
            population[-1] = add

        # Calculate fitness for each individual
        fitness_scores = [calculate_fitness(individual) for individual in population]
        # Find best individual
        best_individual = sort_by_grade(population, fitness_scores)

        final_ans = best_individual[0]
        print(final_ans)
        print("generations: ", generations)
        print("counter_convergence: ", counter_convergence)
        # information for graph
        best_results.append((best_individual[0][1]))
        worst_results.append((best_individual[-1][1]))
        average_results.append(statistics.mean(fitness_scores))

        if best_ans_for_generations != "" and best_ans_for_generations == final_ans[1]:
            counter_convergence -= 1
        else:
            counter_convergence = 200

        best_ans_for_generations = final_ans[1]
        # Create new population through selection, crossover, and mutation
        new_population = create_population_lamarck(best_individual)

        population = new_population
        generations += 1
    PercentPerGeneration(best_results, worst_results, average_results)
    writeRes(final_ans[0])
    return generations

def darwin_algorithm():
    best_results = []
    worst_results = []
    average_results = []
    population = generate_population()
    generations = 0
    best_individual = None
    best_ans_for_generations = ""
    counter_convergence = COUNTER_CONVERGENCE
    while generations < MAX_GENERATIONS and counter_convergence:
        fitness_scores = [calculate_fitness(individual) for individual in population]

        results = beforAndAfterMutate(population)  # make mutation and calculate fitness
        # best_individual = sort_by_grade(population, fitness_scores)
        best_individual = (sorted(results.items(), key=lambda item: item[1], reverse=True))
        final_ans = best_individual[0]
        print(final_ans)
        print("generations: ", generations)
        print("counter_convergence: ", counter_convergence)
        # information for graph
        best_results.append((best_individual[0][1]))
        worst_results.append((best_individual[-1][1]))
        average_results.append(statistics.mean(fitness_scores))

        if best_ans_for_generations != "" and best_ans_for_generations == final_ans[1]:
            counter_convergence -= 1
        else:
            counter_convergence = 200
        best_ans_for_generations = final_ans[1]
        # Create new population through selection, crossover, and mutation
        new_population = create_population_darwin(best_individual).copy()
        population = new_population.copy()
        generations += 1
    PercentPerGeneration(best_results, worst_results, average_results)
    writeRes(final_ans[0])
    return generations

def writeRes(final_ans):
    decrypted_text = cipher_text.translate(str.maketrans(ALPHABET, "".join(final_ans)))
    with open(PLAIN_TEXT_FILE, "w") as file:
        file.write(decrypted_text)
    with open(PERMUTATION_FILE, "w") as file:
        text = ""
        for i in range(26):
            text += ALPHABET[i] + final_ans[i] + "\n"
        file.write(text)

# create new population with lamarck - duplicate the best individuals
def create_population_lamarck(best_individual):
    new_population = []
    count = int(SELECTION_BIASED_PART2)
    top_best_individual = best_individual[:10].copy()
    for best in top_best_individual:
        if len(new_population) >= POPULATION_SIZE:
            break
        for i in range(count):
            new_population.append(best[0].copy())
        count -= 1
    for_rest = best_individual
    # the rest of the population -> do crossover
    crossover_population(new_population, for_rest, "lamarck")
    return new_population

def create_population_darwin(best_individual):
    new_population = []
    count = int(SELECTION_BIASED_PART2)
    top_best_individual = best_individual[:10].copy()
    new_population.append(list(best_individual[0][0]).copy())
    for best in top_best_individual:
        for i in range(count):
            new_copy = mutate(list(best[0]))
            new_population.append(new_copy)
        count -= 1
    # creat new population
    new_group = new_population.copy()
    # add random individual
    size_left = POPULATION_SIZE - len(new_population)
    for i in range(size_left):
        new_group.append(mutate(list(random.choice(best_individual)[0])).copy())
    # new_group = crossover_population(new_group, best_individual, "darwin")
    return new_group

# take population and return new population after crossover
def crossover_population(dest_population, source_population, mode):
    if mode == "lamarck":
        # do this until we get the same size of population
        while len(dest_population) < POPULATION_SIZE:
            parent1 = random.choice(source_population)[0].copy()
            parent2 = random.choice(source_population)[0].copy()
            child = crossover(parent1, parent2)
            if len(child) != 26:
                print("error")
            child = mutate(child)
            if len(child) != 26:
                print("error 2")
            dest_population.append(child)
        return dest_population
    elif mode == "darwin":
        # do this until we get the same size of population
        while len(dest_population) < POPULATION_SIZE:
            parent1 = list(random.choice(source_population)[0])
            parent2 = list(random.choice(source_population)[0])
            child = mutate(crossover(parent1, parent2))
            if len(child) != 26:
                print("error")
            dest_population.append(child)
        return dest_population

# dict of population before And After Mutate, key=before, value=After
def beforAndAfterMutate(population):
    dict_map = {}
    for people in population:
        dict_map["".join(people)] = calculate_fitness(mutate(list(people)))
    return dict_map

def graph_results(best_results, worst_results, average_results):
    PercentPerGeneration(best_results, worst_results, average_results)
    return

if __name__ == "__main__":
    run = 1
    args = process_arguments()
    if args.algorithem:
        run = int(args.algorithem)
    if args.enc:
        CIPHER_TEXT_FILE = args.enc
    if args.freq:
        FREQ_FILE = args.freq
    if args.common:
        COMMON_WORD = args.common
    if args.result:
        PERMUTATION_FILE = args.result
    if args.dict:
        PLAIN_TEXT_FILE = args.dict

    with open(CIPHER_TEXT_FILE, "r") as file:
        cipher_text = file.read().lower()
    sum_enc_word = len((cipher_text.strip().split()))
    real_freq = file_to_dict(FREQ_FILE)
    # A sorted array of the most common letters according to the common letters file
    common_word = file_to_arr(COMMON_WORD)

    sulotionFreq = statistic_letter(cipher_text.strip())
    if (run == 2):
        total_steps = darwin_algorithm()
    elif (run == 3):
        total_steps = lamarck_algorithm()
    else:
        total_steps = genetic_algorithm()
    print(f"Total Steps: {str(total_steps)}")


import torch
import gym
from torch import nn
from typing import Union, Dict, List, Any
import ray
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
from ray.rllib.models.modelv2 import ModelV2, restore_original_dimensions
from ray.rllib.utils.typing import ModelConfigDict, TensorType
from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC

from models.depth_vae import DepthVAE

# from models.rgbd_resnet18_vae import VAE

DEFAULTS = {"z_dim": 64, "depth_weight": 1, "rgb_weight": 1, "elbo_beta": 1}

class DepthRayVAE(TorchModelV2, DepthVAE):
    def __init__(
        self,
        obs_space: gym.spaces.Space,
        action_space: gym.spaces.Space,
        num_outputs: int,
        model_config: ModelConfigDict,
        name: str,
        **custom_model_kwargs,
    ):
        super().__init__(obs_space, action_space, num_outputs, model_config, name)
        self.cfg = dict(DEFAULTS, **custom_model_kwargs)
        # VAE.__init__(self, z_dim=self.cfg["z_dim"], nc=4)
        DepthVAE.__init__(self, z_dim=self.cfg["z_dim"])
        self.rgb_loss_fn = nn.MSELoss(reduction="mean")
        self.depth_loss_fn = nn.MSELoss(reduction="mean")
        self.act_space = gym.spaces.utils.flatdim(action_space)
        self.visdom_imgs: Dict[str, Any] = {}

    def forward(self, input_dict, state, seq_lens):
        """Compute autoencoded image. Note the returned "logits"
        are random, as we want random actions"""
        self.curr_ae_input = self.to_img(input_dict)
        # if torch.any(self.curr_ae_input > 0):
        #        ray.util.pdb.set_trace()
        self.curr_ae_output, self.curr_mu, self.curr_logvar = DepthVAE.forward(
            self, self.curr_ae_input
        )
        batch = input_dict["obs_flat"].shape[0]
        device = input_dict["obs_flat"].device
        self.device = device
        self._curr_value = torch.zeros((batch,), device=device)
        out = torch.zeros((batch, self.act_space), device=device)

        self.visdom_imgs.clear()
        di = self.curr_ae_input[:64].cpu().detach()
        do = self.curr_ae_output[:64].cpu().detach()
        self.visdom_imgs[f"depth_in-{self.cfg}"] = torch.cat(
            (di, di, di), dim=1
        ).numpy()
        self.visdom_imgs[f"depth_out-{self.cfg}"] = torch.cat(
            (do, do, do), dim=1
        ).numpy()
        # Return [batch, action_space]
        return out, state

    def value_function(self):
        return self._curr_value

    def to_img(self, input_dict):
        """Build obs into an image tensor for feeding to nn"""
        depth = input_dict["obs"]["depth"]
        # [batch, channel, cols, rows]
        # To B, dim, h, w
        depth = depth.permute(0, 3, 1, 2)
        """
        x = torch.zeros(
            (batch, channels, cols, rows), dtype=torch.float32, device=rgb.device
        )
        x[:, 0:semantic_tgt_channel] = semantic
        # x[:, semantic_channels:semantic_tgt_channel] = sem_tgt
        x[:, depth_channel] = torch.squeeze(depth)
        """
        return depth

    def custom_loss(
        self, policy_loss: List[torch.Tensor], loss_inputs
    ) -> List[torch.Tensor]:

        self.sem_loss = torch.tensor([0], device=self.curr_ae_output.device)

        self.depth_loss = self.cfg["depth_weight"] * self.depth_loss_fn(
            self.curr_ae_output[:, -1, :, :],
            self.curr_ae_input[:, -1, :, :],
        )
        self.recon_loss = self.sem_loss + self.depth_loss
        self.kld_loss = self.cfg["elbo_beta"] * DepthVAE.kld_loss(
            self, self.curr_mu, self.curr_logvar
        )
        self.combined_loss = self.recon_loss + self.kld_loss

        return [self.combined_loss]

    def metrics(self):
        return {
            "depth_loss": self.depth_loss.detach().item(),
            "semantic_loss": self.sem_loss.detach().item(),
            "combined_loss": self.combined_loss.detach().item(),
        }


import streamlit as st
import csv

# Sidebar Logo
logo_image = "logo.jpg"  # Ruta de la imagen del logotipo
st.sidebar.image(logo_image, use_column_width=True)

def enter_data(firstname, lastname, title, age, nationality, education, registration_status, numcourses, numsemesters):
    """
    Ingresa los datos del usuario.

    Args:
        firstname (str): Nombre del usuario.
        lastname (str): Apellidos del usuario.
        title (str): Título del usuario.
        age (int): Edad del usuario.
        nationality (str): Continente de origen del usuario.
        education (str): Nivel educativo del usuario.
        registration_status (str): Estado de registro del usuario.
        numcourses (int): Número de cursos completados por el usuario.
        numsemesters (int): Número de semestres completados por el usuario.
    """
    accepted = st.checkbox("Acepto los términos y condiciones")

    if accepted:
        if firstname and lastname:
            if st.button("Ingresar Datos"):
                # Guardar los datos en un archivo CSV
                data = [title, firstname, lastname, age, nationality, education, registration_status, numcourses, numsemesters]
                save_data(data)

                st.write("Datos guardados correctamente.")
                st.write("---------------------------------------------")
        else:
            st.warning("Nombre y Apellidos son requeridos.")
    else:
        st.warning("No ha aceptado los términos y condiciones")

def save_data(data):
    """
    Guarda los datos en un archivo CSV.

    Args:
        data (list): Lista con los datos a guardar.
    """
    with open('registros.csv', mode='a', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(data)

# Sidebar Menu
st.sidebar.title("Menú")
selected_menu = st.sidebar.radio("Seleccionar opción", ["Información de usuario", "Solicitud de inscripción"])

# User Info Form
if selected_menu == "Información de usuario":
    st.subheader("Informacion de usuario")
    firstname = st.text_input("Nombre")
    lastname = st.text_input("Apellidos")
    title = st.selectbox("Titulo Académico", ["", "Sr.", "Srita.", "Sra."])
    age = st.number_input("Edad", min_value=18, max_value=110)
    nationality = st.selectbox("Continente", ["Africa", "Antartida", "Europa", "Norteamerica", "Sudamerica", "Oceania"])
    education = st.selectbox("Nivel educativo", ["Educación Superior", "Universitario", "Ciclo Formativo", "Máster Profesional"])
    comentario = st.text_input("Comentario Breve")

    # Validar la longitud del comentario
    if len(comentario) > 120:
        st.warning("El comentario no puede exceder los 120 caracteres.")

    # Course Info
    st.subheader("Informacion de cursos")
    registration_status = st.radio("Estado de registro", ["Registrado", "No registrado"], index=1)
    numcourses = st.number_input("# Cursos completados", min_value=0)
    numsemesters = st.number_input("# Semestres completados", min_value=0)

    # Button

    enter_data(firstname, lastname, title, age, nationality, education, registration_status, numcourses, numsemesters)

# Inscription Request Form
elif selected_menu == "Solicitud de inscripción":
    st.subheader("Solicitud de inscripción")
    # Agrega los campos y la lógica correspondiente para la solicitud de inscripción


# coding=utf-8

import json
import os

import datasets
import gdown

logger = datasets.logging.get_logger(__name__)

_CITATION = """\
@article{park2019cord,
  title={CORD: A Consolidated Receipt Dataset for Post-OCR Parsing},
  author={Park, Seunghyun and Shin, Seung and Lee, Bado and Lee, Junyeop and Surh, Jaeheung and Seo, Minjoon and Lee, Hwalsuk}
  booktitle={Document Intelligence Workshop at Neural Information Processing Systems}
  year={2019}
}
"""

_DESCRIPTION = """\
https://github.com/clovaai/cord
"""
_URL = "https://drive.google.com/uc?id=1MqhTbcj-AHXOqYoeoh12aRUwIprzTJYI"

def gdrive_downloader(url, path):
    gdown.download(url, path, quiet=False)

class CordConfig(datasets.BuilderConfig):
    """BuilderConfig for CORD"""

    def __init__(self, **kwargs):
        """BuilderConfig for CORD.
        Args:
          **kwargs: keyword arguments forwarded to super.
        """
        super(CordConfig, self).__init__(**kwargs)

class Cord(datasets.GeneratorBasedBuilder):
    """Conll2003 dataset."""

    BUILDER_CONFIGS = [
        CordConfig(name="cord", version=datasets.Version(
            "1.0.0"), description="FUNSD dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "bboxes": datasets.Sequence(datasets.Sequence(datasets.Value("int64"))),
                    "roi": datasets.Sequence(datasets.Sequence(datasets.Value("int64"))),
                    "ner_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=['menu.cnt',
                                   'menu.discountprice',
                                   'menu.etc',
                                   'menu.itemsubtotal',
                                   'menu.nm',
                                   'menu.num',
                                   'menu.price',
                                   'menu.sub_cnt',
                                   'menu.sub_etc',
                                   'menu.sub_nm',
                                   'menu.sub_price',
                                   'menu.sub_unitprice',
                                   'menu.unitprice',
                                   'menu.vatyn',
                                   'sub_total.discount_price',
                                   'sub_total.etc',
                                   'sub_total.othersvc_price',
                                   'sub_total.service_price',
                                   'sub_total.subtotal_price',
                                   'sub_total.tax_price',
                                   'total.cashprice',
                                   'total.changeprice',
                                   'total.creditcardprice',
                                   'total.emoneyprice',
                                   'total.menuqty_cnt',
                                   'total.menutype_cnt',
                                   'total.total_etc',
                                   'total.total_price',
                                   'void_menu.nm',
                                   'void_menu.price']
                        )
                    ),
                    "image_path": datasets.Value("string"),
                }
            ),
            supervised_keys=None,
            homepage="https://github.com/clovaai/cord",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: datasets.DownloadManager):
        """Returns SplitGenerators."""
        url_or_urls = ['https://drive.google.com/uc?id=1MqhTbcj-AHXOqYoeoh12aRUwIprzTJYI',
                       'https://drive.google.com/uc?id=1wYdp5nC9LnHQZ2FcmOoC0eClyWvcuARU']

        downloaded_file = dl_manager.extract(
            dl_manager.download_custom(url_or_urls, gdrive_downloader))

        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/train"}
            ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/test"}
            ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/dev"}
            ),
        ]

    def _generate_examples(self, filepaths, mode):
        guid = -1
        for filepath in filepaths:
            filepath_folder  = filepath + mode
            logger.info("⏳ Generating examples from = %s", filepath_folder)
            ann_dir = os.path.join(filepath_folder, "json")
            if not os.path.exists(ann_dir):
                continue
            img_dir = os.path.join(filepath_folder, "image")
            for file in sorted(os.listdir(ann_dir)):
                guid +=1
                tokens = []
                bboxes = []
                ner_tags = []

                file_path = os.path.join(ann_dir, file)
                with open(file_path, "r", encoding="utf8") as f:
                    data = json.load(f)

                image_path = os.path.join(img_dir, file)
                image_path = image_path.replace("json", "png")

                if not os.path.exists(image_path):
                    other_dir_idx = int(not (filepaths.index(filepath)+2)%2)
                    image_path = image_path.replace(
                        filepath, filepaths[other_dir_idx])

                roi = data["roi"]
                if roi:
                    top_left = [roi["x1"], roi["y1"]]
                    bottom_right = [roi["x3"], roi["y3"]]
                    bottom_left = [roi["x4"], roi["y4"]]
                    top_right = [roi["x2"], roi["y2"]]
                    roi = [top_left, top_right, bottom_right, bottom_left]
                else:
                    roi = []

                for item in data["valid_line"]:
                    for word in item['words']:
                        # get word
                        txt = word['text']

                        # get bounding box
                        x1 = word['quad']['x1']
                        y1 = word['quad']['y1']
                        x3 = word['quad']['x3']
                        y3 = word['quad']['y3']

                        box = [x1, y1, x3, y3]

                        # ADDED
                        # skip empty word
                        if len(txt) < 1:
                            continue

                        tokens.append(txt)
                        bboxes.append(box)
                        ner_tags.append(item['category'])

                yield guid, {"id": str(guid), "tokens": tokens, "bboxes": bboxes, "ner_tags": ner_tags, "image_path": image_path, "roi":roi}


from typing import List

class Solution:
    def longestCommonPrefix(self, strs: List[str]) -> str:
        result = ""

        if len(strs) == 0:
            return result

        for i in range(len(strs[0])):
            for j in range(1, len(strs)):
                if i >= len(strs[j]) or strs[0][i] != strs[j][i]:
                    return result
            result += strs[0][i]

        return result

if __name__ == "__main__":
    strs = ["flower", "flow", "flight"]
    print(Solution().longestCommonPrefix(strs))


"""
LayerArtist classes handle the visualization of an individual subset
or dataset
"""
import logging

import numpy as np
from matplotlib.colors import Normalize
from matplotlib.cm import gray
from ..core.exceptions import IncompatibleAttribute
from ..core.util import color2rgb
from ..core.subset import Subset
from .util import view_cascade, get_extent, fast_limits

class ChangedTrigger(object):
    """Sets an instance's _changed attribute to True on update"""
    def __init__(self, default=None):
        self._default = default
        self._vals = {}

    def __get__(self, inst, type=None):
        return self._vals.get(inst, self._default)

    def __set__(self, inst, value):
        changed = value != self.__get__(inst)
        self._vals[inst] = value
        if changed:
            inst._changed = True

class LayerArtist(object):
    def __init__(self, layer, axes):
        """Create a new LayerArtist

        :param layer: Data or subset to draw
        :type layer: :class:`~glue.core.data.Data` or `glue.core.subset.Subset`
        """
        self.layer = layer
        self._axes = axes
        self._visible = True

        self._zorder = 0
        self.view = None
        self.artists = []

        self._changed = True
        self._state = None  # cache of subset state, if relevant

    def redraw(self):
        self._axes.figure.canvas.draw()

    @property
    def zorder(self):
        return self._zorder

    @zorder.setter
    def zorder(self, value):
        for artist in self.artists:
            artist.set_zorder(value)
        self._zorder = value

    @property
    def visible(self):
        return self._visible

    @visible.setter
    def visible(self, value):
        self._visible = value
        for a in self.artists:
            a.set_visible(value)

    @property
    def enabled(self):
        return len(self.artists) > 0

    def update(self, view=None):
        """Redraw this layer"""
        raise NotImplementedError()

    def clear(self):
        """Clear the visulaization for this layer"""
        for artist in self.artists:
            try:
                artist.remove()
            except ValueError:  # already removed
                pass
        self.artists = []

    def _check_subset_state_changed(self):
        """Checks to see if layer is a subset and, if so,
        if it has changed subset state. Sets _changed flag to True if so"""
        if not isinstance(self.layer, Subset):
            return
        state = self.layer.subset_state
        if state is not self._state:
            self._changed = True
            self._state = state

    def _sync_style(self):
        style = self.layer.style
        for artist in self.artists:
            artist.set_markeredgecolor('none')
            artist.set_markerfacecolor(style.color)
            artist.set_marker(style.marker)
            artist.set_markersize(style.markersize)
            artist.set_linestyle('None')
            artist.set_alpha(style.alpha)
            artist.set_zorder(self.zorder)
            artist.set_visible(self.visible and self.enabled)

    def __str__(self):
        return "%s for %s" % (self.__class__.__name__, self.layer.label)

    __repr__ = __str__

class ImageLayerArtist(LayerArtist):
    def __init__(self, layer, ax):
        super(ImageLayerArtist, self).__init__(layer, ax)
        self.norm = None
        self._cmap = gray

    @property
    def cmap(self):
        return self._cmap

    @cmap.setter
    def cmap(self, value):
        self._cmap = value
        for a in self.artists:
            a.set_cmap(value)

    def _default_norm(self, layer):
        vals = np.sort(layer.ravel())
        vals = vals[np.isfinite(vals)]
        result = InvNormalize()
        result.stretch = 'arcsinh'
        result.clip = True
        if vals.size > 0:
            result.vmin = vals[.01 * vals.size]
            result.vmax = vals[.99 * vals.size]
        return result

    def update(self, view):
        self.clear()
        views = view_cascade(self.layer, view)
        artists = []

        lr0 = self.layer[views[0]]
        self.norm = self.norm or self._default_norm(lr0)
        self.norm = self.norm or self._default_norm(lr0)
        self.norm.update_clip(self.layer, view[0])

        for v in views:
            image = self.layer[v]
            extent = get_extent(v)
            artists.append(self._axes.imshow(image, cmap=self.cmap,
                                             norm=self.norm,
                                             interpolation='nearest',
                                             origin='lower',
                                             extent=extent, zorder=0))
        self.artists = artists
        self._sync_style()

    def set_norm(self, vmin=None, vmax=None,
                 bias=None, contrast=None, stretch=None, norm=None,
                 clip_lo=None, clip_hi=None):
        if norm is not None:
            self.norm = norm
            return norm
        if self.norm is None:
            self.norm = InvNormalize()
        if vmin is not None:
            self.norm.vmin = vmin
        if vmax is not None:
            self.norm.vmax = vmax
        if bias is not None:
            self.norm.bias = bias
        if contrast is not None:
            self.norm.contrast = contrast
        if clip_lo is not None:
            self.norm.clip_lo = clip_lo
        if clip_hi is not None:
            self.norm.clip_hi = clip_hi
        if stretch is not None:
            if stretch not in ['linear', 'sqrt', 'arcsinh']:
                raise TypeError(
                    "Stretch must be one of 'linear', 'sqrt', or 'arcsinh'")
            self.norm.stretch = stretch
        return self.norm

    def clear_norm(self):
        self.norm = None

    def _sync_style(self):
        for artist in self.artists:
            artist.set_zorder(self.zorder)
            artist.set_visible(self.visible and self.enabled)

class RGBImageLayerArtist(ImageLayerArtist):
    def __init__(self, layer, ax):
        super(RGBImageLayerArtist, self).__init__(layer, ax)
        self.r = None
        self.g = None
        self.b = None
        self.rnorm = None
        self.gnorm = None
        self.bnorm = None
        self.contrast_layer = 'green'
        self.layer_visible = dict(red=True, green=True, blue=True)

    def set_norm(self, *args, **kwargs):
        spr = super(RGBImageLayerArtist, self).set_norm
        if self.contrast_layer == 'red':
            self.norm = self.rnorm
            self.rnorm = spr(*args, **kwargs)
        if self.contrast_layer == 'green':
            self.norm = self.gnorm
            self.gnorm = spr(*args, **kwargs)
        if self.contrast_layer == 'blue':
            self.norm = self.bnorm
            self.bnorm = spr(*args, **kwargs)

    def update(self, view=None):
        self.clear()
        if self.r is None or self.g is None or self.b is None:
            return

        if view is None:
            view = self._last_view

        if view is None:
            return
        self._last_view = view

        views = view_cascade(self.layer, view)
        artists = []
        for v in views:
            extent = get_extent(v)
            # first argument = component. swap
            r = tuple([self.r] + list(v[1:]))
            g = tuple([self.g] + list(v[1:]))
            b = tuple([self.b] + list(v[1:]))
            r = self.layer[r]
            g = self.layer[g]
            b = self.layer[b]
            self.rnorm = self.rnorm or self._default_norm(r)
            self.gnorm = self.gnorm or self._default_norm(g)
            self.bnorm = self.bnorm or self._default_norm(b)
            if v is views[0]:
                self.rnorm.update_clip(self.layer, self.r)
                self.gnorm.update_clip(self.layer, self.g)
                self.bnorm.update_clip(self.layer, self.b)

            image = np.dstack((self.rnorm(r),
                               self.gnorm(g),
                               self.bnorm(b)))

            if not self.layer_visible['red']:
                image[:, :, 0] *= 0
            if not self.layer_visible['green']:
                image[:, :, 1] *= 0
            if not self.layer_visible['blue']:
                image[:, :, 2] *= 0

            artists.append(self._axes.imshow(image,
                                             interpolation='nearest',
                                             origin='lower',
                                             extent=extent, zorder=0))
        self.artists = artists
        self._sync_style()

class SubsetImageLayerArtist(LayerArtist):

    def update(self, view):
        subset = self.layer
        self.clear()
        logging.debug("View into subset %s is %s", self.layer, view)

        try:
            mask = subset.to_mask(view[1:])
        except IncompatibleAttribute:
            return
        logging.debug("View mask has shape %s", mask.shape)

        #shortcut for empty subsets
        if not mask.any():
            return

        extent = get_extent(view)
        r, g, b = color2rgb(self.layer.style.color)
        mask = np.dstack((r * mask, g * mask, b * mask, mask * .5))
        mask = (255 * mask).astype(np.uint8)
        self.artists = [self._axes.imshow(mask, extent=extent,
                                          interpolation='nearest',
                                          origin='lower',
                                          zorder=5, visible=self.visible)]

class ScatterLayerArtist(LayerArtist):
    xatt = ChangedTrigger()
    yatt = ChangedTrigger()

    def __init__(self, layer, ax):
        super(ScatterLayerArtist, self).__init__(layer, ax)
        self.emphasis = None

    def _recalc(self):
        self.clear()
        assert len(self.artists) == 0

        try:
            x = self.layer[self.xatt].ravel()
            y = self.layer[self.yatt].ravel()
        except IncompatibleAttribute:
            return False
        self.artists = self._axes.plot(x, y)
        return True

    def update(self, view=None):
        self._check_subset_state_changed()
        if self._changed:
            if not self._recalc():
                return
            self._changed = False

        has_emph = False
        if self.emphasis is not None:
            try:
                s = Subset(self.layer.data)
                s.subset_state = self.emphasis & self.layer.subset_state
                x = s[self.xatt].ravel()
                y = s[self.yatt].ravel()
                self.artists.extend(self._axes.plot(x, y))
                has_emph = True
            except IncompatibleAttribute:
                pass

        self._sync_style()
        if has_emph:
            self.artists[-1].set_mec('green')
            self.artists[-1].set_mew(2)
            self.artists[-1].set_alpha(1)

    def get_data(self):
        try:
            return self.layer[self.xatt].ravel(), self.layer[self.yatt].ravel()
        except IncompatibleAttribute:
            return np.array([]), np.array([])

class LayerArtistContainer(object):
    """A collection of LayerArtists"""
    def __init__(self):
        self.artists = []

    def _duplicate(self, artist):
        for a in self.artists:
            if type(a) == type(artist) and a.layer is artist.layer:
                return True
        return False

    def _check_duplicate(self, artist):
        """Raise an error if this artist is a duplicate"""
        if self._duplicate(artist):
            raise ValueError("Already have an artist for this type "
                             "and data")

    def append(self, artist):
        """Add a LayerArtist to this collection"""
        self._check_duplicate(artist)
        self.artists.append(artist)
        artist.zorder = max(a.zorder for a in self.artists) + 1

    def remove(self, artist):
        """Remove a LayerArtist from this collection"""
        try:
            self.artists.remove(artist)
            artist.clear()
        except ValueError:
            pass

    def pop(self, layer):
        """Remove all artists associated with a layer"""
        to_remove = [a for a in self.artists if a.layer is layer]
        for r in to_remove:
            self.remove(r)
        return to_remove

    @property
    def layers(self):
        """A list of the unique layers in the container"""
        return list(set([a.layer for a in self.artists]))

    def __len__(self):
        return len(self.artists)

    def __iter__(self):
        return iter(self.artists)

    def __contains__(self, item):
        if isinstance(item, LayerArtist):
            return item in self.artists
        return any(item is a.layer for a in self.artists)

    def __getitem__(self, layer):
        if isinstance(layer, int):
            return self.artists[layer]
        return [a for a in self.artists if a.layer is layer]

class HistogramLayerArtist(LayerArtist):
    lo = ChangedTrigger(0)
    hi = ChangedTrigger(1)
    nbins = ChangedTrigger(10)
    xlog = ChangedTrigger(False)

    def __init__(self, layer, axes):
        super(HistogramLayerArtist, self).__init__(layer, axes)
        self.ylog = False
        self.cumulative = False
        self.normed = False
        self.y = np.array([])
        self.x = np.array([])
        self._y = np.array([])

        self._scale_state = None

    def has_patches(self):
        return len(self.artists) > 0

    def get_data(self):
        return self.x, self.y

    def clear(self):
        super(HistogramLayerArtist, self).clear()
        self.x = np.array([])
        self.y = np.array([])
        self._y = np.array([])

    def _calculate_histogram(self):
        """Recalculate the histogram, creating new patches"""
        self.clear()
        try:
            data = self.layer[self.att].ravel()
        except IncompatibleAttribute:
            return False

        if data.size == 0:
            return

        if self.lo > np.nanmax(data) or self.hi < np.nanmin(data):
            return
        if self.xlog:
            data = np.log10(data)
            rng = [np.log10(self.lo), np.log10(self.hi)]
        else:
            rng = self.lo, self.hi
        nbinpatch = self._axes.hist(data,
                                    bins=self.nbins,
                                    range=rng)
        self._y, self.x, self.artists = nbinpatch
        return True

    def _scale_histogram(self):
        """Modify height of bins to match ylog, cumulative, and norm"""
        if self.x.size == 0:
            return

        y = self._y.astype(np.float)
        dx = self.x[1] - self.x[0]
        if self.normed:
            div = y.sum() * dx
            if div == 0:
                div = 1
            y /= div
        if self.cumulative:
            y = y.cumsum()
            y /= y.max()

        self.y = y
        bottom = 0 if not self.ylog else 1e-100

        for a, y in zip(self.artists, y):
            a.set_height(y)
            x, y = a.get_xy()
            a.set_xy((x, bottom))

    def _check_scale_histogram(self):
        """
        If needed, rescale histogram to match cumulative/log/normed state.
        """
        state = (self.normed, self.ylog, self.cumulative)
        if state == self._scale_state:
            return
        self._scale_state = state
        self._scale_histogram()

    def update(self, view=None):
        """Sync plot.

        The _change flag tracks whether the histogram needs to be
        recalculated. If not, the properties of the existing
        artists are updated
        """
        self._check_subset_state_changed()
        if self._changed:
            if not self._calculate_histogram():
                return
            self._changed = False
            self._scale_state = None
        self._check_scale_histogram()
        self._sync_style()

    def _sync_style(self):
        """Update visual properties"""
        style = self.layer.style
        for artist in self.artists:
            artist.set_facecolor(style.color)
            artist.set_alpha(style.alpha)
            artist.set_zorder(self.zorder)
            artist.set_visible(self.visible and self.enabled)

class InvNormalize(Normalize):
    """ Simple wrapper to matplotlib Normalize object, that
    handles the case where vmax <= vmin """
    def __init__(self):
        Normalize.__init__(self)
        self.stretch = 'linear'
        self.bias = 0.5
        self.contrast = 0.5
        self.clip_lo = 5.
        self.clip_hi = 95.

    def update_clip(self, data, component):
        vmin, vmax = fast_limits(data, component, self.clip_lo, self.clip_hi)
        self.vmin = vmin
        self.vmax = vmax

    def __call__(self, value):
        self.autoscale_None(value)  # set vmin, vmax if unset
        inverted = self.vmax <= self.vmin

        hi, lo = max(self.vmin, self.vmax), min(self.vmin, self.vmax)
        ra = hi - lo
        mid = lo + ra * self.bias
        mn = mid - ra * self.contrast
        mx = mid + ra * self.contrast

        if self.stretch == 'linear':
            result = (value - mn) * (1.0 / (mx - mn))
            result = np.clip(result, 0, 1)
        elif self.stretch == 'arcsinh':
            b = max(self.bias, 1e-5)
            c = self.contrast
            result = (value - lo) / (1.0 * (hi - lo))
            result = np.arcsinh(result / b) / np.arcsinh((b + c) / b)
            result = np.clip(result, 0, 1)
        elif self.stretch == 'sqrt':
            result = (value - mn) * (1.0 / (mx - mn))
            result = np.clip(result, 0, 1)
            result = np.sqrt(result)
        else:
            raise TypeError("Invalid stretch: %s" % self.stretch)

        if inverted:
            result = 1 - result

        return result


# System imports
import datetime
import json
import os
import re
import shutil
import zipfile

from gettext import gettext as _
from pathlib import Path

# Included imports

from loguru import logger
from utility import cprint, cstrip

# Module imports
from .config import *
from .hardware import *
from .util import *

class PlatformBase():
    MOVE_PM_BASH = False

    def __init__(self, hm):
        self.hm = hm

    def first_run(self):
        """
        Called on first run, this can be used to add custom sources for your platform.
        """
        logger.debug(f"{self.__class__.__name__}: First Run")

    def port_install(self, port_name, port_info, port_files):
        """
        Called on after a port is installed, this can be used to check permissions, possibly augment the bash scripts.
        """
        logger.debug(f"{self.__class__.__name__}: Port Install {port_name}")

    def runtime_install(self, runtime_name, runtime_files):
        """
        Called on after a port is installed, this can be used to check permissions, possibly augment the bash scripts.
        """
        logger.debug(f"{self.__class__.__name__}: Runtime Install {runtime_name}")

    def port_uninstall(self, port_name, port_info, port_files):
        """
        Called on after a port is uninstalled, this can be used clean up special files.
        """
        logger.debug(f"{self.__class__.__name__}: Port Uninstall {port_name}")

    def portmaster_install(self):
        """
        Called on after portmaster is updated, this can be used clean up special files.
        """
        logger.debug(f"{self.__class__.__name__}: PortMaster Install")

    def set_gcd_mode(self, mode=None):
        logger.debug(f"{self.__class__.__name__}: Set GCD Mode {mode}")

    def get_gcd_modes(self):
        return tuple()

    def get_gcd_mode(self):
        logger.debug(f"{self.__class__.__name__}: Get GCD Mode")
        return None

class PlatformGCD_PortMaster:
    """
    gamecontrollerdb standard / xbox mode
    """
    def set_gcd_mode(self, gcd_mode=None):
        gamecontroller_file = self.hm.tools_dir / "PortMaster" / "gamecontrollerdb.txt"
        mode_files = {
            'standard': self.hm.tools_dir / "PortMaster" / ".Backup" / "donottouch.txt",
            'xbox': self.hm.tools_dir / "PortMaster" / ".Backup" / "donottouch_x.txt",
            }

        logger.debug(f"{self.__class__.__name__}: Set GCD Mode: {gcd_mode}")

        if gcd_mode:
            if gcd_mode not in mode_files:
                logger.debug(f"Unknown gcd_mode {gcd_mode}")
                return

            if not mode_files[gcd_mode].is_file():
                logger.debug(f"Unknown gcd_mode {gcd_mode} file.")
                return

            shutil.copy(mode_files[gcd_mode], gamecontroller_file)

            self.hm.cfg_data['gcd-mode'] = gcd_mode
            self.hm.save_config()

        else:
            logger.debug(f"Weird {gcd_mode}")

    def get_gcd_modes(self):
        return ('standard', 'xbox')

    def get_gcd_mode(self):
        gamecontroller_file = self.hm.tools_dir / "PortMaster" / "gamecontrollerdb.txt"

        gcd_mode = self.hm.cfg_data.get('gcd-mode', None)

        if gcd_mode not in self.get_gcd_modes():
            gcd_mode = None

        if gcd_mode is None:
            if gamecontroller_file.is_file():
                if "# Xbox 360 Layout" in gamecontroller_file.read_text():
                    gcd_mode = 'xbox'

            if gcd_mode is None:
                gcd_mode = 'standard'

            self.hm.cfg_data['gcd-mode'] = gcd_mode
            self.hm.save_config()

        logger.debug(f"{self.__class__.__name__}: Get GCD Mode: {gcd_mode}")
        return gcd_mode

class PlatformUOS(PlatformGCD_PortMaster, PlatformBase):
    ...

class PlatformJELOS(PlatformBase):
    def first_run(self):
        self.portmaster_install()

    def portmaster_install(self):
        """
        Copy JELOS PortMaster files here.
        """

        ## Copy the JELOS portmaster stuff into the right place.
        JELOS_PM_DIR = Path("/storage/.config/PortMaster")

        if not JELOS_PM_DIR.is_dir():
            shutil.copytree("/usr/config/PortMaster", JELOS_PM_DIR)

        ## This fixes it if you have the older mapper.txt
        BROKEN_MAPPER = JELOS_PM_DIR / "mapper.txt"
        BROKEN_MAPPER_MD5 = "edb6c56435798cd8a5bc15be71a3c124"
        PM_DIR = HM_TOOLS_DIR / "PortMaster"

        if hash_file(BROKEN_MAPPER) == BROKEN_MAPPER_MD5:
            logger.debug("Replacing broken mapper.txt if it is still there.")
            FIXED_MAPPER = PM_DIR / ".Backup" / "mapper.txt"
            shutil.copy(FIXED_MAPPER, BROKEN_MAPPER)

        ## Copy the files as per usual.
        shutil.copy(JELOS_PM_DIR / "control.txt", PM_DIR / "control.txt")
        shutil.copy(JELOS_PM_DIR / "gptokeyb", PM_DIR / "gptokeyb")
        shutil.copy(JELOS_PM_DIR / "gamecontrollerdb.txt", PM_DIR / "gamecontrollerdb.txt")
        shutil.copy(JELOS_PM_DIR / "mapper.txt", PM_DIR / "mapper.txt")

        for oga_control in JELOS_PM_DIR.glob("oga_controls*"):
            shutil.copy(oga_control, PM_DIR / oga_control.name)

class PlatformArkOS(PlatformGCD_PortMaster, PlatformBase):
    MOVE_PM_BASH = True

class PlatformAmberELEC(PlatformGCD_PortMaster, PlatformBase):
    ...

HM_PLATFORMS = {
    'jelos': PlatformJELOS,
    'arkos': PlatformArkOS,
    'amberelec': PlatformAmberELEC,
    'unofficialos': PlatformUOS,
    'default': PlatformBase,
    # 'default': PlatformAmberELEC,
    }

__all__ = (
    'PlatformBase',
    'HM_PLATFORMS',
    )


from datetime import datetime as dt
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
import argparse

def load_data(path):
    return np.loadtxt(path, delimiter='\t', skiprows=1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--file', type=str, required=True)

    # Graph options
    parser.add_argument('--xlabel', type=str, default='million steps')
    parser.add_argument('--ylabel', type=str, default='score')

    parser.add_argument('--filename', type=str, default='result.png')

    parser.add_argument('--xplim', type=int, default=10)
    parser.add_argument('--yplim', type=int, default=500)
    parser.add_argument('--xnlim', type=int, default=0)
    parser.add_argument('--ynlim', type=int, default=0)

    parser.add_argument('--legend-pos', type=str, default='upper left')

    args = parser.parse_args()

    results = load_data(args.file)

    avg_y = results[:, 1]
    med_y = results[:, 2]
    x = np.asarray(range(len(med_y))) * 0.1

    plt.figure(figsize=(5, 4), dpi=80)

    plt.plot(x, med_y, label='median', linewidth=1)
    plt.plot(x, avg_y, label='average', linewidth=1)

    plt.legend(loc=args.legend_pos, fontsize=8)
    plt.xlim(args.xnlim, args.xplim)
    plt.ylim(args.ynlim, args.yplim)

    plt.xlabel(args.xlabel)
    plt.ylabel(args.ylabel)

    plt.savefig(args.filename)
    plt.show()


"""
Author: Daniel Mohr.

Date: 2017-02-07 (last change).

License: GNU GENERAL PUBLIC LICENSE, Version 3, 29 June 2007.
"""

import logging

from .read_data_from_file import read_data_from_file

def add_logging_level_name(lvl, levelname):
    """
    :Author: Daniel Mohr
    :Email: daniel.mohr@dlr.de
    :Date: 2016-12-04 (last change).
    :License: GNU GENERAL PUBLIC LICENSE, Version 3, 29 June 2007.
    """
    methodname = levelname.lower()
    if hasattr(logging, levelname):
        print("debug: levelname already exists")
    if hasattr(logging, methodname):
        print("debug: methodname already exists in logging")
    if hasattr(logging.getLoggerClass(), methodname):
        print("debug: methodname already exists in logging.getLoggerClass")

    def logforlevel(self, message, *args, **kwargs):
        # pylint: disable=missing-docstring,protected-access
        if self.isEnabledFor(lvl):
            self._log(lvl, message, args, **kwargs)

    def logtoroot(message, *args, **kwargs):
        # pylint: disable=missing-docstring
        logging.log(lvl, message, *args, **kwargs)
    logging.addLevelName(lvl, levelname)
    setattr(logging, levelname, lvl)
    setattr(logging.getLoggerClass(), methodname, logforlevel)
    setattr(logging, methodname, logtoroot)

add_logging_level_name(15, "VERBOSEINFO")

__all__ = ['read_data_from_file']


import os
import json
import tempfile
import hashlib
import cPickle
import copy

import boto
import Image
import numpy as np

import bson.json_util as json_util
from bson.objectid import ObjectId

import tornado.web
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.autoreload
from tornado.options import define, options

import genthor.datasets as gd

define("port", default=27719, help="run on the given port", type=int)

class App(tornado.web.Application):
    """
        Tornado app which serves the API.
    """
    def __init__(self):
        handlers = [(r"/renderimage", RenderImageHandler)]
        settings = dict(debug=True)
        tornado.web.Application.__init__(self, handlers, **settings)

class BaseHandler(tornado.web.RequestHandler):
    def get(self):
        args = self.request.arguments
        for k in args.keys():
            args[k] = args[k][0]
        args = dict([(str(x),y) for (x,y) in args.items()])
        args['preproc'] = json.loads(args['preproc'])
        args['spec'] = json.loads(args['spec'])

        callback = args.pop('callback', None)

        resp = jsonize(self.get_response(args))

        if callback:
            self.write(callback + '(')
        self.write(json.dumps(resp, default=json_util.default))
        if callback:
            self.write(')')

        self.finish()

class RenderImageHandler(BaseHandler):
    def get_response(self, args):
        return image_response(args)

DATASET = None
BUCKET = None
def image_response(args):
    sha1 = hashlib.sha1(json.dumps(args)).hexdigest()
    filename = sha1 + '.png'
    global BUCKET
    if BUCKET is None:
        conn = boto.connect_s3()
        BUCKET = conn.get_bucket('dicarlo-renderedimages')
    k = BUCKET.get_key(filename)
    global DATASET
    if not k:
        if DATASET is None:
            DATASET = gd.GenerativeBase() #pull out into global?
        imarray = DATASET.get_image(args['preproc'], args['spec'])[::-1]
        im = Image.fromarray(imarray, mode=args['preproc']['mode'])
        tmp = tempfile.TemporaryFile()
        im.save(tmp, "png")
        del imarray
        del im
        tmp.seek(0)
        k = BUCKET.new_key(filename)
        k.set_contents_from_string(tmp.read(), headers={'Content-Type' : 'image/png'})
        k.make_public()
        tmp.close()
    url = "http://dicarlo-renderedimages.s3.amazonaws.com/" + filename
    resp = {"url": url}
    return resp

def jsonize(x):
    try:
        json.dumps(x)
    except:
        return SONify(x)
    else:
        return x

def SONify(arg, memo=None):
    if memo is None:
        memo = {}
    if id(arg) in memo:
        rval = memo[id(arg)]
    if isinstance(arg, ObjectId):
        rval = arg
    elif isinstance(arg, np.floating):
        rval = float(arg)
    elif isinstance(arg, np.integer):
        rval = int(arg)
    elif isinstance(arg, (list, tuple)):
        rval = type(arg)([SONify(ai, memo) for ai in arg])
    elif isinstance(arg, dict):
        rval = dict([(SONify(k, memo), SONify(v, memo))
            for k, v in arg.items()])
    elif isinstance(arg, (basestring, float, int, type(None))):
        rval = arg
    elif isinstance(arg, np.ndarray):
        if arg.ndim == 0:
            rval = SONify(arg.sum())
        else:
            rval = map(SONify, arg) # N.B. memo None
    # -- put this after ndarray because ndarray not hashable
    elif arg in (True, False):
        rval = int(arg)
    else:
        raise TypeError('SONify', arg)
    memo[id(rval)] = rval
    return rval

def main():
    """
        function which starts up the tornado IO loop and the app.
    """
    tornado.options.parse_command_line()
    ioloop = tornado.ioloop.IOLoop.instance()
    http_server = tornado.httpserver.HTTPServer(App())
    http_server.listen(options.port)
    tornado.autoreload.start()
    ioloop.start()

if __name__ == "__main__":
    main()


from django.contrib import admin

from duoyidian.models import DishClassifyInfo, DishInfo

class DishClassifyInfoAdmin(admin.ModelAdmin):
    list_display = ('classify', 'status', 'priority', 'create_at', 'update_at')

class DishInfoAdmin(admin.ModelAdmin):
    list_display = ('dish_classify', 'dish_name', 'dish_image', 'dish_price', 'status', 'priority', 'create_at', 'update_at')

admin.site.register(DishClassifyInfo, DishClassifyInfoAdmin)
admin.site.register(DishInfo, DishInfoAdmin)


# -*- coding:utf-8 -*-

from scikits.talkbox.features import mfcc
import numpy as np
import soundfile as sf
import os
from sklearn.ensemble import RandomForestClassifier
# from sklearn.svm import LinearSVC, SVC
# from xgboost import XGBClassifier
from sklearn.externals import joblib

from utils import import_wav_data_in_dir

random_state=24
label_dict = {'hu':0, 'ti':1, 'dc':2}
datapath = 'cry_data/devided_wav_data/'

def main():
    # import wav files
    files = import_wav_data_in_dir(datapath)

    # prepare for training
    # get feature vectors by mfcc
    X = []
    y = []
    for f in files:
        x, sample_rate = sf.read(datapath + f)
        x = np.clip(x, 1e-10, 1)
        ceps,mspec,spec = mfcc(x, nwin=256, nfft=512, fs=8000, nceps=13)
        X.append(np.mean(ceps, axis=0))
        if '-hu.' in f:
            y.append(label_dict['hu'])
        elif '-ti.' in f:
            y.append(label_dict['ti'])
        else:
            y.append(label_dict['dc'])

    X = np.array(X)
    y = np.array(y)

    # training
    clf = RandomForestClassifier(n_estimators=498, random_state=random_state)
    # clf = XGBClassifier(max_depth=8, learning_rate=0.05, n_estimators=700, seed=random_state)
    clf.fit(X, y)

    # save model
    joblib.dump(clf, 'trained_models/clf_rf2.pkl.cmp', compress=True)

if __name__ == '__main__':
    main()


import argparse

from models import *
from utils.datasets import *
from utils.utils import *

def test(
        cfg,
        data_cfg,
        weights,
        batch_size=16,
        img_size=416,
        iou_thres=0.5,
        conf_thres=0.3,
        nms_thres=0.45
):
    device = torch_utils.select_device()

    # Configure run
    data_cfg_dict = parse_data_cfg(data_cfg)
    nC = int(data_cfg_dict['classes'])  # number of classes (80 for COCO)
    test_path = data_cfg_dict['valid']

    # Initialize model
    model = Darknet(cfg, img_size)

    # Load weights
    if weights.endswith('.pt'):  # pytorch format
        model.load_state_dict(torch.load(weights, map_location='cpu')['model'])
    else:  # darknet format
        load_darknet_weights(model, weights)

    model.to(device).eval()

    # Get dataloader
    # dataloader = torch.utils.data.DataLoader(LoadImagesAndLabels(test_path), batch_size=batch_size)  # pytorch
    #该类的说明见notebook和dataset.py
    #(可改，多尺度训练和数据增广入口在load_images_and_labels()里面)
    #类似detect的load_image()，这里构造的dataloader也是一个迭代器，需要通过遍历來访问其两个返回值：torch.from_numpy(img_all), labels_all
    dataloader = LoadImagesAndLabels(test_path, batch_size=batch_size, img_size=img_size)

    mean_mAP, mean_R, mean_P = 0.0, 0.0, 0.0    #mAP,平均召回率，平均准确率
    print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP')) #%11s是每个字符串均占11个字符长度，向右对其，不足补空格
    outputs, mAPs, mR, mP, TP, confidence, pred_class, target_class = [], [], [], [], [], [], [], []
    AP_accum, AP_accum_count = np.zeros(nC), np.zeros(nC)

    #通过遍历访问构造的迭代器：imgs是图片像素张量，targets是该图所有gt的张量
    #下面设涉及维度变化，遍历中间结果放在notebook展示比较清楚，直接搜for batch_i, (imgs, targets) in enumerate(dataloader)
    for batch_i, (imgs, targets) in enumerate(dataloader):
        output = model(imgs.to(device))
        output = non_max_suppression(output, conf_thres=conf_thres, nms_thres=nms_thres)

        # Compute average precision for each sample
        #将预测的输出和真实gt进行压缩用于迭代（re：zip返回的是元组，这个创建迭代器真的好用）
        for sample_i, (labels, detections) in enumerate(zip(targets, output)):
            correct = []
            #没有检测出物体，但是实际gt是有物体
            if detections is None:
                # If there are no detections but there are labels mask as zero AP
                if labels.size(0) != 0: #存在图片：labels.size(0)返回的是第0维度长度，即当前batch的图片数目
                    mAPs.append(0), mR.append(0), mP.append(0)#有图片默认就有gt，但是没有检测出物体，因此p,r均为0
                continue

            # Get detections sorted by decreasing confidence scores
            #原来box是按照类内conf降序，下面转为所有类conf降序
            detections = detections.cpu().numpy()
            detections = detections[np.argsort(-detections[:, 4])]

            # If no labels add number of detections as incorrect
            if labels.size(0) == 0: #负样本，没有标签
                # correct.extend([0 for _ in range(len(detections))])
                mAPs.append(0), mR.append(0), mP.append(0)
                continue
            else:
                target_cls = labels[:, 0]   #摘出gt的类标签放入一维tensor，长度为gt的box个数

                # Extract target boxes as (x1, y1, x2, y2)
                #pred结果已经处理过是正常值，但是gt标注还是416归一化的，这里给她缩放到正常图（416）
                #xywh转成xyxy因为下面的iou计算要用xyxy
                target_boxes = xywh2xyxy(labels[:, 1:5]) * img_size

                detected = []
                #下面对一张图的每个预测的box进行遍历
                for *pred_bbox, conf, obj_conf, obj_pred in detections:
                    #（re)pred_bbox存放预测box的坐标信息，*代表传递多参数
                    pred_bbox = torch.FloatTensor(pred_bbox).view(1, -1)
                    # Compute iou with target boxes(传入形式要为xyxy，他这个可以(re)，参见utils)
                    #这里返回的张量不好理解，见notebook和utils.py
                    iou = bbox_iou(pred_bbox, target_boxes)
                    # Extract index of largest overlap
                    best_i = np.argmax(iou)
                    # If overlap exceeds threshold and classification is correct mark as correct
                    #(atten)这个if很仔细：1.iou大于阈值 2.类别和gt能对应上（易漏，如果不对应，那就是很严重的误检了）
                    if iou[best_i] > iou_thres and obj_pred == labels[best_i, 0] and best_i not in detected:
                        correct.append(1)       #满足上述，说明这个box预测的正确，记为1,否则0
                        detected.append(best_i) #记下这个box是属于哪gt
                    else:
                        correct.append(0)

            # Compute Average Precision (AP) per class
            AP, AP_class, R, P = ap_per_class(tp=correct, conf=detections[:, 4], pred_cls=detections[:, 6],
                                              target_cls=target_cls)

            # Accumulate AP per class
            AP_accum_count += np.bincount(AP_class, minlength=nC)
            AP_accum += np.bincount(AP_class, minlength=nC, weights=AP)

            # Compute mean AP across all classes in this image, and append to image list
            mAPs.append(AP.mean())
            mR.append(R.mean())
            mP.append(P.mean())

            # Means of all images
            mean_mAP = np.mean(mAPs)
            mean_R = np.mean(mR)
            mean_P = np.mean(mP)

        # Print image mAP and running mean mAP
        print(('%11s%11s' + '%11.3g' * 3) % (len(mAPs), dataloader.nF, mean_P, mean_R, mean_mAP))

    # Print mAP per class
    print('%11s' * 5 % ('Image', 'Total', 'P', 'R', 'mAP') + '\n\nmAP Per Class:')

    classes = load_classes(data_cfg_dict['names'])  # Extracts class labels from file
    for i, c in enumerate(classes):
        print('%15s: %-.4f' % (c, AP_accum[i] / (AP_accum_count[i] + 1E-16)))

    # Return mAP
    return mean_mAP, mean_R, mean_P

if __name__ == '__main__':
    parser = argparse.ArgumentParser(prog='test.py')
    parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch')
    parser.add_argument('--cfg', type=str, default='cfg/yolov3.cfg', help='cfg file path')
    parser.add_argument('--data-cfg', type=str, default='cfg/coco.data', help='coco.data file path')
    parser.add_argument('--weights', type=str, default='weights/yolov3.weights', help='path to weights file')
    parser.add_argument('--iou-thres', type=float, default=0.5, help='iou threshold required to qualify as detected')
    parser.add_argument('--conf-thres', type=float, default=0.3, help='object confidence threshold')
    parser.add_argument('--nms-thres', type=float, default=0.45, help='iou threshold for non-maximum suppression')
    parser.add_argument('--img-size', type=int, default=416, help='size of each image dimension')
    opt = parser.parse_args()
    print(opt, end='\n\n')

    with torch.no_grad():
        mAP = test(
            opt.cfg,
            opt.data_cfg,
            opt.weights,
            opt.batch_size,
            opt.img_size,
            opt.iou_thres,
            opt.conf_thres,
            opt.nms_thres
        )