import pandas as pd
import pickle
import project_helper
from tqdm import tqdm
from selenium import webdriver
from selenium.webdriver.edge.service import Service as EdgeService
from webdriver_manager.microsoft import EdgeChromiumDriverManager
from selenium.webdriver.common.by import By
from datetime import datetime
from selenium.webdriver import EdgeOptions
from itertools import islice
from bs4 import BeautifulSoup
import requests

#continue from the previous break
break_batch = 14

headers = {'Host': 'www.sec.gov', 'Connection': 'close',
           'Accept': 'application/json, text/javascript, */*; q=0.01',
           'X-Requested-With': 'XMLHttpRequest',
           'User-Agent': 'ruizhuoj@andrew.cmu.edu'
           }
endpoint = r"https://www.sec.gov/cgi-bin/browse-edgar"
base_url_sec = r"https://www.sec.gov"
sec_api = project_helper.SecAPI()
# browser = webdriver.Edge(service=EdgeService(EdgeChromiumDriverManager().install()))
# browser.set_page_load_timeout(1000)
#break apart dictionaries
def chunks(data, SIZE=100):
    it = iter(data)
    for i in range(0, len(data), SIZE):
        yield {k:data[k] for k in islice(it, SIZE)}

### Get S&P500 CIKs
# Filter the sp500_constituents csv by removing companies that were out after 2017.
# Use the sp500_constituents permnos to filter sp500_data and get a dictionary of tickers and
# CIKs.
sp500_constituents = pd.read_csv("Data/sp500_constituents.csv", dtype={"permno":int}, index_col=0)
sp500_constituents = sp500_constituents[(sp500_constituents["ending"] > "2017-01-01")]
sp500_data = pd.read_csv("Data/sp500_w_addl_id_with_cik.csv", dtype={"cik":str, "permno":int})
sp500_data = sp500_data[["ticker", "permno", "cik"]].set_index("ticker")
sp500_data = sp500_data[sp500_data["permno"].isin(sp500_constituents["permno"])]
sp500_data.drop_duplicates(inplace=True)
sp500_data.dropna(inplace=True)
cik_lookup = sp500_data.to_dict()["cik"]
cik_lookup = dict(islice(cik_lookup.items(), break_batch, len(cik_lookup)))
# download, parse and save 100 tickers at a time
for cik_lookup_s in chunks(cik_lookup, 10):
    break_batch += 1
    sec_data = {ticker: [] for ticker in cik_lookup_s}
    for ticker in cik_lookup_s:
        # define our parameters dictionary
        param_dict_10k = {'action': 'getcompany',
                          'CIK': cik_lookup_s[ticker],
                          'type': '10-k',
                          'dateb': '20220101',
                          'owner': 'exclude',
                          'start': '',
                          'output': '',
                          'count': '10'}
        # request the url, and then parse the response.
        response_10k = requests.get(url=endpoint, params=param_dict_10k, headers=headers)
        soup_10k = BeautifulSoup(response_10k.content, 'html.parser')
        doc_table_10k = soup_10k.find_all('table', class_='tableFile2')
        param_dict_10q = {'action': 'getcompany',
                          'CIK': cik_lookup_s[ticker],
                          'type': '10-Q',
                          'dateb': '20220101',
                          'owner': 'exclude',
                          'start': '',
                          'output': '',
                          'count': '20'}
        # request the url, and then parse the response.
        response_10q = requests.get(url=endpoint, params=param_dict_10q, headers=headers)
        soup_10q = BeautifulSoup(response_10q.content, 'html.parser')
        doc_table_10q = soup_10q.find_all('table', class_='tableFile2')
        # Get 10-Ks
        for row in doc_table_10k[0].find_all('tr'):
            # find all the columns
            cols = row.find_all('td')
            # if there are no columns move on to the next row.
            if len(cols) != 0:
                # grab the text
                filing_type = cols[0].text.strip()
                filing_date = cols[3].text.strip()
                if datetime.strptime(filing_date, '%Y-%m-%d').date() < datetime.strptime("2017", '%Y').date():
                    pass
                else:
                    filing_numb = cols[4].text.strip()
                    # find the links
                    filing_doc_href = cols[1].find('a', {'href': True, 'id': 'documentsbutton'})
                    filing_int_href = cols[1].find('a', {'href': True, 'id': 'interactiveDataBtn'})
                    filing_doc_link = base_url_sec + filing_doc_href['href']
                    sec_data[ticker].append((filing_doc_link, filing_type, filing_date))
        # Get 10-Qs
        for row in doc_table_10q[0].find_all('tr'):
            # find all the columns
            cols = row.find_all('td')
            # if there are no columns move on to the next row.
            if len(cols) != 0:
                # grab the text
                filing_type = cols[0].text.strip()
                filing_date = cols[3].text.strip()
                filing_numb = cols[4].text.strip()
                # find the links
                filing_doc_href = cols[1].find('a', {'href': True, 'id': 'documentsbutton'})
                filing_int_href = cols[1].find('a', {'href': True, 'id': 'interactiveDataBtn'})
                filing_doc_link = base_url_sec + filing_doc_href['href']
                sec_data[ticker].append((filing_doc_link, filing_type, filing_date))
        print(ticker, "request successful")

    fillings_by_ticker = {}
    for ticker, data in sec_data.items():
        fillings_by_ticker[ticker] = {}
        for index_url, file_type, file_date in tqdm(data, desc='Downloading {} Fillings'.format(ticker),
                                                    unit='filling'):
            print(index_url, file_type, file_date)
            if (file_type == '10-K' or file_type == '10-Q'):
                file_url = index_url.replace('-index.htm', '.txt').replace('.txtl', '.txt')
                fillings_by_ticker[ticker][file_date] = sec_api.get(file_url)
    with open(r'E:\10KQ\fillings_by_ticker ' + str(break_batch), 'wb') as handle:
        pickle.dump(fillings_by_ticker, handle, protocol=pickle.HIGHEST_PROTOCOL)
        print("Batch", break_batch, "saved.")


#!/usr/bin/python3
"""lists all State objects from the database hbtn_0e_6_usa"""
from sys import argv
from model_state import Base, State
from sqlalchemy import create_engine
from sqlalchemy.orm import Session

if __name__ == '__main__':
    engine = create_engine('mysql+mysqldb://{}:{}@localhost/{}'
                           .format(argv[1], argv[2],
                                   argv[3],), pool_pre_ping=True)
    name = argv[4]
    Base.metadata.create_all(engine)
    session = Session(engine)
    row = session.query(State).filter(State.name == name).first()
    if row is not None:
        print(row.id)
    else:
        print("Not found")
    session.close()


import os
import cv2

images = os.listdir("/home/anirudh_/images/")
#images.sort()
j = 0
length = len(images)
for i in images:
    img = cv2.imread(r"/home/anirudh_/images/" + i, 1)
    print(i, j, length)
    j+=1
    cv2.imshow(i, img)
    if cv2.waitKey(0) & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        continue
    elif cv2.waitKey(0) & 0xFF == ord('e'):
        cv2.destroyAllWindows()
        os.remove(i)


import pandas, numpy, scipy.ndimage
import os, timeit
import matplotlib.pyplot as plt
import random
import math
import logging
import multiprocessing as mp

home = 'C:/Users/OPEN/Documents/NanZHAO/Projet CIR Fraude/'
path = 'C:/Users/OPEN/Documents/NanZHAO/Projet CIR Fraude/data/store/'
logging.basicConfig(filename = os.path.join(home, 'code/images_info.log'), level = logging.INFO)
process_number = 40

images_all = os.listdir(path)

def get_image_info(im):
	postId = im.split('_')[0]
	Id = im.split(postId+'_')[1]
	logging.info(Id)
	shapes = scipy.ndimage.imread(os.path.join(path, im)).shape
	height = shapes[0]
	width  = shapes[1]
	if len(shapes) == 3:
		channels = shapes[2]
	else:
		channels = 2

	return(pandas.Series({'Id':Id, 'postId':postId, 'height':height, 'width':width, 'channels': channels}))

if __name__ == '__main__':
	start = timeit.default_timer()
	image_info = pandas.DataFrame(columns = ('Id','postId', 'height', 'width', 'channels'))
	count =  0
	p = mp.Pool(process_number)
	for image in images_all:
		res = p.apply_async(get_image_info, (image,)).get(timeout=10)
		#print(res)
		image_info = image_info.append (res, ignore_index=True)
		#image_info.append (res.get(timeout=10))
		#print(image_info.head())
		"""
		count = count + 1
		if (count%2075) == 0:
			print(" %d percentage is done." %(count/2075) )
		"""
	print(image_info.head())
	print(len(image_info))
	image_info.to_json(os.path.join(home, 'data/images_info.json'))
	stop = timeit.default_timer()
	print("Execution time: ", (stop-start))


import json

with open("datasets/sarcasm/Sarcasm_Headlines_Dataset.json", 'r') as f:
    datastore = json.load(f)

data = {
    'sentences': [],
    'labels': [],
    'urls': []
}

for item in datastore:
    data['sentences'].append(item['headline'])
    data['labels'].append(item['is_sarcastic'])
    data['urls'].append(item['article_link'])


"""
First, a few callback functions are defined. Then, those functions are passed to
the Dispatcher and registered at their respective places.
Then, the bot is started and runs until we press Ctrl-C on the command line.
Usage:
Example of a bot-user conversation using ConversationHandler.
Send /start to initiate the conversation.
Press Ctrl-C on the command line or send a signal to the process to stop the
bot.
"""

import logging
from typing import Dict

from telegram import ReplyKeyboardMarkup, Update, ReplyKeyboardRemove
from telegram.ext import (
    Updater,
    CommandHandler,
    MessageHandler,
    Filters,
    ConversationHandler,
    CallbackContext,
)
# user-defined functions and constants
from data_handler import Data_handler
from constants import BOT_TOKEN, DB_PATH # нужно сюда поставить токен бота и путь к файлу с БД

# Enable logging
logging.basicConfig(
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
)

logger = logging.getLogger(__name__)

CHOOSING, TYPING_VAL, TYPING_PARAM = range(3)

start_keyboard = [
    ['Fetch data', 'Insert data'],
    ['Done'],
]
start_markup = ReplyKeyboardMarkup(start_keyboard, one_time_keyboard=True)

body_keyboard = [
    ['Proceed with a new param'],
    ['Done'],
]
body_markup = ReplyKeyboardMarkup(body_keyboard, one_time_keyboard=True)

# Add a custom db wrapper
db = Data_handler(DB_PATH)

def udata_to_str(user_data: Dict[str, str]) -> str:
    """Helper function for formatting the gathered user info."""
    facts = [f'{key} : {value}' for key, value in user_data.items()]
    return "\n".join(facts).join(['\n', '\n'])

def start(update: Update, context: CallbackContext) -> int:
    """Start the conversation and ask user to choose an action."""
    update.message.reply_text(
        "Hi! Homer bot is here, duh.\n"
        "I can fetch you deal price based on params you provide or update the database with your data.\n"
        "What do you want?",
        reply_markup=start_markup,
    )
    context.user_data['param_dict'] = {}

    return CHOOSING

def initiate_fetch(update: Update, context: CallbackContext) -> int:
    """Invite user to provide an *independent param*."""
    update.message.reply_text(
    "Alright, please send me the name of the parameter first:")
    context.user_data['action'] = 'Fetch'

    return TYPING_PARAM

def get_param_name(update: Update, context: CallbackContext) -> int:
    """Ask the user for a name of the *independent param*."""
    text = update.message.text.strip()
    context.user_data['choice'] = text
    # place to hold user data input
    update.message.reply_text(f'Your param is {text.lower()}? Please, provide its value.')

    return TYPING_VAL

def get_param_val(update: Update, context: CallbackContext) -> int:
    """Store info provided by user and ask for the next param if needed."""
    user_data = context.user_data
    text = update.message.text.strip()
    param_name = user_data['choice']
    user_data['param_dict'][param_name] = text
    del user_data['choice']

    # TODO delete this print after debug
    print(user_data)

    if context.user_data['action'] == 'Fetch':
        try:
            result_dict = db.get_param(user_data['param_dict'])
        except KeyError:
            update.message.reply_text(
                "Neat! Just so you know, below are the params and vals you entered:"
                f"{udata_to_str(user_data['param_dict'])}\n"
                "No such parameter in db."
                ,
                reply_markup=body_markup,
            )
            # delete wrong entry from the user_data
            del user_data['param_dict'][param_name]

            return CHOOSING

        # TODO delete this print after debug
        print(user_data)

        try:
            avg_param = sum(result_dict.values())/len(result_dict)
        except ZeroDivisionError:
            avg_param = 'NULL'

        update.message.reply_text(
            "Neat! Just so you know, below are the params and vals you entered:"
            f"{udata_to_str(user_data['param_dict'])}"
            f"There are {len(result_dict)} lines matching params provided."
            f"Corresponding average price is {avg_param}.\n"
            "You can tell me more, or change the nature of interaction."
            ,
            reply_markup=body_markup,
        )

        if len(result_dict) <= 10:
            update.message.reply_text(
                "Below are the ids matching the pattern:"
                f"{udata_to_str(result_dict)}"
            )

        return CHOOSING

    # TODO add messages to user
    # TODO refactor the code into multiple sub functions
    if context.user_data['action'] == 'Insert':
        try:
            db.append_entry(context.user_data['param_dict'])
        except ValueError:
            update.message.reply_text(
                "Neat! Just so you know, below are the params and vals you entered:"
                f"{udata_to_str(user_data['param_dict'])}\n"
                "db has no such column."
                ,
                reply_markup=body_markup,
            )
        else:
            update.message.reply_text(
                "Neat! Just so you know, below are the params and vals you entered:"
                f"{udata_to_str(user_data['param_dict'])}"
                "You can tell me more, or change the nature of interaction."
                ,
                reply_markup=body_markup,
            )
        return CHOOSING

def initiate_insert(update: Update, context: CallbackContext) -> int:
    """Ask the user for a description of a custom param_name."""
    update.message.reply_text(
    "Alright, please send me the name of the parameter first:")
    context.user_data['action'] = 'Insert'

    return TYPING_PARAM

def done(update: Update, context: CallbackContext) -> int:
    """Display the gathered info and end the conversation."""
    user_data = context.user_data
    if 'choice' in user_data:
        del user_data['choice']

    update.message.reply_text(
        f"Below is the data you provided: {udata_to_str(user_data['param_dict'])} Until next time!",
        reply_markup=ReplyKeyboardRemove(),
    )

    user_data.clear()
    return ConversationHandler.END

# TODO change the default text
def main() -> None:
    """Run the bot."""
    # Create the Updater and pass it your bot's token.
    updater = Updater(BOT_TOKEN)

    # Get the dispatcher to register handlers
    dispatcher = updater.dispatcher

    # Add conversation handler with the states CHOOSING, TYPING_PARAM and TYPING_VAL
    conv_handler = ConversationHandler(
        entry_points=[CommandHandler('start', start)],
        states={
            CHOOSING: [
                MessageHandler(
                    Filters.regex('^(Fetch data)$') | Filters.regex('^Proceed with a new param$'),
                    initiate_fetch),
                MessageHandler(Filters.regex('^Insert data$'), initiate_insert),
                # MessageHandler(Filters.regex('^Proceed with a new param$'), initiate_fetch),
            ],
            TYPING_PARAM: [
                MessageHandler(Filters.text & ~(Filters.command | Filters.regex('^Done$')), get_param_name)
            ],
            TYPING_VAL: [
                MessageHandler(
                    Filters.text & ~(Filters.command | Filters.regex('^Done$')),
                    get_param_val,
                )
            ],
        },
        fallbacks=[MessageHandler(Filters.regex('^Done$'), done)],
    )

    dispatcher.add_handler(conv_handler)

    # Start the Bot
    updater.start_polling()

    # Run the bot until you press Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()

if __name__ == '__main__':
    main()


import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
import pandas as pd
import pickle
import re
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.multioutput import MultiOutputClassifier
from sklearn.pipeline import Pipeline
from sqlalchemy import create_engine
import sys

nltk.download(['punkt', 'stopwords', 'wordnet'])

def load_data(database_filepath):
    """
    Reads in data from SQLite database and outputs dataframes for ML model

    INPUT:
        database_filepath: filepath to SQLite database

    OUTPUTS:
        X: features dataframe
        y: target dataframe
        category_names: names of category targets
    """
    # create SQLAlchemy engine and read in data
    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql_table('df', engine)

    # create features (X) and target (y) dataframes
    X = df['message']
    y = df.iloc[:, 4:]

    # retrieve names of targets
    category_names = y.columns

    return X, y, category_names

def tokenize(text):
    """
    Cleans text data in order to use it for machine learning later.

    INPUT:
        text: Raw message

    OUTPUT:
        cleaned_tokens: Cleaned text (w/o urls, normalized, tokenized, w/o stopwords, lemmatized) as list
    """
    # Use regex expression to detect urls in text
    url_regex = "http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
    detected_urls = re.findall(url_regex, text)

    # Replace all urls with placeholder
    for detected_url in detected_urls:
        text = text.replace(detected_url, "url")

    # Normalization
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    # Tokenization
    words = word_tokenize(text)

    # Removal of stopwords
    words = [w for w in words if w not in stopwords.words("english")]

    # Lemmatization (nouns)
    cleaned_tokens = [WordNetLemmatizer().lemmatize(w).strip() for w in words]

    return cleaned_tokens

def build_model():
    """
    Builds a ML Pipeline using a RandomForest classifier and GridSearchCV to tune it to its optimal hyperparameters.

    INPUT:
        None

    OUTPUT:
        model: GridSearch output from cross-validation
    """
    # set up of ML Pipeline
    pipeline = Pipeline([
        ('vect', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', MultiOutputClassifier(RandomForestClassifier(random_state=42)))
    ])

    # set up of hyper-parameter dictionary for GridSearchCV of RandomForestClassifier
    params = {
        'clf__estimator__min_samples_split': [2, 4],  # min number of data points in node before the node is split
        'clf__estimator__max_features': ['log2', 'auto'],  # max number of features considered for splitting a node
        'clf__estimator__n_estimators': [50, 100],  # number of trees in the forest
    }

    # create model
    model = GridSearchCV(pipeline, param_grid=params, scoring='accuracy', verbose=3, n_jobs=1, cv=3)
    return model

def evaluate_model(model, X_test, y_test, category_names):
    """
    Evaluate trained ML model performance against test data.

    INPUTS:
        model: Trained ML model
        X_test: Test data (features)
        y_test: Test data (targets / labels)
        target_names: Names of labels

    OUTPUT:
        Printed results from GridSearch; classification report and accuracy for all categories
    """
    # print results from GridSearch
    model_results_table = pd.concat([pd.DataFrame(model.cv_results_["params"]),
                                     pd.DataFrame(model.cv_results_["mean_test_score"],
                                                  columns=["Accuracy"])], axis=1)

    print("GridSearch Results Table")
    print(model_results_table)

    print("Best-performing parameters from GridSearch:", model.best_params_)

    # make predictions on test data
    y_pred = model.predict(X_test)

    # print classification reports
    print("Classification report incl. overall micro, macro, weighted and sample averages")
    print(classification_report(y_test, y_pred, target_names=category_names, zero_division=0))

    print("Individual classification report incl. accuracy, macro and weighted averages")
    y_pred_df = pd.DataFrame(y_pred, columns=y_test.columns)
    for category in y_test.columns:
        print('-------------------------------------------------\n Category: {}\n'.format(category))
        print(classification_report(y_test[category], y_pred_df[category], zero_division=0))

def save_model(model, model_filepath):
    """
    Save trained model to pickle file

    INPUTS:
        model: trained model
        model_filepath: filepath to save model as pickle file (byte stream)

    OUTPUT:
        pickle file // none
    """
    # save trained model in binary
    with open(model_filepath, 'wb') as file:
        pickle.dump(model, file)

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, y, category_names = load_data(database_filepath)
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database ' \
              'as the first argument and the filepath of the pickle file to ' \
              'save the model to as the second argument. \n\nExample: python ' \
              'train_classifier.py ../data/Disaster.db classifier.pkl')

if __name__ == '__main__':
    main()


import pandas as pd
import dash
from dash import dcc, html, Input, Output
from dash.exceptions import PreventUpdate
from dash.dependencies import State
import base64
import io

# Create the Dash app
app = dash.Dash(__name__)

# Add the Font Awesome CSS link to your app
app.css.append_css({
    "external_url": "https://use.fontawesome.com/releases/v5.15.3/css/all.css"
})

# Define the layout of the dashboard
app.layout = html.Div([
    html.H1([html.I(className='fas fa-chart-bar'), ' Data Visualization Dashboard'], style={'textAlign': 'center', 'color': '#007BFF'}),

    dcc.Upload(
        id='upload-data',
        children=html.Div([
            'Drag and Drop or ',
            html.A('Select Files')
        ]),
        style={
            'width': '50%',
            'height': '60px',
            'lineHeight': '60px',
            'borderWidth': '1px',
            'borderStyle': 'dashed',
            'borderRadius': '5px',
            'textAlign': 'center',
            'margin': 'auto',
            'marginBottom': '20px'
        },
        # Allow multiple files to be uploaded
        multiple=False
    ),dcc.Dropdown(
        id='dropdown',
        options=[],
        value=None,
        style={'width': '50%', 'margin': 'auto', 'marginBottom': '20px'}
    ),

    html.Div([
        dcc.Graph(id='bar-chart', className='graph-container'),
        dcc.Graph(id='line-chart', className='graph-container'),
    ], style={'display': 'flex', 'flexWrap': 'wrap', 'justifyContent': 'center'})

])

# Callback to read the uploaded file and update the dropdown options
@app.callback(
    [Output('dropdown', 'options'),
     Output('dropdown', 'value')],
    [Input('upload-data', 'contents')],
    [State('upload-data', 'filename')]
)
def update_dropdown_options(contents, filename):
    if contents is None:
        raise PreventUpdate

    df = parse_contents(contents, filename)
    dropdown_options = [{'label': col, 'value': col} for col in df.columns]
    dropdown_value = df.columns[0] if not df.empty else None
    return dropdown_options, dropdown_value

# Callback to update the bar chart based on the dropdown selection
@app.callback(
    Output('bar-chart', 'figure'),
    [Input('dropdown', 'value')],
    [State('upload-data', 'contents')],
    [State('upload-data', 'filename')]
)
def update_bar_chart(selected_value, contents, filename):
    if selected_value is None or contents is None:
        raise PreventUpdate

    df = parse_contents(contents, filename)
    data_to_plot = df[['Month', selected_value]].to_dict('list')
    return {
        'data': [{'x': data_to_plot['Month'], 'y': data_to_plot[selected_value], 'type': 'bar'}],
        'layout': {'title': f'{selected_value} by Month'}
    }

# Callback to update the line chart based on the dropdown selection
@app.callback(
    Output('line-chart', 'figure'),
    [Input('dropdown', 'value')],
    [State('upload-data', 'contents')],
    [State('upload-data', 'filename')]
)
def update_line_chart(selected_value, contents, filename):
    if selected_value is None or contents is None:
        raise PreventUpdate

    df = parse_contents(contents, filename)
    data_to_plot = df[['Month', selected_value]].to_dict('list')
    return {
        'data': [{'x': data_to_plot['Month'], 'y': data_to_plot[selected_value], 'type': 'line'}],
        'layout': {'title': f'{selected_value} by Month'}
    }

def parse_contents(contents, filename):
    content_type, content_string = contents.split(',')
    decoded = base64.b64decode(content_string)
    try:
        if 'csv' in filename:
            # Assume that the user uploaded a CSV file
            df = pd.read_csv(io.StringIO(decoded.decode('utf-8')))
        else:
            raise Exception('Unsupported file format')
    except Exception as e:
        print(e)
        return pd.DataFrame()
    return df

# Run the app
if __name__ == '__main__':
    app.run_server(debug=True)


import os
import sys
from tempfile import TemporaryFile

from metomi.rose.config import ConfigDecodeError, ConfigLoader
from metomi.rose.popen import RosePopener, RosePopenError
from metomi.rose.reporter import Reporter

class BadChange(Exception):
    """An error representing a bad change in a commit transaction."""

    FMT_HEAD = "%s: %-4s%s"
    FMT_TAIL_1 = ": %s"
    FMT_TAIL_M = ":\n%s"
    NO_INFO = "SUITE MUST HAVE INFO FILE"
    NO_OWNER = "SUITE MUST HAVE OWNER SPECIFIED IN INFO FILE"
    NO_TRUNK = "SUITE MUST HAVE TRUNK"
    PERM = "PERMISSION DENIED"
    USER = "NO SUCH USER"
    VALUE = "BAD VALUE IN FILE"

    def __init__(self, status, path, reason=PERM, content=""):
        Exception.__init__(self, status, path, reason, content)
        self.status = status
        self.path = path
        self.reason = reason
        self.content = content

    def __str__(self):
        tail = ""
        if self.content and "\n" in self.content:
            tail = self.FMT_TAIL_M % self.content
        elif self.content:
            tail = self.FMT_TAIL_1 % self.content
        return self.FMT_HEAD % (self.reason, self.status, self.path) + tail

class InfoFileError(BadChange):
    """Error representing a bad or missing info file.

    Args:
        reason (str): a user-friendly message.
        exception (Exception): the exception encountered in loading the
            bad info file.
    """

    VALUE = "BAD VALUE IN INFO FILE"

    def __init__(self, reason=BadChange.PERM, exception=None):
        self.reason = reason
        self.exc = exception

    def __str__(self):
        if self.exc:
            return f"{self.reason}: {self.exc}"
        return self.reason

class BadChanges(Exception):
    """An error representing bad changes in a commit transaction."""

    def __str__(self):
        bad_changes = self.args[0]
        return "\n".join(str(bad_change) for bad_change in bad_changes)

class RosieSvnHook(object):
    """A parent class for hooks on a Rosie Subversion repository."""

    DATE_FMT = "%Y-%m-%d %H:%M:%S %Z"
    ID_CHARS_LIST = ["abcdefghijklmnopqrstuvwxyz"] * 2 + ["0123456789"] * 3
    LEN_ID = len(ID_CHARS_LIST)
    TRUNK = "trunk"
    INFO_FILE = "rose-suite.info"
    TRUNK_INFO_FILE = "trunk/rose-suite.info"
    ST_ADDED = "A"
    ST_DELETED = "D"
    ST_MODIFIED = "M"
    ST_UPDATED = "U"
    ST_EMPTY = " "

    def __init__(self, event_handler=None, popen=None):
        if event_handler is None:
            event_handler = Reporter()
        self.event_handler = event_handler
        if popen is None:
            popen = RosePopener(self.event_handler)
        self.popen = popen
        self.path = os.path.dirname(
            os.path.dirname(sys.modules["metomi.rosie"].__file__)
        )

    def _svnlook(self, *args):
        """Return the standard output from "svnlook"."""
        command = ["svnlook", *args]
        return self.popen(*command)[0]

    def _load_info(
        self,
        repos,
        sid,
        branch=None,
        revision=None,
        transaction=None,
        allow_popen_err=False,
    ):
        """Load info file from branch_path in repos @revision.

        Returns a ConfigNode for the "rose-suite.info" of a suite at a
        particular revision or transaction.

        Args:
            repos (str): The path of the repository.
            sid (str): The Rosie suite unique identifier, e.g. "ay327".
            branch (str): The branch that the info file is under.
            revision (int, str): A commit revision number. Cannot be used with
                transaction.
            transaction (str): A commit transaction identifer. Cannot be used
                with revision.
            allow_popen_err (bool): If True, return None if a RosePopenError
                occurs during svnlook command.
        """
        if not branch:
            branch = self.TRUNK
        commit_opts = []
        if transaction is not None:
            if revision is not None:
                raise ValueError(
                    f"Cannot load transaction {transaction} and "
                    f"revision {revision} at the same time"
                )
            commit_opts = ["-t", transaction]
        if revision is not None:
            commit_opts = ["-r", str(revision)]
        info_file_path = os.path.join("/".join(sid), branch, self.INFO_FILE)
        t_handle = TemporaryFile()
        try:
            t_handle.write(
                self._svnlook(
                    "cat", repos, info_file_path, *commit_opts
                ).encode()
            )
        except UnicodeDecodeError as err:
            raise ConfigDecodeError(info_file_path, err)
        except RosePopenError as err:
            if allow_popen_err:
                return None
            raise err
        t_handle.seek(0)
        config_node = ConfigLoader(allow_sections=False).load(t_handle)
        t_handle.close()
        return config_node


import sys
import logging
from abc import ABCMeta, abstractmethod
from math import ceil, floor
from typing import Tuple, List, Union, Dict
from json_checker import And, Or

import numpy as np
import xarray as xr
from scipy.ndimage import binary_dilation

from pandora.img_tools import shift_right_img

class AbstractMatchingCost:
    """
    Abstract Matching Cost class
    """

    __metaclass__ = ABCMeta

    matching_cost_methods_avail: Dict = {}
    _subpix = None
    _window_size = None
    cfg = None
    _band = None
    _step_col = None

    # Default configuration, do not change these values
    _WINDOW_SIZE = 5
    _SUBPIX = 1
    _BAND = None
    _STEP_COL = 1

    # Matching cost schema confi
    schema = {
        "subpix": And(int, lambda input: input > 0 and ((input % 2) == 0) or input == 1),
        "band": Or(str, lambda input: input is None),
        "step": And(int, lambda y: y >= 1),
    }

    def __new__(cls, **cfg: Union[str, int]):
        """
        Return the plugin associated with the matching_cost_method given in the configuration

        :param cfg: configuration {'matching_cost_method': value}
        :type cfg: dictionary
        """

        if cls is AbstractMatchingCost:
            if isinstance(cfg["matching_cost_method"], str):
                try:
                    return super(AbstractMatchingCost, cls).__new__(
                        cls.matching_cost_methods_avail[cfg["matching_cost_method"]]
                    )
                except:
                    raise KeyError("No matching cost method named {} supported".format(cfg["matching_cost_method"]))
            else:
                if isinstance(cfg["matching_cost_method"], unicode):  # type:ignore # pylint:disable=undefined-variable
                    # creating a plugin from registered short name given as unicode (py2 & 3 compatibility)
                    try:
                        return super(AbstractMatchingCost, cls).__new__(
                            cls.matching_cost_methods_avail[cfg["matching_cost_method"].encode("utf-8")]
                        )
                    except:
                        raise KeyError("No matching cost method named {} supported".format(cfg["matching_cost_method"]))
        else:
            return super(AbstractMatchingCost, cls).__new__(cls)
        return None

    @classmethod
    def register_subclass(cls, short_name: str, *args):
        """
        Allows to register the subclass with its short name

        :param short_name: the subclass to be registered
        :type short_name: string
        :param args: allows to register one plugin that contains different methods
        """

        def decorator(subclass):
            """
            Registers the subclass in the available methods

            :param subclass: the subclass to be registered
            :type subclass: object
            """
            cls.matching_cost_methods_avail[short_name] = subclass
            for arg in args:
                cls.matching_cost_methods_avail[arg] = subclass
            return subclass

        return decorator

    @abstractmethod
    def desc(self) -> None:
        """
        Describes the matching cost method
        :return: None
        """
        print("Matching cost description")

    def instantiate_class(self, **cfg: Union[str, int]) -> None:
        """
        :param cfg: optional configuration,  {'window_size': int, 'subpix': int,
                                                'band': str}
        :type cfg: dictionary
        :return: None
        """
        self.cfg = self.check_conf(**cfg)  # type: ignore
        self._window_size = int(self.cfg["window_size"])
        self._subpix = int(self.cfg["subpix"])
        self._band = self.cfg["band"]
        self._step_col = int(self.cfg["step"])

    def check_conf(self, **cfg: Dict[str, Union[str, int]]) -> Dict[str, Union[str, int]]:
        """
        Add default values to the dictionary if there are missing elements and check if the dictionary is correct

        :param cfg: matching cost configuration
        :type cfg: dict
        :return cfg: matching cost configuration updated
        :rtype: dict
        """

        # Give the default value if the required element is not in the conf
        if "window_size" not in cfg:
            cfg["window_size"] = self._WINDOW_SIZE  # type: ignore
        if "subpix" not in cfg:
            cfg["subpix"] = self._SUBPIX  # type: ignore
        if "band" not in cfg:
            cfg["band"] = self._BAND

        if "pandora2d" not in sys.modules:
            if "step" in cfg and cfg["step"] != 1:
                logging.error("Step parameter cannot be different from 1")
                sys.exit(1)
        if "step" not in cfg:
            cfg["step"] = self._STEP_COL  # type: ignore

        return cfg  # type: ignore

    def check_band_input_mc(self, img_left: xr.Dataset, img_right: xr.Dataset) -> None:
        """
        Check coherence band parameter between inputs and matching cost step

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param img_right: right Dataset  containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_right: xarray.Dataset
        :return: None
        """
        if self._band is not None:
            try:
                list(img_right.band_im.data)
            except AttributeError:
                logging.error("Right dataset is monoband: %s band cannot be selected", self._band)
                sys.exit(1)
            try:
                list(img_left.band_im.data)
            except AttributeError:
                logging.error("Left dataset is monoband: %s band cannot be selected", self._band)
                sys.exit(1)
            if (self._band not in list(img_right.band_im.data)) or (self._band not in list(img_left.band_im.data)):
                logging.error("Wrong band instantiate : %s not in img_left or img_right", self._band)
                sys.exit(1)
        else:
            try:
                list(img_right.band_im.data)
            except AttributeError:
                return
            try:
                list(img_left.band_im.data)
            except AttributeError:
                return
            if (img_right.band_im.data is not None) or (img_left.band_im.data is not None):
                logging.error("Band must be instantiated in matching cost step")
                sys.exit(1)

    @abstractmethod
    def compute_cost_volume(
        self, img_left: xr.Dataset, img_right: xr.Dataset, grid_disp_min: np.ndarray, grid_disp_max: np.ndarray
    ) -> xr.Dataset:
        """
        Computes the cost volume for a pair of images

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param img_right: right Dataset  containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_right: xarray.Dataset
        :param grid_disp_min: minimum disparity
        :type grid_disp_min: np.ndarray
        :param grid_disp_max: maximum disparity
        :type grid_disp_max: np.ndarray
        :return: the cost volume dataset with the data variables:

                - cost_volume 3D xarray.DataArray (row, col, disp)
        :rtype: xarray.Dataset
        """

    def allocate_costvolume(
        self,
        img_left: xr.Dataset,
        subpix: int,
        disp_min: int,
        disp_max: int,
        window_size: int,
        metadata: dict,
        np_data: np.ndarray = None,
    ) -> xr.Dataset:
        """
        Allocate the cost volume

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param subpix: subpixel precision = (1 or 2 or 4)
        :type subpix: int
        :param disp_min: minimum disparity
        :type disp_min: int
        :param disp_max: maximum disparity
        :type disp_max: int
        :param window_size: size of the aggregation window
        :type window_size: int, odd number
        :param metadata: dictionary storing arbitrary metadata
        :type metadata: dictionary
        :param step: step
        :type step: int
        :param np_data: the array’s data
        :type np_data: 3D numpy array, dtype=np.float32
        :return: the dataset cost volume with the cost_volume :

                - cost_volume 3D xarray.DataArray (row, col, disp)
        :rtype: xarray.Dataset
        """
        c_row = img_left["im"].coords["row"]
        c_col = img_left["im"].coords["col"]

        # First pixel in the image that is fully computable (aggregation windows are complete)
        row = np.arange(c_row[0], c_row[-1] + 1)  # type: np.ndarray
        col = np.arange(c_col[0], c_col[-1] + 1, self._step_col)  # type: np.ndarray

        disparity_range = AbstractMatchingCost.get_disparity_range(disp_min, disp_max, subpix)

        # Create the cost volume
        if np_data is None:
            np_data = np.zeros((len(row), len(col), len(disparity_range)), dtype=np.float32)

        cost_volume = xr.Dataset(
            {"cost_volume": (["row", "col", "disp"], np_data)},
            coords={"row": row, "col": col, "disp": disparity_range},
        )
        cost_volume.attrs = metadata

        cost_volume.attrs["crs"] = img_left.attrs["crs"]
        cost_volume.attrs["transform"] = img_left.attrs["transform"]

        cost_volume.attrs["window_size"] = window_size

        return cost_volume

    @staticmethod
    def get_disparity_range(disparity_min: int, disparity_max: int, subpix: int) -> np.ndarray:
        """
        Build disparity range and return it.

        :param disparity_min: minimum disparity
        :type disparity_min: int
        :param disparity_max: maximum disparity
        :type disparity_max: int
        :param subpix: subpixel precision = (1 or 2 or 4)
        :return: disparity range
        :rtype: np.ndarray
        """
        if subpix == 1:
            disparity_range = np.arange(disparity_min, disparity_max + 1)
        else:
            disparity_range = np.arange(disparity_min, disparity_max, 1 / float(subpix), dtype=np.float64)
            disparity_range = np.append(disparity_range, [disparity_max])
        return disparity_range

    def point_interval(
        self, img_left: xr.Dataset, img_right: xr.Dataset, disp: float
    ) -> Tuple[Tuple[int, int], Tuple[int, int]]:
        """
        Computes the range of points over which the similarity measure will be applied

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param img_right: right Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_right: xarray.Dataset
        :param disp: current disparity
        :type disp: float
        :return: the range of the left and right image over which the similarity measure will be applied
        :rtype: tuple
        """
        nx_left = int(img_left.dims["col"] / self._step_col)
        nx_right = int(img_right.dims["col"] / self._step_col)

        # range in the left image
        point_p = (max(0 - disp, 0), min(nx_left - disp, nx_left))
        # range in the right image
        point_q = (max(0 + disp, 0), min(nx_right + disp, nx_right))

        # Because the disparity can be floating
        if disp < 0:
            point_p = (int(ceil(point_p[0])), int(ceil(point_p[1])))
            point_q = (int(ceil(point_q[0])), int(ceil(point_q[1])))
        else:
            point_p = (int(floor(point_p[0])), int(floor(point_p[1])))
            point_q = (int(floor(point_q[0])), int(floor(point_q[1])))

        return point_p, point_q

    @staticmethod
    def masks_dilatation(
        img_left: xr.Dataset, img_right: xr.Dataset, window_size: int, subp: int
    ) -> Tuple[xr.DataArray, List[xr.DataArray]]:
        """
        Return the left and right mask with the convention :
            - Invalid pixels are nan
            - No_data pixels are nan
            - Valid pixels are 0

        Apply dilation on no_data : if a pixel contains a no_data in its aggregation window, then the central pixel
        becomes no_data

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param img_right: right Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_right: xarray.Dataset
        :param window_size: window size of the measure
        :type window_size: int
        :param subp: subpixel precision = (1 or 2 or 4)
        :type subp: int
        :return: the left mask and the right masks:

                - left mask :  xarray.DataArray msk 2D(row, col)
                - right mask :  xarray.DataArray msk 2D(row, col)
                - right mask shifted :  xarray.DataArray msk 2D(row, shifted col by 0.5)
        :rtype: tuple (left mask, list[right mask, right mask shifted by 0.5])
        """
        # Create the left mask with the convention : 0 = valid, nan = invalid and no_data
        if "msk" in img_left.data_vars:
            dilatate_left_mask = np.zeros(img_left["msk"].shape)
            # Invalid pixels are nan
            dilatate_left_mask[
                np.where(
                    (img_left["msk"].data != img_left.attrs["valid_pixels"])
                    & (img_left["msk"].data != img_left.attrs["no_data_mask"])
                )
            ] = np.nan
            # Dilatation : pixels that contains no_data in their aggregation window become no_data = nan
            dil = binary_dilation(
                img_left["msk"].data == img_left.attrs["no_data_mask"],
                structure=np.ones((window_size, window_size)),
                iterations=1,
            )
            dilatate_left_mask[dil] = np.nan
        else:
            # All pixels are valid
            dilatate_left_mask = np.zeros((img_left.dims["row"], img_left.dims["col"]))

        # Create the right mask with the convention : 0 = valid, nan = invalid and no_data
        if "msk" in img_right.data_vars:
            dilatate_right_mask = np.zeros(img_right["msk"].shape)
            # Invalid pixels are nan
            dilatate_right_mask[
                np.where(
                    (img_right["msk"].data != img_right.attrs["valid_pixels"])
                    & (img_right["msk"].data != img_right.attrs["no_data_mask"])
                )
            ] = np.nan
            # Dilatation : pixels that contains no_data in their aggregation window become no_data = nan
            dil = binary_dilation(
                img_right["msk"].data == img_right.attrs["no_data_mask"],
                structure=np.ones((window_size, window_size)),
                iterations=1,
            )
            dilatate_right_mask[dil] = np.nan
        else:
            # All pixels are valid
            dilatate_right_mask = np.zeros((img_left.dims["row"], img_left.dims["col"]))

        ny_, nx_ = img_left.dims["row"], img_left.dims["col"]

        row = np.arange(0, ny_)
        col = np.arange(0, nx_)

        # Shift the right mask, for sub-pixel precision. If an no_data or invalid pixel was used to create the
        # sub-pixel point, then the sub-pixel point is invalidated / no_data.
        dilatate_right_mask_shift = xr.DataArray()

        if subp != 1:
            # Since the interpolation of the right image is of order 1, the shifted right mask corresponds
            # to an aggregation of two columns of the dilated right mask

            str_row, str_col = dilatate_right_mask.strides
            shape_windows = (
                dilatate_right_mask.shape[0],
                dilatate_right_mask.shape[1] - 1,
                2,
            )

            strides_windows = (str_row, str_col, str_col)
            aggregation_window = np.lib.stride_tricks.as_strided(dilatate_right_mask, shape_windows, strides_windows)
            dilatate_right_mask_shift = np.sum(aggregation_window, 2)

            # Whatever the sub-pixel precision, only one sub-pixel mask is created,
            # since 0.5 shifted mask == 0.25 shifted mask
            col_shift = np.arange(0 + 0.5, nx_ - 1, step=1)
            dilatate_right_mask_shift = xr.DataArray(
                dilatate_right_mask_shift, coords=[row, col_shift], dims=["row", "col"]
            )

        dilatate_left_mask_xr = xr.DataArray(dilatate_left_mask, coords=[row, col], dims=["row", "col"])
        dilatate_right_mask_xr = xr.DataArray(dilatate_right_mask, coords=[row, col], dims=["row", "col"])

        return dilatate_left_mask_xr, [dilatate_right_mask_xr, dilatate_right_mask_shift]

    @staticmethod
    def get_min_max_from_grid(disp_min: np.ndarray, disp_max: np.ndarray) -> Tuple[int, int]:
        """
        Find the smallest disparity present in disp_min, and the highest disparity present in disp_max

        :param disp_min: minimum disparity
        :type disp_min: np.ndarray
        :param disp_max: maximum disparity
        :type disp_max: np.ndarray
        :return: dmin_min: the smallest disparity in disp_min, dmax_max: the highest disparity in disp_max
        :rtype: Tuple(int, int)
        """
        return int(np.nanmin(disp_min)), int(np.nanmax(disp_max))

    def cv_masked(
        self,
        img_left: xr.Dataset,
        img_right: xr.Dataset,
        cost_volume: xr.Dataset,
        disp_min: np.ndarray,
        disp_max: np.ndarray,
    ) -> None:
        """
        Masks the cost volume :
            - costs which are not inside their disparity range, are masked with a nan value
            - costs of invalid_pixels (invalidated by the input image mask), are masked with a nan value
            - costs of no_data pixels, are masked with a nan value. If a valid pixel contains a no_data in its
                aggregation window, then the cost of the central pixel is masked with a nan value

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param img_right: right Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_right: xarray.Dataset
        :param cost_volume: the cost_volume DataSet with the data variables:

                - cost_volume 3D xarray.DataArray (row, col, disp)
        :type cost_volume: xarray.Dataset
        :param disp_min: minimum disparity
        :type disp_min: np.ndarray
        :param disp_max: maximum disparity
        :type disp_max: np.ndarray
        :return: None
        """
        ny_, nx_, nd_ = cost_volume["cost_volume"].shape

        dmin, _ = self.get_min_max_from_grid(disp_min, disp_max)

        # ----- Masking invalid pixels -----

        # Contains the shifted right images
        img_right_shift = shift_right_img(img_right, self._subpix, self._band)  # type: ignore

        # Computes the validity mask of the cost volume : invalid pixels or no_data are masked with the value nan.
        # Valid pixels are = 0
        mask_left, mask_right = self.masks_dilatation(img_left, img_right, self._window_size, self._subpix)

        for disp in cost_volume.coords["disp"].data:
            i_right = int((disp % 1) * self._subpix)
            point_p, point_q = self.point_interval(img_left, img_right_shift[i_right], disp)

            # Point interval in the left image
            p_mask = (point_p[0], point_p[1])
            # Point interval in the right image
            q_mask = (point_q[0], point_q[1])

            i_mask_right = min(1, i_right)
            dsp = int((disp - dmin) * self._subpix)

            # Invalid costs in the cost volume
            cost_volume["cost_volume"].data[:, p_mask[0] : p_mask[1], dsp] += (
                mask_right[i_mask_right].data[:, q_mask[0] : q_mask[1]] + mask_left.data[:, p_mask[0] : p_mask[1]]
            )

        # ----- Masking disparity range -----

        # Fixed range of disparities
        # Disparity range may be one size bigger in y axis
        if disp_min.shape[0] > ny_:
            disp_min = disp_min[0:ny_, :]
            disp_max = disp_max[0:ny_, :]
        if disp_min.shape[1] > nx_:
            disp_min = disp_min[:, 0:nx_]
            disp_max = disp_max[:, 0:nx_]

        # Mask the costs computed with a disparity lower than disp_min and higher than disp_max
        for dsp in range(nd_):
            masking = np.where(
                np.logical_or(
                    cost_volume.coords["disp"].data[dsp] < disp_min,
                    cost_volume.coords["disp"].data[dsp] > disp_max,
                )
            )
            cost_volume["cost_volume"].data[masking[0], masking[1], dsp] = np.nan

    def allocate_numpy_cost_volume(self, img_left: xr.Dataset, disparity_range: Union[np.ndarray, List]) -> np.ndarray:
        """
        Allocate the numpy cost volume cv = (disp, col, row), for efficient memory management

        :param img_left: left Dataset image containing :

                - im: 2D (row, col) or 3D (band_im, row, col) xarray.DataArray float32
                - disparity (optional): 3D (disp, row, col) xarray.DataArray float32
                - msk (optional): 2D (row, col) xarray.DataArray int16
                - classif (optional): 3D (band_classif, row, col) xarray.DataArray int16
                - segm (optional): 2D (row, col) xarray.DataArray int16
        :type img_left: xarray.Dataset
        :param disparity_range: disparity range
        :type disparity_range: np.ndarray
        :param offset_row_col: offset in row and col
        :type offset_row_col: int
        :return: the cost volume dataset , with the data variables:

                - cost_volume 3D xarray.DataArray (row, col, disp)
        :rtype: xarray.Dataset
        """

        return np.full(
            (len(disparity_range), int(img_left.dims["col"] / self._step_col), int(img_left.dims["row"])),
            np.nan,
            dtype=np.float32,
        )

    @staticmethod
    def crop_cost_volume(cost_volume: np.ndarray, offset: int = 0) -> np.ndarray:
        """
        Return a cropped view of cost_volume.

        If offset, do not consider border position for cost computation.
        :param cost_volume: cost volume to crop
        :type cost_volume: np.ndarray
        :param offset: offset used to crop cost volume
        :type offset: int
        :return: cropped view of cost_volume.
        :rtype: np.ndarray
        """
        return cost_volume[:, offset:-offset, offset:-offset] if offset else cost_volume


from os import listdir, remove

from discord.ext import commands, tasks

from utils import basic
from utils.decos import handle_errors

def setup(bot):
    bot.add_cog(loops(bot))

class loops(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.cache_cleanup.start()

    def cog_unload(self):
        self.cache_cleanup.cancel()

    @tasks.loop(seconds=60)
    @handle_errors
    async def cache_cleanup(self):
        cache_size = basic.get_size("cache")
        if cache_size >= 2000000000:
            cache_folder = listdir("cache")
            cache_folder.sort(reverse=False, key=lambda x: int(''.join(filter(str.isdigit, x))))
            cache_folder = cache_folder[:1000]

            for file in cache_folder:
                remove(f"cache/{file}")

            await self.bot.cache.bulk_remove(int(cache_file[:-8]) for cache_file in cache_folder)

    @cache_cleanup.before_loop
    async def before_loops(self):
        await self.bot.wait_until_ready()


import datetime
from typing import Any, Dict, List, Type, TypeVar

import attr
from dateutil.parser import isoparse

T = TypeVar("T", bound="TablePublication")

@attr.s(auto_attribs=True)
class TablePublication:
    """
    Attributes:
        created_at (datetime.datetime):
        export_job_id (int):
        id (int):
    """

    created_at: datetime.datetime
    export_job_id: int
    id: int
    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        created_at = self.created_at.isoformat()

        export_job_id = self.export_job_id
        id = self.id

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "createdAt": created_at,
                "exportJobId": export_job_id,
                "id": id,
            }
        )

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        created_at = isoparse(d.pop("createdAt"))

        export_job_id = d.pop("exportJobId")

        id = d.pop("id")

        table_publication = cls(
            created_at=created_at,
            export_job_id=export_job_id,
            id=id,
        )

        table_publication.additional_properties = d
        return table_publication

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


#!/usr/bin/env python
from .app import app, db
from .db import Job
from flask import session, abort, request, flash, redirect, render_template, url_for, g, jsonify

@app.route('/login', methods=['GET', 'POST'])
def login():
    error = None
    if request.method == 'POST':
        if request.form['username'] != app.config['USERNAME']:
            error = 'Invalid username'
        elif request.form['password'] != app.config['PASSWORD']:
            error = 'Invalid password'
        else:
            session['logged_in'] = True
            flash('You were logged in', 'success')
            return redirect(url_for('show_jobs'))
    return render_template('login.html', error=error)

@app.route('/logout')
def logout():
    session.pop('logged_in', None)
    flash('You were logged out', 'info')
    return redirect(url_for('show_jobs'))

@app.route('/jobs/', methods=['POST'])
def create_job():
    if not session.get('logged_in'):
        abort(401)

    job = Job(**request.form.to_dict(flat=True))
    db.session.add(job)
    db.session.commit()

    flash('New job was successfully created', 'success')
    return redirect(url_for('show_jobs'))

@app.route('/jobs/', methods=['GET'])
def show_jobs():
    jobs = Job.query.all()
    return render_template('show_jobs.html', jobs=jobs)

@app.route('/jobs/<int:job_id>', methods=['DELETE'])
def delete_job(job_id):
    try:
        Job.query(id=job_id).delete()
    except:
        return jsonify(False)
    return jsonify(True)

if __name__ == "__main__":
    app.run()


#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""

@Author : kazi
@File : subsequences.py
@Time : 3/27/22 3:55 PM
@Desc:
"""

import itertools as it

def sub_sequences(string):
    """
    "Print all sub sequence of a string using itertools.
    Args:
        string:

    Returns:

    """
    result = []
    for i in range(1, len(string) + 1):
        result.append(list(it.combinations(string, i)))
    for c in result:
        for t in c:
            print(''.join(t), end=' ')

'''
s = 'ab'
possible subsequence = a, b, c, ab, ac, bc, abc
'''

def main():
    """
    main function to drive code
    Returns:

    """
    sub_sequences('abc')

if __name__ == "__main__":
    main()


import logging
import scipy
import time
import json
from sentence_transformers import SentenceTransformer
import threading
import zmq
import zmq.decorators as zmqd
# from multiprocessing import Process

class QTCosine(threading.Thread):
    def __init__(self, model_path, stop_sign:threading.Event, recv_port=1991, send_port=1992, classid=0, logger=logging.getLogger("QTCosine")):
        super(QTCosine, self).__init__()
        self.model_path = model_path
        self.model = SentenceTransformer(model_path)
        self.logger = logger
        self.id = classid
        self.recv_port = recv_port
        self.send_port = send_port
        self.stop_sign = stop_sign

    def run(self):
        self._run()

    @zmqd.context()
    @zmqd.socket(zmq.PULL)
    @zmqd.socket(zmq.PUSH)
    def _run(self,_, receiver, sender):
        receiver.connect('tcp://127.0.0.1:%d'%(self.recv_port))
        sender.connect('tcp://127.0.0.1:%d'%(self.send_port))
        self.logger.info("QTCosine[%d] connect recv_port:%d, send_por:%d" % (self.id, self.recv_port, self.send_port))

        while self.stop_sign.is_set() == False:
            events = receiver.poll(1000)
            if events:
                data = receiver.recv().decode("utf-8")
                json_part = json.loads(data.split('\t')[0])
                query = json_part["query"]
                title = json_part["title"]
                texts = [query, title]
                embeddings = self.model.encode(texts, convert_to_numpy=True)
                cosine = 1 - scipy.spatial.distance.cosine(embeddings[0], embeddings[1])

                res_data = "%s\t%f" % (data, cosine)
                res_data = res_data.encode("utf-8")
                sender.send(res_data)

        self.logger.info("QTCosine[%d] stop run" % (self.id))

class DataReader(threading.Thread):
    def __init__(self, input_file, stop_sign:threading.Event, send_port=1991, logger=logging.getLogger(), encoding="utf-8"):
        super(DataReader, self).__init__()
        self.input_file = input_file
        self.stop_sign = stop_sign
        self.encoding = encoding
        self.send_port = send_port
        self.logger = logger

    def run(self):
        self._run()

    @zmqd.context()
    @zmqd.socket(zmq.PUSH)
    def _run(self, _, sender):
        sender.bind('tcp://127.0.0.1:%d'%(self.send_port))
        self.logger.info("DataReader bind send_port:%d" % (self.send_port))
        rfp = open(input_file, "r", encoding=self.encoding)
        while self.stop_sign.is_set() == False:
            for line in rfp:
                data = line.rstrip('\n').encode("utf-8")
                sender.send(data)
        rfp.close()
        self.logger.info("DataReader stop")

class DataReceiver(threading.Thread):
    def __init__(self, output_file, stop_sign:threading.Event, recv_port=1992, logger=logging.getLogger(), encoding="utf-8"):
        super(DataReceiver, self).__init__()
        self.output_file = output_file
        self.stop_sign = stop_sign
        self.encoding = encoding
        self.recv_port = recv_port
        self.logger = logger

    def run(self):
        self._run()

    @zmqd.context()
    @zmqd.socket(zmq.PULL)
    def _run(self, _, receiver):
        receiver.bind('tcp://127.0.0.1:%d'%(self.recv_port))
        self.logger.info("DataReceiver bind recv_port:%d" % (self.recv_port))
        last_time = time.time()
        wfp = open(self.output_file, "w", encoding=self.encoding)
        count = 0
        st = time.time()
        while self.stop_sign.is_set() == False:
            events = receiver.poll(timeout=1000)
            if events:
                data = receiver.recv()
                data = data.decode("utf-8")
                wfp.write("%s\n" % (data))
                count += 1
                last_time = time.time()
            else:
                now = time.time()
                if int(now - last_time) > 20:
                    self.stop_sign.set()
                    self.logger.info("set stop_sign")
            ed = time.time()
            if int(ed - st) >= 60:
                st = time.time()
                speed = float(count) / 60
                self.logger.info("[check speed] %d" % (speed))
                count = 0
        wfp.close()
        self.logger.info("DataReceiver stop")

def predict_query_title_score(input_file, output_file, model_path, thread_num=3):
    logging.basicConfig(level=logging.INFO,
                        format="[%(levelname).1s %(asctime)s] %(message)s",
                        datefmt="%Y-%m-%d_%H:%M:%S")
    logger = logging.getLogger()

    stop_sign = threading.Event()
    data_read_port = 1991
    data_recv_port = 1992
    data_reader = DataReader(input_file, stop_sign, data_read_port, logger=logger)
    data_receiver = DataReceiver(output_file, stop_sign, data_recv_port, logger=logger)

    worker_list = []
    for i in range(thread_num):
        worker = QTCosine(model_path, stop_sign, recv_port=data_read_port, send_port=data_recv_port, classid=i, logger=logger)
        worker_list.append(worker)

    for w in worker_list:
        w.start()

    data_reader.start()
    data_receiver.start()

    for w in worker_list:
        w.join()

    data_reader.join()
    data_receiver.join()

if __name__ == "__main__":
    input_file = "/search/odin/liruihong/websearch/websearch_data/data_20part"
    output_file = "/search/odin/liruihong/websearch/res_data/data_20part_qtfeature"
    model_path = "/search/odin/liruihong/websearch/model_data/fuse-data-epoch5-utf8"
    thread_num = 30
    predict_query_title_score(input_file, output_file, model_path, thread_num = thread_num)


# -*- coding: utf-8 -*-
"""Setup info for building OpenPype 3.0."""
import os
import re
import platform
import distutils.spawn
from pathlib import Path

from cx_Freeze import setup, Executable
from sphinx.setup_command import BuildDoc

openpype_root = Path(os.path.dirname(__file__))

def validate_thirdparty_binaries():
    """Check existence of thirdpart executables."""
    low_platform = platform.system().lower()
    binary_vendors_dir = os.path.join(
        openpype_root,
        "vendor",
        "bin"
    )

    error_msg = (
        "Missing binary dependency {}. Please fetch thirdparty dependencies."
    )
    # Validate existence of FFmpeg
    ffmpeg_dir = os.path.join(binary_vendors_dir, "ffmpeg", low_platform)
    if low_platform == "windows":
        ffmpeg_dir = os.path.join(ffmpeg_dir, "bin")
    ffmpeg_executable = os.path.join(ffmpeg_dir, "ffmpeg")
    ffmpeg_result = distutils.spawn.find_executable(ffmpeg_executable)
    if ffmpeg_result is None:
        raise RuntimeError(error_msg.format("FFmpeg"))

    # Validate existence of OpenImageIO (not on MacOs)
    oiio_tool_path = None
    if low_platform == "linux":
        oiio_tool_path = os.path.join(
            binary_vendors_dir,
            "oiio",
            low_platform,
            "bin",
            "oiiotool"
        )
    elif low_platform == "windows":
        oiio_tool_path = os.path.join(
            binary_vendors_dir,
            "oiio",
            low_platform,
            "oiiotool"
        )
    oiio_result = None
    if oiio_tool_path is not None:
        oiio_result = distutils.spawn.find_executable(oiio_tool_path)
        if oiio_result is None:
            raise RuntimeError(error_msg.format("OpenImageIO"))

# Give ability to skip vaidation
if not os.getenv("SKIP_THIRD_PARTY_VALIDATION"):
    validate_thirdparty_binaries()

version = {}

with open(openpype_root / "openpype" / "version.py") as fp:
    exec(fp.read(), version)

version_match = re.search(r"(\d+\.\d+.\d+).*", version["__version__"])
__version__ = version_match.group(1)

low_platform_name = platform.system().lower()
IS_WINDOWS = low_platform_name == "windows"
IS_LINUX = low_platform_name == "linux"
IS_MACOS = low_platform_name == "darwin"

base = None
if IS_WINDOWS:
    base = "Win32GUI"

# -----------------------------------------------------------------------
# build_exe
# Build options for cx_Freeze. Manually add/exclude packages and binaries

install_requires = [
    "appdirs",
    "cx_Freeze",
    "keyring",
    "clique",
    "jsonschema",
    "pathlib2",
    "pkg_resources",
    "PIL",
    "pymongo",
    "pynput",
    "jinxed",
    "blessed",
    "Qt",
    "qtpy",
    "speedcopy",
    "googleapiclient",
    "httplib2",
    # Harmony implementation
    "filecmp",
    "dns",
    # Python defaults (cx_Freeze skip them by default)
    "dbm",
    "sqlite3",
    "dataclasses",
    "timeit"
]

includes = []
# WARNING: As of cx_freeze there is a bug?
# when this is empty, its hooks will not kick in
# and won't clean platform irrelevant modules
# like dbm mentioned above.
excludes = [
    "openpype"
]
bin_includes = [
    "vendor"
]
include_files = [
    "igniter",
    "openpype",
    "LICENSE",
    "README.md"
]

if IS_WINDOWS:
    install_requires.extend([
        # `pywin32` packages
        "win32ctypes",
        "win32comext",
        "pythoncom"
    ])

icon_path = openpype_root / "igniter" / "openpype.ico"
mac_icon_path = openpype_root / "igniter" / "openpype.icns"

build_exe_options = dict(
    packages=install_requires,
    includes=includes,
    excludes=excludes,
    bin_includes=bin_includes,
    include_files=include_files,
    optimize=0
)

bdist_mac_options = dict(
    bundle_name=f"OpenPype {__version__}",
    iconfile=mac_icon_path
)

executables = [
    Executable(
        "start.py",
        base=base,
        target_name="openpype_gui",
        icon=icon_path.as_posix()
    ),
    Executable(
        "start.py",
        base=None,
        target_name="openpype_console",
        icon=icon_path.as_posix()
    ),
]

if IS_LINUX:
    executables.append(
        Executable(
            "app_launcher.py",
            base=None,
            target_name="app_launcher",
            icon=icon_path.as_posix()
        )
    )

setup(
    name="OpenPype",
    version=__version__,
    description="OpenPype",
    cmdclass={"build_sphinx": BuildDoc},
    options={
        "build_exe": build_exe_options,
        "bdist_mac": bdist_mac_options,
        "build_sphinx": {
            "project": "OpenPype",
            "version": __version__,
            "release": __version__,
            "source_dir": (openpype_root / "docs" / "source").as_posix(),
            "build_dir": (openpype_root / "docs" / "build").as_posix()
        }
    },
    executables=executables,
    packages=[]
)


import numpy as np
import matplotlib.pyplot as plt
import scipy.io.wavfile as wav
import wave
import sounddevice as sd
import glob
import os.path

myPath = 'C:/python/kkmmoouu/fft/wavsamples/realuse/roop2/why'
myExt = '*.wav'
freqframe = []
frqs = [70, 104, 150, 3700, 3260, 6813]

for filename in glob.glob(os.path.join(myPath, myExt)):
    rate, data0 = wav.read(filename)
    data1 = data0
    original = wave.open(filename, 'rb')
    nChannel = original.getnchannels()
    length = original.getnframes()
    original.close()

    if nChannel == 2:
        data2 = data0[:, 0]
        spectre = np.fft.fft(data2)
        freq = np.fft.fftfreq(data2.size, 1/rate)
        mask = 0 < freq
    else:
        spectre = np.fft.fft(data0)
        freq = np.fft.fftfreq(data0.size, 1 / rate)
        mask = 0 < freq

    maxamp = np.abs(np.max(spectre[mask]))

    #보통 500만대고 317만이 최소다 250만 ㄱㄱ
    #sliding window 써보자
    # sliding window
     #avgspectre = []
     #slicerange = 300
     #for i in range(0, np.alen(freq)-(slicerange*2), slicerange):
     #    j = i
     #    spectresum = 0
     #    for j in range(j, j+((slicerange*2)-1), 1):
     #        spectresum = spectresum + np.abs(int(spectre[j]))
     #
     #    avgspectre.append(spectresum/200)
     #
     #for i in range(0, np.alen(freq) - (slicerange*2), slicerange):
     #    j = i
     #    for j in range(j, j+((slicerange*2)-1), 1):
     #        spectre[j] = avgspectre[int(i/slicerange)]

    masks = freq[mask]
    number = 0
    frews = []

    for mask in masks:

        number = number + 1
        if np.abs(spectre[number]) == maxamp:
            maxfreq = round(freq[number], 2)

        if freq[number] < 2000:
            if spectre[number] < 3000000:
                spectre[number] = 0
                freq[number] = 0
        else:
            if spectre[number] < 450000:
                spectre[number] = 0
                freq[number] = 0
        spectre[number] = 0

        for num in range(0, 3):
            if freq[number] > 0:
                if abs(abs(frqs[num]) - freq[number]) < 7:
                    frews.append(frqs[num])

        for num in range(3, 6):
            if freq[number] > 0:
                if abs(abs(frqs[num]) - freq[number]) < 20:
                    frews.append(frqs[num])

    plt.title(filename + " " + np.str(maxfreq) + " hz")
    f = set(frews)
    print(f)

   # plt.title(filename + " " + np.str(maxfreq) + " hz, amp = " + np.str(maxamp))
    #freqframe.append((maxfreq))

    plt.plot(freq, np.abs(spectre), '.')
    plt.xlabel('freq')
    plt.ylabel('amp')
    plt.xlim(0, 10000)
    plt.ylim(0)
    plt.grid()
    plt.show()


import matplotlib.pyplot as plt
import numpy as np

from nerdart import Parser
from nerdart.utils import save_or_show

@save_or_show(__file__)
def plot(args):
    n_rectangles = args.n_rectangles

    x = np.random.random(n_rectangles)
    cm = plt.cm.autumn(np.linspace(0, 1, n_rectangles))
    ax = plt.subplot()

    ax.bar(x, np.tan(x), alpha=0.25, color=cm, edgecolor="k")

    ax.axis("off")

def main():
    parser = Parser()
    parser.add("-n", "--n-rectangles", type=int, default=100)
    args = parser.parse()
    plot(args)

if __name__ == "__main__":
    main()


#!/usr/bin/python

import os, argparse

supportArchs = [
  "x86_64",
  #"aarch64"
]

rootdir = ""
selectedArch = ""

rebuildFlag = False;

def mkdir(dirName : str) :
  os.system("mkdir -p " + rootdir + dirName)
  print("创建了文件夹[" + rootdir + dirName + "]")

def cp(src : str, dest : str) :
  ret = os.system("cp -f " + src + " " + rootdir + dest)
  if (ret != 0) :
    print("复制系统文件[" + rootdir + dest + "]失败,返回值[{0}]".format(ret))
    exit();
  print("复制系统文件[" + rootdir + dest + "]完成")

dependcies = [
  "build/kernel.sys",
  "boot/build/boot.efi",
  ]

dependciesDestinations = [
  "/EXOS/kernel.sys",
  "placeholder__BOOTLOADER",
  ]

def checkDependcies() -> bool :
  if rebuildFlag == True:
    return False
  print("检查依赖中...")
  for depFile in dependcies:
    if (not os.path.isfile(depFile)) :
      return False

  return True

if (len(dependcies) != len(dependciesDestinations)) :
  print("安装脚本依赖列表与目的地列表不对等!请检查依赖列表与目的地列表")

argparser = argparse.ArgumentParser(description="EXOS安装脚本")
argparser.add_argument("path", type=str, help="安装目录")
argparser.add_argument("--rebuild", action="store_true", help="强制重新构建", default=False)
argparser.add_argument("--arch", type=str, help="目标机器架构", default="x86_64")
args = argparser.parse_args()

rebuildFlag = args.rebuild
rootdir = args.path
selectedArch = args.arch

if selectedArch not in supportArchs:
  print("不支持的架构: " + selectedArch)
  exit(-1)

dependciesDestinations[dependciesDestinations.index("placeholder__BOOTLOADER")] = "/efi/boot/boot" + ["x64", "aa64"][supportArchs.index(selectedArch)] + ".efi"

if (not os.access(rootdir, os.W_OK))  :
  os.system("mkdir -p " + rootdir)
  if (not os.access(rootdir, os.W_OK))  :
    print("没有目标文件夹的写入权限!请尝试使用sudo再试一次")
    exit()

if (not checkDependcies()):
  if (os.system("cd boot;python ./build.py --arch " + selectedArch) != 0) :
    print("构建引导出错!!!")
    exit()
  print("引导构建完成")
  if (os.system("python ./build.py --arch " + selectedArch) != 0) :
    print("构建内核出错!!!")
    exit()
  print("内核构建完成")

mkdir("/EXOS")
mkdir("/efi/boot/")
mkdir("/efi/EXOS/")

for i in range(0, len(dependciesDestinations))  :
  cp(dependcies[i], dependciesDestinations[i])

print("安装完成")


from matplotlib import pyplot as plt
import numpy as np
students = {
    'Gautam':40,
    'Ashu':60,
    'Gaurav':50,
}
names = list(students.keys())
values = list(students.values())
plt.bar(names,values,color="green")
plt.title("Student Marks")
plt.xlabel("Names")
plt.ylabel("Marks")
plt.grid(True)
plt.show()


import timeit
from random import randint
import matplotlib.pyplot as plt

def geraLista(tam):
    lista = []
    while tam:
        lista.append(tam)
        tam-=1
    return lista

def desenhaGrafico(x,lista,xl = "Entradas", yl = "Y",nam="out"):
    plt.plot(x,lista, label = "Bubble Sort")
    plt.legend(bbox_to_anchor=(1, 1),bbox_transform=plt.gcf().transFigure)
    plt.ylabel(yl)
    plt.xlabel(xl)
    plt.savefig(nam)

graf_operacoes =[]
def bb(lista):
    count = 0
    for i in range(len(lista)):
        for k in range(i, len(lista)):
            count+=1
            if lista[i] > lista[k]:
                lista[i], lista[k] = lista[k], lista[i]
    graf_operacoes.append(count)

quant = [10000, 20000, 50000, 100000]
graf_tempo = []

for i in range(len(quant)):
    print(quant[i])
    lista = geraLista(quant[i])
    graf_tempo.append(timeit.timeit("bb({})".format(lista),setup="from __main__ import bb",number=1))

desenhaGrafico(quant,graf_tempo,"Tamanho", "Tempo", "saida_time")
desenhaGrafico(quant,graf_operacoes , "Tamanho", "Operações", "saida_operacoes")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings
import datetime
import main.models

class Migration(migrations.Migration):

    dependencies = [
        ('auth', '0006_require_contenttypes_0002'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Answer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('viewed_hint', models.BooleanField(default=False)),
                ('attempts', models.PositiveIntegerField(default=0)),
            ],
            options={
                'ordering': ('id',),
            },
        ),
        migrations.CreateModel(
            name='Exam',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('name', models.CharField(max_length=50)),
                ('info', models.TextField(blank=True)),
                ('time_limit', models.DurationField(verbose_name='Time limit to complete exam', default=datetime.timedelta(0))),
                ('shuffle_sections', models.BooleanField(verbose_name='Randomly shuffle sections', default=False)),
                ('can_attempt_again', models.BooleanField(verbose_name='Can a user attempt this exam multiple times', default=True)),
                ('author', models.TextField(blank=True)),
                ('comment', models.TextField(blank=True)),
                ('postinfo', models.TextField(blank=True)),
                ('attempt_group', models.ForeignKey(to='auth.Group', null=True, related_name='attexam_set')),
                ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL, null=True)),
                ('solutions_group', models.ForeignKey(to='auth.Group', null=True, related_name='solexam_set')),
            ],
        ),
        migrations.CreateModel(
            name='ExamAnswerSheet',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('start_time', models.DateTimeField(null=True)),
                ('end_time', models.DateTimeField(null=True)),
                ('exam', models.ForeignKey(to='main.Exam')),
                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
        ),
        migrations.CreateModel(
            name='McqAnswer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('answer', models.OneToOneField(to='main.Answer')),
            ],
            options={
                'ordering': ('id',),
            },
            bases=(models.Model, main.models.SpecialAnswer),
        ),
        migrations.CreateModel(
            name='McqAnswerToMcqOption',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('mcq_answer', models.ForeignKey(to='main.McqAnswer')),
            ],
        ),
        migrations.CreateModel(
            name='McqOption',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('title', models.CharField(max_length=30, blank=True)),
                ('text', models.TextField()),
                ('is_correct', models.BooleanField(default=False)),
            ],
            options={
                'ordering': ('id',),
            },
        ),
        migrations.CreateModel(
            name='McqQuestion',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('multicorrect', models.BooleanField(default=False)),
            ],
            options={
                'ordering': ('id',),
            },
            bases=(models.Model, main.models.SpecialQuestion),
        ),
        migrations.CreateModel(
            name='Question',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('title', models.CharField(max_length=120, blank=True)),
                ('text', models.TextField(blank=True)),
                ('hint', models.TextField(blank=True)),
                ('solution', models.TextField(blank=True)),
                ('comment', models.TextField(blank=True)),
                ('metadata', models.TextField(blank=True)),
            ],
            options={
                'ordering': ('id',),
            },
        ),
        migrations.CreateModel(
            name='Section',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('name', models.CharField(max_length=50)),
                ('info', models.TextField(blank=True)),
                ('comment', models.TextField(blank=True)),
                ('postinfo', models.TextField(blank=True)),
                ('correct_marks', models.IntegerField(verbose_name='Marks for correct answer', default=1)),
                ('wrong_marks', models.IntegerField(verbose_name='Marks for wrong answer', default=0)),
                ('na_marks', models.IntegerField(verbose_name='Marks for not attempting question', default=0)),
                ('hint_deduction', models.IntegerField(verbose_name='Marks deducted for viewing hint', default=0)),
                ('unlock_marks', models.IntegerField(verbose_name='Minimum marks to attempt this section', null=True)),
                ('unlock_questions', models.PositiveIntegerField(verbose_name='Minimum attempted questions to attempt this section', null=True)),
                ('unlock_both_needed', models.BooleanField(verbose_name='Should both minimum question and minimum marks requirements be fulfilled?', default=True)),
                ('shuffle_options', models.BooleanField(verbose_name='Randomly shuffle options of questions', default=False)),
                ('shuffle_questions', models.BooleanField(verbose_name='Randomly shuffle questions', default=False)),
                ('allowed_attempts', models.IntegerField(verbose_name='Number of attempts allowed per question', default=0)),
                ('show_correct_answer', models.BooleanField(verbose_name='Should correct answer be shown after attempting a question?', default=False)),
                ('show_solution', models.BooleanField(verbose_name='Should solution be shown after attempting a question?', default=False)),
                ('max_questions_to_attempt', models.PositiveIntegerField(verbose_name='Maximum number of questions a student is allowed to attempt', default=0)),
                ('exam', models.ForeignKey(to='main.Exam')),
            ],
            options={
                'ordering': ('id',),
            },
        ),
        migrations.CreateModel(
            name='SectionAnswerSheet',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('exam_answer_sheet', models.ForeignKey(to='main.ExamAnswerSheet')),
                ('section', models.ForeignKey(to='main.Section')),
            ],
            options={
                'ordering': ('id',),
            },
        ),
        migrations.CreateModel(
            name='Tag',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('name', models.CharField(max_length=30, unique=True)),
                ('description', models.TextField(blank=True)),
            ],
        ),
        migrations.CreateModel(
            name='TextAnswer',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('response', models.TextField(blank=True)),
                ('answer', models.OneToOneField(to='main.Answer')),
            ],
            options={
                'ordering': ('id',),
            },
            bases=(models.Model, main.models.SpecialAnswer),
        ),
        migrations.CreateModel(
            name='TextQuestion',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, primary_key=True, auto_created=True)),
                ('ignore_case', models.BooleanField(default=True)),
                ('use_regex', models.BooleanField(default=False)),
                ('correct_answer', models.TextField()),
                ('question', models.OneToOneField(to='main.Question')),
            ],
            options={
                'ordering': ('id',),
            },
            bases=(models.Model, main.models.SpecialQuestion),
        ),
        migrations.AddField(
            model_name='textanswer',
            name='special_question',
            field=models.ForeignKey(to='main.TextQuestion'),
        ),
        migrations.AddField(
            model_name='section',
            name='tags',
            field=models.ManyToManyField(to='main.Tag'),
        ),
        migrations.AddField(
            model_name='question',
            name='section',
            field=models.ForeignKey(to='main.Section'),
        ),
        migrations.AddField(
            model_name='question',
            name='tags',
            field=models.ManyToManyField(to='main.Tag'),
        ),
        migrations.AddField(
            model_name='mcqquestion',
            name='question',
            field=models.OneToOneField(to='main.Question'),
        ),
        migrations.AddField(
            model_name='mcqoption',
            name='special_question',
            field=models.ForeignKey(to='main.McqQuestion'),
        ),
        migrations.AddField(
            model_name='mcqanswertomcqoption',
            name='mcq_option',
            field=models.ForeignKey(to='main.McqOption'),
        ),
        migrations.AddField(
            model_name='mcqanswer',
            name='chosen_options',
            field=models.ManyToManyField(to='main.McqOption', through='main.McqAnswerToMcqOption'),
        ),
        migrations.AddField(
            model_name='mcqanswer',
            name='special_question',
            field=models.ForeignKey(to='main.McqQuestion'),
        ),
        migrations.AddField(
            model_name='exam',
            name='tags',
            field=models.ManyToManyField(to='main.Tag'),
        ),
        migrations.AddField(
            model_name='answer',
            name='section_answer_sheet',
            field=models.ForeignKey(to='main.SectionAnswerSheet'),
        ),
    ]


from sqlalchemy import inspect
from sqlalchemy.ext.asyncio import AsyncEngine

def ok_response(data: dict):
    return {
        "status": "ok",
        "data": data,
    }

def error_response(status: str, message: str, data: dict):
    return {
        "status": status,
        "message": message,
        "data": data,
    }

def use_inspector(conn):
    inspector = inspect(conn)
    return inspector.get_table_names()

async def check_empty_table_exists(cli, tablename: str):
    engine: AsyncEngine = cli.app.database._engine
    async with engine.begin() as conn:
        tables = await conn.run_sync(use_inspector)

    assert tablename in tables