#!/usr/bin/env python
# coding: utf-8

# # ML Pipeline Preparation
# Follow the instructions below to help you create your ML pipeline.
# ### 1. Import libraries and load data from database.
# - Import Python libraries
# - Load dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)
# - Define feature and target variables X and Y

# In[1]:

# import libraries
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.multioutput import MultiOutputClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
from sklearn.decomposition import TruncatedSVD
import pickle

# In[2]:

import nltk
nltk.download(['punkt', 'wordnet'])

# In[3]:

# load data from database
engine = create_engine('sqlite:///InsertDatabaseName.db')
df = pd.read_sql_table("messages_disaster", con=engine)

# In[4]:

df.head()

# In[5]:

X = df["message"]
Y = df.drop(['message', 'genre', 'id', 'original'], axis = 1)

# ### 2. Write a tokenization function to process your text data

# In[6]:

def tokenize(text):
    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()
    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)
    return clean_tokens

# ### 3. Build a machine learning pipeline
# This machine pipeline should take in the `message` column as input and output classification results on the other 36 categories in the dataset. You may find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables.

# In[7]:

pipeline = Pipeline([
        ('vect', CountVectorizer(tokenizer=tokenize)),
        ('tfidf', TfidfTransformer()),
        ('clf', MultiOutputClassifier(RandomForestClassifier()))
    ])

# In[8]:

pipeline.get_params()

# ### 4. Train pipeline
# - Split data into train and test sets
# - Train pipeline

# In[9]:

X_train, X_test, y_train, y_test = train_test_split(X, Y,test_size = 0.2, random_state = 45)
# train classifier
pipeline.fit(X_train, y_train)

# ### 5. Test your model
# Report the f1 score, precision and recall for each output category of the dataset. You can do this by iterating through the columns and calling sklearn's `classification_report` on each.

# In[10]:

def perf_report(model, X_test, y_test):
    '''
    Function to generate classification report on the model
    Input: Model, test set ie X_test & y_test
    Output: Prints the Classification report
    '''
    y_pred = model.predict(X_test)
    for i, col in enumerate(y_test):
        print(col)
        print(classification_report(y_test[col], y_pred[:, i]))

# In[11]:

perf_report(pipeline, X_test, y_test)

# ### 6. Improve your model
# Use grid search to find better parameters.

# In[12]:

parameters =  {'tfidf__use_idf': (True, False),
              'clf__estimator__n_estimators': [50, 100],
              'clf__estimator__min_samples_split': [2, 4]}

cv = GridSearchCV(pipeline, param_grid=parameters)

# In[13]:

cv

# ### 7. Test your model
# Show the accuracy, precision, and recall of the tuned model.
#
# Since this project focuses on code quality, process, and  pipelines, there is no minimum performance metric needed to pass. However, make sure to fine tune your models for accuracy, precision and recall to make your project stand out - especially for your portfolio!

# In[14]:

cv.fit(X_train, y_train)
perf_report(cv, X_test, y_test)

# ### 8. Try improving your model further. Here are a few ideas:
# * try other machine learning algorithms
# * add other features besides the TF-IDF

# In[15]:

#Improve  the pipeline
pipeline2 = Pipeline([
    ('vect', CountVectorizer()),
    ('best', TruncatedSVD()),
    ('tfidf', TfidfTransformer()),
    ('clf', MultiOutputClassifier(AdaBoostClassifier()))
])

# In[16]:

pipeline2.get_params()

# In[17]:

#Train & predict
pipeline2.fit(X_train, y_train)
perf_report(pipeline2, X_test, y_test)

# In[18]:

#Param tunning
parameters2 = { #'vect__ngram_range': ((1, 1), (1, 2)),
              #'vect__max_df': (0.5, 1.0),
              #'vect__max_features': (None, 5000),
              'tfidf__use_idf': (True, False),
              'clf__estimator__n_estimators': [50, 100],
              'clf__estimator__learning_rate': [1,2] }

# In[19]:

cv2 = GridSearchCV(pipeline2, param_grid=parameters2)
cv2

# In[20]:

cv2.fit(X_train, y_train)

# In[21]:

perf_report(cv2, X_test, y_test)

# ### 9. Export your model as a pickle file

# In[22]:

with open('model.pkl', 'wb') as f:
    pickle.dump(cv2, f)

# ### 10. Use this notebook to complete `train.py`
# Use the template file attached in the Resources folder to write a script that runs the steps above to create a database and export a model based on a new dataset specified by the user.

# In[ ]:


"""Set up the environment for doctests

This file is automatically evaluated by py.test. It ensures that we can write
doctests without importing anything. The entire content for qnet, as well as
numpy and sympy will be available in all doctests.
"""
import numpy
import sympy
import qnet
from collections import OrderedDict

# noinspection PyPackageRequirements
import pytest

@pytest.fixture(autouse=True)
def set_doctest_env(doctest_namespace):
    doctest_namespace['numpy'] = numpy
    doctest_namespace['sympy'] = sympy
    doctest_namespace['OrderedDict'] = OrderedDict
    for name in qnet.__all__:
        doctest_namespace[name] = getattr(qnet, name)


#!/usr/bin/env python
# coding: utf-8

# # Index Fund Data Processing
#
# This notebook contains code for data scraping and formatting of index fund prices into time-series JSON files. Index funds processed include Unit Investment Trust Funds (UITF), Mutual Funds (MF), and Exchange-Traded Funds (ETF) which are invested in the Philippine Stock Exchange, track the Philippine Stock Exchange Index (PSEi), and whose portfolios track the PSEi composition. In addition, this notebook also downloads and processes the PSEi and the PSEi Total Return Index (PSEi TRI) time-series prices as benchmarks so the index funds' performances can be compared.

# In[1]:

import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
import json
import os
from datetime import datetime

# In[2]:

overwrite_uitf = False
overwrite_mf = False
overwrite_etf = False
overwrite_index = False

print_json = False

# ## Unit Investment Trust Fund (UITF) Data Extraction
#
# The code below extracts the list of peso equity funds from uitf.com.ph, and upon user selection, will extract the entire price history of the selected fund. Output is a JSON file containing key-value pairs of "Date": "NAVPU".
#
# The UITFs included in this analysis are hand-picked below to be specifically "index funds"; their fund information sheets should specifically note that the portfolio track the PSEi composition.
#
# These funds are:
# - BDO Equity Index Fund
# - BDO PERA Equity Index Fund
# - BPI Philippine Equity Index Fund
# - CTBC Bank - Sun Life Philippine Stock Index Feeder Fund
# - EastWest PSEi Tracker Fund
# - PNB Phil-Index Tracker Fund
# - SB Philippine Equity Index Fund
# - UnionBank Philippine Equity Index Portfolio
# - UCPB Philippine Index Equity Fund

# In[3]:

# URL containing list of Peso equity UITFs
uitf_fund_list_url = "http://www.uitf.com.ph/fund-matrix.php?sortby=bank&sortorder=asc&class_id=1&currency=PHP&btn=Filter"

# In[4]:

# We GET from the URL above, and parse the HTML document to get the 4th

response = requests.get(uitf_fund_list_url)

soup = BeautifulSoup(response.content)
table = soup.find_all('table')[3]

# In[5]:

# We parse the HTML
 and convert it to a Pandas dataframe
# The dataframe's columns are: 'Bank', 'Fund Name', 'Bank ID', 'Fund ID'
# The 'Bank ID' and 'Fund ID' are parsed from the 'View' GET request per row

for row in table.find_all('tr'):
  th_tags = row.find_all('th')
  if len(th_tags) > 0:
    column_names = [" ".join(th.get_text().split()) for th in th_tags]
    uitf_fund_list = pd.DataFrame(columns=(column_names + ["Bank ID", "Fund ID"]))
  td_tags = row.find_all('td')
  if len(td_tags) > 0:
    row_append = pd.Series([" ".join(td.get_text().split()) for td in td_tags], column_names)

    get_attrs = td_tags[-1].find('a').attrs['href'].split('?')[1].split('&')
    fund_id = int(get_attrs[0].split('=')[1])
    bank_id = int(get_attrs[1].split('=')[1])
    bank_fund_id = pd.Series([bank_id, fund_id], ["Bank ID", "Fund ID"])

    row_append = row_append.append(bank_fund_id)

    uitf_fund_list = uitf_fund_list.append(row_append, ignore_index=True)

uitf_fund_list = uitf_fund_list[['Bank', 'Fund Name', 'Bank ID', 'Fund ID']]

# In[6]:

# This is a manual selection (i.e. human-selected) of index equity funds from:
# uitf_fund_list[["Bank", "Fund Name"]].to_numpy().tolist()

index_fund_name = [['BDO Unibank, Inc.', 'BDO EQUITY INDEX FUND'],
 ['BDO Unibank, Inc.', 'BDO PERA EQUITY INDEX FUND'],
 ['BPI Asset Management and Trust Corporation', 'BPI Philippine Equity Index Fund'],
 ['CTBC Bank (Philippines) Corp.', 'CTBC Bank - Sun Life Philippine Stock Index Feeder Fund'],
 ['EastWest Banking Corporation', 'EastWest PSEI Tracker Fund'],
 ['Metropolitan Bank & Trust Co.', 'Metro Philippine Equity Index Tracker Fund'],
 ['Philippine National Bank', 'PNB PHIL-INDEX TRACKER FUND(formerly PNB ENHANCED PHIL-INDEX REFERENCE FUND)'],
 ['Security Bank Corporation', 'SB PHILIPPINE EQUITY INDEX FUND'],
 ['Union Bank', 'UnionBank Philippine Equity Index Portfolio'],
 ['United Coconut Planters Bank', 'UCPB Philippine Index Equity Fund']]

index_fund_name = set(tuple(x) for x in index_fund_name)

uitf_index_fund_list = uitf_fund_list[uitf_fund_list.apply(lambda x: (tuple([x['Bank'], x['Fund Name']]) in index_fund_name), axis=1)].reset_index(drop=True)

uitf_index_fund_list['Fund Name'] = uitf_index_fund_list['Fund Name'].replace(
    'BDO EQUITY INDEX FUND', 'BDO Equity Index Fund').replace(
    'BDO PERA EQUITY INDEX FUND', 'BDO PERA Equity Index Fund').replace(
    'EastWest PSEI Tracker Fund', 'EastWest PSEi Tracker Fund').replace(
    'PNB PHIL-INDEX TRACKER FUND(formerly PNB ENHANCED PHIL-INDEX REFERENCE FUND)', 'PNB Phil-Index Tracker Fund').replace(
    'SB PHILIPPINE EQUITY INDEX FUND', 'SB Philippine Equity Index Fund')

# In[7]:

fmonth = '01'
fday = '01'
fyear = '1970'
tmonth = '12'
tday = '31'
tyear = '2030'

data_dir = 'data'
if not os.path.exists(data_dir):
  os.makedirs(data_dir)

# In[8]:

for index_fund in list(uitf_index_fund_list.iterrows()):
  bank_id = index_fund[1]['Bank ID']
  fund_id = index_fund[1]['Fund ID']

  timeseries_url = 'http://www.uitf.com.ph/daily_navpu_details_json.php?bank_id={}&fund_id={}&fmonth={}&fday={}&fyear={}&tmonth={}&tday={}&tyear={}&btn=Filter'.format(
      bank_id, fund_id, fmonth, fday, fyear, tmonth, tday, tyear
  )

  response = requests.get(timeseries_url)
  navpu_list = [tuple(reversed(x.replace('NAVpu : ', '').replace('', '').replace('Date : ', '').split('
'))) for x in json.loads(response.content)['thlabels']]

  if print_json:
    print(index_fund[1]['Fund Name'])
    print(json.dumps(dict(navpu_list)))
    print('')

  json_file = os.path.join(data_dir, index_fund[1]['Fund Name'] + '.json')

  if not os.path.isfile(json_file) or overwrite_uitf:
    with open(json_file, 'w') as f:
      json.dump(dict(navpu_list), f)

# In[9]:

# Fix data from 'PNB Phil-Index Tracker Fund'
raw_file = os.path.join(data_dir, 'PNB Phil-Index Tracker Fund' + '.json')
with open(raw_file, 'r') as f:
  raw_dict = json.loads(f.readline().strip())
raw_dict.pop(' 26, 2016', None)
with open(raw_file, 'w') as f:
  json.dump(raw_dict, f)

# Fix data from 'UCPB Philippine Index Equity Fund'
raw_file = os.path.join(data_dir, 'UCPB Philippine Index Equity Fund' + '.json')
with open(raw_file, 'r') as f:
  raw_dict = json.loads(f.readline().strip())
raw_dict['Jun 10, 2019'] = '1.1472'
with open(raw_file, 'w') as f:
  json.dump(raw_dict, f)

# Fix data from 'CTBC Bank - Sun Life Philippine Stock Index Feeder Fund'
raw_file = os.path.join(data_dir, 'CTBC Bank - Sun Life Philippine Stock Index Feeder Fund' + '.json')
with open(raw_file, 'r') as f:
  raw_dict = json.loads(f.readline().strip())
raw_dict['Sep 19, 2019'] = '0.99575'
with open(raw_file, 'w') as f:
  json.dump(raw_dict, f)

# ## Mutual Fund (MF) Data Extraction
#
# The MF data was scraped using other tools; this section will transform the raw price data into the standard JSON file format used in the UITF section above.
#
# The MFs included in this analysis are hand-picked below to be specifically "index funds"; their fund information sheets should specifically note that the portfolio track the PSEi composition.
#
# These funds are:
# - First Metro Save and Learn Philippine Index Fund
# - PAMI Equity Index Fund
# - Philequity PSE Index Fund
# - Philippine Stock Index Fund
# - Sun Life Prosperity Philippine Stock Index Fund

# In[10]:

raw_data_dir = 'raw-data'
data_dir = 'data'
if not os.path.exists(data_dir):
  os.makedirs(data_dir)

# In[11]:

# First Metro Save and Learn Philippine Index Fund

fund_name = 'First Metro Save and Learn Philippine Index Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readline().strip()[2:-2].split('],[')

for line in line_list:
  line = line.strip()
  date = int(line.split(',')[0]) // 1000
  date = datetime.utcfromtimestamp(date).strftime('%b %-d, %Y')
  price = line.split(',')[1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_mf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# In[12]:

# PAMI Equity Index Fund

fund_name = 'PAMI Equity Index Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = reversed(f1.readlines())

for line in line_list:
  line = line.strip()
  date = line.split('\t')[0]
  date = datetime.strptime(date, '%Y-%m-%d').strftime('%b %-d, %Y')
  price = line.split('\t')[1][:-1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_mf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# In[13]:

# Philequity PSE Index Fund

fund_name = 'Philequity PSE Index Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readlines()[1:]

for line in line_list:
  line = line.strip()
  date = line.split('\t')[0]
  date = datetime.strptime(date, '%Y-%m-%d').strftime('%b %-d, %Y')
  price = line.split('\t')[1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_mf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# In[14]:

# Philippine Stock Index Fund

fund_name = 'Philippine Stock Index Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  raw_dict = json.loads(f1.readline().strip())

date_list = raw_dict['category']
price_list = raw_dict['values']

for i in range(len(date_list)):
  date = date_list[i]
  date = datetime.strptime(date, '%Y-%m-%d').strftime('%b %-d, %Y')
  price = "{}".format(price_list[i])
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_mf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# In[15]:

# Sun Life Prosperity Philippine Stock Index Fund

fund_name = 'Sun Life Prosperity Philippine Stock Index Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readline().strip()[3:-3].split('\'), (\'')

for line in line_list:
  line = line.strip()
  date = line.split('\', \'')[0]
  date = datetime.strptime(date, '%b %d, %Y').strftime('%b %-d, %Y')
  price = line.split('\', \'')[1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_mf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# ## Exchange-Traded Fund (ETF) Data Extraction
#
# The ETF data was also scraped using other tools; this section will transform the raw price data into the standard JSON file format used in the UITF and MF sections above.
#
# Currently, there is only one ETF in the PSE, and it specifically tracks the PSEi composition; hence, it is an index fund.
#
# This fund is:
# - First Metro Equity Exchange-Traded Fund

# In[16]:

raw_data_dir = 'raw-data'
data_dir = 'data'
if not os.path.exists(data_dir):
  os.makedirs(data_dir)

# In[17]:

# First Metro Equity Exchange-Traded Fund

fund_name = 'First Metro Equity Exchange-Traded Fund'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readline().strip()[2:-2].split('],[')

for line in line_list:
  line = line.strip()
  date = int(line.split(',')[0]) // 1000
  date = datetime.utcfromtimestamp(date).strftime('%b %d, %Y')
  price = line.split(',')[1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_etf:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# ## PSEi and PSEi TRI Data Extraction
#
# The PSEi and PSEi TRI were also scraped / downloaded using other tools and sources. This section will transform the raw price data into the standard JSON file format used in the UITF, MF, and ETF sections above.

# In[18]:

raw_data_dir = 'raw-data'
data_dir = 'data'
if not os.path.exists(data_dir):
  os.makedirs(data_dir)

# In[19]:

# PSEi

fund_name = 'PSEi'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readlines()[1:]

for line in line_list:
  line = line.strip()
  date = line.split(',')[0]
  date = datetime.strptime(date, '%Y-%m-%d').strftime('%b %-d, %Y')
  price = line.split(',')[4]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_index:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)

# In[20]:

# PSEi Total Return

fund_name = 'PSEi Total Return'
raw_file = os.path.join(raw_data_dir, fund_name + '.txt')
json_file = os.path.join(data_dir, fund_name + '.json')

navps = dict()
with open(raw_file, 'r') as f1:
  line_list = f1.readlines()

for line in line_list:
  line = line.strip()
  date = line.split('\t')[0]
  date = datetime.strptime(date, '%m/%d/%Y').strftime('%b %-d, %Y')
  price = line.split('\t')[1]
  navps[date] = price

if print_json:
  print(fund_name)
  print(json.dumps(navps))
  print('')

if not os.path.isfile(json_file) or overwrite_index:
  with open(json_file, 'w') as f2:
    json.dump(navps, f2)


# -*- coding: utf-8 -*-
"""netcdf_P.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1WZQe2U65UyG45e167DVVg9uqyoLkSs9c

**netCDF**

Usually We have to install netCDF4 in Google Collab and Jupyter Notebook for working on it.
"""

pip install netCDF4

import warnings
warnings.filterwarnings('ignore')

import datetime
import numpy as np
import netCDF4 as nc4

ncFileName = 'sample_netcdf.nc4'
modeType   = 'w'
fileFormat = 'NETCDF4'
ncfid = nc4.Dataset(ncFileName, mode=modeType, format=fileFormat)

"""This is just a Demo, If you need any help regarding your work or Project. Please **Email me vatshayan007@gmail.com **"""


"""
Cremona's tables of elliptic curves

Sage includes John Cremona's tables of elliptic curves in an
easy-to-use format. An instance of the class CremonaDatabase()
gives access to the database.

If the optional full CremonaDatabase is not installed, a mini-version
is included by default with Sage.  It contains Weierstrass equations,
rank, and torsion for curves up to conductor 10000.

The large database includes all curves in John Cremona's tables. It
also includes data related to the BSD conjecture and modular degrees
for all of these curves, and generators for the Mordell-Weil
groups. To install it, run the following in the shell::

    sage -i database_cremona_ellcurve

This causes the latest version of the database to be downloaded from
the internet.

Both the mini and full versions of John Cremona's tables are stored in
SAGE_SHARE/cremona as SQLite databases. The mini version has the layout::

    CREATE TABLE t_class(conductor INTEGER, class TEXT PRIMARY KEY, rank INTEGER);
    CREATE TABLE t_curve(class TEXT, curve TEXT PRIMARY KEY, eqn TEXT UNIQUE, tors INTEGER);
    CREATE INDEX i_t_class_conductor ON t_class(conductor);
    CREATE INDEX i_t_curve_class ON t_curve(class);

while the full version has the layout::

    CREATE TABLE t_class(conductor INTEGER, class TEXT PRIMARY KEY, rank INTEGER, L REAL, deg INTEGER);
    CREATE TABLE t_curve(class TEXT, curve TEXT PRIMARY KEY, eqn TEXT UNIQUE, gens TEXT, tors INTEGER, cp INTEGER, om REAL, reg REAL, sha);
    CREATE INDEX i_t_class_conductor ON t_class(conductor);
    CREATE INDEX i_t_curve_class ON t_curve(class);
"""
#*****************************************************************************
#       Copyright (C) 2014 John Cremona <john.cremona@gmail.com>
#       Copyright (C) 2011 R. Andrew Ohana <andrew.ohana@gmail.com>
#       Copyright (C) 2005 William Stein <wstein@gmail.com>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#  as published by the Free Software Foundation; either version 2 of
#  the License, or (at your option) any later version.
#                  http://www.gnu.org/licenses/
#*****************************************************************************

from __future__ import print_function

import os
from sage.misc.prandom import randint

import sage.schemes.elliptic_curves.constructor as elliptic
from sql_db import SQLDatabase, verify_column
from sage.misc.package import is_package_installed
from sage.env import SAGE_SHARE
from sage.misc.all import walltime

import re
import string

_cremonaSkeleton = {
    't_class': {
        'conductor': {'sql':'INTEGER', 'index':True},
        'class':     {'sql':'TEXT',    'primary_key':True},
        'rank':      {'sql':'INTEGER'},
        'L':         {'sql':'REAL'},
        'deg':       {'sql':'INTEGER'}
    },
    't_curve': {
        'class':    {'sql':'TEXT', 'index':True},
        'curve':    {'sql':'TEXT', 'primary_key':True},
        'eqn':      {'sql':'TEXT', 'unique':True},
        'gens':     {'sql':'TEXT'},
        'tors':     {'sql':'INTEGER'},
        'cp':       {'sql':'INTEGER'},
        'om':       {'sql':'REAL'},
        'reg':      {'sql':'REAL'},
        'sha':      {'sql':'NOTYPE'}
    }
}
_miniCremonaSkeleton = {
    't_class': {
        'conductor': {'sql':'INTEGER', 'index':True},
        'class':     {'sql':'TEXT',    'primary_key':True},
        'rank':      {'sql':'INTEGER'}
    },
    't_curve': {
        'class':    {'sql':'TEXT', 'index':True},
        'curve':    {'sql':'TEXT', 'primary_key':True},
        'eqn':      {'sql':'TEXT', 'unique':True},
        'tors':     {'sql':'INTEGER'}
    }
}

for t in _cremonaSkeleton:
    for c in _cremonaSkeleton[t]:
        _cremonaSkeleton[t][c] = verify_column(_cremonaSkeleton[t][c])
    for c in _miniCremonaSkeleton[t]:
        _miniCremonaSkeleton[t][c] = verify_column(_miniCremonaSkeleton[t][c])

def build(name, data_tgz, largest_conductor=0, mini=False, decompress=True):
    """
    Build the CremonaDatabase with given name from scratch
    using the data_tgz tarball.

    .. note::

           For data up to level 350000, this function takes about
           3m40s.  The resulting database occupies 426MB disk space.

    To create the large Cremona database from Cremona's data_tgz
    tarball, obtainable from
    http://homepages.warwick.ac.uk/staff/J.E.Cremona/ftp/data/, run
    the following command::

        sage: d = sage.databases.cremona.build('cremona','ecdata.tgz')   # not tested
    """
    db_path = os.path.join(SAGE_SHARE,'cremona',name.replace(' ','_')+'.db')
    if os.path.exists(db_path):
        raise RuntimeError('Please (re)move %s before building '%db_path \
                + 'database')
    if not os.path.exists(data_tgz):
        raise IOError("The data file is not at %s"%data_tgz)
    t = walltime()

    if decompress:
        cmd = "tar zxvf %s"%data_tgz
        n = os.system(cmd)
        if n:
            raise RuntimeError("Error extracting tarball.")
    if mini:
        c = MiniCremonaDatabase(name,False,True)
    else:
        c = LargeCremonaDatabase(name,False,True)
    # The following line assumes that the tarball extracts to a
    # directory called 'ecdata'
    c._init_from_ftpdata('ecdata', largest_conductor)
    print("Total time: ", walltime(t))

def is_optimal_id(id):
    """
    Returns true if the Cremona id refers to an optimal curve, and
    false otherwise. The curve is optimal if the id, which is of the
    form [letter code][number] has number 1.

    .. note::

       990h3 is the optimal curve in that class, so doesn't obey
       this rule.

    INPUT:

    -  ``id`` - str of form letter code followed by an
       integer, e.g., a3, bb5, etc.

    OUTPUT: bool

    EXAMPLES::

        sage: from sage.databases.cremona import is_optimal_id
        sage: is_optimal_id('b1')
        True
        sage: is_optimal_id('bb1')
        True
        sage: is_optimal_id('c1')
        True
        sage: is_optimal_id('c2')
        False
    """
    return id[-1] == '1' and not id[-2].isdigit()

def cremona_letter_code(n):
    """
    Returns the Cremona letter code corresponding to an integer. For
    example, 0 - a 25 - z 26 - ba 51 - bz 52 - ca 53 - cb etc.

    .. note::

       This is just the base 26 representation of n, where a=0, b=1,
       ..., z=25. This extends the old Cremona notation (counting from
       0) for the first 26 classes, and is different for classes above
       26.

    INPUT:

    -  ``n`` (int) -- a non-negative integer

    OUTPUT: str

    EXAMPLES::

        sage: from sage.databases.cremona import cremona_letter_code
        sage: cremona_letter_code(0)
        'a'
        sage: cremona_letter_code(26)
        'ba'
        sage: cremona_letter_code(27)
        'bb'
        sage: cremona_letter_code(521)
        'ub'
        sage: cremona_letter_code(53)
        'cb'
        sage: cremona_letter_code(2005)
        'czd'

    TESTS::

        sage: cremona_letter_code(QQ)
        Traceback (most recent call last):
        ...
        ValueError: Cremona letter codes are only defined for non-negative integers
        sage: cremona_letter_code(x)
        Traceback (most recent call last):
        ...
        ValueError: Cremona letter codes are only defined for non-negative integers
        sage: cremona_letter_code(-1)
        Traceback (most recent call last):
        ...
        ValueError: Cremona letter codes are only defined for non-negative integers
        sage: cremona_letter_code(3.14159)
        Traceback (most recent call last):
        ...
        ValueError: Cremona letter codes are only defined for non-negative integers
    """
    try:
        m = int(n)
        if n == m:
            n = m
        else:
            n = -1
    except (ValueError, TypeError):
        n = -1

    if n<0:
        raise ValueError("Cremona letter codes are only defined for non-negative integers")

    if n == 0:
        return "a"
    s = ""
    while n != 0:
        s = chr(n%26+97) + s
        n //= 26
    return s

def old_cremona_letter_code(n):
    r"""
    Returns the *old* Cremona letter code corresponding to an integer.
    integer.

    For example::

        1  --> A
        26 --> Z
        27 --> AA
        52 --> ZZ
        53 --> AAA
        etc.

    INPUT:

    -  ``n`` - int

    OUTPUT: str

    EXAMPLES::

        sage: from sage.databases.cremona import old_cremona_letter_code
        sage: old_cremona_letter_code(1)
        'A'
        sage: old_cremona_letter_code(26)
        'Z'
        sage: old_cremona_letter_code(27)
        'AA'
        sage: old_cremona_letter_code(521)
        'AAAAAAAAAAAAAAAAAAAAA'
        sage: old_cremona_letter_code(53)
        'AAA'
        sage: old_cremona_letter_code(2005)
        'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC'
    """
    n -= 1
    k = n%26 + 65
    label = chr(k)*int(n//26 + 1)
    return label

old_cremona_label_regex = re.compile(r'(\d+)([A-Z]*)(\d*)$')
cremona_label_regex = re.compile(r'(\d+)([a-z]*)(\d*)$')
lmfdb_label_regex = re.compile(r'(\d+)\.([a-z]+)(\d*)$')

def parse_cremona_label(label):
    """
    Given a Cremona label that defines an elliptic
    curve, e.g., 11a1 or 37b3, parse the label and return the
    conductor, isogeny class label, and number.

    For this function, the curve number may be omitted, in which case
    it defaults to 1.  If the curve number and isogeny class are both
    omitted (label is just a string representing a conductor), then
    the isogeny class defaults to 'a' and the number to 1.  Valid
    labels consist of one or more digits, followed by zero or more
    letters (either all in upper case for an old Cremona label, or all
    in lower case), followed by zero or more digits.

    INPUT:

    -  ``label`` - str

    OUTPUT:

    -  ``int`` - the conductor
    -  ``str`` - the isogeny class label
    -  ``int`` - the number

    EXAMPLES::

        sage: from sage.databases.cremona import parse_cremona_label
        sage: parse_cremona_label('37a2')
        (37, 'a', 2)
        sage: parse_cremona_label('37b1')
        (37, 'b', 1)
        sage: parse_cremona_label('10bb2')
        (10, 'bb', 2)
        sage: parse_cremona_label('11a')
        (11, 'a', 1)
        sage: parse_cremona_label('11')
        (11, 'a', 1)

    Valid old Cremona labels are allowed::

        sage: parse_cremona_label('17CCCC')
        (17, 'dc', 1)
        sage: parse_cremona_label('5AB2')
        Traceback (most recent call last):
        ...
        ValueError: 5AB2 is not a valid Cremona label

    TESTS::

        sage: from sage.databases.cremona import parse_cremona_label
        sage: parse_cremona_label('x11')
        Traceback (most recent call last):
        ...
        ValueError: x11 is not a valid Cremona label
    """
    m = cremona_label_regex.match(str(label))
    if m is None:
        m = old_cremona_label_regex.match(str(label))
        if m is None:
            raise ValueError(label + " is not a valid Cremona label")

    conductor, iso, num = m.groups()
    if len(iso) == 0:
        iso = "a"
    if len(num) == 0:
        num = "1"

    # convert old cremona labels to new ones
    if iso.upper() == iso and iso[0]*len(iso) == iso:
        iso = cremona_letter_code((len(iso)-1)*26+ord(iso[0])-ord('A'))

    # verify cremona label is valid
    if iso.lower() != iso:
        raise ValueError('%s is not a valid Cremona label'%label)

    return int(conductor), iso, int(num)

def parse_lmfdb_label(label):
    """
    Given an LMFDB label that defines an elliptic curve, e.g., 11.a1
    or 37.b3, parse the label and return the conductor, isogeny class
    label, and number.

    The LMFDB label (named after the L-functions and modular forms
    database), is determined by the following two orders:

    - Isogeny classes with the same conductor are ordered
      lexicographically by the coefficients in the q-expansion of the
      associated modular form.

    - Curves within the same isogeny class are ordered
      lexicographically by the a-invariants of the minimal model.

    The format is <conductor>.<iso><curve>, where the isogeny class is
    encoded using the same base-26 encoding into letters used in
    Cremona's labels.  For example, 990.h3 is the same as Cremona's 990j1

    For this function, the curve number may be omitted, in which case
    it defaults to 1.  If the curve number and isogeny class are both
    omitted (label is just a string representing a conductor), then
    the isogeny class defaults to 'a' and the number to 1.

    INPUT:

    -  ``label`` - str

    OUTPUT:

    -  ``int`` - the conductor
    -  ``str`` - the isogeny class label
    -  ``int`` - the number

    EXAMPLES::

        sage: from sage.databases.cremona import parse_lmfdb_label
        sage: parse_lmfdb_label('37.a2')
        (37, 'a', 2)
        sage: parse_lmfdb_label('37.b')
        (37, 'b', 1)
        sage: parse_lmfdb_label('10.bb2')
        (10, 'bb', 2)
    """
    m = lmfdb_label_regex.match(str(label).lower())
    if m is None:
        raise ValueError(label + " is not a valid LMFDB label")
    conductor, iso, num = m.groups()
    if len(iso) == 0:
        iso = "a"
    if len(num) == 0:
        num = "1"
    return int(conductor), iso, int(num)

def split_code(key):
    """
    Splits class+curve id string into its two parts.

    EXAMPLES::

        sage: import sage.databases.cremona as cremona
        sage: cremona.split_code('ba2')
        ('ba', '2')
    """
    cu = re.split("[a-z]*",key)[1]
    cl =  re.split("[0-9]*",key)[0]
    return (cl,cu)

def class_to_int(k):
    """
    Converts class id string into an integer. Note that this is the
    inverse of cremona_letter_code.

    EXAMPLES::

        sage: import sage.databases.cremona as cremona
        sage: cremona.class_to_int('ba')
        26
        sage: cremona.class_to_int('cremona')
        821863562
        sage: cremona.cremona_letter_code(821863562)
        'cremona'
    """
    kk = [string.ascii_lowercase.index(ch) for ch in list(k)]
    kk.reverse()
    return sum([kk[i]*26**i for i in range(len(kk))])

def cmp_code(key1,key2):
    """
    Comparison function for curve id strings.

    .. note::

       Not the same as standard lexicographic order!

    EXAMPLES::

        sage: import sage.databases.cremona as cremona
        sage: cremona.cmp_code('ba1','z1')
        1

    By contrast::

        sage: cmp('ba1','z1')
        -1
    """
    cl1,cu1 = split_code(key1)
    cl2,cu2 = split_code(key2)
    d = class_to_int(cl1)-class_to_int(cl2)
    if d!=0:  return d
    return cmp(cu1,cu2)

def cremona_to_lmfdb(cremona_label, CDB=None):
    """
    Converts a Cremona label into an LMFDB label.

    See :func:`parse_lmfdb_label` for an explanation of LMFDB labels.

    INPUT:

    - ``cremona_label`` -- a string, the Cremona label of a curve.
      This can be the label of a curve (e.g. '990j1') or of an isogeny
      class (e.g. '990j')
    - ``CDB`` -- the Cremona database in which to look up the isogeny
      classes of the same conductor.

    OUTPUT:

    - ``lmfdb_label`` -- a string, the corresponding LMFDB label.

    EXAMPLES::

        sage: from sage.databases.cremona import cremona_to_lmfdb, lmfdb_to_cremona
        sage: cremona_to_lmfdb('990j1')
        '990.h3'
        sage: lmfdb_to_cremona('990.h3')
        '990j1'

    TESTS::

        sage: for label in ['5077a1','66a3','102b','420c2']:
        ...       assert(lmfdb_to_cremona(cremona_to_lmfdb(label)) == label)
        sage: for label in ['438.c2','306.b','462.f3']:
        ...       assert(cremona_to_lmfdb(lmfdb_to_cremona(label)) == label)
    """
    from sage.libs.pari.all import pari
    m = cremona_label_regex.match(cremona_label)
    if m is None:
        raise ValueError("Invalid Cremona label")
    N, cremona_iso, cremona_number = m.groups()
    if CDB is None:
        CDB = CremonaDatabase()
    classes = CDB.isogeny_classes(N)
    ft = int(53)
    tff = int(255) # This should be enough to distinguish between curves (using heuristics from Sato-Tate for example)
    isos = []
    for i, iso in enumerate(classes):
        alist = iso[0][0]
        E = pari(alist).ellinit(precision=ft)
        isos.append((E.ellan(tff, python_ints=True), cremona_letter_code(i)))
    isos.sort()
    sorted_letters = [iso[1] for iso in isos]
    lmfdb_iso = cremona_letter_code(sorted_letters.index(cremona_iso))
    if len(cremona_number) > 0:
        iso_class = sorted([(curve[0],str(i+1)) for i,curve in enumerate(classes[class_to_int(cremona_iso)])])
        sorted_numbers = [curve[1] for curve in iso_class]
        lmfdb_number = str(sorted_numbers.index(cremona_number)+1)
        return N + '.' + lmfdb_iso + lmfdb_number
    else:
        return N + '.' + lmfdb_iso

def lmfdb_to_cremona(lmfdb_label, CDB=None):
    """
    Converts an LMFDB labe into a Cremona label.

    See :func:`parse_lmfdb_label` for an explanation of LMFDB labels.

    INPUT:

    - ``lmfdb_label`` -- a string, the LMFDB label of a curve.
      This can be the label of a curve (e.g. '990.j1') or of an isogeny
      class (e.g. '990.j')
    - ``CDB`` -- the Cremona database in which to look up the isogeny
      classes of the same conductor.

    OUTPUT:

    - ``cremona_label`` -- a string, the corresponding Cremona label.

    EXAMPLES::

        sage: from sage.databases.cremona import cremona_to_lmfdb, lmfdb_to_cremona
        sage: lmfdb_to_cremona('990.h3')
        '990j1'
        sage: cremona_to_lmfdb('990j1')
        '990.h3'
    """
    from sage.libs.pari.all import pari
    m = lmfdb_label_regex.match(lmfdb_label)
    if m is None:
        raise ValueError("Invalid LMFDB label")
    N, lmfdb_iso, lmfdb_number = m.groups()
    if CDB is None:
        CDB = CremonaDatabase()
    classes = CDB.isogeny_classes(N)
    ft = int(53)
    tff = int(255) # This should be enough to distinguish between curves (using heuristics from Sato-Tate for example)
    isos = []
    for i, iso in enumerate(classes):
        alist = iso[0][0]
        E = pari(alist).ellinit(precision=ft)
        isos.append((E.ellan(tff, python_ints=True), cremona_letter_code(i)))
    isos.sort()
    cremona_iso = isos[class_to_int(lmfdb_iso)][1]
    if len(lmfdb_number) > 0:
        iso_class = sorted([(curve[0],i+1) for i,curve in enumerate(classes[class_to_int(cremona_iso)])])
        cremona_number = str(iso_class[int(lmfdb_number)-1][1])
        return N + cremona_iso + cremona_number
    else:
        return N + cremona_iso

class MiniCremonaDatabase(SQLDatabase):
    """
    The Cremona database of elliptic curves.

    EXAMPLES::

        sage: c = CremonaDatabase()
        sage: c.allcurves(11)
        {'a1': [[0, -1, 1, -10, -20], 0, 5],
         'a2': [[0, -1, 1, -7820, -263580], 0, 1],
         'a3': [[0, -1, 1, 0, 0], 0, 5]}
    """
    def __init__(self, name, read_only=True, build=False):
        """
        Initialize the database.

        TESTS::

            sage: c = CremonaDatabase('cremona mini')
            sage: c.name
            'cremona mini'
        """
        self.name = name
        name = name.replace(' ','_')
        db_path = os.path.join(SAGE_SHARE, 'cremona', name+'.db')
        if build:
            if name is None:
                raise RuntimeError('The database must have a name.')
            if read_only:
                raise RuntimeError('The database must not be read_only.')
            SQLDatabase.__init__(self, db_path, read_only=read_only, \
                    skeleton=_miniCremonaSkeleton)
            return
        if not os.path.isfile(db_path):
            raise ValueError("Desired database (='%s') does not "%self.name \
                    + "exist")
        SQLDatabase.__init__(self, db_path, read_only=read_only)
        if self.get_skeleton() != _miniCremonaSkeleton:
            raise RuntimeError('Database at %s does '%(self.__dblocation__) \
              + 'not appear to be a valid SQL Cremona database.')

    def __iter__(self):
        """
        Returns an iterator through all EllipticCurve objects in the
        Cremona database.

        TESTS::

            sage: it = CremonaDatabase().__iter__()
            sage: next(it).label()
            '11a1'
            sage: next(it).label()
            '11a2'
            sage: next(it).label()
            '11a3'
            sage: next(it).label()
            '14a1'
            sage: skip = [next(it) for _ in range(100)]
            sage: next(it).label()
            '45a3'
        """
        query = "SELECT curve FROM t_curve,t_class USING(class) ORDER BY conductor"
        for c in self.__connection__.cursor().execute(query):
            yield self.elliptic_curve(c[0])

    def __getitem__(self, N):
        """
        If N is an integer, return all data about level N in the database.
        If N is a string it must be a Cremona label, in which case return
        the corresponding elliptic curve, if it is in the database.

        INPUT:

        -  ``N`` - int or str

        OUTPUT: dict (if N is an int) or EllipticCurve (if N is a str)

        TESTS::

            sage: c = CremonaDatabase()
            sage: c[11]['allcurves']['a2']
            [[0, -1, 1, -7820, -263580], 0, 1]
            sage: c['11a2']
            Elliptic Curve defined by y^2 + y = x^3 - x^2 - 7820*x - 263580 over Rational Field
        """
        if isinstance(N, str):
            return self.elliptic_curve(N)

        try:
            N = int(N)
        except ValueError:
            raise KeyError("N (=%s) must be a string or positive integer."%N)

        if N <= 0:
            raise KeyError("N (=%s) must be a string or positive integer."%N)

        ret = {'allcurves': self.allcurves(N)}
        if hasattr(self, 'allbsd'):
            ret['allbsd'] = self.allbsd(N)
        if hasattr(self, 'degphi'):
            ret['degphi'] = self.degphi(N)
        if hasattr(self, 'allgens'):
            ret['allgens'] = self.allgens(N)
        return ret

    def __repr__(self):
        """
        String representation of this database.

        TESTS::

            sage: c = CremonaDatabase('cremona mini')
            sage: c.__repr__()
            "Cremona's database of elliptic curves with conductor at most 9999"
        """
        return "Cremona's database of elliptic curves with conductor at most "\
            + str(self.largest_conductor())

    def allcurves(self, N):
        """
        Returns the allcurves table of curves of conductor N.

        INPUT:

        -  ``N`` - int, the conductor

        OUTPUT:

        -  ``dict`` - id:[ainvs, rank, tor], ...

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.allcurves(11)['a3']
            [[0, -1, 1, 0, 0], 0, 5]
            sage: c.allcurves(12)
            {}
            sage: c.allcurves(12001)['a1']   # optional - database_cremona_ellcurve
            [[1, 0, 0, -101, 382], 1, 1]
        """
        ret = {}
        for c in self.__connection__.cursor().execute('SELECT curve,eqn,' \
            + 'rank,tors FROM t_curve,t_class USING(class) WHERE ' \
            + 'conductor=?',(int(N),)):
            N,iso,num = parse_cremona_label(c[0])
            ret[iso+str(num)] = [eval(c[1]),c[2],c[3]]
        return ret

    def curves(self, N):
        """
        Returns the curves table of all *optimal* curves of conductor N.

        INPUT:

        -  ``N`` - int, the conductor

        OUTPUT:

        -  ``dict`` - id:[ainvs, rank, tor], ...

        EXAMPLES:

        Optimal curves of conductor 37::

            sage: CremonaDatabase().curves(37)
            {'a1': [[0, 0, 1, -1, 0], 1, 1], 'b1': [[0, 1, 1, -23, -50], 0, 3]}

        Note the 'h3', which is the unique case in the tables where
        the optimal curve doesn't have label ending in 1::

            sage: list(sorted(CremonaDatabase().curves(990).keys()))
            ['a1', 'b1', 'c1', 'd1', 'e1', 'f1', 'g1', 'h3', 'i1', 'j1', 'k1', 'l1']

        TESTS::

            sage: c = CremonaDatabase()
            sage: c.curves(12001)['a1']   # optional - database_cremona_ellcurve
            [[1, 0, 0, -101, 382], 1, 1]
        """
        ret = {}
        for c in self.__connection__.cursor().execute('SELECT curve,eqn,' \
            + 'rank,tors FROM t_curve,t_class USING(class) WHERE ' \
            + 'curve=class||1 AND conductor=?',(int(N),)):
            N,iso,num = parse_cremona_label(c[0])
            ret[iso+str(num)] = [eval(c[1]),c[2],c[3]]
        if N == 990:
            del ret['h1']
            ret['h3'] = [[1,-1,1,-1568,-4669],int(1),int(6)]
        return ret

    def coefficients_and_data(self, label):
        """
        Return the Weierstrass coefficients and other data for the
        curve with given label.

        EXAMPLES::

            sage: c, d = CremonaDatabase().coefficients_and_data('144b1')
            sage: c
            [0, 0, 0, 6, 7]
            sage: d['conductor']
            144
            sage: d['cremona_label']
            '144b1'
            sage: d['rank']
            0
            sage: d['torsion_order']
            2

        Check that :trac:`17904` is fixed::

            sage: 'gens' in CremonaDatabase().coefficients_and_data('100467a2')[1] # optional - database_cremona_ellcurve
            True

        """
        # There are two possible strings: the Cremona label and the LMFDB label.
        # They are distinguished by the presence of a period.
        if label.find('.') == -1:
            cremona_label = label
            lmfdb_label = None
        else:
            cremona_label = lmfdb_to_cremona(label)
            lmfdb_label = label

        N, iso, num = parse_cremona_label(cremona_label)
        label = str(N)+iso+str(num)
        if self.get_skeleton() == _miniCremonaSkeleton:
            q = self.__connection__.cursor().execute("SELECT eqn,rank,tors " \
                + 'FROM t_curve,t_class USING(class) WHERE curve=?', (label,))
        else:
            q = self.__connection__.cursor().execute("SELECT eqn,rank,tors," \
                + "deg,gens,cp,om,L,reg,sha FROM t_curve,t_class " \
                + "USING(class) WHERE curve=?",(label,))
        try:
            c = next(q)
        except StopIteration:
            if N < self.largest_conductor():
                message = "There is no elliptic curve with label " + label \
                    + " in the database"
            elif is_package_installed('database_cremona_ellcurve'):
                message = "There is no elliptic curve with label " + label \
                    + " in the currently available databases"
            else:
                message = "There is no elliptic curve with label " \
                    + label + " in the default database; try installing " \
                    + "the optional package database_cremona_ellcurve which " \
                    + "contains the complete Cremona database"
            raise ValueError(message)
        ainvs = eval(c[0])
        data = {'cremona_label': label,
                'rank': c[1],
                'torsion_order': c[2],
                'conductor': N}
        if lmfdb_label:
            data['lmfdb_label'] = lmfdb_label
        if len(c) > 3:
            data['modular_degree'] = (c[3])
            data['gens'] = eval(c[4])
            data['db_extra'] = list(c[5:])
        return ainvs, data

    def data_from_coefficients(self, ainvs):
        """
        Return elliptic curve data for the curve with given
        Weierstrass coefficients.

        EXAMPLES::

            sage: d = CremonaDatabase().data_from_coefficients([1, -1, 1, 31, 128])
            sage: d['conductor']
            1953
            sage: d['cremona_label']
            '1953c1'
            sage: d['rank']
            1
            sage: d['torsion_order']
            2

        Check that :trac:`17904` is fixed::

            sage: ai = EllipticCurve('100467a2').ainvs() # optional - database_cremona_ellcurve
            sage: 'gens' in CremonaDatabase().data_from_coefficients(ai) # optional - database_cremona_ellcurve
            True
        """
        ainvs = str(list(ainvs))
        if self.get_skeleton() == _miniCremonaSkeleton:
            q = self.__connection__.cursor().execute("SELECT curve,rank,tors "
                + 'FROM t_curve,t_class USING(class) WHERE eqn=?',
                (ainvs.replace(' ', ''),))
        else:
            q = self.__connection__.cursor().execute("SELECT curve,rank,tors,"
                + "deg,gens,cp,om,L,reg,sha FROM t_curve,t_class "
                + "USING(class) WHERE eqn=?",
                (ainvs.replace(' ', ''),))
        try:
            c = next(q)
        except StopIteration:
            raise RuntimeError("There is no elliptic curve with coefficients "
                               + ainvs + " in the database")
        label = str(c[0])
        N, iso, num = parse_cremona_label(label)
        data = {'cremona_label': label,
                'rank': c[1],
                'torsion_order': c[2],
                'conductor': N}
        if len(c) > 3:
            data['modular_degree'] = (c[3])
            data['gens'] = eval(c[4])
            data['db_extra'] = list(c[5:])
        return data

    def elliptic_curve_from_ainvs(self, ainvs):
        """
        Returns the elliptic curve in the database of with minimal
        ainvs, if it exists, or raises a RuntimeError exception
        otherwise.

        INPUT:

        -  ``ainvs`` - list (5-tuple of int's); the minimal
           Weierstrass model for an elliptic curve

        OUTPUT: EllipticCurve

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.elliptic_curve_from_ainvs([0, -1, 1, -10, -20])
            Elliptic Curve defined by y^2 + y = x^3 - x^2 - 10*x - 20 over Rational Field
            sage: c.elliptic_curve_from_ainvs([1, 0, 0, -101, 382])  # optional - database_cremona_ellcurve
            Elliptic Curve defined by y^2 + x*y = x^3 - 101*x + 382 over Rational Field

        Old (pre-2006) Cremona labels are also allowed::

            sage: c.elliptic_curve('9450KKKK1')
            Elliptic Curve defined by y^2 + x*y + y = x^3 - x^2 - 5*x + 7 over Rational Field

        Make sure :trac:`12565` is fixed::

            sage: c.elliptic_curve('10a1')
            Traceback (most recent call last):
            ...
            ValueError: There is no elliptic curve with label 10a1 in the database
        """
        data = self.data_from_coefficients(ainvs)
        return elliptic.EllipticCurve(ainvs, **data)

    def elliptic_curve(self, label):
        """
        Return an elliptic curve with given label with some data about it
        from the database pre-filled in.

        INPUT:

        -  ``label`` - str (Cremona or LMFDB label)

        OUTPUT:

        - an :class:`sage.schemes.elliptic_curves.ell_rational_field.EllipticCurve_rational_field`

        .. note::

            For more details on LMFDB labels see :func:`parse_lmfdb_label`.

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.elliptic_curve('11a1')
            Elliptic Curve defined by y^2 + y = x^3 - x^2 - 10*x - 20 over Rational Field
            sage: c.elliptic_curve('12001a1')    # optional - database_cremona_ellcurve
            Elliptic Curve defined by y^2 + x*y = x^3 - 101*x + 382 over Rational Field
            sage: c.elliptic_curve('48c1')
            Traceback (most recent call last):
            ...
            ValueError: There is no elliptic curve with label 48c1 in the database

        You can also use LMFDB labels::

            sage: c.elliptic_curve('462.f3')
            Elliptic Curve defined by y^2 + x*y = x^3 - 363*x + 1305 over Rational Field
        """
        ainvs, data = self.coefficients_and_data(label)
        return elliptic.EllipticCurve(ainvs, **data)

    def iter(self, conductors):
        """
        Return an iterator through all curves in the database with given
        conductors.

        INPUT:

        -  ``conductors`` - list or generator of ints

        OUTPUT: generator that iterates over EllipticCurve objects.

        EXAMPLES::

            sage: [e.cremona_label() for e in CremonaDatabase().iter([11..15])]
            ['11a1', '11a2', '11a3', '14a1', '14a2', '14a3', '14a4', '14a5',
             '14a6', '15a1', '15a2', '15a3', '15a4', '15a5', '15a6', '15a7', '15a8']
        """
        for N in conductors:
            for c in self.__connection__.cursor().execute('SELECT curve ' \
                + 'FROM t_curve,t_class USING(class) WHERE conductor=?', \
                (int(N),)):
                yield self.elliptic_curve(c[0])

    def isogeny_classes(self, conductor):
        """
        Return the allcurves data (ainvariants, rank and torsion) for the
        elliptic curves in the database of given conductor as a list of
        lists, one for each isogeny class. The curve with number 1 is
        always listed first.

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.isogeny_classes(11)
            [[[[0, -1, 1, -10, -20], 0, 5],
             [[0, -1, 1, -7820, -263580], 0, 1],
             [[0, -1, 1, 0, 0], 0, 5]]]
            sage: c.isogeny_classes(12001)   # optional - database_cremona_ellcurve
            [[[[1, 0, 0, -101, 382], 1, 1]],
             [[[0, 0, 1, -247, 1494], 1, 1]],
             [[[0, 0, 1, -4, -18], 1, 1]],
             [[[0, 1, 1, -10, 18], 1, 1]]]
        """
        conductor=int(conductor)
        classes = []
        A = self.allcurves(conductor)
        K = A.keys()
        K.sort(cmp_code)
        for k in K:
            v = A[k]
            # test if not first curve in class
            if not (k[-1] == '1' and k[-2].isalpha()):
                classes[len(classes)-1].append(v)
            else:
                classes.append([v])
        return classes

    def isogeny_class(self, label):
        """
        Returns the isogeny class of elliptic curves that are
        isogenous to the curve with given Cremona label.

        INPUT:

        -  ``label`` - string

        OUTPUT:

        -  ``list`` - list of EllipticCurve objects.

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.isogeny_class('11a1')
            [Elliptic Curve defined by y^2 + y = x^3 - x^2 - 10*x - 20 over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 - x^2 - 7820*x - 263580 over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 - x^2 over Rational Field]
            sage: c.isogeny_class('12001a1')   # optional - database_cremona_ellcurve
            [Elliptic Curve defined by y^2 + x*y = x^3 - 101*x + 382 over Rational Field]
        """
        conductor,iso,num=parse_cremona_label(label)
        q = self.__connection__.cursor().execute("SELECT curve FROM t_curve " \
            + "WHERE class=?",(str(conductor)+iso,))
        return [self.elliptic_curve(c[0]) for c in q]

    def iter_optimal(self, conductors):
        """
        Return an iterator through all optimal curves in the database with given conductors.

        INPUT:

        - ``conductors`` - list or generator of ints

        OUTPUT:

        generator that iterates over EllipticCurve objects.

        EXAMPLES:

        We list optimal curves with conductor up to 20::

            sage: [e.cremona_label() for e in CremonaDatabase().iter_optimal([11..20])]
            ['11a1', '14a1', '15a1', '17a1', '19a1', '20a1']

        Note the unfortunate 990h3 special case::

            sage: [e.cremona_label() for e in CremonaDatabase().iter_optimal([990])]
            ['990a1', '990b1', '990c1', '990d1', '990e1', '990f1', '990g1', '990h3', '990i1', '990j1', '990k1', '990l1']
        """
        for N in conductors:
            if N == 990:
                for c in self.__connection__.cursor().execute('SELECT class ' \
                    + 'FROM t_class WHERE conductor=990'):
                    if c[0][-1] == u'h':
                        yield self.elliptic_curve(c[0]+u'3')
                    else:
                        yield self.elliptic_curve(c[0]+u'1')
                continue
            for c in self.__connection__.cursor().execute('SELECT curve ' \
                + 'FROM t_curve,t_class USING(class) WHERE curve=class||1 ' \
                + 'AND conductor=?',(int(N),)):
                yield self.elliptic_curve(c[0])

    def list(self, conductors):
        """
        Returns a list of all curves with given conductors.

        INPUT:

        - ``conductors`` - list or generator of ints

        OUTPUT:

        - list of EllipticCurve objects.

        EXAMPLES::

            sage: CremonaDatabase().list([37])
            [Elliptic Curve defined by y^2 + y = x^3 - x over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 + x^2 - 23*x - 50 over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 + x^2 - 1873*x - 31833 over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 + x^2 - 3*x + 1 over Rational Field]
        """
        return list(self.iter(conductors))

    def list_optimal(self, conductors):
        """
        Returns a list of all optimal curves with given conductors.

        INPUT:

        -  ``conductors`` - list or generator of ints
            list of EllipticCurve objects.

        OUTPUT:

        list of EllipticCurve objects.

        EXAMPLES::

            sage: CremonaDatabase().list_optimal([37])
            [Elliptic Curve defined by y^2 + y = x^3 - x over Rational Field,
             Elliptic Curve defined by y^2 + y = x^3 + x^2 - 23*x - 50 over Rational Field]
        """
        return list(self.iter_optimal(conductors))

    def largest_conductor(self):
        """
        The largest conductor for which the database is complete.

        OUTPUT:

        -  ``int`` - largest conductor

        EXAMPLES::

            sage: c = CremonaDatabase('cremona mini')
            sage: c.largest_conductor()
            9999
        """
        if hasattr(self, '__largest_conductor__'):
            return self.__largest_conductor__
        #print "Computing largest conductor."
        q = self.__connection__.cursor().execute('SELECT conductor FROM ' \
            + 't_class ORDER BY conductor DESC LIMIT 1')
        self.__largest_conductor__ = next(q)[0]
        return self.__largest_conductor__

    def smallest_conductor(self):
        """
        The smallest conductor for which the database is complete: always 1.

        OUTPUT:

        -  ``int`` - smallest conductor

        .. note::

           This always returns the integer 1, since that is the
           smallest conductor for which the database is complete,
           although there are no elliptic curves of conductor 1.  The
           smallest conductor of a curve in the database is 11.

        EXAMPLES::

            sage: CremonaDatabase().smallest_conductor()
            1
        """
        return 1

    def conductor_range(self):
        """
        Return the range of conductors that are covered by the database.

        OUTPUT: tuple of ints (N1,N2+1) where N1 is the smallest and
        N2 the largest conductor for which the database is complete.

        EXAMPLES::

            sage: c = CremonaDatabase('cremona mini')
            sage: c.conductor_range()
            (1, 10000)
        """
        return 1, self.largest_conductor()+1

    def number_of_curves(self,  N=0, i=0):
        """
        Returns the number of curves stored in the database with conductor
        N. If N = 0, returns the total number of curves in the database.

        If i is nonzero, returns the number of curves in the i-th isogeny
        class. If i is a Cremona letter code, e.g., 'a' or 'bc', it is
        converted to the corresponding number.

        INPUT:

        -  ``N`` - int
        -  ``i`` - int or str

        OUTPUT: int

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.number_of_curves(11)
            3
            sage: c.number_of_curves(37)
            4
            sage: c.number_of_curves(990)
            42
            sage: num = c.number_of_curves()
        """
        if N == 0:
            if hasattr(self, '__number_of_curves__'):
                return self.__number_of_curves__
            q = self.__connection__.cursor().execute('SELECT COUNT(curve) ' \
                + 'FROM t_curve')
            self.__number_of_curves__ = next(q)[0]
            return self.__number_of_curves__
        if i == 0:
            q = self.__connection__.cursor().execute('SELECT COUNT(curve) ' \
                + 'FROM t_curve,t_class USING(class) WHERE conductor=?', \
                (int(N),))
            return next(q)[0]
        if not isinstance(i, str):
            i = cremona_letter_code(i)
        q = self.__connection__.cursor().execute('SELECT COUNT(curve) FROM ' \
            + 't_curve WHERE class=?',(str(N)+i,))
        return next(q)[0]

    def number_of_isogeny_classes(self, N=0):
        """
        Returns the number of isogeny classes of curves in the database of
        conductor N. If N is 0, return the total number of isogeny classes
        of curves in the database.

        INPUT:

        -  ``N`` - int

        OUTPUT: int

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.number_of_isogeny_classes(11)
            1
            sage: c.number_of_isogeny_classes(37)
            2
            sage: num = c.number_of_isogeny_classes()
        """
        if N == 0:
            if hasattr(self, '__number_of_isogeny_classes__'):
                return self.__number_of_isogeny_classes__
            q = self.__connection__.cursor().execute('SELECT COUNT(class) ' \
                + 'FROM t_class')
            self.__number_of_isogeny_classes__ = next(q)[0]
            return self.__number_of_isogeny_classes__
        q = self.__connection__.cursor().execute('SELECT COUNT(class) FROM ' \
            + 't_class WHERE conductor=?',(int(N),))
        return next(q)[0]

    def random(self):
        """
        Returns a random curve from the database.

        EXAMPLES::

            sage: CremonaDatabase().random() # random -- depends on database installed
            Elliptic Curve defined by y^2 + x*y  = x^3 - x^2 - 224*x + 3072 over Rational Field
        """
        N = randint(11, self.largest_conductor())
        q = self.__connection__.cursor().execute('SELECT conductor FROM ' \
            + 't_class WHERE conductor>=? ORDER BY conductor',(int(N),))
        try:
            N = next(q)[0]
        except StopIteration:
            N = 11
        iso = randint(0, self.number_of_isogeny_classes(N)-1)
        iso = cremona_letter_code(iso)
        num = randint(1, self.number_of_curves(N,iso))
        return self.elliptic_curve(str(N)+iso+str(num))

    ###############################################################################
    # Functions for loading data from Cremona's ftpdata directory.
    ###############################################################################
    def _init_from_ftpdata(self, ftpdata, largest_conductor=0):
        """
        Create the SQL Cremona Database from the Cremona data directory,
        which is available from Cremona's website. I.e., just wget
        Cremona's database to a local directory.

        To create the large database from Cremona's text files, see
        sage.databases.cremona.build, do NOT run this method directly.

        EXAMPLES::

            sage: d = sage.databases.cremona.MiniCremonaDatabase(name='cremona', read_only=False, rebuild=True)   # not tested
            sage: d._init_from_ftpdata('.')     # not tested
        """
        if self.__read_only__:
            raise RuntimeError("The database must not be read_only.")

        if not os.path.exists(ftpdata):
            raise RuntimeError("The cremona ftpdata directory '" + ftpdata \
                + "' does not exist.")

        if largest_conductor:
            print("largest conductor =", largest_conductor)
            self.__largest_conductor__ =  largest_conductor

        # Since July 2014 the data files have been arranged in
        # subdirectories (see trac #16903).
        allcurves_dir = os.path.join(ftpdata,'allcurves')
        allbsd_dir = os.path.join(ftpdata,'allbsd')
        allgens_dir = os.path.join(ftpdata,'allgens')
        degphi_dir = os.path.join(ftpdata,'degphi')
        num_curves, num_iso_classes = self._init_allcurves(allcurves_dir, largest_conductor)
        self.__number_of_curves__ = num_curves
        self.__number_of_isogeny_classes__ = num_iso_classes
        if hasattr(self, 'degphi'):
            self._init_degphi(degphi_dir, largest_conductor)
        if hasattr(self, 'allbsd'):
            self._init_allbsd(allbsd_dir, largest_conductor)
        if hasattr(self, 'allgens'):
            self._init_allgens(allgens_dir, largest_conductor)
        self.vacuum()

    def _init_allcurves(self, ftpdata, largest_conductor=0):
        """
        Initialize the allcurves table by reading the corresponding ftpdata
        files and importing them into the database.

        To create the large database from Cremona's text files, see
        sage.databases.cremona.build, do NOT run this method directly.

        INPUT:

        - `ftpdata` (string) -- the name of the directory in which the data is

        -  ``largest_conductor`` - int (default: 0), if 0,
           then only include data up to that conductor.

        OUTPUT:

        -  ``int`` - number_of_curves
        -  ``int`` - number_of_isogeny_classes

       EXAMPLES::

            sage: d = sage.databases.cremona.MiniCremonaDatabase(name='cremona', read_only=False, rebuild=True)   # not tested
            sage: d._init_allcurves('.', 11)    # not tested
            (3, 1)
        """
        if self.__read_only__:
            raise RuntimeError("The database must not be read_only.")
        files = sorted(os.listdir(ftpdata))
        name = 'allcurves'
        num_curves = 0
        num_iso_classes = 0
        con = self.get_connection()
        for F in files:
            if not F[:len(name)] == name:
                continue
            print("Inserting", F)
            class_data = []
            curve_data = []
            for L in open(ftpdata + "/" + F).readlines():
                N, iso, num, ainvs, r, tor = L.split()
                if largest_conductor and int(N) > largest_conductor: break
                cls = N+iso
                cur = cls+num
                if num == "1":
                    class_data.append((N,cls,r))
                    num_iso_classes += 1
                curve_data.append((cur,cls,ainvs,tor))
                num_curves += 1
            con.executemany('INSERT INTO t_class (conductor,class,rank) ' \
                + 'VALUES (?,?,?)', class_data)
            con.executemany('INSERT INTO t_curve (curve,class,eqn,tors) ' \
                + 'VALUES (?,?,?,?)', curve_data)
            print("Committing...")
            print("num_iso_classes =", num_iso_classes)
            self.commit()
            if largest_conductor and int(N) > largest_conductor: break
        return num_curves, num_iso_classes

class LargeCremonaDatabase(MiniCremonaDatabase):
    """
    The Cremona database of elliptic curves.

    EXAMPLES::

        sage: c = CremonaDatabase('cremona')  # optional - database_cremona_ellcurve
        sage: c.allcurves(11)                 # optional - database_cremona_ellcurve
        {'a1': [[0, -1, 1, -10, -20], 0, 5],
        'a2': [[0, -1, 1, -7820, -263580], 0, 1],
        'a3': [[0, -1, 1, 0, 0], 0, 5]}
    """
    def __init__(self, name, read_only=True, build=False):
        """
        Initialize the database.

        TESTS::

            sage: c = CremonaDatabase('cremona')    # optional - database_cremona_ellcurve
            sage: c.name                            # optional - database_cremona_ellcurve
            'cremona'
        """
        self.name = name
        name = name.replace(' ','_')
        db_path = os.path.join(SAGE_SHARE, 'cremona', name+'.db')
        if build:
            if name is None:
                raise RuntimeError('The database must have a name.')
            if read_only:
                raise RuntimeError('The database must not be read_only.')
            SQLDatabase.__init__(self, db_path, read_only=read_only, \
                    skeleton=_cremonaSkeleton)
            return
        if not os.path.isfile(db_path):
            raise ValueError("Desired database (='%s') does not "%self.name \
                    + "exist")
        SQLDatabase.__init__(self, db_path, read_only=read_only)
        if self.get_skeleton() != _cremonaSkeleton:
            raise RuntimeError('Database at %s does '%(self.__dblocation__) \
              + 'not appear to be a valid SQL Cremona database.')

    def allbsd(self, N):
        r"""
        Return the allbsd table for conductor N. The entries are::

            [id, tamagawa_product, Omega_E, L, Reg_E, Sha_an(E)]

        where id is the isogeny class (letter) followed by a number, e.g.,
        b3, and L is `L^r(E,1)/r!`, where E has rank r.

        INPUT:

        -  ``N`` - int, the conductor

        OUTPUT: dict containing the allbsd table for each isogeny class
        in conductor N

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.allbsd(12)            # optional - database_cremona_ellcurve
            {}
            sage: c.allbsd(19)['a3']      # optional - database_cremona_ellcurve
            [1, 4.07927920046493, 0.453253244496104, 1.0, 1]
            sage: c.allbsd(12001)['a1']   # optional - database_cremona_ellcurve
            [2, 3.27608135248722, 1.54910143090506, 0.236425971187952, 1.0]
        """
        ret = {}
        for c in self.__connection__.cursor().execute('SELECT curve,cp,om,L,' \
            + 'reg,sha FROM t_curve,t_class USING(class) WHERE conductor=?', \
            (int(N),)):
            N,iso,num = parse_cremona_label(c[0])
            ret[iso+str(num)] = list(c[1:])
        return ret

    def allgens(self, N):
        """
        Return the allgens table for conductor N.

        INPUT:

        -  ``N`` - int, the conductor

        OUTPUT:

        -  ``dict`` - id:[points, ...], ...

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.allgens(12)            # optional - database_cremona_ellcurve
            {}
            sage: c.allgens(1001)['a1']    # optional - database_cremona_ellcurve
            [[61, 181, 1]]
            sage: c.allgens(12001)['a1']   # optional - database_cremona_ellcurve
            [[7, 2, 1]]
        """
        ret = {}
        for c in self.__connection__.cursor().execute('SELECT curve,gens ' \
            + 'FROM t_curve,t_class USING(class) WHERE conductor=?',(int(N),)):
            N,iso,num = parse_cremona_label(c[0])
            ret[iso+str(num)] = eval(c[1])
        return ret

    def degphi(self, N):
        """
        Return the degphi table for conductor N.

        INPUT:

        -  ``N`` - int, the conductor

        OUTPUT:

        -  ``dict`` - id:degphi, ...

        EXAMPLES::

            sage: c = CremonaDatabase()
            sage: c.degphi(11)            # optional - database_cremona_ellcurve
            {'a1': 1}
            sage: c.degphi(12001)['c1']   # optional - database_cremona_ellcurve
            1640
        """
        ret = {}
        for c in self.__connection__.cursor().execute('SELECT curve,deg FROM' \
            + ' t_curve,t_class USING(class) WHERE curve=class||1 AND ' \
            + 'conductor=?', (int(N),)):
            N,iso,num = parse_cremona_label(c[0])
            ret[iso+str(num)] = c[1]
        return ret

    def _init_degphi(self, ftpdata, largest_conductor=0):
        """
        Initialize the degphi table by reading the corresponding ftpdata
        files and importing them into the database.

        To create the large database from Cremona's text files, see
        sage.databases.cremona.build, do NOT run this method directly.

        EXAMPLES::

            sage: d = sage.databases.cremona.LargeCremonaDatabase(name='cremona', read_only=False, rebuild=True)   # not tested
            sage: d._init_degphi('.')           # not tested
        """
        if self.__read_only__:
            raise RuntimeError("The database must not be read_only.")
        files = sorted(os.listdir(ftpdata))
        name = "degphi"
        con = self.get_connection()
        for F in files:
            if not F[:len(name)] == name:
                continue
            print("Inserting", F)
            class_data = []
            for L in open(ftpdata + "/" + F).readlines():
                N, iso, num, degree, primes, curve = L.split()
                if largest_conductor and int(N) > largest_conductor: break
                class_data.append((degree,N+iso))
            con.executemany('UPDATE t_class SET deg=? WHERE class=?', \
                class_data)
            print("Committing...")
            self.commit()
            if largest_conductor and int(N) > largest_conductor: break

    def _init_allbsd(self, ftpdata, largest_conductor=0):
        """
        Initialize the allbsd table by reading the corresponding ftpdata
        files and importing them into the database.

        To create the large database from Cremona's text files, see
        sage.databases.cremona.build, do NOT run this method directly.

        EXAMPLES::

            sage: d = sage.databases.cremona.LargeCremonaDatabase(name='cremona', read_only=False, rebuild=True)   # not tested
            sage: d._init_allbsd('.')           # not tested
        """
        if self.__read_only__:
            raise RuntimeError("The database must not be read_only.")
        files = sorted(os.listdir(ftpdata))
        name = "allbsd"
        con = self.get_connection()
        for F in files:
            if not F[:len(name)] == name:
                continue
            print("Inserting", F)
            curve_data = []
            class_data = []
            for L in open(ftpdata + "/" + F).readlines():
                N, iso, num, eqn, rank, tor, cp, om, L, reg, sha  = L.split()
                if largest_conductor and int(N) > largest_conductor: break
                cls = N+iso
                if num == "1":
                    class_data.append((L,cls))
                curve_data.append((cp,om,reg,eval(sha),cls+num))
            con.executemany("UPDATE t_class SET L=? WHERE class=?", class_data)
            con.executemany("UPDATE t_curve SET cp=?,om=?,reg=?,sha=? WHERE " \
                    + "curve=?", curve_data)
            print("Committing...")
            self.commit()
            if largest_conductor and int(N) > largest_conductor: break

    def _init_allgens(self, ftpdata, largest_conductor=0):
        """
        Initialize the allgens table by reading the corresponding ftpdata
        files and importing them into the database.

        To create the large database from Cremona's text files, see
        sage.databases.cremona.build, do NOT run this method directly.

        EXAMPLES::

            sage: d = sage.databases.cremona.LargeCremonaDatabase(name='cremona', read_only=False, rebuild=True)   # not tested
            sage: d._init_allgens('.')          # not tested
        """
        if self.__read_only__:
            raise RuntimeError("The database must not be read_only.")
        files = sorted(os.listdir(ftpdata))
        name = "allgens"
        con = self.get_connection()
        for F in files:
            if not F[:len(name)] == name:
                continue
            print("Inserting", F)
            curve_data = []
            for L in open(ftpdata + "/" + F).readlines():
                v = L.split()
                if largest_conductor and int(v[0]) > largest_conductor: break
                gens = '['+','.join(v[6:6+int(v[4])]).replace(':',',')+']'
                curve_data.append((gens,''.join(v[:3])))
            con.executemany("UPDATE t_curve SET gens=? WHERE curve=?", \
                curve_data)
            print("Committing...")
            if largest_conductor and int(v[0]) > largest_conductor: break

_db = None
def CremonaDatabase(name=None,mini=None,set_global=None):
    """
    Initializes the Cremona database with name ``name``. If ``name`` is
    ``None`` it instead initializes large Cremona database (named 'cremona'),
    if available or default mini Cremona database (named 'cremona mini').

    If the Cremona database in question is in the format of the mini database,
    you must set ``mini=True``, otherwise it must be set to ``False``.

    If you would like other components of Sage to use this database, mark
    ``set_global=True``.

    TESTS::

        sage: c = CremonaDatabase()
        sage: isinstance(c, sage.databases.cremona.MiniCremonaDatabase)
        True
        sage: isinstance(c, sage.databases.cremona.LargeCremonaDatabase)  # optional - database_cremona_ellcurve
        True

    Verify that :trac:`12341` has been resolved::

        sage: c = CremonaDatabase('should not exist',mini=True)
        Traceback (most recent call last):
        ...
        ValueError: Desired database (='should not exist') does not exist
        sage: c = CremonaDatabase('should not exist',mini=False)
        Traceback (most recent call last):
        ...
        ValueError: Desired database (='should not exist') does not exist
        sage: from sage.env import SAGE_SHARE
        sage: os.path.isfile(os.path.join(SAGE_SHARE,'cremona','should_not_exist.db'))
        False
    """
    global _db
    if set_global is None:
        set_global = _db is None and name is None
    if name is None and not set_global:
        return _db
    if set_global and name is None:
        if is_package_installed('database_cremona_ellcurve'):
            name = 'cremona'
        else:
            name = 'cremona mini'
    if name == 'cremona':
        mini = False
    elif name == 'cremona mini':
        mini = True
    if mini is None:
        raise ValueError('mini must be set as either True or False')
    if set_global:
        if mini:
            _db = MiniCremonaDatabase(name)
        else:
            _db = LargeCremonaDatabase(name)
        return _db
    if mini:
        return MiniCremonaDatabase(name)
    return LargeCremonaDatabase(name)


from os import error
from TPM2PPF_learntools.core import ThoughtExperiment, CodingProblem, bind_exercises, MultipartProblem
from TPM2PPF_learntools.core.problem import injected
from TPM2PPF_learntools.core.exceptions import Uncheckable
import textwrap

import numpy as np
from astropy.constants import eps0
eps0 = eps0.value

class FiniteVolumeDiscretisation(ThoughtExperiment):
    _bonus = False
    _hint = textwrap.dedent("""
    To answer this question, use the Latex langage for equations (you can look that the other equations in the notebooks by double-cliking on the markdown cells)

    I expect in the question the diffinitions of $V_{e,i}$ (and others) as function of $d_x$ $d_y$ the cell sizes, $\epsilon_0$, and the source therm.
    The discretisation have been shown in the presentation.

    """)  # TODO: continue the hint

    _solution = """You cannot have access to the solution !"""

    def check(self):
        raise Uncheckable

class BoundaryConditionDiscretisation(ThoughtExperiment):
    _bonus = False
    _hint = textwrap.dedent("""
    I expect in the question the deffinitions of $V_i,j$ at $i=1$, $i=N$, $j=1$, $j=M$
    as function of the parameters $U_a$ (or $U_c$) and other variables like the plasma potential in the cells nearby.

    Somethink like :

    1. Dirichlet condition: $V_1,j = U_a + V_2,j$ ... (this is wrong !!)
    2. Neumann Condition:  $V_i,1 = \log(U_c)$ (this is also wrong !!)

    """)  # TODO: continue the hint

    _solution = """You cannot have access to the solution !"""

    def check(self):
        raise Uncheckable

class MatrixDiscretisation(ThoughtExperiment):
    _bonus = False
    _hint = textwrap.dedent("""
    I expect here the details of the matrix `A`.

    If you can factor some parameters, please do it !

    You do not need to write all the elements of $A$. Find ways to give the most important informations
    And find a way to deals with the 2D domain.

    """)  # TODO: continue the hint

    _solution = textwrap.dedent("""
    You cannot have access to the solution !
    However, there is a second tips.

    You have to write a 2D Matrix, with the vectors $V$ and $d$ 1D vectors.
    Hence, you need to "unwrap" the 2D domain into a 1D vector.
    One solutions is to start by iterating over $x$, then $y$ :

    $$
    V = [V_{1,1}, V_{2,1}, ... V_{N,1}, V_{1,2}, V_{2,2} ... V_{N, 2}, ... V_{N-1, M}, V_{N,M}]
    $$
     """
    )

    def check(self):
        raise Uncheckable

class UsingThomas(ThoughtExperiment):
    _bonus = False
    _hint = textwrap.dedent("""
    Look at the caracteristics of the matrix $A$,
    and the conditions to use the Thomas algorythm.
    """)  # TODO: continue the hint

    _solution = textwrap.dedent("""
    The Matrix $A$ is no more Tridiagonal. So the Thomas algorythm cannot be used.
    We will use the SOR algorithm.
     """
    )

    def check(self):
        raise Uncheckable

class ExerciceSorSolver(CodingProblem):
    _vars = ['SOR', "E",  "Ve", "Vo", "Vs", "Vn", "Vc", "rho", "V", "Ua", "Uc" ]
    _hint = "You need to modify the function `SOR` that would solve the tridiagonal matrix linear equation."
    _solution = textwrap.dedent("""
    You cannot have access to the solution !

    But there is another tips :
    The `SOR` function is actually pretty simple, it is almost direct translation for the math to python
    """
    )

    def check(self, SOR, E, Ve, Vo, Vs, Vn, Vc, rho, V, Ua, Uc):

        print("Running the `SOR` function over the system, it can take some time...")

        N = 50
        M = 50
        dx = 1./N
        dy = 1./M

        eps0 = 8.85E-14   # F/cm

        # dielectric layers
        epsdiel=1    # default =1 (no dielectric)

        nxdg = 0 # Nx/10      # dielectric anode side
        nxdd = 0 # Nx/10      # dielectric cathode side

        Vact = SOR(w=1.5)

        Vslot = 0.5 * (np.linspace(Uc, Ua, N) + np.linspace(Uc, Ua, N+2)[1:-1])

        Vtheo = np.zeros( (N,M) )
        for j in range(M):
            Vtheo[:,j] = Vslot

        error = np.sum(np.abs(Vact - Vtheo))

        if False:
            import matplotlib.pyplot as plt

            plt.figure()
            plt.plot(Vact[:, 1], label="comuted")
            plt.plot(Vtheo[:, 1], label = "expected")
            plt.legend()
            plt.xlabel("$x$ axis")
            plt.ylabel("V(i, j=1) [V]")
            plt.show()

        Vact = np.zeros((N,M))  # The solution

        assert error < N*M*1, self._failure_message(error)

    @property
    def _correct_message(self):
        history = self._view.interactions
        return ("Congrats ! "
        "You succedded to write the SOR function to solve the Poisson Equation !")

    def _failure_message(self, error_value):

        return ("For some reasons, when using your function `SOR`,"
                "the answer I obtained was not the value expected !\n"
                "In particular, the total square error is : "
                "$$\sum (V - V_{{theo}}) = {}$$"
                "which is very large !"
        ).format(error_value)

# def SOR_solution(w=1):
#     """arguments: w between 0 and 2
#     Return x a vector of the same size of a"""

#     "Verifying that the inputs are correct"
#     assert w > 0 and w < 2 , "the argument is not between 0 and 2"

#     for i in range(N):
#         for j in range(M):

#             Vact[i, j] = (1-w)*Vact[i, j] + w * (Vo(i,j)*V(i-1, j) +
#             Ve(i,j)*V(i+1, j) +
#             Vs(i,j)*V(i, j-1) +
#             Vn(i,j)*V(i, j+1)
#             + rho(i,j)) / Vc(i,j)

#     # Write your code here

#     return Vact

class SolvingOnce(CodingProblem):
    _var = 'Vact'
    _hint = ("Remembrer, SOR is an iterativ solver !")

    _solution = """You cannot have access to the solution !"""
    def check(self, Vact):

        import matplotlib.pyplot as plt

        N = 50
        M = 50
        Ua = 45
        Uc = 10

        xvect = np.linspace(0, 1, N)

        Vslot = 0.5 * (np.linspace(Uc, Ua, N) + np.linspace(Uc, Ua, N+2)[1:-1])

        Vtheo = np.zeros( (N,M) )
        for j in range(M):
            Vtheo[:,j] = Vslot

        plt.figure(figsize=(5,3))
        plt.title("Plasma potential \n with no charge density")
        plt.ylabel("Potential at $j=1$ [V]")
        plt.xlabel("Position x [cm]")
        plt.plot(xvect, Vtheo[:, 1], ":", label="The solution")
        plt.plot(xvect, Vact[:, 1], label="Your answer `Vact`")
        plt.legend()
        plt.tight_layout()

        error = np.sum(np.abs(Vact - Vtheo))

        assert error < N*M*1, self._failure_message(error)

    @property
    def _correct_message(self):
        history = self._view.interactions
        return ("Congrats ! "
        "You succedded to solve the Poisson Equation !")

    def _failure_message(self, error_value):

        return ("For some reasons, your solution for `V` is not good !"
                "In particular, the total square error I get is : "
                "$$\sum (V - V_{{theo}}) = {}$$"
                "which is very large !"
        ).format(error_value)

class SolvingTwice(CodingProblem):
    _var = 'Vact'
    _hint = ("Remembrer, SOR is an iterativ solver !")

    _solution = """You cannot have access to the solution !"""

    def check(self, Vact):

        import matplotlib.pyplot as plt

        N = 50
        M = 50
        Ua = 45
        Uc = 10
        rho = 1e-10

        x = np.linspace(0, 1, N)

        Vslot = 0.5 * (np.linspace(Uc, Ua, N) + np.linspace(Uc, Ua, N+2)[1:-1])
        Vslot = np.linspace(Uc, Ua, N)

        Vslot += - rho / 8.85e-14 / 2 * ( x*(x - 1))

        Vtheo = np.zeros( (N,M) )
        for j in range(M):
            Vtheo[:,j] = Vslot

        plt.figure(figsize=(5,3))
        plt.title("Plasma potential \n with uniform charge density")
        plt.ylabel("Potential at $j=1$ [V]")
        plt.xlabel("Position x [cm]")
        plt.plot(x, Vtheo[:, 1], ":", label="The solution")
        plt.plot(x, Vact[:, 1], label="Your answer `Vact`")
        plt.legend()
        plt.tight_layout()

        error = np.sum(np.abs(Vact - Vtheo))

        assert error < N*M*2, self._failure_message(error)

    @property
    def _correct_message(self):
        history = self._view.interactions
        return ("Congrats ! "
        "You succedded to solve the Poisson Equation !")

    def _failure_message(self, error_value):

        return ("For some reasons, your solution for `V` is not good !"
                "In particular, the total square error I get is : "
                "$$\sum (V - V_{{theo}}) = {}$$"
                "which is very large !"
        ).format(error_value)

SORfunction = MultipartProblem(UsingThomas, ExerciceSorSolver)
SolvingSimpleCases = MultipartProblem(SolvingOnce, SolvingTwice)

qvars = bind_exercises(globals(), [
    FiniteVolumeDiscretisation,
    BoundaryConditionDiscretisation,
    MatrixDiscretisation,
    SORfunction,
    SolvingSimpleCases,
    ],
    start=1,
    )
__all__ = list(qvars)