# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.13.3
#   kernelspec:
#     display_name: tcv-x21
#     language: python
#     name: tcv-x21
# ---

# %% [markdown] pycharm={"name": "#%% md\n"}
# # Post-processing a GRILLIX simulation
#
# This notebook demonstrates how to perform the post-processing analysis for
# a GRILLIX simulation, contained in the `sample_data` folder.
#
# To limit the repository size, only 5 snapshot write-outs are provided. The sample data is provided primarily to demonstrate the
# method by which you can generate a standard NetCDF file for another simulation,
# to perform an equivalent validation.
#
# Analysis routines are provided in the `tcvx21/grillix_post` folder.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
import tcvx21

# %reload_ext autoreload
# %autoreload 2
# %matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
import xarray as xr
from pathlib import Path
from tcvx21.units_m import Quantity, Dimensionless, convert_xarray_to_quantity
from tcvx21.file_io.json_io_m import read_from_json

import tcvx21.grillix_post as grillix
from tcvx21 import test_session

plt.style.use(tcvx21.style_sheet)

plt.rcParams.update({"mathtext.default": "regular"})
plt.rcParams["figure.facecolor"] = "white"
xr.set_options(keep_attrs=True)

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
file_path = tcvx21.sample_data
time_slice = slice(None)

file_path = Path(file_path)
assert file_path.exists()

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Setting up an interface to the data
#
# We start by reading in the basic simulation data. For GRILLIX, this is
#
# 1. Grid: the $R,Z$ grid used for the simulation (assumed to be axisymmetric).
#    Since we trim our grid at a limiting flux-surface, the data is stored as unstructured
#    $(R, Z, value)$ column data. The grid allows us to map this to a matrix form $value(R, Z)$, which is needed
#    for plotting and analysis.
# 2. Normalisation: a set of `pint.Quantity` values which allows us to go from
#    normalised values to SI values. The `pint.Quantity` class is highly capable,
#    allowing unit tracking through basic operations, and unit conversion.
# 3. `xr.Dataset` snaps: an interface to the GRILLIX snapshot data. The xarray
#    module interfaces with dask to provide a memory-light interface to very
#    large files. It is similar to a `pandas.DataFrame`.
# 4. Equi: an interface to the equilibrium file, providing data about the magnetic
#    field and penalisation (which is how GRILLIX sets boundary conditions). The
#    `flip_z` parameter is used to invert the toroidal field direction.
#
# To limit the size of the tcvx21 repository, we provide only sample data over a few snapshots
# of a low-resolution (2mm) case.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
grid = grillix.components.Grid(file_path / "vgrid.nc")
norm = grillix.components.Normalisation.initialise_from_normalisation_file(
    file_path / "physical_parameters.nml"
)
snaps = grillix.components.read_snaps_from_file(
    file_path, norm, time_slice=slice(None), all_planes=True
)
equi = grillix.components.Equi(
    file_path / "TCV_ortho.nc", file_path / "pen_metainfo.nc", flip_z=True
)

parameter_filepath = grillix.filepath_resolver(file_path, "params.in")
params = grillix.components.convert_params_filepaths(
    parameter_filepath, grillix.components.read_fortran_namelist(parameter_filepath)
)

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Data extraction via "lineouts"
#
# Now that we have the interface prepared, we want to extract values at the
# set of experimental observable positions. We do this via "lineouts", which
# provide an efficient way to get points at specific $R,Z$ positions.
#
# We also need to be able to map from a flux-surface label to the
# $R^u - R^u_{sep}$ coordinate. We do this via the OutboardMidplaneMap.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
n_points = 500
omp_map = grillix.lineouts.OutboardMidplaneMap(grid, equi, norm)
omp = grillix.lineouts.outboard_midplane_chord(grid, equi, n_points=500)
lfs = grillix.lineouts.penalisation_contour(
    grid, equi, level=0.0, contour_index=0, n_points=n_points
)
hfs = grillix.lineouts.penalisation_contour(
    grid, equi, level=0.0, contour_index=1, n_points=n_points
)
ts = grillix.lineouts.thomson_scattering(
    grid, equi, tcvx21.thomson_coords_json, n_points=n_points
)
rdpa = grillix.lineouts.rdpa(grid, equi, omp_map, norm, tcvx21.rdpa_coords_json)
xpt = grillix.lineouts.xpoint(grid, equi, norm)

lineouts = {"omp": omp, "lfs": lfs, "hfs": hfs, "ts": ts, "rdpa": rdpa, "xpt": xpt}

# %% [markdown] pycharm={"name": "#%% md\n"}
# We also want to be able to calculate parallel gradients. In GRILLIX, this
# is done via the FCI method. A simplified trace-and-interpolate method
# is demonstrated here. Note that since we're interpolating in Python,
# this is very slow!
#
# To speed things up a little, we trace only once per lineout, and only
# for the lineouts where we need parallel gradients.
#
# (Unfortunately, dask-based parallelism raises a strange `CancelledError` when
# these loops are attempted to be parallelized)

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
# %%time
grillix.observables.initialise_lineout_for_parallel_gradient(
    lfs,
    grid,
    equi,
    norm,
    params["params_grid"]["npol"],
    stored_trace=file_path / f"lfs_trace_{n_points}.nc",
)
# We don't compute the heat flux for the HFS, so can save time by not tracing for it

# %% [markdown] pycharm={"name": "#%% md\n"}
# As a sanity check, it's always a good idea to plot things before we perform the
# analysis.
#
# The lfs and hfs lineouts in GRILLIX are defined at positions which don't exactly
# correspond to the physical wall. This is because GRILLIX uses a smooth
# penalisation characteristic function. The lfs and hfs positions are defined as
# the surface where we *start* to apply boundary conditions. For more details
# on this method, see Stegmeir et al., 2019.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
divertor_ = read_from_json(tcvx21.divertor_coords_json)

_, ax = plt.subplots(figsize=(10, 10))
plt.contour(grid.r_s, grid.z_s, equi.normalised_flux_surface_label(grid.r_s, grid.z_s))

plt.scatter(
    lineouts["xpt"].r_points,
    lineouts["xpt"].z_points,
    s=1,
    marker=".",
    color="r",
    label="xpt",
)
plt.scatter(
    lineouts["rdpa"].r_points,
    lineouts["rdpa"].z_points,
    s=1,
    marker="+",
    color="k",
    label="rdpa",
)

for key, lineout in lineouts.items():

    if key in ["rdpa", "xpt"]:
        continue

    plt.plot(lineout.r_points, lineout.z_points, label=key, linewidth=2.5)

    if hasattr(lineout, "forward_lineout"):
        plt.plot(
            lineout.forward_lineout.r_points,
            lineout.forward_lineout.z_points,
            label=f"{key}+",
            linewidth=2.5,
        )
    if hasattr(lineout, "reverse_lineout"):
        plt.plot(
            lineout.reverse_lineout.r_points,
            lineout.reverse_lineout.z_points,
            label=f"{key}-",
            linewidth=2.5,
        )

plt.legend()
if equi.flipped_z:
    ax.invert_yaxis()
ax.set_aspect("equal")

plt.plot(
    divertor_["r_points"] / equi.axis_r.values,
    divertor_["z_points"] / equi.axis_r.values * -1.0 if equi.flipped_z else 1.0,
    color="k",
)

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Interfacing with the snaps
#
# The `snaps` object (an `xarray` labelled multi-dimensional array) provides
# convenient access to the simulation data. We mostly focus on 1D lineouts
# which can be compared to the experimental diagnostics, but we demonstrate
# here the use of the analysis components to show the density perturbation.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
fig, ax = plt.subplots()
grid.shape(
    snaps.density.isel(tau=0, phi=0) - snaps.density.mean(dim=("tau", "phi"))
).plot(shading="flat")

ax.invert_yaxis()
ax.set_aspect("equal")
ax.set_title("Density perturbation")

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Demonstrating the lineout functionality
#
# Looks good! Next, let's test the lineout functionality. If you just want
# normalised units, it's very easy using the xarray functionality.
#
# This one-liner has a lot going on, so let's unpack it
# 1. `lineouts[omp]` requests the stored omp `Lineout`
# 2. `.interpolate(` calls the interpolate method
# 3. `snaps.density` requests the density stored in the snaps
# 4. `).mean(dim='phi')` toroidally averages the result (since snaps.density has
#    `phi, tau, points` dimensions, the lineout will have `phi, tau, interp_points`
#    dimensions. We want to plot 2D, so average over `phi` to eliminate this dimension).
# 5. `.plot()` is a thin wrapper of `matplotlib.pyplot.pcolormesh`, which automatically
#    sets the x and y values of the plot from the xarray coordinates

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
lineouts["omp"].interpolate(snaps.density).mean(dim="tau").plot.contourf()

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# What about if we want this in SI units? Unfortunately, xarray isn't completely
# compatible with `pint`, so we have to manually convert to `pint.Quantity` objects
# and then set up the plotting manually.
#
# N.b. the underscore in the variable names is simply to indicate that these variables
# are private -- i.e. they shouldn't be used in other routines (since Python
# automatically promotes variables outside of functions to global scope. This
# is something to be careful about in Jupyter notebooks).
#
# You can see that the result doesn't vary much with time,
# but that's not surprising since 5 snaps is only 8 microseconds
# of plasma time!

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
tau_ = convert_xarray_to_quantity(snaps.tau).to("microseconds")
tau_ -= tau_[0]

lineout_ = lineouts["omp"]
lineout_rho_ = equi.normalised_flux_surface_label(
    lineout_.r_points, lineout_.z_points, grid=False
)
lineout_ru_ = convert_xarray_to_quantity(
    omp_map.convert_rho_to_distance(lineout_rho_)
).to("cm")

lineout_density_ = convert_xarray_to_quantity(
    lineout_.interpolate(snaps.density).mean(dim="phi")
).to("1/m^3")

fig, ax = plt.subplots()

im = ax.contourf(lineout_ru_, tau_, lineout_density_, shading="nearest")
plt.colorbar(im, ax=ax)

ax.set_ylabel("$\\tau$ [$\\mu$ s]")
ax.set_xlabel("$R^u - R^u_{sep}$ [cm]")
ax.set_title("OMP density [$1/m^3$]")

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Synthetic diagnostics and bootstrap
#
# For observables which do not directly map to the simulation dynamic quantities,
# we need a method to estimate what the measured value will be given plasma
# parameters from the simulation. These can be roughly termed *synthetic
# diagnostics*, although we use here only simple approximations. The functions
# are stored in the `experimental_quantities_m.py` file.
#
# We show here how to calculate the skewness of the ion saturation current measured the low-field-side
# target Langmuir probe array.
#
# This method also gives an estimate of the uncertainty of the statistical moment via the
# bootstrap method. The bootstrap method takes several random samples of the signal (potentially
# double-counting some points and omitting others) and then calculates the mean
# and standard deviation of the statistical moments calculated from the random samples.
# Strictly speaking it should be applied for purely random data, rather than correlated
# data, but we use it nevertheless because it is a nice and simple approximation of the
# finite-sampling error.
#
# For the mean, the bootstrap error is tiny, but for the higher-order statistical moments it
# can be appreciable.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
lineout_ = lineouts["lfs"]
lineout_rho_ = equi.normalised_flux_surface_label(
    lineout_.r_points, lineout_.z_points, grid=False
)
lineout_ru_ = convert_xarray_to_quantity(
    omp_map.convert_rho_to_distance(lineout_rho_)
).to("cm")

sound_speed_ = grillix.observables.sound_speed(
    electron_temp=snaps.electron_temp, ion_temp=snaps.ion_temp, norm=norm
)

jsat_ = grillix.observables.ion_saturation_current(
    density=snaps.density, sound_speed=sound_speed_, norm=norm, wall_probe=True
)

jsat_skew_, jsat_skew_err_ = tcvx21.analysis.compute_statistical_moment_with_bootstrap(
    lineout_.interpolate(jsat_).rename({"interp_points": "points"}), moment="skew"
)

jsat_skew_ = convert_xarray_to_quantity(jsat_skew_)
jsat_skew_err_ = convert_xarray_to_quantity(jsat_skew_err_)

fig, ax = plt.subplots()

ax.plot(lineout_ru_, jsat_skew_)
ax.fill_between(
    lineout_ru_, jsat_skew_ + jsat_skew_err_, jsat_skew_ - jsat_skew_err_, alpha=0.5
)

ax.set_xlabel("$R^u - R^u_{sep}$ [cm]")
ax.set_title("LFS ion saturation current skewness")

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Heat flux profile fitting
#
# A more involved synthetic diagnostic is the parallel heat flux to the
# boundaries. From our anomalous heat transmission boundary conditions, we can
# determine the heat flux to the boundaries as
#
# $q_{\parallel,e} = \frac{5}{2}n v_{\parallel,e}T_e + \frac{5}{2}n u_{E\times B}T_e - \chi_{\parallel e,0} T_e^{5/2}\nabla_\parallel T_e$
#
# $q_{\parallel,i} = \frac{5}{2}n u_{\parallel,i}T_i + \frac{5}{2}n u_{E\times B}T_i - \chi_{\parallel i,0} T_i^{5/2}\nabla_\parallel T_i$
#
# We can check the heat flux routines and the contribution from each component here.
# Note that, unless the $E \times B$ contribution is included, the heat flux
# can become negative.
#
# You notice that there's a lot of terms in this equation.
# $q_\parallel$ isn't directly evolved in the code, so it has
# a lower simulation hierarchy than other simpler observables.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
lineout_ = lineouts["lfs"]
lineout_rho_ = equi.normalised_flux_surface_label(
    lineout_.r_points, lineout_.z_points, grid=False
)
lineout_ru_ = convert_xarray_to_quantity(
    omp_map.convert_rho_to_distance(lineout_rho_)
).to("cm")

density = lineout_.interpolate(snaps.density)
ion_velocity = lineout_.interpolate(snaps.velocity)
current = lineout_.interpolate(snaps.current)
electron_temp = lineout_.interpolate(snaps.electron_temp)
electron_temp_parallel_gradient = grillix.observables.compute_gradient_on_plane(
    lineout_, snaps.electron_temp, plane=0
)
ion_temp = lineout_.interpolate(snaps.ion_temp)
ion_temp_parallel_gradient = grillix.observables.compute_gradient_on_plane(
    lineout_, snaps.ion_temp, plane=0
)
effective_parallel_exb = lineout_.interpolate(
    grillix.observables.effective_parallel_exb_velocity(
        grid, equi, norm, snaps.potential
    )
)

q_e_conv = grillix.observables.heat_flux.electron_parallel_heat_convection(
    density, electron_temp, ion_velocity, current, norm
).isel(tau=0, phi=0)
q_i_conv = grillix.observables.heat_flux.ion_parallel_heat_convection(
    density, ion_temp, ion_velocity, norm
).isel(tau=0, phi=0)
q_e_cond = grillix.observables.heat_flux.electron_parallel_heat_conduction(
    electron_temp, electron_temp_parallel_gradient, norm
).isel(tau=0, phi=0)
q_i_cond = grillix.observables.heat_flux.ion_parallel_heat_conduction(
    ion_temp, ion_temp_parallel_gradient, norm
).isel(tau=0, phi=0)
q_e_exb = grillix.observables.heat_flux.exb_effective_parallel_heat_convection(
    density, electron_temp, effective_parallel_exb, norm
).isel(tau=0, phi=0)
q_i_exb = grillix.observables.heat_flux.exb_effective_parallel_heat_convection(
    density, ion_temp, effective_parallel_exb, norm
).isel(tau=0, phi=0)

q_par = grillix.observables.total_parallel_heat_flux(
    density,
    electron_temp,
    electron_temp_parallel_gradient,
    ion_temp,
    ion_temp_parallel_gradient,
    ion_velocity,
    current,
    effective_parallel_exb,
    norm,
).load()

fig, ax = plt.subplots()

ax.plot(lineout_ru_, convert_xarray_to_quantity(q_e_conv), label="q_e_conv")
ax.plot(lineout_ru_, convert_xarray_to_quantity(q_i_conv), label="q_i_conv")
ax.plot(lineout_ru_, convert_xarray_to_quantity(q_e_cond), label="q_e_cond")
ax.plot(lineout_ru_, convert_xarray_to_quantity(q_i_cond), label="q_i_cond")
ax.plot(lineout_ru_, convert_xarray_to_quantity(q_i_exb), label="q_i_exb")
ax.plot(lineout_ru_, convert_xarray_to_quantity(q_e_exb), label="q_e_exb")

ax.plot(
    lineout_ru_,
    convert_xarray_to_quantity(q_par.isel(tau=0, phi=0)),
    "k--",
    label="total",
)

ax.legend()

fig, ax = plt.subplots()
_, lambda_q, _, _, _ = tcvx21.analysis.fit_eich_profile(
    lineout_ru_, convert_xarray_to_quantity(q_par.mean(dim=("tau", "phi"))), plot=True
)

print(f"lambda_q = {lambda_q[0]:4.3}±{lambda_q[1]:4.3}")

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## 2D data from the divertor volume
#
# As well as 1D profiles, we also have 2D profiles from the reciprocating
# divertor probe array. Since the plasma can move relative to the wall,
# we use $R^u - R^u_{omp}$ and $Z$ instead of the original $R, Z$ coordinates.
# This means that our sample grid is not rectangular. We need to sample our
# data on the unstructured grid.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from scipy.interpolate import griddata

def plot_rdpa_field(rdpa_lineout, field, n_rsep=50, n_z=50):

    x = rdpa_lineout.coords["Rsep"]
    y = rdpa_lineout.coords["Z"]
    assert x.shape == y.shape
    z = rdpa_lineout.interpolate(field)

    x_sample = np.linspace(x.min(), x.max(), num=n_rsep)
    y_sample = np.linspace(y.min(), y.max(), num=n_z)
    x_mesh, y_mesh = np.meshgrid(x_sample, y_sample)

    z_sample = griddata(
        points=(x.magnitude, y.magnitude),
        values=z,
        xi=(x_mesh.magnitude, y_mesh.magnitude),
    )

    z_sample *= z.norm

    plt.pcolormesh(x_sample, y_sample, z_sample, shading="nearest")
    plt.colorbar()
    plt.xlabel("$R^u - R^u_{sep}$" + f" [{x.units}]")
    plt.ylabel(f"Z [{y.units}]")
    plt.title(f"RDPA {z.name} [{z_sample.units}]")

plt.figure()
plot_rdpa_field(lineouts["rdpa"], snaps.electron_temp.mean(dim=("phi", "tau")))

if test_session:
    plt.close("all")

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Integral values for sources
#
# As well as profiles, we are also interested in integrals of the sources, to determine whether
# we are using a source in the region of the experimental values.
#
# The sources in GRILLIX are given in the file tcvx21/grillix_analysis/sources_m.py. In GRILLIX we have
# set a 'constant-power, constant-particle' source. That is, we actively adapt our electron-temperature
# source to compensate for the power due to the particle source. This way, we inject a constant
# 150kW exactly. This is deposited in the core as a temperature source. However, since the density source
# also contributes to the power source we compensate for this by adding a negative temperature source
# at the edge.
#
# $P = \frac{3}{2}\int n S_{T_e} + (T_e + T_i) S_n \textrm{d}^3V$
#
# We are using
# $S_n = n_0 / \tau_0 \hat{S}_n$
# where
#
# $\hat{S}_n = \hat{S}_{n0} f_{S,n}(R,Z) = \hat{S}_{n0} \exp\left[-\left(\rho(R,Z)^2 - \rho_{cn}^2 \right)/\rho_{wn}^2\right]$
#
# for $\rho_{cn} = 0.815$ and $\rho_{wn} =0.083$, and
#
# $S_{Te} = T_{0} / \tau_0 \left(\frac{1}{\hat{n}}\hat{S}_{Te} - \frac{\hat{T}_e + \hat{T}_i}{\hat{n}} \hat{S}_n \right)$
#
# where
#
# $\hat{S}_{Te} = \hat{S}_{Te0} f_{S,Te}(R,Z) = \hat{S}_{Te0} \left(1 - \mathcal{S}_3(\rho, \rho_{cT}, \rho_{wT})\right)$
#
# for $\mathcal{S}_3(x, c, w)$ is a third-order [smoothstep](https://en.wikipedia.org/wiki/Smoothstep) function
# centred at $c$ and of transition width $w$.
#
# This allows us to write
#
# $P = \frac{3}{2}\iiint n_0 \hat{n} T_{0} / \tau_0 \left(\frac{1}{\hat{n}}\hat{S}_{Te} - \frac{\hat{T}_e + \hat{T}_i}{\hat{n}} \hat{S}_n \right) + T_0(\hat{T}_e + \hat{T}_i) n_0 / \tau_0 \hat{S}_n \textrm{d}^3V$
#
# $P= \frac{3}{2}\frac{n_0 T_0}{\tau_0} \iiint \hat{S}_{Te} - (\hat{T}_e + \hat{T}_i) \hat{S}_n + (\hat{T}_e + \hat{T}_i) \hat{S}_n \textrm{d}^3V$
#
# $P= \frac{3}{2}\frac{n_0 T_0}{\tau_0} \hat{S}_{Te0} \iiint f_{S,Te}(R,Z) \textrm{d}^3V$
#
# $P= \frac{3}{2}\frac{n_0 T_0}{\tau_0} \hat{S}_{Te0} \mathcal{V}_w$
#
# That is, our power injection is determined entirely by our core electron temperature source.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
grillix.components.integrated_sources(grid, equi, norm, params, snaps)

# %% [markdown] pycharm={"name": "#%% md\n"}
# ## Iterating over all observables
#
# The final step to producing a standard NetCDF file for comparison is to iterate over each element
# of the standard dictionary and fill the dictionary with results.
# We then use the `RecordWriter`
# to convert the standard dictionary into a NetCDF file.
#
# Since the raw data is very large (does not fit in memory), we do a two-step post-processing. In the first step, we iterate
# over the time points in our raw data to compute the raw observables at the observable positions.
# These are iteratively written to an intermediate processing file. In the second
# step, we perform statistics over the processing file, calculating the data used for the validation
# analysis.
#
# The first step is done with the `fill_work_file()` method of the `DataExtract` object.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from tcvx21.grillix_post.work_file_writer_m import WorkFileWriter

(file_path / "processing_file.nc").unlink(missing_ok=True)

data = WorkFileWriter(
    file_path=file_path,
    work_file=file_path / "processing_file.nc",
    toroidal_field_direction="forward",
    make_work_file=True,
)

data.fill_work_file()

# %% [markdown] pycharm={"name": "#%% md\n"}
# In the second step, we need to make a `Record`, which is a NetCDF file formatted according
# to the data template in `observables.json`.
#
# The validation is done in terms of statistical moments, rather than raw values.
#
# To compute statistical values and the uncertainty associated with it, we use the bootstrap
# method. We take samples from all toroidal planes, and 1ms of data (500 snaps).
#
# We select values from the samples available for each observable point *with replacement*. This means that some points may be double-counted,
# and some may be missed entirely.
#
# We repeat this a number of times, to generate a number of *tests*.
#
# Finally, we compute the median of the tests and a confidence interval -- which gives us a
# robust estimator for the statistical moment and its uncertainty.
#
# The 65% and 95% confidence intervals are shown for the different statistical moments of $J_{sat}$
#
# Note that since we only have 5 time points here, the uncertainty is
# higher than for the real result with 500 time points.

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
import xarray as xr

def Q(netcdf_array):
    """Converts netcdf arrays to Quantities"""
    return Quantity(netcdf_array.values, netcdf_array.units)

def plot_region(x, y, dy, label):
    plt.plot(x, y, label=label)
    plt.fill_between(x, y - dy, y + dy, alpha=0.2)

work_file_path = file_path / "processing_file.nc"
values = (
    xr.open_dataset(work_file_path, group="LFS-LP/observables")
    .jsat.isel(tau=slice(-500, None))
    .persist()
)
Rsep = Q(xr.open_dataset(work_file_path, group="LFS-LP").Rsep)

for moment in ["mean", "std", "skew", "kurt"]:
    plt.figure()

    mean, error = tcvx21.analysis.compute_statistical_moment_with_bootstrap(
        values=values, moment=moment, ci=0.95, n_tests=1000
    )
    plot_region(Rsep, Q(mean), Q(error), label="95%")

    mean, error = tcvx21.analysis.compute_statistical_moment_with_bootstrap(
        values=values, moment=moment, ci=0.65, n_tests=1000
    )
    plot_region(Rsep, Q(mean), Q(error), label="65%")

    plt.legend(title="Confidence interval")
    plt.title(f"$J_{{sat}}$ {moment} on low-field-side target")

if test_session:
    plt.close("all")

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from netCDF4 import Dataset

standard_dict = read_from_json(tcvx21.template_file)
simulation_hierarchy = read_from_json(file_path / "simulation_hierarchy.json")
work_file_path = file_path / "processing_file.nc"

dataset = Dataset(work_file_path)

def strip_moment(observable_key):
    if observable.endswith("_std"):
        moment = "std"
        key = observable_key.rstrip("std").rstrip("_")
    elif observable.endswith("_skew"):
        moment = "skew"
        key = observable_key.rstrip("skew").rstrip("_")
    elif observable.endswith("_kurtosis"):
        moment = "kurt"
        key = observable_key.rstrip("kurtosis").rstrip("_")
    else:
        moment = "mean"
        key = observable_key
    return key, moment

def write_observable(diagnostic_key, observable_key, output_dict):

    print(f"\tProcessing {diagnostic_key}:{observable_key}")

    observable_key, moment = strip_moment(observable_key)

    diagnostic = xr.open_dataset(work_file_path, group=diagnostic_key)
    observable = xr.open_dataset(work_file_path, group=f"{diagnostic_key}/observables")[
        observable_key
    ]
    observable = observable.isel(tau=slice(-500, None)).persist()

    output_dict["simulation_hierarchy"] = simulation_hierarchy[
        observable_key if moment != "lambda_q" else moment
    ]

    value, error = tcvx21.analysis.compute_statistical_moment_with_bootstrap(
        observable, moment=moment
    )

    output_dict["values"] = Q(value).to(output_dict["units"]).magnitude
    output_dict["errors"] = Q(error).to(output_dict["units"]).magnitude

    for variable_key in diagnostic.variables.keys():
        variable = diagnostic[variable_key]
        output_key = variable_key.replace("Rsep", "Ru")

        output_dict[output_key] = variable.values
        output_dict[f"{output_key}_units"] = getattr(variable, "units", "")

print("Filling standard dict")

for diagnostic, diagnostic_dict in standard_dict.items():
    for observable, observable_dict in diagnostic_dict["observables"].items():
        write_observable(diagnostic, observable, observable_dict)

print("Done")

# %% [markdown] pycharm={"name": "#%% md\n"}
# Finally, we write this into a standard NetCDF

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from tcvx21.record_c.record_writer_m import RecordWriter

writer = RecordWriter(
    file_path=file_path / "GRILLIX_example.nc",
    descriptor="GRX",
    description=Path(file_path / "description.txt").read_text(),
    allow_overwrite=True,
)

writer.write_data_dict(standard_dict)

# %% [markdown] pycharm={"name": "#%% md\n"}
# To perform the entire setup and file write in a single step, you can also use
# the following command, although this is more for convenience

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from tcvx21.grillix_post.validation_writer_m import (
    convert_work_file_to_validation_netcdf,
)

convert_work_file_to_validation_netcdf(
    work_file=file_path / "processing_file.nc",
    output_file=file_path / "GRILLIX_example.nc",
    simulation_hierarchy=tcvx21.read_from_json(
        tcvx21.grillix_dir / "simulation_hierarchy.json"
    ),
)

# %% [markdown] pycharm={"name": "#%% md\n"}
# We can then load this as a `Record` -- a NetCDF file which has a rigid common
# structure shared between the simulations and the experiment.
#
# This allows us to write interfaces to the data -- i.e. plotting routines -- and
# don't have to worry about which data source the data is coming from

# %% jupyter={"outputs_hidden": false} pycharm={"name": "#%%\n"}
from tcvx21.record_c.record_m import Record

ds = Record(file_path / "GRILLIX_example.nc", color="C1", label="GRILLIX")

plt.figure()
ds.get_observable("LFS-LP", "density").plot()

plt.figure()
ds.get_observable("RDPA", "density").plot(log_cbar=True)

if test_session:
    plt.close("all")
# %%


#!/usr/bin/env python
# coding: utf-8

# **Data cleaning&Classification algorithms comparison**

# **In this study, chronic kidney disease was estimated using classification algorithms. You'll find this kernel;**
#             * The codes I think will be useful for data cleaning
#             * How to Handle Missing Data,what did I do?
#             * Data Visualization
#             *  Classification Algorithms
#                 -KNN
#                 -Navie-Bayes
#                 -Logistic Regression
#                 -Decision Tree
#                 -Random Forest
#                 -Support Vector Machine
#             * Success rate of classification algorithms
#             * Conclusion
#
#

# In[ ]:

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt
import seaborn as sns #for confusion matrix
#For data visualization
import plotly.plotly as py
from plotly.offline import init_notebook_mode, iplot
init_notebook_mode(connected=True)
import plotly.graph_objs as go
# Input data files are available in the "../../../input/mansoordaku_ckdisease/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import warnings
warnings.filterwarnings('ignore')

import os
print(os.listdir("../../../input/mansoordaku_ckdisease"))

# Any results you write to the current directory are saved as output.

# In[ ]:

data=pd.read_csv("../../../input/mansoordaku_ckdisease/kidney_disease.csv")

# In[ ]:

data.info() #data types and feature names.
            #I'll change the data types of some parameters in the following codes

# In[ ]:

data.head() #first 5 samples in dataset

# In[ ]:

data.classification.unique()

# **PREPARE DATA**
#
# **1) 3 unique values appear in the data set. However, there is no value called  "ckd\t. "  I have written the following code to solve this problem.**

# In[ ]:

data.classification=data.classification.replace("ckd\t","ckd")

# In[ ]:

data.classification.unique() #problem solved.

# **2) The "id" parameter will not work for classification, so I'm removing this parameter from the data set.**

# In[ ]:

data.drop("id",axis=1,inplace=True)

# In[ ]:

data.head() #id parameter dropped.

# **3) I changed the target parameter values to 1 and 0 to be able to use the classification algorithms. If the value is "ckd" is 1, if not equal to 0. **

# In[ ]:

data.classification=[1 if each=="ckd" else 0 for each in data.classification]

# In[ ]:

data.head()

# **4) I used the following code to find out how many "NaN" values in which parameter.**

# In[ ]:

data.isnull().sum()

#
# **5)Sometimes instead of Nan in the data set, "?" value can be found. To solve this problem ; **df = data [(data! = '?'). all (axis = 1)]** can be used.**
#

#                        **How to Handle Missing Data,what did I do?**
#
# Pandas provides the dropna() function that can be used to drop either columns or rows with missing data. We can use dropna() to remove all rows with missing data
# Removing rows with missing values can be too limiting on some predictive modeling problems, an alternative is to impute missing values.
# Imputing refers to using a model to replace missing values.
#
# There are many options we could consider when replacing a missing value, for example:
#
#     A constant value that has meaning within the domain, such as 0, distinct from all other values.
#     A value from another randomly selected record.
#     A mean, median or mode value for the column.
#     A value estimated by another predictive model.
#
# Pandas provides the fillna() function for replacing missing values with a specific value.
#
# For example, we can use fillna() to replace missing values with the mean value for each column,
# For example; dataset.fillna(dataset.mean(), inplace=True)

# **6) I can use dropna() to remove all rows with missing data**
#
# There were 25 parameters of 400 samples before writing this code. After writing this code, there are 25 parameters left in 158 examples.
#
# The number of samples decreased but the reliability of the model increased.

# In[ ]:

df=data.dropna(axis=0)
print(data.shape)
print(df.shape)
df.head()

# **7) indexes are not sequential as you can see in the table above. I used the following code to sort indexes.**

# In[ ]:

df.index=range(0,len(df),1)
df.head()

# **8) I corrected some of the parameters.**

# In[ ]:

#you can see that the values have changed.
df.wc=df.wc.replace("\t6200",6200)
df.wc=df.wc.replace("\t8400",8400)
print(df.loc[11,["wc"]])
print(df.loc[20,["wc"]])

# **9) I'll change the data types of some parameters **

# In[ ]:

df.pcv=df.pcv.astype(int)
df.wc=df.wc.astype(int)
df.rc=df.rc.astype(float)
df.info()

# **10)Keep in mind, the goal in this section is to have all the columns as numeric columns (int or float data type), and containing no missing values. We just dealt with the missing values, so let's now find out the number of columns that are of the object data type and then move on to process them into numeric form.**

# In[ ]:

dtype_object=df.select_dtypes(include=['object'])
dtype_object.head()

# **11)display a sample row to get a better sense of how the values in each column are formatted.**

# In[ ]:

for x in dtype_object.columns:
    print("{} unique values:".format(x),df[x].unique())
    print("*"*20)

# **12)The ordinal values to integers, we can use the pandas DataFrame method replace() to"rbc","pc","pcc","ba","htn","dm","cad","appet","pe" and "ane" to appropriate numeric values**

# In[ ]:

dictonary = { "rbc": { "abnormal":1, "normal": 0, }, "pc":{ "abnormal":1, "normal": 0, }, "pcc":{ "present":1, "notpresent":0, }, "ba":{ "notpresent":0, "present": 1, }, "htn":{ "yes":1, "no": 0, }, "dm":{ "yes":1, "no":0, }, "cad":{ "yes":1, "no": 0, }, "appet":{ "good":1, "poor": 0, }, "pe":{ "yes":1, "no":0, }, "ane":{ "yes":1, "no":0, } }

# In[ ]:

#We used categorical values as numerical to replace them.
df=df.replace(dictonary)

# In[ ]:

df.head() #All values are numerical.

# **VISUALIZATION**

# In[ ]:

#HEAT MAP #correlation of parameters
f,ax=plt.subplots(figsize=(15,15))
print()
plt.xticks(rotation=45)
plt.yticks(rotation=45)
print()

# In[ ]:

#box-plot
trace0 = go.Box( y=df.bp, name = 'Bp', marker = dict( color = 'rgb(12, 12, 140)', ) )
trace1 = go.Box( y=df.sod, name = 'Sod', marker = dict( color = 'rgb(12, 128, 128)', ) )
data = [trace0, trace1]
iplot(data)

# In[ ]:

#Line plot
df2=df.copy()
df2["id"]=range(1,(len(df.ba)+1),1)
df2["df2_bp_norm"]=(df2.bp-np.min(df2.bp))/(np.max(df2.bp)-np.min(df2.bp))
df2["df2_hemo_norm"]=(df2.hemo-np.min(df2.hemo))/(np.max(df2.hemo)-np.min(df2.hemo))
#Line Plot
trace1 = go.Scatter( x = df2.id, y = df2.df2_bp_norm, mode = "lines", name = "Blood Press.", marker = dict(color = 'rgba(16, 112, 2, 0.8)'), text= df.age)
trace2 = go.Scatter( x = df2.id, y = df2.df2_hemo_norm, mode = "lines+markers", name = "Hemo", marker = dict(color = 'rgba(80, 26, 80, 0.8)'), text= df.age)
data=[trace1,trace2]
layout=dict(title="Blood Press and Hemoglobin values according the age", xaxis=dict(title="İd",ticklen=5,zeroline=False))
fig=dict(data=data,layout=layout)
iplot(fig)

# **CLASSIFICATION ALGORITHMS**

# In[ ]:

score=[] #these variables will be used to show the algorithm name and its successes.
algorithms=[]

# In[ ]:

#KNN
from sklearn.neighbors import KNeighborsClassifier
y=df["classification"].values
x_data=df.drop(["classification"],axis=1)

#Normalization
x=(x_data-np.min(x_data))/(np.max(x_data)-np.min(x_data))

#Preparing the test and training set
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=1,test_size=0.3)

#model and accuracy
knn=KNeighborsClassifier(n_neighbors=3)
knn.fit(x_train,y_train)
knn.predict(x_test)
score.append(knn.score(x_test,y_test)*100)
algorithms.append("KNN")
print("KNN accuracy =",knn.score(x_test,y_test)*100)

#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=knn.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)

#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title(" KNN Confusion Matrix")
print()
#%%

# In[ ]:

#Navie-Bayes
from sklearn.naive_bayes import GaussianNB
nb=GaussianNB()

#Training
nb.fit(x_train,y_train)
#Test
score.append(nb.score(x_test,y_test)*100)
algorithms.append("Navie-Bayes")
print("Navie Bayes accuracy =",nb.score(x_test,y_test)*100)

#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=nb.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)

#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title("Navie Bayes Confusion Matrix")
print()

# In[ ]:

#RANDOM FOREST
from sklearn.ensemble import RandomForestClassifier
rf=RandomForestClassifier(n_estimators=100,random_state=1)
rf.fit(x_train,y_train)
score.append(rf.score(x_test,y_test)*100)
algorithms.append("Random Forest")
print("Random Forest accuracy =",rf.score(x_test,y_test))

#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=rf.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)

#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title("Random Forest Confusion Matrix")
print()

# In[ ]:

#Support Vector Machine
from sklearn.svm import SVC
svm=SVC(random_state=1)
svm.fit(x_train,y_train)
score.append(svm.score(x_test,y_test)*100)
algorithms.append("Support Vector Machine")
print("svm test accuracy =",svm.score(x_test,y_test)*100)

#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=svm.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)

#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title("Support Vector Machine Confusion Matrix")
print()

# In[ ]:

#Decision Tree
from sklearn.tree import DecisionTreeClassifier
dt=DecisionTreeClassifier()
dt.fit(x_train,y_train)
print("Decision Tree accuracy:",dt.score(x_test,y_test)*100)
score.append(dt.score(x_test,y_test)*100)
algorithms.append("Decision Tree")

#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=dt.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)

#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title("Decision Tree Confusion Matrix")
print()

# In[ ]:

from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(x_train,y_train)
score.append(lr.score(x_test,y_test)*100)
algorithms.append("Logistic Regression")
print("test accuracy {}".format(lr.score(x_test,y_test)))
#Confusion Matrix
from sklearn.metrics import confusion_matrix
y_pred=lr.predict(x_test)
y_true=y_test
cm=confusion_matrix(y_true,y_pred)
#Confusion Matrix on Heatmap
f,ax=plt.subplots(figsize=(5,5))
print()
plt.xlabel("y_pred")
plt.ylabel("y_true")
plt.title("Logistic Regression Confusion Matrix")
print()

# In[ ]:

trace1 = { 'x': algorithms, 'y': score, 'name': 'score', 'type': 'bar' }

# In[ ]:

data = [trace1];
layout = { 'xaxis': {'title': 'Classification Algorithms'}, 'title': 'Comparison of the accuracy of classification algorithms' };
fig = go.Figure(data = data, layout = layout)
iplot(fig)

# **CONCLUSION**
#
# In this study, there were 400 samples and 26 parameters.However, some samples had no parameter values. For this reason, I prepared the data to use the classification algorithms. I did data visualization work.  I applied classification algorithms and compared success rates with each other.   It was a nice work for me. I hope you like it.
# If you have questions and suggestions, you can comment. Because your questions and suggestions are very valuable for me.
#


"""
@brief      test log(time=1s)
"""

import sys
import os
import unittest
import pandas
import numpy
from pyquickhelper.loghelper import fLOG

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.ensae_teaching_cs.faq.faq_pandas import groupby_topn, df_equal

class TestFaqPandas(unittest.TestCase):

    def test_groupby_sort_head(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        ld = [dict(k1="a", k2="b", v=4, i=1),
              dict(k1="a", k2="b", v=5, i=1),
              dict(k1="a", k2="b", v=4, i=2),
              dict(k1="b", k2="b", v=1, i=2),
              dict(k1="b", k2="b", v=1, i=3)]

        exp = [dict(k1="a", k2="b", v=4, i=1),
               dict(k1="b", k2="b", v=1, i=2)]

        df = pandas.DataFrame(ld)
        exp = pandas.DataFrame(exp)

        res = groupby_topn(df, by_keys=["k1", "k2"],
                           sort_keys=["v", "i"], as_index=False)
        b = df_equal(exp, res)
        if not b:
            raise Exception(
                "dataframe not equal\nRES:\n{0}\nEXP\n{1}".format(str(res), str(exp)))

    def test_df_equal(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        exp1 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        assert df_equal(exp1, exp2)

        exp1 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = exp2[["k2", "k1", "v", "i"]]
        assert df_equal(exp1, exp2)

        exp1 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=3, i=1), dict(k1="b", k2="b", v=1, i=2)])
        assert not df_equal(exp1, exp2)

        exp1 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=4, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=numpy.nan, i=1), dict(k1="b", k2="b", v=1, i=2)])
        assert not df_equal(exp1, exp2)

        exp1 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=numpy.nan, i=1), dict(k1="b", k2="b", v=1, i=2)])
        exp2 = pandas.DataFrame(
            [dict(k1="a", k2="b", v=numpy.nan, i=1), dict(k1="b", k2="b", v=1, i=2)])
        assert not df_equal(exp1, exp2)

if __name__ == "__main__":
    unittest.main()


"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

import pytest

import pytablewriter as ptw

from ...._common import print_test_result
from ....data import (
    Data,
    headers,
    mix_header_list,
    mix_value_matrix,
    null_test_data_list,
    value_matrix,
    value_matrix_iter,
    value_matrix_with_none,
)

try:
    import numpy as np  # noqa: W0611

    SKIP_DATAFRAME_TEST = False
except ImportError:
    SKIP_DATAFRAME_TEST = True

normal_test_data_list = [
    Data(
        table="table-name ho'ge",
        indent=0,
        header=headers,
        value=value_matrix,
        expected="""table_name_ho_ge = np.array([
    ["a", "b", "c", "dd", "e"],
    [1, 123.1, "a", 1, 1],
    [2, 2.2, "bb", 2.2, 2.2],
    [3, 3.3, "ccc", 3, "cccc"],
])
""",
    ),
    Data(
        table="empty value",
        indent=0,
        header=headers,
        value=None,
        expected="""empty_value = np.array([
    ["a", "b", "c", "dd", "e"],
])
""",
    ),
    Data(
        table="table with%null-value",
        indent=0,
        header=headers,
        value=value_matrix_with_none,
        expected="""table_with_null_value = np.array([
    ["a", "b", "c", "dd", "e"],
    [1, None, "a", 1, None],
    [None, 2.2, None, 2.2, 2.2],
    [3, 3.3, "ccc", None, "cccc"],
    [None, None, None, None, None],
])
""",
    ),
    Data(
        table="mix data types",
        indent=0,
        header=mix_header_list,
        value=mix_value_matrix,
        expected="""mix_data_types = np.array([
    ["i", "f", "c", "if", "ifc", "bool", "inf", "nan", "mix_num", "time"],
    [1, 1.1, "aa", 1, 1, True, np.inf, np.nan, 1, dateutil.parser.parse("2017-01-01T00:00:00")],
    [2, 2.2, "bbb", 2.2, 2.2, False, np.inf, np.nan, np.inf, "2017-01-02 03:04:05+09:00"],
    [3, 3.33, "cccc", -3, "ccc", True, np.inf, np.nan, np.nan, dateutil.parser.parse("2017-01-01T00:00:00")],
])
""",
    ),
    Data(
        table="mix data types wo header",
        indent=0,
        header=None,
        value=mix_value_matrix,
        expected="""mix_data_types_wo_header = np.array([
    [1, 1.1, "aa", 1, 1, True, np.inf, np.nan, 1, dateutil.parser.parse("2017-01-01T00:00:00")],
    [2, 2.2, "bbb", 2.2, 2.2, False, np.inf, np.nan, np.inf, "2017-01-02 03:04:05+09:00"],
    [3, 3.33, "cccc", -3, "ccc", True, np.inf, np.nan, np.nan, dateutil.parser.parse("2017-01-01T00:00:00")],
])
""",
    ),
    Data(
        table="float-with-null",
        indent=0,
        header=["a", "b"],
        value=[
            ["0.03785679191278808", "826.21158713263"],
            [None, "826.21158713263"],
            [0.1, "1.0499675627886724"],
        ],
        expected="""float_with_null = np.array([
    ["a", "b"],
    [0.03785679191278808, 826.21158713263],
    [None, 826.21158713263],
    [0.1, 1.0499675627886724],
])
""",
    ),
]

table_writer_class = ptw.NumpyTableWriter

class Test_NumpyTableWriter_write_new_line:
    def test_normal(self, capsys):
        writer = table_writer_class()
        writer.write_null_line()

        out, _err = capsys.readouterr()

        assert out == "\n"

class Test_NumpyTableWriter_write_table:
    @pytest.mark.parametrize(
        ["table", "indent", "header", "value", "expected"],
        [
            [data.table, data.indent, data.header, data.value, data.expected]
            for data in normal_test_data_list
        ],
    )
    def test_normal(self, capsys, table, indent, header, value, expected):
        writer = table_writer_class()
        writer.table_name = table
        writer.set_indent_level(indent)
        writer.headers = header
        writer.value_matrix = value
        writer.write_table()

        out, err = capsys.readouterr()
        print_test_result(expected=expected, actual=out, error=err)

        assert out == expected

    @pytest.mark.parametrize(
        ["table", "indent", "header", "value", "expected"],
        [
            [data.table, data.indent, data.header, data.value, data.expected]
            for data in null_test_data_list
        ],
    )
    def test_exception_null(self, table, indent, header, value, expected):
        writer = table_writer_class()
        writer.table_name = table
        writer.set_indent_level(indent)
        writer.headers = header
        writer.value_matrix = value

        assert writer.dumps() == ""

    @pytest.mark.parametrize(
        ["table", "indent", "header", "value", "expected"],
        [
            [data.table, data.indent, data.header, data.value, data.expected]
            for data in [
                Data(
                    table=None,
                    indent=0,
                    header=headers,
                    value=value_matrix,
                    expected="",
                )
            ]
        ],
    )
    def test_exception(self, table, indent, header, value, expected):
        writer = table_writer_class()
        writer.table_name = table
        writer.set_indent_level(indent)
        writer.headers = header
        writer.value_matrix = value

        with pytest.raises(ptw.EmptyTableNameError):
            writer.write_table()

class Test_NumpyTableWriter_write_table_iter:
    @pytest.mark.parametrize(
        ["table", "header", "value", "expected"],
        [
            [
                "tablename",
                ["ha", "hb", "hc"],
                value_matrix_iter,
                """tablename = np.array([
    ["ha", "hb", "hc"],
    [1, 2, 3],
    [11, 12, 13],
    [1, 2, 3],
    [11, 12, 13],
    [101, 102, 103],
    [1001, 1002, 1003],
])
""",
            ]
        ],
    )
    def test_normal(self, capsys, table, header, value, expected):
        writer = table_writer_class()
        writer.table_name = table
        writer.headers = header
        writer.value_matrix = value
        writer.iteration_length = len(value)
        writer.write_table_iter()

        out, _err = capsys.readouterr()

        assert out == expected

    @pytest.mark.parametrize(
        ["table", "header", "value", "expected"],
        [[data.table, data.header, data.value, data.expected] for data in null_test_data_list],
    )
    def test_normal_empty(self, table, header, value, expected):
        writer = table_writer_class()
        writer.table_name = table
        writer.headers = header
        writer.value_matrix = value

        writer.write_table_iter()


![Callysto.ca Banner](https://github.com/callysto/curriculum-notebooks/blob/master/callysto-notebook-banner-top.jpg?raw=true)

%%html

 Code is hidden for ease of viewing. Click the Show/Hide button to see. </>
<form action="javascript:code_toggle()"><input type="submit" id="toggleButton" value="Show Code"></form>

# Modules

import string
import numpy as np
import pandas as pd
import qgrid as q
import matplotlib.pyplot as plt

# Widgets & Display modules, etc..

from ipywidgets import widgets as w
from ipywidgets import Button, Layout
from IPython.display import display, Javascript, Markdown, HTML

# grid features for interactive grids

grid_features = { 'fullWidthRows': True,
                  'syncColumnCellResize': True,
                  'forceFitColumns': True,
                  'rowHeight': 40,
                  'enableColumnReorder': True,
                  'enableTextSelectionOnCells': True,
                  'editable': True,
                  'filterable': False,
                  'sortable': False,
                  'highlightSelectedRow': True}

def rerun_cell( b ):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1,IPython.notebook.get_selected_index()+2)'))

def check_answers(series_input,answer_list):

    # convert valid answer list to string format

    valid_answers = ""

    for item in answer_list:

        valid_answers = valid_answers + item + ","

    valid_answers = valid_answers[:len(valid_answers)-1]

    # compare student's answers to answer list

    for entry in series_input:

        if(entry != '' and entry not in answer_list):

            # display(Markdown("Some of your inputs are invalid. Please enter valid inputs from the following: ", valid_answers))

            return 0

    return 1

name_text = w.Textarea( value='', placeholder='STUDENT NAME', description='', disabled=False , layout=Layout(width='30%', height='32.5px') )
date_text = w.Textarea( value='', placeholder='DATE', description='', disabled=False , layout=Layout(width='30%', height='32.5px') )
profile_button = w.Button(button_style='info',description="Save", layout=Layout(width='15%', height='30px'))

display(name_text)
display(date_text)
display(profile_button)

profile_button.on_click( rerun_cell )

name = name_text.value
date = date_text.value

name_saved = False
date_saved = False

if(name != ''):

    name_text.close()
    display(Markdown("### Student Name: $\hspace{1.5cm}$"+ name ))
    name_saved = True

if(date != ''):

    date_text.close()
    display(Markdown("### $\hspace{2.15cm}$Date: $\hspace{1.5cm}$"+ date ))
    date_saved = True

if(name_saved == True and date_saved == True):

    profile_button.close()

# Budget and Banking

## Assignment Lesson 1

answers_recorded = 0
q1_answered = 0
q2_answered = 0
q3a_answered = 0
q3b_answered = 0
q3c_answered = 0
q3d_answered = 0

For question 1 and 2, choose the best answer.

**Question 1.** A good reason for preparing a budget

if(q1_answered == 1):

    q1_answered += 1
    q1_student_answer = q1_choices.value
    correct_answer = 'd.) All the above are good reasons for preparing a budget'

    if(q1_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q1_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q1_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 1 Answer Choices

    choice_1 = 'a.) Running out of money each month'
    choice_2 = 'b.) To reduce debt'
    choice_3 = 'c.) To save for something special'
    choice_4 = 'd.) All the above are good reasons for preparing a budget'

    answer_choices = [ choice_1,choice_2,choice_3,choice_4 ]

    # Question 1 choices widget

    q1_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q1_choices)

q1_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q1_choices.close()

if(q1_answered >= 2):

    q1_button.close()

else:

    q1_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q1_button.on_click( record_answer )
    display(q1_button)

**Question 2.** Which equation best describes gross and net pay?

if(q2_answered == 1):

    q2_answered += 1
    q2_student_answer = q2_choices.value
    correct_answer = 'b.) Net Pay = Gross Pay - Deductions'

    if(q2_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q2_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q2_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 2 Answer Choices

    choice_1 = 'a.) Gross Pay = Net Pay - Deductions'
    choice_2 = 'b.) Net Pay = Gross Pay - Deductions'
    choice_3 = 'c.) Gross Pay = Net Pay ÷ Deductions'
    choice_4 = 'd.) Net Pay = Gross Pay ÷ Deductions'

    answer_choices = [ choice_1,choice_2,choice_3,choice_4 ]

    q2_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q2_choices)

q2_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q2_choices.close()

if(q2_answered >= 2):

    q2_button.close()

else:

    q2_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q2_button.on_click( record_answer )
    display(q2_button)

**Question 3.** Label the following incomes as **fixed** or **variable**. Explain reasons for each.

$\hspace{0.35cm}$**a.)** Lloyd earns $50.00 for every set of knife he sells.

if(q3a_answered == 1):

    q3a_answered += 1
    q3a_student_answer = q3a_choices.value
    correct_answer = 'Variable'

    if(q3a_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q3a_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q3a_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 3.a Answer Choices

    choice_1 = 'Fixed'
    choice_2 = 'Variable'

    answer_choices = [ choice_1,choice_2 ]

    q3a_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q3a_choices)

q3a_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q3a_choices.close()

if(q3a_answered >= 2):

    q3a_button.close()

else:

    q3a_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q3a_button.on_click( record_answer )
    display(q3a_button)

$\hspace{0.35cm}$**b.)** Each month, Florence is paid 4.5% commission on her first $1,500.00 in sales. If she makes more than this, Florence is paid 6% in commission.

if(q3b_answered == 1):

    q3b_answered += 1
    q3b_student_answer = q3b_choices.value
    correct_answer = 'Variable'

    if(q3b_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q3b_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q3b_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 3.b Answer Choices

    choice_1 = 'Fixed'
    choice_2 = 'Variable'

    answer_choices = [ choice_1,choice_2 ]

    q3b_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q3b_choices)

q3b_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q3b_choices.close()

if(q3b_answered >= 2):

    q3b_button.close()

else:

    q3b_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q3b_button.on_click( record_answer )
    display(q3b_button)

$\hspace{0.35cm}$**c.)** Corinne works 40 hours a week at $17.00 an hour. She is paid bi-weekly (every two weeks).

if(q3c_answered == 1):

    q3c_answered += 1
    q3c_student_answer = q3c_choices.value
    correct_answer = 'Fixed'

    if(q3c_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q3c_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q3c_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 3.c Answer Choices

    choice_1 = 'Fixed'
    choice_2 = 'Variable'

    answer_choices = [ choice_1,choice_2 ]

    q3c_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q3c_choices)

q3c_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q3c_choices.close()

if(q3c_answered >= 2):

    q3c_button.close()

else:

    q3c_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q3c_button.on_click( record_answer )
    display(q3c_button)

$\hspace{0.35cm}$**d.)** Gord earns a salary of $3,000.00 each month.

if(q3d_answered == 1):

    q3d_answered += 1
    q3d_student_answer = q3d_choices.value
    correct_answer = 'Fixed'

    if(q3d_student_answer == correct_answer):

        display(Markdown("### You answered: "))
        display(Markdown(q3d_student_answer))

        display(Markdown("This is correct!"))

    else:

        display(Markdown("### You answered: "))
        display(Markdown(q3d_student_answer))

        display(Markdown("This is incorrect."))
        display(Markdown("### Correct answer: "))
        display(Markdown(correct_answer))

else:

    # Question 3.d Answer Choices

    choice_1 = 'Fixed'
    choice_2 = 'Variable'

    answer_choices = [ choice_1,choice_2 ]

    q3d_choices = w.RadioButtons( options=answer_choices , description="" , disabled=False , layout=Layout(width='100%'))
    display(q3d_choices)

q3d_answered += 1

def record_answer(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range( IPython.notebook.get_selected_index()-1 , IPython.notebook.get_selected_index()+1) '))
    q3d_choices.close()

if(q3d_answered >= 2):

    q3d_button.close()

else:

    q3d_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))
    q3d_button.on_click( record_answer )
    display(q3d_button)

4) Lian works in a retail store $35$ hours a week. She is paid $\$12.75$ an hour bi-weekly (every two weeks). Her last paycheque had a deduction of $\$71.19$ for Income Tax, $\$35.82$ for CPP, and $\$14.85$ for EI.

$\hspace{1.5cm}$a.) Determine Lian's gross pay for two weeks.

**Write your calculations below.**
* Valid inputs: Numbers and decimals.
* Valid operations: `+,-,*` for addition, subtraction, and multiplication, respectively.

**Example input:** `40 * 10 * 2 + 30`

q4a_text = w.Textarea( value='', placeholder='Your calculations for Exercise 4.a.', description='', disabled=False , layout=Layout(width='100%', height='30px') )
q4a_button = w.Button(button_style='info',description="Calculate", layout=Layout(width='15%', height='30px'))

display(q4a_text)
display(q4a_button)

q4a_button.on_click( rerun_cell )

# Obtain user's input

q4a_input = q4a_text.value

# Define the valid character inputs for this exercise

numbers = '0123456789'
operations = '+-*'
others = ' .'
valid_inputs = numbers + operations + others

# Check if every character in user's string input is valid

user_input_valid = all( ch in valid_inputs for ch in q4a_input)

# Check for correctness of user's calculation

if(q4a_input != '' and user_input_valid == True):

    user_answer = eval(q4a_input)

    display(Markdown("### You answered: "))
    display(Markdown("$\$"+str(round(user_answer,2))+"$"))

    if(user_answer == 892.50):

        display(Markdown("Your calculation is correct!"))

        q4a_button.close()
        q4a_text.close()

    else:

        display(Markdown("Your calculation is incorrect. Try again."))

if(q4a_input != '' and user_input_valid == False):

    display(Markdown("Your answer contains invalid inputs or operations."))

$\hspace{1.5cm}$b.) Determine Lian's net pay for two weeks.

q4b_text = w.Textarea( value='', placeholder='Your calculations for Exercise 4.b.', description='', disabled=False , layout=Layout(width='100%', height='30px') )
q4b_button = w.Button(button_style='info',description="Calculate", layout=Layout(width='15%', height='30px'))

display(q4b_text)
display(q4b_button)

q4b_button.on_click( rerun_cell )

# Obtain user's input

q4b_input = q4b_text.value

# Define the valid character inputs for this exercise

numbers = '0123456789'
operations = '+-*'
others = ' .'
valid_inputs = numbers + operations + others

# Check if every character in user's string input is valid

user_input_valid = all( ch in valid_inputs for ch in q4a_input)

# Check for correctness of user's calculation

if(q4b_input != '' and user_input_valid == True):

    user_answer = round(eval(q4b_input),2)
    correct_answer = round(eval("892.5 - 71.19 - 35.82 - 14.85"),2)

    display(Markdown("### You answered: "))
    display(Markdown("$\$"+str(round(user_answer,2))+"$"))

    if(user_answer == correct_answer):

        q4b_button.close()
        q4b_text.close()

        display(Markdown("Your calculation is correct!"))

    else:

        display(Markdown("Your calculation is incorrect. Try again."))

if(q4b_input != '' and user_input_valid == False):

    display(Markdown("Your answer contains invalid inputs or operations."))

---

Character Section Lesson

To conclude the Chapter 1 Assignment, you will be constructing a conservative budget and answering a "what if" question. You will be marked using a rubric that is located on the last page of the booklet.

### Character Background

Emma, a high school student working at a local grocery store as a cashier.

### Personal Background

Emma is 16 years old, in grade 11 and living at home with her parents. She plays soccer and rugby. Emma is the bass player in a band. Currently, she is using an old bass guitar that her dad played in the 1980s, and she would like to buy her own. Her goal is to buy it in five months because the band has a big gig coming up in six months. The new bass costs $828.45 including GST.

### Salary

Emma works part-time at Gobey’s Grocery. She is paid a biweekly amount of $321.13.

Emma babysits occasionally for her neighbours for $10.00 an hour. Several of Emma's babysitting client pay her by cheque, and Emma puts the money directly into her bank account.

### Expenses

Emma’s parents have encouraged her to save for post-secondary schooling. To achieve this, Emma and her parents set up a direct withdrawal from her bank account of $40.00 a paycheque into an RESP account. Emma pays for her own transit pass, which she uses to travel around the city. Emma is very thrifty and creative, so she chooses to shop for clothing at local second-hand stores. Emma’s position as the bass player in a local band requires her to maintain her instrument and provide her own sound equipment.

Emma downloads the bank record of her spending during two months (February and March).

### Notes

* The credit column contains all income (part-time job and babysitting).

* The debit column contains all expenses that Emma has paid.

* An automatic withdrawal of $40.00 each paycheque goes into an RESP account.

* In February, Emma paid rugby fees with cash, which are $180.00.

* At the end of March, Emmas’ soccer team went to an overnight tournament for which she paid $160.00 in cash as her portion of the hotel room.

Character Section Lesson Exercises

**Question 1.** List **at least two** reasons Emma might want to prepare a budget for herself.

emma1_text = w.Textarea( value='', placeholder='Write your answer here for Question 1.', description='', disabled=False , layout=Layout(width='100%', height='75px') )
emma1_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))

display(emma1_text)
display(emma1_button)

emma1_button.on_click( rerun_cell )

emma1_input = emma1_text.value

if(emma1_input != ''):

    emma1_text.close()
    emma1_button.close()
    display(Markdown("### Your answer for Question 1:"))
    display(Markdown(emma1_input))

**Question 2.** Determine Emma’s net income for the month of February and March. Her bank statements are displayed below.

# Prepare dataframes for Emma's February & March Transactions

entry_types = {'Debit ($)': str , 'Credit ($)' : str}

february_transactions_df = pd.read_csv('./data/februarytransactions.csv',converters=entry_types)
february_transactions_df.set_index('Date',inplace=True)
february_transactions_df[['Debit ($)','Credit ($)']] = february_transactions_df[['Debit ($)','Credit ($)']].replace(np.nan,"0.00")

march_transactions_df = pd.read_csv('./data/marchtransactions.csv',converters=entry_types)
march_transactions_df.set_index('Date',inplace=True)
march_transactions_df[['Debit ($)','Credit ($)']] = march_transactions_df[['Debit ($)','Credit ($)']].replace(np.nan,"0.00")

# Control grid features

emma_grid_features = { 'fullWidthRows': True,
                  'syncColumnCellResize': True,
                  'forceFitColumns': True,
                  'rowHeight': 40,
                  'enableColumnReorder': True,
                  'enableTextSelectionOnCells': True,
                  'editable': False,
                  'filterable': False,
                  'sortable': False,
                  'highlightSelectedRow': True}

q_emma_feb = q.show_grid( february_transactions_df , grid_options = emma_grid_features )
q_emma_mar = q.show_grid( march_transactions_df , grid_options = emma_grid_features )

display(Markdown("
Emma's February Transactions
"))

display(q_emma_feb)

**Write your calculations below.**
* Valid inputs: Numbers and decimals.
* Valid operations: `+` for addition.

**Example input:** `35 + 27.13 + 11.32`

feb_text = w.Textarea( value='', placeholder="Enter your calculation here to determine Emma's net income for the month of February. Hint: which column do Emma's income come from?", description='', disabled=False , layout=Layout(width='100%', height='30px') )
feb_button = w.Button(button_style='info',description="Calculate", layout=Layout(width='15%', height='30px'))

display(feb_text)
display(feb_button)

feb_button.on_click( rerun_cell )

# Obtain user's input

feb_input = feb_text.value

# Define the valid character inputs for this exercise

numbers = '0123456789'
operations = '+'
others = ' .'
valid_inputs = numbers + operations + others

# Check if every character in user's string input is valid

user_input_valid = all( ch in valid_inputs for ch in feb_input)

# Check for correctness of user's calculation

if(feb_input != '' and user_input_valid == True):

    user_answer = eval(feb_input)

    display(Markdown("### You answered: "))
    display(Markdown("$\$"+str(user_answer)+"$"))

    if(user_answer == 1042.26):

        feb_button.close()
        feb_text.close()

        display(Markdown("Your calculation is correct!"))

    else:

        display(Markdown("Your calculation is incorrect. Try again."))

if(feb_input != '' and user_input_valid == False):

    display(Markdown("Your answer contains invalid inputs or operations."))

display(Markdown("
Emma's March Transactions
"))

display(q_emma_mar)

**Write your calculations below.**
* Valid inputs: Numbers and decimals.
* Valid operations: `+` for addition.

**Example input:** `35 + 27.13 + 11.32`

mar_text = w.Textarea( value='', placeholder="Enter your calculation here to determine Emma's net income for the month of March. Hint: which column do Emma's income come from?", description='', disabled=False , layout=Layout(width='100%', height='30px') )
mar_button = w.Button(button_style='info',description="Calculate", layout=Layout(width='15%', height='30px'))

display(mar_text)
display(mar_button)

mar_button.on_click( rerun_cell )

# Obtain user's input

mar_input = mar_text.value

# Define the valid character inputs for this exercise

numbers = '0123456789'
operations = '+'
others = ' .'
valid_inputs = numbers + operations + others

# Check if every character in user's string input is valid

user_input_valid = all( ch in valid_inputs for ch in mar_input)

# Check for correctness of user's calculation

if(mar_input != '' and user_input_valid == True):

    user_answer = eval(mar_input)

    display(Markdown("### You answered: "))
    display(Markdown("$\$"+str(user_answer)+"$"))

    if(user_answer == 922.26):

        mar_button.close()
        mar_text.close()

        display(Markdown("Your calculation is correct!"))

    else:

        display(Markdown("Your calculation is incorrect. Try again."))

if(mar_input != '' and user_input_valid == False):

    display(Markdown("Your answer contains invalid inputs or operations."))

**Question 3.** Categorize Emma’s income as fixed, variable or both. Explain.

emma3_text = w.Textarea( value='', placeholder='Write your answer here for Question 3.', description='', disabled=False , layout=Layout(width='100%', height='75px') )
emma3_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))

display(emma3_text)
display(emma3_button)

emma3_button.on_click( rerun_cell )

emma3_input = emma3_text.value

if(emma3_input != ''):

    emma3_text.close()
    emma3_button.close()
    display(Markdown("### Your answer for Question 3:"))
    display(Markdown(emma3_input))

Interactive Exercise: Entering & Saving Spreadsheet Data

**Question 4.** By analyzing the bank statements for the two months, complete the following tables for each month.

$\hspace{0.35cm}$**a.)** For each row in the table, fill the **Expense Category** column with the appropriate expense label. The valid choices are listed below - write the associated upper case letter for each expense. Expand the **Description** column to see the expense entry.

**Categories: **

$$\text{(S) Savings, (U) Utilities, (C) Clothing, (E) Entertainment, (M) Miscellaneous, (P) Personal Care, (T) Transportation, (F) Food}$$

# Note to developers:
# 1. data in tables below obtained by converting relevant pages from Math20-3Unit1Key.pdf into .csv format.
# 2. this part of the interactive focuses on categorizing expenses

# Answer key for this lesson

feb_expenses_answer_key = ['S', 'T', 'F', 'U', 'F', 'F', 'F', 'C', 'E', 'S', 'F', 'F', 'E', 'M', 'M', 'F', 'F', 'M', 'M', 'C']
feb_fv_answer_key = ['V','V','V','V','F','F','F']

# Setting up the dataframe

pd.options.display.max_rows = 50

entry_types = {'Debit ($)': str , 'Balance ($)' : str}

february_df = pd.read_csv('./data/februarydebits.csv',converters=entry_types)
february_df.set_index('Transaction #',inplace=True)
february_df['Debit ($)'] = february_df['Debit ($)'].replace(np.nan,"0.00")
february_df['Expense Category'] = february_df['Expense Category'].replace(np.nan,"")

original_feb_df = february_df[['Date','Description','Debit ($)','Balance ($)']]

# Display interactive grid 1: categorizing expenses

q_february_df = q.show_grid( february_df , grid_options = grid_features )

# Recording answers

q4a_button = w.Button(button_style='info',description="Record Answers", layout=Layout(width='15%', height='30px'))

def record_spreadsheet(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1,IPython.notebook.get_selected_index()+3)'))

display(q_february_df)
display(q4a_button)

q4a_button.on_click( record_spreadsheet )

# Recover entries

q4a_recover_button = w.Button(button_style='info',description="Reset all values", layout=Layout(width='15%', height='30px'))

def recover_entries(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()-1,IPython.notebook.get_selected_index()+0)'))

display(q4a_recover_button)

q4a_recover_button.on_click( recover_entries )

# Obtain the changed dataframe

student_feb_df = q_february_df.get_changed_df()

# Check answers

answer_list = ['S','F','C','E','T','U','M']

answers_valid = check_answers(student_feb_df['Expense Category'],answer_list)

if(answers_valid == 0):

    display(Markdown("Some of your inputs are invalid. Please enter inputs from the following list: S,U,C,E,M,P,T,F"))

# Group the dataframe according to student's expense categories once student's answers are correct

if(answers_valid == 1):

    # Check if every entry matches the answer key

    student_answers = (student_feb_df['Expense Category'].values).tolist()

    if( feb_expenses_answer_key == student_answers):

        display(Markdown("Your selections are correct!"))

        q4a_button.close()
        q4a_recover_button.close()
        q_february_df.close()

        q4a_grid_features = { 'fullWidthRows': True,
                          'syncColumnCellResize': True,
                          'forceFitColumns': True,
                          'rowHeight': 40,
                          'enableColumnReorder': True,
                          'enableTextSelectionOnCells': True,
                          'editable': False,
                          'filterable': False,
                          'sortable': False,
                          'highlightSelectedRow': True}

        student_q4a_df = q.show_grid( student_feb_df , grid_options = q4a_grid_features )

        display(student_q4a_df)

    else:

        display(Markdown("Some of your inputs are incorrect."))

$\hspace{0.35cm}$**b.)** Calculate the total of each expense category you used in (a).

$\hspace{3cm}$This is done for you. Proceed to exercise (c).

q4b_df = pd.read_csv('./data/expensesum.csv')
q4b_df.set_index("Expense Category",inplace=True)
q_q4b_df = q.show_grid( q4b_df , grid_options = grid_features )
display(q_q4b_df)

$\hspace{0.35cm}$**c.)** For each category, determine whether it is **fixed** or **variable**. Enter F for fixed and V for variable.

# Setting up the dataframe

fixed_or_var_df = pd.read_csv('./data/fixedorvar.csv')
fixed_or_var_df.set_index('Expense Category',inplace=True)
fixed_or_var_df['Fixed or Variable'] = fixed_or_var_df['Fixed or Variable'].replace(np.nan,"")

q_fixed_or_var_df = q.show_grid( fixed_or_var_df , grid_options = grid_features)

q4c_button = w.Button(button_style='info',description="Record Answers", layout=Layout(width='15%', height='30px'))

def record_spreadsheet(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1,IPython.notebook.get_selected_index()+2)'))

display(q_fixed_or_var_df)
display(q4c_button)

q4c_button.on_click( record_spreadsheet )

# Obtain the changed dataframe

q4c_df = q_fixed_or_var_df.get_changed_df()

# Check answers

q4c_answer_list = ['V','F','V','F','V','V','V','V']
q4c_student_answer = (q4c_df['Fixed or Variable'].values).tolist()

def check_q4c(student_inputs):

    valid_inputs = 'VF'

    for entry in student_inputs:

        if(entry not in valid_inputs):

            display(Markdown("Please enter valid inputs only."))

            return False

        if(entry == ''):

            display(Markdown("Please fill all the entries in the spreadsheet above."))

            return False

    return True

if(check_q4c(q4c_student_answer) == True):

    if(q4c_student_answer == q4c_answer_list):

        display(Markdown("Your selections are correct!"))

        q4c_grid_features = { 'fullWidthRows': True,
                          'syncColumnCellResize': True,
                          'forceFitColumns': True,
                          'rowHeight': 40,
                          'enableColumnReorder': True,
                          'enableTextSelectionOnCells': True,
                          'editable': False,
                          'filterable': False,
                          'sortable': False,
                          'highlightSelectedRow': True}

        student_q4c_df = q.show_grid( q4c_df , grid_options = q4c_grid_features )

        q_fixed_or_var_df.close()

        display(student_q4c_df)

    else:

        display(Markdown("Some inputs are incorrect."))

**Question 5.** Construct a **monthly** conservative budget skeleton for Emma based upon her February and March data. To do this, compare the data for February and March and use the _highest_ expense amount for each category.

**Note: ** The **February Expense Data** and **March Expense Data** are given below.

entry_types = {'February Totals ($)': str , 'March Totals ($)' : str}

comparison_df = pd.read_csv('./data/expensecomparison.csv',converters = entry_types)
comparison_df.set_index('Expense Categories',inplace=True)
comparison_df['Highest Expense Amount ($)'] = comparison_df['Highest Expense Amount ($)'].replace(np.nan,"")

del comparison_df['Transaction #']

q_comparison_df = q.show_grid( comparison_df , grid_options = grid_features )

expense_button = w.Button(button_style='info',description="Record Answers", layout=Layout(width='15%', height='30px'))

def record_spreadsheet(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1,IPython.notebook.get_selected_index()+3)'))

display(q_comparison_df)
display(expense_button)

expense_button.on_click( record_spreadsheet )

# Recover entries

recover_button = w.Button(button_style='info',description="Reset all values", layout=Layout(width='15%', height='30px'))

def recover_entries(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()-1,IPython.notebook.get_selected_index()+0)'))

display(recover_button)

recover_button.on_click( recover_entries )

# Obtain changed dataframe

student_comparison_df = q_comparison_df.get_changed_df()

# Answers

correct_answers = ['80.00', '73.50', '91.95', '30.15', '189.45', '233.64', '206.00', '116.53']
student_answers = (student_comparison_df['Highest Expense Amount ($)'].values).tolist()

# Function to check if every entry is a valid input

def check_floats(input_array):

    valid_inputs = '0123456789.'

    for answer in input_array:

        current_user_input = all( ch in valid_inputs for ch in answer)

        if(current_user_input == False):

            return False

    return True

# Check if answer is correct

if(check_floats(student_answers) == False):

    display(Markdown("Please enter decimal numbers only."))

else:

    if(student_answers == correct_answers):

        display(Markdown("Your choices are correct!"))

        recover_button.close()
        expense_button.close()
        q_comparison_df.close()

        comparison_grid_features = { 'fullWidthRows': True,
                                  'syncColumnCellResize': True,
                                  'forceFitColumns': True,
                                  'rowHeight': 40,
                                  'enableColumnReorder': True,
                                  'enableTextSelectionOnCells': True,
                                  'editable': False,
                                  'filterable': False,
                                  'sortable': False,
                                  'highlightSelectedRow': True}

        student_comparison_df = q.show_grid( student_comparison_df , grid_options = comparison_grid_features )
        display(student_comparison_df)

    else:

        display(Markdown("Some of your entries are incorrect. Please fill them with the correct values. Make sure to write numbers in decimal form. Write 80.00 for 80, 206.00, etc."))

**Question 6.** If there is excess money, where should Emma put it? If there is not enough income for expenses, suggest how Emma can cover her expenses or reduce her expenses.

q6_text = w.Textarea( value='', placeholder='Write your answer here for Question 6.', description='', disabled=False , layout=Layout(width='100%', height='75px') )
q6_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))

display(q6_text)
display(q6_button)

q6_button.on_click( rerun_cell )

q6_input = q6_text.value

if(q6_input != ''):

    q6_text.close()
    q6_button.close()
    display(Markdown("### Your answer for question 6:"))
    display(Markdown(q6_input))

**Question 7.**

$\hspace{0.35cm}$**a.)** How much will Emma need to save each month to purchase a new base, in 5 months, that costs $\$828.45$ (including GST)?

q7a_text = w.Textarea( value='', placeholder='Write your answer here for Question 7.a.', description='', disabled=False , layout=Layout(width='100%', height='75px') )
q7a_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))

display(q7a_text)
display(q7a_button)

q7a_button.on_click( rerun_cell )

q7a_input = q7a_text.value

if(q7a_input != ''):

    q7a_text.close()
    q7a_button.close()
    display(Markdown("### Your answer for question 7.a:"))
    display(Markdown(q7a_input))

$\hspace{0.35cm}$**b.)** Based upon her budget, will Emma be able to save enough? If not, modify her budget so that she can afford it. A new category has been added to show her savings for the bass.

**Note:** Income and expenses should be balanced at this point.

From the previous exercises, we saw that a conservative income for Emma was $\$922.26$. In this exercise, we want to create a budget amount in such a way that the sum of each category is exactly $\$922.26$.

# Prepare dataframes for Emma's February & March Transactions

ex_7b_df = pd.read_csv('./data/exercise7b.csv')
ex_7b_df[['Budgeted Amount ($)']] = ex_7b_df[['Budgeted Amount ($)']].replace(np.nan,"")
ex_7b_df.set_index('Category',inplace=True)

q_ex_7b = q.show_grid( ex_7b_df , grid_options = grid_features )

display(Markdown("
Emma's Modified Conservative Budget
"))

display(q_ex_7b)

ex_7b_button = w.Button(button_style='info',description="Record Answers", layout=Layout(width='15%', height='30px'))

def record_spreadsheet(button_widget):

    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1,IPython.notebook.get_selected_index()+3)'))

display(ex_7b_button)

ex_7b_button.on_click( record_spreadsheet )

**Question 8.** Determine the percent of Emma’s income on each category.

To calculate percentage of income for each category use the formula:

$$\text{Percent of Income} = \frac{\text{Amount Allotted for Expense}}{\text{Budgeted Income}}\times 100\%$$

**Note:** This part of the exercise is done for you and changes based on your inputs in exercise 7.b., the formula above is for your reference.

# Set up student inputs as a list

ex_7b_student_df = q_ex_7b.get_changed_df()
student_budget_input = ex_7b_student_df['Budgeted Amount ($)']
student_budget_col = (student_budget_input.values).tolist()

# Check if every entry is valid

def check_ex_7b(student_inputs):

    valid_inputs = '0123456789. '

    for entry in student_inputs:

        for ch in entry:

            if(ch not in valid_inputs):

                display(Markdown("Please enter valid inputs only."))

                return False

    if(entry == ''):

        display(Markdown("Please fill all the entries in the spreadsheet above."))

        return False

    return True

# Check if student's budget matches the total income

valid_input = check_ex_7b(student_budget_col)

if(valid_input == True):

    # Convert every entry to float and get the sum

    budget_sum = 0

    for entry in student_budget_col:

        current_entry = eval(entry)
        budget_sum += current_entry

    # If the budget sum matches, then create the percentages chart

    percentages_col = []

    if(budget_sum != 922.26):

        display(Markdown("Your budget proposal does not total $922.26. Please try again."))

    else:

        for entry in student_budget_col:

            current_percent = round( eval(entry)/budget_sum , 4)
            percentages_col.append( round(current_percent*100,4) )

        indices = ['Savings (RESP)','Transportation','Utilities','Food','Clothing','Entertainment','Miscellaneous','Personal Care','Savings (Bass)']

        percentage_df = pd.DataFrame({'Percentage (%): ': percentages_col },index=indices)
        updated_ex_7b_student_df = pd.concat( [ex_7b_student_df,percentage_df] , axis = 1 )

        ex_7b_features = { 'fullWidthRows': True,
                          'syncColumnCellResize': True,
                          'forceFitColumns': True,
                          'rowHeight': 40,
                          'enableColumnReorder': True,
                          'enableTextSelectionOnCells': True,
                          'editable': False,
                          'filterable': False,
                          'sortable': False,
                          'highlightSelectedRow': True}

        q_ex_7b_updated = q.show_grid( updated_ex_7b_student_df , grid_options = ex_7b_features )

        display(Markdown("
Emma's Modified Budget Percentage Breakdown
"))

        display(q_ex_7b_updated)

**Question 9.** Are there any categories that Emma is far above or far below the spending guidelines? How does being a high school student affect Emma’s consideration of the spending guidelines?

q9_text = w.Textarea( value='', placeholder='Write your answer here for Question 9', description='', disabled=False , layout=Layout(width='100%', height='75px') )
q9_button = w.Button(button_style='info',description="Record Answer", layout=Layout(width='15%', height='30px'))

display(q9_text)
display(q9_button)

q9_button.on_click( rerun_cell )

q9_input = q9_text.value

if(q9_input != ''):

    q9_text.close()
    q9_button.close()
    display(Markdown("### Your answer for question 9:"))
    display(Markdown(q9_input))

---

Student Interactive Section

In this section, you will enter your own expense categories and see how your expense percentage breakdown compares to the spending guideline above.

# Create button and dropdown widget

number_of_cat = 13
dropdown_options = [ str(i+1) for i in range(number_of_cat) ]
dropdown_widget = w.Dropdown( options = dropdown_options , value = '3' , description = 'Categories' , disabled=False )

categories_button = w.Button(button_style='info',description="Save", layout=Layout(width='15%', height='30px'))

# Display widgets

display(dropdown_widget)
display(categories_button)

categories_button.on_click( rerun_cell )

# Create dataframe

df_num_rows = int(dropdown_widget.value)
empty_list = [ '' for i in range(df_num_rows) ]
category_list = [ i+1 for i in range(df_num_rows) ]

# Set up data input for dataframe

df_dict = {'Category #': category_list, 'Budget ($)': empty_list , 'Expense Category': empty_list}

student_df = pd.DataFrame(data = df_dict)
student_df.set_index('Category #',inplace=True)

# Reorder column labels

student_df = student_df[['Expense Category','Budget ($)']]

# Set up & display as Qgrid

q_student_df = q.show_grid( student_df , grid_options = grid_features )
display(q_student_df)

# Create & display save entries widget button

save_student_entries_button = w.Button(button_style='info',description="Plot Pie Chart", layout=Layout(width='15%', height='30px'))
display(save_student_entries_button)

save_student_entries_button.on_click( rerun_cell )

# Convert qgrid to dataframe

student_updated_df = q_student_df.get_changed_df()

student_budget_col = student_updated_df['Budget ($)'].values.tolist()
student_labels_col = student_updated_df['Expense Category'].values.tolist()

# Check if a number if float

def isfloat(value):

    try:

        float(value)

        return True

    except ValueError:

        return False

# Function: Check for validity of budget column entries
# Input: Student's budget column values
# Output: Boolean, false if one of the entries is invalid

def check_budget_column(input_list):

    valid_inputs = '0123456789.'

    # Check if inputs are valid

    for entry in input_list:

        if( isfloat(entry) == False):

            return False

    return True

# Function: Calculate the percentage of each expense category
# Input: Student expense category lists and student budget column values
# Output: Percentages column

def get_percentages(input_list):

    total = 0
    percentage_col = []

    # Obtain total

    for entry in input_list:

        entry = eval(entry)
        total += entry

    # Obtain percentages

    for entry in input_list:

        entry = eval(entry)
        current_percentage = entry/total
        percentage_col.append( round(current_percentage*100,2) )

    return percentage_col

# If student input is valid, create a pie chart plot

colors = ['yellowgreen', 'gold', 'lightskyblue', 'lightcoral','darkseagreen','lightcyan','lightpink','coral','tan','slateblue','azure','tomato','lawngreen']

if(check_budget_column(student_budget_col) == True):

    student_values = get_percentages(student_budget_col)
    labels = student_labels_col

    plt.figure(figsize=(20,10))
    plt.rcParams['font.size'] = 20
    plt.title('Your Expense Category Percentage Breakdown',fontsize=25)
    plt.pie(student_values, labels=labels, colors=colors, autopct='%1.1f%%', shadow=True, startangle=35)
    plt.axis('equal')
    plt.show()

else:

    display(Markdown("Please enter decimal numbers only for the budget column"))

[![Callysto.ca License](https://github.com/callysto/curriculum-notebooks/blob/master/callysto-notebook-banner-bottom.jpg?raw=true)](https://github.com/callysto/curriculum-notebooks/blob/master/LICENSE.md)