"""
Python 3.6
PyTorch 0.4
"""

import os
import logging
import math
from configparser import ConfigParser
import functools, itertools

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import imageio
from tqdm import tqdm_notebook

import torch
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import tensorboardX

def print_line(num = 1):
    for _ in range(num):
        logging.debug('================================')

def set_logging(level = logging.INFO):
    """
    set logging level and format.
    """
    logging.basicConfig(level = level, filename = '', format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

def read_config(dataset):
    """
    Read from the config file to get the parameters.
    """
    from Models.AbstractModel import Models
    dataset = dataset.lower()
    assert dataset in Models.keys()
    config = ConfigParser()
    config.read('configuration.ini')

    toreturn = {'dataset':dataset}
    for key, val in dict(config[dataset]).items():
        try:
            toreturn[key] = int(val)
        except:
            toreturn[key] = val

    logging.critical(f'I will train for {toreturn["epoch"]} epochs!')
    return toreturn

def long_tensor_to_onehot(idx, max_idx):
    """ from a one-dimension LongTensor to get the onehot vector.
    >>> long_tensor_to_onehot(torch.LongTensor([1,0,2]), 3)
    tensor([[ 0,  1,  0],
            [ 1,  0,  0],
            [ 0,  0,  1]])
    """
    return torch.zeros(idx.size()[0], max_idx).scatter_(1, idx.view(-1,1), 1).long()

def gen_random_labels(num_instance, max_idx):
    """ get random labels in randomrange(max_idx)
    >>> labels = gen_random_labels(10, 3)
    """
    return torch.multinomial(torch.ones(max_idx), num_instance, replacement = True)

def network_num_parameters(net):
    """
    Compute the number of parameters
    """
    num_params = 0
    for param in net.parameters():
        num_params += param.numel()
    logging.debug(f'Total number of parameters: {num_params}')
    return num_params

def to_np(x):
    """ transform a given tensor to numpy array
    >>> to_np(torch.ones(3,2))
    array([[1., 1.],
           [1., 1.],
           [1., 1.]], dtype=float32)
    """
    if not torch.is_tensor(x):
        raise TypeError('We need tensor here.')

    return x.to( torch.device('cpu') ).numpy()

def cov(x):
    """ calculate covariance matrix of rows
    >>> a = torch.arange(18).view(3,6)
    >>> cov(a)
    tensor([[ 3.5000,  3.5000,  3.5000],
            [ 3.5000,  3.5000,  3.5000],
            [ 3.5000,  3.5000,  3.5000]])
    >>> t = torch.FloatTensor([[1,2,6],[2,4,0],[5,2,3],[9,5,1]])
    >>> cov(t)
    tensor([[  7.0000,  -4.0000,  -1.5000, -10.0000],
            [ -4.0000,   4.0000,  -1.0000,   4.0000],
            [ -1.5000,  -1.0000,   2.3333,   4.0000],
            [-10.0000,   4.0000,   4.0000,  16.0000]])

    """
    mean_x = torch.mean(x, 1, keepdim = True)
    xm = x.sub(mean_x.expand_as(x))
    c = xm.mm(xm.t())
    c = c / (x.size(1) - 1)
    return c

def addGuassianNoise(x, sigma = 0.1):
    """ Add small Gaussian noise to input x
    >>> tmp = addGuassianNoise(torch.ones(12,13), sigma = 1)
    """
    if not torch.is_tensor(x):
        raise TypeError('We need tensor here.')

    to_add = torch.randn(x.size()) * sigma

    return x + to_add

def check_nan(dict_):
    """
    Check nan for a dict.
    >>> dict_ = {'a': 0, 'b': 1, 'c': 0, 'd': 10}
    >>> check_nan(dict_)
    True
    >>> from math import nan
    >>> check_nan({'a': 0, 'b': nan, 'c': 0, 'd': nan})
    ArithmeticError: We find the loss(es) [b, d] nan in PairwiseDis.
    """
    # check nan
    flag_nan = [ key for key, loss in dict_.items() if math.isnan(loss) ]
    if len(flag_nan) > 0:
        str_keys_nan = ', '.join(flag_nan)
        raise ArithmeticError(f'We find the loss(es) [{str_keys_nan}] nan.')
    return True

def pair_iters(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip(a, b)

# ================================ Visualization part ================================

def show_image(x, nrow = 0):
    """ Show the images (in tensors) in a grid where number of row specified nrow
    >>> import torch; show_image(torch.rand(25, 3, 32, 32), nrow = 5)
    """
    if nrow == 0:
        nrow = max( int(np.sqrt(len(x))), 6 )

    x.detach_()
    x = x.cpu()

    x = torchvision.utils.make_grid(x, nrow=nrow, normalize=True, range=(-1,1), padding=4 ).float()
#     x = torchvision.utils.make_grid(x, nrow = nrow, padding = 4, scale_each=True, normalize = True)
    x = to_np(x)

    if len(x.shape) == 4:
        x = x.transpose(0,2,3,1)
    elif len(x.shape) == 3:
        x = x.transpose(1,2,0)

    f, ax = plt.subplots(figsize=(12, 9), dpi=300)
    plt.imshow(x)
    plt.show()

def generate_animation(imgs, save_path, name=None):
    """
    Generate a gif animation of given imgs.
    >>> import torch; generate_animation([torch.randn(3, 32, 32) for _ in range(10) ], 'save_dir/unittest')
    """
    if torch.is_tensor(imgs):
        imgs.requires_grad_(False)

    normalize = functools.partial( torchvision.utils.make_grid, nrow=6, normalize=True, range=(-1,1), padding=4 )
    images = [ to_np(normalize(img)*255).astype(np.uint8).transpose(1, 2, 0) for img in imgs  ]

    if name is None:
        name = 'generated_animation'
    imageio.mimsave( os.path.join(save_path, f'{name}.gif'), images, fps = 40)

def exp_encoded_range_visual(sample, models, code_dim_varying, inverse = [False]*5, num_ = 6, filename = 'imgs/tmp.png'):
    """
    Generate a sequence of samples by varying the encoded code given a sample
    >>> from Dataset import random_samples
    >>> sample = random_samples('mnist')
    >>> from Models.Beta_VAE import Beta_VAE; models = [Beta_VAE(1 , dataset = 'mnist', hidden_dim = 10, gpu_mode = False)]
    >>> code_dim_varying = [0,5,7]
    >>> out = exp_encoded_range_visual(sample, models, code_dim_varying, [False], 3, 'save_dir/unittest/exp_encoded_range_visual.png')
    >>> out.type()
    'torch.FloatTensor'
    """

    assert len(sample) == 1
    samples_batch = []

    with torch.no_grad():

        for i0, (model, code) in tqdm_notebook(enumerate(zip(models, code_dim_varying)), 'exp_encoded_range_visual'):
            if 'infogan' in model.model_name.lower():
                continue
#             print(model.model_name, code)

            samples = model.latent_traversal_given_samples_dim(sample, code, num_range=6)
            if inverse[i0]:
                samples = samples[::-1]
            samples_this_batch = torch.cat(samples, 0)
            samples_batch.append(samples_this_batch)

    samples = torch.cat(samples_batch, 0)
    show_image(samples, nrow = num_ )

    tmp = [ torchvision.utils.make_grid(s, nrow = num_, padding = 4, normalize=True, range=(-1,1), pad_value=1 ).unsqueeze(0) for s in samples_batch ]
    samples = torch.cat(tmp, 0)
    torchvision.utils.save_image(samples, filename = filename, nrow = 1, padding=2, pad_value=1)
    logging.info('Files saved! ' + filename)

    return samples

# ================================ Recording part ================================

def updateDataFrame(index, in_dict, label):
    '''
    Save the dict info in the csv file specified by label, return the DataFrame
    >>> updateDataFrame('label', {'s':2, 't':8}, 'doctest')
           s  t
    label  2  8
    '''

    # save path
    save_path = 'PD_DF'
    csv_file = os.path.join(save_path, f'{label}.csv')
    # load the existing dataframe
    df = loadDataFrame(label)
    # make the dataframe
    new_entry = pd.DataFrame.from_dict({index:in_dict}, orient = 'index')

    # if not dataframe exists
    if df is None:
        df = new_entry
        # if an old entry is in the dataframe
    else:
        if index in df.index:
            df.loc[index] = new_entry.loc[index]
        else:
            # this is a new entry
            df = df.append(new_entry, verify_integrity=True, sort=False)

    # save it
    df.to_csv(csv_file, header='dataframe')

    return df

def loadDataFrame(label):
    '''
    Load the dataframe from csv specified by the label
    >>> loadDataFrame('doctest')
           s  t
    label  2  8
    >>> loadDataFrame('some_label_no_exist')
    Target filed not found.
    '''
    # save path
    save_path = 'PD_DF'
    csv_file = os.path.join(save_path, f'{label}.csv')
    # load it
    try:
        df = pd.read_csv(csv_file, index_col=0)
        return df
    except Exception as e:
        print('Target filed not found.')
        return None

# ================================ Dist and distributions part ================================

def pdist(A, B, sqrt=True):
    """ Pairwise Euclidean distance
    >>> import torch; A = torch.randn(2,4); B = torch.randn(15,4); pdist(A, B).size()
    torch.Size([2, 15])
    """
    A_squared = A.pow(2).sum(1).unsqueeze(1)
    B_squared = B.pow(2).sum(1).unsqueeze(0)
    AB = torch.mm(A, B.t()) # A @ B.t()
    A_B_squared = A_squared + B_squared - 2 * AB
    A_B_squared.clamp_(min=1e-16) # min=0 will cause Nan grad
    return torch.sqrt( A_B_squared ) if sqrt else A_B_squared

def log_prob_standard_normal(x):
    """ The log probability of x in the standard normal distribution.
        asser x.dim in {1,2}
        >>> import torch;from torch.distributions.multivariate_normal import MultivariateNormal
        >>> normal_prior = MultivariateNormal(torch.zeros(5), torch.eye(5) )
        >>> x = torch.randn(15, 5) * 2 + 4
        >>> (log_prob_standard_normal(x) - normal_prior.log_prob(x)).norm(p=1).item() < 1e-8
        True

        # Another implementation
        device = x.device
        dim = x.size()[1]
        normal_prior = MultivariateNormal(torch.zeros(dim, device=device), \
                                          torch.eye(dim, device=device) )
        # probability of p(x)
        log_prob_p = normal_prior.log_prob(x) # B

    """
    if x.dim() == 1: x.unsqueeze_(1)
    dim = x.size()[1]
    d_log_2pi = dim * math.log(2 * math.pi)
    return - 0.5 * ( x.pow(2).sum(1) + d_log_2pi )

def log_prob_of_multiple_dist(mu, log_var, x):
    """ Compute the log probabilty of x subject to distributions define by mu and log_var
    Args:
    mu     - N * d
    logvar - N * d
    x      - B * d

    -0.5 * \|x-mu\|^2 / log_var.exp() - 0.5 * log_var.sum() - 0.5 * log(2*pi) * d

    return B * N

    >>> import torch; from torch.distributions import Normal
    >>> n = Normal(torch.zeros(1), torch.ones(1))
    >>> x = torch.randn(1) * 5 + 4
    >>> pred = log_prob_of_multiple_dist(torch.zeros(1), torch.zeros(1), x)
    >>> (pred - n.log_prob(x) ).abs().item() < 1e-8
    True
    """

    r"""
    This is the simple implementation. Out implementation is 20x faster on CPU and 1000x faster on GPU.

    def t(mu, log_var, x):
        tmp = []
        for mu_, log_var_ in zip(mu, log_var):
            mn = MultivariateNormal(mu_, torch.diag( log_var_.exp() ))
            ground_truth = mn.log_prob(x) # B
            tmp.append(ground_truth)

        # B * N
        out = torch.stack(tmp, dim = 1)
    """

    if mu.dim()==1: mu.unsqueeze_(1)
    if log_var.dim()==1: log_var.unsqueeze_(1)
    if x.dim()==1: x.unsqueeze_(1)

    dim = mu.size()[1]

#         # N * d, mu^2
#         mu.pow(2)
#         # N * d, mu^2 ./ var
#         mu.pow(2).div( log_var.exp() )
#         # N * 1, \sum_i mu_i^2/var_i
#         mu.pow(2).div( log_var.exp() ).sum(1).unsqueeze(1)

#         # B * d, x^2
#         x.pow(2)
#         # 1 * B * d, x^2
#         x.pow(2).unsqueeze(0)
#         # N * 1 * d, var
#         var = log_var.exp().unsqueeze(1)
#         # N * B * d, x^2 ./ var
#         x.pow(2).unsqueeze(0).div( var )
#         # N * B, \sum_i x_i^2/var_i
#         x.pow(2).unsqueeze(0).div( var ).sum(2)

#         # N * B, \sum_i mu_i*x_i/var_i
#         torch.mm(mu.div(log_var.exp), B.t())

    # N * d
    var = log_var.exp() + 1e-8

    mu_div_var_squared = mu.pow(2).div( var ).sum(1).unsqueeze(1) # N * 1
    x_div_var_squared  = x.pow(2).unsqueeze(0).div( var.unsqueeze(1) ).sum(2) # N * B
    mu_x_div_var_squared = torch.mm(mu.div( var ), x.t()) # N * B

    # N * B
    mu_x_var_dist = mu_div_var_squared + x_div_var_squared - 2 * mu_x_div_var_squared
    mu_x_var_dist.clamp_(min=10**-16)

    # N * 1
    log_var_sum = log_var.sum(1, keepdim = True)

    # 0-dim
    d_log_2pi = dim * math.log(2 * math.pi)

    # N * B
    out = -0.5 * (d_log_2pi + log_var_sum + mu_x_var_dist)

    return out.t() # B * N

# ================================ Initialization part ================================

def initialize_weights_kaiming_normal(*nets):
    for net in nets:
        for m in net.modules():
            if isinstance( m, (nn.Conv2d,nn.Linear,nn.ConvTranspose2d) ):
                if hasattr(m,'weight'):
                    torch.nn.init.kaiming_normal_(m.weight.data)
                if hasattr(m.bias,'data'):
                    m.bias.data.zero_()
    logging.debug('Weight initialized with kaiming_normal')

# ================================ Breakpoint tool part ================================

def breakpoint():
    ''' to place break point somewhere '''
    from IPython.core.debugger import set_trace;set_trace()


# coding: utf-8
from pandas import Series, DataFrame
import pandas as pd
import numpy as np

obj = Series([4,5,6,7])
print obj.values
print obj.index

obj2 = Series([4,5,-6,7,8],index=['a','b','c','d','e'])
print obj2
print obj2.index
print obj2['b']
print obj2[obj2 > 0]
print obj2 * 2
print np.exp(obj2)
print 'b' in obj2
print 'f' in obj2

sdata = {'yang':35000,"zhi":23000,"wei":12122}
obj3 = Series(sdata)
print obj3
name = ['yang','zhi','wei','coo']
obj4 = Series(sdata,index=name)
print obj4

print pd.isnull(obj4)
print pd.notnull(obj4)

print obj3 + obj4

obj4.name = 'population'
print obj4.name

obj4.index.name = 'state'
print obj4.index.name
print obj4.index

obj.index = ['bob','steve','jeef','ryan']
print obj


"""
Python tools to read and write 'La Palma' cubes

Merge of some routines written by Tiago Pereira (github:helita), J. de la Cruz Rodriguez,
Alex Pietrow (github:crispy), Carlos Diaz and G. Vissers (ISP/SU 2019)

"""
import numpy as np
import os
import sys

# ========================================================================
def head(name, verbose=False, appendFormat=False):
    """
    Get the header of a legacy 'La Palma' cube

    Parameters
    ----------
    name : str
        name of the file
    verbose : bool, optional
        print out the header information (Default value = True)
    appendFormat : bool, optional
        different format to ensure append operation (Default =False)

    Returns
    -------
    header : tuple
        header information in order (nx, ny, nt, nstokes, dtype, ndims)

    Example
    -------
    >>> h = head('crispex_3950_2016-09-19T09:28:36_scans=11-117_time-corrected_im.fcube')
    ('head:', '[dtype=float32, ndims=3, nx=1734, ny=1240, nt=3317, nstokes=1] -> crispex_3950_2016-09-19T09:28:36_scans=11-117_time-corrected_im.fcube')
    >>> nx, ny, nt, ns, dtype, ndims = head('crispex_3950_2016-09-19T09:28:36_scans=11-117_time-corrected_im.fcube', verbose=False)
    >>> nx, ny, ns
    (1734, 1240, 1)
    """

    inam = 'head:'

    # Open file
    datfil = open(name, 'rb')

    # get header and extract dimensions
    head = (np.fromfile(datfil, dtype=np.dtype('a512'), count=1))[0]

    dum = head.decode("utf-8").split()
    datfil.close()

    ndims = 0
    dtype = 0
    nx = 0
    ny = 0
    nt = 0
    nstokes = 1

    for it in dum:
        du1 = it.split("=")
        if("dims" in du1[0]):
            ndims = int((du1[1].split(','))[0])
        elif("datatype" in du1[0]):
            dtype = int(du1[1].split(',')[0])
        elif("nx" in du1[0]):
            nx = int(du1[1].split(',')[0])
        elif("ny" in du1[0]):
            ny = int(du1[1].split(',')[0])
        elif("nt" in du1[0]):
            try:
                nt = int(du1[1].split(',')[0])
            except:
                pass # (integer label)
        elif("stokes" in du1[0]):
            du2 = du1[1].split(']')
            nstokes = int(np.size(du2[0].split(',')))

    if(dtype == 1):
        dtype = np.dtype('b')
    elif(dtype == 2):
        dtype = np.dtype('h')
    elif(dtype == 3):
        dtype = np.dtype('i')
    elif(dtype == 4):
        dtype = np.dtype('f')
    elif(dtype == 5):
        dtype = np.dtype('d')
    else:
        print((inam, 'Warning, dtype={0} not supported!'.format(dtype)))

    if(verbose):
        print(inam, "[dtype={0}, ndims={1}, nx={2}, ny={3}, nt={4}, nstokes={5}] -> {6}".format(
            dtype, ndims, nx, ny, nt, nstokes, os.path.basename(name)))

    if appendFormat is True:
        # read header and convert to string
        h = np.fromfile(name, dtype='uint8', count=512)
        header = ''
        for s in h[h > 0]:
            header += chr(s)
        # start reading at 'datatype'
        hd = header[header.lower().find('datatype'):]
        hd = hd.split(':')[0].replace(',', ' ').split()
        # Types:   uint8  int16 int32 float32
        typelist = ['u1', 'i2', 'i4', 'f4']
        # extract datatype
        try:
            dtype = typelist[int(hd[0].split('=')[1]) - 1]
        except:
            print(header)
            raise IOError('getheader: datatype invalid or missing')
        # extract endianness
        try:
            if hd[-1].split('=')[0].lower() != 'endian':
                raise IndexError()
            endian = hd[-1].split('=')[1]
        except IndexError:
            print(header)
            raise IOError('getheader: endianess missing.')
        if endian.lower() == 'l':
            dtype = '<' + dtype
        else:
            dtype = '>' + dtype
        # extract dims
        try:
            if hd[2].split('=')[0].lower() != 'dims':
                raise IndexError()
            dims = int(hd[2].split('=')[1])
            if dims not in [2, 3]:
                raise ValueError('Invalid dims=%i (must be 2 or 3)' % dims)
        except IndexError:
            print(header)
            raise IOError('getheader: dims invalid or missing.')
        try:
            if hd[3].split('=')[0].lower() != 'nx':
                raise IndexError()
            nx = int(hd[3].split('=')[1])
        except:
            print(header)
            raise IOError('getheader: nx invalid or missing.')
        try:
            if hd[4].split('=')[0].lower() != 'ny':
                raise IndexError()
            ny = int(hd[4].split('=')[1])
        except:
            print(header)
            raise IOError('getheader: ny invalid or missing.')
        if dims == 3:
            try:
                if hd[5].split('=')[0].lower() != 'nt':
                    raise IndexError()
                nt = int(hd[5].split('=')[1])
            except:
                print(header)
                raise IOError('getheader: nt invalid or missing.')
            shape = (nx, ny, nt)
        else:
            shape = (nx, ny)
        return [shape, dtype, header]

    return nx, ny, nt, nstokes, dtype, ndims

# ========================================================================
def read(cube, spnw=None, spformat='_sp', verb=False):
    """
    Read the full cube from a La Palma format file

    Parameters
    ----------
    cube : str
        filename, has to be .icube or .fcube
    spnw : str or int, optional
        Specific filename of spectral cube OR number of wavelength steps (Default value = None)
    spformat : str, optional
        filename identifier for the spectral cube (Default value = '_sp')
    verb : bool, optional
        Verbose mode. (Default value = False)

    Returns
    -------
    cube_array: ndarray
        5D cube of shape [nt,ns,nw,nx,ny]

    Examples
    --------
    >>> from ISPy.io import lapalma as lp
    >>> cube_a = lp.read('filename.fcube') # It searches for 'filename_sp.fcube' in the same path
    >>> cube_b = lp.read('filename.fcube' , 8)
    >>> cube_c = lp.read('filename.fcube' , 'filename_sp.fcube')

    :Authors:
        Alex Pietrow (ISP/SU 2019), Carlos Diaz (ISP/SU 2019)
    """
    if spnw is None:
        cube_format = cube[:-6]+'{0}.'+cube[-5:]
        f1 = cube_format.format('')
        f2 = cube_format.format(spformat)
        if not os.path.isfile(f2):
            raise ValueError('File '+f2+' was not found. Please '
                +'include the name of spectral file or wavelength steps.')
        nx, ny, ndum, ns, dtype, ndim = head(f1, False)
        nw, nt, ndum, ns, dtype, ndim = head(f2, False)
        cube_array = np.memmap(f1, shape=(
            nt, ns, nw, ny, nx), offset=512, dtype=dtype, mode='r')

    elif type(spnw) is str:
        f2 = str(spnw)
        if not os.path.isfile(f2):
            raise ValueError('File '+f2+' was not found. Please '
                +'include the name of spectral file or wavelength steps.')
        nx, ny, ndum, ns, dtype, ndim = head(f1, False)
        nw, nt, ndum, ns, dtype, ndim = head(f2, False)
        cube_array = np.memmap(f1, shape=(
            nt, ns, nw, ny, nx), offset=512, dtype=dtype, mode='r')

    elif type(spnw) is int:
        nw = int(spnw)
        nx, ny, hh2, ns, dtype, ndim = head(cube, False)
        nt = int(hh2/nw/ns)
        cube_array = np.memmap(cube, shape=(
            nt, ns, nw, ny, nx), offset=512, dtype=dtype, mode='r')

    if(verb):
        print("[dtype={0}, nx={2}, ny={3}, nt={4}, nstokes={5}, nwav={1}] -> {6}".format(
            dtype, nw, nx, ny, nt, ns, os.path.basename(cube)))

    return cube_array

# ========================================================================
def mk_header(image):
    """
    Create a La Palma format header of an image array

    Parameters
    ----------
    image : ndarray
        2D or 3D image array in La Palma ordering (nx, ny, nt)

    Returns
    -------
    header : str
        header of the cube
    """
    from struct import pack
    ss = image.shape
    # only 2D or 3D arrays
    if len(ss) not in [2, 3]:
        raise IndexError(
            'make_header: input array must be 2D or 3D, got %iD' % len(ss))
    dtypes = {'int8': ['(byte)', 1], 'int16': ['(integer)', 2], 'int32': [
        '(long)', 3], 'float32': ['(float)', 4]}
    if str(image.dtype) not in dtypes:
        raise ValueError('make_header: array type' +
                         ' %s not supported, must be one of %s' % (image.dtype, list(dtypes.keys())))
    sdt = dtypes[str(image.dtype)]
    header = ' datatype=%s %s, dims=%i, nx=%i, ny=%i' % (
        sdt[1], sdt[0], len(ss), ss[0], ss[1])
    if len(ss) == 3:
        header += ', nt=%i' % (ss[2])
    # endianess
    if pack('@h', 1) == pack('<h', 1):
        header += ', endian=l'
    else:
        header += ', endian=b'
    return header

# ========================================================================
def writeto(filename, image, extraheader='', dtype=None, verbose=False,
            append=False):
    """
    Submodule of "write". It writes a cube to disk in LaPalma format.
    Partially from https://github.com/ITA-Solar/helita/blob/master/helita/io/lp.py

    Parameters
    ----------
    filename : str
        name of the file
    image : ndarray
        data allocated in memory
    extraheader : str, optional
        extra header information to append to standard header (Default value = '')
    dtype : str, optional
        data type of the image (Default value = None)
    verbose : bool, optional
        verbose mode (Default value = False)
    append : bool, optional
        append `image` to existing file (Default value = False)

    Returns
    -------
    NoneType

    Examples
    --------
    writeto('path/cube.fcube', image, append=True)
    """

    if not os.path.isfile(filename):
        append = False
    # use dtype from array, if none is specified
    if dtype is None:
        dtype = image.dtype
    image = image.astype(dtype)
    if append:
        # check if image sizes/types are consistent with file
        sin, t, h = head(filename, verbose=verbose,
                           appendFormat=True)  # getheader(filename)
        if sin[:2] != image.shape[:2]:
            raise IOError('writeto: trying to write' +
                          ' %s images, but %s has %s images!' %
                          (repr(image.shape[:2]), filename, repr(sin[:2])))
        if np.dtype(t) != image.dtype:
            raise IOError('writeto: trying to write' +
                          ' %s type images, but %s nas %s images' %
                          (image.dtype, filename, np.dtype(t)))
        # add the nt of current image to the header
        hloc = h.lower().find('nt=')
        new_nt = str(sin[-1] + image.shape[-1])
        header = h[:hloc + 3] + new_nt + h[hloc + 3 + len(str(sin[-1])):]
    else:
        header = mk_header(image)
    if extraheader:
        header += ' : ' + extraheader
    # convert string to [unsigned] byte array
    hh = np.zeros(512, dtype='uint8')
    for i, ss in enumerate(header):
        hh[i] = ord(ss)
    # write header to file
    file_arr = np.memmap(filename, dtype='uint8',
                         mode=append and 'r+' or 'w+', shape=(512,))
    file_arr[:512] = hh[:]

    del file_arr
    # offset if appending
    apoff = append and np.prod(sin) * image.dtype.itemsize or 0
    # write array to file
    file_arr = np.memmap(filename, dtype=dtype, mode='r+',
                         order='F', offset=512 + apoff, shape=image.shape)
    file_arr[:] = image[:]

    del file_arr
    if verbose:
        if append:
            print(('Appended %s %s array into %s.' %
                   (image.shape, dtype, filename)))
        else:
            print(('Wrote %s, %s array of shape %s' %
                   (filename, dtype, image.shape)))
    return

# ========================================================================
def write(cube_array, name, stokes=True, sp=False, path=''):
    """
    Write a data cube in La Palma format to disc

    Parameters
    ----------
    cube_array : ndarray
        datacube in form of [t,s,w,x,y]
    name : str
        name of file with .icube/fcube extention
    stokes : bool, optional
        flag for if data has stokes or not. (Default value = True)
    sp : bool, optional
        Save spectral cube of shape. (Default value = False)
    path : str, optional
        Filepath where file needs to be saved.(Default value = '')

    Examples
    --------
    >>> from ISPy.io import lapalma as lp
    >>> lp.write(cube_array, 'cube.fcube', path='fits/')

    :Authors:
        Alex Pietrow (ISP/SU 2019), Carlos Diaz (ISP/SU 2019)
    """

    # Reshaping to save it in the right format:
    intensity = np.moveaxis(cube_array.astype(np.float32), [
                            0, 1, 2, 3, 4], [1, 0, -1, -2, 2])

    # 'write_buf' function is included
    if not stokes:
        nt, nx, ny, nw = intensity.shape
        ax = [(1, 2, 0, 3), (3, 0, 2, 1)]
        rs = [(nx, ny, nt * nw), (nw, nt, ny * nx)]
        extrahd = ''
    else:
        ns, nt, nx, ny, nw = intensity.shape
        ax = [(2, 3, 1, 0, 4), (4, 1, 3, 2, 0)]
        rs = [(nx, ny, nt * ns * nw), (nw, nt, ny * nx * ns)]
        extrahd = ', stokes=[I,Q,U,V], ns=4'

    # this is the image cube:
    im = np.transpose(intensity, axes=ax[0])
    im = im.reshape(rs[0])
    # this is the spectral cube
    if sp:
        sp = np.transpose(intensity, axes=ax[1])
        sp = sp.reshape(rs[1])
        writeto(path+name+'_sp.fcube', sp, extraheader=extrahd)

    writeto(path+name+'.fcube', im, extraheader=extrahd)
    return

# ========================================================================
def get(filename, index, verb=False):
    """
    Read a 2D image (slice) given a known index from a La Palma cube.

    Parameters
    ----------
    filename : str
        file to be opened. Has to be .icube or .fcube
    index : int
        chosen frame, where frame is t*nw*ns + s*nw + w
        t: time or scan number
        s: stokes parameter
        w: wavelength step
    verbose : bool, optional
        Verbose model. (Default value = False)

    Returns
    -------
    image : ndarray
        2D image slice

    Examples
    --------
    >>> image = lp_get('cube.fcube', 0)

    :Authors:
        G. Vissers (ITA UiO, 2016), A.G.M. Pietrow (2018), Carlos Diaz (ISP/SU 2019)
    """
    nx, ny, ndum, nstokes, dt, dum1 = head(filename, verb)
    # header offset + stepping through cube
    offset = 512 + index * nx * ny * np.dtype(dt).itemsize
    image = np.memmap(filename, dtype=dt, mode='r', shape=(nx, ny), offset=offset,
                      order='F')
    return image.T

# ========================================================================
def put(filename, image, append=True, verbose=False, stokes=True):
    """
    Append a new cube/slice to a pre-existent La Palma cube.

    Parameters
    ----------
    filename : str
        name of file with .icube/fcube extention
    image : ndarray
        datacube in form of [t,s,w,x,y]
    append : bool, optional
        append `image` to an existing file (Default value = True)
    verbose : bool, optional
        verbose mode (default False)
    stokes : bool, optional
        data has Stokes parameters (default True)

    Examples
    --------
    >>> lp_put('cube.fcube', cube2)

    To do:
        Insert a slice in a La Palma cube

    :Authors:
        Carlos Diaz,  G. Vissers, A.G.M. Pietrow (ISP/SU 2019)
    """

    # Reshaping to save it in the right format:
    intensity = np.moveaxis(image.astype(np.float32), [
                            0, 1, 2, 3, 4], [1, 0, -1, -2, 2])

    # 'write_buf' function is included
    if not stokes:
        nt, nx, ny, nw = intensity.shape
        ax = [(1, 2, 0, 3), (3, 0, 2, 1)]
        rs = [(nx, ny, nt * nw), (nw, nt, ny * nx)]
        extrahd = ''
    else:
        ns, nt, nx, ny, nw = intensity.shape
        ax = [(2, 3, 1, 0, 4), (4, 1, 3, 2, 0)]
        rs = [(nx, ny, nt * ns * nw), (nw, nt, ny * nx * ns)]
        extrahd = ', stokes=[I,Q,U,V], ns=4'

    # this is the image cube:
    im = np.transpose(intensity, axes=ax[0])
    im = im.reshape(rs[0])

    writeto(filename, im, verbose=verbose, append=append)
    return


import os
import re
import pickle
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.model_selection import train_test_split

PADDING_TOKEN = "~"
START_TOKEN = "^"
END_TOKEN = "$"
# 外文地名最大长度（字母数量）
DM_MAX_LENGTH_SOURCE = 100
# 汉语地名最大长度（汉字数量）
DM_MAX_LENGTH_CHINESE = 50

# DATA_FOLDER = "drive/MyDrive/Colab Notebooks/dmt/data"
DATA_FOLDER = "data"
BATCH_SIZE = 64

def preprocess_dm_source(dm):
    """
    处理外文地名
    1、将多个空格替换成一个空格
    2、删除两端的空格、换行符等
    :param dm: 一个地名
    :return: 处理后的地名
    """
    dm = re.sub(r'[" "]+', " ", dm)
    dm = dm.strip()
    return dm

def preprocess_dm_chinese(dm):
    """
    处理中文地名
    1、删除所有空格
    2、删除两端的空格、换行符等
    :param dm: 一个地名
    :return: 处理后的地名
    """
    dm = re.sub(r' ', "", dm)
    dm = dm.strip()
    return dm

def load_data(file_path):
    """
    读取地名文件，解析出外文和中文的字符总数（去重后），做成字符和索引映射表。
    加工地名数据，首尾增加开始和结束标记。
    :param file_path: 文件路径
    :return: 字符和索引映射表, 地名列表
    """
    df = pd.read_table(file_path)
    df.columns = ['source', 'chinese']
    # 获取外文和中文字符数组
    characters_source = sorted(list(set(df.source.unique().sum())))
    characters_chinese = sorted(list(set(df.chinese.unique().sum())))

    # 添加的开头结尾符号
    special_characters = [PADDING_TOKEN, START_TOKEN, END_TOKEN]
    token_to_idx_source = dict([(char, i) for i, char in enumerate(special_characters + characters_source)])
    token_to_idx_zh = dict([(char, i) for i, char in enumerate(special_characters + characters_chinese)])
    idx_to_token_source = dict([(i, char) for i, char in enumerate(special_characters + characters_source)])
    idx_to_token_zh = dict([(i, char) for i, char in enumerate(special_characters + characters_chinese)])

    # 给地名添加开始和结束符
    df['source'] = df['source'].apply(lambda x: START_TOKEN + preprocess_dm_source(x) + END_TOKEN)
    df['chinese'] = df['chinese'].apply(lambda x: START_TOKEN + preprocess_dm_chinese(x) + END_TOKEN)
    # 获取地名数组，1维
    dm_text_source = df.source.values.tolist()
    dm_text_chinese = df.chinese.values.tolist()
    return (dm_text_source, dm_text_chinese), (token_to_idx_source, token_to_idx_zh), (idx_to_token_source, idx_to_token_zh)

def dm_to_ids(dm_text, token_to_idx):
    """
    将外文地名转换成单词索引数组，中文地名转换成汉字索引数组
    :param dm_text: 一个元组，包含：外文地名数组和中文地名数组，数组都是1维的
    :param token_idx: 一个元组，包含：外文和中文的字符到索引的映射表
    :return: 外文地名单词索引数组 / 中文地名汉字索引数组, 2D, shape(数据集的大小, 语种地名的最大长度-预定义)
    """
    dm_text_source, dm_text_chinese = dm_text
    token_to_idx_source, token_to_idx_zh = token_to_idx
    # 使用固定长度，替代从数据集中查找最大长度
    # max_length_source = max([len(dm) for dm in dm_text_source]) + 2
    # max_length_chinese = max([len(dm) for dm in dm_text_chinese]) + 2
    # print(f"外文地名最大长度：{max_length_source}")
    # print(f"中文地名最大长度：{max_length_chinese}")

    # 将地名数据转换成定长，长度为最长地名的长度。padding内容为：0(字符到索引的映射表中，把PADDING_TOKEN放在第一位，所以它的index是0)
    dm_ids_source = np.zeros((len(dm_text_source), DM_MAX_LENGTH_SOURCE), dtype=np.float32)
    dm_ids_chinese = np.zeros((len(dm_text_chinese), DM_MAX_LENGTH_CHINESE), dtype=np.float32)

    for i in range(len(dm_text_source)):
        one_dm_source = dm_text_source[i]
        one_dm_chinese = dm_text_chinese[i]

        tokens_source = list(one_dm_source)
        tokens_chinese = list(one_dm_chinese)

        end_index_source = 0
        end_index_chinese = 0
        for index, token in enumerate(tokens_source):
            if token != "":
                dm_ids_source[i][index + 1] = token_to_idx_source[token]
                end_index_source = end_index_source + 1

        for index, token in enumerate(tokens_chinese):
            if token != "":
                dm_ids_chinese[i][index + 1] = token_to_idx_zh[token]
                end_index_chinese = end_index_chinese + 1

    return dm_ids_source, dm_ids_chinese

def get_numpy_array_from_dataset(ds):
    source = np.concatenate([inp for (inp, targ) in ds.as_numpy_iterator()])
    target = np.concatenate([targ for (inp, targ) in ds.as_numpy_iterator()])
    return source, target

def preprocess_and_save():
    dm_text, token_to_idx, idx_to_token = load_data(os.path.join(DATA_FOLDER, "argentine-chinese.txt"))

    dm_ids_source, dm_ids_chinese = dm_to_ids(dm_text, token_to_idx)

    # print(dm_ids_source.shape)
    # print(dm_ids_chinese.shape)
    # print(token_idx)
    # print(idx_token)

    # shuffle
    all_dataset = tf.data.Dataset.from_tensor_slices((dm_ids_source, dm_ids_chinese))
    all_dataset = all_dataset.shuffle(6000).batch(BATCH_SIZE, drop_remainder=False)
    all_npa_source, all_npa_chinese = get_numpy_array_from_dataset(all_dataset)

    # split to train and validate
    source_dm_train, source_dm_val, chinese_dm_train, chinese_dm_val = train_test_split(all_npa_source, all_npa_chinese,
                                                                                test_size=0.2)

    # drop remainder
    train_dataset = tf.data.Dataset.from_tensor_slices((source_dm_train, chinese_dm_train))
    train_dataset = train_dataset.batch(BATCH_SIZE, drop_remainder=True)
    val_dataset = tf.data.Dataset.from_tensor_slices((source_dm_val, chinese_dm_val))
    val_dataset = val_dataset.batch(BATCH_SIZE, drop_remainder=True)
    train_npa_source, train_npa_chinese = get_numpy_array_from_dataset(train_dataset)
    val_npa_source, val_npa_chinese = get_numpy_array_from_dataset(val_dataset)

    pickle.dump((
        (train_npa_source, train_npa_chinese),
        (val_npa_source, val_npa_chinese),
        token_to_idx,
        idx_to_token), open(os.path.join(DATA_FOLDER, 'preprocess.p'), 'wb'))

if __name__ == '__main__':
    preprocess_and_save()


import numpy as np
import pandas as pd
import statsmodels.api as sm
import didipack as didi

# the idea of the parameter class is to centralize the parameters of your project in a single class
# since its a class you can create several set of parameters and save them separatly
# you can overite the name function so that the name of the parameter is an automatic function of the parameter set
# you can also use it to loop through different values of the parameters with the function update_param_grid

##################
# example, creating the class
##################
# To use it you need to create our own parameters using inheritance.

# First create a set of little class containing the parameters, here are two examples.
class ParamsModelA:
    def __init__(self):
        self.alpha = 1
        self.beta = 0.5
        self.gamma = 21

class ParamsModelB:
    def __init__(self):
        self.alpha = 0.8
        self.nu = 1
        self.opti = 'adam'
        self.gamma = 21
        self.missing_params = 'nope'

# now create the main Params class which inherit didi.ParamsBasis
class Params(didi.ParamsBasis):
    def __init__(self):

        self.name_detail = ''
        self.name = ''
        self.seed = 12345

        # put here a link to the little class so the model are included
        self.a = ParamsModelA()
        self.b = ParamsModelB()

        self.update_model_name()

    def update_model_name(self):
        # you can overwrite the model name to have an automatic name, function of some key parameters
        # here is an example
        self.name = self.name_detail + 'Aalpha_' + str(self.a.alpha) + 'gamma_' + str(self.b.gamma) + 'Type_' + self.b.opti

##################
# examples of applicaiton
##################
# creating a parameters
par1 = Params()
# note that one of the advantage of this organisation in sub-classes is that it allows multiple parameter with the same name
# here for example both model A and B have an alpha parameters, but they are stored separatly.
print('par1.a.alpha',par1.a.alpha,'par1.b.alpha',par1.b.alpha,)
# this is quite usefull when you want to follow the notation of multiple papers or you have to many parameters for the greek alphabet.

# you can modify some parameters of this specific instatiations
par1.a.alpha = 8
par1.b.gamma = -1
# change the name after changing parameters in case we want to save it
par1.update_model_name()

# creating another set of parameters
par2 = Params() # this is initiated with the default parameters
print('The two parameter set have different values ')
print('par2.a.alpha',par2.a.alpha,'par1.a.alpha',par1.a.alpha)
print('\n','\n')

# printing the summary with the print_values function, can be usefull if you want to check what the model looks like
par1.print_values()
print('\n','\n')

# saving the parameters
par1.save(save_dir='save/'+par1.name)
par2.save(save_dir='save/'+par2.name)
# note that saving create a folder and put a parameter object in it. This is ideal if you want to save results for different parameter set
# whenever you come back to your folder of result, all is always contained in this par object.

# to load a parameter you first create a new parameter object
load_par_1 = Params()
# then you just load with the directory
load_par_1.load('save/'+par1.name)
# the parameter object is backward compatible
# if you update your project and add parameters you can still load it
# it will load all the parameters, set the new one to default value and print a warning.
# in this example, I saved a version of the parameters that did not had the b.missing_params values
# here is what happen when the parameter is loaded
par_missing = Params()
par_missing.load('save/missing')

##################
# grids
##################

# This last function is quite usefull to do some parameter girds search through the parameter class.
# You first create a list of list as below
# each list in the list of list is of dimension 3
# first the name of the sub param clas
# second the name of the parameter
# third the list of parameter you want the frid to come through
par = Params()
grid = [
    ['a','alpha',[0.1,0.2,0.3]],
    ['b','alpha',[0.1,0.2,0.3]],
    ['b','gamma',[1,2]]
]

# the grid has 18 combinations
# you can update the parameters to one paraticular combinations with the command
# the function update_param take
par.update_param_grid(grid, 0)

# the loop below goes through all combinations and print the values to show how the update works
print(("{:>10}"*3).format('a.alpha', 'b.alpha','b.gamma'))
print('-'*35)
for i in range(18):
    par.update_param_grid(grid,i)
    print(("{:>10}"*3).format(par.a.alpha, par.b.alpha,par.b.gamma))


import unittest
import numpy as np
from prml import nn

class TestReshape(unittest.TestCase):

    def test_reshape(self):
        self.assertRaises(ValueError, nn.reshape, 1, (2, 3))

        x = np.random.rand(2, 6)
        p = nn.Parameter(x)
        y = p.reshape(3, 4)
        self.assertTrue((x.reshape(3, 4) == y.value).all())
        y.backward(np.ones((3, 4)))
        self.assertTrue((p.grad == np.ones((2, 6))).all())

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from xlrd import *
import ipysheet as ips
import panel as pn
get_ipython().run_line_magic('matplotlib', 'inline')
from scipy import stats
pn.extension('katex')

# ## Groundwater Exam Solution -  2019-2020 ##
#
# _(The contents presented in this section were re-developed principally by Dr. P. K. Yadav with supervision from Prof. Rudolf Liedl)_

# **Q1. Aquifer Types**   (ca. 5 pts.)
#
#
# a. Differentiate between Aquifer, Aquitard and Aquiclude (3 points)
#
# b. Schematically present a confined aquifer (vertical cross-section) providing essential features with their legends (2 points)

# **Solution 1. a.**
#
# See slide: L03/08
#
# An **aquifer** or a groundwater reservoir can store and transmit significant (= exploitable) amounts of groundwater.
#
# An **aquitard** can store and transmit groundwater but to a much lesser extent than an (adjacent) aquifer.
#
# An **aquiclude** can store groundwater but cannot transmit groundwater.
#

# **Solution 1b** - (L03/11)
#
#
#
#
#
# **Confined Aquifer**
#
# 1. The essential feature of confined aquifer is provided in the figure above.
#

# **Q2. Groundwater storage (3 pts.)**
#
# Dry season in Dresden (2018-19) led to intense extraction of groundawater in rural areas. At one location in a confined aquifer (storage coeff. 4·10-4, total porosity = 30%),  the pressure head was lowered by 150 m. The thickness of the aquifer was measured to be 90 m before the beginning of extraction and the compressibility of the porous medium in that region is estimated 6·10-8 m2/N. Density of water can be assumed to be 1000 kg/m3.
#
# (Hint: $\Delta V_T = \alpha_{pm}\cdot\rho_w \cdot g \cdot  V_T \cdot  \Delta \psi $).
#
#
# a. Approximately how much water was extracted? (1 point)
#
# b. How much land subsidence due to water extraction is expected? (2 point)
#

# **Solution 2**
#
# Given relation:
#
# For part a. (see Tut 02/P4)
#
# $
# S_s  = \frac{\Delta V_w}{V_T\cdot \Delta \psi}
# $
#
# In confined aquifer $S$ is used, which is obtained from:
#
# $S = S_s \cdot m$
#
# $
#  \frac{S}{m} = \frac{\Delta V_w}{A_T\cdot m \cdot \Delta \psi}
# $
#
# So,
#
# $  \frac{\Delta V_w}{A}  = S \cdot \Delta \psi$
#
#
# For part b.
#
#
# $\Delta V_T = \alpha_{pm}\cdot\rho_w \cdot g \cdot  V_T \cdot  \Delta \psi $
#
# $V_T = A\times h$, with $A$ surface area and $h$ aquifer thickness.
#
# $\Delta V_T = A \times \Delta h$, with $\Delta h$ change in thickness.
#
# $A \times \Delta h= \alpha_{pm}\cdot\rho_w \cdot g \cdot  A \cdot h \cdot  \Delta \psi $
#
# $\Delta h = \alpha_{pm}\cdot\rho_w \cdot g \cdot h \cdot  \Delta \psi$, with $\Delta h$ being the land subsidence

# In[2]:

#  solution 2 a

# Given

A = 1 # m², assuming 1 m² aquifer area
h = 90 # m, aquifer height before extraction
d_psi = 150 # m, change in pressure head
S_2 = 4*10**-4 #  specific storage
rho_w = 1000 # Kg/m³, density of water
a_pm = 6* 10**-8 # m²/N = m-s²/kg, compressibility of porous medium
g = 9.81 # m/s², gravity factor

#Solution

d_V_w  = S_2*A*d_psi

print("The water abstraction volume per m\u00b2 aquifer is {0:0.3f}".format(d_V_w), "m\u00b3")

# In[3]:

#  solution 2 b

# Given

A = 1 # m², assuming 1 m² aquifer area
h = 90 # m, aquifer height before extraction
d_psi = 150 # m, change in pressure head
Ss = 4*10**-4 # specific storage
rho_w = 1000 # Kg/m³, density of water
a_pm = 6* 10**-8 # m²/N = m-s²/kg, compressibility of porous medium
g = 9.81 # m/s², gravity factor

#interim calculation
V_T = A*h # m³, Aquifer volume before extraction

#Solution

d_h  = a_pm*rho_w*g*h*d_psi

print("The water abstraction volume is {0:0.2f}".format(d_h), "m")

# **Q3. Aquifer Properties** (ca. 10 pts.)
#
# The hydraulic conductivity of a sample (length 15 cm, diameter 5 cm) is to be determined using a constant-head permeameter. For that 250 ml water is passed through the sample in 30 s while maintaining the head difference of 2.5 cm. Properties of water provided are:
# density of water at 20°C: 1000 kg/m3;                                                  dynamic viscosity of water at 20°C: 1.0087·10-3 Pa·s
#
# a.  Sketch the problem as accurately as possible providing essential features with legends (3 points)
#
#
# b. What will be the conductivity of the sample? (4 points)
#
# What is the intrinsic permeability of the sample? (2 points)
#
# c. What soil type is likely the sample? (1 point)
#
# (Hint: For the calculation of permeability, dynamic viscosity/density ratio is required.)
#
#
#

# **Solution 3** -
#
# **Solution 3a** (L05/15)
#
#
#
#
#

# In[4]:

#Solution 3b** (L05/15)

# Given

L_c = 15 # cm, column length
Dia_c = 5 # cm, diameter column
V_in= 250 # mL, water entering the column
t_c = 30 # s, time required to pass
d_3h = 2.5 # cm, head difference

# interim calculation
A_c = np.pi*Dia_c**2/4 # cm², Area of column
Q_c = V_in/t_c # cm³/s, assume 1mL = 1 cm³, Discharge out of column

#solution
K_c = (Q_c*L_c)/(A_c*d_3h) # cm/s, conductivity

print("The area of the column is {0:0.2f}".format(A_c), "cm\u00b2")
print("The discharge from the aquifer is {0:0.2f}".format(Q_c), "cm\u00b3/s")
print("The conductivity of the sample is {0:0.2f}".format(K_c), "cm/s")
print("The conductivity of the sample is {0:0.4f}".format(K_c/100), "m/s")

# In[5]:

#Solution 3c** (L05/18)

# Given

K_m = K_c/100 # m/s, conductivity
rho_3w = 1000 # Kg/m³, density of water
nu_w = 1.0087*10**-3 # Pa-s = Kg/m-s, dynamic viscosity
g = 9.81 # m/s², gravity factor

#solution
k_c = K_m*nu_w/(rho_3w*g)

print("The permeability of the sample is {0:0.10f}".format(k_c), "m\u00b2")
print("The permeability of the sample is {0:0.2E}".format(k_c), "m\u00b2")

# **Solution 3d**
# (L05/11)
#
# The sample in the column is likely gravel or coarse sand.

# **Q4. Sieve Analysis (ca. 6 pts.)**
#
# Sieve experiments were performed with the bore samples and the following observations were obtained:
#
# | mesh diameter [mm] 	| residue in the sieve [g] 	| Σ total 	| Σ/Σtotal 	|
# |:------------------:	|:------------------------:	|:-------:	|:--------:	|
# |          6         	|             0            	|         	|          	|
# |          2         	|            40            	|         	|          	|
# |         0.6        	|            250           	|         	|          	|
# |         0.2        	|            150           	|         	|          	|
# |        0.06        	|            60            	|         	|          	|
# |     <0.06 (cup)    	|            10            	|         	|          	|
#
#
# a. Draw the granulometric curve in the diagram below.			(ca. 5 pts.)
#
# b. Briefly characterise the sediment.						(ca. 1 pt.)

# In[6]:

#Solution 4 -(L03/18)

dia = [6,2,0.6,0.2, 0.06, 0.001] # mm, diameter <0.06 (cup)= 0.001
mass = [0, 40, 250, 150, 60, 10] # g, the residue in seive

# Calculation steps - filling table
Total_mass = np.sum(mass)  # add the mass column to get total mass
retain_per = mass/Total_mass*100   # retain percentage
retain_per_cumsum = np.cumsum(retain_per) # get the cummulative sum of the reatined
passing_per = 100 - retain_per_cumsum # substract 100-cummsum to get passing %

data = {"mesh diameter [mm]": dia, "residue in the sieve [g]": mass, "Σtotal": retain_per, "Σ/Σtotal": passing_per }

df1= pd.DataFrame(data)
df1

# In[7]:

# plotting
plt.rcParams['axes.linewidth']=2
#plt.rcParams["axes.edgecolor"]='white'
plt.rcParams['grid.linestyle']='--'
plt.rcParams['grid.linewidth']=1
x = np.append([10],dia) # adding data to extend over 6 mm dia
y = np.append([100],passing_per) # adding 100% to plot

fig = plt.figure(figsize=(9,6))
plt.semilogx(x, y, 'x-', color='red')
tics=x.tolist()

plt.grid(which='major', color='k', alpha=0.7)
plt.grid(which='minor', color='k', alpha=0.3)
plt.xticks(x, tics);
plt.yticks(np.arange(0,110,10));
plt.title('grain size distribution');
plt.xlabel('grain size d [mm]');
plt.ylabel('grain fraction < d ins % of total mass');

# **solution 4b**
#
# The sample can be considered uniformly distributed as over 70% of sample falls in the sand size (0.2 mm-2 mm). Therefore, the sample can be considered sandy.

# **Q5. Aquifer characterization** (ca. 8 pts.)
#
# Water levels in m a.s.l. were measured at three observation wells (see figure).
#
#
#
#
# a. Sketch hydraulic head isolines for increments of 0.5 m. (ca. 3 points.)
#
# b. Gravel layer (thickness (t1) = 1.5 m, and conductivity (K1) = 3.7 10-3 is embedded between two sandy layers (t2 = 2 m, K2 = 3·10-4 m/s; and t3 = 3 m, K3 = 4·10-4 m/s). If the hydraulic gradient is 1% and overall discharge is 1 m3/d per unit width of the aquifer, find the effective hydraulic conductivity considering a parallelly layered aquifer.
#
# (Hint: $K_{eff} = \frac{m}{\sum_{i=1}^n \frac{m_i}{K_i}}$ or $K_{eff} =  \sum_{i=1}^n\frac{m_i\cdot K_i}{m}$  ) (ca. 2 points)
#
# c. Distinguish between homogeneity and heterogeneity, and isotropy and anisotropy (ca. 3 points)
#
#
#

# **Solution 5a**
# (L07/08-09)
#
# The isolines and flow direction is provided in the figure below.
#
#
#

# In[8]:

# Solution 5b (L06/08-13)

# Given:

G_t1 = 2 # m, sandy layer top
G_t2 = 1.5 # m, gravel layer middle
G_t3 = 3 # m, sandy layer bottom
K_1 =  3.0*10**-4 # m/s cond. in G_t1
K_2 = 3.7*10**-3 # m/s cond. in G_t2
K_3 =  4.0*10**-4 # m/s cond. in G_t3
i = 1/100 # (), hydraulic gradient 1%
Q_5 = 1 # m³/d per-W, discharge per unit width

#intermediate calculation
m = G_t1+G_t2+G_t3 # m, total aq. thickness

K_ef_h = (1/m) * (G_t1*K_1 + G_t2*K_2 + G_t3*K_3) # m/s, eff. horizontal cond.
K_ef_v = m/(G_t1/K_1 + G_t2/K_2 + G_t3/K_3) # # m/s, eff. vertical cond.

print("The thickness of the aquifer is {0:0.3f}".format(m), "m")
print("The effective horizontal conductivity of the aquifer is {0:0.2E}".format(K_ef_h), "m/s")
print("The effective vertical conductivity of the aquifer is {0:0.2E}".format(K_ef_v), "m/s")

# **Solution 5c** (L06/23)
#
# **Homogeneity**: An aquifer is homogeneous when its parameters are constant throughout the porous medium, i.e. the properties of the medium are independent of space
#
# **Heterogeneity**: Heterogeneous aquifer have its properties varies in space or the properties are space dependent.
#
# **Isotropy**: This relates to properties of aquifer being independent of direction, i.e., $K_v = K_h$
#
# **Anisotropy**: In this case the aquifer properties are direction dependent, i.e., $K_v \neq K_h$.

# **Q6. Well** (ca. 5 pts.)
#
#
# a. Sketch the pumping scenario of an unconfined aquifer (vertical cross section) and label all possible quantities (ca 3 pts.)
#
#
# b. The conductivity of a confined aquifer (8 m thick) is estimated to be $4\cdot 10^{-4}$ m/s. If the steady-state discharge 50 m3/s, using the Theis equation ($s = Q/4\pi T·W(u)$, with $W(u) = 15$), find the drawdown in the aquifer. (2 points)
#

# **Solution 6a**
# (L08/16)
#
# Figure below presents the scenario of a well in an _unconfined_ aquifer.
#
#
#
#

# In[9]:

#Solution 6b

#Given

Q_6 = 50 # m³/s, discharge
K_6 = 4*10**-4 # m/s, conductivity
m_6 = 8 # m, thickness
W_u = 15 # (), well function

# interim cal.
T_6 = K_6 * m_6 # m²/s, Transmissivity T = K*m

# solution
s_6 = (Q_6/(4*np.pi*T_6)) * W_u # m, drawdown

print("The Transmissivity of the aquifer is {0:0.5f}".format(T_6), "m\u00b2/s")
print("The drawdown in the well is {0:0.2f}".format(s_6), "m")

# **Q7. Conservative Transport**  (ca. 7 pts.)
#
# a. How is reactive transport different to conservative transport in the aquifers. (2 points)
#
# b. With suitable sketch distinguish between advective flux and dispersive flux. (2 points)
#
#
# c. A column (L = 1.2 m and Ø = 5 cm) was packed with sandy soil (ne= 35%  K= 0,0002 m/s). The hydraulic head at the inlet and the outlet was set to 230 m and 235 m, resp. The NaCl solution with conc. 10 mg/L was steadily introduced to the column after saturating it with distilled water. The experiment condition was such that diffusive flow could be neglected.  You may make justified assumption for any missing information.
#
# c.i. What will be the advective mass flux at the outlet of the column? (1.5 points)
#
# c.ii. Considering initial concentration difference between inlet and outlet to be 10 mg/L, what    will be the dispersive mass flux at the outlet? (1.5 points)
#
# (Hint: Dispersive and Advective fluxes are either of $ n_e \cdot v\cdot C$ and $n_e\cdot \alpha \cdot v \cdot \Delta C/L$)

# **Solution 7a** (L09/05)
#
# A chemical in groundwater is subject to conservative transport processes if there is:
#
# + no interaction with the solid material,
# + no interaction with other chemicals,
# + no interaction with microbes.
#
# When either of the above are part of the groundwater, the transport process is reactive.
#

# **Solution 7b** (L09/09)
#
# The sketch below distinguish between advective and dispersive fluxes. The figure in the left is of advective process and that in the right results to dispersive flux.
#
#

# In[10]:

#Solution 7c

L_7 = 1.2 # m, col. length
Dia_7 = 5 # cm, col. diameter
ne_7 = 0.35 # (), effective porosity
K_7 = 0.0002 # m/s, conductivity
H_7in = 235 # m, head inlet
H_7out = 230 # m, head outlet
C_7 = 10 # mg/L, NaCl concentration
al_7 = 1 # m, assumed
C_7d = 10 # mg/L

#intermediate calc.
i_7 = (H_7in-H_7out)/L_7 # (), head gradient
v_7dar = K_7*i_7 # m/s, darcy velocity
v_7av = v_7dar/ne_7 # m/s, average linear velocity

# Solution
F_7ad = ne_7*v_7av*C_7 # mg-m/L-s, advective flux
F_7dis = ne_7*al_7*v_7av*C_7d/L_7 # mg-m/L-s, dispersive flux

print("The hydraulic gradient is {0:0.4f}".format(i_7), "")
print("The Darcy velocity is {0:0.4f}".format(v_7dar), "m/s")
print("The average linear velocity is {0:0.4f}".format(v_7av), "m/s")
print("The advective flux is {0:0.10f}".format(F_7ad), "mg-m/L-s")
print("The dispersive flux is {0:0.10f}".format(F_7dis), "mg-m/L-s")

# **Q8. Sorption Isotherms** (ca. 10 pts)
#
# Five batch tests (different initial concentrations – see table below) were performed to determine the sorption properties of a sediment. For each batch 20 g of sediment in 30 mL of water were used. The measured equilibrium solute concentrations are also provided in the table.
#
# a)	Complete the above table								(ca. 3 pts.)
#
# b)	Plot the results in the diagram below and draw a Henry isotherm			(ca. 3 pts.)
#
# c)	How is retardation related to isotherm (ca. 2 points)
# (value and unit!)		(ca. 2 pts.).
#
#
#
#
#

# In[11]:

head = ["Batch nr. ", "Initial Conc. (mg/L) ", "Equi. Conc. (mg/L)", "Sorbed mass (g)", "Sorbed mass/solid (mg/g)" ]
bn = np.array([1,2,3,4,5])
C_0 = np.array([5, 10, 15, 20, 25])# mg/L, initial conc.
C_eq = np.array([2.5, 4.9, 8, 9.8, 13.2])# mg/L, equilibrium conc.
s2 = ips.sheet(rows=6, columns=5, row_headers=False, column_headers=head)
ips.column(0, bn, row_start=0)
ips.column(1, C_0, row_start=0)
ips.column(2, C_eq, row_start=0);
s2

# In[12]:

# SOlution of Problem 10 a (T07/HP9)

# Given
v_ml = 30 # ml of water used in expt.
v_l = v_ml/1000 # L, unit conversion
m_s = 20 # g, solid mass used in expt.

bn = np.array([1,2,3,4,5])
C_0 = np.array([5, 10, 15, 20, 25])# mg/L, initial conc.
C_eq = np.array([2.5, 4.9, 8, 9.8, 13.2])# mg/L, equilibrium conc.
s_m = (C_0-C_eq)*v_l
m_m = s_m/m_s# mg/g, mass ratio

#output
d8 = {"Batch Nr": bn, "Initial Conc. (mg/L)": C_0, "Equi. Conc. (mg/L)": C_eq, "Sorbed mass (g)": s_m, "Sorbed mass/solid (mg/g)" :m_m}
df9 = pd.DataFrame(d8); df9

# In[13]:

# Solution of proble 10 (b) (T07/HP9)
# fit
slope, intercept, r_value, p_value, std_err = stats.linregress(C_eq, m_m) # linear regression

#plot and fit
fig = plt.figure(); plt.plot(C_eq, m_m, 'bo', label=' provided data');
pred = intercept + slope*C_eq # fit line
plt.plot(C_eq, pred, 'r', label='y={:.2E}x+{:.2f}'.format(slope,intercept)) ;
plt.xlabel(r"$C_{eq} $ mg/L"); plt.ylabel(r"$C_{a} $ mg/g");
plt.grid(); plt.legend(fontsize=11);  plt.text(2.2, 0.014,'$R^2 = %0.2f$' % r_value)
plt.text(2.2, 0.012,'$C_a = K_{d}\cdot C_{eq}$');

# **solution 8c**
# (L10/13)
#
# The following relation relates Retardation ($R$) with linear isotherm ($K_d$)
#
# $$
# R = 1+ \frac{1-n_e}{n_e}\rho_s K_d
# $$
#
# with effective porosity $n_e$, solid density $\rho_s$.

# **Q9. Groundwater Modelling** (ca. 8 points.)
#
# a. Distinguish between conceptual model and mathematical model; and between analytical solution and empirical solution (ca. 4 points).
#
# b. Draw a conceptual model for a rectangular aquifer 100 m long and 20 m wide. Discretize the domain with 1/10 of the length length-wise and 1/5 of the width width-wise. Assure that flow in the model is from left to right direction (ca. 3 points).
#
# c. How is a no-flow boundary condition mathematically defined? (ca. 1 point)
#
#

# **Solution 9a**
# (L11/04-06)
#
# A model or also a _conceptual model_ is a representation, an image or a description of a real system.
#
# example for a real system: porous medium with water flowing through the pores (Darcy experiment)
#
# A _mathematical model_ provides a quantitative representation of the relevant system components, processes and impacts in the area of investigation. The quantitative representation is based on mathematical equations.
#
# _Analytical solution_ : These are exact mathematical expressions solving the model equations.
#
# _Emperical solution_ : These are solution based on experimental results.
#
#
#

# **Solution 9b** - (L14/12)
#
#
#

# **Solution 9c**
# (L13/16)
#
# A no-flow boundary condition is special case of second type or Neumann boundary condition. For no flow condition head gradient is equated to zero, i.e., there is no gradient and thus no flow (water flows from high to low head). Mathematically, this is:
#
# $\frac{dh}{dx} = 0 $ for no-flow along $x-$axis, with $h$ representing head.
#
#

# Good Luck.