import pickle
import numpy as np
from scipy.sparse import csr_matrix
from flask import Flask, render_template

app = Flask(__name__)
top_images = []

# Compute page rank
def page_rank(graph, d=.85, maxerr=.001):
    n = graph.shape[0]
    count = 0

    # transform G into stochastic or markov matrix graph
    col_sum = graph.sum(axis=0)
    col_sum[col_sum == 0] = 1
    graph /= col_sum
    a = csr_matrix(graph)

    ro, r = np.zeros(n), np.ones(n)
    r = r / n
    teleportation = np.ones(n) / float(n)
    while np.sum(np.abs(r - ro)) > maxerr:
        ro = r.copy()
        # iterative method
        for i in range(0, n):
            incoming_links = np.array(a[:, i].todense())[:, 0]
            r[i] = ro.dot(incoming_links * d) + teleportation[i] * (1 - d)
        count += 1
    print("Iterations", count)
    return r / sum(r)

@app.route("/")
def visualise_top_images():
    return render_template('pr.html', top_images=top_images)

K = int(input("Enter the value of k: "))

# Get top K dominant images using Page rank
def get_top_images(pr, K):
    # indices = pr.argsort()[-K:][::-1]
    indices = np.argsort(pr)[::-1][:K].tolist()
    images = pickle.load(open("image_store100", "rb"))

    for index in indices:
        top_images.append(images[index] + '.jpg')
    print(top_images)
    app.run()

G = pickle.load(open("Input_graph_final", "rb"))

pr = page_rank(G, d=.85)
print("PageRank", pr)
get_top_images(pr, K)


# coding=utf-8
from ExpertSystem.models import System, SysObject

SESSION_KEY = "ExpertSystem"
SESSION_ES_CREATE_KEY = "ExpertSystem_create"

def init_session(request, system_id):
    system = System.objects.filter(id=system_id)
    sys_objects = SysObject.objects.filter(system=system)
    objects = []
    for object in sys_objects:
        objects.append({
            "name": object.name,
            "weight": 0,
        })
    session = {
        "system_id": system_id,
        "selected_params": {},
        "asked_questions": [],
        "objects": objects
    }

    request.session[SESSION_KEY] = session

def add_to_session(request, asked_questions=None, selected_params=None):
    session = request.session.get(SESSION_KEY)

    if asked_questions:
        session["asked_questions"] += asked_questions

    if selected_params:
        session["selected_params"] = selected_params

    request.session[SESSION_KEY] = session

def clear_session(request):
    if SESSION_KEY in request.session:
        del request.session[SESSION_KEY]

def init_es_create_session(request, system_id):
    # Как только мы начали редактирование системы, или создали новую, мы записываем ее в сессию,
    # это значит, что мы в режиме редактирования и нужно не добавлять сущности, а изменять
    session = {
        "system_id": system_id,
    }

    request.session[SESSION_ES_CREATE_KEY] = session

def clear_es_create_session(request):
    if SESSION_ES_CREATE_KEY in request.session:
        del request.session[SESSION_ES_CREATE_KEY]


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('orderplaced/',views.orderplaced),
    path('vendor/',views.vendor,name='vendor'),
    path('register/user/',views.customerRegister,name='register'),
    path('login/user/',views.customerLogin,name='login'),
    path('login/vendor/',views.vendorLogin,name='rlogin'),
    path('register/vendor/',views.vendorRegister,name='rregister'),
    path('profile/vendor/',views.vendorProfile,name='rprofile'),
    path('profile/user/',views.customerProfile,name='profile'),
    path('user/create/',views.createCustomer,name='ccreate'),
    path('user/update/<int:id>/',views.updateCustomer,name='cupdate'),
    path('vendor/create/',views.createVendor,name='rcreate'),
    path('vendor/update/<int:id>/',views.updateVendor,name='rupdate'),
    path('vendor/orderlist/',views.orderlist,name='orderlist'),
    path('vendor/menu/',views.menuManipulation,name='mmenu'),
    path('logout/',views.Logout,name='logout'),
    path('vendor/<int:pk>/',views.vendorMenu,name='menu'),
    path('checkout/',views.checkout,name='checkout'),
    path('orders/',views.orders,name='orders'),

]


import os

import mock
import contextlib2 as contextlib
import pytest
from vcr.cassette import Cassette, CassetteContextDecorator

from . import util
from okcupyd import settings
from okcupyd import util as okcupyd_util
from okcupyd.session import Session

def pytest_addoption(parser):
    okcupyd_util.add_command_line_options(parser.addoption, use_short_options=False)
    parser.addoption('--live', dest='skip_vcrpy', action='store_true', default=False,
                     help="Skip the patching of http requests in tests. "
                     "USE WITH CAUTION. This will interact with the okcupid "
                     "website and send messages with any provided user credentials.")
    parser.addoption('--scrub', dest='scrub', action='store_true', default=False,
                     help="USE WITH CAUTION. Don't scrub PII from "
                     "http requests/responses. This is useful for recording cassetes.")
    parser.addoption('--resave', dest='resave',
                     action='store_true', default=False,
                     help="Resave cassettes. Use to retoractively scrub cassettes.")
    parser.addoption('--cassette-mode', dest='cassette_mode', action='store',
                     default='once', help="Accept new requests in all tests.")
    parser.addoption('--record', dest='record', action='store_true',
                     default=False, help="Re-record cassettes for all tests.")

def patch(option, *patches, **kwargs):
    negate = kwargs.get('negate', False)
    @pytest.yield_fixture(autouse=True, scope='session')
    def patch_conditionally(request):
        condition = bool(request.config.getoption(option))
        if negate: condition = not condition
        if condition:
            with contextlib.ExitStack() as exit_stack:
                for patch in patches:
                    exit_stack.enter_context(patch)
                yield
        else:
            yield
    return patch_conditionally

patch_settings = patch('credentials_modules',
                       mock.patch.object(settings, 'USERNAME', 'username'),
                       mock.patch.object(settings, 'PASSWORD', 'password'),
                       mock.patch.object(settings, 'AF_USERNAME', 'username'),
                       mock.patch.object(settings, 'AF_PASSWORD', 'password'),
                       mock.patch.object(settings, 'DELAY', 0), negate=True)
patch_save = patch('resave',
                   mock.patch.object(
                       Cassette, '_save',
                       okcupyd_util.n_partialable(Cassette._save)(force=True)))
patch_use_cassette_enabled = patch('skip_vcrpy',
                                   mock.patch.object(CassetteContextDecorator,
                                                     '__enter__'),
                                   mock.patch.object(CassetteContextDecorator,
                                                     '__exit__'))
patch_vcrpy_filters = patch('scrub',
                            mock.patch.object(util, 'SHOULD_SCRUB', False),
                            negate=True)

original_cassette_enter = CassetteContextDecorator.__enter__
def new_cassette_enter(self):
    path, _ = self._args_getter()
    try:
        os.remove(path)
    except:
        pass
    original_cassette_enter(self)
patch_record = patch('record', mock.patch.object(CassetteContextDecorator,
                                                 '__enter__',
                                                 new_cassette_enter))

@pytest.fixture(autouse=True, scope='session')
def set_vcr_options(request):
    util.okcupyd_vcr.record_mode = request.config.getoption('cassette_mode')

@pytest.fixture(autouse=True, scope='session')
def process_command_line_args(request):
    okcupyd_util.handle_command_line_options(request.config.option)

@pytest.fixture
def session():
    with util.use_cassette(cassette_name='session_success'):
        return Session.login()


import time
import numpy as np
from pandas import Series
from Data import reader as dr

GENOME_TAGS = "Data/genome-tags.csv"
IMDB_ACTOR_INFO = "Data/imdb-actor-info.csv"
ML_MOVIES = "Data/mlmovies.csv"
ML_RATINGS = "Data/mlratings.csv"
ML_TAGS = "Data/mltags.csv"
ML_USERS = "Data/mlusers.csv"
MOVIE_ACTORS = "Data/movie-actor.csv"

genome_data = dr.read_data(GENOME_TAGS)
imdb_actor_data = dr.read_data(IMDB_ACTOR_INFO)
movies_data = dr.read_data(ML_MOVIES)
ratings_data = dr.read_data(ML_RATINGS)
tags_data = dr.read_data(ML_TAGS)
users_data = dr.read_data(ML_USERS)
movie_actors_data = dr.read_data(MOVIE_ACTORS)
genome_data['tagid'] = genome_data['tagId']
del genome_data['tagId']
imdb_actor_data = imdb_actor_data.rename(columns={'id': 'actorid'})

'''
print "genome data \n", genome_data.head()
print "imdb actor data \n", imdb_actor_data.head()
print "movies data \n", movies_data.head()
print "ratings data \n", ratings_data.head()
print "tags data \n", tags_data.head()
print "users data \n", users_data.head()
print "movie actor data \n", movie_actors_data.head()
'''

def epoch_time(date_time):
    pattern = '%Y-%m-%d %H:%M:%S'
    return int(time.mktime(time.strptime(date_time, pattern)))

# print tags_data.sort_values(['movieid','tagid'])
tags_data['epoch'] = np.vectorize(epoch_time)(tags_data['timestamp'])
del tags_data['timestamp']
max_epoch = max(tags_data['epoch'])
min_epoch = min(tags_data['epoch'])
tags_data['epoch'] = ((tags_data['epoch'] - min_epoch) / (max_epoch - min_epoch)) + 1

genres = movies_data['genres'].str.split('|').apply(Series, 1).stack()
genres.index = genres.index.droplevel(-1)
genres.name = 'genre'
del movies_data['genres']
movies_data = movies_data.join(genres)


import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import pdb

class StyleGANEncoder(nn.Module):
    """Defines the encoder network for StyleGAN inversion."""

    def __init__(
        self,
        resolution=256,
        w_space_dim=512,
        image_channels=3,
        encoder_channels_base=64,
        encoder_channels_max=1024,
    ):
        super().__init__()
        # resolution = 256
        # w_space_dim = 512
        # image_channels = 3
        # encoder_channels_base = 64
        # encoder_channels_max = 1024

        self.init_res = 4
        self.resolution = resolution
        self.w_space_dim = w_space_dim
        self.image_channels = image_channels
        self.encoder_channels_base = encoder_channels_base
        self.encoder_channels_max = encoder_channels_max

        # Blocks used in encoder.
        self.num_blocks = int(np.log2(resolution))  # 8

        # Layers used in generator.
        self.num_layers = int(np.log2(self.resolution // self.init_res * 2)) * 2
        # (Pdb) self.num_layers -- 14

        in_channels = self.image_channels
        out_channels = self.encoder_channels_base
        for block_idx in range(self.num_blocks):
            if block_idx == 0:
                self.add_module(
                    f"block{block_idx}",
                    FirstBlock(
                        in_channels=in_channels,
                        out_channels=out_channels,
                    ),
                )

            elif block_idx == self.num_blocks - 1:
                in_channels = in_channels * self.init_res * self.init_res
                out_channels = self.w_space_dim * 2 * block_idx
                self.add_module(
                    f"block{block_idx}",
                    LastBlock(
                        in_channels=in_channels,
                        out_channels=out_channels,
                    ),
                )

            else:
                if in_channels != out_channels:
                    self.add_module(
                        f"block{block_idx}",
                        ResBlock1(
                            in_channels=in_channels,
                            out_channels=out_channels,
                        ),
                    )
                else:
                    self.add_module(
                        f"block{block_idx}",
                        ResBlock2(
                            in_channels=in_channels,
                            out_channels=out_channels,
                        ),
                    )
            in_channels = out_channels
            out_channels = min(out_channels * 2, self.encoder_channels_max)

        self.downsample = AveragePoolingLayer()

    def forward(self, x):
        """
        The encoder takes images with `RGB` color channels and range [0, 1]
        as inputs, and encode the input images to W+ space of StyleGAN.
        """
        # (Pdb) x.size() -- [1, 3, 256, 256]

        # move x from [0.0, 1.0] to [-1.0, 1.0]
        x = 2.0 * (x - 0.5)
        # x = x.transpose(2, 0, 1)
        for block_idx in range(self.num_blocks):
            if 0 < block_idx < self.num_blocks - 1:
                x = self.downsample(x)
            x = self.__getattr__(f"block{block_idx}")(x)
        # (Pdb) x.size() -- [1, 7168]

        return x.view(1, 1, self.num_layers, 512)

class AveragePoolingLayer(nn.Module):
    """Implements the average pooling layer.

    Basically, this layer can be used to downsample feature maps from spatial
    domain.
    """

    def __init__(self, scale_factor=2):
        super().__init__()
        self.scale_factor = scale_factor

    def forward(self, x):
        ksize = [self.scale_factor, self.scale_factor]
        strides = [self.scale_factor, self.scale_factor]
        return F.avg_pool2d(x, kernel_size=ksize, stride=strides, padding=0)

class BatchNormLayer(nn.Module):
    """Implements batch normalization layer."""

    def __init__(self, channels, gamma=False, beta=True, decay=0.9, epsilon=1e-5):
        """Initializes with basic settings.

        Args:
          channels: Number of channels of the input tensor.
          gamma: Whether the scale (weight) of the affine mapping is learnable.
          beta: Whether the center (bias) of the affine mapping is learnable.
          decay: Decay factor for moving average operations in this layer.
          epsilon: A value added to the denominator for numerical stability.
        """
        super().__init__()
        self.bn = nn.BatchNorm2d(
            num_features=channels,
            affine=True,
            track_running_stats=True,
            momentum=1 - decay,
            eps=epsilon,
        )
        self.bn.weight.requires_grad = gamma
        self.bn.bias.requires_grad = beta

    def forward(self, x):
        return self.bn(x)

class WScaleLayer(nn.Module):
    """Implements the layer to scale weight variable and add bias.

    NOTE: The weight variable is trained in `nn.Conv2d` layer (or `nn.Linear`
    layer), and only scaled with a constant number, which is not trainable in
    this layer. However, the bias variable is trainable in this layer.
    """

    def __init__(self, in_channels, out_channels, kernel_size, gain=np.sqrt(2.0)):
        super().__init__()
        fan_in = in_channels * kernel_size * kernel_size
        self.scale = gain / np.sqrt(fan_in)
        self.bias = nn.Parameter(torch.zeros(out_channels))

    def forward(self, x):
        return x * self.scale + self.bias.view(1, -1, 1, 1)

class FirstBlock(nn.Module):
    """Implements the first block, which is a convolutional block."""

    def __init__(
        self,
        in_channels,
        out_channels,
        wscale_gain=np.sqrt(2.0),
    ):
        super().__init__()

        self.conv = nn.Conv2d(
            in_channels=in_channels,
            out_channels=out_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False,
        )
        self.scale = 1.0
        self.bn = BatchNormLayer(channels=out_channels)
        self.activate = nn.LeakyReLU(negative_slope=0.2, inplace=True)

        # self = FirstBlock(
        #   (conv): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        #   (bn): BatchNormLayer(
        #     (bn): BatchNorm2d(64, eps=1e-05, momentum=0.09999999999999998, affine=True, track_running_stats=True)
        #   )
        #   (activate): LeakyReLU(negative_slope=0.2, inplace=True)
        # )
        # in_channels = 3
        # out_channels = 64
        # wscale_gain = 1.4142135623730951

    def forward(self, x):
        return self.activate(self.bn(self.conv(x) * self.scale))

class ResBlock1(nn.Module):
    """Implements the residual block1 -- with shortcut."""

    def __init__(
        self,
        in_channels,
        out_channels,
        wscale_gain=np.sqrt(2.0),
    ):
        super().__init__()

        self.conv = nn.Conv2d(
            in_channels=in_channels,
            out_channels=out_channels,
            kernel_size=1,
            stride=1,
            padding=0,
            bias=False,
        )
        self.scale = 1.0
        self.bn = BatchNormLayer(channels=out_channels)

        hidden_channels = min(in_channels, out_channels)

        # First convolutional block.
        self.conv1 = nn.Conv2d(
            in_channels=in_channels,
            out_channels=hidden_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False,
        )
        self.scale1 = wscale_gain / np.sqrt(in_channels * 3 * 3)
        self.wscale1 = WScaleLayer(
            in_channels=in_channels,
            out_channels=hidden_channels,
            kernel_size=3,
            gain=wscale_gain,
        )
        self.bn1 = BatchNormLayer(channels=hidden_channels)

        # Second convolutional block.
        self.conv2 = nn.Conv2d(
            in_channels=hidden_channels,
            out_channels=out_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False,
        )
        self.scale2 = wscale_gain / np.sqrt(hidden_channels * 3 * 3)
        self.wscale2 = WScaleLayer(
            in_channels=hidden_channels,
            out_channels=out_channels,
            kernel_size=3,
            gain=wscale_gain,
        )
        self.bn2 = BatchNormLayer(channels=out_channels)
        self.activate = nn.LeakyReLU(negative_slope=0.2, inplace=True)

    def forward(self, x):
        y = self.activate(self.bn(self.conv(x) * self.scale))
        x = self.activate(self.bn1(self.wscale1(self.conv1(x) / self.scale1)))
        x = self.activate(self.bn2(self.wscale2(self.conv2(x) / self.scale2)))
        return x + y

class ResBlock2(nn.Module):
    """Implements the residual block1 -- without shortcut."""

    def __init__(
        self,
        in_channels,
        out_channels,
        wscale_gain=np.sqrt(2.0),
    ):
        super().__init__()

        hidden_channels = min(in_channels, out_channels)

        # First convolutional block.
        self.conv1 = nn.Conv2d(
            in_channels=in_channels,
            out_channels=hidden_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False,
        )
        self.scale1 = wscale_gain / np.sqrt(in_channels * 3 * 3)
        self.wscale1 = WScaleLayer(
            in_channels=in_channels,
            out_channels=hidden_channels,
            kernel_size=3,
            gain=wscale_gain,
        )
        self.bn1 = BatchNormLayer(channels=hidden_channels)

        # Second convolutional block.
        self.conv2 = nn.Conv2d(
            in_channels=hidden_channels,
            out_channels=out_channels,
            kernel_size=3,
            stride=1,
            padding=1,
            bias=False,
        )
        self.scale2 = wscale_gain / np.sqrt(hidden_channels * 3 * 3)
        self.wscale2 = WScaleLayer(
            in_channels=hidden_channels,
            out_channels=out_channels,
            kernel_size=3,
            gain=wscale_gain,
        )
        self.bn2 = BatchNormLayer(channels=out_channels)
        self.activate = nn.LeakyReLU(negative_slope=0.2, inplace=True)

    def forward(self, x):
        y = x
        x = self.activate(self.bn1(self.wscale1(self.conv1(x) / self.scale1)))
        x = self.activate(self.bn2(self.wscale2(self.conv2(x) / self.scale2)))
        return x + y

class LastBlock(nn.Module):
    """Implements the last block, which is a dense block."""

    def __init__(self, in_channels, out_channels, wscale_gain=1.0):
        super().__init__()

        self.fc = nn.Linear(
            in_features=in_channels, out_features=out_channels, bias=False
        )
        self.scale = wscale_gain / np.sqrt(in_channels)
        self.bn = BatchNormLayer(channels=out_channels)
        # self = LastBlock(
        #   (fc): Linear(in_features=16384, out_features=7168, bias=False)
        #   (bn): BatchNormLayer(
        #     (bn): BatchNorm2d(7168, eps=1e-05, momentum=0.09999999999999998, affine=True, track_running_stats=True)
        #   )
        # )
        # in_channels = 16384
        # out_channels = 7168
        # wscale_gain = 1.0

    def forward(self, x):
        x = x.view(x.shape[0], -1)
        x = self.fc(x) * self.scale
        x = x.view(x.shape[0], x.shape[1], 1, 1)
        return self.bn(x).view(x.shape[0], x.shape[1])

if __name__ == "__main__":
    model = StyleGANEncoder()
    model.eval()

    x = torch.randn(1, 3, 256, 256)

    with torch.no_grad():
        w = model(x)

    print(w.size())


import numpy as np
import librosa
from sklearn.mixture import GaussianMixture
import os
from sklearn.decomposition import PCA
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
import pickle

def load_wavs_as_matrices(filepath):
    x, sr = librosa.load(filepath)
    #mfccs=librosa.feature.mfcc(y,sr)
    #return mfccs.T
    mfccs = librosa.feature.mfcc(x, sr=sr)
    mfccs = mfccs.ravel()#reshape(mfccs.shape[0] * mfccs.shape[1])
    strs = ''
    for i in range(len(mfccs)):
        strs+=str(mfccs[i])+","
    strs = strs[0:len(strs)-1]
    return strs

'''
path = 'dataset'
X = []
Y = []

def load_wavs_as_matrices(filepath):
    x, sr = librosa.load(filepath)
    #mfccs=librosa.feature.mfcc(y,sr)
    #return mfccs.T
    mfccs = librosa.feature.mfcc(x, sr=sr)
    mfccs = mfccs.ravel()#reshape(mfccs.shape[0] * mfccs.shape[1])
    strs = ''
    for i in range(len(mfccs)):
        strs+=str(mfccs[i])+","
    strs = strs[0:len(strs)-1]
    return strs

data = ''
for i in range(0,4121):
    data+="F"+str(i)+","
data = data[0:len(data)-1]
data+="\n"
for root, dirs, directory in os.walk(path):
    for j in range(len(directory)):
        name = os.path.basename(root)
        features = load_wavs_as_matrices(root+"/"+directory[j])
        data+=features+"\n"
        if name == 'forge':
            Y.append(1)
        if name == 'real':
            Y.append(0)

f = open("my.csv", "w")
f.write(data)
f.close()
Y = np.asarray(Y)
np.save("Y.txt",Y)
'''
Y = np.load("Y.txt.npy")
dataset = pd.read_csv("my.csv")
dataset.fillna(0, inplace = True)
X = dataset.values
X = X[:,0:(dataset.shape[1]-1)]
scaler = MinMaxScaler()
X = scaler.fit_transform(X)

indices = np.arange(X.shape[0])
np.random.shuffle(indices)
X = X[indices]
Y = Y[indices]
pca = PCA(n_components = 30)
X = pca.fit_transform(X)

X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2)
#np.save("xtest.txt",X_test)
#np.save("ytest.txt",y_test)
X_test = np.load("xtest.txt.npy")
y_test = np.load("ytest.txt.npy")
print(y_train)
print(y_test)
'''
rfc = GaussianMixture(n_components=2, random_state=42)
rfc.fit(X, Y)
prediction_data = rfc.predict(X_test)
print(prediction_data)
accuracy = accuracy_score(y_test,prediction_data)*100
print(accuracy)
with open('model.txt', 'wb') as file:
    pickle.dump(rfc, file)
file.close()
'''
with open('model.txt', 'rb') as file:
    rfc = pickle.load(file)
file.close()
prediction_data = rfc.predict(X_test)
print(prediction_data)
accuracy = accuracy_score(y_test,prediction_data)*100
print(accuracy)
data = ''
for i in range(0,4121):
    data+="F"+str(i)+","
data = data[0:len(data)-1]
data+="\n"

data+=load_wavs_as_matrices("dataset/real/200003_real.wav")
f = open("test.csv", "w")
f.write(data)
f.close()

test = pd.read_csv("test.csv")
test.fillna(0, inplace = True)
test = test.values
test = test[:,0:(dataset.shape[1]-1)]
#scaler = MinMaxScaler()
test = scaler.transform(test)
test = pca.transform(test)
print(rfc.predict(test))
'''

X = np.asarray(X)
Y = np.asarray(Y)

print(X.shape)
print(Y.shape)
print(X)
#pca = PCA(n_components = 100)
#X = pca.fit_transform(X)
#print(X.shape)
'''


from multiprocessing import Lock
import dquality.common.constants as const

class Result:
    """
    This class is a container of result and parameters linked to the subject of the verification, and the
    verification type.
    """
    def __init__(self, res, index, quality_id, error):
        self.res = res
        self.index = index
        self.quality_id = quality_id
        self.error = error

class Data:
    """
    This class is a container of data. One of the members is a 2D arrray. Other members will be added later
    """
    def __init__(self, slice, ):
        self.slice = slice

class Aggregate:
    """
    This class is a container of results. The results are organized in three dictionaries.
    "bad_indexes": dictionary contains keys that are indexes of slices that not pass one or more quality checks.
    The values are results organized in dictionaries, where the keays are quality check method index.
    "good_indexes" is a similarly organized dictionary that contains indexes for which all quality checks passed.
    "results": a dictionary keyed by quality check id and a value of list of all results for "good" indexes.

    The class has locks, for each quality check type. The lock are used to access the results. One thread is adding
    to the results, and another thread (statistical checks) are reading the stored data to do statistical calculations.

    """
    locks = {const.QUALITYCHECK_MEAN : Lock(), const.QUALITYCHECK_STD : Lock(), const.STAT_MEAN : Lock()}
    lens = {const.QUALITYCHECK_MEAN : 0, const.QUALITYCHECK_STD : 0, const.STAT_MEAN : 0}

    def __init__(self):
        self.bad_indexes = {}
        self.good_indexes = {}
        self.results = {const.QUALITYCHECK_MEAN : [], const.QUALITYCHECK_STD : [], const.STAT_MEAN : []}

    def get_results_len(self, check):
        """
        This returns current length of the results of a given quality check. This operation uses lock, as other
        process writes new results.

        Parameters
        ----------
        check : int
            a value indication quality check id

        Returns
        -------
        int
            the length of the results list for the given quality check
        """
        lock = self.locks[check]
        lock.acquire()
        length = self.lens[check]
        lock.release()
        return length

    def add_result(self, result, check):
        """
        This add a new result for a given quality check to results. This operation uses lock, as other
        process reads the length.

        Parameters
        ----------
        result : Result
            a result instance

        check : int
            a value indication quality check id

        Returns
        -------
        None
        """
        lock = self.locks[check]
        lock.acquire()
        self.results[check].append(result)
        self.lens[check] += 1
        lock.release()

    def handle_result(self,result):
        """
        This function take a result instance that contains information about the result and the subject. If the
        slice did not pass verification for any of the quality check, it will return False. If all quality checks
        passed it will return True. The function maintains the containers; the slice index will be added to a
        "good_index" dictionary if all quality checks passed, and to "bad_index" otherwise.

        Parameters
        ----------
        result : Result
            a result instance

        Returns
        -------
        boolean
            True if all quelity checks passed
            False otherwise
        """
        res = False
        index = result.index
        if result.error is const.NO_ERROR:
            bad_index = self.bad_indexes.get(index)
            if bad_index is None:
                res = True
                good_index = self.good_indexes.get(index)
                if good_index is None:
                    entry = {result.quality_id : result.res}
                    self.good_indexes[index] = entry
                else:
                    good_index[result.quality_id] = result.res
                self.add_result(result.res, result.quality_id)
            else:
                bad_index[result.quality_id] = result.res
        else:
            good_index = self.good_indexes.get(index)
            if good_index is None:
                bad_index = self.bad_indexes.get(index)
                if bad_index is None:
                    entry = {result.quality_id : result.res}
                    self.bad_indexes[index] = entry
                else:
                    bad_index[result.quality_id] = result.res
            else:
                self.good_indexes.pop(index)
                bad_index = good_index
                bad_index[result.quality_id] = result.res
                self.bad_indexes[index] = bad_index
        return res


from django.conf.urls.defaults import *
from django.contrib.admin import autodiscover, site

autodiscover()

urlpatterns = patterns('',
    url(
        regex = r'^admin/',
        view  = include(site.urls),
    ),
    url(
        regex = r'^admin/orderable/',
        view  = include('orderable.urls', namespace='orderable'),
    ),
)


from social_core.exceptions import AuthForbidden
import urllib.request
import os

def download_user_img(image_url, userid):
    if bool(userid) and bool(image_url):
        avatars_dirname = 'users_avatars'
        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        avatars_dir = os.path.join(base_dir, 'media', avatars_dirname)
        img_name = image_url.split('/')[-1]
        extension = img_name.split('.')[1]
        if bool(img_name) and bool(extension):
            full_file_name = os.path.join(avatars_dir, f'{userid}.{extension}')
            link = urllib.request.urlretrieve(image_url, full_file_name)[0]
            if link:
                return os.path.join(avatars_dirname, os.path.split(full_file_name)[1])

def save_user_profile(backend, user, response, *args, **kwargs):
    if backend.name == "google-oauth2":

        print(f'G+ RESPONSE: {response}')

        if 'gender' in response.keys():
            if response['gender'] == 'male':
                user.shopuserprofile.gender = 'M'
            else:
                user.shopuserprofile.gender = 'W'

        if 'tagline' in response.keys():
            user.shopuserprofile.tagline = response['tagline']

        if 'aboutMe' in response.keys():
            user.shopuserprofile.aboutMe = response['aboutMe']

        if 'picture' in response.keys():
            imgurl = response['picture']
            if imgurl:
                avatar_path = download_user_img(imgurl, user.id)
                if avatar_path:
                    user.avatar = avatar_path

        # if 'ageRange' not in response.keys():
        #         raise AuthForbidden('social_core.backends.google.GoogleOAuth2')

        user.save()

    return


from urllib2 import urlopen as uReq
from bs4 import BeautifulSoup as soup
import time
import twilio
import twilio.rest

account_sid = 'AC05a19e314e2e0a36da9d8966556c359c'
auth_token = 'What ever'
twilio_phone_number = 'Theirs'
my_phone_number = 'Yours'

# this is an optional parameter for the frequency you wish
alloted_time  = 10
#this is an optional parameter for the duration you wish the search to occur, the default is one day
duration = 86400

#the following bit is for use in the while loop
iteration = 0;
my_url = 'https://www.google.com'
#optional parameters of time can be changed here:

#connection
def main():
    uClient = uReq(my_url)
    page_html = uClient.read()
    uClient.close()

    page_soup = soup(page_html, "html.parser")
    href_tags = page_soup.find_all(href=True)
    initial_result = len(href_tags)
    time.sleep(10)
    href_tags = page_soup.find_all(href=True)
    new_result = len(href_tags)
    print (new_result - initial_result) # tell you how many links were added or removed
    if new_result - initial_result == 0:
        print ("no change")
    elif new_result - initial_result < 0 or new_result - initial_result > 0:
        client = twilio.rest.Client('AC05a19e314e2e0a36da9d8966556c359c', '8cf175a0d0d3587e9a8ceece40bfa2c6')

        client.messages.create(
            body="Google just changed something on their homepage",
            to=my_phone_number,
            from_=twilio_phone_number
            )

    else:
        print ("nothing noticed")

while iteration < duration / alloted_time:
    main()
    time.sleep(alloted_time)
    iteration += 1


from xai.brain.wordbase.nouns._runway import _RUNWAY

#calss header
class _RUNWAYS(_RUNWAY, ):
	def __init__(self,):
		_RUNWAY.__init__(self)
		self.name = "RUNWAYS"
		self.specie = 'nouns'
		self.basic = "runway"
		self.jsondata = {}


#!/usr/bin/env python
import cmor
import numpy

def main():

    missing = -99.
    cmor.setup(inpath='Tables',
               netcdf_file_action = cmor.CMOR_REPLACE)
    cmor.dataset('historical', 'ukmo', 'HadCM3', '360_day',
                 institute_id = 'ukmo',
                 forcing = 'Nat',
                 model_id = 'HadCM3',
                 contact="Bleachers 140",
                 parent_experiment_rip="r1i3p2",
                 parent_experiment_id="lgm",branch_time=0.)

    table = 'CMIP5_Amon'
    cmor.load_table(table)
    axes = [ {'table_entry': 'time',
              'units': 'days since 2000-01-01 00:00:00',
              },
             {'table_entry': 'latitude',
              'units': 'degrees_north',
              'coord_vals': [0],
              'cell_bounds': [-1, 1]},
             {'table_entry': 'longitude',
              'units': 'degrees_east',
              'coord_vals': [90],
              'cell_bounds': [89, 91]},
             ]

    values = numpy.array([missing], numpy.float32)
    myma = numpy.ma.masked_values(values, missing)
    axis_ids = list()
    for axis in axes:
        axis_id = cmor.axis(**axis)
        axis_ids.append(axis_id)
    varid = cmor.variable('ts', 'K', axis_ids, missing_value = myma.fill_value)

    cmor.write(varid, myma, time_vals = [15], time_bnds = [ [0,30] ])

    cmor.close(varid)

if __name__ == '__main__':

    main()


# -*- coding: utf-8 -*-
import scrapy

class LovemondaysSpider(scrapy.Spider):
    name = 'Programathor'
    allowed_domains = ['programathor.com.br']
    start_urls = ['https://programathor.com.br/jobs/page/1']

    def parse(self, response):
        print(response)
        for job in response.xpath("/html/body/div[3]/div/div[2]/div[2]/div"):
            yield {
            'Vaga' : job.xpath("/a/div/div[2]/div/h3/text()").extract_first(),
            'Empresa' : job.xpath("/a/div/div[2]/div/div[1]/span[1]/text()").extract_first(),
            'Cidade' : job.xpath("/a/div/div[2]/div/div[1]/span[2]/text()").extract_first(),
            'data' : job.xpath("/a/div/div[2]/div/h3/span/text()").extract_first(),
            }
        url = response.url
        print(url)
        pagina = url.split('page/').pop(1)
        pagina = int(pagina)
        pagina = str(pagina +1)
        next_page='https://programathor.com.br/jobs/page/'+pagina

        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)


"""module to summarize and analyze HTCondor log files."""

import logging
import os
import re
import socket
import sys
import numpy as np

from plotille import Figure
from datetime import datetime as date_time, timedelta
from htcondor import JobEventLog, JobEventType as jet
from rich import print as rprint
from rich.progress import Progress, track
from typing import List

# import own class
from htcanalyze.resource import Resource, create_avg_on_resources

class HTCAnalyze:
    """
    This class is able to analyze HTCondor Joblogs.

    The modes:
        analyze,
        summarize,
        analyzed-summary,
        filter_for

    """

    def __init__(self,
                 ext_log="",
                 ext_err=".err",
                 ext_out=".out",
                 show_list=None,
                 rdns_lookup=None,
                 tolerated_usage=None,
                 bad_usage=None,
                 show_legend=True):
        self.ext_log = ext_log
        self.ext_err = ext_err
        self.ext_out = ext_out

        self.show_list = [] if show_list is None else show_list

        self.rdns_cache = dict()
        self.rdns_lookup = False if rdns_lookup is None else rdns_lookup

        self.tolerated_usage = 0.1 if \
            tolerated_usage is None else tolerated_usage
        self.bad_usage = 0.25 if bad_usage is None else bad_usage

        self.show_legend = show_legend  # relevant for histogram

    def read_file(self, file: str, file_ext):
        """
        Read a file.

        :param: file
        :return: content
        """
        output_string = ""
        try:

            if os.path.getsize(file) == 0:
                return output_string

            with open(file, "r") as output_content:
                output_string = output_content.read()
        except NameError as err:
            self.handle_name_error(err, file_ext)
        except FileNotFoundError:
            self.handle_filenotfound_error(file, file_ext)
        except TypeError as err:
            logging.exception(err)
        finally:
            return output_string

    @classmethod
    def handle_name_error(cls, err, file_ext):
        """
        Handle NameError.

        :param err: An exception
        :param file_ext: file extension
        :return: None
        """
        logging.exception(err)
        rprint(f"[red]The smart_output method requires a "
               f"{file_ext} file as parameter[/red]")

    @classmethod
    def handle_filenotfound_error(cls, file, file_ext):
        """
        Handle FileNotFoundError.

        :param file: file that was not found
        :param file_ext: file extension
        :return: None
        """
        relevant = file.split(os.path.sep)[-2:]
        # match the file on ProcessID_ClusterID, if possible
        match = re.match(r".*?([0-9]{3,}_[0-9]+)" + file_ext, relevant[1])
        rprint(
            f"[yellow]There is no related {file_ext} "
            f"file: {relevant[1]} in the directory:\n"
            f"[/yellow][cyan]'{os.path.abspath(relevant[0])}'\n"
            f"with the prefix: {match[1]}[/cyan]"
        )

    def htcondor_stderr(self, file: str) -> str:
        """
        Read HTCondor stderr files.

        :param file: HTCondor stderr file
        :return: filtered content
        """
        output_string = ""

        for line in self.read_file(file, self.ext_err):
            line = line.strip("\n")
            if "err" in line.lower():
                output_string += f"[red]{line}[/red]\n"
            elif "warn" in line.lower():
                output_string += f"[yellow]{line}[/yellow]\n"

        return output_string

    def htcondor_stdout(self, file: str) -> str:
        """
        Read HTCondor stdout files.

        :param: HTCondor stdout file
        :return: content
        """
        return self.read_file(file, self.ext_out)

    def get_job_spec_id(self, file: str) -> str:
        """
        Get job specification id from a HTCondor file.

        Insert a HTCondor file cut of the suffix,
         to get just the job specification id

        Example:
        get_job_spec_id("43221_23.log", ".log") -> "43221_23"

        :param file:
        :param ext_log:
        :return: file prefix
        """
        if self.ext_log.__ne__("") \
                and file[-len(self.ext_log):].__eq__(self.ext_log):
            job_spec_id = file[:-len(self.ext_log)]
        else:
            job_spec_id = os.path.splitext(file)[0]
        return job_spec_id

    def log_to_dict(self, file: str, sec: int = 0
                    ) -> (dict, dict, dict, dict, dict):
        """
        Read the log file with the htcondor module.

        Return five dicts holding information about:
        execution node, used resources, times, used ram history and errors

        :type file: str
        :param file: HTCondor log file
        :param sec: seconds to wait for new events
        :return: job_dict, res_dict, time_dict, ram_history, errors

        Consider that the return values can be None or empty dictionarys
        """
        job_events = list()
        resources = list()
        time_dict = {
            "Submission date": None,
            "Execution date": None,
            "Termination date": None
        }
        ram_history = list()
        occurred_errors = list()

        has_terminated = False
        invalid_file = False

        jel = JobEventLog(file)
        events = list()

        try:
            # Read all currently-available events
            # waiting for 'sec' seconds for the next event.
            for event in jel.events(sec):
                events.append(event)

        except OSError as err:
            logging.exception(err)
            invalid_file = True
            if err.args[0] == "ULOG_RD_ERROR":
                rprint(f"[red]{err}: {file}[/red]")
                reason = "Error while reading log file. " \
                         "File was manipulated or contains gpu usage."
                occurred_errors.append(
                    ["None", "Now", "ULOG_RD_ERROR", reason])
            else:
                rprint(f"[red]Not able to open the file: {file}[/red]")

        for event in events:
            event_type_number = event.get('EventTypeNumber')
            # convert time to datetime object
            date = date_time.strptime(event.get('EventTime'),
                                      "%Y-%m-%dT%H:%M:%S")
            # update submit date, submission host
            if event.type == jet.SUBMIT:
                time_dict["Submission date"] = date

                match_from_host = re.match(r"<(.+):[0-9]+\?(.*)>",
                                           event.get('SubmitHost'))
                if match_from_host:
                    submitted_host = match_from_host[1]
                    job_events.append(('Submitted from', submitted_host))
                # ERROR
                else:
                    invalid_file = True
                    reason = "Can't read user address"
                    occurred_errors.append(
                        [event_type_number, "Now", "invalid user address",
                         reason])
                    job_events.append(('Submitted from', "invalid user"))

                    try:
                        raise_value_error(f"Wrong submission host: {file}")
                    except ValueError as err:
                        logging.exception(err)
                        rprint(f"[red]{err.__class__.__name__}: {err}[/red]")

            # update execution date, execution node
            if event.type == jet.EXECUTE:
                time_dict["Execution date"] = date

                match_to_host = re.match(r"<(.+):[0-9]+\?(.*)>",
                                         event.get('ExecuteHost'))
                if match_to_host:
                    execution_host = match_to_host[1]
                    if self.rdns_lookup:  # resolve
                        execution_host = \
                            self.gethostbyaddrcached(execution_host)

                    job_events.append(('Executing on', execution_host))
                # ERROR
                else:
                    invalid_file = True
                    reason = "Can't read host address"
                    occurred_errors.append(
                        [event_type_number, "Now", "invalid host address",
                         reason])
                    job_events.append(('Executing on', "invalid host"))
                    try:
                        raise_value_error(f"Wrong execution host: {file}")
                    except ValueError as err:
                        logging.exception(err)
                        rprint(f"[red]{err.__class__.__name__}: {err}[/red]")

            # update ram history dict
            if event.type == jet.IMAGE_SIZE:
                size_update = event.get('Size')
                memory_usage = event.get('MemoryUsage')
                resident_set_size = event.get('ResidentSetSize')
                ram_history.append((date, size_update, memory_usage,
                                    resident_set_size))

            # update resource dict and termination date
            if event.type == jet.JOB_TERMINATED:
                has_terminated = True
                time_dict["Termination date"] = date

                # get all resources, replace by np.nan if value is None
                cpu_usage = event.get('CpusUsage', np.nan)
                cpu_requested = event.get('RequestCpus', np.nan)
                cpu_allocated = event.get('Cpus', np.nan)
                disk_usage = event.get('DiskUsage', np.nan)
                disk_requested = event.get('RequestDisk', np.nan)
                disk_allocated = event.get("Disk", np.nan)
                memory_usage = event.get('MemoryUsage', np.nan)
                memory_requested = event.get('RequestMemory', np.nan)
                memory_allocated = event.get('Memory', np.nan)

                # create list with resources
                resources = [Resource("CPU",
                                      cpu_usage,
                                      cpu_requested,
                                      cpu_allocated),
                             Resource("Disk (KB)",
                                      disk_usage,
                                      disk_requested,
                                      disk_allocated),
                             Resource("Memory (MB)",
                                      memory_usage,
                                      memory_requested,
                                      memory_allocated)]

                normal_termination = event.get('TerminatedNormally')
                # differentiate between normal and abnormal termination
                if normal_termination:
                    job_events.insert(0, ("Termination State",
                                          "[green]Normal[/green]"))
                    return_value = event.get('ReturnValue')
                    job_events.append(("Return Value", return_value))
                else:
                    job_events.insert(0, ("Termination State",
                                          "[red]Abnormal[/red]"))
                    signal = event.get('TerminatedBySignal')
                    job_events.append(("Terminated by Signal", signal))

            # update error dict and termination date
            if event.type == jet.JOB_ABORTED:
                has_terminated = True
                time_dict["Termination date"] = date

                reason = event.get('Reason')
                occurred_errors.append(
                    [event_type_number, date.strftime("%m/%d %H:%M:%S"),
                     "Aborted", reason])
                job_events.insert(0, ("Process was", "[red]Aborted[/red]"))

            # update error dict
            if event.type == jet.JOB_HELD:
                reason = event.get('HoldReason')
                occurred_errors.append(
                    [event_type_number, date.strftime("%m/%d %H:%M:%S"),
                     "JOB_HELD", reason])

            # update error dict
            if event.type == jet.SHADOW_EXCEPTION:
                reason = event.get('Message')
                occurred_errors.append((event_type_number,
                                        date.strftime("%m/%d %H:%M:%S"),
                                        "SHADOW_EXCEPTION", reason))

        # End of the file

        # generate a better time dict
        better_time_dict = gen_time_dict(*time_dict.values())

        # Job still running and file valid
        if not invalid_file and not has_terminated:
            if "Total runtime" in better_time_dict["Dates and times"]:
                rprint("[red]This is not supposed to happen,"
                       " check your code[/red]")
                state = "Strange"
            elif "Execution runtime" in better_time_dict["Dates and times"]:
                state = "Executing"
            elif "Waiting time" in better_time_dict["Dates and times"]:
                state = "Waiting"
            else:
                state = "Unknown"
            job_events.insert(0, ("Process is", f"[blue]{state}[/blue]"))
        # file not fully readable
        elif invalid_file:
            better_time_dict = dict()  # times have no meaning here
            job_events.insert(0, ("Error", "[red]Error while reading[/red]"))

        job_events_dict = dict()
        error_dict = dict()
        ram_history_dict = dict()
        # convert job_events to a nice and simple dictionary
        if job_events:
            desc, val = zip(*job_events)
            job_events_dict = {
                "Execution details": list(desc),
                "Values": list(val)
            }

        # convert errors into a dictionary
        if occurred_errors:
            event_numbers, time_list, errors, reasons = zip(*occurred_errors)
            error_dict = {
                "Event Number": list(event_numbers),
                "Time": list(time_list),
                "Error": list(errors),
                "Reason": list(reasons)
            }
        # convert ram_history to a dictionary
        if ram_history:
            time_list, img_size, mem_usage, res_set_size = zip(*ram_history)
            ram_history_dict = {
                "Dates": list(time_list),
                "Image size updates": list(img_size),
                "Memory usages": list(mem_usage),
                "Resident Set Sizes": list(res_set_size)
            }

        return (job_events_dict,
                resources,
                better_time_dict,
                ram_history_dict,
                error_dict)

    def gethostbyaddrcached(self, ip: str):
        """
        Get the hostname by address, with an in-memory cache.

        This prevents excessive queries to DNS servers.

        :param ip: ip represented by a string
        :return: resolved domain name, else give back the IP
        """
        try:
            # try our cache first
            return self.rdns_cache[ip]
        except KeyError:
            # do the lookup
            try:
                rdns = socket.gethostbyaddr(ip)
                logging.debug(f"rDNS lookup successful: "
                              f"{ip} resolved as {rdns[0]}")
                self.rdns_cache[ip] = rdns[0]
                return rdns[0]
            except socket.error:
                logging.debug(f"Unable to perform rDNS lookup for {ip}")
                # cache negative responses too
                self.rdns_cache[ip] = ip
                return ip

    def analyze(self, log_files: List[str]) -> List[dict]:
        """
        Analyze the given log files one by one.

        :param log_files: list of valid HTCondor log files
        :return: list with information of each log file
        """
        logging.info('Starting the analyze mode')

        if not log_files:
            raise_value_error("No files to analyze")

        result_list = list()

        # create progressbar, do not redirect output
        with Progress(transient=True, redirect_stdout=False,
                      redirect_stderr=False) as progress:

            task = progress.add_task("Analysing...", total=len(log_files))

            for file in log_files:
                progress.update(task, advance=1)
                result_dict = dict()

                logging.debug(f"Analysing the HTCondor log file: {file}")
                msg = f"[green]Job analysis of: {file}[/green]"
                result_dict["description"] = msg

                job_dict, resource_list, time_dict, \
                    ram_history, occurred_errors = self.log_to_dict(file)
                if job_dict:
                    result_dict["execution-details"] = job_dict

                if time_dict:
                    result_dict["times"] = time_dict

                if resource_list:
                    result_dict["all-resources"] = resource_list

                # show HTCondor errors
                if occurred_errors:
                    result_dict["errors"] = occurred_errors

                # managing the ram history
                if ram_history:
                    ram = np.array(ram_history.get('Image size updates'))
                    dates = np.array(ram_history.get('Dates'))

                if ram_history and len(ram) > 1:
                    fig = Figure()
                    fig.width = 55
                    fig.height = 15
                    fig.set_x_limits(min_=min(dates))
                    min_ram = int(min(ram))  # raises error if not casted
                    fig.set_y_limits(min_=min_ram)
                    fig.y_label = "Usage"
                    fig.x_label = "Time"

                    # this will use the self written function _
                    # num_formatter, to convert the y-label to int values
                    fig.register_label_formatter(float, _int_formatter)
                    fig.plot(dates, ram, lc='green', label="Continuous Graph")
                    fig.scatter(dates, ram, lc='red', label="Single Values")

                    result_dict["ram-history"] = fig.show(
                        legend=self.show_legend)
                elif ram_history:
                    msg = f"Single memory update found:\n" \
                        f"Memory usage on the {dates[0]} " \
                        f"was updatet to {ram[0]} MB"
                    result_dict["ram-history"] = msg

                if self.show_list:
                    job_spec_id = self.get_job_spec_id(file)
                    if 'htc-err' in self.show_list:
                        result_dict['htc-err'] = self.htcondor_stderr(
                            job_spec_id + self.ext_err)
                    if 'htc-out' in self.show_list:
                        result_dict['htc-out'] = self.htcondor_stdout(
                            job_spec_id + self.ext_out)

                result_list.append(result_dict)

        return result_list

    def summarize(self, log_files: List[str]) -> List[dict]:
        """
        Summarize all given log files.

        Runs through the log files via the log_to_dict function
        Creates average runtimes and average use of resources
        for normal terminated HTCondor Jobs.
        Other types of execution will be ignored

        :return:
        """
        logging.info('Starting the summarize mode')

        # no given files
        if not log_files:
            raise_value_error("No files to summarize")

        # allocated all different datatypes, easier to handle
        result_dict = dict()

        aborted_files = 0
        still_running = 0
        error_reading_files = 0
        other_exception = 0
        normal_runtime = timedelta()
        host_nodes = dict()

        job_resource_list = list()

        for file in track(log_files, transient=True,
                          description="Summarizing..."):
            try:
                (job_dict, job_resources,
                 time_dict, _, _) = self.log_to_dict(file)

                # continue if Process is still running
                if job_dict['Execution details'][0].__eq__("Process is"):
                    still_running += 1
                    continue
                elif job_dict['Execution details'][0].__eq__("Process was"):
                    aborted_files += 1
                    continue
                elif job_dict['Execution details'][0].__eq__("Error"):
                    error_reading_files += 1
                    continue
                elif not job_resources:
                    logging.error("Expected but not found resources")
                    rprint(f"[orange3]Expected but not found "
                           f"resources in {file}[/orange3]")
                    other_exception += 1
                    continue
                elif not job_dict:
                    logging.error("More work is needed developer")
                    rprint(f"[orange3]Process of {file} is strange,\n"
                           f"don't know how to handle this yet[/orange3]")
                    other_exception += 1
                    continue

                if "Total runtime" in time_dict["Dates and times"]:
                    normal_runtime += time_dict['Values'][3]
                host = job_dict['Values'][2]
                if host in host_nodes:
                    host_nodes[host][0] += 1
                    host_nodes[host][1] += time_dict['Values'][3]
                else:
                    host_nodes[host] = [1, time_dict['Values'][3]]

                job_resource_list.append(job_resources)

            # Error ocurres when Job was aborted
            except ValueError or KeyError as err:
                logging.exception(err)
                rprint(f"[red]Error with summarizing: {file}[/red]")
                continue
            except TypeError as err:
                logging.exception(err)
                rprint(f"[red]{err.__class__.__name__}: {err}[/red]")
                sys.exit(3)

        # number of jobs with associated resources,
        # only jobs with normal or abnormal termination state
        n_associated_res = (len(log_files) - aborted_files - still_running -
                            other_exception - error_reading_files)

        average_runtime = normal_runtime / n_associated_res \
            if n_associated_res != 0 else normal_runtime
        average_runtime = timedelta(days=average_runtime.days,
                                    seconds=average_runtime.seconds)

        exec_dict = {
            "Job types": ["Successfully terminated jobs"],
            "Occurrence": [n_associated_res]
        }
        if aborted_files > 0:
            exec_dict["Job types"].append("Aborted jobs")
            exec_dict["Occurrence"].append(aborted_files)
        if still_running > 0:
            exec_dict["Job types"].append("Still running jobs")
            exec_dict["Occurrence"].append(still_running)
        if error_reading_files > 0:
            exec_dict["Job types"].append("Error while reading")
            exec_dict["Occurrence"].append(error_reading_files)
        if other_exception > 0:
            exec_dict["Job types"].append("Other exceptions")
            exec_dict["Occurrence"].append(other_exception)

        result_dict["execution-details"] = \
            sort_dict_by_col(exec_dict, "Occurrence")

        result_dict["description"] = "The following data only implies " \
                                     "on sucessful executed jobs"

        # do not even try further if the only files
        # given have been aborted, are still running etc.
        if n_associated_res == 0:
            return [result_dict]

        create_desc = "The following data implies" \
                      " only on jobs with associated resources"
        if aborted_files > 0 or still_running > 0 \
                or other_exception > 0 or error_reading_files:
            create_desc += "\n[light_grey]" \
                           "Use the analyzed-summary mode " \
                           "for more details about the other jobs" \
                           "[/light_grey]"

        result_dict["summation-description"] = create_desc

        time_desc_list = list()
        time_value_list = list()
        if normal_runtime != timedelta(0, 0, 0):
            time_desc_list.append("Total runtime")
            time_value_list.append(normal_runtime)
        if average_runtime:
            time_desc_list.append("Average runtime")
            time_value_list.append(average_runtime)

        result_dict["times"] = {
            "Times": time_desc_list,
            "Values": time_value_list
        }

        if n_associated_res != 0:  # do nothing, if all valid jobs were aborted

            avg_res_list = create_avg_on_resources(job_resource_list)
            result_dict["all-resources"] = avg_res_list

        if host_nodes:

            executed_jobs = list()
            runtime_per_node = list()
            for val in host_nodes.values():
                executed_jobs.append(val[0])
                average_job_duration = val[1] / val[0]  # timedelta object
                runtime_per_node.append(
                    timedelta(average_job_duration.days,
                              average_job_duration.seconds)
                )

            cpu_dict = {
                "Host Nodes": list(host_nodes.keys()),
                "Executed Jobs": executed_jobs,
                "Average job duration": runtime_per_node
            }

            result_dict["host-nodes"] = sort_dict_by_col(cpu_dict,
                                                         "Executed Jobs")

        return [result_dict]

    def analyzed_summary(self, log_files: List[str]) -> List[dict]:
        """
        Summarize log files and analyze the results.

        This is meant to give the ultimate output
        about every single job state in average etc.

        Creates information based on the job state.
        The common states are:
        - Normal termination
        - Abnormal termination
        - Waiting for execution
        - Currently running
        - Aborted
        - Aborted before submission
        - Aborted before execution
        - error while reading

        :return: list of log information based on the job state
        """
        logging.info('Starting the analyzed summary mode')

        # no given files
        if not log_files:
            raise_value_error("No files for the analyzed-summary")

        # fill this dict with information by the execution type of the jobs
        all_files = dict()
        occurrence_dict = dict()

        for file in track(log_files, transient=True,
                          description="Summarizing..."):

            (job_dict, job_resources, time_dict,
             ram_history, occurred_errors) = self.log_to_dict(file)

            if occurred_errors:
                n_event_err = len(occurred_errors["Event Number"])
                occurred_errors['File'] = [file] * n_event_err

            refactor_job_dict = dict(
                zip(job_dict["Execution details"], job_dict["Values"]))
            job_keys = list(refactor_job_dict.keys())
            to_host = None
            if "Executing on" in job_keys:
                to_host = refactor_job_dict["Executing on"]

            cur_state = job_dict["Values"][0]  # current state

            # if time dict exists
            time_keys = list()
            waiting_time = timedelta()
            runtime = timedelta()
            total_time = timedelta()
            if time_dict:
                refactor_time_dict = dict(
                    zip(time_dict["Dates and times"], time_dict["Values"]))
                time_keys = list(refactor_time_dict.keys())
            if "Waiting time" in time_keys:
                waiting_time = refactor_time_dict["Waiting time"]
            if "Execution runtime" in time_keys:
                runtime = refactor_time_dict["Execution runtime"]
            if "Total runtime" in time_keys:
                total_time = refactor_time_dict["Total runtime"]

            try:
                if cur_state in all_files:
                    # logging.debug(all_files[termination_type])
                    all_files[cur_state][0] += 1  # count number
                    all_files[cur_state][1] += waiting_time
                    all_files[cur_state][2] += runtime
                    all_files[cur_state][3] += total_time

                    # add errors
                    if occurred_errors:
                        for key in occurred_errors.keys():
                            # extend if already existent
                            if key in all_files[cur_state][6].keys():
                                all_files[cur_state][6][key].extend(
                                    occurred_errors[key])
                            else:
                                all_files[cur_state][6] = occurred_errors

                    # resources not empty
                    if all_files[cur_state][4] and job_resources:
                        all_files[cur_state][4].append(job_resources)
                    elif all_files[cur_state][4]:
                        rprint(f"[yellow]{cur_state}: "
                               f"has no resources[/yellow]")

                    # add cpu
                    if to_host is not None:
                        # cpu known ???
                        if to_host in all_files[cur_state][5].keys():
                            all_files[cur_state][5][to_host][0] += 1
                            all_files[cur_state][5][to_host][
                                1] += total_time
                        else:
                            all_files[cur_state][5][to_host] = [1, total_time]
                    elif "Submitted from" in job_dict["Execution details"]:
                        # other waiting jobs ???
                        if 'Waiting for execution' in \
                                all_files[cur_state][5].keys():
                            all_files[cur_state][5][
                                'Waiting for execution'][0] += 1
                            all_files[cur_state][5][
                                'Waiting for execution'][1] += total_time
                        elif "Aborted before execution" in \
                                all_files[cur_state][5].keys():
                            all_files[cur_state][5][
                                'Aborted before execution'][0] += 1
                            all_files[cur_state][5][
                                'Aborted before execution'][1] += total_time
                        else:
                            n_host_nodes = dict()
                            if "Aborted" in cur_state:
                                n_host_nodes['Aborted before'
                                             ' execution'] = [1, total_time]
                            else:
                                n_host_nodes['Waiting for'
                                             ' execution'] = [1, total_time]
                            all_files[cur_state][5] = n_host_nodes
                    else:
                        # other aborted before submission jobs ???
                        if 'Aborted before submission' in \
                                all_files[cur_state][5].keys():
                            all_files[cur_state][5][
                                'Aborted before submission'][0] += 1
                            all_files[cur_state][5][
                                'Aborted before submission'][1] += total_time
                        else:
                            n_host_nodes = dict()
                            n_host_nodes['Aborted before '
                                         'submission'] = [1, total_time]
                            all_files[cur_state][5] = n_host_nodes

                # else new entry
                else:
                    # if host exists
                    if "Executing on" in job_dict["Execution details"]:
                        # to_host = job_dict["Values"][2]
                        n_host_nodes = dict()
                        n_host_nodes[to_host] = [1, total_time]
                    # else if still waiting
                    elif "Submitted from" in job_dict["Execution details"]:
                        n_host_nodes = dict()
                        if "Aborted" in cur_state:
                            n_host_nodes['Aborted before'
                                         ' execution'] = [1, total_time]
                        else:
                            n_host_nodes['Waiting for'
                                         ' execution'] = [1, total_time]
                    # else aborted before submission ?
                    else:
                        n_host_nodes = dict()
                        n_host_nodes['Aborted before'
                                     ' submission'] = [1, total_time]

                    all_files[cur_state] = [1,
                                            waiting_time,
                                            runtime,
                                            total_time,
                                            [job_resources],
                                            n_host_nodes,
                                            occurred_errors]

            # Error ocurres when Job was aborted
            except ValueError or KeyError as err:
                logging.exception(err)
                logging.debug(f"Error with summarizing: {file}")
                rprint(f"[red]Error with summarizing: {file}[/red]")
                continue
            except TypeError as err:
                logging.exception(err)
                logging.debug(f"[red]Error with summarizing: {file}[/red]")
                rprint(f"[red] {err}[/red]")
                sys.exit(3)

        # Now put everything together
        result_list = list()
        for cur_state in all_files:
            term_info = all_files[cur_state]
            result_dict = dict()

            # differentiate between terminated and running processes
            if "Error while reading" in cur_state:
                result_dict["description"] = "" \
                     "##################################################\n" \
                     "## All files, that caused an [red]" \
                     "error while reading[/red]\n" \
                     "##################################################"
            elif cur_state not in ["Waiting", "Executing"]:
                result_dict["description"] = f"" \
                    f"##################################################\n" \
                    f"## All files with the termination state: {cur_state}\n"\
                    f"##################################################"
            else:
                result_dict["description"] = f"" \
                    f"###########################################\n" \
                    f"## All files, that are currently {cur_state}\n" \
                    f"###########################################"

            n = int(term_info[0])
            occurrence_dict[cur_state] = str(n)

            times = np.array([term_info[1], term_info[2], term_info[3]])
            av_times = times / n
            format_av_times = [
                timedelta(days=time.days, seconds=time.seconds)
                for time in av_times]

            time_dict = {
                "Times": ["Waiting time", "Runtime", "Total"],
                "Average": format_av_times,
                "Total": times
            }

            result_dict["times"] = time_dict

            if term_info[4]:
                avg_resources = create_avg_on_resources(term_info[4])
                result_dict["all-resources"] = avg_resources

            executed_jobs = list()
            runtime_per_node = list()
            for val in term_info[5].values():
                executed_jobs.append(val[0])
                average_job_duration = val[1] / val[0]
                runtime_per_node.append(
                    timedelta(average_job_duration.days,
                              average_job_duration.seconds))

            host_nodes_dict = {
                "Host Nodes": list(term_info[5].keys()),
                "Executed Jobs": executed_jobs,
                "Average job duration": runtime_per_node
            }

            result_dict["host-nodes"] = \
                sort_dict_by_col(host_nodes_dict, "Executed Jobs")

            if term_info[6]:
                temp_err = term_info[6]
                del temp_err["Reason"]  # remove reason
                result_dict["errors"] = temp_err

            result_list.append(result_dict)

        new_occ = {
            "Termination type": list(occurrence_dict.keys()),
            "Appearance": list(occurrence_dict.values())
        }
        sorted_occ = sort_dict_by_col(new_occ, "Appearance")

        result_list.insert(0, {"execution-details": sorted_occ})

        return result_list

    def filter_for(self,
                   log_files: List[str],
                   keywords: list,
                   extend=False,
                   mode=None
                   ) -> List[dict]:
        """
        Filter for given keywords.

        The keywords can be extended by:
        "err", "warn", "exception", "aborted", "abortion", "abnormal", "fatal"

        The filtering is NOT case sensitive.

        The filtered files can be analyzed, summarize, etc afterwards,
        else this function will return the files

        :param log_files:
        :param keywords:
        :param extend:
        :param mode:
        :return:
            list with dicts depending on the used mode,
            list of files if mode is None
        """
        logging.info('Starting the filter mode')

        # if the keywords are given as a string, try to create a list
        if isinstance(keywords, list):
            keyword_list = keywords
        else:
            logging.debug(
                f"Filter mode only accepts a string "
                f"or list with keywords, not {keywords}")
            raise TypeError("Expecting a list or a string")

        # if extend is set, keywords like err will
        # also look for keywords like warn exception, aborted, etc.
        if extend:  # apply some rules
            # the error list
            err_list = ["err", "warn", "exception", "aborted", "abortion",
                        "abnormal", "fatal"]

            # remove keyword if already in err_list
            for keyword in keyword_list:
                if keyword.lower() in err_list:
                    keyword_list.remove(keyword)

            keyword_list.extend(err_list)  # extend search

            rprint("[green]Keyword List was extended, "
                   "now search for these keywords:[/green]",
                   keyword_list)
        else:
            rprint("[green]Search for these keywords:[/green]", keyword_list)

        if len(keyword_list) == 1 and keyword_list[0] == "":
            logging.debug("Empty filter, don't know what to do")
            return "[yellow]" \
                   "Don't know what to do with an empty filter,\n" \
                   "if you activate the filter mode in the config file, \n" \
                   "please add a [filter] section with the filter" \
                   "_keywords = your_filter[/yellow]"

        logging.debug(f"These are the keywords to look for: {keyword_list}")

        # now search
        found_at_least_one = False
        found_logs = []
        for file in track(log_files, transient=True,
                          description="Filtering..."):
            found = False
            with open(file, "r") as read_file:
                for line in read_file:
                    for keyword in keyword_list:
                        if re.search(keyword.lower(), line.lower()):
                            if not found_at_least_one:
                                print("Matches:")
                            rprint(f"[grey74]{keyword} in:\t{file}[/grey74] ")
                            found = True
                            found_at_least_one = True
                            break
                    if found:
                        found_logs.append(file)
                        break

        return_dicts = None
        if not found_at_least_one:
            rprint("[red]Unable to find these keywords:[/red]", keyword_list)
            rprint("[red]maybe try again with similar expressions[/red]")

        elif mode is not None:
            print(f"Total count: {len(found_logs)}")
            if mode.__eq__("analyzed-summary"):
                rprint("[magenta]Give an analyzed summary"
                       " for these files[/magenta]")
                return_dicts = self.analyzed_summary(found_logs)
            elif mode.__eq__("summarize"):
                rprint("[magenta]Summarize these files[/magenta]")
                return_dicts = self.summarize(found_logs)
            elif mode.__eq__("analyze"):
                rprint("[magenta]Analyze these files[/magenta]")
                return_dicts = self.analyze(found_logs)

        return return_dicts

def raise_value_error(message: str) -> ValueError:
    """
    Raise Value Error with message.

    :param message: str
    :return:
    """
    raise ValueError(message)

def raise_type_error(message: str) -> TypeError:
    """
    Raise Type Error with message.

    :param message:
    :return:
    """
    raise TypeError(message)

def _int_formatter(val, chars, delta, left=False):
    """
    Format float to int.

    Usage of this is shown here:
    https://github.com/tammoippen/plotille/issues/11
    """
    align = '<' if left else ''
    return '{:{}{}d}'.format(int(val), align, chars)

def gen_time_dict(submission_date: date_time = None,
                  execution_date: date_time = None,
                  termination_date: date_time = None
                  ) -> (timedelta, timedelta, timedelta):
    """
    Generate a time dict.

    Takes in three dates, return the timedelta objects.
    Depending on the given arguments,
    this function will try to calculate
    the time differences between the events.
    If not all three values are given, it will return None

    :param submission_date: Job Sumbission date from the user
    :param execution_date: The date when the job actually started executing
    :param termination_date: The date when the job was finished
    :return: (waiting_time, runtime, total_time)

    """
    if not any(locals().values()):  # all params are None
        return None

    waiting_time = None
    runtime = None
    total_time = None
    today = date_time.now()
    today = today.replace(microsecond=0)  # remove unnecessary microseconds

    time_desc = list()
    time_vals = list()
    running_over_newyear = False

    # calculate the time difference to last year,
    # if the date is higher that today of running jobs
    # this means the execution started before newyear
    if termination_date is None:
        if submission_date and submission_date > today:
            running_over_newyear = True
            submission_date = submission_date.replace(
                year=submission_date.year - 1)
        if execution_date and execution_date > today:
            running_over_newyear = True
            execution_date = execution_date.replace(
                year=execution_date.year - 1)

    if execution_date and submission_date:
        execution_date = execution_date
        # new year ?
        if submission_date > execution_date:
            running_over_newyear = True
            submission_date = submission_date.replace(
                year=submission_date.year - 1)
        waiting_time = execution_date - submission_date

    if termination_date:
        if waiting_time:
            pass
        if execution_date:
            # new year ?
            if execution_date > termination_date:
                running_over_newyear = True
                execution_date = execution_date.replace(
                    year=execution_date.year - 1)
            runtime = termination_date - execution_date
        if submission_date:
            # new year ?
            if submission_date > termination_date:
                running_over_newyear = True
                submission_date = submission_date.replace(
                    year=submission_date.year - 1)
            total_time = termination_date - submission_date
    # Process still running
    elif waiting_time:
        runtime = today - execution_date
    # Still waiting for execution
    elif submission_date:
        waiting_time = today - submission_date

    # now after collecting all available values try to produce a dict
    # if new year was hitted by one of them, show the year as well
    if running_over_newyear:
        if submission_date:
            time_desc.append("Submission date")
            time_vals.append(submission_date)
        if execution_date:
            time_desc.append("Execution date")
            time_vals.append(execution_date)
        if termination_date:
            time_desc.append("Termination date")
            time_vals.append(termination_date)
    else:
        if submission_date:
            time_desc.append("Submission date")
            time_vals.append(submission_date.strftime("%m/%d %H:%M:%S"))
        if execution_date:
            time_desc.append("Execution date")
            time_vals.append(execution_date.strftime("%m/%d %H:%M:%S"))
        if termination_date:
            time_desc.append("Termination date")
            time_vals.append(termination_date.strftime("%m/%d %H:%M:%S"))

    if waiting_time:
        time_desc.append("Waiting time")
        time_vals.append(waiting_time)
    if runtime:
        time_desc.append("Execution runtime")
        time_vals.append(runtime)
    if total_time:
        time_desc.append("Total runtime")
        time_vals.append(total_time)

    time_dict = {
        "Dates and times": time_desc,
        "Values": time_vals
    }

    return time_dict

def sort_dict_by_col(dictionary, column, reverse=True):
    """
    Sort a dictionary by the given column.

    The dictionary must have key: list items
    where all lists have the same length

    The order is reversed
    [1,4,2,3,5] -> [5,4,3,2,1]

    :param dictionary:
    :param column:
    :param reversed:
    :return:
    """
    # sorted_dict = dict.fromkeys(dictionary.keys())
    sorted_dict = {key: [] for key in dictionary.keys()}
    sort_index = list(sorted_dict.keys()).index(column)
    zip_data = zip(*dictionary.values())
    sorted_items = sorted(zip_data, key=lambda tup: tup[sort_index])
    if reverse:
        sorted_items = reversed(sorted_items)
    for item in sorted_items:
        for i, key in enumerate(sorted_dict.keys()):
            sorted_dict[key].append(item[i])

    return sorted_dict


#!/usr/bin/env python
# coding: utf-8

# In[27]:

import numpy as np
from frozen_lake import FrozenLakeEnv
import random

env = FrozenLakeEnv()

def epsilon_greedy_action(env,Q,state,epsilon=0.3):
    n = random.uniform(0,1)
    if n<= epsilon:
        return np.random.randint(env.action_space.n)
    else:
        return np.argmax(Q[state])

def Q_Learning(env, episodes=1000, gamma=0.91, alpha=0.1):
    Q = np.zeros([env.nS,env.nA])

    for i in range(episodes):
        finished = False

        env.reset()

        S = env.s

        while not finished:

            A = epsilon_greedy_action(env,Q,S)

            next_S, R, finished, _ = env.step(A)

            #A = epsilon_greedy_action(env,Q,next_S)

            #Q[S][A] = Q[S][A] + alpha*(R + gamma * Q[next_S][next_A] - Q[S][A])
            Q[S][A] = Q[S][A] + alpha * ( R + gamma* np.max(Q[next_S][A])  - Q[S][A]  )

            S = next_S

    return Q

# In[28]:

#zaokrąglamy do dwóch miejsc po przecinku
Q = Q_Learning(env,10000)
print(Q)

env.close()

# In[ ]:


import socket
import sys
import base64
import os

# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

# Connect the socket to the port where the server is listening
server_address = ('127.0.0.1', 8889)
print(sys.stderr, 'connecting to %s port %s' % server_address)

sock.connect(server_address)

requestfile = "2.png"
request = "download "+requestfile
sock.send(request.encode('utf-8'))
sock.shutdown(socket.SHUT_WR)
#print(request)

data = sock.recv(1024)
if data == b'File not exist':
    print("File not exist")
else:
    f = open("Client/"+requestfile,"wb")
    while (data):
        print(data)
        f.write(data)
        data = sock.recv(1024)
        #f = open("Server/2.png", "rb")
        #print(f.read())
        # print(base64.decode(data))
    print("file " + requestfile + " has been downloaded")
    f.close()

print("Close . . .")
sock.close()


#  Task 1 - Assignment 2

for x in range(2000,3200):
    if x%7 == 0 and x%5!= 0:
        print(x,end=',')

print("end")

#  Task 1 - Assignment 3

first_name =input("Please enter first name ")
last_name = input("Please enter Last Name ")
print(last_name +' ' + first_name)

 # Task 1 - Assignment 4

Formula=(4/3)*(22/7)
r=6
Volume = Formula * r*r*r

print(Volume)

#  Task 2 - Assignment 1

seq_input = input("Please provide a sequence of numbers seperated by comma")
str_split = seq_input.split(",")

list1 = []
for i in range(0,len(str_split)):
    list1.append(str_split[i])
print(list1)

#  Task 2 - Assignment 3
s = 'Y'
while s == 'Y' or s == 'y':
    s = input("You want to reverse word ")
    if s == 'Y' or s == 'y':
        str = input("please enter a word ")
        rev = ""
        for i in range(len(str) -1, -1, -1):
            rev = rev + str[i]
    else:
        break
    print(rev)

#  Task 2 - Assignment 2

for i in range(0, 11):
    if (i <= 5):
        for j in range(0, i):
            print('*', end =' ')
    else:

        for k in range(i,10):
            print('*', end=' ')

    print("\n")

# Task 2 Assignment No 4

Str = "WE,THE PEOPLE OF INDIA,\n having solemnly resolved to constitute India into a " \
"SOVEREIGN, !\n SOCIALIST, SECULAR, DEMOCRATIC REPUBLIC \n and to secure to all" \
" its citizens"

print(Str)


from flask import Blueprint, render_template, url_for, request
import movie_web_app.utilities.utilities as utilities
from flask_paginate import Pagination
from flask_paginate import Pagination, get_page_parameter
from movie_web_app.domainmodel.read_title import read_title

import movie_web_app.adapters.repository as repo

genre_blueprint = Blueprint(
    'genre_bp', __name__
)

@genre_blueprint.route('/genres')
def list_genre():
    return render_template(
        'others/list_genre.html',
        home_url=url_for('home_bp.home'),
        list_movie_url=url_for('movie_bp.movies_by_year'),
        list_actor_url=url_for('actor_bp.list_actor'),
        list_director_url=url_for('director_bp.list_director'),
        list_genre_url=url_for('genre_bp.list_genre'),
        search_url=url_for('search_bp.find_movie'),
        register_url=url_for('authentication_bp.register'),
        genres=repo.repo_instance.get_genres(),
        selected_movies=utilities.get_selected_movies(),
        genre_urls=utilities.get_genres_and_urls(),
    )


# -*- coding:utf-8 -*-
"""RomaFileOps class."""
import os
import fcntl
import logging
import moxing
from vega.core import FileOps

class RomaFileOps(object):
    """This is the class which use the function of roma."""

    @classmethod
    def copy_folder(cls, src, dst):
        """Copy a local dir to s3 bucket.

        :param str src: path to local directory.
        :param str dst: path to s3 directory. eg. `s3://bucket/upload/`

        """
        if dst is None or dst == "":
            return
        if moxing.file.exists(src):
            moxing.file.copy_parallel(src, dst)
        else:
            logging.warn("remote path {} doesn't exist".format(src))

    @classmethod
    def copy_file(cls, src, dst):
        """Copy a local file to s3 bucket.

        :param str src: path to local file.
        :param str dst: path to s3 file. eg. `s3://bucket/upload/a.caffemodel`

        """
        if dst is None or dst == "":
            return
        if moxing.file.exists(src):
            moxing.file.copy(src, dst)
        else:
            logging.warn("remote file {} doesn't exist".format(src))

    @classmethod
    def download_dataset(cls, src_path):
        """If the data path is s3 path, then it will copy the data to /cache.

        and the path will be changed to cache path.
        :param src_path: the data path
        :type src_path: str
        :raises FileNotFoundError: if the file path is not exist, an error will raise
        :return: the final data path
        :rtype: str
        """
        if src_path.split(':')[0] == 's3':
            if src_path[-1] == '/':
                src_path = src_path[0:-1]
            cache_path = '/cache/' + src_path[5:]
            # set a signal file
            src_name = src_path[5:].replace('/', '_')
            signal_file = os.path.join('/cache', 'DATASET_COPY_OK_{}'.format(src_name))
            if not os.path.isfile(signal_file):
                with open(signal_file, 'w') as fp:
                    fp.write('{}'.format(0))
            with open(signal_file, 'r+') as fp:
                # use file lock to make sure only download dataset once on same node
                fcntl.flock(fp, fcntl.LOCK_EX)
                signal = int(fp.readline().strip())
                if signal == 0:
                    if moxing.file.exists(src_path):
                        logging.info("Copy the data to cache.")
                        moxing.file.copy_parallel(src_path, cache_path)
                    else:
                        raise FileNotFoundError(src_path + 'is not existed')
                    with open(signal_file, 'w') as fn:
                        fn.write('{}'.format(1))
                else:
                    logging.info("The data has already existed in the cache.")
                fcntl.flock(fp, fcntl.LOCK_UN)
            final_path = cache_path
        else:
            if os.path.exists(src_path):
                final_path = src_path
            else:
                raise FileNotFoundError(src_path + 'is not existed')
        return final_path

    @classmethod
    def exists(cls, path):
        """Is folder existed or not.

        :param folder: folder
        :type folder: str
        :return: folder existed or not.
        :rtype: bool
        """
        return moxing.file.exists(path)

def replace_file_ops():
    """Replace method of Fileops with RomaFileOps."""
    FileOps.copy_folder = RomaFileOps.copy_folder
    FileOps.copy_file = RomaFileOps.copy_file
    FileOps.download_dataset = RomaFileOps.download_dataset
    FileOps.exists = RomaFileOps.exists


from glob import glob
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import pandas as pd
from collections import defaultdict
import re
import os

def count_text(category_path):
    """
    统计总体语料的分布情况
    :param category_path: 语料路径
    :return: 不同种类的语料字典
    """
    if os.path.exists(category_path):
        # 语料的路径
        category_path = category_path + '/*'  # 匹配所有的目录
        total = {}  # 语料总数
        for dir in glob(category_path):
            total[dir] = len(glob(dir + '/*.txt'))  # 每个类别下文件的数量
        print(total)
        print("文件共{}类,总的文件的数量:{}".format(len(total), sum(total.values())))
    else:
        raise FileExistsError('{}文件的路径不存在'.format(category_path))
    return total

def cut_corpus(path):
    """
    切分语料集 训练集 验证集 测试集，比例0.7:0.15:0.15
    :param path: 语料路径list
    :return: 切分后的数据集和标签
    """
    label = re.findall(r"[\u4e00-\u9fa5]+", path)  # 匹配汉字
    files = glob(path + '/*.txt')  # 匹配txt文件的绝对路径
    # 切分数据集
    train, test = train_test_split(files, test_size=0.3, shuffle=True, random_state=2020)
    valid, test = train_test_split(test, test_size=0.5, shuffle=True, random_state=2021)
    print("train:{} test:{} valid:{}".format(len(train), len(test), len(valid)))
    return train, test, valid, label

def read_data(path, label=None, debug=False, frac=1):
    """
    读取文件中的数据title content
    :param path: 每条语料的路径信息list
    :param debug: 采样模式
    :param frac: 采样的比例
    :return:
    """
    titles = []
    contents = []
    for file in tqdm(path):
        with open(file, 'r', encoding='utf-8') as obj:
            data = obj.readlines()
        title = data[0].strip()
        content = [sen.strip() for sen in data[1:]]
        titles.append(title)
        contents.append(''.join(content))

    title_content = defaultdict(list)

    if len(titles) == len(contents):
        title_content['title'] = titles
        title_content['content'] = contents
        title_content['label'] = [label] * len(titles)
    else:
        raise ValueError('数据titles和contents数量不一致')
    df = pd.DataFrame(title_content, columns=['title', 'content', 'label'])
    if debug:
        # 采样
        df = df.sample(frac=frac, random_state=2020).reset_index(drop=True)
        print('采样的样本数量{}'.format(df.shape[0]))
    return df

def writ_to_csv(dictionary, filename='train'):
    """
    将数据写入csv文件
    :param dictionary: 字典格式
    :return:
    """
    df = pd.DataFrame(dictionary, columns=['title', 'content', 'label'])
    df.to_csv('{}.csv'.format(filename), sep='\t', index=False)
    print()
    print('writing succesfully')

def process(path, filename='train', frac=1):
    """
    读取数据文件将数据写入csv文件 title content label
    :param path: 数据文件的路径dict
    :param filename: 保存文件命名
    :return: None
    """
    print('loading {}'.format(filename))
    sample = []
    for label, data in path.items():
        under_sample = read_data(data, label, debug=True, frac=frac)
        sample.append(under_sample)
    df = pd.concat(sample, axis=0)
    print("{}文件的数据量为:{}".format(filename, df.shape[0]))
    # 保存文件的路径
    base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    save_path = base_path + '/data/' + filename + '.csv'
    df.to_csv(save_path, sep='\t', index=False)
    print('{} writing succesfully'.format(save_path))

if __name__ == '__main__':
    root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    category_path = root_path + '/data/THUCNews/'
    # 语料的路径
    dir_dict = count_text(category_path)
    train_path = defaultdict(list)
    test_path = defaultdict(list)
    valid_path = defaultdict(list)
    for path in dir_dict.keys():
        # 切分数据集
        train, test, valid, label = cut_corpus(path)
        # 保存数据到字典
        train_path[label[0]] = train
        test_path[label[0]] = test
        valid_path[label[0]] = valid

    process(train_path, filename='train', frac=0.6)
    process(test_path, filename='test', frac=0.5)
    process(valid_path, filename='valid', frac=0.5)