# -*- coding: utf-8 -*-

# first neural network with keras tutorial
from numpy import loadtxt
from keras.models import Sequential
from keras.layers import Dense
from keras.models import model_from_json

# load the dataset
dataset = loadtxt('pima-indians-diabetes.csv', delimiter=',')

# split into input (X) and output (y) variables
X = dataset[:,0:8]
y = dataset[:,8]

# define the keras model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

# compile the keras model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# fit the keras model on the dataset
model.fit(X, y, epochs=150, batch_size=10)

# evaluate the keras model
_, accuracy = model.evaluate(X, y)
print('Accuracy: %.2f' % (accuracy*100))

# make class predictions with the model
predictions = model.predict_classes(X)
# summarize the first 5 cases
for i in range(5):
	print('%s => %d (expected %d)' % (X[i].tolist(), predictions[i], y[i]))

# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")

# load json and create model
with open('model.json', 'r') as json_file:
    loaded_model_json = json_file.read()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("model.h5")
print("Loaded model from disk")

# evaluate loaded model on test data
loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
score = loaded_model.evaluate(X, y, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))


#!/usr/bin/env python

import os, os.path
import argparse

parser = argparse.ArgumentParser(description="Mr. Shrinker")

parser.add_argument("percent",
                    help="Please specify how bad you'd like to shrink your file")

parser.add_argument("old_name", help="Please specify the old address to your file")

parser.add_argument("new_name", nargs='?', help="Please specify the new address to your file")

args = parser.parse_args()

if not os.path.exists(args.old_name):
    print('There is no file to shrink')

if args.percent:
    import subprocess
    number = args.percent
    old = args.old_name
    new = args.new_name
    if os.path.isfile(old) == True:
        if new != None:
            params = ['convert', old, '-resize', args.percent, new]
            params = " ".join(params)
            subprocess.call(params, shell=True)
        else:
            params = ['convert', old, '-resize', args.percent, old]
            params = " ".join(params)
            subprocess.call(params, shell=True)
    if os.path.isdir(old) == True:
        new = None
        for (dirpath, dirnames, filenames) in os.walk(old):
            for filename in filenames:
                if filename.endswith('.jpg') or filename.endswith('.png'):
                    right_file = old + '/' + filename
                    params = ['convert', right_file, '-resize', args.percent, right_file]
                    params = " ".join(params)
                    subprocess.call(params, shell=True)


from ftw.solr.interfaces import ISolrConnectionManager
from ftw.solr.interfaces import ISolrSettings
from ftw.upgrade import UpgradeStep
from plone.registry.interfaces import IRegistry
from zope.component import getUtility
import logging

logger = logging.getLogger('opengever.core')

class FixDocsOnlyPartiallyIndexedInSolr(UpgradeStep):
    """Fix docs only partially indexed in solr.

    We think affected documents may have been not indexed in solr at all due to
    a bug with date serialization in `ftw.solr`. The bug was only fixed in
    https://github.com/4teamwork/ftw.solr/pull/158.

    Subsequent edits of the document by users via plone or reindex operations
    during upgrades would have partially indexed or restored the document in
    solr, but only some of the fields as we use atomic updates.

    We've seen that all affected documents do not have the created date set as
    we never manually reindex it, as it never changes. Thus apply a heuristic
    to detect broken solr documents. We assume that broken documents do not
    have the field 'created' set to a value in solr.
    """

    deferrable = True

    @property
    def is_solr_enabled(self):
        registry = getUtility(IRegistry)
        settings = registry.forInterface(ISolrSettings)
        return settings.enabled

    def __call__(self):
        if not self.is_solr_enabled:
            return

        self.solr_connection = getUtility(ISolrConnectionManager).connection
        broken_docs = self.find_broken_solr_docs()
        for doc in broken_docs:
            self.reindex_broken_doc(doc)

    def find_broken_solr_docs(self):
        """This method purposefully bypasses ISolrSearch.

        We have to build our own query as ISolrSearch would insert security
        filters for 'allowedRolesAndUsers'. But there are broken solr docs
        where the 'allowedRolesAndUsers' is missing.
        """
        params = {
            u'offset': 0,
            # this queries for all docs where the `created` field is not set
            u'query': u'-created:[* TO *]',
            # choose a very high number well beyond the number of docs we can
            # possibly have, but also well below java max 4 byte int 2147483647
            u'limit': 99999999,
            # only return the UID field
            u'params': {
                u'fl': u'UID'
            }
        }

        solr_response = self.solr_connection.search(
            params, request_handler=u'/select')
        return solr_response.docs

    def reindex_broken_doc(self, doc):
        """Reindex all fields/indexes for a broken solr document.
        """
        plone_uid = doc[u'UID']
        # Do not duplicate this upgrade step as is. Querying the catalog
        # for each object will trigger flushing of the indexing queue,
        # i.e. the indexing queue gets processed for every object...
        brains = self.catalog_unrestricted_search({u'UID': plone_uid})
        if len(brains) != 1:
            logger.error(
                u'Could not find a unique brain for the UID={}'.format(
                    plone_uid)
            )
            return

        brain = brains[0]
        obj = self.catalog_unrestricted_get_object(brain)
        # `catalog_unrestricted_get_object` takes care of logging already
        if obj:
            # we reindex the whole object as we cannot know which fields are
            # missing, also do catalog reindex for convenience
            obj.reindexObject()


from django.db import models
from legend.models import Parameter, Attribute
import string
# Create your models here.
class OptionForAPI(models.Model):

    parameter = models.ForeignKey(
        Parameter,
        on_delete=models.DO_NOTHING,
        verbose_name="Параметр запроса",
        )

    value = models.CharField(
        verbose_name="Значение параметра",
        help_text="Цифры,знаки",
        max_length=20,
        blank=False,
    )

    def generate_string(self):
        return f'{self.parameter.string}={self.value}'

    class Meta:
        verbose_name = "Пара параметр-значение"
        verbose_name_plural = "Пары параметр-значение"

    def __str__(self):
        return f'{self.parameter.group}| {self.parameter} = {self.value}'

class RequestForApi(models.Model):

    def __init__(self,*args,**kwargs):
        super().__init__(*args,**kwargs)
        #подчистка строки адреса от пробелов при инициализации
        self.addres=''.join(filter(lambda x: x in string.printable and x!=' ', self.addres))

    addres = models.CharField(
        verbose_name="Адрес для API запросов",
        blank=False,
        max_length=150,
    )

    parameters = models.ManyToManyField(
        to = OptionForAPI,
        verbose_name="Параметры",
        blank=True,
    )

    description = models.TextField(
        verbose_name="Пояснение",
        blank=True,
    )

    """def save(self,*args,**kwargs):
        addres = ''.join(filter(lambda x: x in string.printable and x!=' ',self.addres))
        super().save(*args,**kwargs)"""

    def generate_string(self):
        par_list = list(i.generate_string() for i in tuple(self.parameters.all()))
        par_string = '&'.join(par_list)
        return ''.join(filter(lambda x: x in string.printable and x!=' ', f'{self.addres}?{par_string}'))

    def __str__(self):
        return f'id {self.id}| {self.description[:60]}'

    class Meta:
        verbose_name = "Запрос для внешней БД"
        verbose_name_plural = "Запросы для внешней БД"

class CompilationR(models.Model):

    initial_request = models.ForeignKey(
        'RequestForApi',
        verbose_name="Главный запрос",
        on_delete=models.DO_NOTHING
        )

    get_page = models.ForeignKey(
        Parameter,
        verbose_name="Запрос страницы",
        on_delete=models.DO_NOTHING,
        related_name='get_page',
        blank=True,
        default='',
        null=True,
        )

    get_code = models.ForeignKey(
        Parameter,
        verbose_name="Запрос кода",
        on_delete=models.DO_NOTHING,
        related_name='get_code',
        blank=True,
        default='',
        null=True,
        )

    get_data = models.ForeignKey(
        Attribute,
        verbose_name="Данные со страницы",
        on_delete=models.DO_NOTHING,
        related_name='get_data',
        blank=True,
        default=None,
        null=True,
        )

    get_page_count = models.ForeignKey(
        Attribute,
        verbose_name="Общее количество страниц в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_page_count',
        blank=True,
        default='',
        null=True,
        )

    get_record_count = models.ForeignKey(
        Attribute,
        verbose_name="Общее количество записей в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_record_count',
        blank=True,
        default='',
        null=True,
        )

    get_current_page_number = models.ForeignKey(
        Attribute,
        verbose_name="Текущий номер страницы в теле ответа",
        on_delete=models.DO_NOTHING,
        related_name='get_current_page_number',
        blank=True,
        null=True,
        )

    class Meta:
        verbose_name = "Сборник запросов для скачивания БД"
        verbose_name_plural = "Сборники запросов для скачивания БД"

    def __str__(self):
        return f'id {self.id}| {self.initial_request.description[:60]}'


import matplotlib.pyplot as plt
import numpy as np
import read_post
import operator

def runningMean(x, N):
    return np.convolve(x, np.ones((N,))/N)[(N-1):][:(1-N)]

def analysis(file_name):
    incr = 1/12  #every five minutes in hour
    midnight1 = 1424246400 #midnight Tuesday Night
    midnight2 = 1424419200 #midnight Thursday Night
    f = open(file_name)
    number_posts = 0
    everything = {}
    subreddit_num_posts = {}
    index = {}
    times = {}
    for i in f:
        post = read_post.read_post(i)
        bt = post['batch_time']
        sub_id = post['subreddit_id']
        sub_name = post['subreddit']
        if bt <= midnight1 or bt >= midnight2:
            continue
        number_posts += 1
        if number_posts % 50000 == 0:
            print(number_posts);
        if sub_id not in times.keys(): times[sub_id] = {}
        if bt not in times[sub_id].keys(): times[sub_id][bt] = 0
        times[sub_id][bt] += 1
        if bt not in everything.keys(): everything[bt] = 0
        everything[bt] += 1

        if sub_id not in index.keys(): index[sub_id] = sub_name
        elif index[sub_id] != sub_name: print("DUPLACTE")
        if sub_id not in subreddit_num_posts.keys(): subreddit_num_posts[sub_id] = 0
        subreddit_num_posts[sub_id] += 1

    #Create Plotable lists
    batchtime = [(i-midnight1)/3600 for i, j in sorted(everything.items())]
    allposts = {}
    for sub_id in times.keys():
        if sub_id not in allposts: allposts[sub_id] = []
        for bt in everything.keys():
            if bt not in times[sub_id].keys():
                allposts[sub_id].append(0)
            else:
                allposts[sub_id].append(times[sub_id][bt])

    #print
    traffic = 0
    for i,j in [(i,j) for i,j in sorted(subreddit_num_posts.items(),key=operator.itemgetter(1), reverse=True)][:100]:
        print(index[i]+' (' + str(j) + ')')
        traffic += j
    print(traffic)

    for i,j in [(i,j) for i,j in sorted(subreddit_num_posts.items(),key=operator.itemgetter(1), reverse=True)][:7]:
        sub_posts = allposts[i]
        wbatchtime = batchtime[1:]
        wposts = [((i-j)/incr)*k for i,j,k in zip(batchtime[1:],batchtime,sub_posts[1:])]

        plt.figure(1)
        plt.plot(wbatchtime[35:-25],runningMean(wposts[10:],51),label=(index[i]+' ('+str(j)+')'))
        #plt.plot(wbatchtime,wposts)
        plt.title("Subreddits")
        plt.xlabel('Hours (5 minute increment)')
        plt.ylabel('Num of Posts')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
    plt.show()
    print(number_posts)
    print()

if __name__ == "__main__":
    analysis("Data/NewCorpus.txt")


#%%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import networkx as nx

#%%
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch_geometric
from torch_geometric.datasets import Planetoid

#%%
Cora = Planetoid(root = "/Users/mineself2016/study/codeSpace/VSCodeWorkSpace/PythonWorkSpace/PyTorch/Pygeometric/datasets", name = "Cora")

#%%
graph = Cora[0]

#%%
graph

#%%
graph.x.shape

#%%
graph

#%%
graph.edge_index

#%%
e = graph.edge_index.numpy().T.tolist()

#%%
class SAGELayer(nn.Module):
    def __init__(self, edge_index, depth, num_neighbors, in_features, out_features, aggr = "mean"):
        super(SAGELayer, self).__init__()

        self.G = nx.Graph()
        self.G.add_edges_from(edge_index)

        self.linear_1 = nn.Linear(2 * in_features, 20)
        self.linear_2 = nn.Linear(20 * 2, out_features)

        self.depth =  depth
        self.num_neighbors = num_neighbors
        self.aggr = aggr

    def forward(self, x):
        h_old = x
        h_new = None

        for d in range(self.depth):
            if d == 0:
                h_new = torch.zeros(2708, 20)
            elif d == 1:
                h_new = torch.zeros(2708, 10)
            else:
                raise Exception("depth out of range")

            for node_v in self.G.nodes:
                neighbors = self.sample(node_v)
                h_neighbor = self.aggregate(neighbors, h_old)

                h_v = torch.cat([h_old[node_v], h_neighbor], dim = 0)

                if d == 0:
                    h_v = self.linear_1(h_v)
                elif d == 1:
                    h_v = self.linear_2(h_v)
                else:
                    raise Exception("depth out of range")

                h_v = F.sigmoid(h_v)
                # print(h_neighbor[h_neighbor != 0])
                h_v = h_v / torch.norm(h_v, p = 2)

                h_new[node_v] = h_v

            h_old = h_new

        return h_new

    def sample(self, node_v):
        neighbors = list(self.G.adj[node_v])

        if len(neighbors) >= self.num_neighbors:
            neighbors = np.random.choice(neighbors, size = self.num_neighbors, replace = False)
        else:
            neighbors = np.random.choice(neighbors, size = self.num_neighbors, replace = True)

        return neighbors

    def aggregate(self, neighbors, x):
        if self.aggr == "mean":
            idx, value = x[neighbors].max(dim = 0)
            return value

        else:
            print("not mean aggregate")
            return None

#%%
class GraphSAGE(nn.Module):
    def __init__(self, edge_index, depth, num_neighbors, in_features, out_features, aggr = "mean"):
        super(GraphSAGE, self).__init__()
        self.Sage_1 = SAGELayer(edge_index, depth = 2, num_neighbors = 4, in_features = 1433, out_features = 10)
        self.linear = nn.Linear(10, 7)

    def forward(self, x):
        x = self.Sage_1(x)
        x = x.relu()
        x = F.dropout(x, p = 0.5, training = self.training)
        x = self.linear(x)

        return x

model = GraphSAGE(e, depth = 2, num_neighbors = 4, in_features = 1433, out_features = 10)
#%%
print(model)

#%%
optimizer = torch.optim.Adam(model.parameters(), lr = 0.2, weight_decay=5e-4)
criterion = torch.nn.CrossEntropyLoss()

#%%
def train():
      model.train()
      optimizer.zero_grad()  # Clear gradients.

      output = model(graph.x)  # Perform a single forward pass.
      loss = criterion(output[graph.train_mask], graph.y[graph.train_mask])  # Compute the loss solely based on the training nodes.
      loss.backward()  # Derive gradients.
      optimizer.step()  # Update parameters based on gradients.
      return loss

def test():
      model.eval()
      output = model(graph.x)
      pred = output.argmax(dim=1)  # Use the class with highest probability.
      test_correct = pred[graph.test_mask] == graph.y[graph.test_mask]  # Check against ground-truth labels.
      test_acc = int(test_correct.sum()) / int(graph.test_mask.sum())  # Derive ratio of correct predictions.
      return test_acc

#%%
for epoch in range(1, 201):
    loss = train()
    print(f'Epoch: {epoch:03d}, Loss: {loss:.4f}')

    if epoch % 20 == 0:
        test_acc = test()
        print(f'Test Accuracy: {test_acc:.4f}')

#%%

#%%
graph.x

#%%
for node in model.G.nodes:
    print(node)

# #%%
# class GCNModel(nn.Module):
#     def __init__(self, in_features, out_features):
#         super(GCNModel, self).__init__()
#         self.gcn_1 = nn.Linear(in_features, out_features)

#     def forward(self, graph, x):
#         x = self.gcn_1(x)

#         return x

# model = GCNModel(10, 2)
# #%%
# output = model(None, torch.ones(10))

# output
# #%%

# %%


# -*- coding: utf-8 -*-
from __future__ import absolute_import

import pytest
from mock import Mock

from elasticapm.utils import compat, stacks
from elasticapm.utils.stacks import get_culprit, get_stack_info
from tests.utils.stacks import get_me_a_test_frame

class Context(object):
    def __init__(self, dict):
        self.dict = dict

    __getitem__ = lambda s, *a: s.dict.__getitem__(*a)
    __setitem__ = lambda s, *a: s.dict.__setitem__(*a)
    iterkeys = lambda s, *a: compat.iterkeys(s.dict, *a)

def test_get_culprit_bad_module():
    culprit = get_culprit([{
        'module': None,
        'function': 'foo',
    }])
    assert culprit == '<unknown>.foo'

    culprit = get_culprit([{
        'module': 'foo',
        'function': None,
    }])
    assert culprit == 'foo.<unknown>'

    culprit = get_culprit([{
    }])
    assert culprit == '<unknown>.<unknown>'

def test_bad_locals_in_frame():
    frame = Mock()
    frame.f_locals = Context({
        'foo': 'bar',
        'biz': 'baz',
    })
    frame.f_lineno = 1
    frame.f_globals = {}
    frame.f_code.co_filename = __file__.replace('.pyc', '.py')
    frame.f_code.co_name = __name__
    frames = [(frame, 1)]
    results = get_stack_info(frames)
    assert len(results) == 1
    result = results[0]
    assert 'vars' in result
    variables = {
        "foo": "bar",
        "biz": "baz",
    }
    assert result['vars'] == variables

def test_traceback_hide(elasticapm_client):
    def get_me_a_filtered_frame(hide=True):
        __traceback_hide__ = True
        if not hide:
            del __traceback_hide__

        return list(stacks.iter_stack_frames())

    # hide frame from `get_me_a_filtered_frame
    frames = list(stacks.get_stack_info(get_me_a_filtered_frame(hide=True)))
    assert frames[0]['function'] == 'test_traceback_hide'

    # don't hide it:
    frames = list(stacks.get_stack_info(get_me_a_filtered_frame(hide=False)))
    assert frames[0]['function'] == 'get_me_a_filtered_frame'

@pytest.mark.parametrize('elasticapm_client', [{
    'include_paths': ('a.b.c', 'c.d'),
    'exclude_paths': ('c',)
}], indirect=True)
def test_library_frames(elasticapm_client):
    include = elasticapm_client.include_paths_re
    exclude = elasticapm_client.exclude_paths_re
    frame1 = Mock(f_globals={'__name__': 'a.b.c'})
    frame2 = Mock(f_globals={'__name__': 'a.b.c.d'})
    frame3 = Mock(f_globals={'__name__': 'c.d'})

    info1 = stacks.get_frame_info(frame1, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    info2 = stacks.get_frame_info(frame2, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    info3 = stacks.get_frame_info(frame3, 1, False, False, include_paths_re=include, exclude_paths_re=exclude)
    assert not info1['library_frame']
    assert not info2['library_frame']
    assert info3['library_frame']

def test_get_frame_info():
    frame = get_me_a_test_frame()
    frame_info = stacks.get_frame_info(frame, frame.f_lineno,
                                       with_locals=True, with_source_context=True)
    assert frame_info['function'] == 'get_me_a_test_frame'
    assert frame_info['filename'] == 'tests/utils/stacks/__init__.py'
    assert frame_info['module'] == 'tests.utils.stacks'
    assert frame_info['lineno'] == 6
    assert frame_info['context_line'] == '    return inspect.currentframe()'
    assert frame_info['vars'] == {'a_local_var': 42}


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /users/payno/.local/share/virtualenvs/est_venv/lib/python3.7/site-packages/orangecontrib/est/widgets/pymca/ft.py
# Compiled at: 2020-03-05 02:52:24
# Size of source mod 2**32: 9429 bytes
__authors__ = [
 'H. Payno']
__license__ = 'MIT'
__date__ = '06/07/2019'
import functools, logging
from Orange.widgets import gui
from Orange.widgets.settings import Setting
from Orange.widgets.widget import OWWidget
from Orange.widgets.widget import Input, Output
import Orange.data
import PyMca5.PyMcaGui.physics.xas.XASFourierTransformParameters as XASFourierTransformParameters
from silx.gui import qt
from silx.gui.plot import LegendSelector
import est.core.process.pymca.ft
from orangecontrib.est.process import _ProcessForOrangeMixIn, ProcessRunnable
from est.core.types import XASObject
from est.gui.XasObjectViewer import XasObjectViewer, ViewType
from est.gui.XasObjectViewer import _normalized_exafs, _ft_window_plot
from est.gui.XasObjectViewer import _ft_intensity_plot, _ft_imaginary_plot
from orangecontrib.est.progress import QProgress
from orangecontrib.est.utils import Converter
from ..container import _ParameterWindowContainer
_logger = logging.getLogger(__file__)

class FTWindow(qt.QMainWindow):

    def __init__(self, parent=None):
        qt.QMainWindow.__init__(self, parent)
        self.xasObjViewer = XasObjectViewer(mapKeys=['Mu'], spectrumPlots=('FTWindow',
                                                                           'FTIntensity'))
        self.xasObjViewer._spectrumViews[0]._plot.getXAxis().setLabel('K')
        self.xasObjViewer._spectrumViews[1]._plot.getXAxis().setLabel('R (Angstrom)')
        self.xasObjViewer._spectrumViews[1]._plot.getYAxis().setLabel('Arbitrary Units')
        self.setCentralWidget(self.xasObjViewer)
        self._pymcaWindow = _ParameterWindowContainer(parent=self, parametersWindow=XASFourierTransformParameters)
        dockWidget = qt.QDockWidget(parent=self)
        dockWidget.setWidget(self._pymcaWindow)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, dockWidget)
        dockWidget.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        dockWidget.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.legendDockWidget1 = LegendSelector.LegendsDockWidget(parent=self, plot=(self.xasObjViewer._spectrumViews[0]._plot))
        self.legendDockWidget1.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        self.legendDockWidget1.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.legendDockWidget1)
        self.legendDockWidget2 = LegendSelector.LegendsDockWidget(parent=self, plot=(self.xasObjViewer._spectrumViews[1]._plot))
        self.legendDockWidget2.setAllowedAreas(qt.Qt.RightDockWidgetArea | qt.Qt.LeftDockWidgetArea)
        self.legendDockWidget2.setFeatures(qt.QDockWidget.NoDockWidgetFeatures)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.legendDockWidget2)
        self.addDockWidget(qt.Qt.RightDockWidgetArea, self.xasObjViewer._mapView.keySelectionDocker)
        for ope in (_normalized_exafs, _ft_window_plot):
            self.xasObjViewer._spectrumViews[0].addCurveOperation(ope)

        for ope in (_ft_intensity_plot, _ft_imaginary_plot):
            self.xasObjViewer._spectrumViews[1].addCurveOperation(ope)

        self.setWindowFlags(qt.Qt.Widget)
        self.xasObjViewer.viewTypeChanged.connect(self._updateLegendView)
        self._updateLegendView()

    def getNCurves(self):
        return len(self.plot.getAllCurves())

    def _updateLegendView(self):
        index, viewType = self.xasObjViewer.getViewType()
        self.legendDockWidget1.setVisible(viewType is ViewType.spectrum and index == 0)
        self.legendDockWidget2.setVisible(viewType is ViewType.spectrum and index == 1)
        self.xasObjViewer._mapView.keySelectionDocker.setVisible(viewType is ViewType.map)

class FTOW(_ProcessForOrangeMixIn, OWWidget):
    __doc__ = '\n    Widget used for signal extraction\n    '
    name = 'fourier transform'
    id = 'orange.widgets.pymca.xas.ft'
    description = 'Progress fourier transform'
    icon = 'icons/ft.png'
    priority = 4
    category = 'esrfWidgets'
    keywords = ['spectroscopy', 'signal', 'fourier', 'transform', 'fourier transform']
    want_main_area = True
    resizing_enabled = True
    process_function = est.core.process.pymca.ft.PyMca_ft
    _pymcaSettings = Setting(dict())

    class Inputs:
        xas_obj = Input('xas_obj', XASObject, default=True)
        data_table = Input('Data', Orange.data.Table)

    class Outputs:
        res_xas_obj = Output('xas_obj', XASObject)

    def __init__(self):
        super().__init__()
        self._latest_xas_obj = None
        self._window = FTWindow(parent=self)
        layout = gui.vBox(self.mainArea, 'fourier transform').layout()
        layout.addWidget(self._window)
        self._progress = gui.ProgressBar(self, 100)
        if self._pymcaSettings != dict():
            self._window._pymcaWindow.setParameters(self._pymcaSettings)
        self._window._pymcaWindow.sigChanged.connect(self._updateProcess)

    def _updateProcess(self, *arv, **kwargs):
        self._update_settings()
        if self._latest_xas_obj:
            self.process(xas_obj=(self._latest_xas_obj))

    @Inputs.data_table
    def processFrmDataTable(self, data_table):
        if data_table is None:
            return
        self.process(Converter.toXASObject(data_table=data_table))

    @Inputs.xas_obj
    def process(self, xas_obj):
        if xas_obj is None:
            return
        if not self._canProcess():
            _logger.warning('There is some processing on going already, willnot process the new dataset')
        self._latest_xas_obj = xas_obj.copy(create_h5_file=True)
        self._startProcess()
        process_obj = QPyMca_ft()
        process_obj._advancement.sigProgress.connect(self._setProgressValue)
        process_obj.setProperties({'_pymcaSettings': self._window._pymcaWindow.getParameters()})
        thread = self.getProcessingThread()
        thread.init(process_obj=process_obj, xas_obj=(self._latest_xas_obj))
        self._callback_finish = functools.partial(self._endProcess, self._latest_xas_obj)
        thread.finished.connect(self._callback_finish)
        thread.start(priority=(qt.QThread.LowPriority))

    def _update_settings(self):
        self._pymcaSettings = self._window._pymcaWindow.getParameters()

    def _setProgressValue(self, value):
        self._progress.widget.progressBarSet(value)

class QPyMca_ft(est.core.process.pymca.ft.PyMca_ft):
    __doc__ = '\n    Normalization able to give advancement using qt.Signal and QThreadPool\n    '

    def __init__(self):
        est.core.process.pymca.ft.PyMca_ft.__init__(self)
        self._advancement = QProgress('normalization')

    def _pool_process(self, xas_obj):
        self.pool = qt.QThreadPool()
        self.pool.setMaxThreadCount(5)
        for spectrum in xas_obj.spectra:
            runnable = ProcessRunnable(fct=(est.core.process.pymca.ft.process_spectr_ft), spectrum=spectrum,
              configuration=(xas_obj.configuration),
              callback=(self._advancement.increaseAdvancement))
            self.pool.start(runnable)

        self.pool.waitForDone()


import abc
import copy

from bson import json_util, ObjectId
from flask import request
from flask.views import MethodView
from mongoengine.errors import NotUniqueError

from slots_tracker_server.gsheet import write_expense, update_expense
from slots_tracker_server.models import Expense, PayMethods, Categories
from slots_tracker_server.utils import convert_to_object_id, clean_api_object, next_payment_date

class BaseAPI(MethodView):
    __metaclass__ = abc.ABCMeta

    @property
    @abc.abstractmethod
    def api_class(self):
        raise NotImplementedError

    def get(self, obj_id):
        object_id = convert_to_object_id(obj_id)
        instance = self.api_class.objects.get_or_404(id=object_id)
        return [instance.to_json()]

    def post(self, obj_data):
        # TODO: Check that all reference fields are not inactive before creating a new object
        return self.api_class(**obj_data).save()

    def delete(self, obj_id):
        instance = self.api_class.objects.get_or_404(id=obj_id)
        instance.active = False
        instance.save()
        return '', 200

    def put(self, obj_id, obj_data):
        clean_api_object(obj_data)
        object_id = convert_to_object_id(obj_id)

        # Get and updated the object
        instance = self.api_class.objects.get_or_404(id=object_id)
        instance.update(**obj_data)
        return instance.reload()

    @staticmethod
    def get_obj_data():
        return json_util.loads(request.data)

    def objects_id_to_json(self, obj_data):
        for name, document_type in self.api_class.get_all_reference_fields():
            field_id = obj_data.get(name)
            field_data_as_json = document_type.objects.get(id=field_id).to_json()
            obj_data[name] = field_data_as_json

class BasicObjectAPI(BaseAPI):
    __metaclass__ = abc.ABCMeta

    @property
    @abc.abstractmethod
    def api_class(self):
        raise NotImplementedError

    def get(self, obj_id):
        if obj_id:
            obj_data = super(BasicObjectAPI, self).get(obj_id)
        else:
            obj_data = self.api_class.objects(active=True).order_by('-instances').to_json()

        return json_util.dumps(obj_data[0] if obj_id else obj_data)

    def post(self, obj_data=None):
        obj_data = self.get_obj_data()
        try:
            return json_util.dumps(super(BasicObjectAPI, self).post(obj_data).to_json()), 201
        except NotUniqueError:
            return 'Name value must be unique', 400

    def put(self, obj_id, obj_data=None):
        obj_data = self.get_obj_data()
        try:
            new_expense_as_json = super(BasicObjectAPI, self).put(obj_id, obj_data).to_json()
            self.objects_id_to_json(new_expense_as_json)
            return json_util.dumps(new_expense_as_json)
        except NotUniqueError:
            return 'Name value must be unique', 400

class PayMethodsAPI(BasicObjectAPI):
    api_class = PayMethods

class CategoriesAPI(BasicObjectAPI):
    api_class = Categories

class ExpenseAPI(BaseAPI):
    api_class = Expense

    def get(self, obj_id):
        if obj_id:
            obj_data = super(ExpenseAPI, self).get(obj_id)
        else:
            obj_data = self.api_class.objects(active=True).order_by('one_time', '-timestamp').to_json()

        for name, document_type in self.api_class.get_all_reference_fields():
            for item in obj_data:
                item[name] = document_type.objects.get(id=item.get(name)).to_json()

        limit = int(request.args.get('limit', len(obj_data)))
        return json_util.dumps(obj_data[0] if obj_id else obj_data[:limit])

    def post(self, obj_data=None):
        new_expenses_as_json = self.create_multi_expenses()
        return new_expenses_as_json, 201

    def put(self, obj_id, obj_data=None):
        new_expenses_as_json = self.create_multi_expenses(obj_id=obj_id)
        return new_expenses_as_json

    def reference_field_to_object_id(self, obj_data):
        for name, _ in self.api_class.get_all_reference_fields():
            field_data = obj_data.get(name)
            if field_data and not isinstance(field_data, ObjectId):
                field_data_id = field_data.get('_id') if isinstance(field_data, dict) else field_data
                obj_data[name] = convert_to_object_id(field_data_id)

    def create_multi_expenses(self, obj_id=None):
        obj_data = self.get_obj_data()
        self.reference_field_to_object_id(obj_data)

        new_expenses = []
        payments = int(request.args.get('payments', 1))
        payments_data = copy.deepcopy(obj_data)
        payments_data['amount'] = float(payments_data.get('amount')) / payments
        for i in range(payments):
            payments_data['timestamp'] = next_payment_date(obj_data['timestamp'], payment=i)
            new_expenses.append(self.create_expense(payments_data, obj_id))
        return json_util.dumps(new_expenses)

    def create_expense(self, obj_data, obj_id=None):
        if obj_id:
            new_expense = super(ExpenseAPI, self).put(obj_id, obj_data)
        else:
            new_expense = super(ExpenseAPI, self).post(obj_data)
        new_expense_as_json = new_expense.to_json()
        self.objects_id_to_json(new_expense_as_json)

        if obj_id:
            update_expense(new_expense)
        else:
            write_expense(new_expense)

        return new_expense_as_json


import matplotlib.pyplot as plt

def draw_graph(results):
    x = [result.number for result in results]
    y = [result.result.gpa for result in results]
    plt.plot(x, y, '-o')
    plt.grid(linestyle='--')
    plt.xlabel('Semester')
    plt.xlim(1, len(results))
    plt.ylabel('GPA')
    plt.title(f'GPA over past/current semesters', loc='center')
    plt.show()


import os
from abc import ABCMeta, abstractmethod
from configparser import ExtendedInterpolation

from myutils.myconfigparser import myConfigParser
from myutils.logger import get_logger

class Container(object):
    """Defines a container ABC.

    Attributes:
        name: name of the container
        config: configuration
        data: all the data contained
    """
    __metaclass__ = ABCMeta
    logger = get_logger(__name__, __package__+'.log')

    def __init__(self, name, config_file=None, config=None):
        """Defines a new container.

        Parameters:
            name: container name
        """
        assert not (config_file is not None and config is not None)

        self.name = name
        self.data = {}

        if config_file:
            # Load configuration
            self.logger.debug('Loading configuration file: %s', config_file)
            try:
                assert os.path.isfile(config_file)
            except AssertionError:
                self.logger.exception('File %s does not exist', config_file)
                raise IOError
            self.load_config(config_file)
            self.logger.info('Configuration file loaded')
        elif config is not None:
            self.config = config
            self.logger.info('Configuration assigned')
        else:
            self.config = None

    @abstractmethod
    def load_data(self, key, file_name=None):
        """Load *data* from file and save it in *key*"""
        pass

    def __getitem__(self, key):
        if key not in self.data.keys():
            self.load_data(key, None)
        return self.data[key]

    def __setitem__(self, key, value):
        self[key] = value

    def load_data_from_keys(self, keys, file_names):
        """Load all the data in each file in *file_names* and store it in
        *key*

        Parameters:
            keys (iterable): list of keys for each file
            file_names (iterable): list of files to load
        """
        for k,f in zip(keys, file_names):
            self.load_data(k, f)

    def load_config(self, config_file):
        """Load a configuration file.

        Parameters:
            config_file (str): name of the configuration file
        """
        self.config = myConfigParser(interpolation=ExtendedInterpolation())
        self.config.read(config_file)


from flask import Blueprint, render_template, request
from flask_login import login_required

from src.presentation.common import flash_warning, flash_error
from src.domain import Patient
from src.repositories import patient_repository, visit_repository

bp = Blueprint('home', __name__, url_prefix='/home')

@bp.route('')
@login_required
def home():
    return render_template('home.html', last_visits=visit_repository.last())

@bp.route('/search', methods=('POST',))
@login_required
def search():
    name = request.form.get('name', None)

    if not name:
        flash_error('Informe o nome do Paciente!')
        return render_template('home.html')

    patient_list = patient_repository.filter(Patient.name.like('%' + str(name) + '%')).all()

    if patient_list:
        patient_list.sort(key=lambda k: k.name)
    else:
        flash_warning('Paciente %s não encontrado!' % name)

    return render_template('home.html', patients=patient_list)


# sms.py
from twilio.rest import Client
from flask import g
from logging import getLogger
from app import get_key

log = getLogger(__name__)

#-------------------------------------------------------------------------------
def compose(body, to, callback=None):
    """Send SMS message. Returns delivery status.

    :to: destination phone number in international format
         '+17801234567'
    """
    keys = get_key('twilio')

    try:
        client = Client(keys['api']['sid'], keys['api']['auth_id'])
        msg = client.messages.create(
            body=body, to=to, from_=keys['sms']['number'], status_callback=None)
    except Exception as e:
        log.exception('Error creating twilio client or sending msg')
        raise

    log.debug(body)
    return msg.status


import torch
from transformers import BertTokenizer
from transformers.tokenization_utils import BatchEncoding

class YelpTransformer:
    def __init__(self, tokenizer: BertTokenizer):
        super().__init__()
        self.tokenizer = tokenizer

    def transform(self, review: str):
        tokenized_review = self.tokenizer.tokenize(review)
        returned_dict: BatchEncoding = self.tokenizer.encode_plus(
            review,
            add_special_tokens=True,
            max_length=self.tokenizer.model_max_length,
            pad_to_max_length=True,
            is_pretokenized=True,
            return_token_type_ids=True,
            return_attention_mask=True,
        )
        encoded_reviews, token_type_ids, attention_masks = returned_dict.values()

        # tensor に変換
        tokens_tensor = torch.LongTensor(encoded_reviews).to('cpu')
        token_type_ids_tensor = torch.LongTensor(token_type_ids).to('cpu')
        attention_tensor = torch.FloatTensor(attention_masks).to('cpu')

        return tokens_tensor, token_type_ids_tensor, attention_tensor


import sys
import os
from django.core.exceptions import ObjectDoesNotExist
from django.core.management.base import BaseCommand, CommandError
from workshops.models import Award, Badge, Person

class Command(BaseCommand):
    help = 'Report inconsistencies in PDF certificates.'

    def add_arguments(self, parser):
        parser.add_argument(
            'path', help='Path to root directory of certificates repository',
        )

    def handle(self, *args, **options):
        '''Main entry point.'''

        path_to_root = options['path']

        badges = self.get_badges()
        for (name, badge) in badges:
            db_people = self.get_db_people(badge)
            cert_path = os.path.join(path_to_root, name)
            if not os.path.isdir(cert_path):
                print('No directory {0}'.format(name))
            else:
                file_people = self.get_file_people(cert_path)
                self.report_missing('database but not disk', name, db_people - file_people)
                self.report_missing('disk but not database', name, file_people - db_people)

    def get_badges(self):
        '''Get all available badges as list of lower-case name and badge pairs.'''

        return [(b.name.lower(), b) for b in Badge.objects.all()]

    def get_db_people(self, badge):
        '''Get set of usernames of all people with the given badge.'''

        return set(Award.objects.filter(badge=badge).values_list('person__username', flat=True))

    def get_file_people(self, path):
        '''Get names of all people with the given certificate.'''

        return set([os.path.splitext(e)[0]
                    for e in os.listdir(path)
                    if e.endswith('.pdf')])

    def report_missing(self, title, kind, items):
        '''Report missing items.'''
        if items:
            print('{0} {1}'.format(kind, title))
            for i in sorted(list(items)):
                try:
                    p = Person.objects.get(username=i)
                    print(' {0}: {1}'.format(i, p))
                except Person.DoesNotExist:
                    print(' {0}'.format(i))


# -*- coding: utf-8 -*-

import sys    # sys.setdefaultencoding is cancelled by site.py
reload(sys)    # to re-enable sys.setdefaultencoding()
sys.setdefaultencoding('utf-8')
def read_article(path_to_article):
    with open(path_to_article) as my_file:
        article = my_file.read()
    return article.decode('utf-8')

article = read_article(sys.argv[1])
#article = read_article('.\\AML\\1\\article')

from pprint import pprint
import json
def read_json(path_to_json):
    with open(path_to_json) as json_file:
        json_of_article = json.load(json_file)
    return json_of_article

json_of_article = read_json(sys.argv[2])
#json_of_article = read_json('2.out')

from collections import OrderedDict
def get_entities(json_of_article):
    entities = {}
    for j in json_of_article:
        if j != {}:
            s = (j['Type'])
            for pair in j['Boundaries']:
                entities[ pair[0]] = list([pair[1], entity_color[s] ])
    entities = OrderedDict(sorted(entities.items(), key=lambda t:t[0]))
    return entities

color1 ='#' + sys.argv[4][2:];
color2 ='#' + sys.argv[5][2:];
color3 ='#' + sys.argv[6][2:];

entity_color = {'Person':color1, 'PopulatedPlace':color2, 'Organisation':color3}

entities = get_entities(json_of_article)
#print(entities)

from yattag import Doc, indent

def make_html(article, entities, path_to_html):
    doc, tag, text = Doc().tagtext()

    with tag('html'):
        with tag('head'):
            with tag('meta'):
                doc.attr(charset='utf-8')
            with tag('title'):
                text('Marking of article')

        with tag('body'):
            with tag('table'):
                doc.attr(border='2')
                with tag('tr'):
                    with tag('th'):
                        doc.attr(style='padding: 10px;font-size:160%')
                        with tag('p'):
                            doc.attr(style='word-spacing:20px;font-style:italic;')
                            with tag('font'):
                                if(color2 != '#000000'):
                                    doc.attr(color=color2)
                                    text(' Organisation ')
                            with tag('font'):
                                if(color3 != '#000000'):
                                    doc.attr(color=color3)
                                    text(' PopulatedPlace ')
                            with tag('font'):
                                if(color1 != '#000000'):
                                    doc.attr(color=color1)
                                    text(' Person ')
            with tag('pre'):
                doc.attr(style='white-space: pre-wrap')
                with tag('p'):
                    doc.attr(style='width:3000px;font-size:160%')
                    k = 0
                    for d in entities.items():
                        #print article[d[0]: d[1][0]]
                        if (article[k: d[0]] == ' '):
                            text(' ')
                        else:
                            text(article[k: d[0]])
                        with tag('font'):
                            if(d[1][1]!='#000000'):
                                doc.attr(color=d[1][1])
                                with tag('b'):
                                    text(article[d[0]: d[1][0]])
                                    k = d[1][0]
                            else:
                                text(article[d[0]: d[1][0]])
                                k = d[1][0]
                    text(article[k:])

    result = ((doc.getvalue()))
    #print(result)
    Html_file = open(path_to_html, 'w+')
    Html_file.write(result)
    Html_file.close()

make_html(article, entities, sys.argv[3])
#make_html(article, entities, 'h.html')
#print (len(article))


#!/usr/bin/env python
# xia2merger.py
#   Copyright (C) 2011 Diamond Light Source, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# Wrapper to allow code written to merge reflections to be externalised...
#
# 03/MAR/16
# To resolve the naming conflict between this file and the entire xia2 module
# any xia2.* imports in this directory must instead be imported as ..*

from __future__ import absolute_import, division, print_function

from xia2.Toolkit.Merger import merger

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument("hklin", type=str, help="input reflection file")
    parser.add_argument("-n", "--nbins", type=float, help="number of resolution bins")
    parser.add_argument("-r", "--rmerge", type=float, help="limit on Rmerge")
    parser.add_argument(
        "-c", "--completeness", type=float, help="limit on completeness [0,1]"
    )
    parser.add_argument("-i", "--isigma", type=float, help="limit on I/sigma")
    parser.add_argument("-m", "--misigma", type=float, help="limit on Mn(I/sigma)")
    args = parser.parse_args()

    if args.nbins:
        nbins = args.nbins
    else:
        nbins = 100

    m = merger(args.hklin)
    m.calculate_resolution_ranges(nbins=nbins)

    if args.completeness:
        print("COMPLETENESS %f" % m.resolution_completeness(limit=args.completeness))

    if args.rmerge:
        print("RMERGE %f" % m.resolution_rmerge(limit=args.rmerge))

    if args.isigma:
        print("ISIGMA %f" % m.resolution_unmerged_isigma(limit=args.isigma))

    if args.misigma:
        print("MISIGMA %f" % m.resolution_merged_isigma(limit=args.misigma))


add_library('pdf')
from datetime import datetime

def setup():
    global recordPDF, strokeColor
    recordPDF = False
    strokeColor = color(0, 10)
    colorMode(HSB, 360, 100, 100, 100);
    smooth()
    noFill()
    background(360)

def draw():
    global recordPDF, strokeColor
    if(mousePressed):
        pushMatrix()
        translate(width/2,height/2)

        circleResolution = int(map(mouseY+100,0,height,2, 10))
        radius = mouseX-width/2 + 0.5
        angle = TWO_PI/circleResolution

        strokeWeight(2)
        stroke(strokeColor)

        beginShape()
        for i in range(circleResolution+1):
            x = 0 + cos(angle*i) * radius
            y = 0 + sin(angle*i) * radius
            vertex(x, y)
        endShape()
        popMatrix()

def keyReleased():
    global recordPDF, strokeColor
    if (key == DELETE or key == BACKSPACE):
        background(255)
    if (key=='s' or key=='S'):
        saveFrame(datetime.now().strftime("%Y%m%d%H%M%S")+".png")

    if key == "1":
        strokeColor = color(0, 10)
    elif key == "2":
        strokeColor = color(192, 100, 64, 10)
    elif key == "3":
        strokeColor = color(52, 100, 71, 10)

    # pdf export
    if (key =='r' or key =='R'):
        if (recordPDF == False):
            beginRecord(PDF, datetime.now().strftime("%Y%m%d%H%M%S")+".pdf")
            println("recording started")
            recordPDF = True
            colorMode(HSB, 360, 100, 100, 100)
            smooth()
            noFill()
            background(360)
    elif (key == 'e' or key =='E'):
        if (recordPDF):
            println("recording stopped")
            endRecord()
            recordPDF = False
            background(c60)


from flask_app.config.mysqlconnection import connectToMySQL
import re	# the regex module
# create a regular expression object that we'll use later
EMAIL_REGEX = re.compile(r'^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9._-]+\.[a-zA-Z]+$')
from flask import flash

class Email:
    # using a class variable to hold my database name
    db = "email_validation"
    def __init__(self,data):
        self.id = data['id']
        self.email = data['email']
        self.created_at = data['created_at']
        self.updatd_at = data['updated_at']

    # class method to save the email object in the data base
    @classmethod
    def save(cls,data):
        query = "INSERT INTO emails (email) VALUES (%(email)s);"
        return connectToMySQL(cls.db).query_db(query,data)

    # class method to get all the emails fromt the database
    @classmethod
    def get_all(cls):
        query= "SELECT * FROM emails;"
        results = connectToMySQL(cls.db).query_db(query)
        emails = []
        for row in results:
            emails.append( cls(row) )
        return emails

    # class method to delete an email from the database based on the PK.
    @classmethod
    def destroy(cls,data):
        query = "DELETE FROM emails WHERE id = %(id)s;"
        return connectToMySQL(cls.db).query_db(query,data)

    @staticmethod
    def is_valid(email):
        is_valid = True
        query = "SELECT * FROM emails WHERE email = %(email)s;"
        results = connectToMySQL(Email.db).query_db(query,email)
        if len(results) >= 1:
            flash("Email already taken.")
            is_valid=False
        if not EMAIL_REGEX.match(email['email']):
            flash("Invalid Email!!!")
            is_valid=False
        return is_valid


from django.shortcuts import render, redirect, reverse, get_object_or_404
from django.shortcuts import HttpResponse
from django.views.decorators.http import require_POST
from django.contrib import messages
from django.conf import settings

from services.models import Service
from .models import OrderLineItem, Order
from profiles.forms import UserProfileForm
from profiles.models import UserProfile
from .forms import OrderForm
from cart.contexts import cart_contents

import stripe
import json

@require_POST
def cache_checkout_data(request):
    try:
        pid = request.POST.get('client_secret').split('_secret')[0]
        stripe.api_key = settings.STRIPE_SECRET_KEY
        stripe.PaymentIntent.modify(pid, metadata={
            'cart': json.dumps(request.session.get('cart', {})),
            'save_info': request.POST.get('save_info'),
            'username': request.user,
        })
        return HttpResponse(status=200)
    except Exception as e:
        messages.error(request, 'Sorry, your payment cannot be \
            processed right now. Please try again later.')
        return HttpResponse(content=e, status=400)

def checkout(request):
    stripe_public_key = settings.STRIPE_PUBLIC_KEY
    stripe_secret_key = settings.STRIPE_SECRET_KEY

    if request.method == 'POST':
        order_form = OrderForm(request.POST)
        cart = request.session.get('cart', {})

        form_data = {
            'full_name': request.POST['full_name'],
            'email': request.POST['email'],
            'company_name': request.POST['company_name'],
            'phone_number': request.POST['phone_number'],
            'description': request.POST['description'],
            'country': request.POST['country'],
            'zipcode': request.POST['zipcode'],
            'city': request.POST['city'],
            'street_address1': request.POST['street_address'],
            'street_address2': request.POST['second_address'],
            'county': request.POST['region'],
        }

        order_form = OrderForm(form_data)
        if order_form.is_valid():
            order = order_form.save(commit=False)
            pid = request.POST.get('client_secret').split('_secret')[0]
            order.stripe_pid = pid
            order.original_cart = json.dumps(cart)
            order.save()
            for item_id, item_data in cart.items():
                try:
                    service = Service.objects.get(id=item_id)
                    if isinstance(item_data, int):
                        order_line_item = OrderLineItem(
                           order=order,
                           service=service,
                           quantity=item_data,
                        )
                        order_line_item.save()
                except Service.DoesNotExist:
                    messages.error(request, (
                        "One of the services in your cart wasn't found in our database."
                        "Please contact us for assistance")
                    )
                    order.delete()
                    return redirect(reverse('view_cart'))

            request.session['save_info'] = 'save-info' in request.POST
            return redirect(reverse('checkout_success', args=[order.order_number]))

        else:
            messages.error(request, 'There was an error with your form. \
                Please double check your information.')

    else:
        cart = request.session.get('cart', {})
        if not cart:
            messages.error(request, "Cart is empty")
            return redirect(reverse('services'))

    current_cart = cart_contents(request)
    total = current_cart['grand_total']
    stripe_total = round(total * 100)
    stripe.api_key = stripe_secret_key
    intent = stripe.PaymentIntent.create(
        amount=stripe_total,
        currency=settings.STRIPE_CURRENCY,
    )

    if request.user.is_authenticated:
        try:
            profile = UserProfile.objects.get(user=request.user)
            order_form = OrderForm(initial={
                'full_name': profile.user.get_full_name(),
                'email': profile.user.get_full_name(),
                'company_name': profile.default_company_name,
                'phone_number': profile.default_phone_number,
                'country': profile.default_country,
                'zipcode': profile.default_zipcode,
                'city': profile.default_city,
                'street_address': profile.default_street_address,
                'second_address': profile.default_second_address,
                'region': profile.default_region,
            })
        except UserProfile.DoesNotExist:
            order_form = OrderForm()
    else:
        order_form = OrderForm()

    if not stripe_public_key:
        messages.warning(request, 'Stripe public key is missing. \
                Did you forget the env variables?')

    template = 'checkout/checkout.html'
    context = {
            'order_form': order_form,
            'stripe_public_key': stripe_public_key,
            'client_secret': intent.client_secret,
        }
    return render(request, template, context)

def checkout_success(request, order_number):
    """
    Handle successful checkouts
    """

    save_info = request.session.get('save_info')
    order = get_object_or_404(Order, order_number=order_number)

    if request.user.is_authenticated:
        profile = UserProfile.objects.get(user=request.user)
        # Attach the user's profile to the order
        order.user_profile = profile
        order.save()

        if save_info:
            profile_data = {
                'default_company_name': order.company_name,
                'default_phone_number': order.phone_number,
                'default_country': order.country,
                'default_zipcode': order.zipcode,
                'default_city': order.city,
                'default_street_address': order.street_address,
                'default_second_address': order.second_address,
                'default.region': order.region,
            }
            user_profile_form = UserProfileForm(profile_data, instance=profile)
            if user_profile_form.is_valid():
                user_profile_form.save()

    messages.success(request, f'Order successfully processed! \
        Your order number is {order_number}. A confirmation \
        email will be sent to {order.email}.')

    if 'cart' in request.session:
        del request.session['cart']

    template = 'checkout/checkout_success.html'
    context = {
        'order': order,
    }

    return render(request, template, context)


#!/usr/bin/env python

# Dependencies
from lxml import html, etree
import requests
import pandas as pd
import datetime as dt

class rightmove_data(object):
    """The rightmove_data web scraper works by implementing an instance of the class
    on the URL returned by a search on the rightmove website. Go to rightmove.co.uk
    and search for whatever you want, then create an instance of the class on the URL
    returned by the search. The class returns an object which includes various
    methods for extracting data from the search results, the most useful being the
    .get_results() method which returns all results as a pandas DataFrame object.
    """

    def __init__(self, url):

        self.url = url

        try:
            if "searchType=SALE" in self.url:
                self.rent_or_sale = "SALE"
            elif "searchType=RENT" in self.url:
                self.rent_or_sale = "RENT"
        except ValueError:
            print("Not a valid rightmove search URL.")

        self.results_count = self.__results_count()
        self.result_pages_count = self.__result_pages_count()

    def __results_count(self):
        """Returns an integer of the total number of results returned by the search URL."""

        page = requests.get(self.url)
        tree = html.fromstring(page.content)
        xp_result_count = """//span[@class="searchHeader-resultCount"]/text()"""
        return int(tree.xpath(xp_result_count)[0].replace(",", ""))

    def __result_pages_count(self):
        """Returns the number of result pages returned by the search URL.
        There are 24 results on each results page, but note that the
        rightmove website limits results pages to a maximum of 42 pages."""

        page_count = self.results_count // 24

        if self.results_count % 24 > 0:
            page_count += 1

        # Rightmove will return a maximum of 42 results pages, hence:
        if page_count > 42: page_count = 42

        return page_count

    def __get_page_results(self,page_url):
        """This is a hidden method to scrape the data from a single page
        of search results. It is used iteratively by the .get_results()
        method to scrape data from every page returned by the search."""

        # Set the correct xpath for the price.
        if self.rent_or_sale == "RENT":
            xp_prices = """//span[@class="propertyCard-priceValue"]/text()"""
        elif self.rent_or_sale == "SALE":
            xp_prices = """//div[@class="propertyCard-priceValue"]/text()"""

        # Set the xpaths for listing title, property address,
        # listing URL, and agent URL.
        xp_titles = """//div[@class="propertyCard-details"]\
        //a[@class="propertyCard-link"]\
        //h2[@class="propertyCard-title"]/text()"""
        xp_addresses = """//address[@class="propertyCard-address"]//span/text()"""
        xp_weblinks = """//div[@class="propertyCard-details"]\
        //a[@class="propertyCard-link"]/@href"""
        xp_agent_urls = """//div[@class="propertyCard-contactsItem"]\
        //div[@class="propertyCard-branchLogo"]\
        //a[@class="propertyCard-branchLogo-link"]/@href"""

        # Use the requests library to get the whole web page.
        page = requests.get(page_url)

        # Process the html.
        tree = html.fromstring(page.content)

        # Create data lists from Xpaths.
        price_pcm = tree.xpath(xp_prices)
        titles = tree.xpath(xp_titles)
        addresses = tree.xpath(xp_addresses)
        urlbase = "http://www.rightmove.co.uk"
        weblinks = ["{}{}".format(urlbase, tree.xpath(xp_weblinks)[val]) \
                    for val in range(len(tree.xpath(xp_weblinks)))]
        agent_urls = ["{}{}".format(urlbase, tree.xpath(xp_agent_urls)[val]) \
                      for val in range(len(tree.xpath(xp_agent_urls)))]

        # Store the data in a temporary pandas DataFrame.
        data = [price_pcm, titles, addresses, weblinks, agent_urls]
        temp_df = pd.DataFrame(data)
        temp_df = temp_df.transpose()
        temp_df.columns = ["price", "type", "address", "url", "agent_url"]

        # Drop empty rows which come from placeholders in the html.
        temp_df = temp_df[temp_df["address"].notnull()]

        return temp_df

    def get_results(self):
        """Returns a pandas DataFrame with all results returned by the search."""

        # Create DataFrame to store results.
        full_results = pd.DataFrame(columns={"price", "type", "address", "url", "agent_url"})

        # Iterate through pages of results, using the .__get_page_results method to scrape results.
        for page in range(0, self.result_pages_count+1, 1):

            # Create the URL of the specific results page.
            iteration_url = "{}{}{}".format(str(self.url), "&index=", str((page*24)))

            # Create a temporary dataframe of the page results.
            temp_df = self.__get_page_results(iteration_url)

            # Concatenate the temporary dataframe with the full dataframe.
            frames = [full_results, temp_df]
            full_results = pd.concat(frames)

        # Reset the index.
        full_results = full_results.reset_index(drop=True)

        # Convert price column to numeric type.
        full_results.price.replace(regex=True, inplace=True, to_replace=r"\D", value=r"")
        full_results.price = pd.to_numeric(full_results.price)

        # Extract postcodes to a separate column.
        full_results["postcode"] = full_results["address"].str.extract\
        (r"\b([A-Za-z][A-Za-z]?[0-9][0-9]?[A-Za-z]?)\b", expand=True)

        # Extract number of bedrooms from "type" to a separate column.
        full_results["number_bedrooms"] = full_results.type.str.extract(r"\b([\d][\d]?)\b", expand=True)
        full_results.loc[full_results["type"].str.contains("studio", case=False), "number_bedrooms"]=0

        # Clean up annoying white spaces and newlines in "type" column.
        for row in range(len(full_results)):
            type_str = full_results.loc[row, "type"]
            clean_str = type_str.strip("\n").strip()
            full_results.loc[row, "type"] = clean_str

        # Add column with datetime when the search was run (i.e. now).
        now = dt.datetime.today()
        full_results["search_date"] = now

        return full_results


from bs4 import BeautifulSoup
import urllib.request
import os
import time

def main():
    # 先爬取4页代理ip
    # 保存在本地
    get_web()

    # 读取文件 1.txt等格式
    # 提取IP:PORT字段
    get_info()

def get_web():
    get_web_list = ['http://www.xicidaili.com/nn/', 'http://www.xicidaili.com/nt/', 'http://www.xicidaili.com/wn/', 'http://www.xicidaili.com/wt/']

    for x in range(0,len(get_web_list)):

        req = urllib.request.Request(get_web_list[x])
        req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36')

        response = urllib.request.urlopen(req)
        html = response.read().decode('utf-8')

        filename = str(x+1) + ".txt"
        fo = open(filename, "w+")
        fo.write(html)
        fo.close()

        time.sleep(3)

def get_info():

    file_name = ".txt"
    ip_port_list = []

    for x in range(1,5):
        real_name = str(x)+file_name

        info = open(real_name, "r+")
        info = info.read()

        soup=  BeautifulSoup(info, "html.parser")
        test = soup.find_all('td')

        n = 1
        m = 2
        i = 0

        max_num = int(len(test) / 10)

        while i < max_num:
            num_n = str(test[n]).find('	')
            num_m = str(test[m]).find('	')
            ip_port_list.append("%s:%s" % (str(test[n])[4:num_n], str(test[m])[4:num_m]))
            n += 10
            m += 10
            i += 1

    fo = open('IP_PORT.txt', "w")
    for x in range(1,len(ip_port_list)):
        str_test =  ip_port_list[x] + "\n"
        fo.writelines(str_test)
    fo.close()

if __name__ == '__main__':
    main()


import logging

import numpy as np
import pandas as pd
from sklearn.preprocessing import OneHotEncoder, StandardScaler

files_by_day = {
    'mon': ['data/cicids2017/Monday-WorkingHours.pcap_ISCX.csv'],
    'tue': ['data/cicids2017/Tuesday-WorkingHours.pcap_ISCX.csv'],
    'wen': ['data/cicids2017/Wednesday-workingHours.pcap_ISCX.csv'],
    'thu': ['data/cicids2017/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv',
            'data/cicids2017/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv'],
    'fri': ['data/cicids2017/Friday-WorkingHours-Morning.pcap_ISCX.csv',
            'data/cicids2017/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv',
            'data/cicids2017/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv'],
}

# Configuration for generating the split for this dataset
train_files = [files_by_day['fri'][1]]
train_do_clean = True  # Turn any infinite entries into NaN, drop all rows with NaN entries
train_drop_percentage = 0.7  # Drop percentage of rows (after clean)
train_drop_benign_only = False  # If true, only benign traffic rows will be considered for dropping

test_files = [files_by_day['fri'][2]]
test_do_clean = True
test_drop_percentage = 0.8
test_drop_benign_only = True

random_seed = 1337  # Set fixed random seed to make subsampling repeatable

catcols = [' Destination Port',  'Fwd PSH Flags', ' Bwd PSH Flags', ' Fwd URG Flags', ' Bwd URG Flags']

ratecols = [' Down/Up Ratio']

numcols = [' Flow Duration', ' Total Fwd Packets', ' Total Backward Packets',
           'Total Length of Fwd Packets', ' Total Length of Bwd Packets', ' Fwd Packet Length Max',
           ' Fwd Packet Length Min', ' Fwd Packet Length Mean', ' Fwd Packet Length Std', 'Bwd Packet Length Max',
           ' Bwd Packet Length Min', ' Bwd Packet Length Mean', ' Bwd Packet Length Std', 'Flow Bytes/s',
           ' Flow Packets/s', ' Flow IAT Mean', ' Flow IAT Std', ' Flow IAT Max', ' Flow IAT Min', 'Fwd IAT Total',
           ' Fwd IAT Mean', ' Fwd IAT Std', ' Fwd IAT Max', ' Fwd IAT Min', 'Bwd IAT Total', ' Bwd IAT Mean',
           ' Bwd IAT Std', ' Bwd IAT Max', ' Bwd IAT Min', ' Fwd Header Length', ' Bwd Header Length', 'Fwd Packets/s',
           ' Bwd Packets/s', ' Min Packet Length', ' Max Packet Length', ' Packet Length Mean', ' Packet Length Std',
           ' Packet Length Variance', 'FIN Flag Count', ' SYN Flag Count', ' RST Flag Count', ' PSH Flag Count',
           ' ACK Flag Count', ' URG Flag Count', ' CWE Flag Count', ' ECE Flag Count',
           ' Average Packet Size', ' Avg Fwd Segment Size', ' Avg Bwd Segment Size', ' Fwd Header Length.1',
           'Fwd Avg Bytes/Bulk', ' Fwd Avg Packets/Bulk', ' Fwd Avg Bulk Rate', ' Bwd Avg Bytes/Bulk',
           ' Bwd Avg Packets/Bulk', 'Bwd Avg Bulk Rate', 'Subflow Fwd Packets', ' Subflow Fwd Bytes',
           ' Subflow Bwd Packets', ' Subflow Bwd Bytes', 'Init_Win_bytes_forward', ' Init_Win_bytes_backward',
           ' act_data_pkt_fwd', ' min_seg_size_forward', 'Active Mean', ' Active Std', ' Active Max', ' Active Min',
           'Idle Mean', ' Idle Std', ' Idle Max', ' Idle Min']

def fit_encoders(dataset):
    ohe = OneHotEncoder(handle_unknown='ignore')
    ohe.fit(dataset[catcols].values)
    scaler = StandardScaler()
    scaler.fit(dataset[numcols].values)
    return ohe, scaler

def transform(dataset, ohe, scaler):
    a1 = scaler.transform(dataset[numcols])
    a2 = ohe.transform(dataset[catcols]).toarray()
    a3 = dataset[ratecols].values
    labels = (dataset[' Label'].values != 'BENIGN').astype(int)
    return np.append(np.append(a1, a2, axis=1), a3, axis=1), labels

def read_files():
    np.random.seed(random_seed)

    if len(train_files) > 1:
        _train_list = []
        for filename in train_files:
            df = pd.read_csv(filename, index_col=None)
            _train_list.append(df)
        train_dataset = pd.concat(_train_list, axis=0, ignore_index=True)
    else:
        train_dataset = pd.read_csv(train_files[0])

    if train_do_clean:
        train_dataset = train_dataset.replace([np.inf, -np.inf], np.nan)
        train_dataset = train_dataset.dropna()
    benign_count = len(train_dataset[train_dataset[' Label'] == 'BENIGN'])
    logging.debug('Train dataset ...')
    logging.debug('Sample count: '+str(len(train_dataset)))
    logging.debug('Benign count: '+str(benign_count))
    logging.debug('Benign ratio: '+str(benign_count / len(train_dataset)))

    if train_drop_percentage > 0:
        if train_drop_benign_only:
            drop_indices = np.random.choice(train_dataset[train_dataset[' Label'] == 'BENIGN'].index,
                                            int(len(train_dataset) * train_drop_percentage),
                                            replace=False)
        else:
            drop_indices = np.random.choice(train_dataset.index,
                                            int(len(train_dataset) * train_drop_percentage),
                                            replace=False)
        train_dataset = train_dataset.drop(drop_indices)

        benign_count = len(train_dataset[train_dataset[' Label'] == 'BENIGN'])
        logging.debug('After dropping ...')
        logging.debug('Sample count: '+str(len(train_dataset)))
        logging.debug('Benign count: '+str(benign_count))
        logging.debug('Benign ratio: '+str(benign_count / len(train_dataset)))

    if len(test_files) > 1:
        _test_list = []
        for filename in test_files:
            df = pd.read_csv(filename, index_col=None)
            _test_list.append(df)
        test_dataset = pd.concat(_test_list, axis=0, ignore_index=True)
    else:
        test_dataset = pd.read_csv(test_files[0])

    if test_do_clean:
        test_dataset = test_dataset.replace([np.inf, -np.inf], np.nan)
        test_dataset = test_dataset.dropna()
    benign_count = len(test_dataset[test_dataset[' Label'] == 'BENIGN'])
    logging.debug('Test dataset ...')
    logging.debug('Sample count: '+str(len(test_dataset)))
    logging.debug('Benign count: '+str(benign_count))
    logging.debug('Benign ratio: '+str(benign_count / len(test_dataset)))

    if test_drop_percentage > 0:
        if test_drop_benign_only:
            drop_indices = np.random.choice(test_dataset[test_dataset[' Label'] == 'BENIGN'].index,
                                            int(benign_count * test_drop_percentage),
                                            replace=False)
        else:
            drop_indices = np.random.choice(test_dataset.index,
                                            int(benign_count * test_drop_percentage),
                                            replace=False)
        test_dataset = test_dataset.drop(drop_indices)

        benign_count = len(test_dataset[test_dataset[' Label'] == 'BENIGN'])
        logging.debug('After dropping ...')
        logging.debug('Sample count: '+str(len(test_dataset)))
        logging.debug('Benign count: '+str(benign_count))
        logging.debug('Benign ratio: '+str(benign_count / len(test_dataset)))

    np.random.seed(None)  # Set random random seed

    return train_dataset, test_dataset

def get_coherent_split():
    train_dataset, test_dataset = read_files()
    logging.debug(train_dataset.head())
    ohe, scaler = fit_encoders(train_dataset)
    x_train, y_train = transform(train_dataset, ohe, scaler)
    x_test, y_test = transform(test_dataset, ohe, scaler)
    return x_train, y_train, x_test, y_test

def get_random_split():
    train_dataset_coherent, test_dataset_coherent = read_files()

    train_dataset_anomaly_count = len(train_dataset_coherent[train_dataset_coherent[' Label'] != 'BENIGN'])
    train_dataset_benign_count = len(train_dataset_coherent) - train_dataset_anomaly_count
    logging.debug("CICIDS2017 train set anomaly count : " + str(train_dataset_anomaly_count) +
                  ", benign count : " + str(train_dataset_benign_count))

    test_dataset_anomaly_count = len(test_dataset_coherent[test_dataset_coherent[' Label'] != 'BENIGN'])
    test_dataset_benign_count = len(test_dataset_coherent) - test_dataset_anomaly_count
    logging.debug("CICIDS2017 test set anomaly count : " + str(test_dataset_anomaly_count) +
                  ", benign count : " + str(test_dataset_benign_count))

    while True:  # Try to generate splits until valid (should not be necessary with OHE handle_unknown='ignore')
        try:
            # Combine datasets, build new index
            combined_dataset = pd.concat([train_dataset_coherent, test_dataset_coherent], ignore_index=True)
            combined_dataset_benign = combined_dataset[combined_dataset[' Label'] == 'BENIGN']
            combined_dataset_anomaly = combined_dataset[combined_dataset[' Label'] != 'BENIGN']

            # Keep sample count and anomaly/benign ratio from original split
            test_dataset_benign = combined_dataset_benign.sample(n=test_dataset_benign_count)
            train_dataset_benign = combined_dataset_benign.drop(test_dataset_benign.index)
            test_dataset_anomaly = combined_dataset_anomaly.sample(n=test_dataset_anomaly_count)
            train_dataset_anomaly = combined_dataset_anomaly.drop(test_dataset_anomaly.index)
            train_dataset = pd.concat([train_dataset_benign, train_dataset_anomaly])
            test_dataset = pd.concat([test_dataset_benign, test_dataset_anomaly])
            logging.debug("(Shape CICIDS2017 Random Split) Train : " + str(train_dataset.shape) +
                          ", Test : " + str(test_dataset.shape))
            logging.debug("(Shape CICIDS2017 Coherent Split) Train : " + str(train_dataset_coherent.shape) +
                          ", Test : " + str(test_dataset_coherent.shape))
            logging.debug(train_dataset.head())
            ohe, scaler = fit_encoders(train_dataset)
            x_train, y_train = transform(train_dataset, ohe, scaler)
            x_test, y_test = transform(test_dataset, ohe, scaler)
            return x_train, y_train, x_test, y_test
        except ValueError:
            logging.info("Discarding split, retrying...")


from django.db import models

class Node(models.Model):
    slug = models.SlugField(null=False, blank=False)
    content = models.TextField(blank=False, null=False)
    creator = models.CharField(max_length=254, blank=False, null=False)
    created = models.DateTimeField(auto_now_add=True, null=False)
    changelog = models.TextField(null=True, blank=True)

    def __unicode__(self):
        return u'%s' % self.slug

    @property
    def revision(self):
        rev = Node.objects.filter(slug=self.slug, created__lte=self.created)
        return rev.count()


import pytest
import numpy.testing as nptest
import pandas.util.testing as pdtest
from wqio.tests import helpers

import numpy
import pandas
from matplotlib import pyplot

from wqio import hydro

BASELINE_IMAGES = '_baseline_images/hydro_tests'

class fakeStormSublcass(hydro.Storm):
    is_subclassed = True

def setup_storms(filename, baseflow=None):
    storm_file = helpers.test_data_path(filename)
    orig_record = pandas.read_csv(
        storm_file, index_col='date', parse_dates=True
    ).resample('5T').asfreq().fillna(0)

    baseflowcol = None
    if baseflow is not None:
        orig_record = orig_record.assign(baseflow=orig_record['influent'] < baseflow)
        baseflowcol = 'baseflow'

    parsed = hydro.parse_storm_events(orig_record, 3, 5,
                                      precipcol='rain',
                                      inflowcol='influent',
                                      baseflowcol=baseflowcol,
                                      debug=True)
    return parsed

@pytest.fixture
def storms_simple():
    return setup_storms('teststorm_simple.csv')

@pytest.fixture
def storms_firstobs():
    return setup_storms('teststorm_firstobs.csv')

@pytest.fixture
def storms_singular():
    return setup_storms('teststorm_singular.csv')

@pytest.fixture
def storms_with_baseflowcol():
    return setup_storms('teststorm_simple.csv', baseflow=2)

@pytest.mark.parametrize(('wetcol', 'expected'), [('A', 1), ('B', 0)])
def test_wet_first_row(wetcol, expected):
    df = pandas.DataFrame({
        'A': [True, True, False],
        'B': [False, False, True],
        'Z': [0, 0, 0],
    }).pipe(hydro._wet_first_row, wetcol, 'Z')

    assert df.iloc[0].loc['Z'] == expected

def test_wet_window_diff():
    is_wet = pandas.Series([0, 1, 1, 1, 0, 0, 1, 0, 0, 0])
    result = hydro._wet_window_diff(is_wet, 3)
    expected = pandas.Series([numpy.nan, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
    pdtest.assert_series_equal(result, expected)

@pytest.mark.parametrize('parsed', [
    storms_simple(),
    storms_firstobs(),
    storms_singular(),
    storms_with_baseflowcol()
])
def test_number_of_storms(parsed):
    last_storm = parsed['storm'].max()
    assert last_storm == 5

@pytest.mark.parametrize(('parsed', 'sn', 'idx', 'known'), [
    (storms_simple(), 1, 0, '2013-05-18 13:40:00'),
    (storms_simple(), 1, -1, '2013-05-19 01:45:00'),
    (storms_simple(), 2, 0, '2013-05-19 06:10'),
    (storms_simple(), 2, -1, '2013-05-19 11:35'),
    (storms_with_baseflowcol(), 1, 0, '2013-05-18 14:05:00'),
    (storms_with_baseflowcol(), 1, -1, '2013-05-19 01:40:00'),
    (storms_with_baseflowcol(), 2, 0, '2013-05-19 06:35'),
    (storms_with_baseflowcol(), 2, -1, '2013-05-19 11:30'),
    (storms_singular(), 1, 0, '2013-05-18 13:40:00'),
    (storms_singular(), 1, -1, '2013-05-19 01:45:00'),
    (storms_singular(), 2, 0, '2013-05-19 07:00'),
    (storms_singular(), 2, -1, '2013-05-19 07:05'),
    (storms_firstobs(), 1, 0, '2013-05-18 12:05'),
    (storms_firstobs(), 1, -1, '2013-05-19 01:45:00'),
    (storms_firstobs(), 2, 0, '2013-05-19 06:10'),
    (storms_firstobs(), 2, -1, '2013-05-19 11:35'),
])
def test_storm_start_end(parsed, sn, idx, known):
    storm = parsed[parsed['storm'] == sn]
    assert storm.index[idx] == pandas.Timestamp(known)

@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_IMAGES, tolerance=15)
def test_HydroRecord_histogram():
    stormfile = helpers.test_data_path('teststorm_simple.csv')
    orig_record = pandas.read_csv(
        stormfile, index_col='date', parse_dates=True
    ).resample('5T').asfreq().fillna(0)
    hr = hydro.HydroRecord(
        orig_record, precipcol='rain', inflowcol='influent',
        outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
        intereventHours=3
    )

    fig = hr.histogram('Total Precip Depth', [4, 6, 8, 10])
    return fig

class base_HydroRecordMixin(object):
    known_storm_stats_columns = [
        'Storm Number', 'Antecedent Days', 'Season', 'Start Date', 'End Date',
        'Duration Hours', 'Peak Precip Intensity', 'Total Precip Depth',
        'Total Inflow Volume', 'Peak Inflow', 'Total Outflow Volume',
        'Peak Outflow', 'Peak Lag Hours', 'Centroid Lag Hours'
    ]

    def teardown(self):
        pyplot.close('all')

    def test_attributes(self):
        assert hasattr(self.hr, '_raw_data')
        assert isinstance(self.hr._raw_data, pandas.DataFrame)

        assert hasattr(self.hr, 'data')
        assert isinstance(self.hr.data, pandas.DataFrame)
        assert isinstance(self.hr.data.index, pandas.DatetimeIndex)
        for col in self.known_std_columns:
            assert col in self.hr.data.columns.tolist()

        assert hasattr(self.hr, 'all_storms')
        assert isinstance(self.hr.all_storms, dict)

        assert hasattr(self.hr, 'storms')
        assert isinstance(self.hr.storms, dict)

        assert hasattr(self.hr, 'storm_stats')
        assert isinstance(self.hr.storm_stats, pandas.DataFrame)

    def test_check_type_and_columns(self):
        assert isinstance(self.hr.data, pandas.DataFrame)

    def test_check_nan_col(self):
        assert numpy.all(numpy.isnan(self.hr.data['outflow']))

    def test_number_storms(self):
        last_storm = self.hr.data['storm'].max()
        assert last_storm == self.known_number_of_storms

    def test_first_storm_start(self):
        storm1 = self.hr.data[self.hr.data['storm'] == 1]
        assert storm1.index[0] == self.known_storm1_start

    def test_first_storm_end(self):
        storm1 = self.hr.data[self.hr.data['storm'] == 1]
        assert storm1.index[-1] == self.known_storm1_end

    def test_second_storm_start(self):
        storm2 = self.hr.data[self.hr.data['storm'] == 2]
        assert storm2.index[0] == self.known_storm2_start

    def test_second_storm_end(self):
        storm2 = self.hr.data[self.hr.data['storm'] == 2]
        assert storm2.index[-1] == self.known_storm2_end

    def test_interevent_stuff(self):
        assert hasattr(self.hr, 'intereventPeriods')
        assert hasattr(self.hr, 'intereventHours')

        assert self.hr.intereventPeriods == self.known_ie_periods
        assert self.hr.intereventHours == self.known_ie_hours

    def test_storm_stat_columns(self):
        assert self.known_storm_stats_columns == self.hr.storm_stats.columns.tolist()

class Test_HydroRecord_Simple(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_number_of_bigstorms = 2
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')

        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.known_stats_subset = pandas.DataFrame({
            'Storm Number': [1, 5],
            'Antecedent Days': [numpy.nan, 0.708333],
            'Peak Precip Intensity': [1.200, 1.200],
            'Total Precip Depth': [2.76, 4.14],
        })

        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
            intereventHours=self.known_ie_hours
        )

        self.storm_date = pandas.Timestamp('2013-05-19 11:11')
        self.gap_date = pandas.Timestamp('2013-05-19 14:42')
        self.known_storm = 2

    def test_storm_attr_lengths(self):
        assert len(self.hr.storms) == self.known_number_of_bigstorms
        assert len(self.hr.all_storms) == self.known_number_of_storms

    def test_storm_stats(self):

        cols = [
            'Storm Number',
            'Antecedent Days',
            'Peak Precip Intensity',
            'Total Precip Depth',
        ]

        pdtest.assert_frame_equal(self.hr.storm_stats[cols], self.known_stats_subset[cols])

    def test_getStormFromTimestamp_basic(self):
        sn, storm = self.hr.getStormFromTimestamp(self.storm_date)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_string(self):
        datestring = '{}'.format(self.storm_date)
        sn, storm = self.hr.getStormFromTimestamp(datestring)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_datetime(self):
        pydt = self.storm_date.to_pydatetime()
        sn, storm = self.hr.getStormFromTimestamp(pydt)
        assert sn == self.known_storm
        assert storm is None

    def test_getStormFromTimestamp_small(self):
        sn, storm = self.hr.getStormFromTimestamp(self.storm_date, smallstorms=True)
        assert sn == self.known_storm
        assert isinstance(storm, hydro.Storm)

    def test_getStormFromTimestame_big(self):
        sn, storm = self.hr.getStormFromTimestamp(self.known_storm1_start)
        assert sn == 1
        assert isinstance(storm, hydro.Storm)

    def test_getStormFromTimestamp_lookback_6(self):
        sn, storm = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=6)
        assert sn == self.known_storm

    def test_getStormFromTimestamp_lookback_2(self):
        sn, storm = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=2)
        assert sn is None

    def test_getStormFromTimestamp_bad_date(self):
        with helpers.raises(ValueError):
            self.hr.getStormFromTimestamp('junk')

    def test_getStormFromTimestamp_bad_lookback(self):
        with helpers.raises(ValueError):
            sn = self.hr.getStormFromTimestamp(self.gap_date, lookback_hours=-3)

class Test_HydroRecord_Singular(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 07:00')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 07:05')
        self.storm_file = helpers.test_data_path('teststorm_singular.csv')
        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5,
            intereventHours=self.known_ie_hours
        )

class Test_HydroRecord_FirstObservation(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 12:05')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_firstobs.csv')
        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5,
            intereventHours=self.known_ie_hours
        )

class TestHydroRecord_diffStormClass(base_HydroRecordMixin):
    def setup(self):
        self.known_ie_hours = 3
        self.known_ie_periods = 36
        self.known_number_of_storms = 5
        self.known_storm1_start = pandas.Timestamp('2013-05-18 13:40:00')
        self.known_storm1_end = pandas.Timestamp('2013-05-19 01:45:00')
        self.known_storm2_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_storm2_end = pandas.Timestamp('2013-05-19 11:35')
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')

        self.known_std_columns = ['rain', 'influent', 'outflow', 'storm']
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(
            self.orig_record, precipcol='rain', inflowcol='influent',
            outflowcol=None, outputfreqMinutes=5, minprecip=1.5,
            stormclass=fakeStormSublcass,
            intereventHours=self.known_ie_hours
        )

        self.storm_date = pandas.Timestamp('2013-05-19 11:11')
        self.gap_date = pandas.Timestamp('2013-05-19 14:42')
        self.known_storm = 2
        self.known_stats_subset = pandas.DataFrame({
            'Storm Number': [1, 5],
            'Antecedent Days': [-2.409722, 0.708333],
            'Peak Precip Intensity': [1.200, 1.200],
            'Total Precip Depth': [2.76, 4.14],
        })

    def test_subclassed_storms(self):
        for sn in self.hr.all_storms:
            assert self.hr.all_storms[sn].is_subclassed

class Test_Storm(object):
    def setup(self):
        # path stuff
        self.storm_file = helpers.test_data_path('teststorm_simple.csv')
        self.orig_record = pandas.read_csv(
            self.storm_file, index_col='date', parse_dates=True
        ).resample('5T').asfreq().fillna(0)
        self.hr = hydro.HydroRecord(self.orig_record,
                                    precipcol='rain',
                                    inflowcol='influent',
                                    outflowcol='effluent',
                                    outputfreqMinutes=5,
                                    intereventHours=2)
        self.storm = hydro.Storm(self.hr.data, 2,
                                 precipcol=self.hr.precipcol,
                                 inflowcol=self.hr.inflowcol,
                                 outflowcol=self.hr.outflowcol,
                                 freqMinutes=self.hr.outputfreq.n)

        self.known_columns = ['rain', 'influent', 'effluent', 'baseflow', 'storm']
        self.known_index_type = pandas.DatetimeIndex
        self.known_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_end = pandas.Timestamp('2013-05-19 11:55')
        self.known_season = 'spring'
        self.known_precip_start = pandas.Timestamp('2013-05-19 06:10')
        self.known_precip_end = pandas.Timestamp('2013-05-19 09:10')
        self.known_inflow_start = pandas.Timestamp('2013-05-19 06:25')
        self.known_inflow_end = pandas.Timestamp('2013-05-19 11:30')
        self.known_outflow_start = pandas.Timestamp('2013-05-19 06:50')
        self.known_outflow_end = pandas.Timestamp('2013-05-19 11:50')
        self.known_duration_hours = 5.75
        self.known_antecedent_period_days = 0.17708333
        self.known_peak_precip_intensity = 1.32
        self.known_peak_inflow = 123.0
        self.known_peak_outflow = 41.0
        self.known_peak_precip_intensity_time = pandas.Timestamp('2013-05-19 08:00')
        self.known_peak_inflow_time = pandas.Timestamp('2013-05-19 08:35')
        self.known_peak_outflow_time = pandas.Timestamp('2013-05-19 08:45')
        self.known_centroid_precip = pandas.Timestamp('2013-05-19 07:41:58.705023')
        self.known_centroid_inflow = pandas.Timestamp('2013-05-19 08:44:31.252553')
        self.known_centroid_outflow = pandas.Timestamp('2013-05-19 08:59:51.675231')
        self.known_total_precip_depth = 1.39
        self.known_total_inflow_volume = 876600
        self.known_total_outflow_volume = 291900
        self.known_peak_lag_time = 0.1666666667
        self.known_centroid_lag_time = 0.255672966

    def teardown(self):
        pyplot.close('all')
        pass

    def test_columns(self):
        assert sorted(self.known_columns) == sorted(self.storm.data.columns.tolist())

    def test_index_type(self):
        assert isinstance(self.storm.data.index, pandas.DatetimeIndex)

    def test_start(self):
        assert hasattr(self.storm, 'start')
        assert self.storm.start == self.known_start

    def test_end(self):
        assert hasattr(self.storm, 'end')
        assert self.storm.end == self.known_end

    def test_season(self):
        assert hasattr(self.storm, 'season')
        assert self.storm.season == self.known_season

    def test_season_setter(self):
        self.storm.season = 'spring'
        assert self.storm.season == 'spring'

    def test_precip_start(self):
        assert hasattr(self.storm, 'precip_start')
        assert self.storm.precip_start == self.known_precip_start

    def test_precip_end(self):
        assert hasattr(self.storm, 'precip_end')
        assert self.storm.precip_end == self.known_precip_end

    def test_inflow_start(self):
        assert hasattr(self.storm, 'inflow_start')
        assert self.storm.inflow_start == self.known_inflow_start

    def test_inflow_end(self):
        assert hasattr(self.storm, 'inflow_end')
        assert self.storm.inflow_end == self.known_inflow_end

    def test_outflow_start(self):
        assert hasattr(self.storm, 'outflow_start')
        assert self.storm.outflow_start == self.known_outflow_start

    def test_outflow_end(self):
        assert hasattr(self.storm, 'outflow_end')
        assert self.storm.outflow_end == self.known_outflow_end

    def test_duration_hours(self):
        assert hasattr(self.storm, 'duration_hours')
        assert abs(self.storm.duration_hours - self.known_duration_hours) < 0.0001

    def test_antecedent_period_days(self):
        assert hasattr(self.storm, 'antecedent_period_days')
        assert abs(self.storm.antecedent_period_days - self.known_antecedent_period_days) < 0.0001

    def test_peak_precip_intensity(self):
        assert hasattr(self.storm, 'peak_precip_intensity')
        assert abs(self.storm.peak_precip_intensity - self.known_peak_precip_intensity) < 0.0001

    def test_peak_inflow(self):
        assert hasattr(self.storm, 'peak_inflow')
        assert abs(self.storm.peak_inflow - self.known_peak_inflow) < 0.0001

    def test_peak_outflow(self):
        assert hasattr(self.storm, 'peak_outflow')
        assert abs(self.storm.peak_outflow - self.known_peak_outflow) < 0.0001

    def test_peak_precip_intensity_time(self):
        assert hasattr(self.storm, 'peak_precip_intensity_time')
        assert self.storm.peak_precip_intensity_time == self.known_peak_precip_intensity_time

    def test_peak_inflow_time(self):
        assert hasattr(self.storm, 'peak_inflow_time')
        assert self.storm.peak_inflow_time == self.known_peak_inflow_time

    def test_peak_outflow_time(self):
        assert hasattr(self.storm, 'peak_outflow_time')
        assert self.storm.peak_outflow_time == self.known_peak_outflow_time

    def test_centroid_precip(self):
        assert hasattr(self.storm, 'centroid_precip_time')
        assert self.storm.centroid_precip_time.strftime('%x %X') == self.known_centroid_precip.strftime('%x %X')

    def test_centroid_inflow(self):
        assert hasattr(self.storm, 'centroid_inflow_time')
        assert self.storm.centroid_inflow_time.strftime('%x %X') == self.known_centroid_inflow.strftime('%x %X')

    def test_centroid_outflow(self):
        assert hasattr(self.storm, 'centroid_outflow_time')
        assert self.storm.centroid_outflow_time.strftime('%x %X') == self.known_centroid_outflow.strftime('%x %X')

    def test_total_precip_depth(self):
        assert hasattr(self.storm, 'total_precip_depth')
        assert abs(self.storm.total_precip_depth - self.known_total_precip_depth) < 0.0001

    def test_total_inflow_volume(self):
        assert hasattr(self.storm, 'total_inflow_volume')
        assert abs(self.storm.total_inflow_volume - self.known_total_inflow_volume) < 0.0001

    def test_total_outflow_volume(self):
        assert hasattr(self.storm, 'total_outflow_volume')
        assert abs(self.storm.total_outflow_volume - self.known_total_outflow_volume) < 0.0001

    def test_peak_lag_hours(self):
        assert hasattr(self.storm, 'peak_lag_hours')
        assert abs(self.storm.peak_lag_hours - self.known_peak_lag_time) < 0.0001

    def test_centroid_lag_hours(self):
        assert hasattr(self.storm, 'centroid_lag_hours')
        assert abs(self.storm.centroid_lag_hours - self.known_centroid_lag_time) < 0.0001

    def test_summary_dict(self):
        assert hasattr(self.storm, 'summary_dict')
        assert isinstance(self.storm.summary_dict, dict)
        known_keys = [
            'Storm Number',
            'Antecedent Days',
            'Start Date',
            'End Date',
            'Duration Hours',
            'Peak Precip Intensity',
            'Total Precip Depth',
            'Total Inflow Volume',
            'Peak Inflow',
            'Total Outflow Volume',
            'Peak Outflow',
            'Peak Lag Hours',
            'Centroid Lag Hours',
            'Season'
        ]
        keys = list(self.storm.summary_dict.keys())
        assert sorted(keys) == sorted(known_keys)

    def test_is_small(self):
        assert self.storm.is_small(minprecip=5.0)
        assert not self.storm.is_small(minprecip=1.0)

def setup_storm():
    pyplot.rcdefaults()
    storm_file = helpers.test_data_path('teststorm_simple.csv')
    orig_record = (
        pandas.read_csv(storm_file, index_col='date', parse_dates=True)
              .resample('5T')
              .asfreq()
              .fillna(0)
    )

    hr = hydro.HydroRecord(
        orig_record, precipcol='rain', inflowcol='influent',
        outflowcol=None, outputfreqMinutes=5, minprecip=0,
        intereventHours=3
    )

    return hr.storms[2]

@pytest.mark.mpl_image_compare(baseline_dir=BASELINE_IMAGES, tolerance=15)
def test_plot_storm_summary():
    storm = setup_storm()

    def doplot(storm):
        labels = {
            storm.inflowcol: 'Effluent (l/s)',
            storm.precipcol: 'Precip Depth (mm)'

        }
        fig, artists, labels = storm.summaryPlot(outflow=False, serieslabels=labels)
        return fig

    fig = doplot(storm)
    return fig

@pytest.fixture
def drainage_area():
    return hydro.DrainageArea(100, 75, 10)

def test_da_total_area(drainage_area):
    assert drainage_area.total_area == 100

def test_da_imp_area(drainage_area):
    assert drainage_area.imp_area == 75

def test_da_bmp_area(drainage_area):
    assert drainage_area.bmp_area == 10

@pytest.mark.parametrize(('conversion', 'factor', 'expected'), [
    (1.0, 1.0, 82.50),
    (1.0, 0.9, 75.25),
    (2.0, 1.0, 165.00),
    (2.0, 0.9, 150.50)
])
def test_simple_method(drainage_area, conversion, factor, expected):
    depth = 1
    result = drainage_area.simple_method(depth, annual_factor=factor,
                                         volume_conversion=conversion)
    area = (drainage_area.total_area + drainage_area.bmp_area)
    storm_volume = depth * conversion * factor * area
    assert abs(expected - result) < 0.001
    assert storm_volume > result