import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
#from IPython.display import clear_output
import numpy as np
import itertools
from dolfin import *

# Create classes for defining parts of the boundaries
class Left(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[0], 0.0)

class Right(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[0], 1.0)

class Bottom(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[1], 0.0)

class Top(SubDomain):
    def inside(self, x, on_boundary):
        return near(x[1], 1.0)

# Initialize sub-domain instances
left = Left()
top = Top()
right = Right()
bottom = Bottom()

nx = 64
ny = 64

# Define mesh
mesh = UnitSquareMesh(nx, ny)

t = 0.0 # initial time
Time = 1.0         # final time
num_steps = 20     # number of time steps
dt = Time / num_steps # time step size

# Initialize mesh function for boundary domains
boundaries = MeshFunction("size_t", mesh, mesh.topology().dim() - 1)
boundaries.set_all(0)
left.mark(boundaries, 1)
top.mark(boundaries, 2)
right.mark(boundaries, 3)
bottom.mark(boundaries, 4)

# Define input data
a0 = Constant(1.0)
a1 = Constant(0.1)
g = Constant("0.0001")
f = Constant("4.")

# Define function space and basis functions
V = FunctionSpace(mesh, "CG", 2)
u = TrialFunction(V)
v = TestFunction(V)

T_d = Expression('sin(pi*x[1])*exp( -pi*pi*t )*pow(sin(pi*x[0]) , 3)', degree=2, t=0)
T_old = interpolate(T_d, V)

# Define new measures associated with the exterior boundaries
ds = ds(subdomain_data = boundaries)

# Define variational form
F = (u*v*dx - T_old*v*dx
    + dt*inner(a1*grad(u), grad(v))*dx
    - dt*f*v*dx - dt*g*v*ds(1) - dt*g*v*ds(2) - dt*g*v*ds(3) - dt*g*v*ds(4)
    )

# Separate left and right hand sides of equation
a, L = lhs(F), rhs(F)
u = Function(V)

# Solve problem
for n in range(num_steps):

    # Update current time
    t += dt
    # Compute solution
    solve(a == L, u)

    #3D Plot of the Solution
    # matplotlib inline
    x = np.linspace(0.0, 1.0, nx + 1)
    y = np.linspace(0.0, 1.0, ny + 1)

    X, Y = np.meshgrid(x, y)

    vertex_values_u = u.compute_vertex_values()
    Z_vert = vertex_values_u.reshape((nx + 1,ny + 1 ))

    fig = plt.figure()
    ax = plt.axes(projection='3d')

    ax.plot_surface(X, Y, Z_vert, rstride=1, cstride=1,
                cmap='viridis', edgecolor='none')
    #ax.set_title('t = ' +str(round(t, 2)))
    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('T')
    #ax.set_zlim3d(-1.0 , 1.0)

    plt.show()

    print('t = %.2f' % (t))

    clear_output(wait=True)

    T_old.assign(u)


import json
from typing import Any, Dict

import numpy as np
from py4j.java_gateway import get_field

class Sample:
    def __init__(self, sample, java_sample):
        serialized_sample = json.loads(sample.exportToJson())

        self.id = serialized_sample["id"]
        self.target = json.loads(serialized_sample["target"])
        self.neurons = self.deserialize_network(serialized_sample["network"])
        self.output_neuron = self.neurons[-1].index
        self.java_sample = java_sample

    def deserialize_network(self, network):
        neurons = []

        for i, neuron in enumerate(network):
            neuron_object = Neuron(neuron, i)
            neurons.append(neuron_object)

        return neurons

class Neuron:
    def __init__(self, neuron: Dict[str, Any], index):
        self.index = index
        self.name = neuron["name"]
        self.weighted = neuron["weighted"]
        self.activation = neuron.get("activation", None)
        self.inputs = neuron["inputs"]
        self.weights = neuron.get("weights", None)
        self.offset = neuron["offset"]
        self.value = neuron.get("value", None)
        self.pooling = neuron["pooling"]
        self.hook_name = Neuron.parse_hook_name(self.name)

        if self.value:
            self.value = json.loads(self.value)

        if self.weights is not None:
            self.weights = list(self.weights)

        if self.inputs is not None:
            self.inputs = list(self.inputs)

    @staticmethod
    def parse_hook_name(name: str):
        name = name.split(" ")

        if len(name) == 3:
            return name[2]
        return None

class Weight:
    def __init__(self, weight):
        self.index: int = get_field(weight, "index")
        self.name = get_field(weight, "name")
        self.dimensions = tuple(get_field(weight, "value").size())
        self.value = json.loads(get_field(weight, "value").toString())
        self.fixed = get_field(weight, "isFixed")

        if not isinstance(self.value, list):
            self.value = self.value

        if not self.dimensions:
            self.dimensions = (1,)

        if self.fixed:
            self.value = np.array(self.value).reshape(self.dimensions)

    @staticmethod
    def get_unit_weight() -> "Weight":
        weight = Weight.__new__(Weight)
        weight.index = 0
        weight.name = "unit"
        weight.dimensions = (1,)
        weight.value = 1.0
        weight.fixed = True

        return weight

class BuiltDataset:
    """BuiltDataset represents an already built dataset - that is, a dataset that has been grounded and neuralized."""

    def __init__(self, samples):
        self.samples = samples

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, item):
        return self.samples[item]


#!/usr/bin/env python
import wx

class MyFrame(wx.Frame):
    """We simply derive a new class of Frame"""
    def __init__(self,parent,title):
        wx.Frame.__init__(self,parent,title=title,size=(200,100))
        self.control = wx.TextCtrl(self,style=wx.TE_MULTILINE)
        self.CreateStatusBar()

        filemenu = wx.Menu()

        filemenu.Append(wx.ID_ABOUT,'&About','Information about program')
        filemenu.AppendSeparator()
        filemenu.Append(wx.ID_EXIT,'E&xit','Terminate the programe')

        menuBar = wx.MenuBar()
        menuBar.Append(filemenu,'&File')
        self.SetMenuBar(menuBar)

        self.Show(True)
app = wx.App(False)
frame = MyFrame(None,'Small editor')
app.MainLoop()


import numpy as np
import PIL.Image, PIL.ImageDraw, PIL.ImageFilter, PIL.ImageFont, PIL.ImageOps
from psychopy import visual
from .delboeuf import _delboeuf_parameters_sizeinner, _delboeuf_parameters_sizeouter
from ..image import image_circle
from ..image.utilities import _coord_circle

def ebbinghaus_psychopy(window, parameters=None, **kwargs):
    """
    Examples
    ---------
    >>> import pyllusion as ill
    >>> from psychopy import visual, event

    >>> parameters = ill.ebbinghaus_parameters(difficulty=2, illusion_strength=1)

    >>> # Initiate Window
    >>> window = visual.Window(size=[800, 600], fullscr=False,
                               screen=0, winType='pyglet', monitor='testMonitor',
                               allowGUI=False, color="white",
                               blendMode='avg', units='pix')

    >>> # Display illusion
    >>> ill.ebbinghaus_psychopy(window=window, parameters=parameters)

    >>> # Refresh and close window
    >>> window.flip()
    >>> event.waitKeys()  # Press any key to close
    >>> window.close()
    """

    # Create white canvas and get drawing context
    if parameters is None:
        parameters = ebbinghaus_parameters(**kwargs)

    # Loop circles
    for side in ["Left", "Right"]:
        # Draw inner circle
        size_inner = parameters["Size_Inner_" + side]
        radius_inner, x_inner, y_inner = _coord_circle(image=window, diameter=size_inner,
                                         x=parameters["Position_" + side],
                                         y=0, method="psychopy")

        circle_inner = visual.Circle(win=window, units="pix", fillColor="red",
                                     lineColor="red", edges=128,
                                     radius=radius_inner, lineWidth=0.5)
        circle_inner.pos = [x_inner-window.size[0]/2, y_inner-window.size[1]/2]
        circle_inner.draw()

        # outer circle
        ratio = window.size[0] / window.size[1]  # Get width/height ratio to have equidistant circles
        size_outer = parameters["Size_Outer_" + side]
        x_outer = parameters["Position_Outer_x_" + side] / ratio # Adjust for non-squared screen
        x_outer = x_outer + (parameters["Position_" + side] - np.mean(x_outer))
        for i in range(len(parameters["Position_Outer_x_" + side])):
            radius_outer, x_out, y_out = _coord_circle(image=window, diameter=size_outer,
                                                       x=x_outer[i],
                                                       y=parameters["Position_Outer_y_" + side][i],
                                                       method="psychopy")
            circle_outer = visual.Circle(win=window, units="pix",
                                         fillColor="black", lineColor="black",
                                         edges=128, radius=radius_outer, lineWidth=0.5)
            circle_outer.pos = [x_out-window.size[0]/2, y_out-window.size[1]/2]
            circle_outer.draw()

def ebbinghaus_image(
    parameters=None, width=800, height=600, background="white", **kwargs
):
    """Create the Ebbinghaus illusion.
    The Ebbinghaus illusion is an optical illusion of relative size perception,
    where circles of identical size appear as different because of their
    surrounding context.

    Examples
    ---------
    >>> import pyllusion as ill
    >>>
    >>> parameters = ill.ebbinghaus_parameters(difficulty=2, illusion_strength=1)
    >>> ill.ebbinghaus_image(parameters)  #doctest: +ELLIPSIS
    <PIL.Image.Image ...>
    """
    # Create white canvas and get drawing context
    if parameters is None:
        parameters = ebbinghaus_parameters(**kwargs)

    # Background
    image = PIL.Image.new("RGB", (width, height), color=background)

    # Outer circles
    for side in ["Left", "Right"]:
        image = _ebbinghaus_image_draw(image,
                                       parameters,
                                       side=side,
                                       color_inner="red",
                                       color_outer="black")

    return image

def _ebbinghaus_image_draw(image, p, side="Left", color_inner="red", color_outer="black"):

    # Draw inner circle
    image = image_circle(image=image, size=p["Size_Inner_" + side], x=p["Position_" + side], y=0, color=color_inner)

    # Get width/height ratio to have equidistant circles
    ratio = image.size[0] / image.size[1]

    # Adjust for non-squared screen
    x = p["Position_Outer_x_" + side] / ratio
    x = x + (p["Position_" + side] - np.mean(x))

    # Plot each outer circles
    for i in range(len(p["Position_Outer_x_" + side])):
        image = image_circle(image=image, size=p["Size_Outer_" + side], x=x[i], y=p["Position_Outer_y_" + side][i], color=color_outer)

    return image

# ------------------------------------------
# Parameters
# ------------------------------------------

def ebbinghaus_parameters(difficulty=0, size_min=0.25, illusion_strength=0, distance=1, distance_auto=False):
    # Size inner circles
    parameters = _delboeuf_parameters_sizeinner(difficulty=difficulty, size_min=size_min)
    inner_size_left = parameters["Size_Inner_Left"]
    inner_size_right = parameters["Size_Inner_Right"]

    # Position
    position_left = -0.5
    position_right = 0.5

    # Base size outer circles
    outer_size_left = size_min
    outer_size_right = size_min

    # Actual outer size based on illusion
    outer_size_left, outer_size_right = _delboeuf_parameters_sizeouter(outer_size_left,
                                                                       outer_size_right,
                                                                       difficulty=difficulty,
                                                                       illusion_strength=illusion_strength,
                                                                       both_sizes=True)

    # Location outer circles
    l_outer_x, l_outer_y, l_distance_edges = _ebbinghaus_parameters_outercircles(x=position_left,
                                                            y=0,
                                                            size_inner=inner_size_left,
                                                            size_outer=outer_size_left,
                                                            n="auto")
    r_outer_x, r_outer_y, r_distance_edges = _ebbinghaus_parameters_outercircles(x=position_right,
                                                            y=0,
                                                            size_inner=inner_size_right,
                                                            size_outer=outer_size_right,
                                                            n="auto")

    # Get location and distances
    if distance_auto is False:
        distance_centers = distance
        position_left, position_right = -(distance_centers / 2), (distance_centers / 2)
        distance_edges_inner = distance_centers - (inner_size_left/2 + inner_size_right/2)
        distance_edges_outer = distance_centers - l_distance_edges - (outer_size_left/2) - r_distance_edges - (outer_size_right/2)

    else:
        distance_edges_outer = distance
        distance_centers = distance_edges_outer + l_distance_edges + (outer_size_left/2) + r_distance_edges + (outer_size_right/2)
        distance_edges_inner = distance_centers - (outer_size_left/2 + outer_size_right/2)
        position_left, position_right = -(distance_centers / 2), (distance_centers / 2)

    parameters.update({
        "Illusion": "Ebbinhaus",
        "Illusion_Strength": illusion_strength,
        "Illusion_Type": "Congruent" if illusion_strength > 0 else "Incongruent",

        "Size_Outer_Left": outer_size_left,
        "Size_Outer_Right": outer_size_right,

        "Distance_Centers": distance_centers,
        "Distance_Edges_Inner": distance_edges_inner,
        "Distance_Edges_Outer": distance_edges_outer,

        "Size_Inner_Smaller": np.min([inner_size_left, inner_size_right]),
        "Size_Inner_Larger": np.max([inner_size_left, inner_size_right]),
        "Size_Outer_Smaller": np.min([outer_size_left, outer_size_right]),
        "Size_Outer_Larger": np.max([outer_size_left, outer_size_right]),

        "Position_Outer_x_Left": l_outer_x,
        "Position_Outer_y_Left": l_outer_y,
        "Position_Outer_x_Right": r_outer_x,
        "Position_Outer_y_Right": r_outer_y,

        "Position_Left": position_left,
        "Position_Right": position_right
        })

    return parameters

def _ebbinghaus_parameters_outercircles(x=0, y=0, size_inner=0.25, size_outer=0.3, n="auto"):
    # Find distance between center of inner circle and centers of outer circles
    distance = (size_inner / 2) + (size_outer / 2) + 0.01

    # Find n
    if n == "auto":
        perimeter = 2 * np.pi * distance
        n = np.int(perimeter / size_outer)

    # Get position of outer circles
    angle = np.deg2rad(np.linspace(0, 360, num=n, endpoint=False))
    circle_x = x + (np.cos(angle) * distance)
    circle_y = y + (np.sin(angle) * distance)

    return circle_x, circle_y, distance


# -----------------------------------
#   USAGE
# -----------------------------------
# python cnn_face_detection.py --image images/concert.jpg

# -----------------------------
#   IMPORTS
# -----------------------------
# Import the necessary packages
from pyimagesearch.helpers import convert_and_trim_bb
import argparse
import imutils
import time
import dlib
import cv2

# Construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", type=str, required=True, help="Path to the input image file")
ap.add_argument("-m", "--model", type=str, default="mmod_human_face_detector.dat",
                help="Path to dlib's CNN face detector model file")
ap.add_argument("-u", "--upsample", type=int, default=1, help="Number of times to upsample")
args = vars(ap.parse_args())

# Load DLIB'S CNN Face Detector model
print("[INFO] Loading CNN face detection model...")
detector = dlib.cnn_face_detection_model_v1(args["model"])

# Load the input image from disk, resize it and then convert it from BGR to RGB channel ordering
# (which is what the dlib expects)
image = cv2.imread(args["image"])
image = imutils.resize(image, width=600)
rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

# Perform face detection using dlib's face detector
start = time.time()
print("[INFO] Performing face detection with dlib face detector...")
results = detector(rgb, args["upsample"])
end = time.time()
print("[INFO] Face detection took {:.4f} seconds".format(end - start))

# Convert the resulting dlib rectangle objects to bounding boxes, then ensure that the bounding boxes are all within
# the bounds of the input image
boxes = [convert_and_trim_bb(image, r.rect) for r in results]

# Loop over the bounding boxes
for (x, y, w, h) in boxes:
    # Draw the bounding box on the image
    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)

# Show the output image
cv2.imshow("Output", image)
cv2.waitKey(0)


#!/usr/bin/env python
#  -*- mode: python; indent-tabs-mode: nil; -*- coding: utf-8 -*-

"""
RateLimiter.py

Copyright 2010-2014 by Marcello Perathoner

Distributable under the GNU General Public License Version 3 or newer.

A very basic IP rate limiter tool for cherrypy.

begin;
drop table robots.blocks;
create table robots.blocks (
   ip             cidr      primary key,
   host           text      default null,
   has_info       boolean   default false,
   is_blocked     boolean   default true,
   is_whitelisted boolean   default false,
   created        timestamp default now (),
   expires        timestamp default null,
   types          text[]    default null,
   org            text      default null,
   country        text      default null,
   note           text      default null,
   whois          text      default null,
   user_agents    text[]    default null,
   requests       text[]    default null
);

"""

from __future__ import unicode_literals

import datetime
import re
import logging
import threading
import random
import collections
import itertools

import cherrypy
import requests
import netaddr

from libgutenberg.GutenbergDatabase import DatabaseError
import BaseSearcher
import Page
import asyncdns
import ipinfo

# no. of requests to keep in LIFO queue for each IP
KEEP_REQUESTS = 10

RE_SPOOFED_BOTS = re.compile (
    'Googlebot|msnbot|bingbot|Yahoo! Slurp|YandexBot|Baiduspider|ia_archiver')

TRUSTED_XFF_URL = "https://meta.wikimedia.org/w/extensions/TrustedXFF/trusted-hosts.txt"

def log (ip, msg, context = 'RATELIMITER', severity = logging.INFO):
    """ Log with IP. """

    a = []
    if ip:
        a.append (str (ip))
    if msg:
        a.append (str (msg))

    cherrypy.log (' '.join (a), context = context, severity = severity)

def to_cidr (ip):
    """ Build cidr out of ip. """

    if ip and ip != 'unknown':
        if ':' in ip:
            return str (ip) + '/128'
        return str (ip) + '/32'
    return None

# See: https://docs.python.org/3/library/itertools.html
def unique_everseen (iterable):
    "List unique elements, preserving order. Remember all elements ever seen."
    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
    # unique_everseen('ABBCcAD', str.lower) --> A B C D
    seen = set ()
    seen_add = seen.add
    for element in itertools.filterfalse (seen.__contains__, iterable):
        seen_add (element)
        yield element

class IPStruct (object):
    """ A container for a client and proxy address. """

    def __init__ (self):
                                    # ips are always of type string
        self.client_ip = None       # ip of client (or of untrusted proxy)
        self.proxy_ip = None        # ip of proxy or None
        self.proxy_type = None      # None, 'anonymous', 'untrusted', 'trusted', 'private'
        self.whitelisted = False    # if whitelisted ip (eg. Googlebot)
        self.ipinfo = None          # more (dns, whois) info about this ip

    def from_request (self, request, rl):
        """ Init from remote IP and X-Forwarded-For header. """

        xffh = request.headers.get ('X-Forwarded-For') or "%s, %s" % (
            cherrypy.request.remote.ip, cherrypy.request.remote.ip)

        log ('X-FORWARDED-FOR', xffh)

        # split string header into array of IPs
        xff = [x.strip () for x in xffh.split (',')]

        # strip ibiblio haproxy IP
        xff = [x for x in xff if not x.startswith ("2610:28:3091:3001:")]

        xff = list (reversed (list (unique_everseen (xff))))
        if not xff:
            xff = [cherrypy.request.remote.ip]

        try:
            nip = netaddr.ip.IPAddress (xff[0])
        except netaddr.AddrFormatError:
            raise cherrypy.HTTPError (400, 'Bad Request. Invalid IP: %s.' % xff[0])

        self.whitelisted = nip in rl.whitelist

        # the request reaches the app server either this way:
        #
        #   user_proxy* haproxy varnish mod_proxy app_server
        #
        # or this way:
        #
        #   user_proxy* haproxy varnish php_curl app_server
        #
        # so the layout of xff is either:
        #
        # xff[0] address of user    (added by curl)
        # xff[1] address of haproxy (added by varnish)
        # xff[2] address of user    (added by haproxy)
        # xff[3] address of user    (added by user_proxy)
        #
        # or
        #
        # xff[0] address of user    (added by mod_proxy)
        # xff[1] address of haproxy (added by varnish)
        # xff[2] address of user    (added by haproxy)
        # xff[3] address of user    (added by user_proxy)

        self.client_ip = xff[0]
        offset = 1
        if len (xff) >= 2 and xff[0] == xff[1]:
            # skip identic ip
            offset = 2
        if len (xff) > offset:
            # some kind of proxy detected
            self.client_ip = xff[offset]
            self.proxy_ip  = xff[0]

            if self.client_ip == 'unknown':
                self.proxy_type = 'anonymous'
            else:
                try:
                    if netaddr.ip.IPAddress (self.client_ip).is_private ():
                        self.proxy_type = 'private'
                except netaddr.AddrFormatError:
                    raise cherrypy.HTTPError (400, 'Bad Request. Invalid IP: %s.' % self.client_ip)

            if self.proxy_type is None:
                # Check if this request came through a trusted proxy.
                # This check is expensive, so do it only if needed.
                if nip in rl.trusted_xff_list:
                    self.proxy_type = 'trusted'
                else:
                    self.proxy_type = 'untrusted'

    def get_cache_key (self):
        """ Make a key suitable for caching an ipsession.

        This key will distinguish users.

        """
        if self.proxy_type == 'trusted':
            return "%s-%s" % (self.proxy_ip, self.client_ip)
        else:
            return self.proxy_ip or self.client_ip

    def get_ip_to_block (self):
        """ Get the IP to block.

        Because we use apache 'deny from' to block, we need
        the proxy address, not the address behind the proxy.

        """
        return self.proxy_ip or self.client_ip

    def sort_key (self):
        """ Return a key for sorting of IPStructs. """
        return netaddr.ip.IPAddress (self.get_ip_to_block ()).sort_key ()

    def __str__ (self):
        """ Return a human-readable representation. """
        if self.proxy_ip:
            return "%s via %s (%s)" % (self.client_ip, self.proxy_ip, self.proxy_type)
        return self.client_ip

class RatelimiterSession (dict):
    """ A session maintained for every remote IP. """

    def __init__ (self, rl, ips):
        super (RatelimiterSession, self).__init__ ()

        self.ips = ips
        self.sessions = set ()
        now = datetime.datetime.now ()

        # pylint: disable=too-many-function-args
        self.update (
            {
                'blocked':     0,                  # 1 = temp, 2 = perm, 9 = written to db
                'expires':     now,                # expiration of this session
                'last_hit':    now,                # time of last hit
                'hits':        0,                  # all hits
                'rhits':       0,                  # ip score
                'rhits_max':   rl.rhits_max,       # next captcha appears at this score
                'captchas':    0,                  # no. of captchas presented
                'rearms':      0,                  # how many times rearmed (thru captcha)
                'dhits':       0,                  # denied hits
                'user_agents': set (),             # all user agents seen from this ip
                'signatures':  set (),             # all http signatures seen from this ip
                'requests':    collections.deque ([], KEEP_REQUESTS), # last n requests
                'referrers':   collections.deque ([], KEEP_REQUESTS), # last n referrers
                'categories':  collections.deque ([], KEEP_REQUESTS), # last n categories
                'headers':     [],                 # request headers
                'active':      True,               # for stats page
                'css_ok':      None,               # flag for stats page
                'expire_days': 1,                  # expiration of block
            }
        )

        self.set_expiration (rl.expiration)

    def append_trace (self, rl, ua, signature):
        """ Append trace of last X requests. """

        headers = cherrypy.request.headers

        session = cherrypy.session

        ipset = session.get ('ips', set ())
        ipset.add (self.ips.client_ip)
        session['ips'] = ipset

        uaset = session.get ('user_agents', set ())
        uaset.add (ua)
        session['user_agents'] = uaset

        self['signatures'].add (signature)

        request_str = headers.get ('X-Request')
        if request_str is None:
            request_str = cherrypy.request.path_info
            if cherrypy.request.query_string:
                request_str += '?' + cherrypy.request.query_string
        request_str = request_str[:100]
        self['requests'].append (request_str)
        self['referrers'].append (headers.get ('Referer'))

        # categorize request
        for cat in rl.categories:
            if request_str.startswith (cat):
                self['categories'].append (cat)
                break

    def set_add_with_trigger (self, set_, value, trigger_size):
        """ Insert value into set and trigger on size. """

        if value in set_:
            return False
        set_.add (value)
        return len (set_) == trigger_size

    def calc_block (self, rl, ua):
        """ Calculate the penalties for this hit.

        Penalties make rhits grow faster, thus captchas come earlier.

        """

        headers = cherrypy.request.headers
        session = cherrypy.session
        path = cherrypy.request.path_info
        now = datetime.datetime.now ()

        self['rhits'] += 1

        # penalties
        if self['last_hit'] + datetime.timedelta (seconds = rl.min_interval) > now:
            self['rhits'] += rl.rate_malus

        # new session?
        if 'captchas' not in session:
            self['rhits'] += rl.session_malus
            session['captchas'] = 0
        elif session['captchas'] == 0:
            self['rhits'] += rl.no_captcha_malus

        if ('.opds' not in path) and ('.stanza' not in path):
            if self.has_few ('referrers'):
                self['rhits'] += rl.referrer_malus

            if self.has_few ('categories'):
                self['rhits'] += rl.category_malus

            if 'Accept-Encoding' not in headers:
                self['rhits'] += rl.encoding_malus

            if self['css_ok'] == False: # 3-state logic
                self['rhits'] += rl.css_malus

        penalty = headers.get ('X-Penalize-Me')
        if penalty is not None:
            self['rhits'] += max (int (penalty), 1)

        if 'X-Error-404' in headers:
            self['rhits'] += rl.e404_malus

        # soft block to captcha
        if self['blocked'] == 0:
            if self['rhits'] >= self['rhits_max']:
                self['blocked'] = 1

            if self.set_add_with_trigger (self['user_agents'], ua, 5):
                if len (self['signatures']) <= 3:
                    self.log ('Captcha on too few http header signatures')
                    self['blocked'] = 1

        if self['blocked'] == 1:
            if self['rearms'] > rl.rearms_max:
                self.log ('Blocked on rearms_max')
                self['blocked'] = 2

        # hard block
        if self['blocked'] < 2:
            if self['captchas'] > rl.captchas_max:
                self.log ('Blocked on captchas_max')
                self['blocked'] = 2

            penalty = headers.get ('X-Block-Me')
            if penalty is not None:
                self.log ('Blocked on X-Block-Me')
                self['blocked'] = 2
                self['expire_days'] = max (int (penalty), 1)

            if len (session['user_agents']) > rl.ua_limit:
                # spoofed random user-agents
                self.log ('Blocked on session ua_limit')
                self['blocked'] = 2

            if len (session['ips']) > rl.ip_limit:
                # session from multiple ips (probably proxies)
                self.log ('Blocked on session ip_limit')
                self['blocked'] = 2

            if (ua is not None) and (RE_SPOOFED_BOTS.search (ua) is not None):
                # spoofed user-agent because not from whitelisted ip
                self.log ('Blocked on spoofed user agent')
                self['blocked'] = 2

            # if self.ips.proxy_type == 'anonymous':
            #     self.log ('Blocked on anon_proxy')
            #     self['blocked'] = 2

    def extend_expiration (self, timeout):
        """ Extend expiration time to now + timeout. """

        new_expires = datetime.datetime.now () + datetime.timedelta (seconds = timeout)
        self['expires'] = max (new_expires, self['expires'])

    def set_expiration (self, timeout):
        """ Set expiration time to now + timeout. """

        new_expires = datetime.datetime.now () + datetime.timedelta (seconds = timeout)
        self['expires'] = new_expires

    def has_few (self, what):
        """ IPs that always use the same referrer or always hit the
        same category are most likely bots. """

        return self['hits'] > KEEP_REQUESTS and len (set (list (self[what]))) < 2

    def get_tags (self):
        """ Get all tags from ipinfo and add oir own. """
        tags = list (self.ips.ipinfo.tags)
        if self['css_ok'] == False: # 3-state
            tags.append ('NO_CSS')
        return sorted (tags)

    def captcha_answer (self, response):
        """ Check answer to captcha. """

        if response.is_valid:
            # session.captchas NOT ipsession.captchas
            captchas = cherrypy.session.get ('captchas', 0)
            cherrypy.session['captchas'] = captchas + 1
            self.rearm ()
            self.log ('Correct answer', 'CAPTCHA')
        else:
            self.log ('Wrong answer', 'CAPTCHA')

    def rearm (self):
        """ Rest counters. """

        rl = cherrypy.tools.rate_limiter # pylint: disable=E1101

        if self['blocked'] < 2 and self['rearms'] <= rl.rearms_max:
            self.log ('Rearmed')
            self['rearms'] += 1
            self['blocked'] = 0
            self['captchas'] = 0
            # recharge, but don't add up multiple recharges
            self['rhits_max'] = min (self['rhits'] + rl.rhits_max,
                                     self['rhits_max'] + rl.rhits_max)
            self.set_expiration (rl.expiration)

    def block_me (self):
        """ Immediately block this IP. """

        self.log ('Blocked on block_me')
        self['blocked'] = 2

    def unblock_me (self):
        """ Unblock this IP. """

        self.rearm ()

    def log (self, msg, context = 'RATELIMITER'):
        """ Log with IP. """

        log (str (self.ips), str (msg), context)

    def log_state (self):
        """ Log the session state. """

        tup = (
            self['blocked'],
            self['hits'],
            self['rhits'],
            self['rhits_max'],
            self['requests'][-1],
            )
        self.log ("%d %d %d/%d %s" % tup)

        if 'is-catalog-maintainer' in cherrypy.request.cookie:
            cherrypy.response.headers['X-Ratelimiter-Info'] = "b:%d h:%d %d/%d %s" % tup

class RateLimiterTool (cherrypy.Tool):
    """ Limit rate of access per IP. """

    lock = threading.Lock ()
    dblock = threading.Lock () # database accesses
    cache = {}

    # http header fields that should change if user-agent changes,
    # but bots that spoof user-agent won't bother to randomize

    http_headers_no_random = set (
        """Accept Accept-Language Accept-Charset Accept-Encoding
        Cache-Control Connection Dnt Keep-Alive Pragma"""
        .split ())

    http_headers_ignore = set (
        """X-Forwarded-For X-Forwarded-Server X-Forwarded-Host
        X-Request X-Varnish X-Block-Me X-Penalize-Me Remote-Addr
        Cookie"""
        .split ())

    def __init__ (self):
        """ Initialize the Rate Limiter.

        Needs to be on before_request_body on priority > 30 because
        tools.proxy (on prio 30) needs a chance to rewrite the IP.

        """

        super (RateLimiterTool, self).__init__ ('before_request_body', None)
        self.callable = self.limit
        self._priority = 60

        conf = cherrypy.config.get

        # ipsession expiration in seconds
        self.expiration       = conf ('tools.rate_limiter.expiration',      30 * 60)

        # cleanup thread frequency
        self.frequency        = conf ('tools.rate_limiter.frequency',       60)

        # when rhits are reached a captcha must be solved
        self.rhits_max        = conf ('tools.rate_limiter.rhits_max',       500)

        # this many captchas are presented before hard block
        self.captchas_max     = conf ('tools.rate_limiter.captchas_max',    10)

        # how many times rearm thru captcha works
        self.rearms_max       = conf ('tools.rate_limiter.rearms_max',      20)

        # more than this many user agents on *session* will get ip blocked
        self.ua_limit         = conf ('tools.rate_limiter.ua_limit',         4)

        # more than this many ips on *session* will get ip blocked
        self.ip_limit         = conf ('tools.rate_limiter.ip_limit',        20)

        # leave this much time between requests or get rate_malus
        self.min_interval     = conf ('tools.rate_limiter.min_interval',     1)

        self.session_malus    = conf ('tools.rate_limiter.session_malus',   20)
        self.no_captcha_malus = conf ('tools.rate_limiter.no_captcha_malus', 1)
        self.e404_malus       = conf ('tools.rate_limiter.e404_malus',     200)
        self.rate_malus       = conf ('tools.rate_limiter.rate_malus',      10)
        self.encoding_malus   = conf ('tools.rate_limiter.encoding_malus',   5)
        self.referrer_malus   = conf ('tools.rate_limiter.referrer_malus',   5)
        self.category_malus   = conf ('tools.rate_limiter.category_malus',   0)
        self.css_malus        = conf ('tools.rate_limiter.css_malus',        0)

        self.whitelist        = self.load_whitelist (conf)
        self.trusted_xff_list = self.load_trusted_xff_list (conf)

        self.users = 0           # no. of active users
        self.hits = 0            # no. of hits
        self.whitelist_hits = 0  # no. of hits from whitelisted ips
        self.denied_hits = 0     # no. of denied accesses

        self.hit_accumulator = 0
        self.whitelist_hit_accumulator = 0
        self.denied_hit_accumulator = 0

        self.categories = '/ebooks/search/ /ebooks/ /files/ /cache/'.split ()

        # reasons
        # suggest: ajax call, user cannot see the captcha
        # captcha: user must see captcha even if soft-blocked
        # stats: don't block the sysop
        # re must match whole path !
        self.whitelisted_paths = re.compile (r'/ebooks/suggest/|/stats/|/w/captcha/.*')

    @staticmethod
    def load_whitelist (conf):
        """ Load IP whitelist. """

        wl = netaddr.ip.sets.IPSet ()
        for cidr in conf ('tools.rate_limiter.whitelist', '').splitlines ():
            cidr = cidr.partition ('#')[0]
            cidr = cidr.strip ()
            if cidr:
                wl.add (netaddr.ip.IPNetwork (cidr))
                # print cidr
        return wl

    @staticmethod
    def load_trusted_xff_list (conf):
        """ Load a list of proxies whose X-Forwarded-For header we trust. """

        tl = netaddr.ip.sets.IPSet ()
        try:
            url = conf ('tools.rate_limiter.trusted_xff_url')
            if url:
                log (None, "Requesting trusted xff list from: %s ..." % url)
                trusted_xff_list = requests.get (url)
                log (None, "Scanning trusted xff list ...")
                cidrs = []
                to_resolve = {}
                skip = True # horrible hack to skip AOL proxies in list
                for line in trusted_xff_list.text.splitlines ():
                    if line.startswith ('# British Columbia school network'):
                        skip = False
                    if skip:
                        continue
                    line = line.partition ('#')[0]
                    line = line.strip ()
                    if line:
                        try:
                            cidr = netaddr.ip.IPNetwork (line)
                            cidrs.append (cidr)
                        except (ValueError, netaddr.core.AddrFormatError):
                            # not an IP, maybe a hostname we can resolve
                            to_resolve[line] = None
                trusted_xff_list.close ()
                tl.update (cidrs)
                if to_resolve:
                    log (None, "Resolving hosts in xff list ...")
                    asyncdns.bulk_query (to_resolve, asyncdns.adns.rr.A, 5)
                    tl.update ( (ip[0] for ip in to_resolve.values () if ip is not None) )

        except Exception as what:
            log (None, what)

        for cidr in conf ('tools.rate_limiter.trusted_xff_list', '').splitlines ():
            cidr = cidr.partition ('#')[0]
            cidr = cidr.strip ()
            if cidr:
                tl.add (netaddr.ip.IPNetwork (cidr))
                # print cidr

        log (None, "Loaded {count} trusted xff ips.".format (count = len (tl)))
        return tl

    def e404 (self):
        """ 404 penalty when hitting non-existing ebook nos.
        for people who write naive scraper scripts. """
        cherrypy.ipsession['rhits'] += self.e404_malus

    def block (self, ip):
        """ Block IP. """

        ipsession = self.cache.get (ip)
        if ipsession is not None:
            ipsession.block_me ()
            return True
        return False

    def unblock (self, ip):
        """ Unblock IP. """

        ipsession = self.cache.get (ip)
        if ipsession is None:
            return False

        ipsession.unblock_me ()

        conn = cherrypy.engine.pool.connect ()
        c  = conn.cursor ()
        with self.dblock:
            try:
                c.execute ('begin')
                c.execute ('set transaction isolation level serializable')
                query = 'UPDATE robots.blocks SET expires = now () WHERE ip = %(ip)s'
                params = { 'ip': to_cidr (ip) }
                c.execute (query, params)
                c.execute ('commit')
            except DatabaseError as what:
                cherrypy.log ("SQL Error: %s\n" % what,
                              context = 'RATELIMITER', severity = logging.ERROR)
                cherrypy.log ("Query was: %s\n" % c.mogrify (query, params),
                              context = 'RATELIMITER', severity = logging.ERROR)
                c.execute ('rollback')

        return True

    @staticmethod
    def get_challenge ():
        """ Generate a challenge. Called from genshi templates. """

        # currently the challenge gets applied as xml id
        challenge = cherrypy.session['challenge'] = 'id' + str (random.randint (1000, 9999))
        return challenge

    @staticmethod
    def check_challenge ():
        """ User agent handles cookies, JS and CSS if answer equals challenge. """

        # Note that we cannot insert a challenge into every html page,
        # only in those the app server generates. Thus we may get back
        # the answer cookie for one challenge more than once.

        s = cherrypy.session
        if 'challenge' not in s:
            return None # no challenge emitted yet
        cookies = cherrypy.request.cookie
        if 'bonus' not in cookies:
            return False

        challenge = s['challenge']
        answer = cookies['bonus'].value

        if 'is-catalog-maintainer' in cookies:
            # send some debug info
            cherrypy.response.headers['X-CSS-Bonus'] = (
                "Challenge: %s - Response: %s" % (challenge, answer))

        return answer == challenge

    def _get_ip_session (self):
        """ Retrieve or create an ipsession for this request. """

        ips = IPStruct ()
        ips.from_request (cherrypy.request, self)

        # get or set ipsession object
        cache_key = ips.get_cache_key ()
        try:
            ipsession = self.cache[cache_key]
        except KeyError:
            ipsession = RatelimiterSession (self, ips)
            # To guard against `list (dict.items ())` failing in
            # `self.reset ()` because of 'RuntimeError: dictionary
            # changed size during iteration'
            with self.lock:
                self.cache[cache_key] = ipsession

        return ipsession

    def _get_headers_array (self, headers):
        """ All request headers in one array of string. """

        a = []
        for k, v in headers.items ():
            a.append (k + ': ' + v)
        return a

    def _get_http_signature (self, headers):
        """ Get a signature of the header fields.

        Rationale: to detect randomly spoofed user-agents we
        check if the signature stays the same over a range of
        (possibly spoofed) user-agents.

        """

        a = []
        for k, v in headers.items ():
            if k in self.http_headers_ignore or k.startswith ('X-'):
                continue
            if k in self.http_headers_no_random:
                a.append (k + ': ' + v)
            else:
                a.append (k)
        # FIXME: we have no way of getting the header order
        # coordinate with ibiblio varnish team?
        # FIXME: just return a hash after debugging this
        return '|'.join (sorted (a))

    def limit (self, **dummy_kwargs):
        """ Do accounting for ip and return 403 if ip exceeded quota. """

        ipsession = self._get_ip_session ()

        cherrypy.serving.ipsession = ipsession
        if not hasattr (cherrypy, "ipsession"):
            cherrypy.ipsession = cherrypy._ThreadLocalProxy ('ipsession') # pylint: disable=W0212

        ipsession.sessions.add (cherrypy.session.id)

        headers = cherrypy.request.headers

        ua = headers.get ('User-Agent')
        signature = self._get_http_signature (headers)

        whitelisted = (ipsession.ips.whitelisted or
                       self.whitelisted_paths.match (cherrypy.request.path_info))

        with self.lock:
            ipsession['active'] = True
            ipsession.extend_expiration (self.expiration)
            ipsession.append_trace (self, ua, signature)
            ipsession['css_ok'] = self.check_challenge () # store value for stats page

            if not whitelisted:
                ipsession.calc_block (self, ua)

            ipsession['last_hit'] = datetime.datetime.now () # set this after calc_block
            ipsession.log_state ()

            # allow hit
            if whitelisted:
                ipsession.log ('whitelisted')
                cherrypy.response.headers['X-Whitelisted'] = 'ratelimiter'
                ipsession['hits'] += 1
                self.whitelist_hit_accumulator += 1
                return

            if ipsession['blocked'] == 0:
                ipsession['hits'] += 1
                self.hit_accumulator += 1
                return

            # soft or hard deny
            self.denied_hit_accumulator += 1
            cherrypy.lib.caching.expires (0, True)

            if ipsession['blocked'] == 1:
                ipsession['captchas'] += 1
                ipsession.extend_expiration (10 * self.expiration) # watch 'em longer
                ipsession.log ('Redirected to captcha')
                cherrypy.response.headers['X-Ratelimiter-Denied'] = "503"
                raise cherrypy.HTTPRedirect ('//www.gutenberg.org/w/captcha/question/')
            else:
                ipsession['dhits'] += 1
                ipsession.set_expiration (self.expiration)
                if not ipsession['headers']:
                    ipsession['headers'] = self._get_headers_array (headers)
                ipsession.log ('Blocked 403')
                cherrypy.response.headers['X-Ratelimiter-Denied'] = "403"
                raise cherrypy.HTTPError (403, 'Forbidden')

    def reset (self):
        """ Reset the hit counters. """

        try:
            users = 0
            now = datetime.datetime.now ()
            active_window = now - datetime.timedelta (seconds = 60)

            # To guard against `list (self.cache.items ())` failing because
            # of 'RuntimeError: dictionary changed size during iteration'
            with self.lock:
                sessions = list (self.cache.items ())

            # We have to lock self.cache while deleting items, so to keep
            # the lock short, we first compute a list of keys to delete.
            expired_keys = []
            for key, data in sessions:
                if data['expires'] < now:
                    expired_keys.append (key)
                if data['last_hit'] < active_window:
                    data['active'] = False
                if data['active']:
                    users += 1

            with self.lock:
                for key in expired_keys:
                    try:
                        del self.cache[key]
                    except KeyError:
                        cherrypy.log ('KeyError in reset ()', context = 'RATELIMITER',
                                      severity = logging.ERROR)
                self.users = users
                self.hits = self.hit_accumulator
                self.whitelist_hits = self.whitelist_hit_accumulator
                self.denied_hits = self.denied_hit_accumulator

                self.hit_accumulator = 0
                self.whitelist_hit_accumulator = 0
                self.denied_hit_accumulator = 0

            BaseSearcher.formats_acc.reset ()
            BaseSearcher.formats_sum_acc.reset ()

            cherrypy.log ('Run reset ()', context = 'RATELIMITER', severity = logging.INFO)

        except Exception as what:
            # make sure background thread never stops
            cherrypy.log ("Error in reset (): %s" % what,
                          context = 'RATELIMITER', severity = logging.ERROR)

    def debug_reset_all (self):
        """ Reset all hit counters unconditionally.

        Used only by the test suite.

        """
        for key in list (self.cache.keys ()):
            try:
                del self.cache[key]
            except KeyError:
                pass
        cherrypy.log ('Run debug_reset_all ()', context = 'RATELIMITER', severity = logging.INFO)

    @staticmethod
    def get_ipinfo_for (sessions):
        """ Retrieve IP info for all sessions.

        Retrieves info asynchronously in parallel for all sessions.

        """

        adns = asyncdns.AsyncDNS (asyncdns.RESOLV_CONF)
        for ipsession in sessions:
            try:
                if ipsession.ips.ipinfo is None:
                    ip = ipsession.ips.get_ip_to_block ()
                    ipsession.ips.ipinfo = ipinfo.IPInfo (adns, ip)
            except Exception as what:
                cherrypy.log ("Error in get_ipinfo_for (): %s" % what,
                              context = 'RATELIMITER', severity = logging.ERROR)
        adns.wait ()
        adns.cancel ()

    @staticmethod
    def ua_decode (ua):
        """ Try if we can upgrade the encoding to utf-8.

        'headers' items are already decoded to unicode by
        cherrypy, but I found that many foreign (chinese) user
        agents use but fail to specify the utf-8 encoding, so
        that cherrypy wrongly decodes from iso-8859-1.
        """

        if ua is None:
            return 'NO_UA'
        try:
            return ua.encode ('iso-8859-1').decode ('utf-8')
        except UnicodeError:
            return ua

    def to_database (self):
        """ Write IPs of most blatant offenders to database.

        In get_ip_info_for we collect information from many services
        which may be slow or down, so we run in an extra thread to
        avoid delaying the reset thread.

        """

        with self.dblock:  # make sure we don't reenter if we take too long
            try:
                blocked_sessions = [s for s in self.cache.values ()
                                    if s is not None and s['blocked'] == 2]
                self.get_ipinfo_for (blocked_sessions)

                cursor = cherrypy.engine.pool.connect ().cursor ()
                cursor.execute ('set transaction isolation level serializable')

                for ipsession in blocked_sessions:
                    try:
                        ips = ipsession.ips
                        info = ips.ipinfo

                        expire_days = ipsession['expire_days']
                        if info.is_blackhat ():
                            if info.is_dialup ():
                                # at least 2 days
                                expire_days = max (2, expire_days)
                            else:
                                # at least 7 days
                                expire_days = max (7, expire_days)

                        params = {
                            'ip':          to_cidr (ips.get_ip_to_block ()),
                            'client_ip':   to_cidr (ips.client_ip),
                            'proxy_ip':    to_cidr (ips.proxy_ip),
                            'proxy_type':  ips.proxy_type,
                            'host':        info.hostname,
                            'expires':     (datetime.datetime.now () +
                                            datetime.timedelta (days = expire_days)),
                            'types':       ipsession.get_tags (),
                            'whois':       ' - '.join (filter (None, info.whois.values ())),
                            'user_agents': sorted ([self.ua_decode (ua)
                                                    for ua in ipsession['user_agents']]),
                            'requests':    list (ipsession['requests']),
                            'headers':     [self.ua_decode (h) for h in ipsession['headers']],
                            'count':       1,
                            'hits':        ipsession['hits'],
                            'country':     None,
                            'org':         None,
                            'cidr':        None,
                            'asn':         None,
                            }
                        params.update (info.whois)

                        cursor.execute ('begin')
                        query = """select count, hits from robots.blocks
                                   where ip = %(ip)s and not is_whitelisted"""
                        cursor.execute (query, params)
                        rows = cursor.fetchall ()
                        if len (rows) > 0:
                            count = rows[0][0]
                            hits = rows[0][1]
                            params['count'] = count + 1
                            params['hits']  = hits + ipsession['hits']
                            params['expires'] = (
                                datetime.datetime.now () +
                                datetime.timedelta (
                                    days = expire_days * (2 ** min (count, 10))))
                            query = """update robots.blocks set
                                       count = %(count)s,
                                       expires = %(expires)s,
                                       types = %(types)s,
                                       user_agents = %(user_agents)s,
                                       requests = %(requests)s,
                                       headers = %(headers)s,
                                       hits = %(hits)s
                                       where ip = %(ip)s"""
                        else:
                            query = """insert into robots.blocks
                                       (ip, client_ip, proxy_ip, proxy_type,
                                        host, expires, types, country, org, cidr, asn,
                                        whois, user_agents, requests, headers, hits)
                                       values
                                       (%(ip)s, %(client_ip)s, %(proxy_ip)s, %(proxy_type)s,
                                        %(host)s, %(expires)s, %(types)s,
                                        %(country)s, %(org)s, %(cidr)s, %(asn)s,
                                        %(whois)s, %(user_agents)s, %(requests)s,
                                        %(headers)s, %(hits)s)"""
                        cursor.execute (query, params)
                        cursor.execute ('commit')
                        ipsession['blocked'] = 9
                        # keep on stats page for 5 minutes
                        ipsession.set_expiration (5 * 60)
                        cherrypy.log ("BLOCKED: %s\n" % cursor.mogrify (query, params),
                                      context = 'RATELIMITER', severity = logging.INFO)
                    except DatabaseError as what:
                        cherrypy.log ("SQL Error: %s\n" % what,
                                      context = 'RATELIMITER', severity = logging.ERROR)
                        cherrypy.log ("Query was: %s\n" % cursor.mogrify (query, params),
                                      context = 'RATELIMITER', severity = logging.ERROR)
                        cursor.execute ('rollback')
                    except Exception as what:
                        cherrypy.log ("Error in to_database (): %s" % what,
                                      context = 'RATELIMITER', severity = logging.ERROR)
                        cursor.execute ('rollback')

                cherrypy.log ('Run to_database ()',
                              context = 'RATELIMITER', severity = logging.INFO)

            except Exception as what:
                # make sure background thread never stops
                cherrypy.log ("Error in to_database (): %s" % what,
                              context = 'RATELIMITER', severity = logging.ERROR)

class BlockPage (Page.Page):
    """ Block IP. """

    def index (self, **kwargs): # pylint: disable=R0201
        """ Block IP. """

        if cherrypy.tools.rate_limiter.block (kwargs['ip']):  # pylint: disable=E1101
            return "<p>blocked</p>"
        return "<p>not found</p>"

class UnblockPage (Page.Page):
    """ Unblock IP. """

    def index (self, **kwargs): # pylint: disable=R0201
        """ Unblock IP. """

        if cherrypy.tools.rate_limiter.unblock (kwargs['ip']):  # pylint: disable=E1101
            return "<p>unblocked</p>"
        return "<p>not found</p>"

class TracebackPage (Page.Page):
    """ Produce a traceback in the logs. """

    def index (self, **dummy_kwargs): # pylint: disable=R0201
        """ Traceback. """

        raise Exception

class RateLimiterResetPlugin (cherrypy.process.plugins.Monitor):
    """ Plugin to start the counter reset thread.

    We cannot start any threads before daemonizing,
    so we must start the timer thread by this plugin.

    """

    def __init__ (self, bus):
        # frequency of reset () calls in seconds
        frequency = cherrypy.config.get ('tools.rate_limiter.frequency', 60)
        # pylint: disable=E1101
        super (RateLimiterResetPlugin, self).__init__ (
            bus, cherrypy.tools.rate_limiter.reset, frequency)
        self.name = 'rate_limiter_reset'

class RateLimiterDatabasePlugin (cherrypy.process.plugins.Monitor):
    """ Plugin to start the database writer thread.

    We cannot start any threads before daemonizing,
    so we must start the timer thread by this plugin.

    """

    def __init__ (self, bus):
        # frequency of database calls in seconds
        frequency = cherrypy.config.get ('tools.rate_limiter_database.frequency', 60)
        # pylint: disable=E1101
        super (RateLimiterDatabasePlugin, self).__init__ (
            bus, cherrypy.tools.rate_limiter.to_database, frequency)
        self.name = 'rate_limiter_database'

cherrypy.process.plugins.RateLimiterReset    = RateLimiterResetPlugin
cherrypy.process.plugins.RateLimiterDatabase = RateLimiterDatabasePlugin


from typing import Iterator
from copy import deepcopy

class Decompressor:
    """
    Wraps generators of the Compressor class.
    Adds back the redundant data that was removed during the compression process.
    """

    def __init__(self, source_generator: Iterator[dict]):
        """
        Instantiates the wrapper with a target generator containing compressed data

        Args:
            source_generator (Iterator[dict]): target generator
        """

        self.source_generator = source_generator
        self.last_packet = None

    def _grow_dict(self, reference: dict, target: dict):
        """
        Updates the target dict with data from the reference dict,
         if a new version of it is not already in target.

        Args:
            reference (dict): reference generator
            target (dict): target generator
        """

        for k, v in reference.items():

            if target is None:
                target = {}

            if k not in target:
                target[k] = deepcopy(v)
            elif isinstance(v, dict):
                self._grow_dict(reference[k], target[k])
            else:
                reference[k] = deepcopy(target[k])

        for k, v in target.items():
            if k not in reference:
                reference[k] = deepcopy(v)

    def uncompressed_generator(self) -> Iterator[dict]:
        """
        Generator that returns the decompressed data.

        Returns:
            Iterator[dict]: Decompressed generator
        """

        for data_packet in self.source_generator:

            if self.last_packet is None:
                self.last_packet = deepcopy(data_packet)
                yield data_packet
            else:
                self._grow_dict(self.last_packet, data_packet)
                yield data_packet

class JITDecompressor:
    """
    Wraps generators of the Compressor class.
    Adds back the redundant data that was removed during the compression process.
    """

    def __init__(self):
        """
        Instantiates the wrapper
        """

        self.last_packet = None

    def _grow_dict(self, reference: dict, target: dict):
        """
        Updates the target dict with data from the reference dict,
         if a new version of it is not already in target.

        Args:
            reference (dict): reference generator
            target (dict): target generator
        """

        for k, v in reference.items():

            if k not in target or target[k] is None:
                target[k] = deepcopy(v)
            elif isinstance(v, dict):
                self._grow_dict(reference[k], target[k])
            else:
                reference[k] = deepcopy(target[k])

        for k, v in target.items():
            if k not in reference:
                reference[k] = deepcopy(v)

    def rewind(self):
        self.last_packet = None

    def decompress_next_packet(self, source_packet) -> dict:
        """
        Returns the decompressed data.
        # TODO make the other extend this class

        Returns:
            dict: Decompressed packet
        """

        if self.last_packet is None:
            self.last_packet = deepcopy(source_packet)
            return source_packet
        else:
            self._grow_dict(self.last_packet, source_packet)
            return source_packet


import cv2
img = cv2.imread('../img/graph2.png')
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
gray = cv2.GaussianBlur(gray, (11,11), 0)
gray = cv2.Canny(gray, 100, 300)
ret, binary = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
contours, hierarchy = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(img, contours, -1, (0, 0, 255), 3)
cv2.imshow('try1', img)
#cv2.imwrite('threshold1.png', img)
cv2.waitKey()


import csv
import logging
import re
import sys

import requests
from lxml import html

# time pipenv run python fetch_running_order.py 2017 && wc data/running_order_j2017_ju.tsv

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')

distances = {
    "ve": {
        2011: [6.9, 6.2, 5.1, 8.5],
        2012: [5.7, 5.8, 7.2, 8.4],
        2013: [8.2, 6.2, 6.2, 8.7],
        2014: [5.1, 5.0, 6.7, 7.4],
        2015: [8.0, 6.0, 6.2, 8.8],
        2016: [7.1, 6.7, 6.0, 9.1],
        2017: [6.7, 6.6, 5.7, 8.0],
        2018: [6.2, 6.2, 5.4, 7.9]
    },
    "ju": {
        2011: [11.5, 11.4, 13.6, 8.3, 8.5, 10.5, 15.6],
        2012: [12.7, 12.7, 14.1, 7.7, 8.1, 10.2, 15.1],
        2013: [12.2, 13.0, 14.4, 7.8, 7.7, 11.7, 15.1],
        2014: [10.1, 11.5, 10.2, 7.6, 7.7, 10.7, 14.0],
        2015: [13.8, 12.3, 15.8, 8.1, 8.6, 12.6, 14.6],
        2016: [10.7, 12.8, 14.1, 8.6, 8.7, 12.4, 16.5],
        2017: [12.8, 14.3, 12.3, 7.7, 7.8, 11.1, 13.8],
        2018: [11.0, 11.9, 12.7, 8.8, 8.7, 10.8, 15.1]
    }
}

def fetch_running_order(year, ve_or_ju):
    def leg_dist(leg):
        dist = distances[ve_or_ju][year]
        return dist[leg - 1]

    def parse_team_base_name(team_name):
        p = "^(.+) ([0-9]+)$"
        matches = re.match(p, team_name)
        team_base_name = matches.group(1)
        return team_base_name

    def fetch_order(url):
        logging.info("Fetching " + url)
        page = requests.get(url, timeout=15)
        tree = html.fromstring(page.content.decode('ISO-8859-1').encode("utf-8").strip())
        rows = tree.xpath('//*[@id="site_main"]/table/tr')
        output_rows = []
        current_team_id = ""
        current_team_base_name = ""
        current_team_name = ""
        current_team_country = "NONE"
        for row in rows:
            team_id = next(iter(row.xpath('.//td[1]/text()') or []), None)
            if team_id is not None:
                team_name = next(iter(row.xpath('.//td[2]/text()')))
                current_team_country = next(iter(row.xpath('.//img/@title') or []), "NA")
                current_team_id = team_id
                team_base_name = parse_team_base_name(team_name)
                current_team_name = team_name
                current_team_base_name = team_base_name
                logging.info(
                    "Team line: " + current_team_country + " " + team_id + " " + team_name + " -> " + team_base_name)
            else:
                leg = next(iter(row.xpath('.//td[2]/text()') or []), None)
                name = next(iter(row.xpath('.//td[3]/text()') or []), None)
                if name is not None and name != " " and leg is not None:
                    leg = int(leg.strip())
                    name = name.strip()
                    logging.info(current_team_id + " " + current_team_name + " " + str(leg) + " '" + name + "'")
                    output_rows.append(
                        [current_team_id, current_team_name, current_team_base_name, current_team_country, leg,
                         leg_dist(leg), name])
        return output_rows

    out_file_name = f'data/running_order_j{year}_{ve_or_ju}.tsv'
    csv_file = open(out_file_name, 'w')

    csvwriter = csv.writer(csv_file, delimiter="\t", quoting=csv.QUOTE_ALL)
    header = ["team_id", "team", "team_base_name", "team_country", "leg", "leg_dist", "name"]

    csvwriter.writerow(header)

    url = f"https://registration.jukola.com/?kieli=en&kisa=j{year}&view=1&sarja={ve_or_ju}&jarj=kilpnro&jj=1"
    rows = fetch_order(url)
    for row in rows:
        csvwriter.writerow(row)

    csv_file.close()
    logging.info("Wrote " + out_file_name)

year = int(sys.argv[1])

fetch_running_order(year, "ve")
fetch_running_order(year, "ju")


from bs4 import BeautifulSoup
import re
import requests
import pandas as pd
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15'
            }

#funct to get the urls to each recipe in the search
def geturlsFN(response ,text):
    """"
    scrapes urls for recipes that result when searching for a recipe of FN
    """
    reciperesult = text.find_all('section', attrs={'class': 'o-RecipeResult o-ResultCard'})
    recipe_urls = ['http:'+ tag.find('a').get('href') for tag in reciperesult]
    return recipe_urls
def getratingsFN(response ,text):
    """"
    scrapes ratings for recipes that result when searching for a recipe of FN
    """
    s=str(text.find_all('div', attrs={'class': 'o-ResultCard__m-MediaBlock m-MediaBlock'})).split('
')[1:]
    ratings=[(str(re.findall('title=\".*? of 5 stars',x))[9:-13]) for x in s]
    return ratings
def getnumreviewFN(response ,text):
    """"
    scrapes #reviews for recipes that result when searching for a recipe of FN
    """
    s=str(text.find_all('div', attrs={'class': 'o-ResultCard__m-MediaBlock m-MediaBlock'})).split('
')[1:]
    numratings=[(str(re.findall('gig-rating-ratingsum\"\>.*? Review',x))[24:-9]) for x in s]
    return numratings

#obtain urls ratings and #ratings for each page

def scraperecipeurls(searchurl):
    #gets the urls that result from the search and the num of
    #reviews and avg num of stars per recipe
    firstpage=f'{searchurl}{1}/CUSTOM_FACET:RECIPE_FACET'
    print(firstpage)
    response = requests.get(firstpage, headers=headers)
    text = BeautifulSoup(response.text, 'html.parser')

    numberofpages=int(re.sub(r"[\n\t\s]*", "", text.find_all('a', attrs={'class': 'o-Pagination__a-Button'})[-2].string) )
    allurls=[]
    allratings=[]
    allratingnums=[]
    for i in range(numberofpages):
        response = requests.get(f'{searchurl}{i+1}/CUSTOM_FACET:RECIPE_FACET', headers=headers)
        text = BeautifulSoup(response.text, 'html.parser')
        if response.status_code !=200:
            print(f"status trouble {i+1}")
        pageurls=geturlsFN(response ,text)
        allurls=allurls+pageurls
        pageratings=getratingsFN(response ,text)
        allratings=allratings+pageratings
        pageratingnums=getnumreviewFN(response ,text)
        allratingnums=allratingnums+pageratingnums
    return [allurls,allratings,allratingnums]

def getrecipedataFN(urls):
    """"
    funct that takes in a list of urls of recipes and extracts
    the title of the recipe, the ingredients of the cake portion and the number of servings it yields
    """
    title=[]
    servings=[]
    ingredients=[]
    for i in range(len(urls)):
        response = requests.get(urls[i], headers=headers)
        text = BeautifulSoup(response.text, 'html.parser')
        title.append(text.find_all('span', attrs={'class': 'o-AssetTitle__a-HeadlineText'})[0].string)
        try:
            servings.append(str(text.find_all('span',text=re.compile('Yield:'))[0].find_next_siblings())[43:-8])
        except:
            servings.append('NA')
        if text.find_all('h6',attrs={'class':'o-Ingredients__a-SubHeadline'})==[]:
            ingredients.append([str(re.findall('value=\".*?\"',str(x)))[9:-3] for x in text.find_all('input',attrs={'class':'o-Ingredients__a-Ingredient--Checkbox'})[1:]])
        else:
            s=str(text.find_all('div',attrs={'class':'o-Ingredients__m-Body'})).split("Frosting:")[0]
            ingredients.append([x[7:-1] for x in re.findall('value=\".*?\"',s)][1:])
    return [title,servings,ingredients]

#get attributes for each recipe- cost and calories will be calculated based on ingreidnts later
searchurl=['https://www.foodnetwork.com/search/white-cupcakes-/p/','https://www.foodnetwork.com/search/yellow-cupcakes-/p/']#'https://www.foodnetwork.com/search/vanilla-cupcakes-/p/']#'https://www.foodnetwork.com/search/lemon-cake-/p/']#'https://www.foodnetwork.com/search/white-cake-/p/','https://www.foodnetwork.com/search/yellow-cake-/p/','https://www.foodnetwork.com/search/angel-food-cake-/p/']#'https://www.foodnetwork.com/search/vanilla-cake-/p/']

alldata=[]
for i in range(len(searchurl)):

    [allurls,allratings,allratingnums]=scraperecipeurls(searchurl[i])
    [title,servings,ingredients]=getrecipedataFN(allurls)
    alldata=alldata+list(zip(title,servings,ingredients,allratings,allratingnums))

recipedf=pd.DataFrame(alldata,columns =['Recipe_title','servings','ingredients','stars','Num_of_reviews'])
recipedf.to_csv('recipes3.csv')
print('DONE')


from prody import *
import subprocess
import os
import re
from colorama import init, Fore, Back, Style

init()

def prRed(skk):
	print("\033[91m {}\033[00m" .format(skk))

def prGreen(skk):
	print("\033[92m {}\033[00m" .format(skk))

def prYellow(skk):
	print("\033[93m {}\033[00m" .format(skk))

def unique(list1):
	unique_list = [ ]
	for i in list1:
		if i not in unique_list:
			unique_list.append(i)
	return unique_list

def Diff(li1, li2):
    return (list(list(set(li1)-set(li2)) + list(set(li2)-set(li1))))

def cofactor_status(category):
	if category == '4' or category == '1':
		prRed("Your cofactor is doesn't exist")
		cofactor_code = ' '
	elif category == '2' or category == '3':
		prRed("Your cofactor is a small organic molecule")
		cofactor_code = input("please enter the resname of the cofactor: \n")
	else:
		sys.exit('ligand is not either a peptide or small organic molecule: so this code will not work')
	return cofactor_code

def ligand_status(category):
	if category == '3' or category == '4':
		prRed('Your ligand is a poly-peptide')
		ligand_code = input("Please enter the chain_ID: \n")
	elif category == '1' or category == '2':
		prRed('Your ligand is organic molecule')
		ligand_code = input("Please enter the resname of the ligand:\n")
	else:
		sys.exit('ligand is not either a peptide or small organic molecule: so this code will not work')
	return ligand_code

def ligand_write():
	tleap_file.write('# loading parameters for ligand \n \n')
	tleap_file.write('loadamberprep ligand_clean_h.prepi \n')
	tleap_file.write('loadamberparams ligand.frcmod \n \n')
	tleap_file.write('ligand = loadmol2 ligand_clean_h.mol2 \n')

def cofactor_write():
	tleap_file.write('# loading parameters for cofactor \n \n')
	tleap_file.write('loadamberprep cofactor.prepi \n')
	tleap_file.write('loadamberparams cofactor.frcmod \n')
	tleap_file.write('cofactor = loadmol2 cofactor.mol2 \n \n')

def tleap_source():
	tleap_file.write('source /home/bhuvanesh/softwares/miniconda3/envs/amber/dat/leap/cmd/leaprc.protein.ff14SB\n')
	tleap_file.write('source /home/bhuvanesh/softwares/miniconda3/envs/amber/dat/leap/cmd/leaprc.gaff2 \n')
	tleap_file.write('source /home/bhuvanesh/softwares/miniconda3/envs/amber/dat/leap/cmd/leaprc.water.tip3p \n\n\n')

def ismetal(sub,mcode):
	met = sub.select("resname {0}".format(mcode))
	print(met,type(met))
	if met is None:
		return False
	else:
		return True

def chainsplit(chain_id):
	substructure = structure.select("chain {0}".format(chain_id))
	protein = substructure.select("protein and noh".format(chain_id))
	water = substructure.select("water chain {0}".format(chain_id))
	protein_file = 'protein_noh_chain_{0}.pdb'.format(chain_id)
	if water is not None:
		water_file = 'water_{0}.pdb'.format(chain_id)
		writePDB(water_file,water)
		tleap_file.write('water_{0} = loadpdb {1} \n'.format(chain_id,water_file))
		water_list.append('water_{0}'.format(chain_id))
	writePDB(protein_file,protein)
	tleap_file.write('receptor_{0} = loadpdb {1} \n'.format(chain_id,protein_file))
	receptor_list.append('receptor_{0}'.format(chain_id))

	print('metal_resname',len(metal_resname))
	if len(metal_resname) != 0:
		for ele in metal_resname:
			if ismetal(substructure,ele):
				print(ele)
				print('This protein has {0} metal'.format(ele))
				metal = substructure.select("resname {0}".format(ele))
				metal_file = 'metal_{0}_{1}.pdb'.format(ele,chain_id)
				writePDB(metal_file,metal)
				metal_list.append(metal_file)
				tleap_file.write('metal_{0}_{1} = loadpdb metal_{0}_{1}.pdb'.format(ele,chain_id))

def missing(name, pdbid, chain):
	sub = parsePDB(name)
	subs = sub.select('calpha')
	res = subs.getResnums()

	cy = 0

	for i in range(len(res)):
		try:
			if res[i+1]-res[i] != 1:
				cy = cy + 1
				if cy == 1:
					prRed('{0} has missing residues in it, please model them'.format(name[-23:]))
					prGreen('The residues adjacent to missing residues are follows:')
				prGreen('{0} {1}'.format(res[i],res[i+1]))
		except:
			print('')

	if cy != 0:
		print(Fore.YELLOW + "There are missing residues, Therefore modelling chain: {}".format(chain) + Style.RESET_ALL)
		print("python /mnt/e/ProteinLigand/protein_ligand_preparation/ModellingScript.py {}.pdb {}".format(pdbid, chain))
		subprocess.run("python /mnt/e/ProteinLigand/protein_ligand_preparation/ModellingScript.py {}.pdb {}".format(pdbid, chain), shell=True)

	if cy == 0:
		prGreen('{0} doesnot have missing residues'.format(name[-23:]))

# path = '/home/divya/Post-doc-projects/BindingMoaD/bmoadClassifiedProteins/'
path = '/mnt/e/ProteinLigand/bmoadInputFiles/'

index = int(input(Fore.YELLOW + "Enter the index:\n" + Style.RESET_ALL))

pdbid = input(Fore.YELLOW + "Enter the pdbid: \n" + Style.RESET_ALL)

file = str(index)+ '.' + pdbid

tleap_folder = os.path.join(path,file,'tleap.in')

folder = os.path.join(path,file)

cofactor_list = [ ]
receptor_list = [ ]
water_list = [ ]
metal_list = [ ]

try:
	os.mkdir(folder)
except:
	prYellow("Requested file is already exist!!!")

os.chdir(folder)
pathPDBFolder(folder)

tleap_file = open(tleap_folder,'w')

tleap_source()

structure = parsePDB(pdbid,compressed=False)

het = structure.select('hetero')
het1 = het.getResnames()
het = het1.tolist()
het1 = unique(het)

prRed('#.....................................................#')
prGreen('The hetero atoms in the protein are {0}'.format(het1))
prRed('#.....................................................#')

# Checking for the category of the ligand-cofactor
prGreen("Please select the category of the ligand: \n 1. ligand-organic molecule:cofactor-doesnot exist \
	\n 2. ligand-organic molecule:cofactor-organic_molecule\
	 \n 3. ligand-peptide : cofactor-organic_molecule \
	 \n 4. ligand-peptide : cofactor-doesnot exist \
	 \n 5. Others")

option = input(Fore.YELLOW + "Please type the number corresponding to the ligand-cofactor category: \n" + Style.RESET_ALL)

metal_status = input(Fore.YELLOW + "Is your protein has metal? [Y/N] \n" + Style.RESET_ALL)

if metal_status == 'Y':
	n_types = int(input(Fore.YELLOW + "Please enter how different types of metals you had in the pdb \n" + Style.RESET_ALL))
	prYellow("Please enter the resnames of the metals")
else:
	n_types = 0

metal_resname = [ ]

for i in range(n_types):
	ele = input()
	metal_resname.append(ele)

ligand_file = 'ligand.pdb'
cofactor_file = 'cofactor.pdb'

# This piece of code for simple ligand case and cofactor:

if option == '1' or option == '2':
	ligand_code = ligand_status(option)

	ligand = structure.select("resname {0}".format(ligand_code))
	chain = unique(ligand.getChids())
	chain.sort()

	nchains = unique(structure.select('protein').getChids())
	nchains.sort()

	if chain == nchains:
		substructure = structure.select("chain {0}".format(chain[0]))
		ligand_struct = substructure.select("resname {0}".format(ligand_code))
		ligand_struct.setResnames("LIG")
		writePDB(ligand_file,ligand_struct)

		# Loading amber parameters into tleap program
		ligand_write()

		if option == '2':
			cofactor_code = cofactor_status(option)
			cofact = substructure.select("resname {0}".format(cofactor_code))
			writePDB(cofactor_file,cofact)

			# Loading amber parameters into tleap program
			cofactor_write()
			cofactor_list.append('cofactor')

		chainsplit(chain[0])

	else:
		Chains_IDS = structure.select("protein within 8 of resname {0} chain {1}" \
			.format(ligand_code,chain[0]))
		Chains_to_model = unique(Chains_IDS.getChids())
		Chains_to_model.sort()
		ligand_struct = structure.select("resname {0} chain {1}".format(ligand_code,chain[0]))
		ligand_struct.setResnames('LIG')
		writePDB(ligand_file,ligand_struct)

		ligand_write()

		if option == '2':
			cofactor_code = cofactor_status(option)
			cofact = structure.select("resname {0} chain {1}".format(cofactor_code,chain[0]))
			writePDB(cofactor_file,cofact)

			# Loading amber parameters into tleap program
			cofactor_write()
			cofactor_list.append('cofactor')

		for chid in Chains_to_model:
			chainsplit(chid)

elif option == '3' or option == '4': # ligand is peptide and cofactor is organic molecule
	cofactor_code = cofactor_status(option)
	ligand_code = ligand_status(option)
	ligand = structure.select("protein chain {0}".format(ligand_code))
	writePDB('peptide.pdb',ligand)
	tleap_file.write('ligand = loadpdb peptide.pdb \n')

	water = structure.select("water chain {0}".format(ligand_code))
	if water is not None:
		writePDB('water_{0}.pdb'.format(ligand_code),water)
		tleap_file.write('water_{0} = loadpdb water_{0}.pdb \n'.format(ligand_code))
		water_list.append('water_{0}'.format(ligand_code))

	chain = unique(ligand.getChids())

	nchains = structure.select("protein within 8 of chain {0}".format(ligand_code)).getChids()
	nchains1 = unique(nchains.tolist())

	nchains = Diff(nchains1,list(ligand_code))
	nchains.sort()
	print(nchains)

	if option == '3':
		cofact = structure.select("resname {0} within 8 of chain {1}".format(cofactor_code,ligand_code))
		writePDB(cofactor_file,cofact)
		# Loading amber parameters into tleap program
		cofactor_write()
		cofactor_list.append('cofactor')

	for chid in nchains:
		chainsplit(chid)

else: # ligand and cofactor falls under different category
	cofactor_status(option)

string = ''

for i in receptor_list:
	string = string + i + ' '
for i in cofactor_list:
	string = string + i + ' '

string = string + 'ligand' + ' '

if len(metal_list) != 0:
	for i in metal_list:
		string = string + i + ' '

for i in water_list:
	string = string + i + ' '

with open('{1}/{0}.pdb'.format(pdbid,folder),'r') as inFile:
	text = inFile.read()
	if re.search("SSBOND", text):
		tleap_file.write('\n\n')
		tleap_file.write("# Protein has disulphate bonds \n")
		prRed('Protein has disulphate bonds please include the information in tleap.in program')
	else:
		tleap_file.write("# Protein doesn't have disulphate bonds \n")

f = open('string.dat','w')
f.write(string)

# Identifying the missing residues in protein chain

for prefix in receptor_list:
	f = '{0}/{1}.{2}/protein_noh_chain_{3}.pdb'.format(path,index,pdbid,prefix[-1])
	os.chdir('{0}/{1}.{2}'.format(path,index,pdbid))
	missing(f, pdbid, prefix[-1])


import requests
import json
import gw_config

URL = gw_config.URL + 'setonoff.php'

def run():
    data = {'cmd': 'set', 'value': '1'}
    ret = requests.post(URL, json=data)
    ret = ret.json()
    print(ret['errno'], ret['msg'])

if __name__ == '__main__':
    run()


import mnist
import warnings
from time import gmtime, strftime
from timeit import default_timer as timer

warnings.filterwarnings("ignore")

def Train_Network(n_network, n):

    iterations = n

    print("Preparing Training Data Set.")
    training_images = [x.reshape(1, x.shape[0] * x.shape[1]).tolist()[0] for x in mnist.train_images()]
    print("Preparing Training Data Set Completed.\n")

    print("Training Start.")
    training_labels = mnist.train_labels()

    answers = [[0 for _ in range(10)] for _ in range(len(training_labels))]

    for x in range(len(answers)):
        answers[x][training_labels[x]] = 1

    training_start = timer()

    for y in range(iterations):

        iteration_start = timer()

        for x in range(len(training_images)):
            n_network.train(training_images[x], answers[x])

        iteration_end = timer()

        print("{0:.1f}".format((y + 1) / iterations * 100) + "% Completed. Elapsed Time:",
              "{0:.1f}".format(iteration_end - iteration_start) + "s. Remaining Time ~",
              "{0:.1f}".format((iterations - y - 1) * (iteration_end - iteration_start)) + "s.")

    training_end = timer()
    print("Training End. Elapsed time: " + strftime("%Hh, %Mm, %Ss.", gmtime(training_end - training_start)) + "\n")

def Test_Network(n_network):
    print("Preparing Test Data Set.")
    test_images = [x.reshape(1, x.shape[0] * x.shape[1]).tolist()[0] for x in mnist.test_images()]
    print("Preparing Test Data Set Completed.\n")
    print("Testing Start.")

    test_labels = mnist.test_labels()

    successes = 0

    for x in range(len(test_images)):
        output = n_network.feedforward(test_images[x])

        pos = 0
        max_probability = 0
        for z in range(10):
            if output[z] > max_probability:
                max_probability = output[z]
                pos = z

        if pos == test_labels[x]:
            successes += 1

    print("Testing End.\n")
    print("Successes : {0:.2f}%".format(successes / len(test_labels) * 100))

def Test_Network_With_Image(n_network, image):
    import png

    print("Gathering Image Metadata.\n")
    reader = png.Reader(filename=image)
    w, h, pixels, metadata = reader.read_flat()
    pixels = [pixels[x] for x in [x * 4 for x in range(w * h)]]
    output = n_network.feedforward(pixels)

    position = 0
    max_probability = 0

    for x in range(len(output)):
        if output[x] > max_probability:
            max_probability = output[x]
            position = x

    print("Prediction: {0}".format(position) + " Certainty: {0:.2f}".format(output[position] * 100) + "%")


#!/usr/bin/env python

import json
import os

file_paths = [
    "/usr/lib/code/product.json",
    "/usr/share/codium/resources/app/product.json"
]

fileChanged = False

for path in file_paths:
    print('Checking for file: {}'.format(path))
    if os.path.isfile(path):
        with open(path, "r") as f:
            config = json.load(f)
            config["extensionsGallery"]["serviceUrl"] = "https://marketplace.visualstudio.com/_apis/public/gallery"
            config["extensionsGallery"]["cacheUrl"] = "https://vscode.blob.core.windows.net/gallery/index"
            config["extensionsGallery"]["itemUrl"] = "https://marketplace.visualstudio.com/items"

        with open(path, "w") as f:
            json.dump(config, f, indent=2)

        print('  Changed marketplace in {}'.format(path))
        fileChanged = True

if not fileChanged:
    print('Warning: No marketplaces could be found to be changed')
    exit(1)


import tensorflow as tf
from tensorflow.python.tools.freeze_graph import freeze_graph
from core import simple_mnist_cnn
from core import MNISTBatch
import math
import argparse
import os

def calculate_learning_rate(step=0, lr_max=1e-6, lr_min=1e-10, lr_decay=2e4):
    return lr_min + (lr_max - lr_min) * math.exp(-step / lr_decay)

def print_header():
    print('{}|{}|{}|{}|{}'.format('epoch', 'train loss','test loss',
        'train accuracy', 'test accuracy'))

def print_progress(epoch, train_loss, test_loss, train_accuracy, test_accuracy):
    print('{:03d}{:2}|{:4.4f}{:4}|{:4.4f}{:3}|{:.4f}{:8}|{:.3f}{:11} \
        '.format(epoch, '', train_loss, '', test_loss, '', train_accuracy, '',
        test_accuracy, ''))

def parse_args():
    parser = argparse.ArgumentParser(description='Traing Script for MNIST')
    parser.add_argument('--save', dest='save_dir', type=str, default='/tmp/mnist')
    parser.add_argument('--epochs', dest='epochs', type=int, default=600)
    parser.add_argument('--steps', dest='steps', type=int, default=100)
    parser.add_argument('--train-batch', dest='train_batch', type=int, default=100)
    parser.add_argument('--test-batch', dest='test_batch', type=int, default=1000)
    parser.add_argument('--lr-max', dest='lr_max', type=float, default=1e-6)
    parser.add_argument('--lr-min', dest='lr_min', type=float, default=1e-10)
    parser.add_argument('--lr-decay', dest='lr_decay', type=float, default=2e4)
    parser.add_argument('--checkpoint', dest='checkpoint', type=int, default=100)
    return parser.parse_args()

def main():
    args = parse_args()
    train_dir = os.path.join(args.save_dir, 'train')
    test_dir = os.path.join(args.save_dir, 'test')
    ckpt_dir = os.path.join(args.save_dir, 'ckpt')
    freeze_dir = os.path.join(args.save_dir, 'freeze')

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # Do not clutter training progress.

    tf.reset_default_graph()

    learning_rate = tf.placeholder(tf.float32)

    input_tensor, logits, labels = simple_mnist_cnn()

    mnist = MNISTBatch()

    saver = tf.train.Saver()

    with tf.name_scope('output'):
        softmax = tf.nn.softmax(logits, name='softmax')

    with tf.name_scope('cross_entropy'):
        cross_entropy = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels))
        tf.summary.scalar('cross_entropy', cross_entropy)

    with tf.name_scope('accuracy'):
        correct_prediction = tf.equal(tf.argmax(logits, 1), labels)
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        tf.summary.scalar('accuracy', accuracy)

    with tf.name_scope('train'):
        train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        train_writer = tf.summary.FileWriter(train_dir)
        test_writer = tf.summary.FileWriter(test_dir)

        train_writer.add_graph(sess.graph)
        summary = tf.summary.merge_all()

        test_tensor, test_labels = mnist.test(batch_size=args.test_batch)

        print_header()
        for epoch in range(args.epochs):
            train_tensor, train_labels = mnist.train(batch_size=args.train_batch)

            [train_loss, train_accuracy, train_summary] = sess.run([cross_entropy, accuracy, summary],
                feed_dict={input_tensor: train_tensor, labels: train_labels})

            [test_loss, test_accuracy, test_summary] = sess.run([cross_entropy, accuracy, summary],
                feed_dict={input_tensor: test_tensor, labels: test_labels})

            train_writer.add_summary(train_summary, epoch)
            test_writer.add_summary(test_summary, epoch)
            print_progress(epoch, train_loss, test_loss, train_accuracy, test_accuracy)

            for step in range(args.steps):
                global_step = step + epoch * args.steps
                lr = calculate_learning_rate(global_step, args.lr_max, args.lr_min, args.lr_decay)
                sess.run([train_step], feed_dict={input_tensor: train_tensor, labels: train_labels, learning_rate: lr})

            if epoch % args.checkpoint is 0:
                saver.save(sess, os.path.join(ckpt_dir, 'model_' + str(epoch) + '.ckpt'))

        ckpt_file = saver.save(sess, os.path.join(ckpt_dir, 'model'), global_step=global_step)

        tf.train.write_graph(sess.graph.as_graph_def(), freeze_dir, 'model.pb', as_text=False)

        freeze_graph(os.path.join(freeze_dir, 'model.pb'), '', True, os.path.join(ckpt_dir, ckpt_file),
            'output/softmax', 'save/restore_all', 'save/Const:0', os.path.join(freeze_dir, 'frozenmodel.pb'), True, '')

if __name__ == '__main__':
    main()