# importing section
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import scipy as sp
# using the Csv file
df = pd.read_csv('output.csv')

# Checking the first 5 entries of dataset
df.head()
# using the Csv file
df = pd.read_csv('output.csv')

# Checking the first 5 entries of dataset
df.head()
headers = ["symboling", "normalized-losses", "make",
		"fuel-type", "aspiration","num-of-doors",
		"body-style","drive-wheels", "engine-location",
		"wheel-base","length", "width","height", "curb-weight",
		"engine-type","num-of-cylinders", "engine-size",
		"fuel-system","bore","stroke", "compression-ratio",
		"horsepower", "peak-rpm","city-mpg","highway-mpg","price"]

df.columns=headers
df.head()
data = df

# Finding the missing values
data.isna().any()

# Finding if missing values
data.isnull().any()
# converting mpg to L / 100km
data['city-mpg'] = 235 / df['city-mpg']
data.rename(columns = {'city_mpg': "city-L / 100km"}, inplace = True)

print(data.columns)

# checking the data type of each column
data.dtypes
data.price.unique()

# Here it contains '?', so we Drop it
data = data[data.price != '?']

# checking it again
data.dtypes
data['length'] = data['length']/data['length'].max()
data['width'] = data['width']/data['width'].max()
data['height'] = data['height']/data['height'].max()

# binning- grouping values
bins = np.linspace(min(data['price']), max(data['price']), 4)
group_names = ['Low', 'Medium', 'High']
data['price-binned'] = pd.cut(data['price'], bins,
							labels = group_names,
							include_lowest = True)

print(data['price-binned'])
plt.hist(data['price-binned'])
plt.show()
# categorical to numerical variables
pd.get_dummies(data['fuel-type']).head()

# descriptive analysis
# NaN are skipped
data.describe()
# examples of box plot
plt.boxplot(data['price'])

# by using seaborn
sns.boxplot(x ='drive-wheels', y ='price', data = data)

# Predicting price based on engine size
# Known on x and predictable on y
plt.scatter(data['engine-size'], data['price'])
plt.title('Scatterplot of Enginesize vs Price')
plt.xlabel('Engine size')
plt.ylabel('Price')
plt.grid()
plt.show()
# Grouping Data
test = data[['drive-wheels', 'body-style', 'price']]
data_grp = test.groupby(['drive-wheels', 'body-style'],
						as_index = False).mean()

data_grp
# pivot method
data_pivot = data_grp.pivot(index = 'drive-wheels',
							columns = 'body-style')
data_pivot

# heatmap for visualizing data
plt.pcolor(data_pivot, cmap ='RdBu')
plt.colorbar()
plt.show()
# Analysis of Variance- ANOVA
# returns f-test and p-value
# f-test = variance between sample group means divided by
# variation within sample group
# p-value = confidence degree
data_annova = data[['make', 'price']]
grouped_annova = data_annova.groupby(['make'])
annova_results_l = sp.stats.f_oneway(
							grouped_annova.get_group('honda')['price'],
							grouped_annova.get_group('subaru')['price']
									)
print(annova_results_l)

# strong corealtion between a categorical variable
# if annova test gives large f-test and small p-value

# Correlation- measures dependency, not causation
sns.regplot(x ='engine-size', y ='price', data = data)
plt.ylim(0, )


from __future__ import absolute_import, division, print_function
__metaclass__ = type

DOCUMENTATION = """
---
module: icx_firmware_upgrade
version_added: "1.3.0"
author: "Ruckus Wireless (@Commscope)"
short_description: Upgrades firmware of ICX switches
description:
  - This module copies new firmware to switch, flashes it to primary/secondary and reboots the switch.
notes:
  - Tested against ICX 10.1.
options:
  server_type:
    description:
      - Data transfer protocol to be used.
    type: str
    choices: ['scp', 'https', 'tftp']
  server_address:
    description:
      - IPV4/IPV6 address of the scp/https/tftp server.
    type: str
  server_port:
    description:
      - The port number of the server. Default values will be selected based on protocol type.
        Default scp:22, https:443
    type: int
  partition:
    description:
     - Partition to be used for upgrade.
    type: str
    required: true
    choices: ['primary','secondary','fips-ufi-primary-sig','fips-ufi-secondary-sig']
  scp_user:
    description:
      - remote username to be used for scp login.
    type: str
  scp_pass:
    description:
      - remote password to be used for scp login.
    type: str
  boot_only:
    description:
      - Boot the switch if True.
    default: no
    type: bool
    required: true
  save_running_config:
    description:
      - execute Write memory Command.
    default: no
    type: bool
  filename:
    description:
      - The path of the .bin/.sig file.
    type: str
"""

EXAMPLES = """
- name: upgrade firmware using scp
  community.network.icx_firmware_upgrade:
    server_type: scp
    server_address: 10.20.1.1
    partition: secondary
    filename: SPR08030.bin
    boot_only: false
    save_running_config: False
    scp_user: alethea
    scp_pass: alethea123
- name: upgrade firmware using tftp
  community.network.icx_firmware_upgrade:
    server_type: tftp
    server_address: 2001:db8::1
    partition: fips-ufi-primary-sig
    filename: signature_ufi.sig
    boot_only: False
    save_running_config: False
- name: upgrade firmware using https
  community.network.icx_firmware_upgrade:
    server_type: https
    server_address: 10.20.1.8
    partition: primary
    filename: SPR08030.bin
    boot_only: False
    save_running_config: False
- name: run only boot command
  community.network.icx_firmware_upgrade:
    boot_only: True
    save_running_config: True
"""

RETURN = """
changed:
  description: C(true) when file is copied and switch flashed with it. C(false) otherwise.
  returned: always
  type: bool
"""

from copy import deepcopy
import re

from ansible.module_utils._text import to_text
from ansible_collections.commscope.icx.plugins.module_utils.network.icx.icx import run_commands, exec_scp
from ansible.module_utils.basic import AnsibleModule, env_fallback
from ansible.module_utils.connection import ConnectionError
from ansible_collections.ansible.netcommon.plugins.module_utils.network.common.utils import remove_default_spec

def map_params_to_obj(module):
    commands = dict()
    command = ''
    res_commands = []
    server_type = module.params['server_type']
    server_address = module.params['server_address']
    server_port = module.params['server_port']
    filename = module.params['filename']
    partition = module.params['partition']
    if partition in ['fips-ufi-primary-sig', 'fips-ufi-secondary-sig']:
        boot_partition = partition.split('-')[2]
    else:
        boot_partition = partition
    boot_only = module.params['boot_only']
    save_running_config = module.params['save_running_config']
    if not boot_only:
        if server_type == 'tftp':
            command = 'copy tftp flash %s %s %s' % (server_address, filename, partition)
            commands['command'] = command

        elif server_type == 'scp':
            if server_port:
                command = 'copy scp flash %s %d %s %s' % (server_address, server_port, filename, partition)
            else:
                command = 'copy scp flash %s %s %s' % (server_address, filename, partition)
            commands['command'] = command
            commands['scp_user'] = module.params['scp_user']
            commands['scp_pass'] = module.params['scp_pass']

        elif server_type == 'https':
            if server_port:
                command = 'copy https flash %s %s %s port %d' % (server_address, filename, partition, server_port)
            else:
                command = 'copy https flash %s %s %s' % (server_address, filename, partition)

            commands['command'] = command
        res_commands.append(commands)
        if save_running_config:
            res_commands.append('write memory')
            res_commands.append('boot system flash %s yes' % (boot_partition))
        else:
            res_commands.append('boot system flash %s yes' % (boot_partition))
    else:
        if save_running_config:
            res_commands.append('write memory')
            res_commands.append('boot system flash %s yes' % (boot_partition))
        else:
            res_commands.append('boot system flash %s yes' % (boot_partition))

    return res_commands

def checkValidations(module):
    server_type = module.params['server_type']
    server_address = module.params['server_address']
    # server_port = module.params['server_port']
    filename = module.params['filename']
    boot_only = module.params['boot_only']
    scp_user = module.params['scp_user']
    scp_pass = module.params['scp_pass']

    if not boot_only:
        if server_type is None:
            module.fail_json(msg="server_type is required")
        if server_address is None:
            module.fail_json(msg="server_address is required")
        if filename is None:
            module.fail_json(msg="filename is required")
        if server_type == 'scp':
            if scp_user is None:
                module.fail_json(msg="scp_user is required")
            if scp_pass is None:
                module.fail_json(msg="scp_pass is required")

def main():
    """ main entry point for module execution
    """
    argument_spec = dict(
        server_type=dict(type='str', choices=['scp', 'https', 'tftp'], required=False),
        server_address=dict(type='str', required=False),
        server_port=dict(type='int'),
        partition=dict(type='str', choices=['primary', 'secondary', 'fips-ufi-primary-sig',
                                            'fips-ufi-secondary-sig'], required=True),
        filename=dict(type='str', required=False),
        boot_only=dict(type='bool', required=True),
        save_running_config=dict(type='bool'),
        scp_user=dict(type='str', required=False),
        scp_pass=dict(type='str', required=False, no_log=True),
    )

    module = AnsibleModule(argument_spec=argument_spec,
                           supports_check_mode=True)
    checkValidations(module)
    result = {'changed': False}

    commands = map_params_to_obj(module)

    if module.params['save_running_config']:
        if module.params['boot_only']:
            result['commands'] = [commands[0], commands[1]]
        else:
            result['commands'] = [commands[0]['command'], commands[1], commands[2]]
    else:
        if module.params['boot_only']:
            result['commands'] = [commands[0]]
        else:
            result['commands'] = [commands[0]['command'], commands[1]]

    if commands:
        if not module.check_mode:
            if module.params['server_type'] == 'scp':
                responses = exec_scp(module, commands[0])
                responses = run_commands(module, commands[1:])
            else:
                responses = run_commands(module, commands)
        result['changed'] = True
        result['responses'] = responses

    module.exit_json(**result)

if __name__ == '__main__':
    main()


import monai
import numpy as np
from typing import Any, Callable, Optional, Tuple, Union, Mapping, Hashable, List, Sequence
from torch.utils.data._utils.collate import default_collate

# This module contains all the extra components needed for data loading and segmentation model training,
# some of which need to exist in any environment where the saved model is to be deployed.

# Custom transform to convert a collection of segmentation mask arrays to a single one-hot encoded array
class MasksToOneHotD(monai.transforms.MapTransform):
    def __init__(self, keys: monai.config.KeysCollection,
                 keyList: List[Hashable], newKeyName: str) -> None:
        super().__init__(keys)
        self.keyList = keyList
        self.newKeyName = newKeyName
        assert(len(keyList)>0)

    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Mapping[Hashable, np.ndarray]:

        # (if this were to be contributed, I'd have to pay attention to whether keys are in data)
        # (also I'd want to raise more clear exceptions than these asserts)

        assert(all(key in self.keys for key in self.keyList))
        assert(all(key in data.keys() for key in self.keyList))
        assert(self.newKeyName not in data.keys())

        background_mask = (sum(data[key] for key in self.keyList)==0).astype('int8')

        # Assumes these were numpy arrays.
        # If they were torch tensors we'd have to do "torch.stack" and use argument "dim" instead of "axis"
        data[self.newKeyName] = np.stack(
            [background_mask] + [data[key] for key in self.keyList],
            axis=0
        )

        return data

# Custom transform to convert a collection of segmentation mask arrays to a single mask, one-hot encoded (e.g. union left and right lung into one thing)
# Always allows missing keys, just being a no-op if any keys are missing
class UnionMasksD(monai.transforms.MapTransform):
    def __init__(self, keys: monai.config.KeysCollection,
                 keyList: List[Hashable], newKeyName: str) -> None:
        super().__init__(keys)
        self.keyList = keyList
        self.newKeyName = newKeyName
        assert(len(keyList)>0)

    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Mapping[Hashable, np.ndarray]:

        if not all(key in data.keys() for key in self.keyList):
            return data

        assert(all(key in self.keys for key in self.keyList))
        assert(self.newKeyName not in data.keys())

        sum_of_masks = sum(data[key] for key in self.keyList)
        background_mask = (sum_of_masks==0).astype('int8')
        foreground_mask = (sum_of_masks!=0).astype('int8')

        # Assumes these were numpy arrays.
        # If they were torch tensors we'd have to do "torch.stack" and use argument "dim" instead of "axis"
        data[self.newKeyName] = np.stack(
            [background_mask, foreground_mask],
            axis=0
        )

        return data

def rgb_to_grayscale(x):
    """Given a numpy array with shape (H,W,C), return "grayscale" one of shape (H,W);
    behaves as no-op if given array with shape already being (H,W)"""
    if len(x.shape)==2: return x
    elif len(x.shape)==3: return x.mean(axis=2)
    else: raise Exception("rgb_to_grayscale: unexpected number of axes in array")

# Define a custom transform that drops out blob shapes of random values sampled from the image value

def blobbed(img, R):
    """Add a random metaball blob to an image, using the random state R"""
    img_max = img.max()
    val = R.uniform(max(img.min(),img_max/2),img_max)
    number_of_metaballs = R.randint(10,11)

    metaball_radius_mean = img.shape[2]/16
    metaball_radius_std = img.shape[2]/32

    metaball_radii = R.normal(metaball_radius_mean, scale = metaball_radius_std, size = number_of_metaballs)

    blob_center_y = R.uniform(0,img.shape[1])
    blob_center_x = R.uniform(0,img.shape[2])
    blob_std_y = img.shape[2]/8
    blob_std_x = img.shape[1]/8

    metaball_y_locations = R.normal(blob_center_y, scale = blob_std_y, size = number_of_metaballs)
    metaball_x_locations = R.normal(blob_center_x, scale = blob_std_x, size = number_of_metaballs)

    y, x = np.ogrid[tuple(slice(0, i) for i in img.shape[1:])]
    mask = sum(
        r**2 / ((y-y_loc)**2 + (x-x_loc)**2)
        for r, y_loc, x_loc in zip(metaball_radii, metaball_y_locations, metaball_x_locations)
    ) > 1
    img[0][mask]=val

    return img

class RandBlobDropoutD(monai.transforms.RandomizableTransform, monai.transforms.MapTransform):
    """
    Bleh.
    """

    backend = [monai.utils.enums.TransformBackends.NUMPY]

    def __init__(
        self,
        keys: monai.config.KeysCollection,
        prob: float = 0.1,
        allow_missing_keys: bool = False,
    ):
        monai.transforms.MapTransform.__init__(self, keys, allow_missing_keys)
        monai.transforms.RandomizableTransform.__init__(self, prob=prob)

    def __call__(self, data):
        d = dict(data)
        self.randomize(None)
        if not self._do_transform:
            return d

        for key in self.key_iterator(d):
            d[key] = blobbed(d[key], self.R)

        return d

# A custom version of list_data_collate copied straight out of MONAI codebase and slightly modified.
def list_data_collate_no_meta(batch: Sequence):
    """
    Enhancement for PyTorch DataLoader default collate.
    If dataset already returns a list of batch data that generated in transforms, need to merge all data to 1 list.
    Then it's same as the default collate behavior.

    Note:
        Need to use this collate if apply some transforms that can generate batch data.

    """
    elem = batch[0]
    data = [i for k in batch for i in k] if isinstance(elem, list) else batch
    key = None
    try:
        elem = batch[0]
        if isinstance(elem, Mapping):
            ret = {}
            for k in elem:
                key = k

                # Ebrahim's modification: skip any meta_dicts. Collating them is glitchy in my use-case.
                if "_meta_dict" in key: continue

                ret[k] = default_collate([d[k] for d in data])
            return ret
        return default_collate(data)
    except RuntimeError as re:
        re_str = str(re)
        if "equal size" in re_str:
            if key is not None:
                re_str += f"\nCollate error on the key '{key}' of dictionary data."
            re_str += (
                "\n\nMONAI hint: if your transforms intentionally create images of different shapes, creating your "
                + "`DataLoader` with `collate_fn=pad_list_data_collate` might solve this problem (check its "
                + "documentation)."
            )
        raise RuntimeError(re_str) from re
    # except TypeError as re:
    #     re_str = str(re)
    #     if "numpy" in re_str and "Tensor" in re_str:
    #         if key is not None:
    #             re_str += f"\nCollate error on the key '{key}' of dictionary data."
    #         re_str += (
    #             "\n\nMONAI hint: if your transforms intentionally create mixtures of torch Tensor and numpy ndarray, "
    #             + "creating your `DataLoader` with `collate_fn=pad_list_data_collate` might solve this problem "
    #             + "(check its documentation)."
    #         )
    #     raise TypeError(re_str) from re


import pytest

from xyzspaces.utils import get_xyz_token

XYZ_TOKEN = get_xyz_token()

@pytest.mark.skipif(not XYZ_TOKEN, reason="No token found.")
def test_get_alias(api, space_id):
    """Get space statistics."""
    path = f"/hub/spaces/{space_id}/statistics"
    stats = api.get(path=path).json()
    assert stats["type"] == "StatisticsResponse"


import json
import os
import time

from nova.openstack.common import lockutils

TWENTY_FOUR_HOURS = 3600 * 24

@lockutils.synchronized('storage-registry-lock', 'nova-', external=True)
def register_storage_use(storage_path, hostname):
    """Idenfity the id of this instance storage."""

    # NOTE(mikal): this is required to determine if the instance storage is
    # shared, which is something that the image cache manager needs to
    # know. I can imagine other uses as well though.

    d = {}
    id_path = os.path.join(storage_path, 'compute_nodes')
    if os.path.exists(id_path):
        with open(id_path) as f:
            d = json.loads(f.read())

    d[hostname] = time.time()

    with open(id_path, 'w') as f:
        f.write(json.dumps(d))

@lockutils.synchronized('storage-registry-lock', 'nova-', external=True)
def get_storage_users(storage_path):
    """Get a list of all the users of this storage path."""

    d = {}
    id_path = os.path.join(storage_path, 'compute_nodes')
    if os.path.exists(id_path):
        with open(id_path) as f:
            d = json.loads(f.read())

    recent_users = []
    for node in d:
        if time.time() - d[node] < TWENTY_FOUR_HOURS:
            recent_users.append(node)

    return recent_users


from warnings import catch_warnings
from flask import Flask, request, render_template, jsonify
import pickle
import pandas as pd
import sklearn
from flask_cors import CORS

app = Flask(__name__)
CORS(app)
model = pickle.load(open("model.pkl", "rb"))

@app.route("/")
@app.route("/home")
def home():
    return jsonify({
        "about": "This is a Flask REST API for predicting diabetes",
        "written_by": "Prodipto Roy Dipto",
        "method": "GET/POST",
        "url": "base_url/predict",
        "query_params": {
            "age": "number",
            "gender": "1/0",
            "polyuria": "1/0",
            "polydipsia": "1/0",
            "sudden": "1/0",
            "itching": "1/0",
            "irritability": "1/0",
            "delayed": "1/0",
            "partial": "1/0",
            "alopecia": "1/0"
        }
    })

@app.route("/predict", methods=["GET", "POST"])
def predict():
    try:
        # pregnancy = float(request.form["pregnancy"])
        # glucose = int(request.form['glucose'])
        # response.headers.add('Access-Control-Allow-Origin', '*')

        prediction = model.predict([[float(request.args['age']),
                                float(request.args['gender']),
                                float(request.args['polyuria']),
                                float(request.args['polydipsia']),
                                float(request.args['sudden']),
                                float(request.args['itching']),
                                float(request.args['irritability']),
                                float(request.args['delayed']),
                                float(request.args['partial']),
                                float(request.args['alopecia'])
                            ]])

        output = round(prediction[0])

        if output == 0:
            return jsonify({'label': 0})
        else:
            return jsonify({'label': 1})
    except Exception:
        return jsonify({
                "message": "Not enough or wrong type of data provided!"
            })

@app.route("/androidpredict", methods=["GET", "POST"])
def androidpredict():
    try:
        prediction = model.predict([[float(request.form['age']),
                                float(request.form['gender']),
                                float(request.form['polyuria']),
                                float(request.form['polydipsia']),
                                float(request.form['sudden']),
                                float(request.form['itching']),
                                float(request.form['irritability']),
                                float(request.form['delayed']),
                                float(request.form['partial']),
                                float(request.form['alopecia'])
                            ]])

        output = round(prediction[0])

        if output == 0:
            return "No"
        else:
            return "Yes"
    except Exception:
        return jsonify({
                "message": "Not enough or wrong type of data provided!"
            })

if __name__ == "__main__":
    app.run(debug=True)

"""
import flask
from flask import request, render_template, jsonify
from flask_cors import CORS
import pickle

app = flask.Flask(__name__)
CORS(app)

model = pickle.load(open("model.pkl", "rb"))

@app.route('/')
def home():
    return '<h1>API is working.. </h1>'

@app.route('/predict',methods=['GET','POST'])
def predict():
    prediction = model.predict([[float(request.args['pregnancy']),
                            float(request.args['glucose']),
                            float(request.args['bloodpressure']),
                            float(request.args['thickness']),
                            float(request.args['insulin']),
                            float(request.args['bmi']),
                            float(request.args['pedigree']),
                            float(request.args['age'])
                           ]])

    return str(round(prediction[0]))

if __name__ == "__main__":
    app.run(debug=True)
"""


from urllib.parse import urljoin
import requests
from pprint import pprint

# insert your token below
TOKEN = ''
VERSION = '5.124'
API_BASE_URL = 'https://api.vk.com/method/'

class Vk_Friends:
    BASE_URL = API_BASE_URL

    def __init__(self, token=TOKEN, version=VERSION):
        self.token = token
        self.version = version

    def get_mutual_friends(self, other_id):
        mutual_friends_url = urljoin(API_BASE_URL, 'friends.getMutual')
        result = requests.get(mutual_friends_url, params={
            'target_uid': other_id,
            'access_token': self.token,
            'v': self.version
        })

        return result.json()['response']

if __name__ == '__main__':
    myself = Vk_Friends()
    pprint(myself.get_mutual_friends(372488475))


#!/usr/bin/python3
''' Function that queries the Reddit API '''
import requests

def top_ten(subreddit):
    '''
    Prints the titles of the first 10 hot posts listed for a given subreddit.
    '''
    url = "https://www.reddit.com/r/{}/hot.json?limit=10".format(subreddit)
    # Set a custom User-Agent header as Reddit requires it
    headers = {"User-Agent": "MyRedditBot"}

    response = requests.get(url, headers=headers)

    # Check if the subreddit exists and the request was successful
    if response.status_code == 200:
        data = response.json()
        posts = data["data"]["children"]
        for post in posts:
            title = post["data"]["title"]
            print(title)
    else:
            print("None")


import numpy as np
import pandas as pd
import re

from recordclass import RecordClass
from typing import Any, Dict, List, NamedTuple, Optional, Set, Text

from state_data_mapper import nces_metadata_joiner_lib

class ColumnMapping(NamedTuple):
  """Defines how to process columns in state data source file."""
  # Processed output file column name.
  target_column: Text
  # Source file column name.
  source_column: Optional[Text] = None
  # Pandas data type (for casting and validation).
  dtype: Any = None
  # Converter function/lambda.
  converter: Any = None
  # Converter dictionary. If provided, will create a converter that maps values
  # according to this dictionary (if the key is present, otherwise keep the
  # original value).
  converter_dict: Dict[Any, Any] = None
  # Constant value for target column (if provided, source_column is ignored).
  constant: Any = None
  # Expected null values in source column.
  na_values: Optional[List] = None
  # Is temporary column. Will be use for intermediate manipulation, but will not
  # be in the final data frame.
  is_temporary: bool = False
  # Calculate this column from other columns.
  calculation: Any = None
  # List of values for rows that need to be filtered out.
  filter_values: List[Any] = None
  # Drop rows from the data frame if this column value is NA.
  dropna: bool = False

class StateConfig(NamedTuple):
  """Contains configuration for a state data source file."""
  # State name, e.g. "Colorado".
  state: Text
  # State abbreviation, e.g. "CO".
  state_abbreviation: Text
  # State data source XLSX filepath. Expects sheet named "Data for {state}".
  source_filepath: Text
  # Processed output CSV filepath.
  target_filepath: Text
  # List of column mapping from source file to target file.
  column_mappings: List[ColumnMapping] = []
  # Source sheet name containing data. If not specified, then read from
  # "Data for {state}".
  source_sheet_name: Optional[Text] = None
  # If more than one, source sheet name(s) containing data.
  # If not specified, then read from "Data for {state}".
  source_sheet_names_list: List[Text] = []
  # If True, then output rows are deduped.
  dedupe_rows: bool = True
  # Filter values file path. XLSX file whose rows with matching values will be
  # filtered out of the case data file.
  filter_values_file: Text = None
  # Filter value file columns to match on.
  filter_values_file_match_on: List[Text] = []
  # Filter value file columns to match on with fuzzy text matching.
  filter_values_file_fuzzy_match_on: List[Text] = []
  nces_id_lookup_file: Text = None
  nces_id_lookup_file_match_on: List[Text] = []
  nces_id_lookup_file_fuzzy_match_on: List[Text] = []

class ColumnReadReport(RecordClass):
  """Report summarizing the processed output column."""
  state: Text               # State name.
  column: Text              # Column name in the processed output.
  dtype: Any                # Pandas data type.
  count: int = None         # Number of non-null values.
  null_count: float = None  # Number of null values.
  min: Any = None           # Minimum.
  max: Any = None           # Maximum.
  mean: Any = None          # Mean (only defined for numeric columns).
  mode: Any = None          # Mode.

def converter_dict_getter(converter_dict: Dict[Any, Any]):
  """Returns a function that looks up a key in a dict or returns the key if
  the value is not found."""
  def get(x: Any):
    return converter_dict.get(x, x)
  return get

def coerce_int(value):
  coerced_numeric_value = pd.to_numeric(value, errors="coerce")
  if (pd.notna(coerced_numeric_value) and
      int(coerced_numeric_value) == coerced_numeric_value):
    return str(int(coerced_numeric_value))
  else:
    return value

def normalize(s: Text) -> Text:
  """Normalize a string for easier matching."""
  if not isinstance(s, str):
    return s
  tokens = list()
  for token in re.split('[^a-zA-Z0-9]', s.lower()):
    if token:
      tokens.append(token.strip())
  return " ".join(tokens)

def strip_and_clean_string(input: Text) -> Text:
  """Remove newlines and trailing spaces."""
  return " ".join(input.split())

def left_join(
    left_df: pd.DataFrame,
    right_df: pd.DataFrame,
    on: List[Text],
    on_fuzzy_match_columns: List[Text]):
  """Left join data frames on given columns. Handles string fuzzy match.

  Args:
    left_df: left data frame
    right_df: right data frame.
    on: list of columns to match on.
    on_fuzzy_match_columns: additional string columns to fuzzy match on.

  Returns:
    Left joined data frame.
  """
  assert set(on).issubset(left_df.columns)
  assert set(on).issubset(right_df.columns)
  assert not set(on_fuzzy_match_columns).issubset(on)
  left_df = left_df.copy(deep=True)
  right_df = right_df.copy(deep=True)
  normalized_columns = []
  # Create temporary string columns with normalized versions of the strings for
  # fuzzy matching.
  for column in on_fuzzy_match_columns:
    assert column in left_df.columns
    assert column in right_df.columns
    column_normalized = "_normalized_" + column
    normalized_columns.append(column_normalized)
    left_df[column_normalized] = left_df[column].transform(normalize)
    right_df[column_normalized] = right_df[column].transform(normalize)
    # Drop the original column from the right data frame (we don't need it
    # anymore now that the normalized version of it available.
    right_df = right_df.drop(column, axis=1)

  merged_df = left_df.merge(right_df, on=on + normalized_columns, how="left")
  # Drop the normalized columns from the merged dataframe.
  for column in normalized_columns:
    merged_df.drop(column, axis=1, inplace=True)
  return merged_df

def filter_matching_rows(
    left_df: pd.DataFrame,
    right_df: pd.DataFrame,
    on: List[Text],
    on_fuzzy_match_columns: List[Text],
    filter_out=True):
  """Filter matching rows from left data frame.

  Args:
    left_df: left data frame
    right_df: right data frame.
    on: list of columns to match on.
    on_fuzzy_match_columns: additional string columns to fuzzy match on.
    filter_out: If True, matching rows are filtered out. If False,
                non-matching rows are filtered out.

  Returns:
    Left data frame without matching rows from right data frame.
  """
  right_df = right_df[on + on_fuzzy_match_columns].copy(deep=True)
  right_df["_drop"] = True
  merged_df = left_join(left_df, right_df, on, on_fuzzy_match_columns)
  if filter_out:
    merged_df = merged_df[merged_df["_drop"].isna() |
                          (merged_df["_drop"] == False)]
  else:
    merged_df = merged_df[merged_df["_drop"] == True]
  merged_df.drop("_drop", axis=1, inplace=True)
  return merged_df

def process_fixed_length_codes(codes: Text, length: int) -> Text:
  """Ensures NCES codes are fixed length strings of numbers.

  If there is more than one code, then they should be comma-separated.
  """
  return ",".join([str(int(float(code.strip()))).zfill(length)
                   for code in codes.split(",")])

def process_state_data(
    state_config: StateConfig,
    required_columns: List[Text]=[],
    nces_school_metadata: Text=None,
    nces_district_metadata: Text=None
):
  """Read and process data for a single state based on config.

  Args:
    state_config: configuration for state data source.
    required_columns: List of mandatory output columns. If no data is available,
                      they will be empty in the output.
    nces_school_metadata: XLSX filepath containing NCESSchoolID, NCESDistrictID,
                          Charter, SchoolType columns.
    nces_district_metadata: XLSX filepath containing, NCESDistrictID, Charter,
                            DistrictType columns.

  Returns:
    (df, report_df) where df is the processed DataFrame and report_df is the
    DataFrame report summarizing the data.
  """

  # Process column mappings configuration.
  source_dtype_map = dict()
  target_dtype_map = dict()
  usecols = list()
  converters = dict()
  constant_columns = dict()
  column_rename_map = dict()
  na_values = dict()
  temporary_columns = list()
  for mapping in state_config.column_mappings:
    if mapping.is_temporary:
      temporary_columns.append(mapping.target_column)

    if mapping.constant:
      constant_columns[mapping.target_column] = mapping.constant
      assert not mapping.source_column, (
          f"For column {mapping.target_column} provide "
          "either constant or source column.")
      if mapping.dtype:
        target_dtype_map[mapping.target_column] = mapping.dtype
      continue
    if not mapping.source_column:
      continue

    # If this is a calculated column, then no need to read it from XLSX.
    if mapping.calculation:
      continue

    # Pandas XLSX read parameters
    usecols.append(mapping.source_column)
    column_rename_map[mapping.source_column] = mapping.target_column
    if mapping.converter_dict:
      converters[mapping.source_column] = converter_dict_getter(
          mapping.converter_dict)
    elif mapping.converter:
      converters[mapping.source_column] = mapping.converter
    elif mapping.dtype:
      source_dtype_map[mapping.source_column] = mapping.dtype
    if mapping.na_values:
      na_values[mapping.source_column] = mapping.na_values

  if state_config.source_sheet_names_list:
    df_list = []
    for sheet_name in state_config.source_sheet_names_list:
      # Read XLSX file.
      partial_df = pd.read_excel(
          state_config.source_filepath,
          # Sheet with this name must exist.
          sheet_name=sheet_name,
          usecols=usecols,  # Drop unmapped columns.
          dtype=source_dtype_map,  # Cast columns
          converters=converters,  # Drop unmapped columns.
          na_values=na_values  # Null values in source file.
      )
      df_list.append(partial_df)
    df = pd.concat(df_list)
  else:
    sheet_name = state_config.source_sheet_name
    if not sheet_name:
      sheet_name = f"Data for {state_config.state}"

    # Read XLSX file.
    df = pd.read_excel(
        state_config.source_filepath,
        # Sheet with this name must exist.
        sheet_name=sheet_name,
        usecols=usecols,  # Drop unmapped columns.
        dtype=source_dtype_map,  # Cast columns
        converters=converters,  # Drop unmapped columns.
        na_values=na_values  # Null values in source file.
    )

  # Rename columns.
  df.rename(columns=column_rename_map, errors="raise", inplace=True)

  # Constant columns.
  for column, value in constant_columns.items():
    df[column] = value
    if column in target_dtype_map:
      df[column] = df[column].astype(target_dtype_map[column])

  # Calculated columns.
  for mapping in state_config.column_mappings:
    if mapping.calculation:
      df[mapping.target_column] = df.apply(mapping.calculation, axis=1).astype(
          mapping.dtype)

  # Reorder columns in order of the config.
  df = df[[mapping.target_column for mapping in state_config.column_mappings]]

  # Validate dtypes.
  for mapping in state_config.column_mappings:
    if mapping.dtype:
      assert df[mapping.target_column].equals(
          df[mapping.target_column].astype(mapping.dtype)), (
          f"Invalid dtype '{mapping.dtype}' for "
          f"column '{mapping.target_column}' which has type "
          f"{df[mapping.target_column].dtype}.")

  # Remove rows that need to be filtered out from filter values in the config.
  for mapping in state_config.column_mappings:
    if mapping.filter_values:
      df = df[~df[mapping.target_column].isin(set(mapping.filter_values))]
    if mapping.dropna:
      df.dropna(subset=[mapping.target_column])

  # Remove rows that need to be filtered out based on the filter XLSX file.
  if state_config.filter_values_file:
    filter_values_df = pd.read_excel(state_config.filter_values_file)
    filter_columns = state_config.filter_values_file_match_on +\
                     state_config.filter_values_file_fuzzy_match_on
    assert filter_columns
    assert set(filter_columns).issubset(df.columns)
    assert set(filter_columns).issubset(filter_values_df.columns)
    df = filter_matching_rows(
        df,
        filter_values_df,
        state_config.filter_values_file_match_on,
        state_config.filter_values_file_fuzzy_match_on)

  # Dates should be mm/dd/YY.
  for mapping in state_config.column_mappings:
    if str(mapping.dtype).startswith("datetime64"):
      df[mapping.target_column] = df[mapping.target_column].dt.strftime(
          "%m/%d/%y")

  # Dedupe rows.
  if state_config.dedupe_rows:
    df.drop_duplicates(inplace=True)

  # Strip and clean up SchoolName, DistrictName, and City if present.
  for column in ["SchoolName", "DistrictName", "City"]:
    if column in df:
      df[column] = df[column].map(
          lambda value: strip_and_clean_string(value)
          if pd.notna(value) else value
      )

  # For integers in object type columns, cast as value as integer to avoid '.0'
  # suffix.
  for column in df:
    if not column.startswith("NCES") and df[column].dtype == object:
      df[column] = df[column].map(coerce_int)

  # Join NCES IDs
  if state_config.nces_id_lookup_file:
    nces_id_lookup_df = pd.read_excel(
        state_config.nces_id_lookup_file,
        dtype={
            "NCESSchoolID": pd.StringDtype(),
            "NCESDistrictID": pd.StringDtype(),
        })
    # Ensure that the NCES codes are strings.
    # if "NCESDistrictID" in nces_id_lookup_df:
    #   nces_id_lookup_df["NCESDistrictID"] =\
    #     nces_id_lookup_df["NCESDistrictID"].map(
    #       lambda value: process_fixed_length_codes(value, 7)
    #       if pd.notna((value)) else value
    #     )
    # if "NCESSchoolID" in nces_id_lookup_df:
    #   nces_id_lookup_df["NCESSchoolID"] = nces_id_lookup_df["NCESSchoolID"].map(
    #       lambda value: process_fixed_length_codes(value, 12)
    #       if pd.notna((value)) else value
    #   )
    filter_columns = state_config.nces_id_lookup_file_match_on + \
                     state_config.nces_id_lookup_file_fuzzy_match_on
    assert filter_columns
    assert set(filter_columns).issubset(df.columns)
    assert set(filter_columns).issubset(nces_id_lookup_df.columns)
    df = left_join(
        df,
        nces_id_lookup_df,
        state_config.nces_id_lookup_file_match_on,
        state_config.nces_id_lookup_file_fuzzy_match_on)

  # Join additional NCES metadata.
  # Either both nces_school_metadata and nces_district_metadata are available
  # or both are unavailable
  assert ((nces_school_metadata and nces_district_metadata) or (
      not nces_school_metadata and not nces_district_metadata))
  if nces_school_metadata and nces_district_metadata:
    nces_school_metadata_df = pd.read_excel(
        nces_school_metadata,
        usecols=["NCESSchoolID", "NCESDistrictID", "SchoolType", "Charter"],
        dtype=pd.StringDtype()
    )
    nces_district_metadata_df = pd.read_excel(
        nces_district_metadata,
        usecols=["NCESDistrictID", "DistrictType", "Charter"],
        dtype=pd.StringDtype()
    )
    if "NCESSchoolID" in df:
      assert "NCESDistrictID" in df
      df = nces_metadata_joiner_lib.join_nces_metadata_for_schools(
          df, nces_school_metadata_df, nces_district_metadata_df)
    elif "NCESDistrictID" in df:
      df = nces_metadata_joiner_lib.join_nces_metadata_for_districts(
          df, nces_district_metadata_df)

  # Drop temporary columns.
  df.drop(temporary_columns, axis=1, inplace=True)

  # Create report.
  report_rows = list()
  for column in df:
    column_report = ColumnReadReport(
        state_config.state,
        column,
        dtype=df[column].dtype,
        count=df[column].count(),
        null_count=df[column].isna().sum())
    column_report.column = column
    if df[column].dtype == object:
      column_report.min = df[column].astype("string").min()
      column_report.max = df[column].astype("string").max()
    else:
      column_report.min = df[column].min()
      column_report.max = df[column].max()
    mode = list(df[column].mode())
    column_report.mode = mode[0] if mode else np.NaN
    if pd.api.types.is_numeric_dtype(df[column]):
      column_report.mean = df[column].mean()
    report_rows.append(column_report)

  # Mandatory columns.
  for column in required_columns:
    if column not in df:
      df[column] = pd.Series(dtype="object")
      column_report = ColumnReadReport(
          state_config.state,
          column,
          dtype="object",
          count=df[column].count(),
          null_count=df[column].isna().sum())
      report_rows.append(column_report)

  # Reorder columns in the global column order.
  unexpected_columns = []
  if required_columns:
    for column in required_columns:
      assert(column in df.columns), column
    for column in df.columns:
      if column not in required_columns:
        print("Following column was unexpected:", column)
        unexpected_columns.append(column)
    df = df[required_columns + unexpected_columns]

  report_df = pd.DataFrame.from_records(
      report_rows,
      columns=["state", "column", "dtype", "count",
               "null_count", "min", "max", "mean", "mode"])

  # Write file.
  if state_config.target_filepath:
    df.to_excel(state_config.target_filepath, index=False)
    df.to_csv(state_config.target_filepath.replace(".xlsx", ".csv"),
              index=False)

  return df, report_df

def process_all_states(
    config: List[StateConfig],
    report_filepath: Text=None,
    required_columns: List[Text]=[],
    states_to_process: List[Text]=[],
    nces_school_metadata: Text=None,
    nces_district_metadata: Text=None
):
  """Process all states in list of configs.

  Args:
    config: List of StateConfigs defining how to process each file.
    report_filepath: If provided, then write the processed data summary
                     report to this path.
    required_columns: List of mandatory output columns. If no data is available,
                      they will be empty in the output.
    states_to_process: List of states to process from the config. If not
                       provided, then all states from the config will be
                       processed.
    nces_school_metadata: XLSX filepath containing NCESSchoolID, NCESDistrictID,
                          Charter, SchoolType columns.
    nces_district_metadata: XLSX filepath containing, NCESDistrictID, Charter,
                            DistrictType columns.

  Returns:
    (state_dfs, state_report_dfs): state_dfs is the list of processed state
    DataFrames and state_report_dfs is the list corresponding reports.
  """
  states_to_process = [state.lower() for state in states_to_process]
  state_dfs = list()
  state_report_dfs = list()
  for state_config in config:
    if states_to_process:
      if state_config.state.lower() not in states_to_process and \
          state_config.state_abbreviation.lower() not in states_to_process:
        continue
    print(f"Reading {state_config.state} from {state_config.source_filepath}.")
    df, report_df = process_state_data(
        state_config, required_columns, nces_school_metadata,
        nces_district_metadata)
    state_dfs.append(df)
    state_report_dfs.append(report_df)
    pd.set_option("display.max_columns", None, "display.width", 300)
  if report_filepath:
    print(f"Writing processed read reports to {report_filepath}...")
    pd.concat(state_report_dfs).to_csv(report_filepath, index=False)
  return state_dfs, state_report_dfs

def read_config_file(config_filepath: Text):
  """Read config file."""
  with open(config_filepath, "r") as f:
    contents = f.read()
    return eval(contents)


from django.urls import path
from tenant import views

app_name = 'tenant'

urlpatterns = [
    # Home
    path('', views.Home_Tenant, name='tenant'),
    path('no-dorm', views.no_dorm.as_view(), name='no_dorm'),
    path('dorm/', views.has_dorm.as_view(), name='has_dorm'),

    # Messages
    path('messages/', views.Messages_Tenant, name='messages'),
    path('no-dorm/messages', views.No_Dorm_Messages.as_view(), name='no_dorm-messages'),
    path('dorm/messages', views.Has_Dorm_Messages.as_view(), name='has_dorm-messages'),
    path('dorm/messages/list', views.messages_list, name='messages_list'),

    path('favorites/', views.TenantFavorites.as_view(), name='tenant_favorites'),
    path('search/<int:order>/', views.TenantDormSearch.as_view(), name='dorm_search'),
    path('search/map/', views.tenant_map, name='tenant_map'),
    path('search/property/<int:pk>/', views.ViewPropertyDetailView.as_view(), name='view_property'),
    path('search/property/favorite/<int:pk>/', views.favorite_property, name='favorite_property'),

    # NOTE Temporary, created so i can view my template
    path('search/property/application/<int:pk>', views.ApplicationCreateView.as_view(), name='application_form'),
    path('dorm/request', views.Request.as_view(), name='request'),

    # path('dorm/messages/individual', views.TenantIndMessages.as_view(), name='individual_messages'),
    path('dorm/messages/<str:room_name>/', views.tenant_ind_messages, name='room'),
    path('dorm/messages/room/create/', views.create_room, name='create_room'),

    path('dorm/history/', views.HistoryListView.as_view(), name='payment_history'),

]


import copy
import functools
import importlib

import tensorflow as tf

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.training import basic_session_run_hooks
from tensorflow.python.training import session_run_hook
from tensorflow.python.training import saver
from tensorflow.python.training import device_setter

from tensorflow.core.framework import node_def_pb2

from tensorflow.python.framework import device as pydev
from tensorflow.python.framework import ops

from tensorflow.python.ops import math_ops
from tensorflow.python.ops import array_ops

from .optimizer import get_optimizer_fn

from dvolver import *

from dvolver.representations.nasneta import nasnet_utils

from dvolver.representations.nasneta.nasnet import *
from dvolver.representations.nasneta.nasnet import _update_hparams, _build_nasnet_base

tf_nasnet = importlib.import_module('dvolver.representations.nasneta.nasnet')

arg_scope = tf.contrib.framework.arg_scope
slim = tf.contrib.slim

def dvolver_nas_config(model_params):
  return tf.contrib.training.HParams(
      stem_multiplier=model_params.stem_multiplier,
      drop_path_keep_prob=model_params.drop_path_keep_prob,
      num_cells=model_params.num_cells,
      use_aux_head=model_params.use_aux_head,
      num_conv_filters=model_params.num_conv_filters,
      dense_dropout_keep_prob=model_params.dense_dropout_keep_prob,
      filter_scaling_rate=model_params.filter_scaling_rate,
      num_reduction_layers=model_params.num_reduction_layers,
      data_format='NCHW',
      skip_reduction_layer_input=model_params.skip_reduction_layer_input,
      # This is used for the drop path probabilities since it needs to increase
      # the drop out probability over the course of training.
      total_training_steps=model_params.max_steps,
  )

def get_nasnet_arg_scope(model_type):
  return getattr(tf_nasnet,
                 'nasnet_' + model_type + '_arg_scope')

def get_used_hiddenstates(hiddenstate_indices, B):
  used_hiddenstates = [0] * (2 + B)

  for i in hiddenstate_indices:
    used_hiddenstates[i] = 1

  return used_hiddenstates

def get_normal_cell_params(params):
  assert len(params) % 2 == 0
  return params[:len(params)//2]

def get_reduction_cell_params(params):
  assert len(params) % 2 == 0
  return params[len(params)//2:]

class Dvolver_NasNetACell(nasnet_utils.NasNetABaseCell):
  """Dvolver NASNetA Generic (Normal/Reduction) Cell."""

  def __init__(self, params, num_conv_filters, drop_path_keep_prob, total_num_cells,
               total_training_steps):
    assert (len(params)-1) % 4 == 0 # 4 parameters per block

    B = int((len(params)-1) / 4)

    regular_topology = params[:-1]
    extra_concat = params[-1:][0]

    operations = regular_topology[1::2]
    hiddenstate_indices = regular_topology[0::2]
    used_hiddenstates = get_used_hiddenstates(hiddenstate_indices, B)

    for i in extra_concat:
      used_hiddenstates[i] = 0 # add extra connection for h_i

    super(Dvolver_NasNetACell, self).__init__(num_conv_filters, operations,
                                              used_hiddenstates,
                                              hiddenstate_indices,
                                              drop_path_keep_prob,
                                              total_num_cells,
                                              total_training_steps)

def get_additional_cells(model_type):
  if model_type == 'cifar':
    return 0
  else:
    return 2

def get_stem_type(model_type):
  if model_type == 'cifar':
    return 'cifar'
  else:
    return 'imagenet'

def dvolver_build_nasnet(model_type,
                         params,
                         images,
                         num_classes,
                         is_training=True,
                         final_endpoint=None,
                         config=None):
  """Build NASNet model"""

  if not config:
    raise ValueError('config should not be None')

  hparams = copy.deepcopy(config)
  _update_hparams(hparams, is_training)

  # Calculate the total number of cells in the network
  # Add the reduction cells
  total_num_cells = hparams.num_cells + hparams.num_reduction_layers
  # If ImageNet, then add stem cells
  total_num_cells += get_additional_cells(model_type)

  normal_cell = Dvolver_NasNetACell(get_normal_cell_params(params),
                                    hparams.num_conv_filters,
                                    hparams.drop_path_keep_prob,
                                    total_num_cells,
                                    hparams.total_training_steps)
  reduction_cell = Dvolver_NasNetACell(get_reduction_cell_params(params),
                                       hparams.num_conv_filters,
                                       hparams.drop_path_keep_prob,
                                       total_num_cells,
                                       hparams.total_training_steps)

  with arg_scope([slim.dropout, nasnet_utils.drop_path, slim.batch_norm],
                 is_training=is_training):
    with arg_scope([slim.avg_pool2d,
                    slim.max_pool2d,
                    slim.conv2d,
                    slim.batch_norm,
                    slim.separable_conv2d,
                    nasnet_utils.factorized_reduction,
                    nasnet_utils.global_avg_pool,
                    nasnet_utils.get_channel_index,
                    nasnet_utils.get_channel_dim],
                   data_format=hparams.data_format):
      return _build_nasnet_base(images,
                                normal_cell=normal_cell,
                                reduction_cell=reduction_cell,
                                num_classes=num_classes,
                                hparams=hparams,
                                is_training=is_training,
                                stem_type=get_stem_type(model_type),
                                final_endpoint=final_endpoint)

def get_network_fn(model_type, params, num_classes, weight_decay=0.0, is_training=False):
  """Returns a network_fn such as `logits, end_points = network_fn(images)`.
  Args:
    model_type: NASNet-A model type (cifar, mobile or large)
    params: DVOLVER cell representation
    num_classes: The number of classes to use for classification. If 0 or None,
      the logits layer is omitted and its input features are returned instead.
    weight_decay: The l2 coefficient for the model weights.
    is_training: `True` if the model is being used for training and `False`
      otherwise.
  Returns:
    network_fn: A function that applies the model to a batch of images. It has
      the following signature:
          net, end_points = network_fn(images)
      The `images` input is a tensor of shape [batch_size, height, width, 3]
      with height = width = network_fn.default_image_size. (The permissibility
      and treatment of other sizes depends on the network_fn.)
      The returned `end_points` are a dictionary of intermediate activations.
      The returned `net` is the topmost layer, depending on `num_classes`:
      If `num_classes` was a non-zero integer, `net` is a logits tensor
      of shape [batch_size, num_classes].
      If `num_classes` was 0 or `None`, `net` is a tensor with the input
      to the logits layer of shape [batch_size, 1, 1, num_features] or
      [batch_size, num_features]. Dropout has not been applied to this
      (even if the network's original classification does); it remains for
      the caller to do this or not.
  Raises:
    ValueError: If network `name` is not recognized.
  """
  @functools.wraps(dvolver_build_nasnet)
  def network_fn(images, **kwargs):
    arg_scope = get_nasnet_arg_scope(model_type)(weight_decay=weight_decay)
    with slim.arg_scope(arg_scope):
      return dvolver_build_nasnet(model_type, params, images, num_classes, is_training=is_training, **kwargs)

  return network_fn

def get_model_fn(rep_params, model_params, enable_summary):
  """
  Returns a function that will build the NASNet-A model
  Args:
    * rep_params: list of parameters with the following representation:
              B = 5 # number of blocks per cell
              bi = block i
              h0 = hidden state on the left for the block
              h1 = hidden state on the right for the block
              op0 = left operation for the block
              op1 = right operation for the block
              combination = always ADD
              individual representation: [# normal cell
                                          b0.h0, b0.op0, b0.h1, b0.op1,
                                          b1.h0, b1.op0, b1.h1, b1.op1,
                                          b2.h0, b2.op0, b2.h1, b2.op1,
                                          b3.h0, b3.op0, b3.h1, b3.op1,
                                          b4.h0, b4.op0, b4.h1, b4.op1,
                                          # reduction cell
                                          b0.h0, b0.op0, b0.h1, b0.op1,
                                          b1.h0, b1.op0, b1.h1, b1.op1,
                                          b2.h0, b2.op0, b2.h1, b2.op1,
                                          b3.h0, b3.op0, b3.h1, b3.op1,
                                          b4.h0, b4.op0, b4.h1, b4.op1,
                                         ]

  """

  optimizer_fn = get_optimizer_fn(model_params, rep_params)

  weight_decay = model_params.weight_decay
  nb_classes = model_params.nb_classes
  data_format = model_params.data_format
  label_smoothing = model_params.label_smoothing
  aux_weight = model_params.aux_weight
  clip_gradient_norm = model_params.clip_gradient_norm
  train_mode = TrainMode(model_params.train_mode)
  total_training_steps = model_params.max_steps
  model_type = model_params.model_type

  config = dvolver_nas_config(model_params)

  def inference(features, mode):
    """Pure inference function"""

    network_fn = get_network_fn(model_type,
                                rep_params,
                                num_classes=nb_classes,
                                weight_decay=weight_decay,
                                is_training=(mode == tf.estimator.ModeKeys.TRAIN))

    inputs = features

    # if data_format == 'channels_first'
    # Reshape X to 4-D tensor: [batch_size, channels, height, width]
    # if data_format == 'channels_last'
    # Reshape X to 4-D tensor: [batch_size, height, width, channels]

    if data_format == 'channels_last':
      inputs = tf.transpose(inputs, [0, 2, 3, 1], name='transposeNCHW-NHWC')
      config.set_hparam('data_format', 'NHWC')

    logits, end_points = network_fn(inputs,
                                    config=config)

    return logits, end_points

  def inference_wrapper(features, mode):
    logits, _ = inference(features, mode)

    return logits

  def model_fn(features, labels, mode, params):
    """NASNet-A model body.
    Args:
      features: a list of tensors, one for each tower
      labels: a list of tensors, one for each tower
      mode: ModeKeys.TRAIN or EVAL
      params: Hyperparameters suitable for tuning
    Returns:
      A EstimatorSpec object.
    """

    optimizer = optimizer_fn()

    logits, end_points = inference(features, mode)

    # if enable_summary:
    #   tf.summary.image('input', tf.transpose(features, [0, 2, 3, 1]), max_outputs=10)

    # compute total_loss
    losses = []

    # get current scopes to avoid cross tower stuff in replicated multi gpu training
    current_var_scope = tf.get_variable_scope()

    # Auxiliary Loss
    with tf.name_scope('aux_loss'):
      if 'AuxLogits' in end_points:
        onehot_labels = tf.one_hot(labels, nb_classes)
        aux_loss = tf.losses.softmax_cross_entropy(logits=end_points['AuxLogits'],
                                                   onehot_labels=onehot_labels,
                                                   weights=aux_weight,
                                                   label_smoothing=label_smoothing)
        losses.append(aux_loss)
        if enable_summary:
          tf.summary.scalar('aux_loss', aux_loss)

    # Crosse entropy loss
    with tf.name_scope('train_loss'):
      cross_loss = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels, weights=1.0)
      if enable_summary:
        tf.summary.scalar('cross_loss', cross_loss)

    # regularization losses
    regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
    if enable_summary:
      tf.summary.scalar('regularization_loss', tf.add_n(regularization_losses, name='regularization_losses'))

    losses += [cross_loss] + regularization_losses

    # final total loss
    total_loss = tf.add_n(losses, name='total_loss')

    # needed for batch norm
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

    if model_params.moving_average_decay:
      moving_average_variables = slim.get_model_variables(current_var_scope)
      variable_averages = tf.train.ExponentialMovingAverage(model_params.moving_average_decay,
                                                            tf.train.get_global_step())
      update_ops.append(variable_averages.apply(moving_average_variables))

    with tf.control_dependencies(update_ops):
      #train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
      gradients_and_vars = optimizer.compute_gradients(total_loss,
                                                       var_list=tf.trainable_variables(current_var_scope.name))

      gradients = [gradient for gradient, var in gradients_and_vars]

      gradients, _ = tf.clip_by_global_norm(gradients, clip_gradient_norm)

      gradients_and_vars = [(gradient, x[1]) for gradient, x in zip(gradients, gradients_and_vars)]

      train_op = optimizer.apply_gradients(gradients_and_vars, global_step=tf.train.get_global_step())

    # Calculate root mean squared error as additional eval metric
    class_ids = math_ops.argmax(logits, 1, name="class_ids")
    class_ids = array_ops.expand_dims(class_ids, axis=(1,))
    weights = 1
    if enable_summary:
      eval_metric_ops = {
        "accuracy": tf.metrics.accuracy(labels=labels, predictions=class_ids, weights=weights, name="accuracy"),
        "top 5": tf.metrics.recall_at_k(tf.cast(labels,tf.int64), logits, 5),
      }
    else:
      eval_metric_ops = {}

    # Provide an estimator spec for `ModeKeys.EVAL` and `ModeKeys.TRAIN` modes.
    return tf.estimator.EstimatorSpec(mode=mode,
                                      loss=total_loss,
                                      train_op=train_op,
#                                      training_hooks=train_hooks,
                                      eval_metric_ops=eval_metric_ops)

  return inference_wrapper, model_fn


# -*- coding: utf-8 -*-
# @Time    : 2018/9/8 22:20
# @Author  : geekspeng
# @Email   : geekspeng@icloud.com
import os
from datetime import datetime

from iso639 import languages as isoLanguages
from markupsafe import Markup

from app import db, uploads
from app.utils import helper

books_authors_link = db.Table(
    'books_authors_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('author', db.Integer, db.ForeignKey('authors.id'), primary_key=True)
)

books_tags_link = db.Table(
    'books_tags_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('tag', db.Integer, db.ForeignKey('tags.id'), primary_key=True)
)

books_series_link = db.Table(
    'books_series_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('series', db.Integer, db.ForeignKey('series.id'), primary_key=True)
)

books_ratings_link = db.Table(
    'books_ratings_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('rating', db.Integer, db.ForeignKey('ratings.id'), primary_key=True)
)

books_languages_link = db.Table(
    'books_languages_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('lang_code', db.Integer, db.ForeignKey('languages.id'), primary_key=True)
)

books_publishers_link = db.Table(
    'books_publishers_link',
    db.Column('book', db.Integer, db.ForeignKey('books.id'), primary_key=True),
    db.Column('publisher', db.Integer, db.ForeignKey('publishers.id'), primary_key=True)
)

class Comments(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    book = db.Column(db.Integer, db.ForeignKey('books.id'))
    text = db.Column(db.String(64))

    def __repr__(self):
        return "<Comments({0})>".format(self.text)

class Authors(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(128))
    sort = db.Column(db.String(128))
    link = db.Column(db.String(128), default='')

    def __repr__(self):
        return "<Authors('{0},{1}{2}')>".format(self.name, self.sort, self.link)

    @staticmethod
    def get_author_by_name(name):
        return Authors.query.filter_by(name=name).first()

    @staticmethod
    def add_author(name):
        db_author = Authors.get_author_by_name(name)
        if not db_author:
            db_author = Authors(name=name, sort=helper.get_sorted_author(name))
            db.session.add(db_author)
            db.session.commit()

class Data(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    book = db.Column(db.Integer, db.ForeignKey('books.id'))
    format = db.Column(db.String(20))
    uncompressed_size = db.Column(db.Integer)
    name = db.Column(db.String(128))

    def __repr__(self):
        return "<Data('{0},{1}{2}{3}')>".format(self.book, self.format, self.uncompressed_size, self.name)

    @staticmethod
    def get_data_by_format(file_format):
        return Data.query.filter_by(format=file_format).first()

class Identifiers(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    type = db.Column(db.String(128))
    val = db.Column(db.String(128))
    book = db.Column(db.Integer, db.ForeignKey('books.id'))

    def format_type(self):
        if self.type == "amazon":
            return "Amazon"
        elif self.type == "isbn":
            return "ISBN"
        elif self.type == "doi":
            return "DOI"
        elif self.type == "goodreads":
            return "Goodreads"
        elif self.type == "google":
            return "Google Books"
        elif self.type == "kobo":
            return "Kobo"
        else:
            return self.type

    def __repr__(self):
        if self.type == "amazon":
            return "https://amzn.com/{0}".format(self.val)
        elif self.type == "isbn":
            return "http://www.worldcat.org/isbn/{0}".format(self.val)
        elif self.type == "doi":
            return "http://dx.doi.org/{0}".format(self.val)
        elif self.type == "goodreads":
            return "http://www.goodreads.com/book/show/{0}".format(self.val)
        elif self.type == "douban":
            return "https://book.douban.com/subject/{0}".format(self.val)
        elif self.type == "google":
            return "https://books.google.com/books?id={0}".format(self.val)
        elif self.type == "kobo":
            return "https://www.kobo.com/ebook/{0}".format(self.val)
        else:
            return ""

class Languages(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    lang_code = db.Column(db.String(20))

    def __repr__(self):
        return "<Languages('{0}')>".format(self.lang_code)

    @staticmethod
    def get_language_by_code(lang_code):
        return Languages.query.filter_by(lang_code=lang_code).first()

    @staticmethod
    def add_language(lang_code):
        db_language = Languages.get_language_by_code(lang_code)
        if not db_language:
            db_language = Languages(lang_code=lang_code)
            db.session.add(db_language)
            db.session.commit()

class Publishers(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(128))
    sort = db.Column(db.String(128))

    def __repr__(self):
        return "<Publishers('{0},{1}')>".format(self.name, self.sort)

class Ratings(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    rating = db.Column(db.Integer)

    def __repr__(self):
        return "<Ratings('{0}')>".format(self.rating)

class Series(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(128))
    sort = db.Column(db.String(128), default='')

    def __repr__(self):
        return u"<Series('{0},{1}')>".format(self.name, self.sort)

    @staticmethod
    def get_series_by_name(name):
        return Series.query.filter_by(name=name).first()

    @staticmethod
    def add_series(name):
        db_series = Series.get_series_by_name(name)
        if not db_series:
            db_series = Series(name=name)
            db.session.add(db_series)
            db.session.commit()

class Tags(db.Model):
    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    name = db.Column(db.String(32))

    def __repr__(self):
        return "<Tags('{0})>".format(self.name)

    @staticmethod
    def get_tag_by_name(name):
        return Tags.query.filter_by(name=name).first()

    @staticmethod
    def add_tag(name):
        db_tag = Tags.get_tag_by_name(name=name)
        if not db_tag:
            db_tag = Tags(name)
            db.session.add(db_tag)
            db.session.commit()

class Books(db.Model):
    DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00"

    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(128))
    sort = db.Column(db.String(128), default='')
    author_sort = db.Column(db.String(128))
    timestamp = db.Column(db.String(128), default=datetime.utcnow)
    pubdate = db.Column(db.String(128), default=datetime(101, 1, 1))
    series_index = db.Column(db.String(128))
    last_modified = db.Column(db.String(128), default=datetime.utcnow)
    path = db.Column(db.String(128))
    has_cover = db.Column(db.Boolean,  default=False)
    uuid = db.Column(db.String(128))

    authors = db.relationship('Authors', secondary=books_authors_link, backref='books')
    tags = db.relationship('Tags', secondary=books_tags_link, backref='books')
    comments = db.relationship('Comments', backref='books')
    data = db.relationship('Data', backref='books')
    series = db.relationship('Series', secondary=books_series_link, backref='books')
    ratings = db.relationship('Ratings', secondary=books_ratings_link, backref='books')
    languages = db.relationship('Languages', secondary=books_languages_link, backref='books')
    publishers = db.relationship('Publishers', secondary=books_publishers_link, backref='books')
    identifiers = db.relationship('Identifiers', backref='books')

    def __repr__(self):
        return "<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
                                                                self.timestamp, self.pubdate, self.series_index,
                                                                self.last_modified, self.path, self.cover_path)

    @property
    def atom_timestamp(self):
        return (self.timestamp or '').replace(' ', 'T')

    @staticmethod
    def delete_book(book_id):
        db_book = Books.query.filter_by(id=book_id).first()
        if db_book:
            file_path = uploads.path(db_book.path)
            # delete readbook
            ReadBook.query.filter_by(book_id=book_id).delete()
            # delete author
            for author in db_book.authors:
                db_book.authors.remove(author)
                if len(author.books) == 0:
                    db.session.delete(author)
            # delete tag
            for tag in db_book.tags:
                db_book.tags.remove(tag)
                if len(tag.books) == 0:
                    db.session.delete(tag)
            # delete series
            for series in db_book.series:
                db_book.series.remove(series)
                if len(series.books) == 0:
                    db.session.delete(series)
            # delete language
            for language in db_book.languages:
                db_book.languages.remove(language)
                if len(language.books) == 0:
                    db.session.delete(language)
            # delete publisher
            for publisher in db_book.publishers:
                db_book.publishers.remove(publisher)
                if len(publisher.books) == 0:
                    db.session.delete(publisher)
            # delete data
            for data in db_book.data:
                db_book.data.remove(data)
                if data.books is None:
                    db.session.delete(data)
            # delete comment
            for comment in db_book.comments:
                db_book.comments.remove(comment)
                if comment.books is None:
                    db.session.delete(comment)
            db.session.delete(db_book)
            db.session.commit()
            helper.delete_book(file_path)

    @staticmethod
    def add_book(meta):
        # add book
        has_cover = True if meta.cover_path else False
        db_book = Books(title=meta.title, author_sort=helper.get_sorted_author(meta.author),
                        series_index=meta.series_id, path=meta.file_path, has_cover=has_cover)
        # add author
        if meta.author != "":
            db_author = Authors.get_author_by_name(meta.author)
            if not db_author:
                db_author = Authors(name=meta.author, sort=helper.get_sorted_author(meta.author))
                db.session.add(db_author)
            db_book.authors.append(db_author)
        # add series
        if meta.series != "":
            db_series = Series.get_series_by_name(meta.series)
            if not db_series:
                db_series = Series(name=meta.series)
                db.session.add(db_series)
            db_book.series.append(db_series)
        # add language
        if meta.languages != "":
            lang_code = isoLanguages.get(name=meta.languages).part3
            db_language = Languages.get_language_by_code(lang_code)
            if not db_language:
                db_language = Languages(lang_code=lang_code)
                db.session.add(db_language)
            db_book.languages.append(db_language)
        # add tag
        if meta.tags != "":
            for tag in meta.tags.split(','):
                db_tag = Tags.get_tag_by_name(name=tag.strip())
                if not db_tag:
                    db_tag = Tags(name=tag.strip())
                    db.session.add(db_tag)
                db_book.tags.append(db_tag)
        # add data
        db_data = Data(books=db_book, format=meta.file_extension,
                       uncompressed_size=meta.file_size, name=meta.title)
        db.session.add(db_data)
        db_book.data.append(db_data)
        # add comment
        upload_comment = Markup(meta.description).unescape()
        if upload_comment != "":
            db_comment = Comments(books=db_book, text=upload_comment)
            db.session.add(db_comment)
            db_book.comments.append(db_comment)
        db.session.add(db_book)
        db.session.commit()
        return db_book

    @staticmethod
    def search_book(term):
        return db.session.query(Books).filter(
            db.or_(Books.tags.any(Tags.name.ilike("%" + term + "%")),
                   Books.series.any(Series.name.ilike("%" + term + "%")),
                   Books.authors.any(Authors.name.ilike("%" + term + "%")),
                   Books.publishers.any(Publishers.name.ilike("%" + term + "%")),
                   Books.title.ilike("%" + term + "%"))).all()

    @staticmethod
    # get the pagination of books from database
    def get_books_pagination(page, per_page, db_filter, order):
        pagination = Books.query.filter(db_filter).order_by(order).paginate(
            page, per_page=per_page, error_out=False)
        return pagination

    # Convert existing book entry to new format
    @staticmethod
    def convert_book_format(book_id, convert_tool_path, old_book_format, new_book_format):
        db_book = Books.query.get(book_id)
        db_data = Data.query.filter_by(book=book_id, format=old_book_format).first()
        if db_book is None or db_data is None:
            return "{format} format not found for this book".format(format=old_book_format)
        old_file_name = db_data.name + '.' + old_book_format.lower()
        new_file_name = db_data.name + '.' + new_book_format.lower()
        old_file_path = uploads.path(old_file_name, db_book.path)
        new_file_path = uploads.path(new_file_name, db_book.path)
        if os.path.isfile(old_file_path):
            if not os.path.isfile(new_file_name):
                result = helper.convert_book(convert_tool_path, old_file_path, new_file_path)
                if result is None:
                    new_format = Data(book=db_book.id, name=db_book.data[0].name, format=new_book_format,
                                      uncompressed_size=os.path.getsize(new_file_path))
                    db_book.data.append(new_format)
                    db.session.commit()
                return result
            else:
                return "{format} format already exists".format(format=new_file_name)
        else:
            return "{format} not found: {fn}".format(format=old_book_format, fn=db_book.title + "." + old_book_format.lower())

class ReadBook(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), unique=False)
    book_id = db.Column(db.Integer, db.ForeignKey('books.id'), unique=False)
    is_read = db.Column(db.Boolean, unique=False)

class Bookmark(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'),  unique=False)
    book_id = db.Column(db.Integer, db.ForeignKey('books.id'),  unique=False)
    format = db.Column(db.String(20))
    bookmark_key = db.Column(db.String(128))


from collections import deque

class Solution:
    def numOfMinutes(
        self, n: int, headID: int, manager: List[int], informTime: List[int]
    ) -> int:
        tree = [[] for _ in range(n)]
        q = deque()
        for i in range(n):
            if manager[i] == -1:
                q.append(i)
            else:
                tree[manager[i]].append(i)

        time = [0] * n
        ans = 0
        while q:
            v = q.popleft()
            ans = max(ans, time[v])
            for w in tree[v]:
                time[w] = time[v] + informTime[v]
                q.append(w)

        return ans


#!/usr/bin/env python3

import argparse
import os
import sys

importlogging

import frameoverframe.autotrace
import frameoverframe.autotrace_sequence

log = logging.getLogger("frameoverframe")

def collect_args():
    parser = argparse.ArgumentParser(description="process videos frame by frame.")
    parser.add_argument("input", default=None, nargs="*", help="video file, mov,mp4,m4a etc")
    parser.add_argument(
        "-c",
        "--centerline",
        action="store_true",
        default=False,
        dest="centerline",
        help="Trace the centerline of blobs. Produces lines only. ",
    )
    parser.add_argument(
        "-s",
        "--scalefactor",
        type=int,
        default=50,
        help="resize the image size before autotracing. \
                        Scale is a percentage. 50=half size, defalut = 50. \
                        4k images must be scaled down or autotrace will crash.",
    )
    parser.add_argument(
        "-j",
        "--jobs",
        default="0",
        type=int,
        help="Number of parallel prcesseses to spawn. default=0 (same as cpu count)",
    )
    return parser

def main():
    """do the main thing"""

    parser = collect_args()

    args = parser.parse_args()

    if not args.input:
        parser.print_help()
        return 0

    for single_input in args.input:

        #  print('single_input=', single_input)

        single_input = os.path.abspath(single_input)

        if os.path.isfile(single_input):
            frameoverframe.autotrace_sequence.process_file(single_input, args)

        if os.path.isdir(single_input):
            print("Cannot process directories. input must be a video file.")
            return 1

    return 0

if __name__ == "__main__":
    sys.exit(main())


#!/usr/bin/env python
import os.path
import tornado.escape
import tornado.httpserver
import tornado.ioloop
import tornado.options

import json
import handlers

from tornado.options import define

define("port", default=5000, help="run on the given port", type=int)

class Application(tornado.web.Application):
    def __init__(self):
        urls = [
            (r"/issues/(\d+)/subscribe", handlers.SubscribeHandler),
            (r"/issues/?", handlers.IssuesHandler),
            (r"/?", handlers.MainHandler),
            (r"/(callbacks/github)?", handlers.GithubCallbackHandler)
        ]
        settings = dict(
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            debug=True,
        )
        tornado.web.Application.__init__(self, urls, **settings)

def main():
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(Application())
    http_server.listen(tornado.options.options.port)

    tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    main()


import tool
import json
import util
import requests

__all__ = ['GeosMETA']

def processGetResponse(response):
    """Utility method to process the requests response
    for get methods.

    :param response: requests response object
    :returns: JSON from response if response contains it
    :raises Exception: if problem processing response
                       or error received from server
    """

    code = requests.codes.ok
    return processResponse(response, code)

def processPostResponse(response):
    """Utility method to process the requests response
    for post methods.

    :param response: requests response object
    :returns: JSON from response if response contains it
    :raises Exception: if problem processing response
                       or error received from server
    """

    code = requests.codes.created
    return processResponse(response, code)

def processResponse(response, statusCode):
    """Utility method to process the requests response.

    :param response: requests response object
    :param statusCode: HTTP status code to check response against
    :returns: JSON from response if response contains it
    :raises Exception: if problem processing response
                       or error received from server
    """

    if response.status_code == statusCode:
        try:
            responseJSON = response.json()
        except ValueError:
            responseJSON = None
        return responseJSON
    else:
        # Gets status codes 4XX and 5XX only

        # moving to seek to hold onto the err message from server
        # as it gives validation errors.
        # response.raise_for_status()

        message = "Status code: %s" %(response.status_code)
        try:
            responseJSON = response.json()
        except Exception:
            responseJSON = None
            response.raise_for_status() # maybe here is not the place...??
        if responseJSON is not None:
# why make a string? just return reponse??? Mike Jan 2016
#...exception at least until python 3 needs a str...
# could make one we can then interpret back into  json.... ??
            reason = json.dumps(responseJSON) # this not present with 404 code ['_issues'])
            message += "\nReason: " + reason
        raise Exception(message)

class GeosMETA(object):
    def __init__(self, configFilePath = None):
        """Constructor.

        :param configFilePath: The location of the geosmeta configuration
                               file. If not present this defaults to
                               $HOME/.geosmeta/geosmeta.cfg
        """

        # Tool object to talk to the GeosMeta server
        self.tool = tool.Tool(configFilePath=configFilePath)

    def getAccounts(self):
        """Get account details from system.

        :returns: JSON response
        :raises Exception: if problem getting accounts
        """
        message = ''
        resource = 'accounts'
        response = self.tool.performGet(resource, message)

        return processGetResponse(response)

    def getAccount(self,username):
        """Get account details from system for a user.

        :param username: username to lookup
        :returns: JSON response
        :raises Exception: if problem getting accounts
        """
        message = ''
        resource = 'accounts/' + username
        response = self.tool.performGet(resource, message)

        return processGetResponse(response)

    def postAccount(self,accountJSON):
        """Post user account to database.

        :param accountJSON: JSON containing account details
        :returns: requests response object
        """
        return self.tool.performPost('accounts', accountJSON)

    def createUser(self,firstname, lastname, username, email, secret, roles):
        """Create a user in the Geosmeta system

        :param firstname: Firstname of the user
        :param lastname: Lastname of the user
        :param username: Username of the user - becomes id of user doc
        :param email: Email of the user
        :param secret: Secret key
        :param roles: List of roles
        :returns: id of user if creation successful
        :raises Exception: if problem creating user
        """
        account = {'_id': username,
                   'firstname': firstname,
                   'lastname': lastname,
                   'email': email,
                   'secret': secret,
                   'roles': roles
                   }
        response = self.postAccount(json.dumps(account))

        return processPostResponse(response)

    def getProjectRoles(self,project_id):
        """Get the roles for a project.
        :param project_id: project to get roles from
        :returns: JSON response
        :raises Exception: if problem getting the document
        """

        message = ''
        resource = 'project_roles/%s'%project_id
        #+ projectName
#        params = {'where': '{"project": "' +  project_id + '"}' }
        response = self.tool.performGet(resource, message)#, params)

        return processGetResponse(response)

    def fromAbbrevToResource(self, abbrev):
        if abbrev=="A":
            resource = 'activities'
        elif abbrev=="F":
            resource= 'gmfiles'
        elif abbrev=="E":
            resource= 'entities'
            print("entities not in use - Stopping.")
            sys.exit()
        else:
            print("bad collection specified")
            sys.exit()
        return resource

    def getDoc(self,collection,projectName, id,  selectedFields=None, verbose=False):
        """Get a document.
        :param collection
        :param projectName: project to get activity from
        :param id: id to retrieve
        :returns: JSON response
        :raises Exception: if problem getting the document
        """
        if projectName ==None:
                  projectName=self.tool.config._projectname

        message = ''
        params= ''
        resource = "%s/%s"%(self.fromAbbrevToResource(collection), id)

        hdr_ext={"project": projectName}
        if selectedFields:
             hdr_ext["selectedFields"]=selectedFields
        if verbose:
            print(resource)
            print(params)
            print(message)
        response = self.tool.performGet(resource, message, hdr_ext=hdr_ext)

        return processGetResponse(response)
    def getActivity(self, projectName, id):
        return self.getDoc("A",projectName, id)

    def getProjectCollection(self,collection,projectName, selectedFields=None):
        """Get all collection's documents for a given project.
        :param projectName: project to get activities from
        :returns: JSON response
        :raises Exception: if problem getting the document
        """
        if projectName ==None:
                  projectName=self.tool.config._projectname

        resource = self.fromAbbrevToResource(collection)

        hdr_ext={"project": projectName}
        if selectedFields:
             hdr_ext["selectedFields"]=selectedFields
        message = ''
        params = {'where': '{"project": "' + projectName + '"}'}
        #print params
        response = self.tool.performGet(resource, message, params=params, hdr_ext=hdr_ext)

        return processGetResponse(response)

    def getHREF(self,href, projectName,selectedFields=None, verbose=False):
        """Get all activity documents for a given project.
        :returns: JSON response
        :raises Exception: if problem getting the document
        """
#        import pdb; pdb.set_trace()
        message = ''
        resource = href.split('?')[0] #'activities'
        params =href.split('?')[1]
        if verbose:
            print("getHREF params: "+params)
            print("resource:" + resource)
        if projectName ==None:
                  projectName=self.tool.config._projectname

        hdr_ext={"project": projectName}
        if selectedFields:
             hdr_ext["selectedFields"]=selectedFields

        response = self.tool.performGet(resource, message, params,hdr_ext)

        return processGetResponse(response)

    def findActivities(self,collection, projectName, query, kwargs):
        """Find/search activity documents in the project that satisfies the query .
        :param projectName: project to get activities from
        :param query: query parameters
        :returns: JSON response
        :raises Exception: if problem getting the documents
        """
        if projectName ==None:
                  projectName=self.tool.config._projectname

        hdr_ext={"project": projectName}
        if kwargs:
          if 'selectedFields' in kwargs:
             hdr_ext["selectedFields"]=kwargs["selectedFields"]
          if 'summary' in kwargs:
              hdr_ext["summaryOnly"]='T'
#        print "hdr_ext "+str(hdr_ext)
        message = ''
        resource = self.fromAbbrevToResource(collection)

        params = {'where': '{"$and":[{"project": "' + projectName + '"},{' + query + '}]}'}
        print(params)

        response = self.tool.performGet(resource, message,
                                         params=params, hdr_ext=hdr_ext)
        return processGetResponse(response)

    def findFileUse(self,projectName,filename,testField):
        """ Look for filename in the specified list (in/output_files)
            and return the number and ids of docs that have
            status of Current
        """
        if testField == "input_file_docs" or testField =="output_file_docs":
           query='"$and":[{"%s.filename":"%s"}]'%(testField,filename)
           kwargs={'summaryOnly':'T'}
           opdict=self.findActivities('A',projectName, query, kwargs)
		# assuming not more than 25 docs returned as not handling that.
           currentList=list()
           for iitem in opdict["_items"]:
              if iitem:
                 if iitem["gmstatus"] == "Current":
                    currentList.append(iitem["_id"])
           return opdict["_meta"]["total"], currentList

        else:
           print("bad argument to findFileUse")
           print("must be 'input_files' or 'output_files'")
           return -1,[]

    def getProject(self,project_id=None):
        """Get projects from the database.
        :param project_id: project to get details for (optional)
        :returns: JSON response
        :raises Exception: if problem getting the project
        """

        message = ''
        if project_id ==None:
                  project_id=self.tool.config._projectname

        resource = 'projects/%s'%project_id
        response = self.tool.performGet(resource, message)

        return processGetResponse(response)

    def getProjects(self,project_id=None, researchGroupName=None):
        """Still in use?? Get projects from the database.
        :param project_id: project to get details for (optional)
        :param researchGroupName: research group to get details for (optional)
        :returns: JSON response
        :raises Exception: if problem getting the project
        """

        message = ''
        resource = 'projects'
        if project_id ==None:
                  project_id=self.tool.config._projectname
        if project_id is None:
            if researchGroupName is None:
                params = None
            else:
                params = {'where': '{"research_group": "' + researchGroupName +'"}'}
        else:
            if researchGroupName is None:
                params = {'where': '{"title": "' + projectName +'"}'}
            else:
                params = {'where': '{"research_group": "' + researchGroupName
                            + '", "title": "' + projectName + '"}'}

        response = self.tool.performGet(resource, message, params)

        return processGetResponse(response)

    def getResearchGroups(self,researchGroupName=None):
        """Get Research Groups from the database.
        :param researchGroupName: research group to get details for (optional)
        :returns: JSON response
        :raises Exception: if problem getting the project
        """

        message = ''
        resource = 'research_groups'
        if researchGroupName is None:
            params = None
        else:
            params = {'where': '{"title": "' + researchGroupName +'"}'}

        response = self.tool.performGet(resource, message, params)

        return processGetResponse(response)

    def postResearchGroup(self,researchGroupJSON):
        return self.tool.performPost('research_groups', researchGroupJSON)

    def addResearchGroup(self,title, description, comment):
        """ Adds a new research group

        :param title: Unique (required) title of the research group
        :param description: Description of research group
        :param comment: Comment
        :return: id of the research group if created successfully
        """
        creator = self.tool.config.username

        research_group = {'title': title,
                          'creator': creator}

        if description is not None:
            research_group['shortDescription'] = description

        if comment is not None:
            research_group['comment'] = comment

        response = self.postResearchGroup(json.dumps(research_group))

        return processPostResponse(response)

    def postProject(self,projectJSON):
        return self.tool.performPost('projects', projectJSON)

    def postProjectRoles(self,projectRolesJSON):
        return self.tool.performPost('project_roles', projectRolesJSON)

    def addProject(self,project_id, researchGroup, description, comment):
        """ Adds a new project

        :param project_id: Unique title of the project (required, no spaces allowed)
        :param researchGroup: Name of research group to add project to
        :param description: Description of project
        :param comment: Comment
        :return: id of the project if created successfully
        """
        creator = self.tool.config.username

        project = {'_id': project_id,
                   'research_group': researchGroup,
                   'creator': creator}

        if description is not None:
            project['shortDescription'] = description

        if comment is not None:
            project['comment'] = comment

        response = self.postProject(project )#json.dumps(project))

        processPostResponse(response)

        response_json = response.json()
        if response_json['_status'] == "OK":
            project_roles = {'_id': project_id,
                         'read_access': [],
                         'write_access': []}

            responseRoles = self.postProjectRoles(project_roles)#json.dumps(project_roles))

            processPostResponse(responseRoles)

            response_json = responseRoles.json()
            if response_json['_status'] == "OK":
                id = response_json['_id']
                return id
            else:
                raise Exception("Roles Status not OK")
        else:
            raise Exception("New project status not OK")

    def postActivity(self,activityJSON):
        return self.tool.performPost('activities', activityJSON)

    def patchActivity(self,resource, changesJSON, etag):
        return self.tool.performPatch(resource, changesJSON, etag)

    def updateDocStatus(self,activityID, etag, newStatus, message,goDownStream):
        """ Updates a gmDoc's status field, and optionally change status
recirsively down the tree
        :param activityID: _id of the gmDoc to be changed
        :param etag: correct _etag of the current document
        :param newStatus: C(urretn) or E(rror)
        :param message: brief description of why status is changed
        :param goDownStream: True if all docs downstream of this are to have
new status
        :return: id of the activity if update successful
        """
        import time
        stati={'C':"Current",'E':"Error"} # already validated.
        resource = 'activities/' + activityID

        changes = {'statNew':
                         {'message':message,
                          "status":stati[newStatus],
                          "byWhom":self.tool.config.username,
                          "goDownStream":goDownStream}}

        changesJSON = json.dumps(changes)

        return self.patchActivity(resource, changesJSON, etag)

    def updateActivity(self,activityID, etag, field, value):
        """ Updates an activity field with a value provided.
        :param activityID: _id of the activity document to be changed
        :param etag: correct _etag of the current document
        :param field: field to be changed
        :param value: new value for the field
        :return: id of the activity if update successful
        """

        resource = 'activities/' + activityID
        changes = {field: value}
        changesJSON = json.dumps(changes)

        return self.patchActivity(resource, changesJSON, etag)

    def addDoc(self,project, metaFields):
        """ Adds a new document

        :param project: optional: Project the activity is associated with
        :param metaFields: A valid JSON file or a python dict
        :return: id of the activity document if created successfully
        """
        creator = self.tool.config.username

        if isinstance(metaFields,str):
          with open(metaFields) as theFile:
            theContent = json.load(theFile)
        elif isinstance(metaFields,dict):
             theContent = metaFields
        else:
               response = "cant use argument metaFields: Not JSON file or python dict - quitting"
               return response

        if project ==None:
                  project=self.tool.config._projectname

        activity = { 'project': project,
                    'creator': creator,
                    'gmstatus': "Current",
                              # it has to be current if we are writign it!
                    'gmdata': theContent}
        response = self.postActivity(activity)#json.dumps(activity))

        return processPostResponse(response)