"""Check Docker storage driver and usage."""
import json
import re
from openshift_checks import OpenShiftCheck, OpenShiftCheckException
from openshift_checks.mixins import DockerHostMixin

class DockerStorage(DockerHostMixin, OpenShiftCheck):
    """Check Docker storage driver compatibility.

    This check ensures that Docker is using a supported storage driver,
    and that loopback is not being used (if using devicemapper).
    Also that storage usage is not above threshold.
    """

    name = "docker_storage"
    tags = ["health", "preflight"]

    dependencies = ["python-docker-py"]
    storage_drivers = ["devicemapper", "overlay", "overlay2"]
    max_thinpool_data_usage_percent = 90.0
    max_thinpool_meta_usage_percent = 90.0
    max_overlay_usage_percent = 90.0

    # TODO(lmeyer): mention these in the output when check fails
    configuration_variables = [
        (
            "max_thinpool_data_usage_percent",
            "For 'devicemapper' storage driver, usage threshold percentage for data. "
            "Format: float. Default: {:.1f}".format(max_thinpool_data_usage_percent),
        ),
        (
            "max_thinpool_meta_usage_percent",
            "For 'devicemapper' storage driver, usage threshold percentage for metadata. "
            "Format: float. Default: {:.1f}".format(max_thinpool_meta_usage_percent),
        ),
        (
            "max_overlay_usage_percent",
            "For 'overlay' or 'overlay2' storage driver, usage threshold percentage. "
            "Format: float. Default: {:.1f}".format(max_overlay_usage_percent),
        ),
    ]

    def run(self):
        msg, failed = self.ensure_dependencies()
        if failed:
            return {
                "failed": True,
                "msg": "Some dependencies are required in order to query docker storage on host:\n" + msg
            }

        # attempt to get the docker info hash from the API
        docker_info = self.execute_module("docker_info", {})
        if docker_info.get("failed"):
            return {"failed": True,
                    "msg": "Failed to query Docker API. Is docker running on this host?"}
        if not docker_info.get("info"):  # this would be very strange
            return {"failed": True,
                    "msg": "Docker API query missing info:\n{}".format(json.dumps(docker_info))}
        docker_info = docker_info["info"]

        # check if the storage driver we saw is valid
        driver = docker_info.get("Driver", "[NONE]")
        if driver not in self.storage_drivers:
            msg = (
                "Detected unsupported Docker storage driver '{driver}'.\n"
                "Supported storage drivers are: {drivers}"
            ).format(driver=driver, drivers=', '.join(self.storage_drivers))
            return {"failed": True, "msg": msg}

        # driver status info is a list of tuples; convert to dict and validate based on driver
        driver_status = {item[0]: item[1] for item in docker_info.get("DriverStatus", [])}

        result = {}

        if driver == "devicemapper":
            result = self.check_devicemapper_support(driver_status)

        if driver in ['overlay', 'overlay2']:
            result = self.check_overlay_support(docker_info, driver_status)

        return result

    def check_devicemapper_support(self, driver_status):
        """Check if dm storage driver is supported as configured. Return: result dict."""
        if driver_status.get("Data loop file"):
            msg = (
                "Use of loopback devices with the Docker devicemapper storage driver\n"
                "(the default storage configuration) is unsupported in production.\n"
                "Please use docker-storage-setup to configure a backing storage volume.\n"
                "See http://red.ht/2rNperO for further information."
            )
            return {"failed": True, "msg": msg}
        result = self.check_dm_usage(driver_status)
        return result

    def check_dm_usage(self, driver_status):
        """Check usage thresholds for Docker dm storage driver. Return: result dict.
        Backing assumptions: We expect devicemapper to be backed by an auto-expanding thin pool
        implemented as an LV in an LVM2 VG. This is how docker-storage-setup currently configures
        devicemapper storage. The LV is "thin" because it does not use all available storage
        from its VG, instead expanding as needed; so to determine available space, we gather
        current usage as the Docker API reports for the driver as well as space available for
        expansion in the pool's VG.
        Usage within the LV is divided into pools allocated to data and metadata, either of which
        could run out of space first; so we check both.
        """
        vals = dict(
            vg_free=self.get_vg_free(driver_status.get("Pool Name")),
            data_used=driver_status.get("Data Space Used"),
            data_total=driver_status.get("Data Space Total"),
            metadata_used=driver_status.get("Metadata Space Used"),
            metadata_total=driver_status.get("Metadata Space Total"),
        )

        # convert all human-readable strings to bytes
        for key, value in vals.copy().items():
            try:
                vals[key + "_bytes"] = self.convert_to_bytes(value)
            except ValueError as err:  # unlikely to hit this from API info, but just to be safe
                return {
                    "failed": True,
                    "values": vals,
                    "msg": "Could not interpret {} value '{}' as bytes: {}".format(key, value, str(err))
                }

        # determine the threshold percentages which usage should not exceed
        for name, default in [("data", self.max_thinpool_data_usage_percent),
                              ("metadata", self.max_thinpool_meta_usage_percent)]:
            percent = self.get_var("max_thinpool_" + name + "_usage_percent", default=default)
            try:
                vals[name + "_threshold"] = float(percent)
            except ValueError:
                return {
                    "failed": True,
                    "msg": "Specified thinpool {} usage limit '{}' is not a percentage".format(name, percent)
                }

        # test whether the thresholds are exceeded
        messages = []
        for name in ["data", "metadata"]:
            vals[name + "_pct_used"] = 100 * vals[name + "_used_bytes"] / (
                vals[name + "_total_bytes"] + vals["vg_free_bytes"])
            if vals[name + "_pct_used"] > vals[name + "_threshold"]:
                messages.append(
                    "Docker thinpool {name} usage percentage {pct:.1f} "
                    "is higher than threshold {thresh:.1f}.".format(
                        name=name,
                        pct=vals[name + "_pct_used"],
                        thresh=vals[name + "_threshold"],
                    ))
                vals["failed"] = True

        vals["msg"] = "\n".join(messages or ["Thinpool usage is within thresholds."])
        return vals

    def get_vg_free(self, pool):
        """Determine which VG to examine according to the pool name. Return: size vgs reports.
        Pool name is the only indicator currently available from the Docker API driver info.
        We assume a name that looks like "vg--name-docker--pool";
        vg and lv names with inner hyphens doubled, joined by a hyphen.
        """
        match = re.match(r'((?:[^-]|--)+)-(?!-)', pool)  # matches up to the first single hyphen
        if not match:  # unlikely, but... be clear if we assumed wrong
            raise OpenShiftCheckException(
                "This host's Docker reports it is using a storage pool named '{}'.\n"
                "However this name does not have the expected format of 'vgname-lvname'\n"
                "so the available storage in the VG cannot be determined.".format(pool)
            )
        vg_name = match.groups()[0].replace("--", "-")
        vgs_cmd = "/sbin/vgs --noheadings -o vg_free --units g --select vg_name=" + vg_name
        # should return free space like "  12.00g" if the VG exists; empty if it does not

        ret = self.execute_module("command", {"_raw_params": vgs_cmd})
        if ret.get("failed") or ret.get("rc", 0) != 0:
            raise OpenShiftCheckException(
                "Is LVM installed? Failed to run /sbin/vgs "
                "to determine docker storage usage:\n" + ret.get("msg", "")
            )
        size = ret.get("stdout", "").strip()
        if not size:
            raise OpenShiftCheckException(
                "This host's Docker reports it is using a storage pool named '{pool}'.\n"
                "which we expect to come from local VG '{vg}'.\n"
                "However, /sbin/vgs did not find this VG. Is Docker for this host"
                "running and using the storage on the host?".format(pool=pool, vg=vg_name)
            )
        return size

    @staticmethod
    def convert_to_bytes(string):
        """Convert string like "10.3 G" to bytes (binary units assumed). Return: float bytes."""
        units = dict(
            b=1,
            k=1024,
            m=1024**2,
            g=1024**3,
            t=1024**4,
            p=1024**5,
        )
        string = string or ""
        match = re.match(r'(\d+(?:\.\d+)?)\s*(\w)?', string)  # float followed by optional unit
        if not match:
            raise ValueError("Cannot convert to a byte size: " + string)

        number, unit = match.groups()
        multiplier = 1 if not unit else units.get(unit.lower())
        if not multiplier:
            raise ValueError("Cannot convert to a byte size: " + string)

        return float(number) * multiplier

    def check_overlay_support(self, docker_info, driver_status):
        """Check if overlay storage driver is supported for this host. Return: result dict."""
        # check for xfs as backing store
        backing_fs = driver_status.get("Backing Filesystem", "[NONE]")
        if backing_fs != "xfs":
            msg = (
                "Docker storage drivers 'overlay' and 'overlay2' are only supported with\n"
                "'xfs' as the backing storage, but this host's storage is type '{fs}'."
            ).format(fs=backing_fs)
            return {"failed": True, "msg": msg}

        # check support for OS and kernel version
        o_s = docker_info.get("OperatingSystem", "[NONE]")
        if "Red Hat Enterprise Linux" in o_s or "CentOS" in o_s:
            # keep it simple, only check enterprise kernel versions; assume everyone else is good
            kernel = docker_info.get("KernelVersion", "[NONE]")
            kernel_arr = [int(num) for num in re.findall(r'\d+', kernel)]
            if kernel_arr < [3, 10, 0, 514]:  # rhel < 7.3
                msg = (
                    "Docker storage drivers 'overlay' and 'overlay2' are only supported beginning with\n"
                    "kernel version 3.10.0-514; but Docker reports kernel version {version}."
                ).format(version=kernel)
                return {"failed": True, "msg": msg}
            # NOTE: we could check for --selinux-enabled here but docker won't even start with
            # that option until it's supported in the kernel so we don't need to.

        return self.check_overlay_usage(docker_info)

    def check_overlay_usage(self, docker_info):
        """Check disk usage on OverlayFS backing store volume. Return: result dict."""
        path = docker_info.get("DockerRootDir", "/var/lib/docker") + "/" + docker_info["Driver"]

        threshold = self.get_var("max_overlay_usage_percent", default=self.max_overlay_usage_percent)
        try:
            threshold = float(threshold)
        except ValueError:
            return {
                "failed": True,
                "msg": "Specified 'max_overlay_usage_percent' is not a percentage: {}".format(threshold),
            }

        mount = self.find_ansible_mount(path)
        try:
            free_bytes = mount['size_available']
            total_bytes = mount['size_total']
            usage = 100.0 * (total_bytes - free_bytes) / total_bytes
        except (KeyError, ZeroDivisionError):
            return {
                "failed": True,
                "msg": "The ansible_mount found for path {} is invalid.\n"
                       "This is likely to be an Ansible bug. The record was:\n"
                       "{}".format(path, json.dumps(mount, indent=2)),
            }

        if usage > threshold:
            return {
                "failed": True,
                "msg": (
                    "For Docker OverlayFS mount point {path},\n"
                    "usage percentage {pct:.1f} is higher than threshold {thresh:.1f}."
                ).format(path=mount["mount"], pct=usage, thresh=threshold)
            }

        return {}


'''
Feed in JSON-compatible search requests, output is also JSON
--------------------------------------------------
3 options:
(1) list all the column values
    -- output type: JSON
    -- required fields: from_file=filename

(2) list all the unique row values of a specified column OR all columns
    -- output type: JSON
    -- required fields: from_file=filename&conditions=[col1, col2, ...]
    -- Notes:
            If NO condition is specified, shows unique row values of ALL columns

(3) list all the rows of specified col&row values
    -- output type: JSON
    -- required fields: from_file=filename&conditions={col1: [row1, row2, ...], col2: [row1, row2, ...], ...}
    -- Notes:
            Shows 10 random results by default, otherwise specified by row_cnt
--------------------------------------------------
test URLs:
all columns:
            http://127.0.0.1:5000/fetch_parameter_keys?from_file=Merchant_Details_ForEmese.csv
unique values:
            http://127.0.0.1:5000/fetch_unique_values?from_file=Merchant_Details_ForEmese.csv&conditions=[merch_ctry_nm, merch_regn_nm, prtnr_country, prtnr_region, payment_currency]
or          http://127.0.0.1:5000/fetch_unique_values?from_file=Merchant_Details_ForEmese.csv
specified search:
            http://127.0.0.1:5000/fetch_request_combo?from_file=Merchant_Details_ForEmese.csv&conditions={merch_ctry_nm: [UKRAINE, MEXICO, UNITED STATES OF AMERICA]}&row_cnt=5
test conditions:
            http://127.0.0.1:5000/fetch_unique_values?from_file=Merchant_Details_ForEmese.csv&conditions=[payment_currency]
            http://127.0.0.1:5000/fetch_request_combo?from_file=Merchant_Details_ForEmese.csv&conditions={merch_ctry_nm: [UNITED STATES OF AMERICA], payment_currency: ['CNY']}&row_cnt=5
test results:
                    {merch_ctry_nm: [UKRAINE, MEXICO, UNITED STATES OF AMERICA]} -- 2580
                    {merch_ctry_nm: [UKRAINE, MEXICO]} -- 120
                    {merch_ctry_nm: [MEXICO]} -- 104
                    {merch_ctry_nm: [UKRAINE]} -- 16
                    {payment_currency: ['COP', 'USD', 'AUD', 'PLN', 'MYR']} -- 1747
                    {payment_currency: ['USD']} -- 1492
                    {payment_currency: ['COP', 'AUD', 'PLN', 'MYR']} -- 255
                    {payment_currency: ['AUD']} -- 103
                    {payment_currency: ['MYR']} -- 99
                    {payment_currency: ['PLN']} -- 51
                    {ovr_flag_logourl: [T], ovr_flag_externalprofileid: [T], ovr_flag_displayname: [T], ovr_flag_websiteurl: [T], ovr_flag_customersupporturl: [T], ovr_flag_threedsdata_isthreedssuppresschallenge: [T]}
--------------------------------------------------
suggest use to get the json string:
import urllib
url = 'http://127.0.0.1:5000/fetch_request_combo?from_file=Merchant_Details_ForEmese.csv&conditions=...'
f = urllib.urlopen(url)
json_str = f.read()
'''

from flask import Flask, request, render_template, jsonify, json, session
import pandas as pd
from search_functions import *
import io

app = Flask(__name__)

global uni_contents
uni_contents = {}
global uni_rows
uni_rows = {}
global df
df = None

@app.route('/')
def index():
    return render_template("index.html")

@app.route('/col_names', methods=['GET', 'POST'])
def search1():
    if request.method == 'POST':
        try:
            fileobj = request.files['inputFile']
            filename = str(fileobj.filename)
            # stream = io.StringIO(f.stream.read().decode("UTF8"), newline=None)
            stream = io.StringIO(fileobj.stream.read().decode("UTF8"))
            global df
            df = pd.read_csv(stream, sep=",")
            global uni_contents
            uni_contents = csv_scrap(filename, df)
        except:
            return "Please provide a file name"
        try:
            col_list = col_scrap(df)
            return render_template("col_names.html", col_list=col_list)
        except:
            return "Invalid file name"
    return "error"

@app.route('/prev_unique_cols', methods=['GET', 'POST'])
def search2():
    try:
        name = request.args.get('from_file')
    except:
        return "Please provide a file name"
    try:
        uni_contents = csv_scrap(name)
    except:
        return "Invalid file: '{}' ".format(name)
    uni_rows = {}
    condition_str = request.args.get('conditions')
    if condition_str==None or condition_str=='[]':
        for col in uni_contents:
            uni_rows[col] = uni_contents[col].keys()
        return json.dumps(uni_rows)
    try:
        condition_str = condition_str.replace("'", "")
        ind, conditions = condition_parser(condition_str)
    except:
        return "Invalid condition format. ( [col1, col2, ...] or [] )"
    for col in conditions:
        try:
            uni_rows[col] = uni_contents[col].keys()
        except KeyError:
            return "Invalid column name: '{}' ".format(str(col))
    return json.dumps(uni_rows)

@app.route('/unique_cols', methods=['GET', 'POST'])
def unique_cols():
    if request.method == 'POST':
        cols_selected = request.form.getlist('multiselect')
        global uni_rows
        uni_rows = {}
        # return json.dumps(uni_contents)
        for col in cols_selected:
            try:
                uni_rows[str(col)] = uni_contents[str(col)].keys()
            except KeyError:
                return "Invalid column name: '{}' ".format(str(col))
        # return json.dumps(uni_rows)
        return render_template("unique_cols.html", uni_rows=uni_rows, uni_contents=uni_contents)
    return 'error'

@app.route('/combo_search', methods=['GET', 'POST'])
def combo_search():
    if request.method == 'POST':
        query_pairs = {}
        for col in uni_rows:
            query_pairs[col] = request.form.getlist(col)
            # print col, request.form.getlist(col)
        # return json.dumps(query_pairs)

        try:
            row_cnt = int(request.form['row_cnt'])
            if row_cnt <= 0:
                return "'row_cnt' should be >= 1 or None. \n Showing 0 result"
        except:
            row_cnt = 10
        try:
            tot_num, ans = random_lookup(uni_contents, query_pairs, row_cnt, df)
        except KeyError as e:
            if e[0] == 'row':
                return "Row value: '{}' from column: '{}'. Not found ".format(e[1][0], e[1][1])
            if e[0] == 'column':
                return "Invalid column value: '{}'  not found".format(e[1])
            return "Please check query column/row values"
        except NameError as e:
            return "Invalid filename: {}".format(e[0])
        if len(ans) == 0:
            return "No data found"
        table = request.form['table']
        if table == 'y':
            return render_template("combo_search.html", num=len(ans), tot_num=tot_num, con=dict_to_str(query_pairs),
                                   data=pd.concat(ans).to_html())
        dict = pd.concat(ans).to_dict()
        t_dict = transpose_dict(dict)
        return json.dumps(t_dict)

    return render_template('index.html')

@app.route('/send', methods=['GET', 'POST'])
def send():
    if request.method == 'POST':
        '''
            http://127.0.0.1:5000/csv_search/v1.0/search?filename=Merchant_Details_ForEmese.csv&col=payment_cardbrand&row="AMEX","DISCOVER","ELECTRON","ELO","MASTERCARD","VISA"
            :return:
            '''
        try:
            f = request.files['inputFile']
            filename = str(f.filename)
        except:
            return "Please provide a file name"
        try:
            condition_str = str(request.form['conditions']).replace("'", "")
        except:
            return "Please provide conditions"
        if not condition_str_checker_3(condition_str):
            return "Please provide a valid condition"
        try:
            row_cnt = int(request.form['row_cnt'])
            if row_cnt <= 0:
                return "'row_cnt' should be >= 1 or None. \n Showing 0 result"
        except:
            row_cnt = 10
        try:
            if len(condition_str) < 1:
                return "Please provide conditions"
            ind, conditions = condition_parser(condition_str)
        except Exception:
            return "Please check condition format \n e.g. " \
                   "conditions={col1: [row1, row2, ...], col2: [row1, row2, ...], ...} "
        try:
            ans = random_lookup(f, conditions, row_cnt)
        except KeyError as e:
            if e[0] == 'row':
                return "Row value: '{}' from column: '{}'. Not found ".format(e[1][0], e[1][1])
            if e[0] == 'column':
                return "Invalid column value: '{}'  not found".format(e[1])
            return "Please check query column/row values"
        except NameError as e:
            return "Invalid filename: {}".format(e[0])
        if len(ans) == 0:
            return "No data found"
        table = request.form['table']
        if table == 'y':
            return render_template("combo_search.html", filename=filename, con=condition_str, num=len(ans),
                                   data=pd.concat(ans).to_html())
        dict = pd.concat(ans).to_dict()
        t_dict = transpose_dict(dict)
        return json.dumps(t_dict)

    return render_template('index.html')

@app.route('/fetch_request_combo')
def search3():
    '''
    http://127.0.0.1:5000/csv_search/v1.0/search?filename=Merchant_Details_ForEmese.csv&col=payment_cardbrand&row="AMEX","DISCOVER","ELECTRON","ELO","MASTERCARD","VISA"
    :return:
    '''
    try:
        name = request.args.get('from_file')
    except:
        return "Please provide a file name"
    try:
        condition_str = request.args.get('conditions').replace("'", "")
    except:
        return "Please provide conditions"
    if not condition_str_checker_3(condition_str):
        return "Please provide a valid condition"
    try:
        row_cnt = int(request.args.get('row_cnt'))
        if row_cnt<=0:
            return "'row_cnt' should be >= 1 or None. \n Showing 0 result"
    except:
        row_cnt = 10
    try:
        if len(condition_str)<1:
            return "Please provide conditions"
        ind, conditions = condition_parser(condition_str)
    except Exception:
        return "Please check condition format \n e.g. " \
               "conditions={col1: [row1, row2, ...], col2: [row1, row2, ...], ...} "
    try:
        ans = random_lookup(name, conditions, row_cnt)
    except KeyError as e:
        if e[0]=='row':
            return "Row value: '{}' from column: '{}'. Not found ".format(e[1][0], e[1][1])
        if e[0]=='column':
            return "Invalid column value: '{}'  not found".format(e[1])
        return "Please check query column/row values"
    except NameError as e:
        return "Invalid filename: {}".format(e[0])
    if len(ans)==0:
        return "No data found"
    table = request.args.get('table')
    if table == 'true':
        return render_template("combo_search.html", filename=name, con=condition_str, num=len(ans),
                               data=pd.concat(ans).to_html())
    dict = pd.concat(ans).to_dict()
    t_dict = transpose_dict(dict)
    return json.dumps(t_dict)

if __name__ == '__main__':
    # app.secret_key = 'super secret key'
    # app.config['SESSION_TYPE'] = 'filesystem'

    app.run(debug=True, host='0.0.0.0')
    # app.run(host='0.0.0.0')


# pylint: disable=redefined-outer-name
import tempfile as tf
import typing as tp
from unittest import mock

import attr
import git
import plumbum as pb
import pytest
import testdata as td

from benchbuild import source
from benchbuild.source import FetchableSource, Variant

Variants = tp.Iterable[Variant]

class SimpleSource(FetchableSource):
    test_versions: tp.List[str] = attr.ib()

    def __init__(
        self, local: str, remote: tp.Union[str, tp.Dict[str, str]],
        test_versions: tp.List[str]
    ):
        super().__init__(local, remote)

        self.test_versions = test_versions

    @property
    def default(self) -> Variant:
        return Variant(owner=self, version=self.test_versions[0])

    def version(self, target_dir: str, version: str) -> pb.LocalPath:
        return pb.local.path('.') / f'bb-test-{version}'

    def versions(self) -> Variants:
        return [Variant(self, v) for v in self.test_versions]

@pytest.fixture
def versions_a():
    return ["1.0a", "2.0a"]

@pytest.fixture
def versions_b():
    return ["1.0b", "2.0b"]

@pytest.fixture
def src_a(versions_a):
    return SimpleSource(
        local='src_A_local', remote='src_A_remote', test_versions=versions_a
    )

@pytest.fixture
def src_b(versions_b):
    return SimpleSource(
        local='src_B_local', remote='src_B_remote', test_versions=versions_b
    )

def test_base_context(src_a):
    var = src_a.default
    ctx = source.context(var)

    assert ctx[src_a.local].owner == src_a
    assert ctx[src_a.local] == var

    with pytest.raises(KeyError):
        ctx['non-existing']  # pylint: disable=pointless-statement

def test_base_to_str(src_a, src_b, versions_a, versions_b):
    vars_a = src_a.versions()
    vars_b = src_b.versions()

    version_text = source.to_str(*vars_a)
    assert version_text == ",".join(versions_a)
    version_text = source.to_str(*vars_b)
    assert version_text == ",".join(versions_b)

    vars_ab = versions_a + versions_b
    version_text = source.to_str(*vars_a, *vars_b)
    assert version_text == ",".join(vars_ab)

def test_base_default(src_a, src_b):
    default_versions = source.default(src_a, src_b)

    assert src_a.local in default_versions
    assert default_versions[src_a.local] == src_a.default
    assert src_b.local in default_versions
    assert default_versions[src_b.local] == src_b.default

def test_base_product(src_a, src_b):
    all_versions = list(source.product(src_a, src_b))
    assert all([len(v) == 2 for v in all_versions])
    assert len(all_versions) == len(src_a.versions()) * len(src_b.versions())

def test_git_clone_needed():
    pass

def test_git_maybe_shallow():
    pass

def test_http_normalize_remote():
    pass

def test_http_versioned_target_name():
    pass

def test_http_download_single_version():
    pass

def test_http_download_required():
    pass

class VersionSource(FetchableSource):
    known_versions: tp.List[str]

    def __init__(
        self, local: str, remote: tp.Union[str, tp.Dict[str, str]],
        known_versions: tp.List[str]
    ):
        super().__init__(local, remote)

        self.known_versions = known_versions

    @property
    def default(self) -> Variant:
        return Variant(owner=self, version='1')

    def version(self, target_dir: str, version: str) -> pb.LocalPath:
        return '.'

    def versions(self) -> tp.List[Variant]:
        return [Variant(self, str(v)) for v in self.known_versions]

@pytest.fixture
def make_source():

    def _make_version_source(versions: tp.List[int]):
        str_versions = [str(v) for v in versions]
        return VersionSource('ls', 'rs', str_versions)

    return _make_version_source

def test_single_versions_filter(make_source):
    src_1 = make_source([0])
    src_2 = make_source(range(2))

    src = source.SingleVersionFilter(src_1, '0')
    src_vs = [str(v) for v in src.versions()]
    assert ['0'] == src_vs

    src = source.SingleVersionFilter(src_2, '-1')
    src_vs = [str(v) for v in src.versions()]
    assert [] == src_vs

    src = source.SingleVersionFilter(src_2, '1')
    src_vs = [str(v) for v in src.versions()]
    assert ['1'] == src_vs

    src = source.SingleVersionFilter(src_2, '2')
    src_vs = [str(v) for v in src.versions()]
    assert [] == src_vs

def test_versions_product():
    pass

def describe_git():

    def submodule_versions_do_not_get_expanded():
        git_repo = source.Git('remote.git', 'local.git', clone=False)
        git_repo.versions = mock.MagicMock(name='versions')
        git_repo.versions.return_value = ['1', '2', '3']

        git_repo_sub = source.GitSubmodule(
            'remote.sub.git', 'local.git/sub', clone=False
        )
        git_repo_sub.versions = mock.MagicMock(name='versions')
        git_repo_sub.versions.return_value = ['sub1', 'sub2', 'sub3']

        variants = list(source.product(git_repo, git_repo_sub))
        expected_variants = [('1',), ('2',), ('3',)]

        assert variants == expected_variants

    @mock.patch('benchbuild.source.git.base.target_prefix')
    def repo_can_be_fetched(mocked_prefix, simple_repo):
        base_dir, repo = simple_repo
        mocked_prefix.return_value = str(base_dir)

        a_repo = source.Git(remote=repo.git_dir, local='test.git')
        cache_path = a_repo.fetch()

        assert (base_dir / 'test.git').exists()
        assert cache_path == str(base_dir / 'test.git')

    @mock.patch('benchbuild.source.git.base.target_prefix')
    def repo_clones_submodules(mocked_prefix, repo_with_submodule):
        base_dir, repo = repo_with_submodule
        mocked_prefix.return_value = str(base_dir)

        a_repo = source.Git(remote=repo.git_dir, local='test.git')
        a_repo.fetch()

        for submodule in repo.submodules:
            expected_sm_path = base_dir / 'test.git' / submodule.path
            assert expected_sm_path.exists()
            assert expected_sm_path.list() != []

    @mock.patch('benchbuild.source.git.base.target_prefix')
    def submodule_can_be_fetched_outside_main(
        mocked_prefix, repo_with_submodule
    ):
        base_dir, repo = repo_with_submodule
        mocked_prefix.return_value = str(base_dir)

        a_repo = source.Git(remote=repo.git_dir, local='test.git')
        a_repo.fetch()

        for submodule in repo.submodules:
            expected_sub_path_outside = base_dir / 'outside_main.git'
            expected_sub_path_inside = base_dir / 'test.git' / submodule.path

            a_sub_repo = source.GitSubmodule(
                remote=submodule.url, local='outside_main.git'
            )
            a_sub_repo.fetch()

            assert expected_sub_path_outside.exists()
            assert expected_sub_path_outside.list() != []

            assert expected_sub_path_inside.exists()
            assert expected_sub_path_inside.list() != []

    @mock.patch('benchbuild.source.git.base.target_prefix')
    def submodule_can_be_fetched_inside_fetched_main(
        mocked_prefix, repo_with_submodule
    ):
        base_dir, repo = repo_with_submodule
        mocked_prefix.return_value = str(base_dir)

        a_repo = source.Git(remote=repo.git_dir, local='test.git')
        a_repo.fetch()

        for submodule in repo.submodules:
            expected_sub_path = base_dir / 'test.git' / submodule.path
            expected_flat_sub_path = base_dir / f'test.git-{submodule.path}'

            sub_path = f'test.git/{submodule.path}'
            a_sub_repo = source.GitSubmodule(
                remote=submodule.url, local=str(sub_path)
            )
            cache_patch = a_sub_repo.fetch()

            assert expected_sub_path.exists()
            assert expected_flat_sub_path.exists()
            assert expected_sub_path.list() != []
            assert expected_flat_sub_path.list() != []
            assert cache_patch == expected_flat_sub_path

    @mock.patch('benchbuild.source.git.base.target_prefix')
    def repo_can_list_versions(mocked_prefix, simple_repo):
        base_dir, repo = simple_repo
        mocked_prefix.return_value = str(base_dir)
        master = repo.head.reference

        a_repo = source.Git(remote=repo.git_dir, local='test.git')
        expected_versions = [v.newhexsha[0:a_repo.limit] for v in master.log()]
        found_versions = [str(v) for v in reversed(a_repo.versions())]

        assert expected_versions == found_versions


from . import matrix_gen
import numpy as np
import random

CARDS_IN_GAME = 20
class Bot:

    def __init__(self):
        self.gamma = 0.7
        self.Q_matrix = matrix_gen.Matrix(CARDS_IN_GAME, 0).matrix #problem??

        self.points = 0
        #self.score = []
        pass

    #adds scores to txt file for graphs
    def file_append(self,score):
        #print(str(score), "l33")
        #write scores to file
        with open("bots/intelligent_agent/scores.txt", "a+") as score_file:
            score_file.write(str(score) + "\n")

    #makes an array of all cards left unknown to int_agent
    def cards_left(self, card_list, playerHand, playedCards):
        comparison = np.concatenate((playerHand,playedCards))
        new_list = card_list
        for card in comparison:
            new_list.remove(card)
        return new_list

    # selects the next move
    def select_move(self,playerHand, R_matrix , opponentPlayedCard):
        move_selection = R_matrix[playerHand,]
        possible_moves = np.where(move_selection >= 0)[0]
        #selection of all possible moves with that hand

        move = playerHand[int(np.random.choice(possible_moves,1))]
        """
        print(playerHand)
        print(move)"""
        return move

    def get_move(self, state):

        playedCards = []  # all previous played cards
        #populating playedCards Array if there was a move prior
        if state.get_prev_trick() != [None, None]:
            for element in state.get_prev_trick():
                playedCards.append(element)

        moves = state.moves() #legal moves
        trump_suite = state.get_trump_suit()

        playerHand = matrix_gen.get_hand(moves)  #players hand of playable cards / used for the cost matrix
        opponentMove = state.get_opponents_played_card()

        if playerHand[1] == True: #if theres a trump exchange available
            chosen_move = playerHand[0]
            return chosen_move

        available_cards = self.cards_left(list(range(CARDS_IN_GAME)),playerHand[0],playedCards)

        R_matrix = matrix_gen.Reward_Matrix(state.whose_turn(), trump_suite, playedCards, playerHand[0], opponentMove).populate()

        selected_move = self.select_move(playerHand[0], R_matrix, opponentMove)
        for move in moves: #since selected move is only the card we need to select the move properly
            if selected_move == move[0]: chosen_move = move

        self.score = self.brain_update(opponentMove, chosen_move, self.gamma, R_matrix, available_cards)
        self.file_append(self.score)

        print(self.Q_matrix)
        """
        print("Trained Q matrix:")
        #print(self.Q_matrix / self.Q_matrix *100)
        print(self.Q_matrix / np.sum(self.Q_matrix))"""

        np.savetxt("Q_matrix.csv", self.Q_matrix, delimiter= ",", fmt = "%f")

        return chosen_move

    def brain_update(self, opponentMove, move, gamma, reward_matrix, available_cards):
        def shorten_array(optimal_play):
            print(optimal_play)
            if optimal_play.shape[0] > 1:
                # optimal_play = int(np.random.choice(optimal_play, size=1))
                card_val_array = [card % 5 for card in optimal_play]
                i = np.where(card_val_array == np.max(card_val_array))
                optimal_play = np.random.choice(optimal_play[i], 1)
            else: optimal_play = int(optimal_play)
            return optimal_play

        if opponentMove == None:#if opponent didnt play select random card as opponentMove
            opponentMove = np.random.choice(available_cards)#assumes oppontent plays random card
            optimal_play = shorten_array(np.where(self.Q_matrix[move[0],] == np.max(self.Q_matrix[move[0],]))[1])
        else: #if opponent already played a card
            optimal_play = shorten_array(np.where(self.Q_matrix[:,opponentMove] == np.max(self.Q_matrix[:,opponentMove]))[0])

        max_reward = self.Q_matrix[optimal_play, opponentMove]
        self.Q_matrix[move[0], opponentMove] = reward_matrix[move[0], opponentMove] + gamma * max_reward

        return (np.sum(self.Q_matrix))


# Emiko, a catgirl discord bot.
# (c) CutePikachu / MiteBCool Technology LLC
# Licensed under the MIT License

# Imports
import discord
import random
import logging
import asyncio
from subprocess import call

# Set up logging, to discord.log
logger=logging.getLogger('discord')
logger.setLevel(logging.INFO)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

# Initialise the client
client = discord.Client()

# Get the token from token.txt
token = open('token.txt').readline()

# Message handler
@client.event
async def on_message(message):

    # Ignore the message if it's sent from us, so people can't self-command with echo
    if message.author == client.user:
        return

    # Pet command
    if message.content == "+pet":
        line = random.choice(open('responses_pet.txt').readlines())
        await client.send_message(message.channel, line)

    # Ping command
    if message.content == "+ping":
        await client.send_message(message.channel, "Pong!")

    # Poke command
    if message.content == "+poke" or message.content == "+touch":
        line = random.choice(open('responses_poke.txt').readlines())
        await client.send_message(message.channel, line)

    # Echo command
    if message.content.startswith("+echo"):
        fuck = message.content.split('=')
        await client.send_message(message.channel, fuck[1])

    # Exit command
    if message.content == "+exit" or message.content == "+quit":
        if not message.author.id == "235019900618407937":
            line = random.choice(open('responses_unauthorised.txt').readlines())
            await client.send_message(message.channel, line)
        else:
            await client.send_message(message.channel, "See you later!")
            quit()

    # Update command, for easy updating
    if message.content == "+update":
        if not message.author.id == "235019900618407937":
            line = random.choice(open('responses_unauthorised.txt').readlines())
            await client.send_message(message.channel, line)
        else:
            await client.send_message(message.channel, "Updating, brb")
            print("Updating...")
            call(["git", "pull"])
            print("Restarting...")
            call(["run.bat"])
            quit()

    # Restart command
    if message.content == "+restart":
        if not message.author.id == "235019900618407937":
            line = random.choice(open('responses_unauthorised.txt').readlines())
            await client.send_message(message.channel, line)
        else:
            await client.send_message(message.channel, "brb")
            print("Restarting...")
            call(["run.bat"])
            quit()

    # Set game command
    if message.content.startswith("+setgame"):
        if not message.author.id == "235019900618407937":
            line = random.choice(open('responses_unauthorised.txt').readlines())
            await client.send_message(message.channel, line)
        else:
            blah = message.content.split(':')
            await client.change_presence(game=discord.Game(name=blah[1], type=1))

# Client ready handler
@client.event
async def on_ready():
    print("Connected to Discord!")
    print("-------")
    print("Bot: " + client.user.name)
    print("Client id:" + client.user.id)
    print("-------")

# Actually log in
client.run(token)


import datetime

from django.test import TestCase

from pgcrypto_fields import aggregates, proxy
from pgcrypto_fields import fields

from .factories import EncryptedModelFactory
from .models import EncryptedModel

EMAIL_PGP_FIELDS = (fields.EmailPGPPublicKeyField,)
PGP_FIELDS = EMAIL_PGP_FIELDS + (
    fields.IntegerPGPPublicKeyField,
    fields.TextPGPPublicKeyField,
    fields.DatePGPPublicKeyField,
)

class TestPGPMixin(TestCase):
    """Test `PGPMixin` behave properly."""
    def test_check(self):
        """Assert `max_length` check does not return any error."""
        for field in PGP_FIELDS:
            self.assertEqual(field(name='field').check(), [])

    def test_max_length(self):
        """Assert `max_length` is ignored."""
        for field in PGP_FIELDS:
            self.assertEqual(field(max_length=42).max_length, None)

    def test_db_type(self):
        """Check db_type is `bytea`."""
        for field in PGP_FIELDS:
            self.assertEqual(field().db_type(), 'bytea')

class TestEmailPGPMixin(TestCase):
    """Test emails fields behave properly."""
    def test_max_length_validator(self):
        """Check `MaxLengthValidator` is not set."""
        for field in EMAIL_PGP_FIELDS:
            field_validated = field().run_validators(value='value@value.com')
            self.assertEqual(field_validated, None)

class TestEncryptedTextFieldModel(TestCase):
    """Test `EncryptedTextField` can be integrated in a `Django` model."""
    model = EncryptedModel

    def test_fields(self):
        """Assert fields are representing our model."""
        fields = self.model._meta.get_all_field_names()
        expected = (
            'id',
            'email_pgp_pub_field',
            'integer_pgp_pub_field',
            'pgp_pub_field',
            'pgp_pub_date_field',
            'pgp_pub_null_boolean_field',
        )
        self.assertItemsEqual(fields, expected)

    def test_value_returned_is_not_bytea(self):
        """Assert value returned is not a memoryview instance."""
        EncryptedModelFactory.create()

        instance = self.model.objects.get()

        self.assertIsInstance(instance.email_pgp_pub_field, unicode)
        self.assertIsInstance(instance.integer_pgp_pub_field, int)
        self.assertIsInstance(instance.pgp_pub_field, unicode)

    def test_fields_descriptor_is_not_instance(self):
        """`EncryptedProxyField` instance returns itself when accessed from the model."""
        self.assertIsInstance(
            self.model.pgp_pub_field,
            proxy.EncryptedProxyField,
        )

    def test_value_query(self):
        """Assert querying the field's value is making one query."""
        expected = 'bonjour'
        EncryptedModelFactory.create(pgp_pub_field=expected)

        instance = self.model.objects.get()

        with self.assertNumQueries(0):
            instance.pgp_pub_field

    def test_value_pgp_pub(self):
        """Assert we can get back the decrypted value."""
        expected = 'bonjour'
        EncryptedModelFactory.create(pgp_pub_field=expected)
        instance = self.model.objects.get()
        value = instance.pgp_pub_field
        self.assertEqual(value, expected)

    def test_value_pgp_date_pub(self):
        """Assert we can get back the decrypted value."""
        expected = datetime.date.today()
        EncryptedModelFactory.create(pgp_pub_date_field=expected)

        instance = self.model.objects.get()
        value = instance.pgp_pub_date_field
        self.assertEqual(value, expected)

    def test_value_pgp_date_pub_null(self):
        """Assert we can get back the decrypted value."""
        EncryptedModelFactory.create(pgp_pub_date_field=None)
        instance = self.model.objects.get()
        self.assertIsNone(instance.pgp_pub_date_field)

    def test_value_pgp_null_boolean_pub(self):
        """Assert we can get back the decrypted values."""
        EncryptedModelFactory.create(pgp_pub_null_boolean_field=None)
        instance = self.model.objects.last()
        self.assertIsNone(instance.pgp_pub_null_boolean_field)

        EncryptedModelFactory.create(pgp_pub_null_boolean_field=True)
        instance = self.model.objects.last()
        self.assertTrue(instance.pgp_pub_null_boolean_field)

        EncryptedModelFactory.create(pgp_pub_null_boolean_field=False)
        instance = self.model.objects.last()
        self.assertFalse(instance.pgp_pub_null_boolean_field)

    def test_value_pgp_pub_multipe(self):
        """Assert we get back the correct value when the table contains data."""
        expected = 'bonjour'
        EncryptedModelFactory.create(pgp_pub_field='au revoir')
        created = EncryptedModelFactory.create(pgp_pub_field=expected)

        instance = self.model.objects.get(pk=created.pk)
        value = instance.pgp_pub_field

        self.assertEqual(value, expected)

    def test_instance_not_saved(self):
        """Assert not saved instance return the value to be encrypted."""
        expected = 'bonjour'
        instance = EncryptedModelFactory.build(pgp_pub_field=expected)
        self.assertEqual(instance.pgp_pub_field, expected)
        self.assertEqual(instance.pgp_pub_field, expected)

    def test_decrypt_annotate(self):
        """Assert we can get back the decrypted value."""
        expected = 'bonjour'
        EncryptedModelFactory.create(
            pgp_pub_field=expected,
        )

        queryset = self.model.objects.annotate(
            aggregates.PGPPublicKeyAggregate('pgp_pub_field'),
        )
        instance = queryset.get()
        self.assertEqual(instance.pgp_pub_field__decrypted, expected)

    def test_decrypt_filter(self):
        """Assert we can get filter the decrypted value."""
        expected = 'bonjour'
        EncryptedModelFactory.create(
            pgp_pub_field=expected,
        )

        queryset = self.model.objects.annotate(
            aggregates.PGPPublicKeyAggregate('pgp_pub_field'),
        )
        instance = queryset.filter(pgp_pub_field__decrypted=expected).first()
        self.assertEqual(instance.pgp_pub_field__decrypted, expected)

    def test_update_attribute_pgp_pub_field(self):
        """Assert pgp field can be updated through its attribute on the model."""
        expected = 'bonjour'
        instance = EncryptedModelFactory.create()
        instance.pgp_pub_field = expected
        instance.save()

        updated_instance = self.model.objects.get()
        self.assertEqual(updated_instance.pgp_pub_field, expected)

    def test_update_one_attribute(self):
        """Assert value are not overriden when updating one attribute."""
        expected = 'initial value'
        new_value = 'new_value'

        instance = EncryptedModelFactory.create(
            pgp_pub_field=expected,
        )
        instance.pgp_sym_field = new_value
        instance.save()

        updated_instance = self.model.objects.get()
        self.assertEqual(updated_instance.pgp_pub_field, expected)

    def test_pgp_int_public_key_negative_number(self):
        """Assert negative value is saved with an `IntegerPGPPublicKeyField` field."""
        expected = -1
        instance = EncryptedModelFactory.create(integer_pgp_pub_field=expected)

        self.assertEqual(instance.integer_pgp_pub_field, expected)

    def test_pgp_int_public_key_null_number(self):
        """Assert negative value is saved with an `IntegerPGPPublicKeyField` field."""
        expected = None
        instance = EncryptedModelFactory.create(integer_pgp_pub_field=expected)
        self.assertEqual(instance.integer_pgp_pub_field, expected)

    def test_null(self):
        """Assert `NULL` values are saved."""
        instance = EncryptedModel.objects.create()
        fields = self.model._meta.get_all_field_names()
        fields.remove('id')
        for field in fields:
            self.assertEqual(getattr(instance, field), None)


from rest_framework import routers

from django.conf.urls import url, patterns, include
import views

router = routers.SimpleRouter()
router.register(r'posts', views.PostsViewSet, 'posts')

urlpatterns = patterns(
    '',
    url(r'^user/$', views.user),
    url(r'^', include(router.urls)),
)


# -*- coding: utf-8 -*-
"""
Created on Tue Feb  2 19:56:52 2021

@author: Abdul Basit
"""

import pandas as pd
import numpy as np
import re
from sklearn.linear_model import LogisticRegression
import argparse
import os
import numpy as np
from sklearn.metrics import mean_squared_error
import joblib
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from azureml.core.run import Run
from azureml.data.dataset_factory import TabularDatasetFactory

def clean_data(data):

    # Dict for cleaning data
    #months = {"jan":1, "feb":2, "mar":3, "apr":4, "may":5, "jun":6, "jul":7, "aug":8, "sep":9, "oct":10, "nov":11, "dec":12}
    #weekdays = {"mon":1, "tue":2, "wed":3, "thu":4, "fri":5, "sat":6, "sun":7}

    # Clean and one hot encode data
    x_df = data.to_pandas_dataframe().dropna()
    def remove_zero_fares(row):
        if row.Fare == 0:
            row.Fare = np.NaN
        return row
    # Apply the function
    x_df = x_df.apply(remove_zero_fares, axis=1)
    x_df['Fare'] = x_df.Fare.fillna(x_df.Fare.median())
    # If person has cabin, there is expected to be some value. Else, the feature has a NaN value
    x_df['Has_Cabin'] = ~x_df.Cabin.isnull()
    # Handling null values in Embarked - impute using mode
    x_df['Embarked']=x_df.Embarked.fillna('S')
    # Removing age null values
    x_df['Age'] = x_df.Age.fillna(x_df.Age.median())
    #Binning Age and fare -> Categorical features
    categorical_age = pd.cut(x_df['Age'], 5)
    categorical_fare = pd.qcut(x_df['Fare'], 4)
    # Mapping Age
    x_df.loc[ x_df['Age'] <= 16, 'Age'] = 0
    x_df.loc[(x_df['Age'] > 16) & (x_df['Age'] <= 32), 'Age'] = 1
    x_df.loc[(x_df['Age'] > 32) & (x_df['Age'] <= 48), 'Age'] = 2
    x_df.loc[(x_df['Age'] > 48) & (x_df['Age'] <= 64), 'Age'] = 3
    x_df.loc[ x_df['Age'] > 64, 'Age'] = 4 ;
    x_df['Age'] = x_df['Age'].astype(int)

    # Mapping Fare
    x_df.loc[ x_df['Fare'] <= 7.925, 'Fare'] = 0
    x_df.loc[(x_df['Fare'] > 7.925) & (x_df['Fare'] <= 14.5), 'Fare'] = 1
    x_df.loc[(x_df['Fare'] > 14.5) & (x_df['Fare'] <= 31.275), 'Fare'] = 2
    x_df.loc[ x_df['Fare'] > 31.275, 'Fare'] = 3
    x_df['Fare'] = x_df['Fare'].astype(int)
    x_df.Cabin.fillna('Unknown',inplace=True)
    # Extract first letter
    x_df['Cabin'] = x_df['Cabin'].map(lambda x : x[0])
    #New Feature
    family_size = x_df['SibSp'] + x_df['Parch'] + 1
    # Creation of four groups
    x_df['Fam_type'] = pd.cut(family_size, [0,1,4,7,11], labels=['Solo', 'Small', 'Big', 'Very big'])
    # Title Feature from Name
    def get_title(name):
        title_search = re.search(' ([A-Za-z]+)\.', name)
        # If the title exists, extract and return it.
        if title_search:
            return title_search.group(1)
        return ""
    # Create a new feature Title, containing the titles of passenger names
    x_df['Title'] = x_df['Name'].apply(get_title)
    # Substitute rare female titles
    x_df['Title'].replace(['Mme', 'Ms', 'Lady', 'Mlle', 'the Countess', 'Countess', 'Dona'], 'Miss', inplace=True)
    # Substitute rare male titles
    x_df['Title'].replace(['Major', 'Col', 'Capt', 'Don', 'Sir', 'Jonkheer'], 'Mr', inplace=True)
    features_to_drop = ['Name', 'SibSp', 'Parch', 'Ticket']
    x_df.drop(features_to_drop, axis = 1, inplace = True)
    categorical_features = ['Pclass', 'Sex', 'Fam_type', 'Title', 'Embarked', 'Cabin']
    x_df = pd.get_dummies(x_df, columns = categorical_features, drop_first=True)
    x_df = x_df.drop(columns=['Sex_male', 'Has_Cabin'])

    x_df = x_df.iloc[:, 2:]
    y_df = x_df.iloc[:, 1]
    return x_df, y_df

data = "https://raw.githubusercontent.com/Basit040/Capstone-Azure-Machine-Learning/main/titanic.csv"
#creating data in Tabular format via TabularDatasetFactory
ds = TabularDatasetFactory.from_delimited_files(data)
run = Run.get_context()

# clean data function
# Extracting x and y from clean data function
x, y = clean_data(ds)

# Split data into train and test sets.
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25)

def main():
    # Add arguments to script
    parser = argparse.ArgumentParser()

    parser.add_argument('--C', type=float, default=1.0, help="Inverse of regularization strength. Smaller values cause stronger regularization")
    parser.add_argument('--max_iter', type=int, default=100, help="Maximum number of iterations to converge")

    args = parser.parse_args()

    run.log("Regularization Strength:", np.float(args.C))
    run.log("Max iterations:", np.int(args.max_iter))

    model = LogisticRegression(C=args.C, max_iter=args.max_iter).fit(x_train, y_train)

    accuracy = model.score(x_test, y_test)
    run.log("Accuracy", np.float(accuracy))

if __name__ == '__main__':
    main()


'''
Pyimagesearch Tutorial - Part One

Last Updated: 2016-May-26
First Created: 2016-May-25

Python 2.7
Chris

http://www.pyimagesearch.com/2016/02/01/opencv-center-of-contour/
http://www.pyimagesearch.com/2016/02/08/opencv-shape-detection/
http://www.pyimagesearch.com/2016/02/15/determining-object-color-with-opencv/
'''

import imutils
import cv2

# load the image, convert to grayscale, blur it slightly and threshold it.

my_image = 'shapes_and_colors.jpg'

image = cv2.imread(my_image)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray, (5, 5), 0)
thresh = cv2.threshold(blurred, 60, 255, cv2.THRESH_BINARY)[1]

# show picture

cv2.imshow('image', thresh)
cv2.waitKey(0) #waits specified time for key press, can also specify which key

# find contours in the threshold image
# cnts[0] if opencv2.4, cnts[1] if opencv3

cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if imutils.is_cv2() else cnts[1]

# loop over the contours

for c in cnts:
    # compute the center of the contour
    M = cv2.moments(c)
    try:
        cX = int(M['m10'] / M['m00'])
        cY = int(M['m01'] / M['m00'])
    except: # some M results = 0, just move on
        cX = 0
        cY = 0

    # draw the contour and center of the shape on the image
    cv2.drawContours(image, [c], -1, (0, 255, 0), 2)
    cv2.circle(image, (cX, cY), 7, (255, 255, 255), -1)
    cv2.putText(image, 'center', (cX - 20, cY - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)

# show the image
cv2.imshow('image', image)
cv2.waitKey(0) #waits specified time for key press, can also specify which key


"""
Input: array A of N integers
Output: largest integer K > 0 such that both values K and -K exist in array A, or 0 otherwise

Example
Input: [3, 2, -2, 5, -3]
Output: 3

Input: [1, 2, 3, -4]
0

Ideas
- if we have seen our current element's opposite, then update the max_k
- otherwise, if our current element can possibly beat the current max_k, then add it to our seen set
    - if it can't beat our current max_k, then there's no point in adding it to our set
"""

def largest_k(A):
    """
    Runtime: O(n), Space complexity: O(n)
    """
    max_k = 0
    seen = set()
    for k in A:
        if (k < 0 and abs(k) in seen) or (k > 0 and -k in seen):
            max_k = max(max_k, abs(k))
        elif abs(k) > max_k:
            seen.add(k)
    return max_k

assert largest_k([3, 2, -2, 5, -3]) == 3
assert largest_k([1, 2, 3, -4]) == 0


#!/usr/bin/env python3.6
# -*- coding: utf-8 -*-
"""Provides an exhaustive template that downloads, builds and wraps a software based on ``autoconf`` and ``make``.
"""

import os
import stdlib
import stdlib.fetch
import stdlib.extract
import stdlib.patch
import stdlib.split.system
import stdlib.deplinker.elf

from stdlib.template.configure import configure
from stdlib.template.make import make

def build(
    build_folder='.',
    fetch=stdlib.fetch.fetch,
    extract=stdlib.extract.flat_extract_all,
    patch=stdlib.patch.patch_all,
    configure=configure,
    compile=make,
    check=lambda: make('check', fail_ok=True),
    install=lambda: make('install', f'DESTDIR={stdlib.build.current_build().install_cache}'),
    split=stdlib.split.system.system,
    deplinker=stdlib.deplinker.elf.elf_deplinker,
):
    """Download, build and wrap a library based on ``autoconf`` and ``make``.

    This exhaustive template is made of 9 steps:
        * ``fetch``
        * ``extract``
        * ``patch``
        * ``configure``
        * ``compile``
        * ``check``
        * ``install``
        * ``split``
        * ``dependency linking``

    For each one of these steps, a function is called. This template simply calls each of them in the above order.
    All of these functions can be given as arguments, but each one of them has a default value that is explained below.
    If any of those functions is ``None``, the step is skipped.

    **Fetch**

        This step is used to download the source code. The default value is :py:func:`.fetch` with no argument.

    **Extract**

        This step is used to extract the downloaded source code. The default value is :py:func:`.flat_extract_all` with no argument.

    **Patch**

        This step is used to patch the downloaded source code. The default value is :py:func:`.patch_all` with no argument.

    From now on, the current working directory is changed in favor of ``build_folder`` (which defaults to ``.``).
    If the directory pointed by ``build_folder`` doesn't exist, it is created.

    This is useful for the few ``configure`` scripts that don't work if they aren't executed in a standalone directory.

    **Configure**

        This step uses the ``configure`` script to configure the source code. The default value is :py:func:`.configure` with no argument.

        When ``build_folder`` isn't ``.``, it is usually necessary to override this step with an other call to :py:func:`.configure` with
        the argument ``binary='../configure'``.

    **Compile**

        This step compiles the source code. The default value is :py:func:`.make` with no argument.

    **Check**

        This step runs the unit and integration tests. The default value is :py:func:`.make` with the arguments ``'check'`` and ``fail_ok=True``.

    **Install**

        This step installs the software in the install cache. The default value is :py:func:`.make` with the arguments ``'install'`` and
        ``f'DESTDIR={stdlib.build.current_build().install_cache}'``

        If this step is overriden, the ``DESTDIR`` variable is crucial and should be reused. Otherwise, and unless the ``configure`` script has been
        configured to work without it, the installation can damage and overwrite parts of the host system.

    **Split**

        This step automatically splits the output of the build into multiple packages. The default value is :py:func:`~stdlib.split.system.system`.
        Alternative splitters can be found in the :py:mod:`~stdlib.split` module.

    **Dependency Linking**

        This step automatically finds requirements for the generated packages. The default value is :py:func:`~stdlib.deplinker.elf.elf_deplinker`.
        Alternative dependency linkers can be found in the :py:mod:`~stdlib.deplinker` module.

    """
    build = stdlib.build.current_build()

    stdlib.log.ilog("Step 1/9: Fetch")
    if fetch is not None:
        with stdlib.log.pushlog():
            fetch()

    stdlib.log.ilog("Step 2/9: Extract")
    if extract is not None:
        with stdlib.log.pushlog():
            extract()

    stdlib.log.ilog("Step 3/9: Patch")
    if patch is not None:
        with stdlib.log.pushlog():
            patch()

    packages = dict()

    os.makedirs(build_folder, exist_ok=True)
    with stdlib.pushd(build_folder):

        os.environ['DESTDIR'] = build.install_cache

        stdlib.log.ilog("Step 4/9: Configure")
        if configure is not None:
            with stdlib.log.pushlog():
                configure()

        stdlib.log.ilog("Step 5/9: Compile")
        if compile is not None:
            with stdlib.log.pushlog():
                compile()

        stdlib.log.ilog("Step 6/9: Check")
        if check is not None:
            with stdlib.log.pushlog():
                check()

        stdlib.log.ilog("Step 7/9: Install")
        if install is not None:
            with stdlib.log.pushlog():
                install()

        stdlib.log.ilog("Step 8/9: Split")
        if split is not None:
            with stdlib.log.pushlog():
                packages = split()

                if len(packages) > 0:
                    stdlib.log.ilog("The following packages were generated:")

                    with stdlib.log.pushlog():
                        for package in packages.values():
                            stdlib.log.ilog(str(package))

        stdlib.log.ilog("Step 9/9: Dependency Linking")
        if deplinker is not None:
            with stdlib.log.pushlog():
                deplinker(packages)

    return packages


import asyncio
import builtins
import collections
import signal
import subprocess
import sys
import threading
import time
import unittest.mock

import pytest

from hat.util import aio

@pytest.mark.asyncio
async def test_first():
    queue = aio.Queue()
    queue.put_nowait(1)
    queue.put_nowait(2)
    queue.put_nowait(3)
    queue.close()

    result = await aio.first(queue, lambda i: i == 2)
    assert result == 2

    assert not queue.empty()
    assert queue.get_nowait() == 3
    assert queue.empty()

    queue = aio.Queue()
    queue.put_nowait(1)
    queue.put_nowait(2)
    queue.put_nowait(3)
    queue.close()
    result = await aio.first(queue, lambda i: i == 4)
    assert result is None
    assert queue.empty()

@pytest.mark.asyncio
async def test_queue():
    queue = aio.Queue()
    assert not queue.closed.done()
    f = asyncio.ensure_future(queue.get())
    assert not f.done()
    queue.put_nowait(1)
    assert 1 == await f
    for _ in range(5):
        queue.close()
        assert queue.closed.done()
    with pytest.raises(aio.QueueClosedError):
        await queue.get()

@pytest.mark.asyncio
async def test_queue_get_until_empty():
    queue = aio.Queue()
    queue.put_nowait(1)
    queue.put_nowait(2)
    queue.put_nowait(3)
    result = await queue.get_until_empty()
    assert result == 3

def test_queue_get_nowait_until_empty():
    queue = aio.Queue()
    queue.put_nowait(1)
    queue.put_nowait(2)
    queue.put_nowait(3)
    result = queue.get_nowait_until_empty()
    assert result == 3

@pytest.mark.asyncio
async def test_queue_with_size():
    queue_size = 5
    queue = aio.Queue(queue_size)
    assert queue.maxsize == queue_size

    for _ in range(queue_size):
        queue.put_nowait(None)

    with pytest.raises(aio.QueueFullError):
        queue.put_nowait(None)

@pytest.mark.asyncio
async def test_queue_put():
    queue = aio.Queue(1)
    await queue.put(1)
    put_future = asyncio.ensure_future(queue.put(1))
    asyncio.get_event_loop().call_soon(queue.close)
    with pytest.raises(aio.QueueClosedError):
        await put_future

@pytest.mark.asyncio
async def test_queue_put_cancel():
    queue = aio.Queue(1)
    await queue.put(1)
    put_future = asyncio.ensure_future(queue.put(1))
    asyncio.get_event_loop().call_soon(put_future.cancel)
    with pytest.raises(asyncio.CancelledError):
        await put_future

@pytest.mark.asyncio
async def test_queue_async_iterable():
    queue = aio.Queue()
    data = collections.deque()

    for i in range(10):
        queue.put_nowait(i)
        data.append(i)

    queue.close()

    async for i in queue:
        assert i == data.popleft()

    assert queue.empty()
    assert len(data) == 0

def test_init_asyncio_without_libuv(monkeypatch):

    def invalid_import(*args, **kwargs):
        raise ModuleNotFoundError

    platform = sys.platform
    with monkeypatch.context() as ctx:
        ctx.setattr(sys, 'platform', 'win32')
        ctx.setattr(builtins, '__import__', invalid_import)
        if platform == 'win32':
            aio.init_asyncio()
        else:
            with pytest.raises(AttributeError):
                aio.init_asyncio()

@pytest.mark.skipif(sys.platform == 'win32',
                    reason="pthread_kill not supported")
def test_run_asyncio():
    ident = threading.get_ident()

    def thread():
        time.sleep(0.01)
        signal.pthread_kill(ident, signal.SIGINT)

    t = threading.Thread(target=thread)
    t.start()

    async def f():
        await asyncio.Future()

    with pytest.raises(asyncio.CancelledError):
        aio.run_asyncio(f())
    t.join()

# TODO check implementation for posible deadlock
def test_run_asyncio_with_subprocess(tmp_path):

    py_path = tmp_path / 'temp.py'
    run_path = tmp_path / 'temp.run'
    with open(py_path, 'w', encoding='utf-8') as f:
        f.write(f"from hat.util import aio\n"
                f"import asyncio\n"
                f"import sys\n"
                f"async def f():\n"
                f"    open(r'{run_path}', 'w').close()\n"
                f"    await asyncio.Future()\n"
                f"try:\n"
                f"    aio.run_asyncio(f())\n"
                f"except asyncio.CancelledError:\n"
                f"    sys.exit(0)\n"
                f"except Exception:\n"
                f"    sys.exit(10)\n"
                f"sys.exit(5)\n")

    p = subprocess.Popen(['python', str(py_path)],
                         creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP
                                        if sys.platform == 'win32' else 0))
    while not run_path.exists():
        assert p.poll() is None
        time.sleep(0.01)
    p.send_signal(signal.CTRL_BREAK_EVENT
                  if sys.platform == 'win32'
                  else signal.SIGINT)
    assert p.wait() == 0

@pytest.mark.skipif(sys.platform == 'win32',
                    reason="pthread_kill not supported")
def test_run_asyncio_with_multiple_signals():
    ident = threading.get_ident()

    def thread():
        time.sleep(0.01)
        for _ in range(5):
            signal.pthread_kill(ident, signal.SIGINT)
            time.sleep(0.001)

    t = threading.Thread(target=thread)
    t.start()

    async def f():
        await aio.uncancellable(asyncio.sleep(0.02), raise_cancel=False)
        return 1

    assert aio.run_asyncio(f()) == 1
    t.join()

@pytest.mark.asyncio
async def test_call():

    def f1(x):
        return x

    async def f2(x):
        await asyncio.sleep(0)
        return x

    result = await aio.call(f1, 123)
    assert result == 123

    result = await aio.call(f2, 123)
    assert result == 123

@pytest.mark.asyncio
async def test_call_on_cancel():
    exceptions = aio.Queue()
    called = asyncio.Future()
    group = aio.Group(exceptions.put_nowait)

    async def closing(called):
        called.set_result(True)
        assert group.closing.done()
        assert not group.closed.done()

    group.spawn(aio.call_on_cancel, closing, called)
    assert not called.done()

    await group.async_close()
    assert called.done()
    assert exceptions.empty()

@pytest.mark.asyncio
async def test_group():
    group = aio.Group()
    futures = [group.wrap(asyncio.Future()) for _ in range(100)]
    assert not any(future.done() for future in futures)
    await group.async_close()
    assert all(future.done() for future in futures)

@pytest.mark.asyncio
async def test_group_spawn_async_close():

    async def task():
        group.spawn(group.async_close)
        await asyncio.Future()

    group = aio.Group()
    group.spawn(task)
    await group.closed

@pytest.mark.asyncio
async def test_group_subgroup():
    g1 = aio.Group()
    g2 = g1.create_subgroup()
    f = g2.wrap(asyncio.Future())
    g1.close()

    assert g1.closing.done()
    assert g2.closing.done()
    assert not f.done()
    assert not g1.closed.done()
    assert not g2.closed.done()

    with pytest.raises(Exception):
        g1.create_subgroup()
    await g1.async_close()
    with pytest.raises(Exception):
        g1.create_subgroup()

    assert f.done()
    assert g1.closed.done()
    assert g2.closed.done()

@pytest.mark.asyncio
async def test_group_async_close_subgroup_without_tasks():
    g1 = aio.Group()
    g2 = g1.create_subgroup()
    await g1.async_close()

    assert g1.closed.done()
    assert g2.closed.done()

@pytest.mark.asyncio
async def test_group_spawn_subgroup_in_closing_subgroup():
    exceptions = aio.Queue()
    g1 = aio.Group(lambda e: exceptions.put_nowait(e))
    g2 = g1.create_subgroup()

    async def task():
        try:
            await asyncio.Future()
        except asyncio.CancelledError:
            with pytest.raises(Exception):
                g1.create_subgroup()

    g2.spawn(task)
    await g1.async_close()
    assert g1.closed.done()
    assert g2.closed.done()
    assert exceptions.empty()

@pytest.mark.asyncio
async def test_group_spawn_when_not_open():
    g = aio.Group()
    g.spawn(asyncio.Future)
    g.close()
    with pytest.raises(Exception):
        g.spawn(asyncio.Future)
    await g.async_close()
    with pytest.raises(Exception):
        g.spawn(asyncio.Future)
    with pytest.raises(Exception):
        g.wrap(asyncio.Future())

def test_group_close_empty_group():
    g = aio.Group()
    assert not g.closing.done()
    assert not g.closed.done()
    g.close()
    assert g.closing.done()
    assert g.closed.done()

@pytest.mark.asyncio
async def test_group_context():
    async with aio.Group() as g:
        f = g.spawn(asyncio.Future)
        assert not f.done()
    assert f.done()

@pytest.mark.asyncio
async def test_group_custom_exception_handler():

    def exception_cb(e):
        nonlocal e2
        e2 = e

    async def f():
        raise e1

    e1 = Exception()
    e2 = None
    g = aio.Group(exception_cb)
    with pytest.raises(Exception):
        await g.spawn(f)
    await g.async_close()

    assert e1 is e2

@pytest.mark.asyncio
async def test_group_default_exception_handler():

    async def f():
        raise e

    e = Exception()
    g = aio.Group()
    with unittest.mock.patch('hat.util.aio.mlog.warning') as mock:
        with pytest.raises(Exception):
            await g.spawn(f)
    await g.async_close()

    _, kwargs = mock.call_args
    assert kwargs['exc_info'] is e

@pytest.mark.asyncio
async def test_uncancellable():
    f1 = asyncio.Future()

    async def f():
        return await f1

    f2 = aio.uncancellable(f(), raise_cancel=False)
    f3 = asyncio.ensure_future(f2)
    asyncio.get_event_loop().call_soon(f3.cancel)
    f1.set_result(123)
    result = await f3
    assert result == 123

@pytest.mark.asyncio
async def test_uncancellable_vs_shield():

    async def set_future(f, value):
        await asyncio.sleep(0.001)
        f.set_result(value)

    future = asyncio.Future()
    t1 = asyncio.shield(set_future(future, 1))
    t2 = asyncio.ensure_future(t1)
    asyncio.get_event_loop().call_soon(t2.cancel)
    with pytest.raises(asyncio.CancelledError):
        await t2
    assert not future.done()
    await future
    assert future.result() == 1

    future = asyncio.Future()
    t1 = aio.uncancellable(set_future(future, 1), raise_cancel=True)
    t2 = asyncio.ensure_future(t1)
    asyncio.get_event_loop().call_soon(t2.cancel)
    with pytest.raises(asyncio.CancelledError):
        await t2
    assert future.done()
    assert future.result() == 1

    future = asyncio.Future()
    t1 = aio.uncancellable(set_future(future, 1), raise_cancel=False)
    t2 = asyncio.ensure_future(t1)
    asyncio.get_event_loop().call_soon(t2.cancel)
    await t2
    assert future.done()
    assert future.result() == 1

@pytest.mark.asyncio
async def test_executor():
    executor = aio.create_executor()
    result = await executor(lambda: threading.current_thread().name)
    assert threading.current_thread().name != result


import numpy as np

from .common import *
from . import yin

def normalized_pdf(a, b, begin, end, number):
    x = np.arange(0, number, dtype = np.float64) * ((end - begin) / number)
    v = np.power(x, a - 1.0) * np.power(1.0 - x, b - 1.0)
    for i in reversed(range(number - 1)):
        if(v[i] < v[i + 1]):
            v[i] = v[i + 1]
    return v / np.sum(v)

def extractF0(obsProbList):
    nHop = len(obsProbList)

    out = np.zeros(nHop, dtype = np.float64)
    for iHop, (freqProb) in enumerate(obsProbList):
        if(len(freqProb) > 0):
            out[iHop] = freqProb.T[0][np.argmax(freqProb.T[1])]

    return out

class Processor:
    def __init__(self, sr, **kwargs):
        self.samprate = float(sr)
        self.hopSize = kwargs.get("hopSize", roundUpToPowerOf2(self.samprate * 0.0025))

        self.minFreq = kwargs.get("minFreq", 80.0)
        self.maxFreq = kwargs.get("maxFreq", 1000.0)
        self.maxIter = 4
        self.prefilter = kwargs.get("prefilter", True)

        self.valleyThreshold = kwargs.get("valleyThreshold", 1.0)
        self.valleyStep = kwargs.get("valleyStep", 0.01)

        self.probThreshold = kwargs.get("probThreshold", 0.02)
        self.weightPrior = kwargs.get("weightPrior", 5.0)
        self.bias = kwargs.get("bias", 1.0)

        self.pdf = kwargs.get("pdf", normalized_pdf(1.7, 6.8, 0.0, 1.0, 128))

    def __call__(self, x, w = None, removeDC = True):
        nX = len(x)
        nHop = getNFrame(nX, self.hopSize)
        pdfSize = len(self.pdf)

        if(removeDC):
            x = simpleDCRemove(x)
        if(self.prefilter):
            x = yin.doPrefilter(x, self.maxFreq, self.samprate)

        out = []
        for iHop in range(nHop):
            windowSize = 0
            newWindowSize = max(roundUpToPowerOf2(self.samprate / self.minFreq * 4), self.hopSize * 2)
            iIter = 0
            while(newWindowSize != windowSize and iIter < self.maxIter):
                windowSize = newWindowSize
                frame = getFrame(x, iHop * self.hopSize, windowSize)

                buff = yin.difference(frame)
                buff = yin.cumulativeDifference(buff)
                valleyIndexList = yin.findValleys(buff, self.minFreq, self.maxFreq, self.samprate, threshold = self.valleyThreshold, step = self.valleyStep)
                nValley = len(valleyIndexList)
                if(valleyIndexList):
                    possibleFreq = min(self.maxFreq, max(self.samprate / valleyIndexList[-1] - 20.0, self.minFreq))
                    newWindowSize = max(int(np.ceil(self.samprate / possibleFreq * 4)), self.hopSize * 2)
                    if(newWindowSize % 2 == 1):
                        newWindowSize += 1
                    iIter += 1

            freqProb = np.zeros((nValley, 2), dtype = np.float64)
            probTotal = 0.0
            weightedProbTotal = 0.0
            for iValley, valley in enumerate(valleyIndexList):
                ipledIdx, ipledVal = parabolicInterpolation(buff, valley)
                freq = self.samprate / ipledIdx
                v0 = 1 if(iValley == 0) else min(1.0, buff[valleyIndexList[iValley - 1]] + 1e-10)
                v1 = 0 if(iValley == nValley - 1) else max(0.0, buff[valleyIndexList[iValley + 1]]) + 1e-10
                prob = 0.0
                for i in range(int(v1 * pdfSize), int(v0 * pdfSize)):
                    prob += self.pdf[i] * (1.0 if(ipledVal < i / pdfSize) else 0.01)
                prob = min(prob, 0.99)
                prob *= self.bias
                probTotal += prob
                if(ipledVal < self.probThreshold):
                    prob *= self.weightPrior
                weightedProbTotal += prob
                freqProb[iValley] = freq, prob

            # renormalize
            if(nValley > 0 and weightedProbTotal != 0.0):
                freqProb.T[1] *= probTotal / weightedProbTotal

            out.append(freqProb)

        return out