'''
License:
  "NASA Open Source Agreement 1.3"

Description:
  Utility module for ESPA project.
  This is a shared module to hold simple utility functions.

History:
  Original implementation by David V. Hill, USGS/EROS
  Updated Jan/2014 by Ron Dilley, USGS/EROS
'''

import os
import sys
import datetime
import commands
import random
import re
from email.mime.text import MIMEText
from smtplib import SMTP

# local objects and methods
import settings

# ============================================================================
#def scenecache_is_alive(url='http://edclpdsftp.cr.usgs.gov:50000/RPC2'):
#    """Determine if the specified url has an http server
#    that accepts POST calls
#
#    Keyword args:
#    url -- The url of the server to check
#
#    Return:
#    True -- If the contacted server is alive and accepts POST calls
#    False -- If the server does not accept POST calls or the
#             server could not be contacted
#    """
#
#    try:
#        return urllib2.urlopen(url, data="").getcode() == 200
#    except Exception, e:
#        if settings.DEBUG:
#            print("Scene cache could not be contacted")
#            print(e)
#        return False

# ============================================================================
#def scenecache_client():
#    """Return an xmlrpc proxy to the caller for the scene cache
#
#    Returns -- An xmlrpclib ServerProxy object
#    """
#    url = 'http://edclpdsftp.cr.usgs.gov:50000/RPC2'
#    # url = os.environ['ESPA_SCENECACHE_URL']
#    if scenecache_is_alive(url):
#        return xmlrpclib.ServerProxy(url)
#    else:
#        msg = "Could not contact scene_cache at %s" % url
#        raise RuntimeError(msg)

# ============================================================================
def date_from_doy(year, doy):
    '''Returns a python date object given a year and day of year'''

    d = datetime.datetime(int(year), 1, 1) + datetime.timedelta(int(doy) - 1)

    if int(d.year) != int(year):
        raise Exception("doy [%s] must fall within the specified year [%s]" %
                        (doy, year))
    else:
        return d

# ============================================================================
def is_number(s):
    '''Determines if a string value is a float or int.

    Keyword args:
    s -- A string possibly containing a float or int

    Return:
    True if s is a float or int
    False if s is not a float or int
    '''
    try:
        float(s)
        return True
    except ValueError:
        return False

# ============================================================================
def execute_cmd(cmd):
    '''
    Description:
      Execute a command line and return the terminal output or raise an
      exception

    Returns:
        output - The stdout and/or stderr from the executed command.
    '''

    output = ''

    (status, output) = commands.getstatusoutput(cmd)

    if status < 0:
        message = "Application terminated by signal [%s]" % cmd
        if len(output) > 0:
            message = ' Stdout/Stderr is: '.join([message, output])
        raise Exception(message)

    if status != 0:
        message = "Application failed to execute [%s]" % cmd
        if len(output) > 0:
            message = ' Stdout/Stderr is: '.join([message, output])
        raise Exception(message)

    if os.WEXITSTATUS(status) != 0:
        message = "Application [%s] returned error code [%d]" \
                  % (cmd, os.WEXITSTATUS(status))
        if len(output) > 0:
            message = ' Stdout/Stderr is: '.join([message, output])
        raise Exception(message)

    return output

# ============================================================================
def strip_zeros(value):
    '''
    Description:
      Removes all leading zeros from a string
    '''

    while value.startswith('0'):
        value = value[1:len(value)]
    return value

# ============================================================================
def get_cache_hostname():
    '''
    Description:
      Poor mans load balancer for accessing the online cache over the private
      network
    '''

    host_list = settings.ESPA_CACHE_HOST_LIST

    def check_host_status(hostname):
        cmd = "ping -q -c 1 %s" % hostname

        try:
            execute_cmd(cmd)
        except Exception:
            return -1
        return 0

    def get_hostname():
        hostname = random.choice(host_list)
        if check_host_status(hostname) == 0:
            return hostname
        else:
            for x in host_list:
                if x == hostname:
                    host_list.remove(x)
            if len(host_list) > 0:
                return get_hostname()
            else:
                raise Exception("No online cache hosts available...")

    return get_hostname()

def send_email(recipient, subject, body):
    '''Sends an email to a receipient on the behalf of espa'''

    if not validate_email(recipient):
        raise TypeError("Invalid email address provided:%s" % recipient)

    msg = MIMEText(body)
    msg['Subject'] = subject
    msg['To'] = recipient
    msg['From'] = settings.ESPA_EMAIL_ADDRESS
    s = SMTP(host=settings.ESPA_EMAIL_SERVER)
    s.sendmail(msg['From'], msg['To'], msg.as_string())
    s.quit()

    return True

def validate_email(email):
    '''Compares incoming email address against regular expression to make sure
    its at least formatted like an email

    Keyword args:
    email -- String to validate as an email address

    Return:
    True if the string is a properly formatted email address
    False if not
    '''
    #pattern = '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$'
    #some clown used a single quote in his email address... sigh.
    email = email.replace("'", "\'")
    pattern = r'^[A-Za-z0-9._%+-\\\']+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$'
    return re.match(pattern, email.strip())

def tar_files(tarred_full_path, file_list, gzip=False):
    '''
    Description:
      Create a tar ball (*.tar) of the specified file(s).
      OR
      Create a tar.gz ball (*.tar.gz) of the specified file(s).
    '''

    flags = '-cf'
    target = '%s.tar' % tarred_full_path

    # If zipping was chosen, change the flags and the target name
    if gzip:
        flags = '-czf'
        target = '%s.tar.gz' % tarred_full_path

    cmd = ['tar', flags, target]
    cmd.extend(file_list)
    cmd = ' '.join(cmd)

    output = ''
    try:
        output = execute_cmd(cmd)
    except Exception:
        msg = "Error encountered tar'ing file(s): Stdout/Stderr:"
        if len(output) > 0:
            msg = ' '.join([msg, output])
        else:
            msg = ' '.join([msg, "NO STDOUT/STDERR"])
        # Raise and retain the callstack
        raise Exception(msg)

    return target

def gzip_files(file_list):
    '''
    Description:
      Create a gzip for each of the specified file(s).
    '''

    # Force the gzip file to overwrite any previously existing attempt
    cmd = ['gzip', '--force']
    cmd.extend(file_list)
    cmd = ' '.join(cmd)

    output = ''
    try:
        output = execute_cmd(cmd)
    except Exception:
        msg = "Error encountered compressing file(s): Stdout/Stderr:"
        if len(output) > 0:
            msg = ' '.join([msg, output])
        else:
            msg = ' '.join([msg, "NO STDOUT/STDERR"])
        # Raise and retain the callstack
        raise Exception(msg)

def checksum_local_file(filename):
    '''
    Description:
      Create a checksum for the specified file.
    '''

    cmd = ' '.join(['cksum', filename])

    cksum_result = ''
    try:
        cksum_result = execute_cmd(cmd)
    except Exception:
        msg = "Error encountered generating checksum: Stdout/Stderr:"
        if len(cksum_result) > 0:
            msg = ' '.join([msg, cksum_result])
        else:
            msg = ' '.join([msg, "NO STDOUT/STDERR"])
        # Raise and retain the callstack
        raise Exception(msg)

    return cksum_result


from django.conf.urls import patterns, include, url
from django.contrib import admin

admin.autodiscover()

import views
urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'TimetableWeb.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^admin/', include(admin.site.urls)),
    url(r'^accounts/register/$', views.registration),
    url(r'^accounts/login/$', 'django.contrib.auth.views.login', { "template_name": "accounts/login.html" }),
    #url(r'^$', views.main),
    url(r'^accounts/logout/$', 'django.contrib.auth.views.logout_then_login' ),
    url(r'^accounts/profile/$', views.later),
    url(r'^$', views.main),
)
urlpatterns += patterns('event.views',
    url(r'^add_event/$', 'add_event'),
    url(r'^day/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/$', 'day_events'),
    url(r'^event/(?P<cur_id>\d+)/$', 'active_event'),
)


"""Service to retrieve news articles from GDELT news tracker."""
import requests
import pandas as pd
from newspaper import Article, ArticleException

import training_scripts.domain.sampler as sampler

EXCEPTION_CAUSING_URLS = [
    'https://www.newsweek.com/',
    'https://www.forbes.com',
    'https://www.malaysiasun.com/',
]

GDELT_LATEST_UPDATE_URL = 'http://data.gdeltproject.org/gdeltv2/lastupdate.txt'

GDELT_MASTER_LIST_URL = 'http://data.gdeltproject.org/gdeltv2/masterfilelist.txt'

GDELT_COLUMNS = [
    'GlobalEventID',
    'Day',
    'MonthYear',
    'Year',
    'FractionDate',
    'Actor1Code',
    'Actor1Name',
    'Actor1CountryCode',
    'Actor1KnownGroupCode',
    'Actor1EthnicCode',
    'Actor1Religion1Code',
    'Actor1Religion2Code',
    'Actor1Type1Code',
    'Actor1Type2Code',
    'Actor1Type3Code',
    'Actor2Code',
    'Actor2Name',
    'Actor2CountryCode',
    'Actor2KnownGroupCode',
    'Actor2EthnicCode',
    'Actor2Religion1Code',
    'Actor2Religion2Code',
    'Actor2Type1Code',
    'Actor2Type2Code',
    'Actor2Type3Code',
    'IsRootEvent',
    'EventCode',
    'EventBaseCode',
    'EventRootCode',
    'QuadClass',
    'GoldsteinScale',
    'NumMentions',
    'NumSources',
    'NumArticles',
    'AvgTone',
    'Actor1Geo_Type',
    'Actor1Geo_Fullname',
    'Actor1Geo_CountryCode',
    'Actor1Geo_ADM1Code',
    'Actor1Geo_ADM2Code',
    'Actor1Geo_Lat',
    'Actor1Geo_Long',
    'Actor1Geo_FeatureID',
    'Actor2Geo_Type',
    'Actor2Geo_Fullname',
    'Actor2Geo_CountryCode',
    'Actor2Geo_ADM1Code',
    'Actor2Geo_ADM2Code',
    'Actor2Geo_Lat',
    'Actor2Geo_Long',
    'Actor2Geo_FeatureID',
    'Action2Geo_Type',
    'Action2Geo_Fullname',
    'Action2Geo_CountryCode',
    'Action2Geo_ADM1Code',
    'Action2Geo_ADM2Code',
    'Action2Geo_Lat',
    'Action2Geo_Long',
    'Action2Geo_FeatureID',
    'DATEADDED',
    'SOURCEURL',
]

def scrape_gdelt_dataset(url,
                         balance_data,
                         sample_size,
                         blacklisted_domains=EXCEPTION_CAUSING_URLS):
    """Scrape articles linked GDELT dataset and write them to files"""
    df_gdelt = _format_gdelt_dataframe(url, balance_data, sample_size)
    for _, row in df_gdelt.iterrows():
        try:
            article_dict = _build_article_dict(row, blacklisted_domains)
            yield article_dict
        except ArticleException as exception:
            print(repr(exception))

def scrape_latest_gdelt_dataset(balance_data,
                                sample_size,
                                blacklisted_domains=EXCEPTION_CAUSING_URLS):
    """Scrape latest GDELT dataset"""
    latest_dataset_url = _get_latest_gdelt_dataset_url()
    for result in scrape_gdelt_dataset(latest_dataset_url, balance_data,
                                       sample_size, blacklisted_domains):
        yield result

def scrape_latest_gdelt_datasets(n_datasets,
                                balance_data,
                                sample_size):
    """Scrape latest GDELT datasets"""
    for dataset_url in _get_number_of_gdelt_dataset_urls(n_datasets):
        for result in scrape_gdelt_dataset(dataset_url, balance_data, sample_size):
            yield result

def _get_latest_gdelt_dataset_url():
    """Get URL for latest GDELT dataset"""
    information_on_latest_dataset = requests.get(GDELT_LATEST_UPDATE_URL).text
    line_with_latest_dataset, *_ = information_on_latest_dataset.splitlines()
    *_, latest_dataset_url = line_with_latest_dataset.split()
    return latest_dataset_url

def _get_number_of_gdelt_dataset_urls(n_datasets):
    """Get URL for latest n GDELT datasets"""
    gdelt_master_list = requests.get(GDELT_MASTER_LIST_URL).text
    gdelt_master_list = gdelt_master_list.splitlines()
    gdelt_datasets = [
        l for l in gdelt_master_list if l.endswith('.export.CSV.zip')
    ]
    for dataset in gdelt_datasets[-n_datasets:]:
        *_, dataset_url = dataset.split()
        yield dataset_url

def _format_gdelt_dataframe(url, balance_data, sample_size):
    """Create dataframe from URL containing zipped CSV of GDELT data"""
    df_gdelt = pd.read_csv(url, names=GDELT_COLUMNS, delimiter='\t')
    df_gdelt = df_gdelt.sample(frac=sample_size)
    df_gdelt = df_gdelt.drop_duplicates(subset=['SOURCEURL'])
    if balance_data:
        df_gdelt = sampler.sample_dataframe(df_gdelt, 'Actor1Geo_CountryCode')
    return df_gdelt

def _extract_article_contents(url):
    """Get title and text from URL"""
    article = Article(url)
    article.download()
    article.parse()
    article_content = {
        'TITLE': article.title,
        'TEXT': article.text,
        'METADATA': article.meta_data,
    }
    return article_content

def _build_article_dict(row, blacklisted_domains):
    """Create article dictionary from dataframe row"""
    article_dict = {}
    article_dict['GDELT'] = dict(row.dropna())
    url = article_dict['GDELT']['SOURCEURL']
    if any(url.startswith(u) for u in blacklisted_domains):
        return article_dict
    article_dict['ARTICLE'] = _extract_article_contents(url)
    return article_dict


import numpy as np
import pandas as pd

from sklearn.svm import SVR
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, Matern, RationalQuadratic

from aiaa_utils import get_dataset, test_interpolation
from aiaa_utils import plot_loo, plot_comparison_loo, split_output_input, plot_changing_error

def test_loo(inputs, outputs, model, speeds_test, num_samples, speeds=[]):
    # GET DATASET
    dataset = get_dataset(num_samples=num_samples, speeds=speeds)
    # EXECUTE TEST SEQUENCE
    df_mape = all_df_test_pred = pd.DataFrame([], columns=outputs)
    for speed in speeds_test:
        df_train, df_test = split_output_input(dataset, speeds_test=[speed])
        df_test_pred, list_mape = test_interpolation(inputs, outputs, model, df_train, df_test)
        df_mape = df_mape.append(pd.DataFrame([list_mape], columns=outputs))
        all_df_test_pred = pd.concat([all_df_test_pred, df_test_pred], sort=False)

    return df_mape, all_df_test_pred

############## MODELS ##############
svr = SVR(kernel='rbf', C=200, gamma="scale", epsilon=1e-10)
svr2 = SVR(kernel='rbf', C=200, epsilon=1e-10)
svr3 = SVR(kernel='rbf', C=20, gamma="scale", epsilon=1e-10)
svr4 = SVR(kernel='rbf', C=20, epsilon=1e-10)

gpr_rbf = GaussianProcessRegressor(kernel=RBF(length_scale_bounds=(3, 10)))
gpr_matern = GaussianProcessRegressor(kernel=Matern(length_scale_bounds=(3, 10), nu=5, length_scale=0.5))
gpr_quad = GaussianProcessRegressor(kernel=RationalQuadratic(length_scale_bounds=(3, 10), alpha_bounds=(3, 10)))

############## TRAINING SPECIFICATIONS ##############
outputs = ['Pressure_ratio', 'Polytropic_efficiency']
inputs = ['speeds_scaled', 'mass_flow_scaled']
speeds = np.arange(10000, 100000, 10000)

############## TEST SIMPLE LOO ##############
# num_samples = 25
# models = [svr, gpr_matern]
# list_results_models = []
# for model in models:
#     mape_results, list_df_test_pred = test_loo(inputs, outputs, model, speeds, num_samples)
#     print('MAPE mean: ', np.mean(mape_results.values))
#     list_results_models.append(mape_results)
# dict_legend = {'title': 'Models', 'tested_features': ['SVR', 'GPR']}
# plot_loo(list_results_models, speeds, outputs, dict_legend)

############## TEST KERNELS ##############
# num_samples = 25
# kernels = [gpr_rbf] #, gpr_matern
# speed = 30000
# list_results_kernels = []
# for kernel in kernels:
#     mape_results, list_df_test_pred = test_loo(inputs, outputs, kernel, speeds, num_samples)
#     print('MAPE mean: ', np.mean(mape_results.values))
#     list_results_kernels.append(list_df_test_pred)
#
# dict_legend = {'title': '', 'tested_features': ['True Values', 'RBF', 'Matern']}
# plot_comparison_loo(list_results_kernels, speed, outputs, dict_legend)

############## TEST SVR FEATURES ##############
num_samples = 25
speed = 30000
models = [svr, svr2, svr3, svr4]
list_results_models, list_results_structure = [], []
for model in models:
    mape_results, list_df_test_pred = test_loo(inputs, outputs, model, speeds, num_samples)
    print('MAPE mean: ', np.mean(mape_results.values))
    list_results_models.append(mape_results)
    list_results_structure.append(list_df_test_pred)
dict_legend = {'title': '', 'tested_features': ['C=200, scale', 'C=200, 1/n_features',
                                                'C=20, scale', 'C=20, 1/n_features']}
plot_loo(list_results_models, speeds, outputs, dict_legend)
dict_legend = {'title': '', 'tested_features': ['True Values', 'C=200, scale', 'C=200, 1/n_features',
                                                'C=20, scale', 'C=20, 1/n_features']}
plot_comparison_loo(list_results_structure, speed, outputs, dict_legend)

############## TEST NUM SAMPLES ##############
# list_num_samples = [5, 15, 20, 25]
# speed = 50000
# speeds = [20000, 30000, 40000, 50000, 60000, 70000]
# list_results_num_samples, list_mape_results = [], []
# for num_samples in list_num_samples:
#     mape_results, list_df_test_pred = test_loo(inputs, outputs, svr, speeds, num_samples, speeds=speeds)
#     list_results_num_samples.append(list_df_test_pred)
#     list_mape_results.append(mape_results)
#
# dict_legend = {'title': '', 'tested_features': ['True Values', '25', '5']}
# # plot_comparison_loo(list_results_num_samples, speed, outputs, dict_legend)
# plot_changing_error(list_mape_results, list_num_samples, outputs)

############## TEST NUM TRAINING ##############
# speed = [50000]
# all_speeds = [10000, 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000]
# num_iterations = [1, 2, 3, 4, 5, 6]
# num_samples = 25
# list_results_num_samples, list_mape_results = [], []
# for num_iteration in num_iterations:
#     if num_iteration % 2:
#         all_speeds = all_speeds[:-1]
#     else:
#         all_speeds = all_speeds[1:]
#     print('SPEEDS', all_speeds)
#     mape_results, list_df_test_pred = test_loo(inputs, outputs, svr, speed, num_samples, speeds=all_speeds)
#     list_results_num_samples.append(list_df_test_pred)
#     list_mape_results.append(mape_results)
#
# dict_legend = {'title': '', 'tested_features': ['True Values', '25', '5']}
# plot_changing_error(list_mape_results, num_iterations, outputs)

############## TEST COMPARISON MODELS ##############
# num_samples = 25
# list_models = [svr, gpr_matern]
# speed = 30000
# list_results_models = []
# for model in list_models:
#     _, list_df_test_pred = test_loo(inputs, outputs, model, speeds, num_samples)
#     list_results_models.append(list_df_test_pred)
#
# dict_legend = {'title': '', 'tested_features': ['True Values', 'SVR', 'GPR']}
# plot_comparison_loo(list_results_models, speed, outputs, dict_legend)


#!/usr/bin/env python
#
# LSST Data Management System
#
# Copyright 2008-2016  AURA/LSST.
#
# This product includes software developed by the
# LSST Project (http://www.lsst.org/).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program.  If not,
# see <https://www.lsstcorp.org/LegalNotices/>.
#
import unittest
from collections import namedtuple

import lsst.utils.tests
from lsst.afw.image import Filter
from lsst.obs.hsc import HscMapper

class CameraTestCase(lsst.utils.tests.TestCase):

    def setUp(self):
        self.mapper = HscMapper(root=".")
        self.camera = self.mapper.camera

    def tearDown(self):
        del self.camera
        del self.mapper

    def testName(self):
        self.assertEqual(self.camera.getName(), "HSC")

    def testNumCcds(self):
        self.assertEqual(len(list(self.camera.getIdIter())), 112)

    def testCcdSize(self):
        for ccd in self.camera:
            self.assertEqual(ccd.getBBox().getWidth(), 2048)
            self.assertEqual(ccd.getBBox().getHeight(), 4176)

    def testFilters(self):
        # Check that the mapper has defined some standard filters.
        # Note that this list is not intended to be comprehensive -- we
        # anticipate that more filters can be added without causing the test
        # to break -- but captures the standard HSC broad-band filters.
        FilterName = namedtuple("FilterName", ["alias", "canonical"])
        filterNames = (
            FilterName(alias="HSC-G", canonical="g"),
            FilterName(alias="HSC-R", canonical="r"),
            FilterName(alias="HSC-I", canonical="i"),
            FilterName(alias="HSC-Z", canonical="z"),
            FilterName(alias="HSC-Y", canonical="y"),
            FilterName(alias="NONE", canonical="UNRECOGNISED")
        )

        for filterName in filterNames:
            self.assertIn(filterName.alias, self.mapper.filters)
            self.assertEqual(Filter(filterName.alias).getCanonicalName(), filterName.canonical)

class TestMemory(lsst.utils.tests.MemoryTestCase):
    def setUp(self):
        HscMapper.clearCache()
        lsst.utils.tests.MemoryTestCase.setUp(self)

def setup_module(module):
    lsst.utils.tests.init()

if __name__ == "__main__":
    lsst.utils.tests.init()
    unittest.main()


#!python
#cython: language_level=3
import numpy as np
from collections import defaultdict
from . import srg

def lower_upper_bound(A: np.array, b: np.array, bounds: np.array) -> None:
    for row, lower_upper_bound in zip(A, b):
        nonzeros = np.nonzero(row)[0]
        # print(row, lower_upper_bound, nonzeros)
        # [0 0 0 0 1] 1 [4]
        # [0 0 1 1 0] 1 [2 3]
        # [1 0 1 0 0] 0 [0 2]
        # [1 1 0 0 0] 1 [0 1]
        for loc in nonzeros:
            bounds[loc] = min(bounds[loc], lower_upper_bound)

def zero_bound_loc(bounds: np.array) -> np.array:
    return np.where(bounds == 0)[0]

def one_element_row_locs(A: np.array) -> set:
    '''
    example input:

    [0 0 0 0 1] 1 [4]
    [0 0 1 1 0] 1 [2 3]
    [1 0 1 0 0] 0 [0 2]
    [1 1 0 0 0] 1 [0 1]

    defaultdict(<class 'list'>, {0: [4], 1: [2, 3], 2: [0, 2], 3: [0, 1]})

    return {(0,4)}

    '''
    nz = defaultdict(list)
    for rnz, cnz in zip(*A.nonzero()):
        nz[rnz].append(cnz)

    # reduce rcs so that columns are unique. Corresponding rows dont matter
    # {(0, 1), (3, 0), (1, 1), (4, 0)} => { 1:1, 0:4 }
    crs = {clist[0]:r for r, clist in nz.items() if len(clist) == 1}

    return {(r, c) for c, r in crs.items()}

if __name__ == '__main__':
    def test1():
        A = srg.array([[0, 0, 0, 0, 1],
                       [0, 0, 1, 1, 0],
                       [1, 0, 1, 0, 0],
                       [1, 1, 0, 0, 0]])
        b = srg.array([1, 1, 0, 1])
        bound = srg.array([1, 1, 1, 1, 1])

        print('original upper bounds', bound)
        lower_upper_bound(A, b, bound)
        print('new lower upper bounds', bound)
        assert np.array_equal(bound, [0, 1, 0, 1, 1])

        rcs = one_element_row_locs(A)
        print('one element rows (r,c)=', rcs)
        assert np.array_equal(rcs.pop(), (0, 4))

    A = srg.array([[0, 1], [0, 1], [0, 0], [1, 0], [1, 0], [0, 0]])
    b = srg.array([4, 4, 0, 4, 4, 0])
    bound = srg.array([4, 4])

    print('original upper bounds', bound)
    lower_upper_bound(A, b, bound)
    print('new lower upper bounds', bound)

    rcs = one_element_row_locs(A)
    print('one element rows (r,c)=', rcs)


#!/usr/bin/env python3.6
from skimage.measure import label, regionprops

print('hi')

import torch.nn.functional as F
import matplotlib.pyplot as plt
from pathlib import Path
from PIL import Image
import numpy as np
import torch
from torch.autograd import Variable
import sys
import os
import csv

sys.path.append('/home/eljurros/spare-workplace/surface-loss-master')
from utils import dice_coef, dice_batch, save_images, tqdm_, haussdorf, probs2one_hot, class2one_hot, numpy_haussdorf
root = '/media/eljurros/Transcend/Decathlone/benchmark/isles/FOLD_1'
net_path = '/media/eljurros/Transcend/Decathlone/benchmark/isles/FOLD_1/results/eroded_distance/best2.pkl'
net = torch.load(net_path, map_location=torch.device('cpu'))
#print(net)
fieldnames = ['SLICE_ID', 'dice','haus',  'c_error']
n_classes = 4
#assert os.path.exists(os.path.join(net_path.split(os.path.basename(net_path))[0], 'predictions'))== False
print('started this stupid work')
exp_path = net_path.split('/best2.pkl')[0]
name =os.path.basename(exp_path)
folder_path = Path(exp_path, 'CSV_RESULTS')

folder_path.mkdir(parents=True, exist_ok=True)
file_path = os.path.join(exp_path, name)
fold_clean_H1 = open(os.path.join(folder_path, '{}_clean.csv'.format(name)), "w")
fold_all_H1 = open(os.path.join(folder_path, '{}_all.csv'.format(name)), "w")

fold_all_H1.write(f"file, dice, haussdorf,connecterror \n")
fold_clean_H1.write(f"file, dice, haussdorf,connecterror \n")

path=os.path.join(net_path.split(os.path.basename(net_path))[0])

pred_path = Path(path,'predictions')
gt_path = Path(path,'gt')

pred_path.mkdir(parents=True, exist_ok=True)
gt_path.mkdir(parents=True, exist_ok=True)

for _,_,files in os.walk(os.path.join(root, 'img')):

    print('walking into', os.path.join(root, 'img'))
    for file in files:
        print(file)
        image = np.array(Image.open(os.path.join(root,'img', file)))
        gt = np.array(Image.open(os.path.join(root,'gt', file)))
        if len(np.unique(gt)) >0:
            #print('infering {} of shape {} and classes {}, max {} and min {} '.format( file, image.shape, np.unique(gt), image.max(), image.min()))
            image = image.reshape(-1, 5, 256, 256)/255.00
            image = torch.tensor(image, dtype=torch.float)
            image = Variable(image, requires_grad=True)
            pred = net(image)
            pred = F.softmax(pred, dim=1).to('cpu')
            predicted_output = probs2one_hot(pred.detach())
            np.save(os.path.join(path, 'predictions', '{}'.format(file)), pred.to('cpu').detach().numpy())
            dice = dice_coef(predicted_output.to('cpu'), class2one_hot(torch.tensor(gt).to('cpu'), n_classes))
            hauss = haussdorf(predicted_output,class2one_hot(torch.tensor(gt).to('cpu'), n_classes))
            plt.imsave(os.path.join(path, 'predictions', '{}.png'.format(file.split('.npy')[0])), np.argmax(predicted_output,1)[0])
            plt.imsave(os.path.join(path, 'gt', '{}.png'.format(file.split('.npy')[0])), gt)

            pred_label = len(np.unique(label(np.array(pred.argmax(axis = 1).detach().numpy()))))
            gt_label = len(np.unique(label(gt)))
            pred_label = len(np.unique(label(predicted_output[0][1])))
            gt_label = len(np.unique(label(class2one_hot(torch.tensor(gt), n_classes)[0][2])))
            error = np.abs(pred_label - gt_label)

            print(f"{file}, {np.float(dice[0][2])}, {np.float(hauss[0][2])},{np.float(error)} \n")

            fold_all_H1.write(f"{file}, {np.float(dice[0][2])}, {np.float(hauss[0][2])},{np.float(error)} \n")

            if len(np.unique(gt)) == 2:
                fold_clean_H1.write(f"{file}, {np.float(dice[0][2])}, {np.float(hauss[0][2])},{np.float(error)} \n")
                fold_clean_H1.flush()
            #folders.write("hi")
            fold_all_H1.flush()


from app import app, memcache
import flask

@app.route('/fibonachi', methods=('GET', ))
def fibonachi():
    required_args = ['from', 'to']
    try:
        f, t = [int(flask.request.args[x]) for x in required_args]
    except (KeyError, TypeError):
        flask.abort(400)

    if f > t or t < 0 or f < 0:
        flask.abort(400)

    res = []
    offset = 10
    fk = f - f % offset
    tk = t - t % offset + offset
    start = fk
    ref1, ref2 = None, None  # The last 2 numbers in previous block in order to calculate ones in current block

    # Try to figure out whether previous block exists
    # If not, check whole sequence from 0
    if start > 0:
        prev_cached = memcache.client.get(str(start - offset))
        if prev_cached:
            ref1, ref2 = [int(x) for x in prev_cached.decode().split(',')][-2:]
        else:
            start = 0

    for k in range(start, tk, offset):
        cached = memcache.client.get(str(k))
        seq = None

        if not cached:
            seq = list(get_fibonacci(ref1, ref2, offset))
            ref1, ref2 = seq[-2:]
            memcache.client.set(str(k), ','.join(str(x) for x in seq))
        else:
            seq = [int(x) for x in cached.decode().split(',')]
            ref1, ref2 = seq[-2:]

        # print(seq)
        if k >= fk:
            res.extend(seq or [int(x) for x in cached.decode().split(',')])

    res = [int(x) for x in res[f % offset:-offset + (t % offset)]]
    return flask.jsonify(res)

def get_fibonacci(ref1, ref2, count):
    """
        Return Fibonacci sequence of numbers starting next after ref2. I.e. the first item will be ref1+ref2
        If both ref1 and ref2 are None then returns the seqence from start: 0, 1, 1, 2,...
    """
    p, q = ref1, ref2
    c = count

    if ref1 is None and ref2 is None:
        p, q = 0, 1
        yield p
        yield q
        c -= 2
    elif p > q:
        raise ValueError('ref1 more than ref2')

    while c > 0:
        s = p + q
        p = q
        q = s
        yield s
        c -= 1


'''Utility functions for dealing with locations and distances'''
import itertools
import pandas as pd
import numpy as np
import geopy.distance as geodist
import datetime
import maps
import requests

def extract_locations(data):
    '''Get list of (lat, lng) tuples from dataframe'''
    recs = data[['Latitude', 'Longitude']].to_records(index=False)
    return [list(x) for x in recs]

def distance_matrix(row_locs, col_locs, row_names, col_names):
    '''
    Get straight line distances between locations, along with a cutoff value
        for each row which can be used to define "nearby" locations for that
        row, relative to the closest location
        row_locs, col_locs -- lists of (lat, lng) tuples
        row_names, col_names -- indices identifying locations
    '''
    dist_vals = [geodist.distance(l1, l2).miles
                 for l1, l2 in itertools.product(row_locs, col_locs)]
    dist_vals = np.reshape(dist_vals, (len(row_locs), len(col_locs)))
    out = pd.DataFrame(dist_vals, columns=col_names, index=row_names)
    out['Min_dist'] = out.min(axis=1)
    out['Cutoff'] = out.Min_dist.apply(lambda m: max(m * 1.5, m + 30))
    out = out.drop(columns='Min_dist')
    return out

def _to_gmap_coor(coors):
    return '|'.join([','.join([str(x) for x in y]) for y in coors])

def get_request(api_request):
    return requests.get(api_request).json()

def get_depart_time():
    ''' Return peak traffic time at 5:30PM in a weekday
        that is in the future, to be used as an argument for
        Google Maps Distance Matrix API departure_time'''
    today = datetime.datetime.today()
    tmrw = today + datetime.timedelta(days=1)
    while tmrw.weekday() >= 5: # saturday is 5, sunday is 6
        tmrw += datetime.timedelta(days=1)
    peak_hour = datetime.time(17,30,0) # 5:30PM
    depart_time = datetime.datetime.combine(tmrw,peak_hour)
    return int(depart_time.timestamp())

def gmap_distance_matrix(origins,destinations,mode='driving-traffic'):
    if mode=='driving-traffic':
        mode = 'driving'
        depart_time = get_depart_time()
    else:
        depart_time = None
    api_template = 'https://maps.googleapis.com/maps/api/distancematrix/json?origins={}&destinations={}'
    origins,destinations = _to_gmap_coor(origins),_to_gmap_coor(destinations)
    api_call = api_template.format(origins,destinations)
    if depart_time: api_call += "&departure_time={}".format(depart_time)
    api_call += "&key={}".format(maps.get_key())
    return get_request(api_call)


from __future__ import division
from __future__ import print_function

import time
import tensorflow as tf

from gcn.utils import *
from gcn.models import GCN, MLP

# Set random seed
seed = 123
np.random.seed(seed)
tf.set_random_seed(seed)

# Settings
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('dataset', 'cora', 'Dataset string.')  # 'cora', 'citeseer', 'pubmed'
flags.DEFINE_string('model', 'gcn', 'Model string.')  # 'gcn', 'gcn_cheby', 'dense'
flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('epochs', 2000, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 16, 'Number of units in hidden layer 1.')
flags.DEFINE_float('dropout', 0.5, 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')
flags.DEFINE_integer('early_stopping', 10, 'Tolerance for early stopping (# of epochs).')
flags.DEFINE_integer('max_degree', 3, 'Maximum Chebyshev polynomial degree.')

# Load data
# adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data(FLAGS.dataset)
dataset_str = FLAGS.dataset
names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
objects = []
for i in range(len(names)):
    with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
        if sys.version_info > (3, 0):
            objects.append(pkl.load(f, encoding='latin1'))
        else:
            objects.append(pkl.load(f))

x, y, tx, ty, allx, ally, graph = tuple(objects)
test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
test_idx_range = np.sort(test_idx_reorder)

if dataset_str == 'citeseer':
    # Fix citeseer dataset (there are some isolated nodes in the graph)
    # Find isolated nodes, add them as zero-vecs into the right position
    test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
    tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
    tx_extended[test_idx_range-min(test_idx_range), :] = tx
    tx = tx_extended
    ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
    ty_extended[test_idx_range-min(test_idx_range), :] = ty
    ty = ty_extended

features = sp.vstack((allx, tx)).tolil()
features[test_idx_reorder, :] = features[test_idx_range, :]
adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))

labels = np.vstack((ally, ty))
labels[test_idx_reorder, :] = labels[test_idx_range, :]

idx_test = test_idx_range.tolist()
idx_train = range(len(y))
idx_val = range(len(y), len(y)+500)

train_mask = sample_mask(idx_train, labels.shape[0])
val_mask = sample_mask(idx_val, labels.shape[0])
test_mask = sample_mask(idx_test, labels.shape[0])

y_train = np.zeros(labels.shape)
y_val = np.zeros(labels.shape)
y_test = np.zeros(labels.shape)
y_train[train_mask, :] = labels[train_mask, :]
y_val[val_mask, :] = labels[val_mask, :]
y_test[test_mask, :] = labels[test_mask, :]
import pdb; pdb.set_trace()

# Some preprocessing
features = preprocess_features(features)
if FLAGS.model == 'gcn':
    support = [preprocess_adj(adj)]
    num_supports = 1
    model_func = GCN
elif FLAGS.model == 'gcn_cheby':
    support = chebyshev_polynomials(adj, FLAGS.max_degree)
    num_supports = 1 + FLAGS.max_degree
    model_func = GCN
elif FLAGS.model == 'dense':
    support = [preprocess_adj(adj)]  # Not used
    num_supports = 1
    model_func = MLP
else:
    raise ValueError('Invalid argument for model: ' + str(FLAGS.model))

# Define placeholders
placeholders = {
    'support': [tf.sparse_placeholder(tf.float32) for _ in range(num_supports)],
    'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),
    'labels': tf.placeholder(tf.float32, shape=(None, y_train.shape[1])),
    'labels_mask': tf.placeholder(tf.int32),
    'dropout': tf.placeholder_with_default(0., shape=()),
    'num_features_nonzero': tf.placeholder(tf.int32)  # helper variable for sparse dropout
}

# Create model
model = model_func(placeholders, input_dim=features[2][1], logging=True)

# Initialize session
sess = tf.Session()

# Define model evaluation function
def evaluate(features, support, labels, mask, placeholders):
    t_test = time.time()
    feed_dict_val = construct_feed_dict(features, support, labels, mask, placeholders)
    outs_val = sess.run([model.loss, model.accuracy], feed_dict=feed_dict_val)
    return outs_val[0], outs_val[1], (time.time() - t_test)

# Init variables
sess.run(tf.global_variables_initializer())

cost_val = []

# Train model
for epoch in range(FLAGS.epochs):

    t = time.time()
    # Construct feed dictionary
    feed_dict = construct_feed_dict(features, support, y_train, train_mask, placeholders)
    feed_dict.update({placeholders['dropout']: FLAGS.dropout})

    # Training step
    outs = sess.run([model.opt_op, model.loss, model.accuracy], feed_dict=feed_dict)

    # Validation
    cost, acc, duration = evaluate(features, support, y_val, val_mask, placeholders)
    cost_val.append(cost)

    # Print results
    print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(outs[1]),
          "train_acc=", "{:.5f}".format(outs[2]), "val_loss=", "{:.5f}".format(cost),
          "val_acc=", "{:.5f}".format(acc), "time=", "{:.5f}".format(time.time() - t))

    if epoch > FLAGS.early_stopping and cost_val[-1] > np.mean(cost_val[-(FLAGS.early_stopping+1):-1]):
        print("Early stopping...")
        break

print("Optimization Finished!")

# Testing
test_cost, test_acc, test_duration = evaluate(features, support, y_test, test_mask, placeholders)
print("Test set results:", "cost=", "{:.5f}".format(test_cost),
      "accuracy=", "{:.5f}".format(test_acc), "time=", "{:.5f}".format(test_duration))


from django.http import Http404
from django.shortcuts import render, redirect
from django.template import TemplateDoesNotExist

from ladders.logic.util import get_ladder_or_404, get_watcher

def handle_form_and_redirect_to_ladder(request, ladder_id, form_class, template, context=None, form_name='form'):
    if not context:
        context = {}
    ladder = get_ladder_or_404(pk=ladder_id)
    if request.POST:
        form = form_class(ladder, request.POST)
        if form.is_valid():
            form.save()
            return redirect(ladder)
    else:
        form = form_class(ladder)
    context[form_name] = form
    return view_with_ladder(request, ladder, template, context=context)

def handle_form(request, ladder_id, form_class, template, context=None, form_name='form'):
    if not context:
        context = {}
    ladder = get_ladder_or_404(pk=ladder_id)
    if request.POST:
        form = form_class(ladder, request.POST)
        if form.is_valid():
            form.save()
            form.success = "Form submitted successfully"
    else:
        form = form_class(ladder)
    context[form_name] = form
    return view_with_ladder(request, ladder, template, context=context)

def view_with_ladder(request, ladder, template, context=None):
    if not context:
        context = {}
    context.update({'ladder': ladder, 'watcher': get_watcher(request.user, ladder)})
    try:
        return render(request, template, context)
    except TemplateDoesNotExist:
        raise Http404("Template does not exist: %s" % template)


#!/usr/bin/env python
#-*- coding:utf-8 -*-

import os

from collections import defaultdict

from .lowlevel import load_any, unload_any, filter_extensions, VALID_BODY_KERNEL_TYPES
from .._helpers import ignored, cleanpath, iterable_path, TimeWindows

__all__ = ['Kernel']

class Kernel(object):
    '''A loaded kernel file.

    .. WARNING:: The constructor should never be used directly. Use one of the
                 methods in the *See also* section.

    Parameters
    ----------
    path: str
        Relative or absolute path to the kernel file/directory.
    kernel_type: {'sp', 'c', 'pc', 'f', 'ls', 'sc'}
        Used to identify what information the kernel is holding and how to load
        it.
    force_reload: bool, optional
        Reload already loaded kernel files.
        By default an error is raised if a kernel file is already loaded.

    Raises
    ------
    ValueError
        If the kernel file is already loaded.

    Attributes
    ----------
    *classattribute* LOADED: set of Kernel
        All loaded kernels.
    *classattribute* TIMEWINDOWS_POS: dict of int -> list
        Maps a list of start-end-tuples for all known time windows of its
        position to the id of a Body.
    *classattribute* TIMEWINDOWS_ROT: dict of int -> list
        Maps a list of start-end-tuples for all known time windows of its
        rotation to the id of a Body.
    path: str
        Absolute path to the kernel file.
    name: str
        Filename of the kernel file.
    type: {'sp', 'c', 'pc', 'f', 'ls', 'sc'}
        Identifier for kernel loading mechanism.
    info: {'pos', 'rot', 'none'}
        What kind of spatial information the kernel contains.
    ids: set of int
        The IDs of all bodies about which the kernel has information.

    See also
    --------
    load: Load multiple files.
    load_single: More controllable way to load single files.
    unload: Unload kernels.
    '''
    LOADED = set()
    TIMEWINDOWS_POS = defaultdict(TimeWindows)
    TIMEWINDOWS_ROT = defaultdict(TimeWindows)

    def __new__(cls, path, kernel_type, force_reload=False):
        path = cleanpath(path)
        hit = {hash(k): k for k in cls.LOADED}.get(hash(path))
        if hit:
            if not force_reload:
                msg = "Kernel already loaded: {}"
                raise ValueError(msg.format(path))
            else:
                hit._unload()
                return hit
        else:
            return object.__new__(cls)

    def __init__(self, path, kernel_type, force_reload=False):
        self.path = cleanpath(path)
        self.name = os.path.basename(self.path)
        self.type = kernel_type
        # Load the kernel and handle time windows
        self.info, self._intervals = load_any(path, self.type)
        self.ids = frozenset(self._intervals.keys())
        with ignored(AttributeError):
            windows = self._timewindows()
            for key, vals in self._intervals.iteritems():
                windows[key] += vals
        # Make self available for unloading
        self.__class__.LOADED.add(self)

    def _unload(self):
        self.__class__.LOADED.remove(self)
        with ignored(AttributeError):
            windows = self._timewindows()
            for key, vals in self._intervals.iteritems():
                windows[key] -= vals
                if not windows[key]:
                    del windows[key]
        unload_any(self)

    def _timewindows(self):
        windows = 'TIMEWINDOWS_' + self.info.upper()
        return getattr(self.__class__, windows)

    def __hash__(self):
        return hash(self.path)

    def __str__(self):
        return 'Kernel {} ({})'.format(self.name, self.type)

    def __repr__(self):
        return '{}({}, {})'.format(self.__class__.__name__, self.path,
            self.type)

    @classmethod
    def load(cls, path='.', recursive=True, followlinks=False, force_reload=False):
        '''Load a kernel file or all kernel files in a directory tree.

        Refer to `spiceminer.load` for full documentation.

        See also
        --------
        spiceminer.load_single: More controllable way to load single files.
        spiceminer.unload: Unload kernels.
        '''
        path = cleanpath(path)
        # Find and sort kernel files.
        body_kernels = set()
        misc_kernels = set()
        for dir_path, _, fnames in iterable_path(path, recursive, followlinks):
            for name in fnames:
                with ignored(ValueError):
                    kernel_type = filter_extensions(name)
                    filepath = os.path.join(dir_path, name)
                    # Split kernels into those containing bodies and others
                    if kernel_type in VALID_BODY_KERNEL_TYPES:
                        body_kernels.add((filepath, kernel_type))
                    else:
                        misc_kernels.add((filepath, kernel_type))
        # Make sure, we actually found at least one valid file.
        if not (body_kernels or misc_kernels):
            msg = 'No valid files found on path'
            raise IOError(2, msg, path)
        # Load collected kernel files (misc first to avoid missing leapseconds).
        loaded = set()
        for path, extension in misc_kernels:
            k = cls(path, extension, force_reload)
            loaded.add(k)
        for path, extension in body_kernels:
            k = cls(path, extension, force_reload)
            loaded.add(k)
        return loaded

    @classmethod
    def load_single(cls, path, extension=None, force_reload=False):
        '''Load a single kernel file. Allows non-standart extensions.

        Refer to `spiceminer.load_single` for full documentation.

        See also
        --------
        spiceminer.load: Load multiple files.
        spiceminer.unload: Unload kernels.
        '''
        if extension is not None:
            kernel_type = filter_extensions('.' + extension)
        else:
            kernel_type = filter_extensions(path)
        k = cls(path, kernel_type, force_reload)
        return k

    @classmethod
    def unload(cls, path='.', recursive=True, followlinks=False):
        '''Unload a kernel file or all kernel files in a directory tree.

        Refer to `spiceminer.unload` for full documentation.

        See also
        --------
        spiceminer.load: Load multiple files.
        spiceminer.load_single: More controllable way to load single files.
        '''
        path = cleanpath(path)
        if not os.path.exists(path):
            msg = 'unload() expected a valid file or directory path, got {}'
            raise ValueError(msg.format(path))
        # Find and sort kernel files.
        hashes = set()
        for dir_path, _, fnames in iterable_path(path, recursive, followlinks):
            for name in fnames:
                # Ignore extension to be able to unload all kernels
                hashes.add(hash(os.path.join(dir_path, name)))
        # Unload collected kernel files.
        unloaded = {k for k in cls.LOADED if hash(k) in hashes}
        for k in unloaded:
            k._unload()
        return unloaded


import numpy as np
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import cv2

def points_extraction(images, nx, ny):
    """
    perform the extraction of object points and image points
    :param images: list of images
    :param nx: the number of inside corners in x
    :param ny: the number of inside corners in y
    :return: object points, image points and retvalues
    """

    # Arrays to store object points and image points from all the images.
    objpoints = []  # 3d points in real world space
    imgpoints = []  # 2d points in image plane.
    retlist = []

    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
    objp = np.zeros((ny * nx, 3), np.float32)
    objp[:, :2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)

    for idx, fname in enumerate(images):
        img = cv2.imread(fname)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # Find the chessboard corners
        ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)

        retlist.append(ret)

        # If found, add object points, image points
        if ret == True:
            objpoints.append(objp)
            imgpoints.append(corners)

    #         # Draw and display the corners
    #         cv2.drawChessboardCorners(img, (8, 6), corners, ret)
    #         # write_name = 'corners_found'+str(idx)+'.jpg'
    #         # cv2.imwrite(write_name, img)
    #         cv2.imshow('img', img)
    #         cv2.waitKey(500)
    #
    # cv2.destroyAllWindows()
    return objpoints, imgpoints, retlist

# Define a function that takes an image, number of x and y points,
# camera matrix and distortion coefficients
def corners_unwarp(img, nx, ny, mtx, dist):
    # Use the OpenCV undistort() function to remove distortion
    undist = cv2.undistort(img, mtx, dist, None, mtx)
    # Convert undistorted image to grayscale
    gray = cv2.cvtColor(undist, cv2.COLOR_BGR2GRAY)
    # Search for corners in the grayscaled image
    ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)

    if ret == True:
        # If we found corners, draw them! (just for fun)
        cv2.drawChessboardCorners(undist, (nx, ny), corners, ret)
        # Choose offset from image corners to plot detected corners
        # This should be chosen to present the result at the proper aspect ratio
        # My choice of 100 pixels is not exact, but close enough for our purpose here
        offset = 100 # offset for dst points
        # Grab the image shape
        img_size = (gray.shape[1], gray.shape[0])

        # For source points I'm grabbing the outer four detected corners
        src = np.float32([corners[0], corners[nx-1], corners[-1], corners[-nx]])
        # For destination points, I'm arbitrarily choosing some points to be
        # a nice fit for displaying our warped result
        # again, not exact, but close enough for our purposes
        dst = np.float32([[offset, offset], [img_size[0]-offset, offset],
                                     [img_size[0]-offset, img_size[1]-offset],
                                     [offset, img_size[1]-offset]])
        # Given src and dst points, calculate the perspective transform matrix
        M = cv2.getPerspectiveTransform(src, dst)
        # Warp the image using OpenCV warpPerspective()
        warped = cv2.warpPerspective(undist, M, img_size)

    # Return the resulting image and matrix
    return warped, M

# Define a function that applies Sobel x or y,
# then takes an absolute value and applies a threshold.
# Note: calling your function with orient='x', thresh_min=5, thresh_max=100
# should produce output like the example image shown above this quiz.
def abs_sobel_thresh(img, is_gray=True, orient='x', sobel_kernel=3, sob_thresh=(0, 255)):
    # Apply the following steps to img
    # 1) Convert to grayscale

    gray = np.copy(img)
    if is_gray == False:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 2) Take the derivative in x or y given orient = 'x' or 'y'
    if (orient == 'x'):
        sobel = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    elif (orient == 'y'):
        sobel = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Take the absolute value of the derivative or gradient
    abs_sobel = np.absolute(sobel)
    # 4) Scale to 8-bit (0 - 255) then convert to type = np.uint8
    scaled_sobel = np.uint8(255 * abs_sobel / np.max(abs_sobel))
    # 5) Create a mask of 1's where the scaled gradient magnitude
    # is > thresh_min and < thresh_max
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel >= sob_thresh[0]) & (scaled_sobel <= sob_thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    # binary_output = np.copy(img) # Remove this line
    return binary_output

# Define a function that applies Sobel x and y,
# then computes the magnitude of the gradient
# and applies a threshold
def mag_thresh(img, is_gray=True, sobel_kernel=3, mag_thresh=(0, 255)):
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = np.copy(img)
    if is_gray == False:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Calculate the magnitude
    abs_sobelxy = np.sqrt(sobelx ** 2 + sobely ** 2)
    # 4) Scale to 8-bit (0 - 255) and convert to type = np.uint8
    scaled_sobel = np.uint8(255 * abs_sobelxy / np.max(abs_sobelxy))
    # 5) Create a binary mask where mag thresholds are met
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel >= mag_thresh[0]) & (scaled_sobel <= mag_thresh[1])] = 1
    # 6) Return this mask as your binary_output image

    return binary_output

# Define a function that applies Sobel x and y,
# then computes the direction of the gradient
# and applies a threshold.
def dir_threshold(img, is_gray=True, sobel_kernel=3, thresh=(0, np.pi / 2)):
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = np.copy(img)
    if is_gray == False:
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Take the absolute value of the x and y gradients
    abs_sobelx = np.absolute(sobelx)
    abs_sobely = np.absolute(sobely)
    # 4) Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient
    gr_orient = np.arctan2(abs_sobely, abs_sobelx)
    # 5) Create a binary mask where direction thresholds are met
    binary_output = np.zeros_like(gr_orient)
    # 6) Return this mask as your binary_output image
    binary_output[(gr_orient >= thresh[0]) & (gr_orient <= thresh[1])] = 1

    return binary_output

# Define a function that thresholds the S-channel of HLS
# Use exclusive lower bound (>) and inclusive upper (<=)
def hls_select(img, thresh=(0, 255)):
    # 1) Convert to HLS color space
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
    # 2) Apply a threshold to the S channel
    S = hls[:,:,2]
    # 3) Return a binary image of threshold result
    binary_output = np.zeros_like(S)
    binary_output[(S > thresh[0]) & (S <= thresh[1])] = 1

    return binary_output

def hist(img):
    # TO-DO: Grab only the bottom half of the image
    # Lane lines are likely to be mostly vertical nearest to the car
    bottom_half = img[img.shape[0] // 2:, :]

    # TO-DO: Sum across image pixels vertically - make sure to set `axis`
    # i.e. the highest areas of vertical lines should be larger values
    histogram = np.sum(bottom_half, axis=0)

    return histogram

def find_lane_pixels(binary_warped):
    # Take a histogram of the bottom half of the image
    histogram = np.sum(binary_warped[binary_warped.shape[0] // 2:, :], axis=0)
    # Create an output image to draw on and visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0] // 2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # HYPERPARAMETERS
    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 45
    # Set minimum number of pixels found to recenter window
    minpix = 65

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(binary_warped.shape[0] // nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window + 1) * window_height
        win_y_high = binary_warped.shape[0] - window * window_height
        ### TO-DO: Find the four below boundaries of the window ###
        win_xleft_low = leftx_current - margin  # Update this
        win_xleft_high = leftx_current + margin  # Update this
        win_xright_low = rightx_current - margin  # Update this
        win_xright_high = rightx_current + margin  # Update this

        # Draw the windows on the visualization image
        cv2.rectangle(out_img, (win_xleft_low, win_y_low),
                      (win_xleft_high, win_y_high), (0, 255, 0), 2)
        cv2.rectangle(out_img, (win_xright_low, win_y_low),
                      (win_xright_high, win_y_high), (0, 255, 0), 2)

        ### TO-DO: Identify the nonzero pixels in x and y within the window ###
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                          (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                           (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]

        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)

        ### TO-DO: If you found > minpix pixels, recenter next window ###
        ### (`right` or `leftx_current`) on their mean position ###
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

        # pass # Remove this when you add your function

    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    try:
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)
    except ValueError:
        # Avoids an error if the above is not implemented fully
        pass

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    return leftx, lefty, rightx, righty, out_img

def fit_polynomial(binary_warped):
    # Find our lane pixels first
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped)

    ### TO-DO: Fit a second order polynomial to each using `np.polyfit` ###
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)

    # Generate x and y values for plotting
    ploty = np.linspace(0, binary_warped.shape[0] - 1, binary_warped.shape[0])
    try:
        left_fitx = left_fit[0] * ploty ** 2 + left_fit[1] * ploty + left_fit[2]
        right_fitx = right_fit[0] * ploty ** 2 + right_fit[1] * ploty + right_fit[2]
    except TypeError:
        # Avoids an error if `left` and `right_fit` are still none or incorrect
        print('The function failed to fit a line!')
        left_fitx = 1 * ploty ** 2 + 1 * ploty
        right_fitx = 1 * ploty ** 2 + 1 * ploty

    ## Visualization ##
    # Colors in the left and right lane regions
    # out_img[lefty, leftx] = [255, 0, 0]
    # out_img[righty, rightx] = [0, 0, 255]

    # Plots the left and right polynomials on the lane lines
    # plt.plot(left_fitx, ploty, color='yellow')
    # plt.plot(right_fitx, ploty, color='yellow')

    return leftx, lefty, rightx, righty, left_fitx, right_fitx, ploty, out_img

def fit_poly(img_shape, leftx, lefty, rightx, righty):
    ### TO-DO: Fit a second order polynomial to each with np.polyfit() ###
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    # Generate x and y values for plotting
    ploty = np.linspace(0, img_shape[0] - 1, img_shape[0])
    ### TO-DO: Calc both polynomials using ploty, left_fit and right_fit ###
    left_fitx = left_fit[0] * ploty ** 2 + left_fit[1] * ploty + left_fit[2]
    right_fitx = right_fit[0] * ploty ** 2 + right_fit[1] * ploty + right_fit[2]

    return left_fitx, right_fitx, ploty

def search_around_poly(binary_warped, left_fit, right_fit, margin = 45):
    # HYPERPARAMETER
    # Choose the width of the margin around the previous polynomial to search
    # The quiz grader expects 100 here, but feel free to tune on your own!
    # margin = 45

    # Grab activated pixels
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])

    ### TO-DO: Set the area of search based on activated x-values ###
    ### within the +/- margin of our polynomial function ###
    ### Hint: consider the window areas for the similarly named variables ###
    ### in the previous quiz, but change the windows to our new search area ###

    left_lane_inds = (
            (nonzerox > (left_fit[0] * (nonzeroy ** 2) + left_fit[1] * nonzeroy + left_fit[2] - margin))
            & (nonzerox < (left_fit[0] * (nonzeroy ** 2) + left_fit[1] * nonzeroy + left_fit[2] + margin))
    )
    right_lane_inds = (
            (nonzerox > (right_fit[0] * (nonzeroy ** 2) + right_fit[1] * nonzeroy + right_fit[2] - margin))
            & (nonzerox < (right_fit[0] * (nonzeroy ** 2) + right_fit[1] * nonzeroy + right_fit[2] + margin))
    )

    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    # Fit new polynomials
    left_fitx, right_fitx, ploty = fit_poly(binary_warped.shape, leftx, lefty, rightx, righty)

    ## Visualization ##
    # Create an image to draw on and an image to show the selection window
    out_img = np.dstack((binary_warped, binary_warped, binary_warped)) * 255
    window_img = np.zeros_like(out_img)
    # Color in left and right line pixels
    out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
    out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]

    # Generate a polygon to illustrate the search window area
    # And recast the x and y points into usable format for cv2.fillPoly()
    left_line_window1 = np.array([np.transpose(np.vstack([left_fitx - margin, ploty]))])
    left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx + margin, ploty])))])
    left_line_pts = np.hstack((left_line_window1, left_line_window2))

    right_line_window1 = np.array([np.transpose(np.vstack([right_fitx - margin, ploty]))])
    right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx + margin, ploty])))])
    right_line_pts = np.hstack((right_line_window1, right_line_window2))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(window_img, np.int_([left_line_pts]), (0, 255, 0))
    cv2.fillPoly(window_img, np.int_([right_line_pts]), (0, 255, 0))
    result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)

    bwn_window_img = np.zeros_like(out_img)
    left_lane_line_window = np.array([np.flipud(np.transpose(np.vstack([left_fitx + margin, ploty])))])
    right_lane_line_window = np.array([np.transpose(np.vstack([right_fitx - margin, ploty]))])
    lane_line_pts = np.hstack((left_lane_line_window, right_lane_line_window))

    cv2.fillPoly(bwn_window_img, np.int_([lane_line_pts]), (255, 0, 0))
    result = cv2.addWeighted(result, 1, bwn_window_img, 0.3, 0)

    # Plot the polynomial lines onto the image
    plt.plot(left_fitx, ploty, color='yellow')
    plt.plot(right_fitx, ploty, color='yellow')
    ## End visualization steps ##

    return result

# def generate_data():
#     '''
#     Generates fake data to use for calculating lane curvature.
#     In your own project, you'll ignore this function and instead
#     feed in the output of your lane detection algorithm to
#     the lane curvature calculation.
#     '''
#     # Set random seed number so results are consistent for grader
#     # Comment this out if you'd like to see results on different random data!
#     np.random.seed(0)
#     # Generate some fake data to represent lane-line pixels
#     ploty = np.linspace(0, 719, num=720)  # to cover same y-range as image
#     quadratic_coeff = 3e-4  # arbitrary quadratic coefficient
#     # For each y position generate random x position within +/-50 pix
#     # of the line base position in each case (x=200 for left, and x=900 for right)
#     leftx = np.array([200 + (y ** 2) * quadratic_coeff + np.random.randint(-50, high=51)
#                       for y in ploty])
#     rightx = np.array([900 + (y ** 2) * quadratic_coeff + np.random.randint(-50, high=51)
#                        for y in ploty])
#
#     leftx = leftx[::-1]  # Reverse to match top-to-bottom in y
#     rightx = rightx[::-1]  # Reverse to match top-to-bottom in y
#
#     # Fit a second order polynomial to pixel positions in each fake lane line
#     left_fit = np.polyfit(ploty, leftx, 2)
#     right_fit = np.polyfit(ploty, rightx, 2)
#
#     return ploty, left_fit, right_fit

def generate_data(ym_per_pix, xm_per_pix):
    '''
    Generates fake data to use for calculating lane curvature.
    In your own project, you'll ignore this function and instead
    feed in the output of your lane detection algorithm to
    the lane curvature calculation.
    '''
    # Set random seed number so results are consistent for grader
    # Comment this out if you'd like to see results on different random data!
    np.random.seed(0)
    # Generate some fake data to represent lane-line pixels
    ploty = np.linspace(0, 719, num=720)  # to cover same y-range as image
    quadratic_coeff = 3e-4  # arbitrary quadratic coefficient
    # For each y position generate random x position within +/-50 pix
    # of the line base position in each case (x=200 for left, and x=900 for right)
    leftx = np.array([200 + (y ** 2) * quadratic_coeff + np.random.randint(-50, high=51)
                      for y in ploty])
    rightx = np.array([900 + (y ** 2) * quadratic_coeff + np.random.randint(-50, high=51)
                       for y in ploty])

    leftx = leftx[::-1]  # Reverse to match top-to-bottom in y
    rightx = rightx[::-1]  # Reverse to match top-to-bottom in y

    # Fit a second order polynomial to pixel positions in each fake lane line
    # Fit new polynomials to x,y in world space
    left_fit_cr = np.polyfit(ploty * ym_per_pix, leftx * xm_per_pix, 2)
    right_fit_cr = np.polyfit(ploty * ym_per_pix, rightx * xm_per_pix, 2)

    return ploty, left_fit_cr, right_fit_cr

def measure_curvature_pixels(ploty, left_fit, right_fit):
    '''
    Calculates the curvature of polynomial functions in pixels.
    '''
    # Start by generating our fake example data
    # Make sure to feed in your real data instead in your project!
    #ploty, left_fit, right_fit = generate_data()

    # Define y-value where we want radius of curvature
    # We'll choose the maximum y-value, corresponding to the bottom of the image
    y_eval = np.max(ploty)

    # Calculation of R_curve (radius of curvature)
    left_curverad = ((1 + (2 * left_fit[0] * y_eval + left_fit[1]) ** 2) ** 1.5) / np.absolute(2 * left_fit[0])
    right_curverad = ((1 + (2 * right_fit[0] * y_eval + right_fit[1]) ** 2) ** 1.5) / np.absolute(2 * right_fit[0])

    return left_curverad, right_curverad

def measure_curvature_real():
    '''
    Calculates the curvature of polynomial functions in meters.
    '''
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30 / 720  # meters per pixel in y dimension
    xm_per_pix = 3.7 / 650  # meters per pixel in x dimension

    # Start by generating our fake example data
    # Make sure to feed in your real data instead in your project!
    ploty, left_fit_cr, right_fit_cr = generate_data(ym_per_pix, xm_per_pix)

    # Define y-value where we want radius of curvature
    # We'll choose the maximum y-value, corresponding to the bottom of the image
    y_eval = np.max(ploty)

    ##### TO-DO: Implement the calculation of R_curve (radius of curvature) #####
    left_curverad = ((1 + (2 * left_fit_cr[0] * y_eval * ym_per_pix + left_fit_cr[1]) ** 2) ** 1.5) / np.absolute(
        2 * left_fit_cr[0])  ## Implement the calculation of the left line here
    right_curverad = ((1 + (2 * right_fit_cr[0] * y_eval * ym_per_pix + right_fit_cr[1]) ** 2) ** 1.5) / np.absolute(
        2 * right_fit_cr[0])  ## Implement the calculation of the right line here

    return left_curverad, right_curverad

def region_of_interest(img, vertices):
    """
    Applies an image mask.

    Only keeps the region of the image defined by the polygon
    formed from `vertices`. The rest of the image is set to black.
    `vertices` should be a numpy array of integer points.
    """
    # defining a blank mask to start with
    mask = np.zeros_like(img)

    # defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(img.shape) > 2:
        channel_count = img.shape[2]  # i.e. 3 or 4 depending on your image
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255

    # filling pixels inside the polygon defined by "vertices" with the fill color
    cv2.fillPoly(mask, vertices, ignore_mask_color)

    # returning the image only where mask pixels are nonzero
    masked_image = cv2.bitwise_and(img, mask)

    return masked_image

def draw_track(undist, warped, Minv, left_fitx, right_fitx, ploty):
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    pts = np.hstack((pts_left, pts_right))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 0))

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undist.shape[1], undist.shape[0]))
    # Combine the result with the original image
    result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)

    plt.imshow(result)

def draw_track_with_lines(undist, warped, Minv, left_fitx, right_fitx, ploty, margin = 25):

    # Create an image to draw the lines on
    warp_zero = np.zeros_like(warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    # Generate a polygon to illustrate the search window area
    # And recast the x and y points into usable format for cv2.fillPoly()
    left_line_window1 = np.array([np.transpose(np.vstack([left_fitx - margin, ploty]))])
    left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx + margin, ploty])))])
    left_line_pts = np.hstack((left_line_window1, left_line_window2))

    right_line_window1 = np.array([np.transpose(np.vstack([right_fitx - margin, ploty]))])
    right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx + margin, ploty])))])
    right_line_pts = np.hstack((right_line_window1, right_line_window2))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([left_line_pts]), (0, 255, 0))
    cv2.fillPoly(color_warp, np.int_([right_line_pts]), (0, 255, 0))
    # result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)

    # bwn_window_img = np.zeros_like(out_img)
    left_lane_line_window = np.array([np.flipud(np.transpose(np.vstack([left_fitx + margin, ploty])))])
    right_lane_line_window = np.array([np.transpose(np.vstack([right_fitx - margin, ploty]))])
    lane_line_pts = np.hstack((left_lane_line_window, right_lane_line_window))

    cv2.fillPoly(color_warp, np.int_([lane_line_pts]), (255, 0, 0))
    #result = cv2.addWeighted(result, 1, bwn_window_img, 0.3, 0)

    # # Recast the x and y points into usable format for cv2.fillPoly()
    # pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    # pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    # pts = np.hstack((pts_left, pts_right))
    #
    # # Draw the lane onto the warped blank image
    # cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 0))

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undist.shape[1], undist.shape[0]))
    # Combine the result with the original image
    result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)

    plt.imshow(result)


import sys
import random
import math
import itertools
from fractions import Fraction

from scipy.spatial import Delaunay

sys.path.append("../../../")
from dan.lib.helper import *
from solid import *
from solid.utils import *

class LayerStructure:
    def get_layers(self, shrink_offset=0):
        layers = []
        for h in range(self.height):
            layer = []
            for a in range(self.sections):
                angle, dist = self.calc(h, a)
                dist -= shrink_offset
                layer.append(
                    Vec3(math.cos(angle) * dist, math.sin(angle) * dist, h * 300)
                )
            layers.append(layer)
        return layers

class RandomMeshStructure:
    def torus_dist2(self, p1, p2, w, h):
        return (
            min(abs(p1.x - p2.x), w - abs(p1.x - p2.x)) ** 2
            + min(abs(p1.y - p2.y), h - abs(p1.y - p2.y)) ** 2
        )

    def get_object(self, shrink_offset=0, seed=None):
        if not seed:
            seed = self.seed
        if not seed:
            seed = 0
        random.seed(seed)

        width = self.avg_rad * 2 * math.pi
        points = []

        # Put points along top and bottom
        edge_count = int(math.floor(width / self.min_point_dist))
        edge_width = width / edge_count
        for i in range(edge_count):
            points.append(Vec3(edge_width * i, 0))
            points.append(Vec3(edge_width * i, self.height))

        edge_points = [i for i in range(len(points))]

        # Fill the area with points
        while True:
            found = False
            for i in xrange(1000):
                point = Vec3(random.uniform(0, width), random.uniform(0, self.height))
                collides = False
                for p in points:
                    if (
                        self.torus_dist2(point, p, width, self.height)
                        < self.min_point_dist ** 2
                    ):
                        collides = True
                        break

                if not collides:
                    points.append(point)
                    found = True
                    break

            if not found:
                break

        # Bin the points into veritcal columns
        dsects_count = 4
        dsects_inc = width / dsects_count
        dsects = [[] for i in range(dsects_count)]
        for p in points:
            slot = int(p.x / dsects_inc)
            dsects[slot].append(p)

        # Create the Delaunay areas
        dareas = []
        ref_lists = [{} for i in range(dsects_count)]
        for i in range(dsects_count):
            if i - 1 < 0:
                bindex = i - 1 + dsects_count
                before = dsects[bindex]
                before = [Vec3(p.x - width, p.y) for p in before]
            else:
                bindex = i - 1
                before = dsects[i - 1]

            if i + 1 >= dsects_count:
                aindex = i + 1 - dsects_count
                after = dsects[aindex]
                after = [Vec3(p.x + width, p.y) for p in after]
            else:
                aindex = i + 1
                after = dsects[i + 1]

            for j in range(len(before)):
                ref_lists[i][j] = (bindex, j, True)

            for j in range(len(dsects[i])):
                ref_lists[i][j + len(before)] = (i, j, False)

            for j in range(len(after)):
                ref_lists[i][j + len(dsects[i]) + len(before)] = (aindex, j, True)

            dareas.append(before + dsects[i] + after)

        # Calculate Tris
        tris = []
        for i in range(dsects_count):
            tris.append(
                Delaunay([p.to_list()[:2] for p in dareas[i]], qhull_options="QJ")
            )

        def find_index(i, p):
            if p in ref_lists[i]:
                l, li, fake = ref_lists[i][p]
                point = dsects[l][li]

            for cpi in range(len(points)):
                if point is points[cpi]:
                    return cpi, fake

        # Make list of real tris
        real_tris = []
        for i in range(dsects_count):
            for t in tris[i].simplices:
                a, af = find_index(i, t[0])
                b, bf = find_index(i, t[1])
                c, cf = find_index(i, t[2])
                if af and bf and cf:
                    continue
                if a in edge_points and b in edge_points and c in edge_points:
                    continue
                real_tris.append([c, b, a])

        # Remove duplicates
        real_tris.sort()
        real_tris = list(k for k, _ in itertools.groupby(real_tris))
        real_tris = filter(lambda x: None not in x, real_tris)

        def realp(point):
            dist = self.calc(point.x / width, point.y / self.height) - shrink_offset
            a = point.x / width * math.pi * 2
            return Vec3(math.cos(a) * dist, math.sin(a) * dist, point.y)

        real_points = [realp(p) for p in points]

        real_points.append(Vec3(0, 0, 0))
        real_points.append(Vec3(0, 0, self.height))

        for i in range(edge_count):
            real_tris.append(
                [len(real_points) - 2, i * 2, (i * 2 + 2) % (edge_count * 2)]
            )

            real_tris.append(
                [(i * 2 + 3) % (edge_count * 2), i * 2 + 1, len(real_points) - 1]
            )

        return polyhedron(points=[p.to_list() for p in real_points], faces=real_tris)

class WallPiece:
    def get_object(self):
        points = list(self.points)  # Shallow Copy
        tris = Delaunay([p.to_list()[:2] for p in points], qhull_options="Qt")
        tris = [[t[0], t[1], t[2]] for t in tris.simplices]
        tris = [list(reversed(l)) for l in tris]

        edge_count = {}
        for tri in tris:
            for i in range(3):
                a = tri[i]
                b = tri[(i + 1) % 3]
                key = (a, b)
                edge = edge_count.get(key)
                if edge == None:
                    key = (b, a)
                    edge = edge_count.get(key)

                if edge == None:
                    edge_count[key] = 1
                else:
                    edge_count[key] = edge_count[key] + 1

        edges = []
        for key, value in edge_count.iteritems():
            if value == 1:
                edges.append(key[0])
                edges.append(key[1])

        edges = list(set(edges))
        avg_point = reduce(lambda x, i: x + points[i], edges, Vec3())
        avg_point /= len(edges)
        avg_point.z = 0

        def get_angle(a):
            x = a.x - avg_point.x
            y = a.y - avg_point.y
            return atan2(y, x)

        def sort_fun(a, b):
            return 1 if get_angle(points[a]) < get_angle(points[b]) else -1

        edges.sort(sort_fun)

        new_points = []
        for p in edges:
            np = points[p].copy()
            np.z = 0
            new_points.append(np)

        for i in range(len(new_points)):
            t1 = [edges[i], len(points) + i, len(points) + ((i + 1) % len(new_points))]
            t2 = [
                edges[i],
                len(points) + ((i + 1) % len(new_points)),
                edges[(i + 1) % len(new_points)],
            ]
            tris.append(t1)
            tris.append(t2)

        points += new_points

        points.append(avg_point)
        for i in range(len(new_points)):
            tris.append(
                list(
                    reversed(
                        [
                            len(points) - 1,
                            len(points) - 1 - len(new_points) + i,
                            len(points)
                            - 1
                            - len(new_points)
                            + ((i + 1) % len(new_points)),
                        ]
                    )
                )
            )

        # return union() ( [translate( points[v].to_list() ) ( linear_extrude( 10 ) ( text( str(i), size=100, valign="center", halign="center" ) ) ) for i, v in enumerate( edges ) ])
        return polyhedron(points=[p.to_list() for p in points], faces=tris)

    def get_list(self):
        sorted_points = sorted(self.points, lambda a, b: cmp(a.y, b.y))
        sorted_points = filter(lambda x: x.z != 0, sorted_points)
        out = ""
        in_in_mm = 0.0393701
        win = self.width * in_in_mm
        hin = self.height * in_in_mm
        for i in xrange(len(sorted_points)):
            p = sorted_points[i].copy()
            height_in_mm = p.z
            p *= in_in_mm

            def convert_to_fraction(v):
                main = int(v * 16) / 16
                fraction = Fraction(int(v * 16) / 16.0 - main)
                return str(main) + " " + str(fraction)

            top_left = ", ".join([convert_to_fraction(v) for v in p.to_list()[:-1]])
            bottom_right = Vec3(win - p.x, hin - p.y, p.z)
            bottom_right = ", ".join(
                [convert_to_fraction(v) for v in bottom_right.to_list()[:-1]]
            )

            line = [i] + [top_left, bottom_right, convert_to_fraction(p.z)]
            line = [str(e) for e in line]
            out += " || ".join(line) + "\n"
        return out


"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
Harvard CS124 - Spring 2018
Philippe Noël - Pset 3 - Crypto
Python version of Crypto
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
from collections import defaultdict, deque
# global dictionnaries to contain all of our sets
# parent is like the parents set, in some sense
# it contains the parent of each node and that's it
parent = dict()
rank = dict()
# this array will contain the bandwiths for each query from 1 to t
bandwiths = []
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""" Subroutine code for merge subroutine of MergeSort algorithm """
def merge(lst1, lst2):
    # save lengths of lists for runtime
    lst1_len = len(lst1)
    lst2_len = len(lst2)
    # initialize output list and iteration indices
    oupt_lst = []
    index1, index2 = 0, 0
    # merging loop for non-empty lists
    while index1 != lst1_len and index2 != lst2_len:
        if lst1[index1] > lst2[index2]:
            oupt_lst.append(lst1[index1])
            index1 += 1
        else:
            oupt_lst.append(lst2[index2])
            index2 += 1
    # subcase if one list is now empty
    if index1 == lst1_len:
        oupt_lst += lst2[index2:]
    else:
        oupt_lst += lst1[index1:]
    return oupt_lst
""" Subroutine code for MergeSort algorithm """
def MergeSort(lst):
    # save length of list for runtime
    lst_len = len(lst)
    # base case
    if lst_len == 0 or lst_len == 1:
        return lst
    else:
        # recursive call
        mid = int(lst_len / 2)
        first_half = MergeSort(lst[:mid])
        second_half = MergeSort(lst[mid:])
        return merge(first_half, second_half)
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""" Subroutine code for Kruskal's makeset """
def makeset(vertex):
    # add our new vertex as a singleton value inside our set
    # and set its rank to 0
    parent[vertex] = vertex
    rank[vertex] = 0
""" Subroutine code for Kruskal's set find """
def find(vertex):
    # if our vertex is not the parent, recurse down the subtree
    if parent[vertex] != vertex:
        parent[vertex] = find(parent[vertex])
    # return p(x)
    return parent[vertex]
""" Subroutine code for Kruskal's set union """
def union(vertex1, vertex2, servers, users):
    # vertex1 and vertex2 are required to be roots
    root1 = find(vertex1)
    root2 = find(vertex2)
    # if rank of root1 > rank of root 2, we swap the two roots
    # since root2 has higher rank, so we should attach the smallest
    # tree to it
    if rank[root1] > rank[root2]:
        # switch the roots
        tmp = root2
        root2 = root1
        root1 = tmp
        # switch the users count
        tmp1 = users[root2]
        users[root2] = users[root1]
        users[root1] = tmp1
        # switch the servers count
        tmp2 = servers[root2]
        servers[root2] = servers[root1]
        servers[root1] = tmp2
    # if the rank are the same, make them as one root and increment rank
    if rank[root1] == rank[root2]:
        rank[root2] += 1
    # update the tree
    parent[root1] = root2
    # update users
    users[root2] += users[root1]
    # update servers
    servers[root2] += servers[root1]
    # return the root of the new connected component
    return root2
""" Subroutine code for Kruskal's algorithm """
def Kruskal(graph, weights, servers, users):
    # making all the sets is done when reading input
    # each weight in the weights list represent one edge
    # they are already sorted in decreasing order
    for w in weights:
        # get the current edge we are looking at
        edge = graph[w].popleft()
        # if not single node, add edge to our tree and merge the sets
        if find(edge[0]) != find(edge[1]):
            # pass along servers and users as when we will merge elements,
            # we check if they can provide an allocation and if so allocate
            # pass in weight since it will be the bandwith
            connected_comp_root = union(edge[0], edge[1], servers, users)
            # get number of servers and users in this new connected comp
            num_servers = servers[connected_comp_root]
            num_users = users[connected_comp_root]
            # iterate over all the users in the component
            for user in range(num_users):
                # if there is still space in servers
                if num_servers > 0:
                    # append a new bottleneck bandwith
                    bandwiths.append(w)
                    # decrease capacity available by 1
                    num_servers -= 1
                    # remove the user just assigned
                    users[connected_comp_root] -= 1
            # once done looping over the users, set the number of servers
            # in the component to the remaining number of servers for when
            # we will merge the components
            servers[connected_comp_root] = num_servers
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
""" Subroutine code for Crypto algorithm """
def crypto():
    # global var for more easily keeping track of bandwith cuz lazy
    global bandwiths
    # read n (vertices) and m (edges)
    n, m = map(int, input().split())
    # weights list -- will contain duplicate, should be ok
    weights = []
    # initialize the graph as an adjacency list, though about making adjacency
    # matrix, but it will be very inefficient since most of it would be empty
    # weights as keys and edges list as value
    graph = defaultdict(deque)
    # get all the edges and their weights
    for edge in range(m):
        # get the edge and its weight
        u, v, w = map(int, input().split())
        # add weight to our weight array that will be sorted
        weights.append(w)
        # fill in the vertices V_sets
        # this is the MAKESET operation from Kruskal
        # we do it here to save a linear loop later
        makeset(u)
        makeset(v)
        # add edge and its weight to our graph
        graph[w].append((u, v))
    # get k (servers) and t (users)
    k, t = map(int, input().split()) # these are kinda useless since Python
    # get the arrays that contain the servers locations, their capacities,
    # and the users locations
    servs_locations = list(map(int, input().split()))
    servs_capacities = list(map(int, input().split()))
    users_locations = list(map(int, input().split()))
    # generate a dictionary that contains the number of spots available in
    # each connected component (key is the root of the component)
    servers = dict()
    for index in range(k):
        servers[servs_locations[index]] = servs_capacities[index]
    # generate a dictionary that contains the number of users in each
    # connected component (key is the root of the component)
    users = dict()
    for index in range(t):
        users[users_locations[index]] = 1
    # fill the rest of both dictionnaries with 0 values since no user nor serv
    for index in range(n):
        if servers.get(index + 1) == None:
            servers[index + 1] = 0
        if users.get(index + 1) == None:
            users[index + 1] = 0
    # get Q (number of queries) and the Q queries
    Q = int(input()) # useless value since we're using Python
    queries = list(map(int, input().split()))
    # sort the weights in decreasing order (we are building a Max Span Tree)
    # this is an inverted MergeSort algorithm
    sorted_weights = MergeSort(weights)
    # Modified Kruskal algorithm that builds a maximum spanning tree while
    # assigning users to servers and storing it in the bandwiths array
    Kruskal(graph, sorted_weights, servers, users)
    # iterate over all the queries and take the bandwith computed
    # previously and return it. We take - 1 since array is indexed 0
    for query in queries:
        print(bandwiths[query - 1])

""" Function call for testing """
crypto()


"""
 Blogger services
 This service relies on Google GData

 @author Jean-Lou Dupont
"""
import os
import re

from google.appengine.api import urlfetch

from gdata import service
import gdata.urlfetch

# Use urlfetch instead of httplib
gdata.service.http_request_handler = gdata.urlfetch

from libs.cache import *

class WBBlogger( object ):
    """
    Blogger
    """

    def __init__(self):
        self.p = re.compile("(.+)post-(.*)$")

    def getPost(self, blog_id, post_id ):

        feed = self.getPosts(blog_id)
        if ( feed == 0 ):
            return

        for entry in feed.entry:
            id = entry.id.text
            title = entry.title.text
            content = entry.content.text

            if ( self._findPost(entry, post_id, id) ):
                return [title, content]

        return [None, None]

    def getPosts(self, blogid):
        """
        Returns a feed with all the posts associated with blog identified with 'id'
        """
        return self._getBloggerPosts(blogid)

    # since the response is cached, the method name
    # is used to build the cache key
    @cacheResponse(300)
    def _getBloggerPosts(self, blogid):
        blogger_service = service.GDataService( )
        blogger_service.source  = 'web-bloks/blogger'
        blogger_service.service = 'blogger'
        blogger_service.server  = 'www.blogger.com'

        try:    feed = blogger_service.GetFeed('/feeds/' + blogid + '/posts/default')
        except: feed = 0;

        return feed

    def _findPost(self, entry, post_id, search ):
        m = self.p.match( search )

        if (m is None):
            return False

        rid = m.group(2)
        return ( post_id == rid )


#!/usr/local/bin/python3

"""
Python command line chunks data parser into JSON format.
Example usage:
$ python3 convert_to_json.py donorids.txt combined.chunkcounts.out combined.chunklengths.out my_chunk_file
"""

import os.path
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import numpy as np
import json
__author__ = 'Marina Vabistsevits'

def parse_args():
    """Parse command line arguments"""
    parser = ArgumentParser(description=__doc__, formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('donor_ids_file', help="Column file with samples IDs (SID) and population IDs (PID)")
    parser.add_argument('chunkcounts', help="Text file of chunkcounts per sample")
    parser.add_argument('chunklengths', help="Text file of chunklengths per sample")
    parser.add_argument('out_file_prefix', nargs='?', default='chunksdata',
                        help="Output JSON file prefix (without .json)")
    parser.add_argument('output_directory', nargs='?', default='./', help="Directory to write to")
    args = parser.parse_args()
    return args

def main():
    args = parse_args()

    # check inputs
    ids_file = args.donor_ids_file
    if not os.path.isfile(ids_file):
        print("Either file is missing or is not readable:", ids_file)
    counts_file = args.chunkcounts
    if not os.path.isfile(counts_file):
        print("Either file is missing or is not readable:", counts_file)
    lenghts_file = args.chunklengths
    if not os.path.isfile(lenghts_file):
        print("Either file is missing or is not readable:", lenghts_file)
    outdir = args.output_directory
    outfile = args.out_file_prefix

    # creating PID : SIDs  dictionary
    ids_array = np.genfromtxt(ids_file, dtype='str')
    pop_dict = {}
    for sample in ids_array:
        sid = str(sample[0])
        pid = str(sample[1])
        if pid in pop_dict:  # fill in the dictionary FID : SIDs
            pop_dict[pid].append(sid)
        else:
            pop_dict[pid] = [sid]

    # create SID : count and SID : length dictionaries
    counts_dict = chunkfile_to_dict(counts_file)
    lenghts_dict = chunkfile_to_dict(lenghts_file)

    # check that the number of samples (donors (SID) is the same in all 3 supplied files)
    if len(lenghts_dict.keys()) != len(counts_dict.keys()):
        print("The number of donor samples is different between chunk files!")
    elif len(lenghts_dict.keys()) != ids_array.shape[0]:
        print("The number of donor samples in ids file does not match lenghts file!")
    elif len(counts_dict.keys()) != ids_array.shape[0]:
        print("The number of donor samples in ids file does not match counts file!")

    # create per PID counts/lenghts dictionaries
    counts_dict_values = make_population_chunkdata_dict(pop_dict, counts_dict)
    lenghts_dict_values = make_population_chunkdata_dict(pop_dict, lenghts_dict)

    # creating nested dict and saving json
    chunks_dict_output = {"chunk_counts": counts_dict_values, "chunk_lengths": lenghts_dict_values}

    with open(str(outdir + outfile + '.json'), 'w') as f:
        json.dump(chunks_dict_output, f, ensure_ascii=False)

# FUNCTIONS

def make_population_chunkdata_dict(popdict, chunkdict):
    """Take population dict (PID: [list of SIDs] and chunkdict (SID: count/length)
    and output a new dictionary - PID: [list of counts/lenghts for that PID's samples) """

    population_chunkdict = {}  # dictionary to keep chunk data per population (PIDs as keys)

    for pid in popdict.keys():  # take a population
        for sid in popdict[pid]:  # iterate over all samples in it

            if sid in chunkdict:  # take that sample in the current chunkdict
                if pid in population_chunkdict:  # check if it's already in the dict
                    population_chunkdict[pid].append(float(chunkdict[sid]))  # update this PID's value list
                else:
                    population_chunkdict[pid] = [float(chunkdict[sid])]  # create new key
    return population_chunkdict

def chunkfile_to_dict(chunkfile):
    """Read in a chunkfile (counts or lengths), ignore #header (if present),
       transpose to be 2-column array (columns: recipient, sample),
        create a dictionary with key:value as SID: count/length """

    with open(chunkfile) as f:
        lines = (line for line in f if not line.startswith('#'))
        chunk_array = np.loadtxt(lines, delimiter=' ', dtype=object)
        chunk_array = chunk_array[:, 1:]  # remove 'Sample1' and 'Recipient' column
        chunk_array_t = np.transpose(chunk_array)

    chunk_dict = dict(
        zip(chunk_array_t[:, 0], chunk_array_t[:, 1]))  # key is column 0 (SID), value is column 1 (count/length)
    return chunk_dict

if __name__ == '__main__':
    main()


import feedparser

from .resource import ResourceCrawler
from ..utils import normalize_language

class FeedCrawler(ResourceCrawler):
    RESOURCE_TYPE = 'feed'
    SOURCE_EXCHANGE = 'feeds'
    SOURCE_KEY = 'crawl_feed'
    RESULT_EXCHANGE = 'articles'

    async def crawl(self, feed, contents, headers=None, result_producer=None):
        self.log.info(f'crawling {feed["type"]} feed {feed["url"]}')
        try:
            parsed = feedparser.parse(contents)
        except Exception as exc:
            self.log.warning(
                f'error parsing feed {feed["url"]}; sending '
                f'nack: {exc}')
            raise

        if not parsed or not parsed.get('feed'):
            raise ValueError(f'empty feed: {feed["url"]}')

        if result_producer is None:
            # Shouldn't happen
            return

        # Initial best guess at the post language.  An unfortunate
        # misfeature of RSS is that language is a feed-global
        # attribute; it does not support feeds with multiple language
        # entries.  Atom does support this in principle though not sure
        # if it's actually used.  In practice most feeds are
        # monolingual.
        lang = normalize_language(
                parsed['feed'], default=feed.get('lang', 'en'))

        for entry in parsed.get('entries', []):
            link = entry.get('link')
            if not link:
                continue

            await result_producer.proxy.save_article(
                    article={'url': link, 'lang': lang})

        return None

if __name__ == '__main__':
    FeedCrawler.main()


import glob
import yaml
from bunch import Bunch, bunchify

yamlConfigs = None

def get_yaml_configs():
    global yamlConfigs
    if yamlConfigs:
        return yamlConfigs

    yamlConfigs = Bunch()

    for yaml_file in ['reference', 'custom']:
        yamlConfigs[yaml_file] = bunchify(yaml.load(file('static/{}/{}.yml'.format(yaml_file, yaml_file), 'r')))

    return yamlConfigs