#!/usr/bin/env python3
import json
import requests
# include standard modules
import argparse

class Restful:
	url = 'https://jsonplaceholder.typicode.com'

	def run(self, args):
		if args.METHOD == 'get':
			Restful.get(args.ENDPOINT)
		if args.output and args.METHOD == 'get':
			Restful.getOutput(args.ENDPOINT, args.output)
		if args.METHOD == 'post':
			Restful.post(args.ENDPOINT, args.data)
		if args.output and args.METHOD == 'post':
			Restful.postOutput(args.ENDPOINT, args.data, args.output)

	def get(self, endpoint):
		response = requests.get(self.url + endpoint)
		self.handleError(response.status_code)
		print(response.text)

	def getOutput(self, endpoint, file):
		response = requests.get(self.url + endpoint)
		self.handleError(response.status_code)
		self.output(file,response.text)

	def post(self, endpoint, data):
		response = requests.post(self.url + endpoint, data)
		self.handleError(response.status_code)
		print(response.text)

	def postOutput(self, endpoint, data, file):
		response = requests.post(self.url + endpoint, data)
		self.handleError(response.status_code)
		self.output(file,response.text)

	def output(self, file, data):
		with open(file, 'w') as outfile:
			json.dump(data, outfile)

	def handleError(self, status_code):
		if status_code >= 200 and status_code <= 299:
			print(status_code)
		else:
			print(str(status_code) + '- FAILURE')

if __name__ == '__main__':
	# initiate the parser
	parser = argparse.ArgumentParser()
	parser.add_argument('METHOD', metavar='{get, post}', help='Request method')
	parser.add_argument('ENDPOINT', metavar='endpoint', help='Request endpoint URI fragment')
	parser.add_argument('-d', '--data', help="Data to send with request")
	parser.add_argument("-o", "--output", help="Output to .json or .csv file (default: dump to stdout)")

	# read arguments from the command line
	args = parser.parse_args()
	Restful = Restful()
	Restful.run(args)


import numpy as np
from numpy import asarray
from PIL import Image
import zipfile
import os

def segmentation(board):

    BORDER_SIZE = 3
    card_shape = (150,150)

    board = np.array(board)

    cards = []

    """border_matrix = np.empty([card_shape[0] + BORDER_SIZE , card_shape[1] + BORDER_SIZE])

    for x in range(card_shape[0] + BORDER_SIZE):
        for y in range(card_shape[1] + BORDER_SIZE):
            if min(x,y) < BORDER_SIZE or min(card_shape[0] - x -1,card_shape[1] - y -1) < BORDER_SIZE:
                border_matrix[x,y] = 1
            else:
                border_matrix[x,y] = 0

    number_border_pixel = border_matrix.sum()
    """

    vector = np.array([0 for i in range(card_shape[0] + 2*BORDER_SIZE)], dtype=float)
    vector[0:BORDER_SIZE] = 1/(2*BORDER_SIZE)
    vector[-BORDER_SIZE:] = 1/(2*BORDER_SIZE)

    img_board = Image.fromarray(np.array(board))
    grayscale_board = np.array(img_board.convert('L'))

    """print(grayscale_board[30,30])
    img = Image.fromarray(grayscale_board)
    img.show()"""
    means = []
    for column in range(0,board.shape[0] - card_shape[0] - 2*BORDER_SIZE):
        for row in range(0,board.shape[1] - card_shape[1] - 2*BORDER_SIZE ):
            subArray = grayscale_board[column:column+card_shape[0]+ 2*BORDER_SIZE,row:row+card_shape[1]+ 2*BORDER_SIZE]
            mean = np.average((np.dot(vector,subArray)+np.dot(subArray,vector))/2)
            means.append(mean)
            if mean <=20:
                cards.append(board[column + BORDER_SIZE:column + BORDER_SIZE + card_shape[0],
                                   row + BORDER_SIZE:row + BORDER_SIZE + card_shape[1]])
    return cards

def applyThreshold(image1,threshold):
    formatted_card = np.copy(image1)
    for x,column in enumerate(image1):
        for y,pixel in enumerate(column):
            formatted_card[x,y] = 0 if pixel < threshold else 255
    return formatted_card

def imageComparison(image1, image2):

    img1 = Image.fromarray(np.array(image1))
    grayscale_image1 = np.array(img1.convert('L'))
    #grayscale_image1 = applyThreshold(grayscale_image1,128)

    img2 = Image.fromarray(np.array(image2))
    grayscale_image2 = np.array(img2.convert('L'))
    #grayscale_image2 = applyThreshold(grayscale_image2,128)

    average_distance = np.mean(np.square(np.absolute(grayscale_image1 - grayscale_image2)))

    return average_distance < 20

def detection(board):
    background = Image.open("background.png")
    arrayBackground = asarray(background)

    cards = segmentation(board)

    return [card for card in cards if not imageComparison(background,card)]

"""
image = Image.open('board.jpg')
faceup = detection(image)

print(str(len(faceup)) + " cards face up")
for card in faceup:
    img = Image.fromarray(np.array(card))
    img.show()
    s
def zipped(image_array):
    zipf = zipfile.ZipFile('Name.zip','w', zipfile.ZIP_DEFLATED)
    for image in image_array:
        zipf.write(image.tobytes())
    zipf.close()
    return zipf

"""


import os
import hashlib
import json
from urllib.request import urlopen

################################################################################
# get_api_json
#
# Call a JSON api and cache the results in `api_cache/`. Future calls to the
# same API will be read from the cache. To expire the cache, delete the file.
################################################################################
def get_api_json(url: str):
    if not os.path.exists("api_cache"):
        os.makedirs("api_cache")

    url_hash = hashlib.new('sha256')
    url_hash.update(url.encode())
    url_cache = os.path.join("api_cache", url_hash.hexdigest() + ".json")

    if os.path.exists(url_cache):
        with open(url_cache, "r") as f:
            return json.load(f)

    try:
        response = urlopen(url).read().decode("utf-8")
    except Exception as e:
        print(url)
        raise e

    with open(url_cache, "w") as f:
        f.write(response)
    return json.loads(response)


import codecademylib
import pandas as pd
from matplotlib import pyplot as plt

species = pd.read_csv('species_info.csv')

species.fillna('No Intervention', inplace = True)

protection_counts = species.groupby('conservation_status')\
    .scientific_name.nunique().reset_index()\
    .sort_values(by = 'scientific_name')

#protection_counts = pd.DataFrame(protection_counts.\
#                                 to_records())
#protection_counts.drop(['index'], axis=1, inplace = True)
print(protection_counts)

###
# Graphing
###
x_val = range(len(protection_counts['conservation_status']))

plt.figure(figsize = [10,4])
ax = plt.subplot(1,1,1)
ax.bar(x_val, protection_counts['scientific_name'], align = 'center')
ax.set_xlabel('Conservation Status')
ax.set_ylabel('Number of Species')
ax.set_title('Conservation Status by Species')
ax.set_xticks(x_val)
ax.set_xticklabels(protection_counts['conservation_status'])
plt.show()


import json

from django.shortcuts import render, redirect
from django.http import HttpResponse
from nms.models import model_encode
from nms.models import model_encode_verbose

###
### Request Handling
###

def get_request_var(request, name):
    var = None
    try:
        var = request.POST[name]
    except:
        try:
            var = request.GET[name]
        except:
            pass

    if var == '':
        var = None

    return var

###
### Responses
###

def error_dict(http_status_code=400):
    error_dict = _response_dict()
    error_dict['error_code'] = 1
    error_dict['error_msg'] = 'Unknown error'
    error_dict['http_status_code'] = http_status_code
    return error_dict

def success_dict(http_status_code=200):
    success_dict = _response_dict()
    success_dict['error_code'] = 0
    success_dict['error_msg'] = ''
    success_dict['http_status_code'] = http_status_code
    return success_dict

def render_template(request, template, template_values=None, verbose=False, minify=True):
    http_status = 200
    if template_values:
        try:
            http_status = template_values['http_status_code']
        except:
            pass
    return render(request, template, template_values, status=http_status)

def render_json(request, json_values, verbose=False, minify=True):
    http_status = json_values['http_status_code']
    encode = model_encode
    if verbose:
        encode = model_encode_verbose

    response_data = json.dumps(json_values, default=encode)
    return HttpResponse(response_data, content_type="application/json", status=http_status)

def render_400(request, template_values=None):
    return render(request, '400.html', template_values, status=400)

def render_500(request, template_values=None):
    return render(request, '500.html', template_values, status=500)

def response_json_500(request, json_values=None):
    json_values = json_values or error_dict(http_status_code=500)
    json_values['error_code'] = 500
    return render_json(request, json_values=json_values)

def response_json_503(request, json_values=None):
    json_values = json_values or error_dict(http_status_code=503)
    json_values['error_code'] = 503
    json_values['error_msg'] = 'Service not available'
    return render_json(request, json_values=json_values)

def redirect_to_url(url):
    return redirect(url)

###
### PRIVATE
###

def _response_dict():
    response_dict = {}
    response_dict['error_code'] = 0
    response_dict['error_msg'] = ''
    response_dict['http_status_code'] = 200
    return response_dict


import pytest
import importlib
import sys
import random
import numpy as np
import torch
from kevin_toolbox.patches.for_test import check_consistency
from kevin_toolbox.patches.for_torch.compatible import linalg
from kevin_toolbox.env_info import version

"""
建议在最新版本的 pytorch 以及 gpu 可用的环境下运行本测试脚本
    本模块 for_torch.compatible 对低版本的兼容，本质上是用低版本中的函数去实现高版本的功能
    因此只有在高版本的 pytorch 下才能同时运行低版本的实现，并与高版本的运行结果进行比对
"""

def test_norm():
    print("test compatible.linalg.norm()")

    ver_bak = torch.__version__

    for _ in range(100):
        # 随机构建输入的
        shape = [np.random.randint(1, 10) for _ in range(np.random.randint(1, 4))]
        x = torch.rand(shape)
        if np.random.randint(2) % 2 == 1 and len(shape) > 1:
            # If :attr:`dim` is a `2`-`tuple`, the matrix norm will be computed.
            dim = sorted(np.random.choice(range(len(shape)), size=2, replace=False).tolist())
            ord = random.choice(["fro", 1, 2])
        else:
            # If :attr:`dim` is an `int`, the vector norm will be computed
            dim = np.random.randint(len(shape))
            ord = random.choice([1, 2, np.inf])
        if torch.cuda.is_available() and np.random.randint(2) % 2 == 1:
            x = x.to(device=torch.device("cuda"))
        keepdim = np.random.randint(2) % 2 == 1

        #
        res = []
        for ver in ["1.0", "1.12"]:
            ver = ver_bak if version.compare(ver, ">", ver_bak, mode="short") else ver
            torch.__version__ = ver
            importlib.reload(sys.modules["kevin_toolbox.patches.for_torch.compatible.linalg.norm"])
            res.append(linalg.norm(x, ord=ord, dim=dim, keepdim=keepdim))

        # 检验
        check_consistency(*res, tolerance=1e-10)

    torch.__version__ = ver_bak

def test_tile():
    print("test compatible.linalg.svd()")

    ver_bak = torch.__version__

    for _ in range(100):
        # 随机构建输入的
        shape = [np.random.randint(1, 10) for _ in range(np.random.randint(2, 4))]
        x = torch.rand(shape)
        if torch.cuda.is_available() and np.random.randint(2) % 2 == 1:
            x = x.to(device=torch.device("cuda"))
        full_matrices = np.random.randint(2) % 2 == 1

        #
        res = []
        for ver in ["1.0", "1.12"]:
            ver = ver_bak if version.compare(ver, ">", ver_bak, mode="short") else ver
            torch.__version__ = ver
            importlib.reload(sys.modules["kevin_toolbox.patches.for_torch.compatible.linalg.svd"])
            res.append(linalg.svd(x, full_matrices=full_matrices))

        # 检验
        for temp in zip(*res):
            check_consistency(*temp, tolerance=1e-10)

    torch.__version__ = ver_bak


import os
from colorama import Fore,Style
import requests
import time
import urllib3
import sys

def parse_args():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--domain', type=str, required=True, help='Target Domain.')
    parser.add_argument('-o', '--output', type=str, required=False, help='Output to file.')
    return parser.parse_args()

def banner():
    global version
    print(' ')

    time.sleep(1)

def parse_url(url):
    try:
        host = urllib3.util.url.parse_url(url).host
    except Exception as e:
        print(f'{Fore.RED}[*] Invalid domain, try again..')
    return host

def write_subs_to_file(subdomain, output_file):
    with open(output_file, 'a') as fp:
        fp.write(subdomain + '\n')

def main():
    banner()
    subdomains = []

    args = parse_args()
    target = parse_url(args.domain)
    output = args.output

    req = requests.get(f'https://crt.sh/?q=%.{target}&output=json')

    if req.status_code != 200:
        print(f'{Fore.RED}[*] Information Not Available!')

    for (key,value) in enumerate(req.json()):
        subdomains.append(value['name_value'])

    subs = sorted(set(subdomains))

    for s in subs:
        print(f'{s}\n')
        if output is not None:
            write_subs_to_file(s, output)

if __name__=='__main__':
    main()


import nltk
from nltk.corpus import brown

from nltk.corpus import PlaintextCorpusReader
corpus_root = 'C://Users//Stoeoeoe//Dropbox//Thesis 2016//Data//documents'
corpus = PlaintextCorpusReader(corpus_root, '.*\.txt', encoding='utf-8')

cfd = nltk.ConditionalFreqDist(
          (genre, word)
          for genre in brown.categories()
          for word in brown.words(categories=genre))

print(wordlists.words("1.txt"))


# -*- coding: utf-8 -*-

'''

Author  :   Sunanda Bansal (sunanda0343@gmail.com)
Year    :   2021

'''

# Importing Libraries
import warnings
import numpy as np
import pandas as pd

from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering as AHC

def cluster_word_vectors(
    word_vectors,
    clustering_algorithm,
    n_clusters=1500,
    distance_threshold=None,
    linkage="ward",
    random_state=0,
    **clustering_kwargs
    ):
    '''
    Clusters word vectors

    Parameters
    ----------
    word_vectors : pandas.Series
        Pandas Series with words as the index and vector (numpy array) as the
        corresponding value.

    clustering_algorithm : str
        Specifies which clustering algorithm to use. Accepts "kmeans" for K-Means
        or "ahc" for Agglomerative Hierarchical Clustering (AHC).

    n_clusters : int, optional
        The number of clusters to cluster the word vectors into. The default is 1500.

    distance_threshold : float, optional
        For Agglomerative Hierarchical Clustering (AHC), the clusters with linkage
        distance smaller than this value will be merged. If a value is declared
        for this parameter then `n_clusters` must be set to None.
        The default is None.

    linkage : str, optional
        The linkage critera for Agglomerative Hierarchical Clustering (AHC).
        The default is None.

    **clustering_kwargs : TYPE
        Additional keyword arguments are passed to respective clustering functions
        based on the value `clustering_algorithm`. For refer to the documentation
        of sklearn library for KMeans and AgglomerativeClustering clustering
        algorithms.

    Returns
    -------
    labels : list
        A list of cluster labels for each of the word vectors.

    '''

    if clustering_algorithm == "kmeans":
        clustering_model = KMeans(
                                    n_clusters          = n_clusters,
                                    random_state        = random_state,
                                    **clustering_kwargs
                                 )

    elif clustering_algorithm == "ahc" :
        clustering_model = AHC(
                                    n_clusters          = n_clusters,
                                    distance_threshold  = distance_threshold,
                                    linkage             = linkage,
                                    compute_full_tree   = True if n_clusters is None else False,
                                    **clustering_kwargs
                                )
    else:
        raise Exception('''Argument clustering_algorithm accepts "kmeans" or "ahc" only.''')

    clustering_model = clustering_model.fit(word_vectors)

    labels = clustering_model.labels_

    if len(set(labels)) == 1:
        warnings.warn("Based on the parameters provided, only 1 word cluster was found.")

    return labels

def get_document_vectors(
    tokenized_texts,
    words,
    cluster_labels,
    weight_function="cfidf",
    training_tokenized_texts=None,
    normalize=True
    ):

    '''
    Computes the document vectors given word clusters and weight function.

    CF-IDF
    ======
    Required for CF-iDF cluster weight computation

        cf(i,j)  = count of every occurrence of any terms of cluster i that are present in document j
        cdf(i)   = cluster document frequency of cluster i
                 = number of document where any term from cluster i appeared in

    Parameters
    ----------
    tokenized_texts : list of lists of str
        A list of texts where each text is further a list of tokens.

    words : list
        The words that were clustered.

    cluster_labels : list
        For each word, this list gives the label of the cluster it belongs to.

    weight_function : str, optional
        Provides the default weighting scheme to be used. The available
        options are "cfidf" or "tfidf_sum".
        The default is "cfidf".

    training_tokenized_texts : list of list of str, optional
        The training texts are used to calculate the document frequencies.
        The default is None.

    normalize : bool, optional
        If True, then the document vectors will be length normalized (L2).
        The default is True.

    Returns
    -------
    labels : list of lists
        A list of document vector for each documents in the tokenized_texts

    '''

    # If training texts are not provided
    if training_tokenized_texts == None:
        training_tokenized_texts = tokenized_texts

    # Total number of texts (used in tfidf and cfidf weight calculations)
    N = len(training_tokenized_texts)
    vector_size = len(set(cluster_labels))

    # Dictionaries that record the document IDs each term or word cluster appears in
    df = {word:[] for word in words}
    cdf = {cl:[] for cl in range(vector_size)}
    word_clusters = {word: cluster_label for word, cluster_label in zip(words, cluster_labels)}

    # Calculates term-document-frequency and cluster-document-frequency from the [training] data
    for idx, tokenized_text in enumerate(training_tokenized_texts):
        for token in set(tokenized_text):

            # If token is in a part of word clusters
            if token in words:

                # Append document ID if it isn't already in the list of documents
                if idx not in df[token]:
                    df[token].append(idx)
                if idx not in cdf[word_clusters[token]]:
                    cdf[word_clusters[token]].append(idx)

    # Maps the token and word clusters to their corresponding document frequencies
    df = {token:len(df[token]) for token in df}
    cdf = {cluster_label:len(cdf[cluster_label]) for cluster_label in cdf}

    # Calculate inverse document frequencies in preparation for weighting schemes
    if weight_function == "cfidf":
        # CF-IDF Prep
        cdf_vector = np.array([cdf[cl] for cl in range(vector_size)])
        icdf_vector = np.log(N/cdf_vector + 0.01)

    elif weight_function == "tfidf_sum":
        # TF-iDF SUM Prep
        idf = {token:np.log(N/df[token]) for token in df}

    wcde_doc_vectors = [] # WcDe Document vector corresponding to each document
    cf_all = [] # Cluster Frequency vectors corresponding to each document

    # Process each document and generate its document vector
    for idx, tokenized_text in enumerate(tokenized_texts):

        # Default vector - origin
        cluster_weights = [0]*vector_size

        if len(tokenized_text) > 0:

            if weight_function == "cfidf":

                # Get cluster frequency -
                #   cf(i,j) =   count of every occurrence of any terms of
                #               cluster i that are present in document j

                cf_vector = [0]*vector_size  # default
                for token in tokenized_text:
                    if token in words:
                        cf_vector[word_clusters[token]] += 1
                cf_vector = np.array(cf_vector)
                cf_all.append(cf_vector)

                # Calculate CF-iDF weight of all the clusters in the document
                cluster_weights = cf_vector * icdf_vector

            elif weight_function == "tfidf_sum":
                # Calculate the weight of clusters in the documents as sum of
                # tfidf values of its members

                for token in set(tokenized_text):
                    if token in words:
                        cluster_weights[word_clusters[token]] += tokenized_text.count(token)*idf[token]

                cluster_weights = np.array(cluster_weights)

            # Length Normalize
            if bool(normalize):
                cluster_weights = cluster_weights/np.sqrt(np.sum(np.square(cluster_weights)))

        wcde_doc_vectors.append(list(np.asarray(cluster_weights)))

    return wcde_doc_vectors


""" Unit tests for app/env_canada.py """

import os
import sys
import logging
from typing import Optional
import pytest
import requests
import shapely
import datetime
from sqlalchemy.orm import Session
from pytest_mock import MockerFixture
import app.utils.time as time_utils
import app.db.database
import app.db.crud.weather_models
import app.jobs.env_canada
import app.jobs.common_model_fetchers
import app.weather_models.process_grib
from app.weather_models import ProjectionEnum
from app.stations import StationSourceEnum
from app.db.models.weather_models import (PredictionModel, ProcessedModelRunUrl,
                                          PredictionModelRunTimestamp)
from app.tests.weather_models.test_env_canada_gdps import MockResponse

logger = logging.getLogger(__name__)

@pytest.fixture()
def mock_database(monkeypatch):
    """ Mocked out database queries """
    geom = ("POLYGON ((-120.525 50.77500000000001, -120.375 50.77500000000001,-120.375 50.62500000000001,"
            " -120.525 50.62500000000001, -120.525 50.77500000000001))")
    shape = shapely.wkt.loads(geom)

    hrdps_url = 'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/' \
        '00/001/20200521T00Z_MSC_HRDPS_RH_AGL-2m_RLatLon0.0225_PT001H.grib2'
    hrdps_processed_model_run = ProcessedModelRunUrl(url=hrdps_url)
    hrdps_prediction_model = PredictionModel(id=3, abbreviation='HRDPS', projection='ps2.5km',
                                             name='High Resolution Deterministic Prediction System')
    hrdps_prediction_model_run = PredictionModelRunTimestamp(
        id=1, prediction_model_id=hrdps_prediction_model.id, prediction_run_timestamp=time_utils.get_utc_now(),
        prediction_model=hrdps_prediction_model, complete=True)

    def mock_get_prediction_model(session, model, projection) -> Optional[PredictionModel]:
        return hrdps_prediction_model

    def mock_get_hrdps_prediction_model_run_timestamp_records(*args, **kwargs):
        return [(hrdps_prediction_model_run, hrdps_prediction_model)]

    def mock_get_processed_file_record(session, url: str):
        # We only want the one file to be processed - otherwise our test takes forever
        if url != hrdps_url:
            return hrdps_processed_model_run
        return None

    def mock_get_prediction_run(*args, **kwargs):
        return hrdps_prediction_model_run

    monkeypatch.setattr(app.weather_models.process_grib, 'get_prediction_model', mock_get_prediction_model)
    monkeypatch.setattr(app.jobs.common_model_fetchers, 'get_prediction_model_run_timestamp_records',
                        mock_get_hrdps_prediction_model_run_timestamp_records)
    monkeypatch.setattr(app.jobs.env_canada, 'get_processed_file_record', mock_get_processed_file_record)
    monkeypatch.setattr(app.db.crud.weather_models, 'get_prediction_run', mock_get_prediction_run)

@pytest.fixture()
def mock_get_processed_file_record(monkeypatch):
    """ Mock "get_processed_file_record" to only return the None on the 1st call. """
    called = False

    def get_processed_file_record(session: Session, url: str):
        nonlocal called
        if called:
            return ProcessedModelRunUrl()
        called = True
        return None

    monkeypatch.setattr(app.jobs.env_canada, 'get_processed_file_record', get_processed_file_record)

@pytest.fixture()
def mock_download(monkeypatch):
    """ fixture for env_canada.download """
    def mock_requests_get_hrdps(*args, **kwargs):
        """ mock env_canada download method for HRDPS """
        dirname = os.path.dirname(os.path.realpath(__file__))
        filename = os.path.join(
            dirname, '20230317T18Z_MSC_HRDPS_RH_AGL-2m_RLatLon0.0225_PT001H.grib2')
        with open(filename, 'rb') as file:
            content = file.read()
        return MockResponse(status_code=200, content=content)
    monkeypatch.setattr(requests, 'get', mock_requests_get_hrdps)

def test_get_hrdps_download_urls():
    """ test to see if get_download_urls methods gives the correct number of urls """
    # -1 because 000 hour has no APCP_SFC_0
    total_num_of_urls = 49 * len(app.jobs.env_canada.HRDPS_GRIB_LAYERS) - 1
    assert len(list(app.jobs.env_canada.get_high_res_model_run_download_urls(
        time_utils.get_utc_now(), 0))) == total_num_of_urls

@pytest.mark.usefixtures('mock_get_processed_file_record')
def test_process_hrdps(mock_download, mock_database):
    """ run process method to see if it runs successfully. """
    # All files, except one, are marked as already having been downloaded, so we expect one file to
    # be processed.
    sys.argv = ["argv", "HRDPS"]
    assert app.jobs.env_canada.process_models(StationSourceEnum.TEST) == 1

def test_main_fail(mocker: MockerFixture, monkeypatch):
    """ Run the main method, check that message is sent to rocket chat, and exit code is EX_SOFTWARE """
    sys.argv = ["argv", "HRDPS"]

    def mock_process_models():
        raise Exception()

    rocket_chat_spy = mocker.spy(app.jobs.env_canada, 'send_rocketchat_notification')
    monkeypatch.setattr(app.jobs.env_canada, 'process_models', mock_process_models)

    with pytest.raises(SystemExit) as excinfo:
        app.jobs.env_canada.main()

    # Assert that we exited with an error code.
    assert excinfo.value.code == os.EX_SOFTWARE
    # Assert that rocket chat was called.
    assert rocket_chat_spy.call_count == 1

def test_parse_high_res_model_url_correct_format():
    """ Given a grib file download URL in the correct format, env_canada.parse_high_res_model_url
    should return the correct info pulled from the URL """
    test_cases = [
        {
            'url': 'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/06/012/20230322T06Z_MSC_HRDPS_TMP_AGL-2m_RLatLon0.0225_PT012.grib2',
            'variable_name': 'TMP_AGL-2m',
            'projection': ProjectionEnum.HIGH_RES_CONTINENTAL,
            'model_run_timestamp': datetime.datetime(2023, 3, 22, 6, 0, tzinfo=datetime.timezone.utc),
            'prediction_timestamp': datetime.datetime(2023, 3, 22, 18, 0, tzinfo=datetime.timezone.utc)
        },
        {
            'url': 'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/06/012/20230322T06Z_MSC_HRDPS_WIND_AGL-10m_RLatLon0.0225_PT012.grib2',
            'variable_name': 'WIND_AGL-10m',
            'projection': ProjectionEnum.HIGH_RES_CONTINENTAL,
            'model_run_timestamp': datetime.datetime(2023, 3, 22, 6, 0, tzinfo=datetime.timezone.utc),
            'prediction_timestamp': datetime.datetime(2023, 3, 22, 18, 0, tzinfo=datetime.timezone.utc)
        },
        {
            'url': 'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/06/012/20230322T06Z_MSC_HRDPS_APCP_Sfc_RLatLon0.0225_PT012.grib2',
            'variable_name': 'APCP_Sfc',
            'projection': ProjectionEnum.HIGH_RES_CONTINENTAL,
            'model_run_timestamp': datetime.datetime(2023, 3, 22, 6, 0, tzinfo=datetime.timezone.utc),
            'prediction_timestamp': datetime.datetime(2023, 3, 22, 18, 0, tzinfo=datetime.timezone.utc)
        },
        {
            'url': 'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/12/084/20230322T12Z_MSC_HRDPS_RH_AGL-2m_RLatLon0.0225_PT084.grib2',
            'variable_name': 'RH_AGL-2m',
            'projection': ProjectionEnum.HIGH_RES_CONTINENTAL,
            'model_run_timestamp': datetime.datetime(2023, 3, 22, 12, 0, tzinfo=datetime.timezone.utc),
            'prediction_timestamp': datetime.datetime(2023, 3, 26, 0, 0, tzinfo=datetime.timezone.utc)
        },
    ]

    for case in test_cases:
        actual_variable_name, actual_projection, actual_model_run_timestamp, actual_prediction_timestamp \
            = app.jobs.env_canada.parse_high_res_model_url(
                case['url'])
        assert actual_variable_name == case['variable_name']
        assert actual_projection == case['projection']
        assert actual_model_run_timestamp == case['model_run_timestamp']
        assert actual_prediction_timestamp == case['prediction_timestamp']

def test_part_high_res_model_url_incorrect_format():
    """ Tests basic functionality to detect issues in HRDPS URL formats - Exceptions should be raised
    because given URLs are missing vital pieces of info """
    bad_urls = [
        'https://dd.weather.gc.ca/bad_format',
        'https://dd.weather.gc.ca/model_hrdps/moo',
        'https://dd.weather.gc.ca/model_hrdps/continental/25km/',
        'https://dd.weather.gc.ca/model_hrps/wrong_model_name',
        'https://dd.weather.gc.ca/model_hrdps/continental/2.5km/00/MSC_ASDF_ASDFASDF.grib2'
    ]

    for test_case in bad_urls:
        with pytest.raises(Exception):
            app.jobs.env_canada.parse_high_res_model_url(test_case)


from django.urls import path, re_path
from TicketApp import views, apiviews

'''
List of api routes

                    User Routes:
Get, Delete
-----------------------------
Show all users = api/users/
Show one user by username = api/users?username={username}
Show one user by email = api/users?email={email}
Delete one user = api/users/<username>

Post, Put Routes
-----------------------------
id required in body for Put
Add/Update user with request body = api/users/

                    Address Routes:
Get, Delete
-----------------------------
Show all addresses = api/addresses/
Show/Delete one address = api/addresses/<id>

Post, Put Routes
-----------------------------
id required in body for Put
Add/Update address with request body = api/addresses/

                    Offer Routes:
Get, Delete
-----------------------------
Show all offers = api/offers/
Show all of a buyers and/or sellers offers = api/offers?buyer={username}&seller={username}
Show/Delete offer = api/offers/<id>

Post, Put Routes
-----------------------------
id required in body for Put
buyer username may be blank or omitted entirely
buyer (if provided) and seller usernames must exist or else error returned, ids for event and area must exist
Add/Update offer with request body = api/offers/

                    Venue Routes:
Get, Delete
-----------------------------
Show all venues = api/venues/
Show/Delete venue = api/venues/<id>

Post, Put Routes
-----------------------------
id required in body for Put
id for address must exist
Add/Update venue with request body = api/venues/

                    Area Routes:
Get, Delete
-----------------------------
Show all areas = api/areas/
Show/Delete area = api/areas/<id>

Post, Put Routes
-----------------------------
id required in body for Put
id for venue must exist
Add/Update area with request body = api/areas/

                    Price Routes:
Get, Delete
-----------------------------
Show all prices = api/prices/
Show/Delete price = api/prices/<id>

Post, Put Routes
-----------------------------
id required in body for Put
id for area must exist
Add/Update price with request body = api/prices/

                    Performer Routes:
Get, Delete
-----------------------------
Show all performers = api/performers/
Show/Delete performer = api/performers/<id>

Post, Put Routes
-----------------------------
id required in body for Put
Add/Update performer with request body = api/performers/

                    Genre Routes:
Get, Delete
-----------------------------
Show all genres = api/genres/
Show/Delete genre = api/genres/<id>

Post, Put Routes
-----------------------------
id required in body for Put
Add/Update genre with request body = api/genres/

                    EventPerformer Routes:
Get, Delete
-----------------------------
Show all eventperformers = api/eventperformers/
Show all events with performer = api/eventperformers?performer={id}
Show all of an events performers = api/eventperformers?event={id}
Check if event has a performer = api/eventperformers?event={id}&performer={id} | Will return json { ok : [] } in case false
Show/Delete eventperformer = api/eventperformers/<id>

Post, Put Routes
-----------------------------
id required in body for Put
event and performer ids must exist
date is in the format YYYY-MM-DDThh:mm e.g 2023-01-01T23:59
Add/Update eventperformer with request body = api/eventperformers/

                    EventGenre Routes:
Get, Delete
-----------------------------
Show all eventgenres = api/eventgenres/
Show all events with genre = api/eventgenres?genre={name}
Show all of an events genres = api/eventgenres?event={id}
Check if event has a genre = api/eventgenres?event={id}&genre={name} | Will return json { ok : [] } in case false
Show/Delete eventgenre = api/eventgenres/<id>

Post, Put Routes
-----------------------------
id required in body for Put
event id and genre name must exist
Add/Update eventgenre with request body = api/eventgenres/

                    Event Routes:
Get, Delete
-----------------------------
Show all events = api/events/
Show all of a venues events = api/events?venue={id}
Show/Delete event = api/events/<id>

Post, Put Routes
-----------------------------
id required in body for Put
venue id must exist
date is in the format YYYY-MM-DDThh:mm e.g 2023-01-01T23:59
Add/Update event with request body = api/events/

'''

urlpatterns = [

    re_path(r'^users/?$', apiviews.UserApi, name="all-users"),
    re_path(r'^users/(?P<username>\w+)', apiviews.UserApi, name="one-user"),

    re_path(r'^addresses/?$', apiviews.AddressApi, name="all-addresses"),
    re_path(r'^addresses/(?P<id>[0-9]+)', apiviews.AddressApi, name="one-address"),

    re_path(r'^offers/?$', apiviews.OfferApi, name="all-offers"),
    re_path(r'^offers/(?P<id>[0-9]+)', apiviews.OfferApi, name="one-offer"),

    re_path(r'^venues/?$', apiviews.VenueApi, name="all-venues"),
    re_path(r'^venues/(?P<id>[0-9]+)', apiviews.VenueApi, name="one-venue"),

    re_path(r'^areas/?$', apiviews.AreaApi, name="all-areas"),
    re_path(r'^areas/(?P<id>[0-9]+)', apiviews.AreaApi, name="one-area"),

    re_path(r'^prices/?$', apiviews.PriceApi, name="all-prices"),
    re_path(r'^prices/(?P<id>[0-9]+)', apiviews.PriceApi, name="one-price"),

    re_path(r'^performers/?$', apiviews.PerformerApi, name="all-perfomers"),
    re_path(r'^performers/(?P<id>[0-9]+)', apiviews.PerformerApi, name="one-performer"),

    re_path(r'^genres/?$', apiviews.GenreApi, name="all-genres"),
    re_path(r'^genres/(?P<id>[0-9]+)', apiviews.GenreApi, name="one-genre"),

    re_path(r'^eventperformers/?$', apiviews.EventPerformerApi, name="all-eventperformers"),
    re_path(r'^eventperformers/(?P<id>[0-9]+)', apiviews.EventPerformerApi, name="one-eventperformer"),

    re_path(r'^eventgenres/?$', apiviews.EventGenreApi, name="all-eventgenres"),
    re_path(r'^eventgenres/(?P<id>[0-9]+)', apiviews.EventGenreApi, name="one-eventgenre"),

    re_path(r'^events/?$', apiviews.EventApi, name="all-events"),
    re_path(r'^events/(?P<id>[0-9]+)', apiviews.EventApi, name="one-event"),

]


'''
datasets.py: Customized dataset provider.
---- Dataset: object containing training set and test set
'''
import os
import pandas as pd
import numpy as np
import scipy.io as scio
from joblib import dump, load
from sklearn.preprocessing import StandardScaler

def extract_d1(datapath):
    '''
    input: data path -> list
    output: X_train, y_train, X_test, y_test -> array-like object
    '''
    train_path = datapath[0]
    test_path = datapath[1]
    X_train = np.loadtxt(train_path[0])
    y_train = np.loadtxt(train_path[1])
    X_test = np.loadtxt(test_path[0])
    y_test = np.loadtxt(test_path[1])
    return X_train, y_train, X_test, y_test

class Dataset:
    '''
    Dataset of training set and test set
    '''

    def __init__(self, descr, datapath=None, extraction=extract_d1, preprocessing=False, persistency=False, artifact_dir="../artifacts/", LOAD=None):
        '''
        Parameters:
        -- descr: Description of dataset, e.g. the usage or components of dataset, which would be used in persistency directory -> str
        -- datapath: Path of the original data file where you want to load the data -> str
        -- extraction: Function need-to-implement, which is used to extract the array like data from origin data file -> function
        -- preprocessing: Use standard scaler to scale X and normalize y to 0 mean.
        -- persistency: Whether to execute persistency of the data loaded -> bool
        -- artifact_dir: Persistency top directory, e.g /home/xx/proj/data -> str
        -- LOAD: Load from existed pickle? cover the "extraction" and "datapath"-> bool
        '''

        self.descr = str(descr)
        if LOAD:
            if not artifact_dir.endswith("/"):
                artifact_dir = f"{artifact_dir}/"
            self.X_train = load(os.path.join(f"{artifact_dir}data/", LOAD, "X_train.pickle"))
            self.y_train = load(os.path.join(f"{artifact_dir}data/", LOAD, "y_train.pickle"))
            self.X_test = load(os.path.join(f"{artifact_dir}data/", LOAD, "X_test.pickle"))
            self.y_test = load(os.path.join(f"{artifact_dir}data/", LOAD, "y_test.pickle"))

        else:
            X_scaler = None
            y_mean = 0
            self.X_train, self.y_train, self.X_test, self.y_test = extraction(datapath)
            if preprocessing:
                # self.y_train = self.y_train.reshape(-1, 1)
                # self.y_test = self.y_test.reshape(-1, 1)
                self.X_scaler = StandardScaler()
                # self.y_scaler = StandardScaler()
                self.X_scaler.fit(self.X_train)
                # self.y_scaler.fit(self.y_train)
                self.X_train = self.X_scaler.transform(self.X_train)
                # self.y_train = self.y_scaler.transform(self.y_train)
                self.X_test = self.X_scaler.transform(self.X_test)
                # self.y_test = self.y_scaler.transform(self.y_test)
                # self.y_train = self.y_train.reshape(-1,)
                # self.y_test = self.y_test.reshape(-1,)
            if persistency:
                PREFIX = "data/{}/".format(self.descr)
                if not os.path.exists(os.path.join(artifact_dir, PREFIX)):
                    os.mkdir(os.path.join(artifact_dir, PREFIX))
                if preprocessing and X_scaler:
                    dump(X_scaler, os.path.join(artifact_dir, PREFIX, "Xscaler_ymean_{}.pickle".format(y_mean)))
                dump(self.X_train, os.path.join(artifact_dir, PREFIX, "X_train.pickle"))
                dump(self.y_train, os.path.join(artifact_dir, PREFIX, "y_train.pickle"))
                dump(self.X_test, os.path.join(artifact_dir, PREFIX, "X_test.pickle"))
                dump(self.y_test, os.path.join(artifact_dir, PREFIX, "y_test.pickle"))

    def convert(self, to_type):
        '''
        Convert self.xxx to relavant type:
        Parameters:
        -- to_type: The type you want to transform, only support [np.ndarray, torch.Tensor, pd.DataFrame] -> type
        '''
        _type_str = ["Array", "Tensor", "DataFrame"]
        _type = [(np.ndarray, np.generic), torch.Tensor, pd.DataFrame]
        if to_type not in _type_str:
            raise ValueError("to_type must be one of {}".format(_type_str))
        transform = [lambda a: t(a) for t in [np.array, torch.Tensor, pd.DataFrame]]
        for i, _to_type in enumerate(_type_str):
            if to_type == _to_type:
                if not isinstance(self.X_train, _type[i]):
                    try:
                        self.X_train = transform[i](self.X_train)
                    except:
                        raise ValueError("cannot transform to {}".format(_type_str[i]))
                if not isinstance(self.y_train, _type[i]):
                    try:
                        self.y_train = transform[i](self.y_train)
                    except:
                        raise ValueError("cannot transform to {}".format(_type_str[i]))
                if not isinstance(self.X_test, _type[i]):
                    try:
                        self.X_test = transform[i](self.X_test)
                    except:
                        raise ValueError("cannot transform to {}".format(_type_str[i]))
                if not isinstance(self.y_test, _type[i]):
                    try:
                        self.y_test = transform[i](self.y_test)
                    except:
                        raise ValueError("cannot transform to {}".format(_type_str[i]))

    def reverse_scaler(self, y):
        return self.y_scaler.inverse_transform(y)


import sys

import cv2
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import BSpline, make_interp_spline

class AutoFocus:
    def __init__(self) -> None:
        self.k = 700
        self.height_list = []
        self.height_listmag_list = []
        self.x = 10

    def autofocus(self):
        """
        Generate the function comment for the given function body in a markdown code block with the correct language syntax.
        This code defines a function called autofocus that performs some image processing operations. It takes no arguments and returns
        two arrays, xnew and power_smooth.

        The function iterates over a range of values, appends values to a list, and performs image processing operations on each iteration.
        It uses the OpenCV library to apply Sobel operators to calculate the gradient magnitude of the image. It then calculates the median
        of the gradient magnitudes and filters out values below a certain threshold. The function also calculates the sum of the filtered
        gradient magnitudes and appends it to another list.

        Finally, the function converts the lists to numpy arrays, creates a new array xnew using the minimum and maximum values from
        height_list, and applies a spline interpolation to smooth the height_listmag_list values. The smoothed values are stored in power_smooth
        and both xnew and power_smooth are returned from the function.
        Returns:
            xnew (numpy.ndarray): An array of x-axis values.
            power_smooth (numpy.ndarray): An array of smoothed y-axis values.
        """
        for i in range(21):
            self.height_list.append(k)
            k +=10

        self.k = 700
        for i in range(21):
            img = cv2.imread(str(self.k) + ".bmp",0)

            #print("cropped/" + str(k) + ".bmp___________________")
            sobelx = cv2.Sobel(img,cv2.CV_64F,1,0)
            sobely = cv2.Sobel(img,cv2.CV_64F,0,1)

            Gmag = np.sqrt(sobelx*sobelx + sobely*sobely)
            Gmag = Gmag.flatten()
            #Gdir = np.arctan2(sobely, sobelx) * (180 / np.pi)

            #Gmag = np.where(Gmag == 0, np.nan, Gmag)
            Gmag = Gmag[Gmag!=0]

            ortanca_deger = np.median(Gmag)
            print(ortanca_deger)
            #print(str(k) + " - MEDIAN = ", ortanca_deger)

            ortanca_deger3 = ortanca_deger*self.x
            #Gmag = np.where(Gmag < (ortanca_deger3), np.nan, Gmag)
            Gmag = Gmag[Gmag>ortanca_deger3]

            print(str(self.k) + " - SUM    = ", np.sum(Gmag))
            self.height_listmag_list.append(np.sum(Gmag))
            self.k +=10

        self.height_list = np.array(self.height_list); self.height_listmag_list = np.array(self.height_listmag_list)

        xnew = np.linspace(self.height_list.min(), self.height_list.max(), 30000)
        spl = make_interp_spline(self.height_list, self.height_listmag_list, k=3)  # type: BSpline
        power_smooth = spl(xnew)

        return xnew, power_smooth

    def plot(self, xnew, power_smooth):
        """
        Plots the given data using matplotlib.
        This code defines a method called plot that uses matplotlib to plot data. It takes in two parameters, xnew and power_smooth,
        which are arrays of x-values and y-values respectively. The method plots the data using plt.plot, adds a legend, sets labels
        for the x-axis and y-axis, scales the y-axis using a symlog scale, and sets a title for the plot. Finally, it displays the plot
        using plt.show.
        Parameters:
            xnew (array-like): The x-values of the data.
            power_smooth (array-like): The y-values of the data.
        Returns:
            None
        """
        plt.plot(xnew, power_smooth, label="P Values for Every ε Position")
        #plt.plot(self.height_list, self.height_listmag_list, label="P Values for Every ε Position")
        plt.legend()
        plt.xlabel("ε Values")
        plt.ylabel("P Values")
        plt.yscale('symlog')
        plt.title("ς = {}".format(self.x))
        plt.show()


from modules.base_module import RanaModule
from core import geo
from threading import Thread
import math
import time

REROUTE_CHECK_INTERVAL = 5000 # in ms
#in m/s, about 46 km/h - if this speed is reached, the rerouting threshold is multiplied
# by REROUTING_THRESHOLD_MULTIPLIER
INCREASE_REROUTING_THRESHOLD_SPEED = 20
REROUTING_DEFAULT_THRESHOLD = 30
# not enabled at the moment - needs more field testing
REROUTING_THRESHOLD_MULTIPLIER = 1.0
# how many times needs the threshold be crossed to
# trigger rerouting
REROUTING_TRIGGER_COUNT = 3

MAX_CONSECUTIVE_AUTOMATIC_REROUTES = 3
AUTOMATIC_REROUTE_COUNTER_EXPIRATION_TIME = 600 # in seconds

# only import GKT libs if GTK GUI is used
from core import gs

if gs.GUIString == "GTK":
    import pango
    import pangocairo

def getModule(*args, **kwargs):
    return TurnByTurn(*args, **kwargs)

class TurnByTurn(RanaModule):
    """A turn by turn navigation module."""

    def __init__(self, *args, **kwargs):
        RanaModule.__init__(self, *args, **kwargs)
        # initial colors
        self.navigationBoxBackground = (0, 0, 1, 0.3) # very transparent blue
        self.navigationBoxText = (1, 1, 1, 1) # non-transparent white
        self.TBTWorker = None
        self.TBTWorkerEnabled = False
        self.goToInitialState()
        self.automaticRerouteCounter = 0 # counts consecutive automatic reroutes
        self.lastAutomaticRerouteTimestamp = time.time()
        # reroute even though the route was not yet reached (for special cases)
        self.overrideRouteReached = False

    def goToInitialState(self):
        """restore initial state"""
        self.route = None
        self.currentStepIndex = 0
        self.currentStepIndicator = None
        self.espeakFirstAndHalfTrigger = False
        self.espeakFirstTrigger = False
        self.espeakSecondTrigger = False
        self.currentDistance = None
        self.currentStep = None
        self.navigationBoxHidden = False
        self.mRouteLength = 0
        self.locationWatchID = None
        self.onRoute = False
        #rerouting is enabled once the route is reached for the first time
        self.routeReached = False
        self.reroutingThresholdMultiplier = 1.0
        self.reroutingThresholdCrossedCounter = 0

    def firstTime(self):
        icons = self.m.get('icons', None)
        if icons:
            icons.subscribeColorInfo(self, self.colorsChangedCallback)

    def colorsChangedCallback(self, colors):
        self.navigationBoxBackground = colors['navigation_box_background'].getCairoColor()
        self.navigationBoxText = colors['navigation_box_text'].getCairoColor()

    def handleMessage(self, message, messageType, args):
        if message == 'start':
            if messageType == 'ms':
                fromWhere = args
                self.startTBT(fromWhere)
        elif message == 'stop':
            self.stopTBT()
        elif message == 'reroute': # manual rerouting
            # reset automatic reroute counter
            self.automaticRerouteCounter = 0
            self._reroute()
        elif message == "toggleBoxHiding":
            self.log.info("toggling navigation box visibility")
            self.navigationBoxHidden = not self.navigationBoxHidden
        elif message == "switchToPreviousTurn":
            self.switchToPreviousStep()
        elif message == "switchToNextTurn":
            self.switchToNextStep()
        elif message == "showMessageInsideNotification":
            currentStep = self.getCurrentStep()
            if currentStep:
                message = "turn description:\n%s" % currentStep.description
                if self.dmod.hasNotificationSupport():
                    self.dmod.notify(message, 7000)
                    #TODO: add support for modRana notifications once they support line wrapping

    def _rerouteAuto(self):
        """this function is called when automatic rerouting is triggered"""

        # check time from last automatic reroute
        dt = time.time() - self.lastAutomaticRerouteTimestamp
        if dt >= AUTOMATIC_REROUTE_COUNTER_EXPIRATION_TIME:
            # reset the automatic reroute counter
            self.automaticRerouteCounter = 0
            self.log.debug('automatic reroute counter expired, clearing')

        # on some routes, when we are moving away from the start of the route, it
        # is needed to reroute a couple of times before the correct way is found
        # on the other hand there should be a limit on the number of times
        # modRana reroutes in a row
        #
        # SOLUTION:
        # 1. enable automatic rerouting even though the route was not yet reached
        # (as we are moving away from it)
        # 2. do this only a limited number of times (up to 3 times in a row)
        # 3. the counter is reset by manual rerouting, by reaching the route or after 10 minutes

        if self.automaticRerouteCounter < MAX_CONSECUTIVE_AUTOMATIC_REROUTES:
            self.log.debug('faking that route was reached to enable new rerouting')
            self.overrideRouteReached = True
        else:
            self.log.info('tbt: too many consecutive reroutes (%d),', self.automaticRerouteCounter)
            self.log.info('reach the route to enable automatic rerouting')
            self.log.info('or reroute manually')

        # increment the automatic reroute counter & update the timestamp
        self.automaticRerouteCounter += 1
        self.lastAutomaticRerouteTimestamp = time.time()

        # trigger rerouting
        self._reroute()

    def _reroute(self):
        # 1. say rerouting is in progress
        voiceMessage = "rerouting"
        voice = self.m.get('voice', None)
        if voice:
            voice.say(voiceMessage, "en") # make sure rerouting said with english voice
        time.sleep(2) #TODO: improve this
        # 2. get a new route from current position to destination
        self.sendMessage("ms:route:reroute:fromPosToDest")
        # 3. restart routing for to this new route from the closest point
        self.sendMessage("ms:turnByTurn:start:closest")

    def drawMapOverlay(self, cr):
        if self.route:
            # get current step
            currentStep = self.getCurrentStep()
            proj = self.m.get('projection', None)
            # draw the current step indicator circle
            if currentStep and proj:
                (lat, lon) = currentStep.getLL()
                (pointX, pointY) = proj.ll2xy(lat, lon)
                cr.set_source_rgb(1, 0, 0)
                cr.set_line_width(4)
                cr.arc(pointX, pointY, 12, 0, 2.0 * math.pi)
                cr.stroke()
                cr.fill()

    def drawScreenOverlay(self, cr):
        if self.route: # is there something relevant to draw ?

            # get current step
            currentStep = self.getCurrentStep()

            # draw the routing message box

            # we need to have the viewport available
            vport = self.get('viewport', None)
            menus = self.m.get('menu', None)
            if vport and menus:
                (sx, sy, w, h) = vport
                (bx, by, bw, bh) = (w * 0.15, h * 0.20, w * 0.7, h * 0.4)
                buttonStripOffset = 0.25 * bh

                # construct parametric background for the cairo drawn buttons
                background = "generic:;0;;1;5;0"

                if self.navigationBoxHidden:
                    # * show button
                    showButtonWidth = bw * 0.2
                    # the show button uses custom parameters
                    parametricIconName = "center:show;0.1>%s" % background
                    menus.drawButton(cr, bx + (bw - showButtonWidth), by, showButtonWidth, buttonStripOffset, "",
                                     parametricIconName, "turnByTurn:toggleBoxHiding")

                else:
                    # draw the info-box background
                    cr.set_source_rgba(*self.navigationBoxBackground)
                    cr.rectangle(bx, by + buttonStripOffset, bw, bh - buttonStripOffset)
                    cr.fill()

                    # create a layout for our drawing area
                    pg = pangocairo.CairoContext(cr)
                    layout = pg.create_layout()

                    # get the current turn message
                    message = currentStep.description

                    # display current distance to the next point & other unit conversions
                    units = self.m.get('units', None)
                    if units and self.currentDistance:
                        distString = units.m2CurrentUnitString(self.currentDistance, 1, True)
                        if currentStep.distanceFromStart:
                            currentDistString = units.m2CurrentUnitString(currentStep.distanceFromStart, 1, True)
                        else:
                            currentDistString = "?"
                        routeLengthString = units.m2CurrentUnitString(self.mRouteLength, 1, True)
                    else:
                        distString = ""
                        currentDistString = ""
                        routeLengthString = ""

                    # TODO: find why there needs to be a newline on the end
                    message = "%s : %s\n" % (distString, message)

                    border = min(bw / 50.0, bh / 50.0)
                    # compute how much space is actually available for the text
                    usableWidth = bw - 2 * border
                    usableHeight = bh - 6 * border - buttonStripOffset
                    layout.set_width(int(usableWidth * pango.SCALE))
                    layout.set_wrap(pango.WRAP_WORD)
                    layout.set_markup(message)
                    layout.set_font_description(pango.FontDescription("Sans Serif 24")) #TODO: custom font size ?
                    (lw, lh) = layout.get_size()
                    if lw == 0 or lh == 0:
                        # no need to draw a zero area layout
                        return

                    # get coordinates for the area available for text
                    ulX, ulY = (bx + border, by + border + buttonStripOffset)
                    cr.move_to(ulX, ulY)
                    cr.save()
                    if lh > usableHeight: # is the rendered text larger than the usable area ?
                        clipHeight = 0
                        # find last completely visible line
                        cut = False
                        for index in range(0, layout.get_line_count() - 1):
                            lineHeight = layout.get_line(index).get_pixel_extents()[1][3]
                            if clipHeight + lineHeight <= usableHeight:
                                clipHeight = clipHeight + lineHeight
                            else:
                                cut = True # signalize we cut off some lines
                                break

                        textEndY = by + border + clipHeight + buttonStripOffset

                        if cut:
                            # notify the user that a part of the text was cut,
                            # by drawing a red line and a scissors icon

                            # draw the red line
                            cr.set_source_rgb(1, 0, 0)
                            cr.set_line_width(bh * 0.01)
                            cr.move_to(bx, textEndY)
                            cr.line_to(bx + bw, textEndY)
                            cr.stroke()
                            # draw the scissors icon
                            cutSide = bw / 10
                            menus.drawButton(cr, bx + bw, textEndY - cutSide / 2.0, cutSide, cutSide, "",
                                             "center:scissors_right;0>%s" % background,
                                             "turnByTurn:showMessageInsideNotification")
                            #TODO: show the whole message in a notifications after clicking the scissors
                            # (this needs line wrapping support in modRana notifications)

                        # clip out the overflowing part of the text
                        cr.rectangle(ulX, ulY, usableWidth, clipHeight)
                        cr.translate(ulX, ulY)
                        cr.clip()

                    cr.set_source_rgba(*self.navigationBoxText)
                    pg.show_layout(layout)
                    cr.restore()

                    # use the bottom of the infobox to display info
                    (bottomX, bottomY) = (bx, by + bh - 6 * border)
                    if self.routeReached and self._automaticReroutingEnabled():
                        arString = "automatic rerouting enabled"
                    else:
                        arString = "tap this box to reroute"
                    note = "%s/%s, %d/%d    %s" % (
                    currentDistString, routeLengthString, self.currentStepIndex + 1, self.getMaxStepIndex() + 1,
                    arString)
                    menus.drawText(cr, "%s" % note, bottomX, bottomY, bw, 6 * border, 0,
                                   rgbaColor=self.navigationBoxText)
                    # make clickable
                    clickHandler = self.m.get('clickHandler', None)
                    if clickHandler:
                        action = "turnByTurn:reroute"
                        clickHandler.registerXYWH(bx, by + buttonStripOffset, bw, bh - buttonStripOffset, action)

                    # draw the button strip
                    hideButtonWidth = bw * 0.2
                    switchButtonWidth = bw * 0.4

                    # * previous turn button
                    menus.drawButton(cr, bx, by, switchButtonWidth, buttonStripOffset, "",
                                     "center:less;0.1>%s" % background, "turnByTurn:switchToPreviousTurn")
                    # * next turn button
                    menus.drawButton(cr, bx + switchButtonWidth, by, switchButtonWidth, buttonStripOffset, "",
                                     "center:more;0.1>%s" % background, "turnByTurn:switchToNextTurn")
                    # * hide button
                    menus.drawButton(cr, bx + 2 * switchButtonWidth, by, hideButtonWidth, buttonStripOffset, "",
                                     "center:hide;0.1>%s" % background, "turnByTurn:toggleBoxHiding")

    def _automaticReroutingEnabled(self):
        return self.get('reroutingThreshold', REROUTING_DEFAULT_THRESHOLD)

    def sayTurn(self, message, distanceInMeters, forceLanguageCode=False):
        """say a text-to-speech message about a turn
           this basically wraps the simple say method from voice and adds some more information,
           like current distance to the turn
           """
        voice = self.m.get('voice', None)
        units = self.m.get('units', None)

        if voice and units:
            (distString, short, long) = units.humanRound(distanceInMeters)

            if distString == "0":
                distString = ""
            else:
                distString = '
in <emphasis level="strong">' + distString + ' ' + long + '</emphasis>

'
                # TODO: language specific distance strings
            text = distString + message

            #      """ the message can contain unicode, this might cause an exception when printing it
            #      in some systems (SHR-u on Neo, for example)"""
            #      try:
            #        print("saying: %s" % text)
            #        pass
            #      except UnicodeEncodeError:
            #        print("voice: printing the current message to stdout failed do to unicode conversion error")

            if forceLanguageCode:
                espeakLanguageCode = forceLanguageCode
            else:
                # the espeak language code is the first part of this whitespace delimited string
                espeakLanguageCode = self.get('directionsLanguage', 'en en').split(" ")[0]
            return voice.say(text, espeakLanguageCode)

    def getMaxStepIndex(self):
        return self.route.message_point_count - 1

    def getStartingStep(self, which='first'):
        if self.route:
            if which == 'first':
                return self.getStep(0)
            if which == 'closest':
                return self.getClosestStep()

    def getClosestStep(self):
        """get the geographically closest step"""
        proj = self.m.get('projection', None) # we also need the projection module
        pos = self.get('pos', None) # and current position
        if pos and proj:
            (lat1, lon1) = pos
            tempSteps = self.route.message_points
            for step in tempSteps:
                (lat2, lon2) = step.getLL()
                step.currentDistance = geo.distance(lat1, lon1, lat2, lon2) * 1000 # km to m
            closestStep = sorted(tempSteps, key=lambda x: x.currentDistance)[0]

            return closestStep

    def getStep(self, index):
        """return steps for valid index, None otherwise"""
        maxIndex = self.getMaxStepIndex()
        if index > maxIndex or index < -(maxIndex + 1):
            self.log.error("wrong turn index: %d, max index is: %d", index, maxIndex)
            return None
        else:
            return self.route.get_message_point_by_index(index)

    def setStepAsCurrent(self, step):
        """set a given step as current step"""
        mpId = self.route.get_message_point_index(step)
        self._setCurrentStepIndex(mpId)

    def getCurrentStep(self):
        """return current step"""
        return self.route.get_message_point_by_index(self.currentStepIndex)

    def getStepID(self, step):
        return self.route.get_message_point_index(step)

    def getCurrentStepVisitStatus(self):
        """report visit status for  current step"""
        return self.getCurrentStep().visited

    def markCurrentStepAsVisited(self):
        """mark current step as visited"""
        self.getCurrentStep().visited = True

    def switchToPreviousStep(self):
        """switch to previous step and clean up"""
        nextIndex = self.currentStepIndex - 1
        if nextIndex >= 0:
            self._setCurrentStepIndex(nextIndex)
            self.espeakFirstTrigger = False
            self.espeakSecondTrigger = False
            self.log.info("switching to previous step")
        else:
            self.log.info("previous step reached")

    def switchToNextStep(self):
        """switch to next step and clean up"""
        maxIndex = self.getMaxStepIndex()
        nextIndex = self.currentStepIndex + 1
        if nextIndex <= maxIndex:
            self._setCurrentStepIndex(nextIndex)
            self.espeakFirstAndHalfTrigger = False
            self.espeakFirstTrigger = False
            self.espeakSecondTrigger = False
            self.log.info("switching to next step")
        else:
            self.log.info("last step reached")
            self._lastStepReached()

    def _lastStepReached(self):
        """handle all tasks that are needed once the last step is reached"""
        #disable automatic rerouting
        self._stopTBTWorker()
        # automatic rerouting needs to be disabled to prevent rerouting
        # once the destination was reached

    def _setCurrentStepIndex(self, index):
        self.currentStepIndex = index
        self._doNavigationUpdate()

    def enabled(self):
        """return True if enabled, false otherwise"""
        if self.route:
            return True
        else:
            return False

    def startTBT(self, fromWhere='first'):
        """start Turn-by-turn navigation"""

        # clean up any possible previous navigation data
        self.goToInitialState()

        # NOTE: turn and step are used interchangeably in the documentation
        m = self.m.get('route', None)
        if m:
            route = m.get_current_directions()
            if route: # is the route nonempty ?
                self.route = route
                # get route in radians for automatic rerouting
                self.radiansRoute = route.get_points_lle_radians(drop_elevation=True)
                # start rerouting watch
                self._startTBTWorker()

                # show the warning message
                self.sendMessage('ml:notification:m:use at own risk, watch for cliffs, etc.;3')
                # for some reason the combined distance does not account for the last step
                self.mRouteLength = route.length

                # some statistics
                metersPerSecSpeed = self.get('metersPerSecSpeed', None)
                dt = m.route_lookup_duration
                self.log.info("route lookup took: %f s" % dt)
                if dt and metersPerSecSpeed:
                    dm = dt * metersPerSecSpeed
                    self.log.info("distance traveled during lookup: %f m" % dm)
                    # the duration of the road lookup and other variables are currently not used
                # in the heuristics but might be added later to make the heuristics more robust

                # now we decide if we use the closest turn, or the next one,
                # as we might be already past it and on our way to the next turn
                cs = self.getClosestStep() # get geographically closest step
                pos = self.get('pos', None) # get current position
                pReachedDist = int(self.get('pointReachedDistance', 30)) # get the trigger distance
                nextTurnId = self.getStepID(cs) + 1
                nextStep = self.getStep(nextTurnId)
                # check if we have all the data needed for our heuristics
                self.log.info("trying to guess correct step to start navigation")
                if nextStep and pos and pReachedDist:
                    (lat, lon) = pos
                    (csLat, csLon) = cs.getLL()
                    (nsLat, nsLon) = nextStep.getLL()
                    pos2nextStep = geo.distance(lat, lon, nsLat, nsLon) * 1000
                    pos2currentStep = geo.distance(lat, lon, csLat, csLon) * 1000
                    currentStep2nextStep = geo.distance(csLat, csLon, nsLat, nsLon) * 1000
                    #          self.log.debug("pos",(lat,lon))
                    #          self.log.debug("cs",(csLat,csLon))
                    #          self.log.debug("ns",(nsLat,nsLon))
                    self.log.debug("position to next turn: %f m" % pos2nextStep)
                    self.log.debug("position to current turn: %f m" % pos2currentStep)
                    self.log.debug("current turn to next turn: %f m" % currentStep2nextStep)
                    self.log.debug("turn reached trigger distance: %f m" % pReachedDist)

                    if pos2currentStep > pReachedDist:
                        #this means we are out of the "capture circle" of the closest step

                        # what is more distant, the closest or the next step ?
                        if pos2nextStep < currentStep2nextStep:
                            # we are mostly probably already past the closest step,
                            # so we switch to the next step at once
                            self.log.debug("already past closest turn, switching to next turn")
                            self.setStepAsCurrent(nextStep)
                            # we play the message for the next step,
                            # with current distance to this step,
                            # to assure there is some voice output immediately after
                            # getting a new route or rerouting"""
                            plaintextMessage = nextStep.ssmlMessage
                            self.sayTurn(plaintextMessage, pos2nextStep)
                        else:
                            # we have probably not yet reached the closest step,
                            # so we start navigation from it
                            self.log.debug("closest turn not yet reached")
                            self.setStepAsCurrent(cs)

                    else:
                        # we are inside the  "capture circle" of the closest step,
                        # this means the navigation will trigger the voice message by itself
                        # and correctly switch to next step
                        # -> no need to switch to next step from here
                        self.log.debug("inside reach distance of closest turn")
                        self.setStepAsCurrent(cs)

                else:
                    # we dont have some of the data, that is needed to decide
                    # if we start the navigation from the closest step of from the step that is after it
                    # -> we just start from the closest step
                    self.log.debug("not enough data to decide, using closest turn")
                    self.setStepAsCurrent(cs)
        self._doNavigationUpdate() # run a first time navigation update
        self.locationWatchID = self.watch('locationUpdated', self.locationUpdateCB)
        self.log.info("started and ready")

    def stopTBT(self):
        """stop Turn-by-turn navigation"""
        # remove location watch
        if self.locationWatchID:
            self.removeWatch(self.locationWatchID)
            # cleanup
        self.goToInitialState()
        self._stopTBTWorker()
        self.log.info("stopped")

    def locationUpdateCB(self, key, newValue, oldValue):
        """position changed, do a tbt navigation update"""
        if key == "locationUpdated": # just to be sure
            self._doNavigationUpdate()
        else:
            self.log.error("invalid key: %r", key)

    def _doNavigationUpdate(self):
        """do a navigation update"""
        # make sure there really are some steps
        if not self.route:
            self.log.error("no route")
            return
        pos = self.get('pos', None)
        if pos is None:
            self.log.error("skipping update, invalid position")
            return

        # get/compute/update necessary the values
        (lat1, lon1) = pos
        currentStep = self.getCurrentStep()
        lat2, lon2 = currentStep.getLL()
        currentDistance = geo.distance(lat1, lon1, lat2, lon2) * 1000 # km to m
        self.currentDistance = currentDistance # update current distance

        # use some sane minimum distance
        distance = int(self.get('minAnnounceDistance', 100))

        # GHK: make distance speed-sensitive
        #
        # I came up with this formula after a lot of experimentation with
        # gnuplot.  The idea is to give the user some simple parameters to
        # adjust yet let him have a lot of control.  There are five
        # parameters in the equation:
        #
        # lowSpeed	Speed below which the pre-announcement time is constant.
        # lowTime	Announcement time at and below lowSpeed.
        # highSpeed	Speed above which the announcement time is constant.
        # highTime	Announcement time at and above highSpeed.
        # power	Exponential power used in the formula; good values are 0.5-5
        #
        # The speeds are in m/s.  Obviously highXXX must be greater than lowXXX.
        # If power is 1.0, announcement times increase linearly above lowSpeed.
        # If power < 1.0, times rise rapidly just above lowSpeed and more
        # gradually approaching highSpeed.  If power > 1.0, times rise
        # gradually at first and rapidly near highSpeed.  I like power > 1.0.
        #
        # The reasoning is that at high speeds you are probably on a
        # motorway/freeway and will need extra time to get into the proper
        # lane to take your exit.  That reasoning is pretty harmless on a
        # high-speed two-lane road, but it breaks down if you are stuck in
        # heavy traffic on a four-lane freeway (like in Los Angeles
        # where I live) because you might need quite a while to work your
        # way across the traffic even though you're creeping along.  But I
        # don't know a good way to detect whether you're on a multi-lane road,
        # I chose speed as an acceptable proxy.
        #
        # Regardless of speed, we always warn a certain distance ahead (see
        # "distance" above).  That distance comes from the value in the current
        # step of the directions.
        #
        # BTW, if you want to use gnuplot to play with the curves, try:
        # max(a,b) = a > b ? a : b
        # min(a,b) = a  mps speed > point reached distance
            # also enlarge the rerouting threshold as it looks like it needs to be larger
            # when moving at high speed to prevent unnecessary rerouting
            if metersPerSecSpeed > pointReachedDistance * 0.75:
                pointReachedDistance = metersPerSecSpeed * 2
            #        self.log.debug("tbt: enlarging point reached distance to: %1.2f m due to large speed (%1.2f m/s)". (pointReachedDistance, metersPerSecSpeed)

            if metersPerSecSpeed > INCREASE_REROUTING_THRESHOLD_SPEED:
                self.reroutingThresholdMultiplier = REROUTING_THRESHOLD_MULTIPLIER
            else:
                self.reroutingThresholdMultiplier = 1.0

            # speed & time based triggering
            lowSpeed = float(self.get('minAnnounceSpeed', 13.89))
            highSpeed = float(self.get('maxAnnounceSpeed', 27.78))
            highSpeed = max(highSpeed, lowSpeed + 0.1)
            lowTime = int(self.get('minAnnounceTime', 10))
            highTime = int(self.get('maxAnnounceTime', 60))
            highTime = max(highTime, lowTime)
            power = float(self.get('announcePower', 2.0))
            warnTime = (max(lowSpeed, metersPerSecSpeed) - lowSpeed) ** power \
                       * (highTime - lowTime) / (highSpeed - lowSpeed) ** power \
                       + lowTime
            warnTime = min(highTime, warnTime)
            distance = max(distance, warnTime * metersPerSecSpeed)

            if self.get('debugTbT', False):
                self.log.debug("#####")
                self.log.debug("min/max announce time: %d/%d s", lowTime, highTime)
                self.log.debug("trigger distance: %1.2f m (%1.2f s warning)", distance, distance / float(metersPerSecSpeed))
                self.log.debug("current distance: %1.2f m", currentDistance)
                self.log.debug("current speed: %1.2f m/s (%1.2f km/h)", metersPerSecSpeed, metersPerSecSpeed * 3.6)
                self.log.debug("point reached distance: %f m", pointReachedDistance)
                self.log.debug("1. triggered=%r, 1.5. triggered=%r, 2. triggered=%r",
                               self.espeakFirstTrigger, self.espeakFirstAndHalfTrigger, self.espeakSecondTrigger)
                if warnTime > 30:
                    self.log.debug("optional (20 s) trigger distance: %1.2f", 20.0 * metersPerSecSpeed)

            if currentDistance <= pointReachedDistance:
                # this means we reached the point"""
                if self.espeakSecondTrigger == False:
                    self.log.debug("triggering espeak nr. 2")
                    # say the message without distance
                    plaintextMessage = currentStep.ssmlMessage
                    # consider turn said even if it was skipped (ignore errors)
                    self.sayTurn(plaintextMessage, 0)
                    self.markCurrentStepAsVisited() # mark this point as visited
                    self.espeakFirstTrigger = True # everything has been said, again :D
                    self.espeakSecondTrigger = True # everything has been said, again :D
                self.switchToNextStep() # switch to next step
            else:
                if currentDistance <= distance:
                    # this means we reached an optimal distance for saying the message"""
                    if self.espeakFirstTrigger == False:
                        self.log.debug("triggering espeak nr. 1")
                        plaintextMessage = currentStep.ssmlMessage
                        if self.sayTurn(plaintextMessage, currentDistance):
                            self.espeakFirstTrigger = True # first message done
                if self.espeakFirstAndHalfTrigger == False and warnTime > 30:
                    if currentDistance <= (20.0 * metersPerSecSpeed):
                        # in case that the warning time gets too big, add an intermediate warning at 20 seconds
                        # NOTE: this means it is said after the first trigger
                        plaintextMessage = currentStep.ssmlMessage
                        if self.sayTurn(plaintextMessage, currentDistance):
                            self.espeakFirstAndHalfTrigger = True # intermediate message done

            ## automatic rerouting ##

            # is automatic rerouting enabled from options
            # enabled == threshold that is not not None
            if self._automaticReroutingEnabled():
                # rerouting is enabled only once the route is reached for the first time
                if self.onRoute and not self.routeReached:
                    self.routeReached = True
                    self.automaticRerouteCounter = 0
                    self.log.info('route reached, rerouting enabled')

                # did the TBT worker detect that the rerouting threshold was reached ?
                if self._reroutingConditionsMet():
                    # test if enough consecutive divergence point were recorded
                    if self.reroutingThresholdCrossedCounter >= REROUTING_TRIGGER_COUNT:
                        # reset the routeReached override
                        self.overrideRouteReached = False
                        # trigger rerouting
                        self._rerouteAuto()
                else:
                    # reset the counter
                    self.reroutingThresholdCrossedCounter = 0

    def _reroutingConditionsMet(self):
        return (self.routeReached or self.overrideRouteReached) and not self.onRoute

    def _followingRoute(self):
        """are we still following the route or is rerouting needed"""
        start1 = time.clock()
        pos = self.get('pos', None)
        proj = self.m.get('projection', None)
        if pos and proj:
            pLat, pLon = pos
            # we use Radians to get rid of radian conversion overhead for
            # the geographic distance computation method
            radiansLL = self.radiansRoute
            pLat = geo.radians(pLat)
            pLon = geo.radians(pLon)
            if len(radiansLL) == 0:
                self.log.error("Divergence: can't follow a zero point route")
                return False
            elif len(radiansLL) == 1: # 1 point route
                aLat, aLon = radiansLL[0]
                minDistance = geo.distanceApproxRadians(pLat, pLon, aLat, aLon)
            else: # 2+ points route
                aLat, aLon = radiansLL[0]
                bLat, bLon = radiansLL[1]
                minDistance = geo.distancePointToLineRadians(pLat, pLon, aLat, aLon, bLat, bLon)
                aLat, aLon = bLat, bLon
                for point in radiansLL[1:]:
                    bLat, bLon = point
                    dist = geo.distancePointToLineRadians(pLat, pLon, aLat, aLon, bLat, bLon)
                    if dist < minDistance:
                        minDistance = dist
                    aLat, aLon = bLat, bLon
                # the multiplier tries to compensate for high speed movement
            threshold = float(
                self.get('reroutingThreshold', REROUTING_DEFAULT_THRESHOLD)) * self.reroutingThresholdMultiplier
            self.log.debug("Divergence from route: %1.2f/%1.2f m computed in %1.0f ms",
            minDistance * 1000, float(threshold), (1000 * (time.clock() - start1)))
            return minDistance * 1000 < threshold

    def _startTBTWorker(self):
        self.log.info("starting worker thread")
        startThread = True
        if not self.TBTWorker: # reuse previous thread or start new one
            self.TBTWorkerEnabled = True
            t = Thread(target=self._TBTWorker)
            t.daemon = True
            t.start()
            self.TBTWorker = t
        else:
            self.log.info("reusing worker thread")

    def _stopTBTWorker(self):
        self.TBTWorkerEnabled = False
        self.TBTWorker = None

    def _TBTWorker(self):
        """this function is run in its own thread and check if
        we are following the current route"""
        self.log.info("TBTWorker: started")
        while self.route and self.TBTWorkerEnabled:
            # first make sure automatic rerouting is enabled
            # eq. reroutingThreshold != None
            if self._automaticReroutingEnabled():
            # check if we are still following the route
            #        self.log.debug('TBTWorker: checking divergence from route')
                self.onRoute = self._followingRoute()
                if self._reroutingConditionsMet():
                    self.log.info('TBTWorker: divergence detected')
                    # switch to quick updates
                    for i in range(0, REROUTING_TRIGGER_COUNT + 1):
                        time.sleep(1)
                        onRoute = self._followingRoute()
                        if onRoute: # divergence stopped
                            self.onRoute = onRoute
                            self.log.info('TBTWorker: false alarm')
                            break
                        else: # still diverging from current route
                            self.onRoute = onRoute
                            # increase divergence counter
                            self.reroutingThresholdCrossedCounter += 1
                            self.log.debug('TBTWorker: increasing divergence counter (%d)',
                                           self.reroutingThresholdCrossedCounter)
            time.sleep(REROUTE_CHECK_INTERVAL / 1000.0)
        self.log.info("TBTWorker: shutting down")

    def shutdown(self):
        # cleanup
        self.stopTBT()


import pygame
from numpy import random as rnd

class Mamam():

  # inisialisasi atribut pada class Mamam
  def __init__(self, arena, color=(255, 0, 0), name="mamam"):
    self.name = name
    food_pos_x = rnd.randint(0, arena.get_column_sum() - 1) # mamam position
    food_pos_y = rnd.randint(0, arena.get_row_sum() - 1) # mamam position
    self.pos = (food_pos_x, food_pos_y)
    self.color = color
    self.surface = arena.get_surface()
    self.width = arena.get_column_distance()
    self.height = arena.get_row_distance()
    self.arena = arena
    arena.assign_member(self)

  # getter untuk posisi
  def get_pos(self):
    return self.pos

  # method untuk mengubah posisi makanan
  def change_pos(self):
    food_pos_x = rnd.randint(0, self.arena.get_column_sum() - 1)
    food_pos_y = rnd.randint(0, self.arena.get_row_sum() - 1)
    self.pos = (food_pos_x, food_pos_y)

  # method untuk merender objek makanan
  def draw(self):
    start_x = self.width * self.pos[0]
    start_y = self.height * self.pos[1]
    pygame.draw.rect(self.surface, self.color, (start_x, start_y, self.width, self.height))

  # memberikan representasi string
  def __repr__(self):
    return self.name


"""Implementation of various utilities used in ModularCMA-ES package."""

import warnings
import typing
from inspect import Signature, Parameter, getmodule
from functools import wraps
from time import time

import numpy as np

class Descriptor:
    """Data descriptor."""

    def __set_name__(self, owner, name):
        """Set name attribute."""
        self.name = name

    def __set__(self, instance, value):
        """Set value on instance."""
        instance.__dict__[self.name] = value

    def __delete__(self, instance):
        """Delete attribute from the instance __dict__."""
        del instance.__dict__[self.name]

class InstanceOf(Descriptor):
    """Data descriptor checks for correct types."""

    def __init__(self, dtype):
        """Set dtype."""
        self.dtype = dtype

    def __set__(self, instance, value):
        """Set the value of instance to value, checks type of argument.

        Raises
        ------
        TypeError
            If type of the argument does not match self.dtype

        """
        if type(value) != type(None):

            if (
                type(value) != self.dtype
                and not (
                    isinstance(value, np.generic) and type(value.item()) == self.dtype
                )
                and str(self.dtype)[1:] != value.__class__.__name__
            ):
                raise TypeError(
                    "{} should be {} got type {}: {}".format(
                        self.name, self.dtype, type(value), str(value)[:50]
                    )
                )
            if hasattr(value, "__copy__"):
                value = value.copy()
        super().__set__(instance, value)

class AnyOf(Descriptor):
    """Descriptor, checks of value is Any of a specified sequence of options."""

    def __init__(self, options=None):
        """Set options."""
        self.options = options

    def __set__(self, instance, value):
        """Set the value of instance to value, checks value of argument to match self.options.

        Raises
        ------
        TypeError
            If type of the argument does not match self.dtype

        """
        if value not in self.options:
            raise ValueError(
                "{} should be any of [{}]. Got: {}".format(
                    self.name, self.options, value)
            )
        super().__set__(instance, value)

class AnnotatedStructMeta(type):
    """Metaclass for class for AnnotatedStruct.

    Wraps all parameters defined in the class body with
    __annotations__ into a signature. It additionally wraps each
    parameter into a descriptor using __annotations__,
    allowing for type checking.
    Currently, only two types of descriptors are implementated,
    InstanceOf and typing.AnyOf, the first implements simple type validation,
    the latter implements validation though the use of sequence of
    allowed values.
    """

    def __new__(cls: typing.Any, name: str, bases: tuple, attrs: dict) -> typing.Any:
        """Control instance creation of classes that have AnnotatedStructMeta as metaclass.

        All cls attributes that are defined in __annotations__ are wrapped
        into either an typing.AnyOf or an InstanceOf descriptor, depending on
        the type of the annotation. If the annotation is a sequence, the first
        element is used as a default value.

        Parameters
        ----------
        cls: typing.Any
            The empty body of the class to be instantiated
        name: str
            The name of the cls
        bases: dict
            The base classes of the cls
        attrs: dict
            The attributes of the cls

        Returns
        -------
        A new cls object

        """
        parameters = []
        for key, annotation in attrs.get("__annotations__", {}).items():
            default_value = attrs.get(key, Parameter.empty)

            if isinstance(annotation, (list, tuple)):
                attrs[key] = AnyOf(annotation)
            else:
                if (
                    not type(annotation) == type
                    and getmodule(type(annotation)) != typing
                ):
                    raise TypeError(
                        f"Detected wrong format for annotations of AnnotatedStruct.\n\t"
                        f"Format should be <name>: <type> = <default_value>\n\t"
                        f"Got: {name}: {annotation} = {default_value}"
                    )
                attrs[key] = InstanceOf(annotation)
            parameters.append(
                Parameter(
                    name=key,
                    default=default_value,
                    kind=Parameter.POSITIONAL_OR_KEYWORD,
                )
            )

        clsobj = super().__new__(cls, name, bases, attrs)
        setattr(clsobj, "__signature__", Signature(parameters=parameters))
        return clsobj

class AnnotatedStruct(metaclass=AnnotatedStructMeta):
    """Custom class for defining structs.

    Automatically sets parameters defined in the signature.
    AnnotatedStruct objects, and children thereof, require
    the following structure:
        class Foo(AnnotatedStruct):
            variable_wo_default : type
            variable_w_default  : type = value

    The metaclass will automatically assign a decriptor object
    to every variable, allowing for type checking.
    The init function will be dynamically generated, and user specified values
    in the *args **kwargs, will override the defaults.
    The *args will follow the order as defined in the class body:
        i.e. (variable_wo_default, variable_w_default,)

    Attributes
    ----------
    __signature__: Signature
        The calling signature, instantiated by the metaclass
    __bound__ : Signature
        The bound signature, bound to the *args and **kwargs

    """

    def __init__(self, *args, **kwargs) -> None:
        """Bind *args and **kwargs to a signature instantiated by the metaclass."""
        self.__bound__ = self.__signature__.bind(*args, **kwargs)
        self.__bound__.apply_defaults()
        for name, value in self.__bound__.arguments.items():
            setattr(self, name, value)

    def __repr__(self) -> str:
        """Representation for a AnnotatedStruct object."""
        return "<{}: ({})>".format(
            self.__class__.__qualname__,
            ", ".join(
                "{}={}".format(name, getattr(self, name))
                for name, value in self.__bound__.arguments.items()
            ),
        )

    def set_default(self, name: str, default_value: typing.Any) -> None:
        """Helper method to set default parameters."""
        current = getattr(self, name)
        if type(current) == type(None):
            setattr(self, name, default_value)

def timeit(func):
    """Decorator function for timing the excecution of a function.

    Parameters
    ----------
    func: typing.Callable
        The function to be timed

    Returns
    -------
    typing.Callable
        a wrapped function

    """
    @wraps(func)
    def inner(*args, **kwargs):
        start = time()
        res = func(*args, **kwargs)
        print("Time elapsed", time() - start)
        return res

    return inner

def ert(evals, budget):
    """Computed the expected running time of a list of evaluations.

    Parameters
    ----------
    evals: list
        a list of running times (number of evaluations)
    budget: int
        the maximum number of evaluations

    Returns
    -------
    float
        The expected running time

    float
        The standard deviation of the expected running time
    int
        The number of successful runs

    """
    try:
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            evals = np.array(evals)
            n_succ = (evals < budget).sum()
            _ert = float(evals.sum()) / int(n_succ)
        return _ert, np.std(evals), n_succ
    except ZeroDivisionError:
        return float("inf"), np.nan, 0


# basicsqlite3.py

import sqlite3

# สร้าง database
conn = sqlite3.connect('expense.db') # หรือ .sqlite3
# สร้างตัวดำเนินการหรือ execution (อยากได้อะไรใช้ตัวนี้ได้เลย)
c = conn.cursor()

# สร้าง table ด้วยภาษา sql
'''['รหัสรายการ (transactionid)' TEXT,
'วัน-เวลา(datetime)' TEXT,
'รายการ(title)' TEXT,
'ค่าใช้จ่าย(expense)' REAL (float),
'จำนวน(quantity)' INTEGER,
'รวม(total)' REAL]
'''

# ถ้าหลายบรรทัดเมื่อไรต้องใช้ """__""" ไม่สามารถใช้ "_" ได้
# transaction is sqlite keyword

c.execute("""CREATE TABLE IF NOT EXISTS expenselist(
				ID INTEGER PRIMARY KEY AUTOINCREMENT,
				transactionid TEXT,
				datetime TEXT,
				title TEXT,
				expense REAL,
				quantity INTEGER,
				total REAL
			)""")

def insert_expense(transactionid,datetime,title,expense,quantity,total):
	ID = None # ID คือตัวที่ 7 ของ ?
	with conn:
		c.execute("""INSERT INTO expenselist VALUES (?,?,?,?,?,?,?)""",
			(ID,transactionid,datetime,title,expense,quantity,total))
		conn.commit() # การบันทึกข้อมูลลงในฐานข้อมูล ถ้าไม่รันตัวนี้จะไม่บันทึก
		print('Insert Success!')

def show_expense():
	with conn:
		c.execute("SELECT * FROM expenselist")
		expense = c.fetchall() # คำสั่งให้ดึงข้อมูลเข้ามาใส่ในตัวแปรที่ชื่อ expense
		print(expense)

	return expense

def update_expense(transactionid,title,expense,quantity,total):   # expense,quantity,total):
	with conn:
		c.execute("""UPDATE expenselist SET
			title=?,
			expense=?,
			quantity=?,
			total=? WHERE transactionid=?""",
			([title,expense,quantity,total,transactionid]))
	conn.commit()
	print('Data updated')

def delete_expense(transactionid):
	with conn:
		c.execute("DELETE FROM expenselist WHERE transactionid=(?)",([transactionid]))
	conn.commit()
	print('Data deleted')

# insert_expense('20213234456','วันเสาร์ 2021-06-19','ข้าวสาร',45,2,90) # - C - CREATE
# show_expense() # - R - READ
# update_expense('20213234456','omelette',50,2,100) # - U - UPDATE
# delete_expense('20213234457') # - D - DELETE

show_expense()

print('success')


import torch
import torch.nn as nn
from typing import List
from timm.models.registry import register_model

class YourConvNet(nn.Module):
    """
    This is a template for your custom ConvNet.
    It is required to implement the following three functions: `get_downsample_ratio`, `get_feature_map_channels`, `forward`.
    You can refer to the implementations in `pretrain\models\resnet.py` for an example.
    """

    def get_downsample_ratio(self) -> int:
        """
        This func would ONLY be used in `SparseEncoder's __init__` (see `pretrain/encoder.py`).

        :return: the TOTAL downsample ratio of the ConvNet.
        E.g., for a ResNet-50, this should return 32.
        """
        raise NotImplementedError

    def get_feature_map_channels(self) -> List[int]:
        """
        This func would ONLY be used in `SparseEncoder's __init__` (see `pretrain/encoder.py`).

        :return: a list of the number of channels of each feature map.
        E.g., for a ResNet-50, this should return [256, 512, 1024, 2048].
        """
        raise NotImplementedError

    def forward(self, inp_bchw: torch.Tensor, hierarchical=False):
        """
        The forward with `hierarchical=True` would ONLY be used in `SparseEncoder.forward` (see `pretrain/encoder.py`).

        :param inp_bchw: input image tensor, shape: (batch_size, channels, height, width).
        :param hierarchical: return the logits (not hierarchical), or the feature maps (hierarchical).
        :return:
            - hierarchical == False: return the logits of the classification task, shape: (batch_size, num_classes).
            - hierarchical == True: return a list of all feature maps, which should have the same length as the return value of `get_feature_map_channels`.
              E.g., for a ResNet-50, it should return a list [1st_feat_map, 2nd_feat_map, 3rd_feat_map, 4th_feat_map].
                    for an input size of 224, the shapes are [(B, 256, 56, 56), (B, 512, 28, 28), (B, 1024, 14, 14), (B, 2048, 7, 7)]
        """
        raise NotImplementedError

@register_model
def your_convnet_small(pretrained=False, **kwargs):
    raise NotImplementedError
    return YourConvNet(**kwargs)

@torch.no_grad()
def convnet_test():
    from timm.models import create_model
    cnn = create_model('your_convnet_small')
    print('get_downsample_ratio:', cnn.get_downsample_ratio())
    print('get_feature_map_channels:', cnn.get_feature_map_channels())

    downsample_ratio = cnn.get_downsample_ratio()
    feature_map_channels = cnn.get_feature_map_channels()

    # check the forward function
    B, C, H, W = 4, 3, 224, 224
    inp = torch.rand(B, C, H, W)
    feats = cnn(inp, hierarchical=True)
    assert isinstance(feats, list)
    assert len(feats) == len(feature_map_channels)
    print([tuple(t.shape) for t in feats])

    # check the downsample ratio
    feats = cnn(inp, hierarchical=True)
    assert feats[-1].shape[-2] == H // downsample_ratio
    assert feats[-1].shape[-1] == W // downsample_ratio

    # check the channel number
    for feat, ch in zip(feats, feature_map_channels):
        assert feat.ndim == 4
        assert feat.shape[1] == ch

if __name__ == '__main__':
    convnet_test()