import asyncio
import json
import threading
import time
import wave
import queue
from dataclasses import dataclass
from typing import Any, Callable, List, Union

import numpy as np
import pvporcupine
import vosk
from maglevapi.basic import quick_kwargs
from pyaudio_mixer import InputTrack
from pyaudio_mixer.utils import BasicFX
from speech_recognition import AudioFile, Recognizer, UnknownValueError
from tensorflow.keras.models import load_model

from .constants import *

@dataclass
class Result:
    """
    Result Types
    ------------
    `active_listening` :
        What is being heard at all times. Text data is written to `payload`.
    `woke` :
        Once the wakeword has been recognized (or .wake_up() has been called manually). Used wakeword is written to `payload`, could be none.
    `error` :
        When an error has been caught. Original exception is written to `exception`.
    `recognized` :
        When the phrase has been fully recognized. Text data is written to `payload`. Audio data is written to `audio_data` as a ndarray.
    `unknown` :
        When the recognized phrase is unknown.
    `realtime_recognition` :
        What the recognizer is currently recognizing (realtime speech recognition). Text data is written to `payload`
    `voice_activity` :
        Whether there is voice activity going on. Probability of a voice activity is written to `payload`
    """

    type: str
    payload: Any = None
    exception: Exception = None
    audio_data: np.ndarray = None

    @property
    def raw_audio_data(self) -> bytes:
        if self.audio_data is None:
            return
        return self.audio_data.tobytes()

class VAD:
    """
    Voice Activity Detection.

    Parameters
    ----------
    `model` : str
        Path to the .h5 file of this voice activity detection model.
    `threshold` : float
        Sensitivity threshold.

    Methods
    -------
    `async def detect(data: bytes)` :
        returns True if the classified data is a voice, otherwise False.
    """

    def __init__(self, model: str, threshold: float = 0.8) -> None:
        self.model = load_model(model)
        self.threshold = threshold

        self.buffer = []

    def format_predictions(self, preds: list) -> list:
        predictions = [[i, float(r)] for i, r in enumerate(preds)]
        predictions.sort(key=lambda x: x[1], reverse=True)
        return predictions

    async def detect(self, data: np.ndarray) -> bool:
        data = np.frombuffer(data.tobytes(), dtype="int16")

        # Get the decibel values
        dbs = 20 * np.log10(np.abs(np.fft.rfft(data[:2048])))

        # Collect the decibel values  for the frequencies of interest and round them to 2
        features = list(np.round(dbs[3:VAD_FREQUENCIES], 2))
        self.buffer.append(features)

        if len(self.buffer) == int(SAMPLERATE / CHUNK_SIZE * VAD_DURATION):
            total = np.array([x for y in self.buffer for x in y])
            self.buffer.clear()

            # Make the predictions
            predictions = self.model(np.array([total]))[0]
            predictions = self.format_predictions(predictions)

            idx, prob = predictions[0]
            if idx == 1 and prob >= self.threshold:
                return (True, prob)
        return (False, 0)

class VRS:
    """
    Initializes the VRS (Voice Recognition Software)

    Notes
    -----
    - It is required for you to install nest_asyncio and patch your asyncio via `nest_asyncio.apply()`.

    Parameters
    ----------
    `vad` : VAD
        Voice activity detection class.
    `vosk_path` : str
        Path to the vosk model.
    `input_tracks` : List[InputTrack]
        List of input track to get audio data from.
    `wakewords` : List[str]
        List of wakewords to recognize.
    `callback` : Callable
        Coroutine used to get results.
    `loop` : AbstractEventLoop
        Asyncio event loop.
    `save_path` : str
        Where to save the final file that will be recognized via google speech recognizer. File must be a .wav format (e.g, "temporary_speech.wav")
    `wakeword_sensitivity` : float
        Sensitivity for the wakewords. Defaults to 0.5.
    `logging` : Logger
        Logger object from py-prettylog. Defaults to None. If provided, all logs coming from the VRS will be in a group called "vrs"
    `output_track` : OutputTrack
        If provided, every audio data coming from the input tracks will be redirected here. Defaults to None.
    `disable_vosk` : bool
        Whether to disable the vosk recognizer. Defaults to False.
    `speech_lengths` : Tuple[float, float]
        How many seconds of speech has to occur before cutting off. First tuple represents the initial
    `offline` : bool
        Whether to use the vosk recognizer only (i.e, we're running offline and don't want to send the audio file to google's speech recognition). Defaults to False.
    `google_recognizer_key` : str
        Google speech recognition key. Defaults to None which is the default key the SpeechRecognition package uses.
    `preprocessing_chain` : dict
        Key, Value pair effects where in key is the effect function from the `BasicFX` class and the value must be a dictionary that acts as the parameter to that effect function.
    """

    def __init__(
        self,
        vad: VAD,
        vosk_path: str,
        input_tracks: List[InputTrack],
        wakewords: List[str],
        callback: Callable,
        loop: asyncio.AbstractEventLoop,
        save_path: str,
        **kwargs
    ) -> None:

        # Required Parameters
        self.vad = vad
        self.vosk_path = vosk_path
        self.input_tracks = input_tracks if isinstance(
            input_tracks, list) else [input_tracks]
        self.wakewords = wakewords if isinstance(
            wakewords, list) else [wakewords]
        self._callback = callback
        self.loop = loop
        self.save_path = save_path

        # Kwargs
        self.wakeword_sensitivity, \
            self.logging, \
            self.output_track, \
            self.disable_vosk, \
            self.speech_lengths, \
            self.offline, \
            self.google_recognizer_key, \
            self.preprocessing_chain \
            = quick_kwargs(
                ("wakeword_sensitivity", 0.5),
                ("logging", None),
                ("output_track", None),
                ("disable_vosk", False),
                ("speech_lengths", (6.0, 0.9)),
                ("offline", False),
                ("google_recognizer_key", None),
                ("preprocessing_chain", {}),
                kwargs=kwargs
            )

        # Initialize pvporcupine (accurate wakeword recognizer)
        self.porcupine = None
        if any(kw in pvporcupine.KEYWORDS for kw in self.wakewords):
            _kws = [x for x in self.wakewords if x in pvporcupine.KEYWORDS]
            self.porcupine = pvporcupine.create(
                keywords=_kws,
                sensitivities=[self.wakeword_sensitivity] * len(_kws)
            )
        self.wakewords = set(self.wakewords)

        # Initialize the vosk recognizer (realtime offline speech recognition)
        if not self.disable_vosk:
            self.vosk_model = vosk.Model(self.vosk_path)
        self.vosk = None
        self.restart_vosk()

        # Initialize the speechrecognition module. This module is used for the final speech recognition.
        self.sr_recognizer = Recognizer()

        # Initialize the list of available effects (for preprocessing)
        self.preprocessor = BasicFX(
            dtype=self.input_tracks[0].stream.dtype,
            samplerate=self.input_tracks[0].stream.samplerate
        )
        if self.preprocessing_chain:
            for _e in self.preprocessing_chain.keys():
                if _e not in [x.__name__ for x in self.preprocessor.effects]:
                    raise ValueError(f"unknown effect '{_e}'")

        ######
        self.listen = True
        self.stopped = True
        self.callback_params = {}
        self.q = queue.Queue(maxsize=15)
        self.__woke = False
        self.__stop_signal = False
        self.__last_vosk_reset = time.time()
        self.__to_be_recognized = []
        self.__last_va = None
        self.__last_recognized = time.time()

    def restart_vosk(self) -> None:
        if not self.disable_vosk:
            self.vosk = vosk.KaldiRecognizer(self.vosk_model, SAMPLERATE)

    def read_tracks(self) -> np.ndarray:
        try:
            return self.input_tracks[0].read()
        except AttributeError:

            if self.logging:
                self.logging.warning(
                    "Unknown error 'AttributeError' occurred.", groups=LOGGING_GROUP)

            return

    def reset_variables(self) -> None:
        self.__woke = False
        self.__last_vosk_reset = time.time()
        self.__last_va = None
        self.__to_be_recognized.clear()
        self.callback_params = {}

        if not self.disable_vosk:
            self.vosk.FinalResult()

    def __input_streamer__(self) -> None:
        while not self.__stop_signal:
            data = self.read_tracks()
            if data is not None:
                try:
                    self.q.put(data, block=False)
                except queue.Full:
                    pass

            time.sleep(0.0001)

        for _ in range(10):
            try:
                self.q.put(None, block=False)
            except queue.Full:
                pass

    def __preprocess_data(self, data: np.ndarray) -> np.ndarray:

        for effect in self.preprocessor.effects:
            params = self.preprocessing_chain.get(effect.__name__)
            if params is not None:
                if effect.__name__ not in ["noise_reduction"] and len(data) > 2048:
                    _buffer = []
                    for d in np.array_split(data, len(data) / 2048):
                        _buffer.append(effect(d, **params))
                    data = np.concatenate(_buffer)
                else:
                    data = effect(data, **params)

        return data

    async def callback(self, *args, **kwargs) -> None:
        await self._callback(*args, **kwargs, **self.callback_params)

    async def start(self) -> None:
        if self.logging:
            self.logging.debug("Starting VRS...", group=LOGGING_GROUP)

        def f():
            self.loop.run_until_complete(self.__vrs__())

        threading.Thread(target=f, daemon=True).start()
        threading.Thread(target=self.__input_streamer__, daemon=True).start()

        if self.logging:
            self.logging.debug(
                "Waiting for VRS to be ready...", group=LOGGING_GROUP)

        while self.stopped:
            await asyncio.sleep(0.001)

        if self.logging:
            self.logging.success(
                "VRS Started successfully.", group=LOGGING_GROUP)

    async def stop(self, blocking: bool = True) -> None:
        if self.logging:
            self.logging.debug("Stopping the VRS...", group=LOGGING_GROUP)
        self.__stop_signal = True

        if blocking:
            while not self.stopped:
                await asyncio.sleep(0.01)

        if self.logging:
            self.logging.success(
                "VRS Stopped successfully", group=LOGGING_GROUP)

    async def wait_for_wakeword(self, data: np.ndarray) -> Union[None, str]:
        wakeword_used = None
        if self.porcupine:
            idx = self.porcupine.process(data)
            if idx > -1:
                wakeword_used = [
                    x for x in self.wakewords if x in pvporcupine.KEYWORDS][idx]

        if not self.disable_vosk:
            _as_bytes = data.tobytes()
            __reset = False
            if len(_as_bytes) != 0:
                self.vosk.AcceptWaveform(_as_bytes)
                __partial = json.loads(self.vosk.PartialResult())

                if __partial["partial"]:
                    await self.callback(Result("active_listening", __partial["partial"]))

                    if len(__partial["partial"].split()) > 25:
                        __reset = True

                if (time.time() - self.__last_vosk_reset) > 30.0:
                    self.__last_vosk_reset = time.time()
                    __reset = True

                if not wakeword_used:
                    ___wakeword = [
                        x for x in self.wakewords if x in __partial["partial"]]
                    if ___wakeword:
                        wakeword_used = ___wakeword[0]

            if __reset:
                self.vosk.FinalResult()
                self.restart_vosk()

        if wakeword_used:
            if not self.disable_vosk:
                self.vosk.FinalResult()

        return wakeword_used

    async def wake_up(self, wakeword: str = None, **kwargs) -> None:
        result = Result("woke", wakeword)
        self.__woke = True

        self.callback_params = kwargs
        await self.callback(result)

    async def fully_recognize(self) -> None:
        self.__last_recognized = time.time()

        if not self.__to_be_recognized:
            return await self.callback(Result("error", exception=ValueError("empty __to_be_recognized variable.")))

        preprocessed = self.__preprocess_data(np.concatenate(self.__to_be_recognized))

        # Match data types if they do not match
        if preprocessed.dtype != self.input_tracks[0].stream.dtype:
            preprocessed = preprocessed.astype(self.input_tracks[0].stream.dtype)

        if self.offline:
            if not self.disable_vosk:
                self.vosk.FinalResult()
                self.vosk.AcceptWaveform(preprocessed.tobytes())

                result = Result("recognized", json.loads(
                    self.vosk.FinalResult())["text"], audio_data=preprocessed)
                return await self.callback(result)
            return await self.callback(Result("error", exception=ValueError("vosk should not be disabled if offline is True")))

        # Save the file and recognize it using the SpeechRecognizer library (more accurate since it uses google's)
        itrack = self.input_tracks[0]
        wf = wave.open(self.save_path, "wb")
        wf.setnchannels(itrack.stream.channels)
        wf.setsampwidth(2)
        wf.setframerate(itrack.stream.samplerate)
        wf.writeframes(preprocessed.tobytes())
        wf.close()

        # Convert it into a AudioFile that the SpeechRecognizer module can use
        try:
            with AudioFile(self.save_path) as src:
                audio = self.sr_recognizer.record(src)
        except Exception as e:
            return await self.callback(Result("error", "Error trying to convert the saved file into a AudioFile object.", e))

        # Attempt to recognize using google's recognizer
        try:
            content = self.sr_recognizer.recognize_google(
                audio, key=self.google_recognizer_key)
            r = Result("recognized", content, audio_data=preprocessed)
        except UnknownValueError:
            r = Result("unknown")
        except Exception as e:
            r = Result("error", "Error trying to recognize the audio.", e)
        finally:
            return await self.callback(r)

    async def __vrs__(self) -> None:
        while not self.__stop_signal:
            self.stopped = False
            if not self.listen:
                continue

            data = self.q.get()
            if data is None:
                continue

            # Write the data to the output if it exists. (i.e., we'll here what we're saying)
            if self.output_track:
                self.output_track.write(data, wait=False)

            # 1: Constantly listen until a wakeword has been mentioned
            # A active_listening callback is also constantly being emitted
            if not self.__woke:
                wakeword_used = await self.wait_for_wakeword(data)

                if (time.time() - self.__last_recognized) < 1.8:
                    wakeword_used = None

                if wakeword_used:
                    await self.wake_up(wakeword_used)
                else:
                    continue

            # 2: Wakeword has been said, record until no more voice activity.
            # A realtime_recognition callback is also constantly being emitted
            self.__to_be_recognized.append(data)
            if not self.disable_vosk:
                self.vosk.AcceptWaveform(data.tobytes())
                partial = json.loads(self.vosk.PartialResult()).get("partial")
                if partial:
                    await self.callback(Result("realtime_recognition", partial))

            # 2.1: Listen for voice activities and stop "recording" once there is no more voice activity.
            # A voice_activity callback is also being emitted for every voice activity
            va, prob = await self.vad.detect(data)
            if va:

                if self.__last_va is None:
                    required_length = self.speech_lengths[0]
                else:
                    required_length = self.speech_lengths[1]

                self.__last_va = time.time()
                await self.callback(Result("voice_activity", prob))

            if self.__last_va is not None:
                duration = time.time() - self.__last_va
                if duration > required_length:
                    await self.fully_recognize()
                    self.reset_variables()

        """Code below will only be reached once stop() is called or an exception occurred inside the loop."""
        await asyncio.sleep(0.1)
        self.stopped = True
        self.__stop_signal = False
        self.q.queue.clear()


from __future__ import print_function
import os
import time
import tensorflow as tf
import numpy as np
import random
from utils import *

# Set gpus
gpus = [0]
os.environ["CUDA_VISIBLE_DEVICES"]=','.join([str(i) for i in gpus])
num_gpus = len(gpus) # number of GPUs to use

### parameters setting
DATA_DIR = './datasets/CIHP'
LIST_PATH = './datasets/CIHP/list/train_rev.txt'
DATA_ID_LIST = './datasets/CIHP/list/train_id.txt'
SNAPSHOT_DIR = './checkpoint/CIHP_pgn'
LOG_DIR = './logs/CIHP_pgn'

N_CLASSES = 20
INPUT_SIZE = (512, 512)
BATCH_I = 1
BATCH_SIZE = BATCH_I * len(gpus)
SHUFFLE = True
RANDOM_SCALE = True
RANDOM_MIRROR = True
LEARNING_RATE = 1e-5
MOMENTUM = 0.9
POWER = 0.9
p_Weight = 50
e_Weight = 0.005
Edge_Pos_W = 2
with open(DATA_ID_LIST, 'r') as f:
    TRAIN_SET = len(f.readlines())
SAVE_PRED_EVERY = TRAIN_SET / BATCH_SIZE + 1   # save model per epoch  (number of training set / batch)
NUM_STEPS = SAVE_PRED_EVERY * 100 + 1  # 100 epoch

def main():
    RANDOM_SEED = random.randint(1000, 9999)
    tf.set_random_seed(RANDOM_SEED)

    ## Create queue coordinator.
    coord = tf.train.Coordinator()
    h, w = INPUT_SIZE

    ## Load reader.
    with tf.name_scope("create_inputs"):
        reader = ImageReaderPGN(DATA_DIR, LIST_PATH, DATA_ID_LIST, INPUT_SIZE, RANDOM_SCALE, RANDOM_MIRROR, SHUFFLE, coord)
        image_batch, label_batch, edge_batch = reader.dequeue(BATCH_SIZE)

    tower_grads = []
    reuse1 = False
    # Define loss and optimisation parameters.
    base_lr = tf.constant(LEARNING_RATE)
    step_ph = tf.placeholder(dtype=tf.float32, shape=())
    learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / NUM_STEPS), POWER))
    optim = tf.train.MomentumOptimizer(learning_rate, MOMENTUM)

    for i in xrange (num_gpus):
        with tf.device('/gpu:%d' % i):
            with tf.name_scope('Tower_%d' % (i)) as scope:
                if i == 0:
                    reuse1 = False
                else:
                    reuse1 = True
                next_image = image_batch[i*BATCH_I:(i+1)*BATCH_I,:]
                next_label = label_batch[i*BATCH_I:(i+1)*BATCH_I,:]
                next_edge = edge_batch[i*BATCH_I:(i+1)*BATCH_I,:]
                # Create network.
                with tf.variable_scope('', reuse=reuse1):
                    net = PGNModel({'data': next_image}, is_training=False, n_classes=N_CLASSES, keep_prob=0.9)

                # parsing net
                parsing_out1 = net.layers['parsing_fc']
                parsing_out2 = net.layers['parsing_rf_fc']

                # edge net
                edge_out1_final = net.layers['edge_fc']
                edge_out1_res5 = net.layers['fc1_edge_res5']
                edge_out1_res4 = net.layers['fc1_edge_res4']
                edge_out1_res3 = net.layers['fc1_edge_res3']
                edge_out2_final = net.layers['edge_rf_fc']

                # combine resize
                edge_out1 = tf.image.resize_images(edge_out1_final, tf.shape(next_image)[1:3,])
                edge_out2 = tf.image.resize_images(edge_out2_final, tf.shape(next_image)[1:3,])
                edge_out1_res5 = tf.image.resize_images(edge_out1_res5, tf.shape(next_image)[1:3,])
                edge_out1_res4 = tf.image.resize_images(edge_out1_res4, tf.shape(next_image)[1:3,])
                edge_out1_res3 = tf.image.resize_images(edge_out1_res3, tf.shape(next_image)[1:3,])

                ### Predictions: ignoring all predictions with labels greater or equal than n_classes
                raw_prediction_p1 = tf.reshape(parsing_out1, [-1, N_CLASSES])
                raw_prediction_p2 = tf.reshape(parsing_out2, [-1, N_CLASSES])
                label_proc = prepare_label(next_label, tf.stack(parsing_out1.get_shape()[1:3]), one_hot=False) # [batch_size, h, w]
                raw_gt = tf.reshape(label_proc, [-1,])
                indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, N_CLASSES - 1)), 1)
                gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
                prediction_p1 = tf.gather(raw_prediction_p1, indices)
                prediction_p2 = tf.gather(raw_prediction_p2, indices)

                raw_edge = tf.reshape(tf.sigmoid(edge_out2_final), [-1,])
                edge_cond = tf.multiply(tf.cast(tf.greater(raw_edge, 0.1), tf.int32), tf.cast(tf.less_equal(raw_gt, N_CLASSES - 1), tf.int32))
                edge_mask = tf.squeeze(tf.where(tf.equal(edge_cond, 1)), 1)
                gt_edge = tf.cast(tf.gather(raw_gt, edge_mask), tf.int32)
                p1_lc = tf.gather(raw_prediction_p1, edge_mask)
                p2_lc = tf.gather(raw_prediction_p2, edge_mask)

                ### Pixel-wise softmax loss.
                loss_p1_gb = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction_p1, labels=gt))
                loss_p2_gb = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction_p2, labels=gt))
                loss_p1_lc = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=p1_lc, labels=gt_edge))
                loss_p2_lc = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=p2_lc, labels=gt_edge))
                loss_p1 = loss_p1_lc + loss_p1_gb
                loss_p2 = loss_p2_lc + loss_p2_gb

                ### Sigmoid cross entropy
                edge_pos_mask = tf.equal(next_edge, 1)
                edge_neg_mask = tf.logical_not(edge_pos_mask)
                edge_pos_mask = tf.cast(edge_pos_mask, tf.float32)
                edge_neg_mask = tf.cast(edge_neg_mask, tf.float32)

                total_pixels = tf.cast(tf.shape(next_edge)[1] * tf.shape(next_edge)[2], tf.int32)
                pos_pixels = tf.reduce_sum(tf.to_int32(next_edge))
                neg_pixels = tf.subtract(total_pixels, pos_pixels)
                pos_weight = tf.cast(tf.divide(neg_pixels, total_pixels), tf.float32)
                neg_weight = tf.cast(tf.divide(pos_pixels, total_pixels), tf.float32)

                parsing_mask = tf.cast(tf.greater(next_label, 0), tf.float32)
                edge_gt = tf.cast(next_edge, tf.float32)

                t_loss_e1 = tf.nn.sigmoid_cross_entropy_with_logits(logits=edge_out1, labels=edge_gt)
                loss_e1_pos_gb = tf.reduce_sum(tf.multiply(t_loss_e1, edge_pos_mask), [1, 2])
                loss_e1_neg_gb = tf.reduce_sum(tf.multiply(t_loss_e1, edge_neg_mask), [1, 2])
                loss_e1_pos_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1, parsing_mask), edge_pos_mask), [1, 2])
                loss_e1_neg_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1, parsing_mask), edge_neg_mask), [1, 2])
                loss_e1_pos = (loss_e1_pos_gb + loss_e1_pos_lc)* pos_weight
                loss_e1_neg = (loss_e1_neg_gb + loss_e1_neg_lc) * neg_weight
                loss_e1 = tf.reduce_mean(loss_e1_pos * Edge_Pos_W + loss_e1_neg)

                t_loss_e2 = tf.nn.sigmoid_cross_entropy_with_logits(logits=edge_out2, labels=edge_gt)
                loss_e2_pos_gb = tf.reduce_sum(tf.multiply(t_loss_e2, edge_pos_mask), [1, 2])
                loss_e2_neg_gb = tf.reduce_sum(tf.multiply(t_loss_e2, edge_neg_mask), [1, 2])
                loss_e2_pos_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e2, parsing_mask), edge_pos_mask), [1, 2])
                loss_e2_neg_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e2, parsing_mask), edge_neg_mask), [1, 2])
                loss_e2_pos = (loss_e2_pos_gb + loss_e2_pos_lc)* pos_weight
                loss_e2_neg = (loss_e2_neg_gb + loss_e2_neg_lc) * neg_weight
                loss_e2 = tf.reduce_mean(loss_e2_pos * Edge_Pos_W + loss_e2_neg)

                t_loss_e1_res5 = tf.nn.sigmoid_cross_entropy_with_logits(logits=edge_out1_res5, labels=edge_gt)
                loss_e1_res5_pos_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res5, edge_pos_mask), [1, 2])
                loss_e1_res5_neg_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res5, edge_neg_mask), [1, 2])
                loss_e1_res5_pos_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res5, parsing_mask), edge_pos_mask), [1, 2])
                loss_e1_res5_neg_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res5, parsing_mask), edge_neg_mask), [1, 2])
                loss_e1_res5_pos = (loss_e1_res5_pos_gb + loss_e1_res5_pos_lc)* pos_weight
                loss_e1_res5_neg = (loss_e1_res5_neg_gb + loss_e1_res5_neg_lc) * neg_weight
                loss_e1_res5 = tf.reduce_mean(loss_e1_res5_pos * Edge_Pos_W + loss_e1_res5_neg)

                t_loss_e1_res4 = tf.nn.sigmoid_cross_entropy_with_logits(logits=edge_out1_res4, labels=edge_gt)
                loss_e1_res4_pos_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res4, edge_pos_mask), [1, 2])
                loss_e1_res4_neg_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res4, edge_neg_mask), [1, 2])
                loss_e1_res4_pos_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res4, parsing_mask), edge_pos_mask), [1, 2])
                loss_e1_res4_neg_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res4, parsing_mask), edge_neg_mask), [1, 2])
                loss_e1_res4_pos = (loss_e1_res4_pos_gb + loss_e1_res4_pos_lc)* pos_weight
                loss_e1_res4_neg = (loss_e1_res4_neg_gb + loss_e1_res4_neg_lc) * neg_weight
                loss_e1_res4 = tf.reduce_mean(loss_e1_res4_pos * Edge_Pos_W + loss_e1_res4_neg)

                t_loss_e1_res3 = tf.nn.sigmoid_cross_entropy_with_logits(logits=edge_out1_res3, labels=edge_gt)
                loss_e1_res3_pos_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res3, edge_pos_mask), [1, 2])
                loss_e1_res3_neg_gb = tf.reduce_sum(tf.multiply(t_loss_e1_res3, edge_neg_mask), [1, 2])
                loss_e1_res3_pos_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res3, parsing_mask), edge_pos_mask), [1, 2])
                loss_e1_res3_neg_lc = tf.reduce_sum(tf.multiply(tf.multiply(t_loss_e1_res3, parsing_mask), edge_neg_mask), [1, 2])
                loss_e1_res3_pos = (loss_e1_res3_pos_gb + loss_e1_res3_pos_lc)* pos_weight
                loss_e1_res3_neg = (loss_e1_res3_neg_gb + loss_e1_res3_neg_lc) * neg_weight
                loss_e1_res3 = tf.reduce_mean(loss_e1_res3_pos * Edge_Pos_W + loss_e1_res3_neg)

                loss_parsing = loss_p1 + loss_p2
                loss_edge = loss_e1 + loss_e2 + loss_e1_res5 + loss_e1_res4 + loss_e1_res3
                reduced_loss = loss_parsing * p_Weight + loss_edge * e_Weight

                trainable_variable = tf.trainable_variables()
                grads = optim.compute_gradients(reduced_loss, var_list=trainable_variable)

                tower_grads.append(grads)

                tf.add_to_collection('loss_p', loss_parsing)
                tf.add_to_collection('loss_e', loss_edge)
                tf.add_to_collection('reduced_loss', reduced_loss)

    # Average the gradients
    grads_ave = average_gradients(tower_grads)
    # apply the gradients with our optimizers
    train_op = optim.apply_gradients(grads_ave)

    loss_p_ave = tf.reduce_mean(tf.get_collection('loss_p'))
    loss_e_ave = tf.reduce_mean(tf.get_collection('loss_e'))
    loss_ave = tf.reduce_mean(tf.get_collection('reduced_loss'))

    loss_summary_p = tf.summary.scalar("loss_p_ave", loss_p_ave)
    loss_summary_e = tf.summary.scalar("loss_e_ave", loss_e_ave)
    loss_summary_ave = tf.summary.scalar("loss_ave", loss_ave)
    loss_summary = tf.summary.merge([loss_summary_ave, loss_summary_p, loss_summary_e])
    summary_writer = tf.summary.FileWriter(LOG_DIR, graph=tf.get_default_graph())

    # Saver for storing checkpoints of the model.
    all_saver_var = tf.global_variables()
    restore_var = [v for v in all_saver_var if 'parsing' not in v.name and 'edge' not in v.name and 'Momentum' not in v.name]
    saver = tf.train.Saver(var_list=all_saver_var, max_to_keep=100)
    loader = tf.train.Saver(var_list=restore_var)

    # Set up tf session and initialize variables.
    config = tf.ConfigProto(allow_soft_placement=True,log_device_placement=False)
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    init = tf.global_variables_initializer()
    sess.run(init)

    if load(loader, sess, SNAPSHOT_DIR):
        print(" [*] Load SUCCESS")
    else:
        print(" [!] Load failed...")

    # Start queue threads.
    threads = tf.train.start_queue_runners(coord=coord, sess=sess)

    # Iterate over training steps.
    for step in range(NUM_STEPS):
        start_time = time.time()
        loss_value = 0
        feed_dict = { step_ph : step }

        # Apply gradients.
        summary, loss_value, par_loss, edge_loss, _ = sess.run([loss_summary, reduced_loss, loss_parsing, loss_edge, train_op], feed_dict=feed_dict)
        summary_writer.add_summary(summary, step)
        if step % SAVE_PRED_EVERY == 0:
            save(saver, sess, SNAPSHOT_DIR, step)

        duration = time.time() - start_time
        print('step {:d} \t loss = {:.3f}, parsing_loss = {:.3f}, edge_loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, par_loss, edge_loss, duration))

def average_gradients(tower_grads):
  """Calculate the average gradient for each shared variable across all towers.
  Note that this function provides a synchronization point across all towers.
  Args:
    tower_grads: List of lists of (gradient, variable) tuples. The outer list
      is over individual gradients. The inner list is over the gradient
      calculation for each tower.
  Returns:
     List of pairs of (gradient, variable) where the gradient has been averaged
     across all towers.
  """
  average_grads = []
  for grad_and_vars in zip(*tower_grads):
    # Note that each grad_and_vars looks like the following:
    #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
    grads = []
    for g, _ in grad_and_vars:
      # Add 0 dimension to the gradients to represent the tower.
      expanded_g = tf.expand_dims(g, 0)

      # Append on a 'tower' dimension which we will average over below.
      grads.append(expanded_g)

    # Average over the 'tower' dimension.
    grad = tf.concat(axis=0, values=grads)
    grad = tf.reduce_mean(grad, 0)

    # Keep in mind that the Variables are redundant because they are shared
    # across towers. So .. we will just return the first tower's pointer to
    # the Variable.
    v = grad_and_vars[0][1]
    grad_and_var = (grad, v)
    average_grads.append(grad_and_var)
  return average_grads

if __name__ == '__main__':
    main()

##########################################


import cv2, os, time, math
import numpy as np
from face_alignment import FaceMaskDetection
from tools import model_restore_from_pb
import tensorflow

if tensorflow.__version__.startswith('1.'):
    import tensorflow as tf
else:
    import tensorflow.compat.v1 as tf
    tf.disable_v2_behavior()
print("Tensorflow version: ",tf.__version__)

img_format = {'png','jpg','bmp'}

def video_init(camera_source=0,resolution="480",to_write=False,save_dir=None):
    writer = None
    resolution_dict = {"480":[480,640],"720":[720,1280],"1080":[1080,1920]}

    #camera source connection
    cap = cv2.VideoCapture(camera_source)
    #resolution decision
    if resolution_dict.get(resolution) is not None:
        width = resolution_dict[resolution][1]
        height = resolution_dict[resolution][0]
        cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
    else:
        height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
        width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
        print("video size is auto set")

    if to_write is True:
        fourcc = cv2.VideoWriter_fourcc(*'XVID')
        save_path = 'demo.avi'
        if save_dir is not None:
            save_path = os.path.join(save_dir,save_path)
        writer = cv2.VideoWriter(save_path, fourcc, 30, (int(width), int(height)))

    return cap,height,width,writer

def stream(pb_path, node_dict,ref_dir,camera_source=0,resolution="480",to_write=False,save_dir=None):
# def stream(camera_source=0, resolution="480", to_write=False, save_dir=None):
    #var
    frame_count = 0
    FPS = "loading"
    face_mask_model_path = r'face_mask_detection.pb'
    margin = 40
    id2class = {0: 'Mask', 1: 'NoMask'}
    batch_size = 32
    threshold = 0.8

    #Video streaming initialization
    cap,height,width,writer = video_init(camera_source=camera_source, resolution=resolution, to_write=to_write, save_dir=save_dir)

    # face detection face recognition init
    fmd = FaceMaskDetection(face_mask_model_path, margin, GPU_ratio=None)
    sess, tf_dict = model_restore_from_pb(pb_path, node_dict, GPU_ratio=None)

    tf_input = tf_dict['input']
    tf_phase_train = tf_dict['phase_train']
    tf_embeddings = tf_dict['embeddings']
    model_shape = tf_input.shape
    print("The mode shape of face recognition:",model_shape)
    feed_dict = {tf_phase_train: False}

    # if tf_input.shape[1].value is None:
    #     model_shape = (None, 160, 160, 3)
    # else:
    #     model_shape = (None, tf_input.shape[1].value, tf_input.shape[2].value, 3)
    # print("The mode shape of face recognition:",model_shape)
    # feed_dict = dict()
    if 'keep_prob' in tf_dict.keys():
        tf_keep_prob = tf_dict['keep_prob']
        feed_dict[tf_keep_prob] = 1.0
    # if 'phase_train' in tf_dict.keys():
    #     tf_phase_train = tf_dict['phase_train']
    #     feed_dict[tf_phase_train] = False

    #read images from the database
    d_t = time.time()
    paths = [file.path for file in os.scandir(ref_dir) if file.name[-3:] in img_format]
    len_ref_path = len(paths)
    if len_ref_path == 0:
        print("No images in ", ref_dir)
    else:
        ites = math.ceil(len_ref_path / batch_size)
        embeddings_ref = np.zeros([len_ref_path, tf_embeddings.shape[-1]], dtype=np.float32)

        for i in range(ites):
            num_start = i * batch_size
            num_end = np.minimum(num_start + batch_size, len_ref_path)

            batch_data_dim =[num_end - num_start]
            batch_data_dim.extend(model_shape[1:])
            batch_data = np.zeros(batch_data_dim,dtype=np.float32)

            for idx,path in enumerate(paths[num_start:num_end]):
                # img = cv2.imread(path)
                img = cv2.imdecode(np.fromfile(path, dtype=np.uint8), 1)
                if img is None:
                    print("read failed:",path)
                else:
                    #print("model_shape:",model_shape[1:3])
                    img = cv2.resize(img,(model_shape[2],model_shape[1]))
                    img = img[:,:,::-1]
                    batch_data[idx] = img
            batch_data /= 255
            feed_dict[tf_input] = batch_data
            #get data embedding
            embeddings_ref[num_start:num_end] = sess.run(tf_embeddings,feed_dict=feed_dict)

        d_t = time.time() - d_t

        print("ref embedding shape",embeddings_ref.shape)
        print("It takes {} secs to get {} embeddings".format(d_t, len_ref_path))

    # tf setting for calculating distance
    if len_ref_path > 0:
        with tf.Graph().as_default():
            tf_tar = tf.placeholder(dtype=tf.float32, shape=tf_embeddings.shape[-1])
            tf_ref = tf.placeholder(dtype=tf.float32, shape=tf_embeddings.shape)
            tf_dis = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(tf_ref, tf_tar)), axis=1))
            config = tf.ConfigProto(log_device_placement=True,
                                    allow_soft_placement=True,
                                    )
            config.gpu_options.allow_growth = True
            sess_cal = tf.Session(config=config)
            sess_cal.run(tf.global_variables_initializer())

        feed_dict_2 = {tf_ref: embeddings_ref}

    #Get an image 功能加入while loop中
    while(cap.isOpened()):
        ret, img = cap.read()
        if ret is True:
            #---------------image processing--------------------------------------------
            img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            img_rgb = img_rgb.astype(np.float32)
            img_rgb /= 255

            #--------------------face detection-------------------------------
            img_fd = cv2.resize(img_rgb, fmd.img_size)
            img_fd = np.expand_dims(img_fd, axis=0)

            bboxes, re_confidence, re_classes, re_mask_id = fmd.inference(img_fd, height, width)
            if len(bboxes) > 0:
                for num, bbox in enumerate(bboxes):
                    class_id = re_mask_id[num]
                    if class_id == 0:
                        color = (0, 255, 0)  # (B,G,R) --> Green(with masks)
                    else:
                        color = (0, 0, 255)  # (B,G,R) --> Red(without masks)
                    cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[0] + bbox[2], bbox[1] + bbox[3]), color, 2)
                    # cv2.putText(img, "%s: %.2f" % (id2class[class_id], re_confidence[num]), (bbox[0] + 2, bbox[1] - 2),
                    #             cv2.FONT_HERSHEY_SIMPLEX, 0.8, color)

                    # ----face recognition
                    name = ""
                    if len_ref_path > 0:
                        img_fr = img_rgb[bbox[1]:bbox[1] + bbox[3], bbox[0]:bbox[0] + bbox[2], :]  # crop
                        img_fr = cv2.resize(img_fr, (model_shape[2], model_shape[1]))  # resize
                        img_fr = np.expand_dims(img_fr, axis=0)  # make 4 dimensions

                        feed_dict[tf_input] = img_fr
                        embeddings_tar = sess.run(tf_embeddings, feed_dict=feed_dict)
                        feed_dict_2[tf_tar] = embeddings_tar[0]
                        distance = sess_cal.run(tf_dis, feed_dict=feed_dict_2)
                        arg = np.argmin(distance)  # index of the smallest distance

                        if distance[arg] < threshold:
                            name = paths[arg].split("\\")[-1].split(".")[0]

                    cv2.putText(img, "{},{}".format(id2class[class_id], name), (bbox[0] + 2, bbox[1] - 2),
                                cv2.FONT_HERSHEY_SIMPLEX, 0.8, color)

            # #----FPS calculation
            # if frame_count == 0:
            #     t_start = time.time()
            # frame_count += 1
            # if frame_count >= 10:q
            #     FPS = "FPS=%1f" % (10 / (time.time() - t_start))
            #     frame_count = 0

            # cv2.putText(img, text, coor, font, size, color, line thickness, line type)
            # cv2.putText(img, FPS, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)
            # cv2.putText(img, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 3)

            #----image display
            cv2.imshow("Yue's test", img)

            #----image writing
            if writer is not None:
                writer.write(img)

            #press the q to quit, s to save
            key = cv2.waitKey(1) & 0xFF
            if key == ord('q'):
                break
            elif key == ord('s'):
                if len(bboxes) > 0:
                    img_temp = img[bbox[1]:bbox[1] + bbox[3], bbox[0]:bbox[0] + bbox[2], :]
                    save_path = "img_crop.jpg"
                    save_path = os.path.join(ref_dir,save_path)
                    cv2.imwrite(save_path,img_temp)
                    print("An image is saved to ",save_path)

        else:
            print("get images failed")
            break

    #----release
    cap.release()
    cv2.destroyAllWindows()
    if writer is not None:
        writer.release()

if __name__ == "__main__":
    #调用模型
    pb_path = r"E:\PycharmProjects\Train_faceNet_model\model_saver\pb_modelepoch20.pb"
    node_dict = {'input': 'input:0',
                 'keep_prob': 'keep_prob:0',
                 'phase_train': 'phase_train:0',
                 'embeddings': 'embeddings:0',
                 }
    #对比数据库
    ref_dir = r"E:\PycharmProjects\Realtime_face_recognition\test_database_4\test_reduce"
    stream(pb_path, node_dict, ref_dir, camera_source=0, resolution='480', to_write=True, save_dir=None)


from keras.models import load_model
import cv2
import numpy as np
from random import choice

REV_CLASS_MAP = {
    0: "rock",
    1: "paper",
    2: "scissors",
    3: "none"
}

def mapper(val):
    return REV_CLASS_MAP[val]

def calculate_winner(move1, move2):
    if move1 == move2:
        return "Tie"

    if move1 == "rock":
        if move2 == "scissors":
            return "User"
        if move2 == "paper":
            return "Computer"

    if move1 == "paper":
        if move2 == "rock":
            return "User"
        if move2 == "scissors":
            return "Computer"

    if move1 == "scissors":
        if move2 == "paper":
            return "User"
        if move2 == "rock":
            return "Computer"

model = load_model("rock-paper-scissors-model.h5")

cap = cv2.VideoCapture(0)

prev_move = None

while True:
    ret, frame = cap.read()
    if not ret:
        continue

    # rectangle for user to play
    cv2.rectangle(frame, (100, 100), (500, 500), (255, 255, 255), 2)
    # rectangle for computer to play
    cv2.rectangle(frame, (800, 100), (1200, 500), (255, 255, 255), 2)

    # extract the region of image within the user rectangle
    roi = frame[100:500, 100:500]
    img = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (227, 227))

    # predict the move made
    pred = model.predict(np.array([img]))
    move_code = np.argmax(pred[0])
    user_move_name = mapper(move_code)

    # predict the winner (human vs computer)
    if prev_move != user_move_name:
        if user_move_name != "none":
            computer_move_name = choice(['rock', 'paper', 'scissors'])
            winner = calculate_winner(user_move_name, computer_move_name)
        else:
            computer_move_name = "none"
            winner = "Waiting..."
    prev_move = user_move_name

    # display the information
    font = cv2.FONT_HERSHEY_SIMPLEX
    cv2.putText(frame, "Your Move: " + user_move_name,
                (50, 50), font, 1.2, (255, 255, 255), 2, cv2.LINE_AA)
    cv2.putText(frame, "Computer's Move: " + computer_move_name,
                (750, 50), font, 1.2, (255, 255, 255), 2, cv2.LINE_AA)
    cv2.putText(frame, "Winner: " + winner,
                (400, 600), font, 2, (0, 0, 255), 4, cv2.LINE_AA)

    if computer_move_name != "none":
        icon = cv2.imread(
            "images/{}.png".format(computer_move_name))
        icon = cv2.resize(icon, (400, 400))
        frame[100:500, 800:1200] = icon

    cv2.imshow("Rock Paper Scissors", frame)

    k = cv2.waitKey(10)
    if k == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


import numpy as np
import cv2
import imutils
import time
from sklearn.metrics import pairwise
from imutils.video import FPS
import copy
import os
import sys
import tensorflow as tf
import pathlib
from collections import defaultdict

from utils import tracking_utils
from utils import signalDetection_utils
from utils import estimate_collide_utils
from utils import estimate_stepping_utils
from utils import lane_detection_utils
from utils import break_light_utils
from utils import functions

from utils import ops as utils_ops
from utils import label_map_util

# print(category_index)
colors = np.random.uniform(0, 255, size=(100, 3))
font = cv2.FONT_HERSHEY_SIMPLEX
blackLower = (0 , 0 , 0)
blackUpper = (180 , 255 , 35)

def click_and_crop(event, x, y, flags, param):
    global refPt
    # if the left mouse button was clicked, record the starting (x, y) coordinates
    if event == cv2.EVENT_LBUTTONDOWN:
        refPt.append([x, y])

def confirm_day_or_night(frame , flag_night_counter):
    blurred = cv2.GaussianBlur(frame, (11, 11), 0)
    hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV)
    mask = cv2.inRange(hsv, blackLower , blackUpper)
    mask = cv2.erode(mask, None, iterations=2)
    mask = cv2.dilate(mask , None, iterations=2)
    # cv2.imshow('black',imutils.resize(mask,width=250))
    # cv2.imshow('frame',frame)
    pixel_ct = 0
    pixel_len = 0
    for i in mask:
      pixel_ct = pixel_ct + np.sum(i==0)
      pixel_len = pixel_len + len(i)
    ratio = pixel_ct / pixel_len
    print("ratio = ",ratio)
    if ratio < 0.6:
        flag_night_counter = flag_night_counter + 1
        return flag_night_counter
    else:
        flag_night_counter = flag_night_counter - 1
        return flag_night_counter

net = cv2.dnn.readNet("../yolov3.weights", "../yolov3.cfg")
classes = []
with open("../coco.names", "r") as f:
    classes = [line.strip() for line in f.readlines()]
layer_names = net.getLayerNames()
output_layers = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]

# flagPerson and areaPerson are pedestrian crash
flagPerson = 0
areaPerson = 0
areaDetails = []

# crash_count_frames are vehicle warning variables
crash_count_frames = 0

# signalCounter and flagSignal are traffic signal variables
signalCounter = -99999
flagSignal = [0] * 10

# number and prev_frame are tracking variables
number = 0
prev_frame = []

# flagLanes are lane variables
flagLanes = [0] * 20

def yolo_infer(dashPointer , lanePointer , frame):
    global flagPerson , areaPerson , areaDetails
    global crash_count_frames
    global signalCounter , flagSignal
    global prev_frame , number
    global flagLanes

    image_np = np.array(frame)
    lane_image = copy.deepcopy(image_np)
    height,width,channel = image_np.shape

    mask = 255*np.ones_like(image_np)
    vertices = np.array(dashPointer, np.int32)
    cv2.fillPoly(mask, [vertices], [0,0,0])
    image_np = cv2.bitwise_and(image_np, mask)

    # Detecting objects
    blob = cv2.dnn.blobFromImage(image_np, 0.00392, (416,416), (0, 0, 0), True, crop=False)
    net.setInput(blob)
    outs = net.forward(output_layers)

    confidencesCars , boxesCars = [] , []
    confidencesLights , boxesLights = [] , []
    confidencesPersons , boxesPersons = [] , []

    for out in outs:
        for detection in out:
            scores = detection[5:]
            class_id = np.argmax(scores)
            if (class_id == 2 or class_id == 5  or class_id == 7):

                scr = scores[class_id]
                center_x = detection[0] * width
                center_y = detection[1] * height

                w = detection[2] * width
                h = detection[3] * height
                xmin = center_x - w / 2
                ymin = center_y - h / 2
                if (w*h >=800):
                    confidencesCars.append(float(scr))
                    boxesCars.append([int(xmin) , int(ymin) , int(w) , int(h)])
            elif class_id == 0:
                # print(scores)

                scr = scores[class_id]
                center_x = detection[0] * width
                center_y = detection[1] * height

                w = detection[2] * width
                h = detection[3] * height
                xmin = center_x - w / 2
                ymin = center_y - h / 2
                confidencesPersons.append(float(scr))
                boxesPersons.append([int(xmin) , int(ymin) , int(w) , int(h)])

            elif class_id == 9:
                scr = scores[class_id]
                center_x = detection[0] * width
                center_y = detection[1] * height

                w = detection[2] * width
                h = detection[3] * height
                xmin = center_x - w / 2
                ymin = center_y - h / 2
                confidencesLights.append(float(scr))
                boxesLights.append([int(xmin) , int(ymin) , int(w) , int(h)])

    indexesLights = cv2.dnn.NMSBoxes(boxesLights, confidencesLights, 0.5, 0.4)
    indexesCars = cv2.dnn.NMSBoxes(boxesCars, confidencesCars, 0.5, 0.4)
    indexesPersons = cv2.dnn.NMSBoxes(boxesPersons, confidencesPersons, 0.5, 0.4)

    image_np , signalCounter , flagSignal = signalDetection_utils.signalDetection(indexesLights , boxesLights , image_np , signalCounter , flagSignal)
    image_np , prev_frame , number = tracking_utils.tracking(indexesCars , boxesCars , image_np , prev_frame , number)
    image_np , crash_count_frames = estimate_collide_utils.estimate_collide(indexesCars , boxesCars , image_np , crash_count_frames)
    image_np , flagPerson , areaPerson , areaDetails = estimate_stepping_utils.estimate_stepping(indexesPersons , boxesPersons , image_np, flagPerson , areaPerson , areaDetails)

    cv2.putText(image_np,"DAY",(width - 200 ,50), font, 2, (167,133,0) , 2 , cv2.LINE_AA)

    image_np , flagLanes = lane_detection_utils.draw_lines(lanePointer , dashPointer , lane_image , image_np , flagLanes)

    cv2.imshow("finally", image_np)

def selectRegions(image  , text , flag):
    global refPt
    clone = copy.deepcopy(image)
    while True:
        key = cv2.waitKey(1) & 0xFF
        # display the image and wait for a keypress
        if flag==1:
            cv2.putText(image, text ,  (240,30), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'r' key to reset everything.",  (290,70), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'd' key if the region selection is done.",  (180,110), font , 1.2, [0,255,255], 2,cv2.LINE_AA)
        else:
            cv2.putText(image, text ,  (240,30), font , 1.2, [0,255,0], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'r' key to reset everything.",  (290,70), font , 1.2, [0,255,0], 2,cv2.LINE_AA)
            cv2.putText(image, "Press 'd' key if the region selection is done.",  (180,110), font , 1.2, [0,255,0], 2,cv2.LINE_AA)

        for pt in range(len(refPt)-1):
            pt1 , pt2 = refPt[pt] , refPt[pt+1]
            cv2.line(image, (pt1[0],pt1[1]), (pt2[0],pt2[1]), [0,255,255], 3)

        cv2.imshow("ROI", image)
        if key == ord("r"):
            image = copy.deepcopy(clone)
            refPt = []
        elif key == ord("d"):
            if flag == 1:
                return 0
            elif flag == 2 and len(refPt) > 2:
                return 0
        elif key == ord('q'):
            return 1

def night():
    global refPt
    _ , image = cap.read()
    image=imutils.resize(image, width=1280)

    ctt = 0

    Quit = selectRegions(image  , "Click points to select your vehicle dash." , 1)
    dashPointer = refPt
    if len(dashPointer) <= 2:
        dashPointer = [[0,0], [0,0], [0,0]]
    refPt = []
    print("For dash: ",dashPointer)
    if Quit == 1:
        return
    cv2.destroyWindow("ROI")

    fps = FPS().start()
    while True:
        _,frame = cap.read()
        frame=imutils.resize(frame, width=1280)

        if _ == False:
            break

        # print(ctt ,fps._numFrames)
        # ctt = ctt + 1
        break_light_utils.break_light(dashPointer , frame)

        # cv2.imshow("original",frame)
        key = cv2.waitKey(1) & 0xFF
        fps.update()
        if key == ord('q'):
            break
    fps.stop()
    print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

def day():
    global refPt
    _ , image = cap.read()
    image=imutils.resize(image, width=1280)

    ctt = 0

    Quit = selectRegions(copy.deepcopy(image)  , "Click points to select your vehicle dash." , 1)
    dashPointer = refPt
    if len(dashPointer) <= 2:
        dashPointer = [[0,0], [0,0], [0,0]]
    refPt = []
    print("For dash: ",dashPointer)
    if Quit == 1:
        return

    Quit = selectRegions(copy.deepcopy(image)  , "Click points to select bird's eye view." , 2)
    lanePointer = refPt
    print("For lanes: ",lanePointer)
    if Quit == 1:
        return

    cv2.destroyWindow("ROI")

    fps = FPS().start()
    while True:
        _,frame = cap.read()
        frame = imutils.resize(frame, width=1280)
        if _ == False:
            break
        # print(ctt ,fps._numFrames)
        # ctt = ctt + 1

        yolo_infer(dashPointer , lanePointer , frame)

        # cv2.imshow("original",frame)
        key = cv2.waitKey(1) & 0xFF
        fps.update()
        if key == ord('q'):
            break

    fps.stop()
    print("[INFO] elasped time: {:.2f}".format(fps.elapsed()))
    print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

refPt = []                  # to store refernece pointers
flag_night_counter = 0      # counter to count night frames

cap=cv2.VideoCapture('../videos/r.mp4')
start_frame = 0*30
cap.set(1,start_frame)
_ , image = cap.read()
image=imutils.resize(image, width=1280)
cv2.namedWindow("ROI")
cv2.setMouseCallback("ROI", click_and_crop)

for z in range(10):
    (grabbed, frame) = cap.read()
    frame=imutils.resize(frame, width=1280)
    height,width,channel = frame.shape

    flag_night_counter = confirm_day_or_night(frame , flag_night_counter)
print("flag_night_counter = ",flag_night_counter)
cap.set(1 , start_frame)

if flag_night_counter > 4:
    night()
else:
    day()

cv2.destroyAllWindows()
cap.release()