from __future__ import annotations

import logging
import os
from collections import defaultdict
from copy import deepcopy
from pathlib import Path
from typing import List, Optional, Union, Any, Dict, Set, Tuple

from typing_extensions import TypedDict

from checkov.common.graph.graph_builder import Edge
from checkov.common.graph.graph_builder import reserved_attribute_names
from checkov.common.graph.graph_builder.graph_components.attribute_names import CustomAttributes
from checkov.common.graph.graph_builder.local_graph import LocalGraph
from checkov.common.graph.graph_builder.utils import calculate_hash, join_trimmed_strings, filter_sub_keys
from checkov.common.runners.base_runner import strtobool
from checkov.common.util.parser_utils import get_abs_path, get_tf_definition_key_from_module_dependency
from checkov.common.util.type_forcers import force_int
from checkov.terraform.checks.utils.dependency_path_handler import unify_dependency_path
from checkov.terraform.context_parsers.registry import parser_registry
import checkov.terraform.graph_builder.foreach_handler as foreach_module
from checkov.terraform.graph_builder.graph_components.block_types import BlockType
from checkov.terraform.graph_builder.graph_components.blocks import TerraformBlock
from checkov.terraform.graph_builder.graph_components.generic_resource_encryption import ENCRYPTION_BY_RESOURCE_TYPE
from checkov.terraform.graph_builder.graph_components.module import Module
from checkov.terraform.graph_builder.utils import (
    get_referenced_vertices_in_value,
    attribute_has_nested_attributes, remove_index_pattern_from_str,
)
from checkov.terraform.graph_builder.utils import is_local_path
from checkov.terraform.graph_builder.variable_rendering.renderer import TerraformVariableRenderer

MODULE_RESERVED_ATTRIBUTES = ("source", "version")
CROSS_VARIABLE_EDGE_PREFIX = '[cross-variable] '

class Undetermined(TypedDict):
    module_vertex_id: int
    attribute_name: str
    variable_vertex_id: int

class TerraformLocalGraph(LocalGraph[TerraformBlock]):
    def __init__(self, module: Module) -> None:
        super().__init__()
        self.vertices: list[TerraformBlock] = []
        self.module = module
        self.map_path_to_module: Dict[str, List[int]] = {}
        self.relative_paths_cache = {}
        self.abspath_cache: Dict[str, str] = {}
        self.dirname_cache: Dict[str, str] = {}
        self.vertices_by_module_dependency_by_name: Dict[Tuple[str, str], Dict[BlockType, Dict[str, List[int]]]] = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
        self.vertices_by_module_dependency: Dict[Tuple[str, str], Dict[BlockType, List[int]]] = defaultdict(lambda: defaultdict(list))
        self.enable_foreach_handling = strtobool(os.getenv('CHECKOV_ENABLE_FOREACH_HANDLING', 'False'))
        self.foreach_blocks: Dict[str, List[int]] = {BlockType.RESOURCE: [], BlockType.MODULE: []}

    def build_graph(self, render_variables: bool) -> None:
        self._create_vertices()
        logging.info(f"[TerraformLocalGraph] created {len(self.vertices)} vertices")
        self._build_edges()
        logging.info(f"[TerraformLocalGraph] created {len(self.edges)} edges")
        if self.enable_foreach_handling:
            try:
                foreach_handler = foreach_module.ForeachHandler(self)
                foreach_handler.handle_foreach_rendering(self.foreach_blocks)
                self._arrange_graph_data()
                self._build_edges()
                logging.info(f"[TerraformLocalGraph] finished handling foreach values with {len(self.vertices)} vertices and {len(self.edges)} edges")
            except Exception as e:
                logging.info(f'Failed to process foreach handling, error: {str(e)}', exc_info=True)

        self.calculate_encryption_attribute(ENCRYPTION_BY_RESOURCE_TYPE)
        if render_variables:
            logging.info(f"Rendering variables, graph has {len(self.vertices)} vertices and {len(self.edges)} edges")
            renderer = TerraformVariableRenderer(self)
            renderer.render_variables_from_local_graph()
            self.update_vertices_breadcrumbs_and_module_connections()
            self.update_nested_modules_address()
            if strtobool(os.getenv("CHECKOV_EXPERIMENTAL_CROSS_VARIABLE_EDGES", "True")):
                # experimental flag on building cross variable edges for terraform graph
                logging.info("Building cross variable edges")
                edges_count = len(self.edges)
                self._build_cross_variable_edges()
                logging.info(f"Found {len(self.edges) - edges_count} cross variable edges")

    def _create_vertices(self) -> None:
        logging.info("Creating vertices")
        self.vertices: List[TerraformBlock] = [None] * len(self.module.blocks)
        for i, block in enumerate(self.module.blocks):
            self.vertices[i] = block
            self._add_block_data_to_graph(i, block)
            if self.enable_foreach_handling and (foreach_module.FOREACH_STRING in block.attributes or foreach_module.COUNT_STRING in block.attributes) \
                    and block.block_type in (BlockType.MODULE, BlockType.RESOURCE):
                self.foreach_blocks[block.block_type].append(i)

    def _add_block_data_to_graph(self, idx: int, block: TerraformBlock) -> None:
        self.vertices_by_block_type[block.block_type].append(idx)
        self.vertices_block_name_map[block.block_type][block.name].append(idx)

        if block.block_type == BlockType.MODULE:
            # map between file paths and module vertices indexes from that file
            self.map_path_to_module.setdefault(block.path, []).append(idx)

        self.vertices_by_module_dependency[(block.module_dependency, block.module_dependency_num)][block.block_type].append(idx)
        self.vertices_by_module_dependency_by_name[(block.module_dependency, block.module_dependency_num)][block.block_type][block.name].append(idx)

        self.in_edges[idx] = []
        self.out_edges[idx] = []

    def _arrange_graph_data(self) -> None:
        # reset all the relevant data
        self.vertices_by_block_type = defaultdict(list)
        self.vertices_block_name_map = defaultdict(lambda: defaultdict(list))
        self.map_path_to_module = {}
        self.vertices_by_module_dependency = defaultdict(lambda: defaultdict(list))
        self.vertices_by_module_dependency_by_name = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
        self.edges = []
        for i in range(len(self.vertices)):
            self.out_edges[i] = []
            self.in_edges[i] = []

        for i, block in enumerate(self.vertices):
            self._add_block_data_to_graph(i, block)

    def _set_variables_values_from_modules(self) -> List[Undetermined]:
        undetermined_values: List[Undetermined] = []
        resources_types = self.get_resources_types_in_graph()
        for module_vertex_id in self.vertices_by_block_type.get(BlockType.MODULE, []):
            module_vertex = self.vertices[module_vertex_id]
            for attribute_name, attribute_value in module_vertex.attributes.items():
                matching_variables = self.vertices_block_name_map.get(BlockType.VARIABLE, {}).get(attribute_name, [])
                for variable_vertex_id in matching_variables:
                    variable_dir = os.path.dirname(self.vertices[variable_vertex_id].path)
                    # TODO: module_vertex.path is always a string and the retrieved dict value is a nested list
                    #   therefore this condition is always false. Fixing it results in some variables not being rendered.
                    #   see test: tests.graph.terraform.variable_rendering.test_render_scenario.TestRendererScenarios.test_account_dirs_and_modules
                    if module_vertex.path in self.module.module_dependency_map.get(variable_dir, []):
                        has_var_reference = get_referenced_vertices_in_value(
                            value=attribute_value, aliases={}, resources_types=resources_types
                        )
                        if has_var_reference:
                            undetermined_values.append(
                                {
                                    "module_vertex_id": module_vertex_id,
                                    "attribute_name": attribute_name,
                                    "variable_vertex_id": variable_vertex_id,
                                }
                            )
                        var_default_value = self.vertices[variable_vertex_id].attributes.get("default")
                        if (
                            not has_var_reference
                            or not var_default_value
                            or get_referenced_vertices_in_value(
                                value=var_default_value, aliases={}, resources_types=resources_types
                            )
                        ):
                            self.update_vertex_attribute(
                                variable_vertex_id, "default", attribute_value, module_vertex_id, attribute_name
                            )
        return undetermined_values

    def _get_aliases(self) -> Dict[str, Dict[str, BlockType]]:
        """
        :return aliases: map between alias names that are found inside the blocks and the block type their aliased to.
        """
        return {
            vertex.name: {CustomAttributes.BLOCK_TYPE: vertex.block_type}
            for vertex in self.vertices
            if "alias" in vertex.attributes
        }

    def get_module_vertices_mapping(self) -> None:
        """
        For each vertex, if it's originated in a module import, add to the vertex the index of the
        matching module vertex as 'source_module'
        """
        block_dirs_to_modules: Dict[Tuple[str, str], Dict[str, Set[int]]] = defaultdict(dict)
        for dir_name, paths_to_modules in self.module.module_dependency_map.items():
            # for each directory, find the module vertex that imported it
            for path_to_module in paths_to_modules:
                if not path_to_module:
                    continue
                path_to_module_str = unify_dependency_path(path_to_module)
                if block_dirs_to_modules.get((dir_name, path_to_module_str)):
                    continue
                module_file = get_abs_path(path_to_module[-1])
                module_list = self.map_path_to_module.get(module_file, [])
                for module_index in module_list:
                    module_vertex = self.vertices[module_index]
                    if get_path_with_nested_modules(module_vertex) == path_to_module_str:
                        module_vertex_dir = self.get_dirname(module_vertex.path)
                        module_source = module_vertex.attributes.get("source", [""])[0]
                        module_version = module_vertex.attributes.get("version", ["latest"])[0]
                        dest_module_path = self._get_dest_module_path(
                            curr_module_dir=module_vertex_dir,
                            dest_module_source=module_source,
                            dest_module_version=module_version
                        )
                        if dest_module_path == dir_name:
                            module_dependency_num = self.module.module_address_map.get((module_vertex.path, module_vertex.name))
                            if module_dependency_num:
                                block_dirs_to_modules[(dir_name, path_to_module_str)].setdefault(module_dependency_num, set()).add(module_index)

        for vertex in self.vertices:
            # match the right module vertex according to the vertex path directory
            module_dependency_nums = block_dirs_to_modules.get((self.get_dirname(vertex.path), vertex.module_dependency))
            if module_dependency_nums:
                module_indices = module_dependency_nums.get(vertex.module_dependency_num)
                if module_indices:
                    vertex.source_module = module_indices

    def _build_edges(self) -> None:
        logging.info("Creating edges")
        self.get_module_vertices_mapping()
        aliases = self._get_aliases()
        resources_types = self.get_resources_types_in_graph()
        for origin_node_index, vertex in enumerate(self.vertices):
            self._build_edges_for_vertex(origin_node_index, vertex, aliases, resources_types)

    def _build_edges_for_vertex(self, origin_node_index: int, vertex: TerraformBlock, aliases: Dict[str, Dict[str, BlockType]], resources_types: List[str], cross_variable_edges: bool = False, referenced_module: Optional[Dict[str, Any]] = None):
        referenced_module_idx = referenced_module.get("idx") if referenced_module else None
        referenced_module_path = referenced_module.get("path") if referenced_module else None
        for attribute_key, attribute_value in vertex.attributes.items():
            if attribute_key in reserved_attribute_names or attribute_has_nested_attributes(
                    attribute_key, vertex.attributes
            ):
                continue
            referenced_vertices = get_referenced_vertices_in_value(
                value=attribute_value,
                aliases=aliases,
                resources_types=resources_types,
            )
            for vertex_reference in referenced_vertices:
                # for certain blocks such as data and resource, the block name is composed from several parts.
                # the purpose of the loop is to avoid not finding the node if the name has several parts
                sub_values = [remove_index_pattern_from_str(sub_value) for sub_value in vertex_reference.sub_parts]
                for i in range(len(sub_values)):
                    reference_name = join_trimmed_strings(char_to_join=".", str_lst=sub_values, num_to_trim=i)
                    if referenced_module is not None:
                        dest_node_index = self._find_vertex_index_relative_to_path(
                            vertex_reference.block_type, reference_name, referenced_module_path, vertex.module_dependency,
                            vertex.module_dependency_num, referenced_module_idx
                        )
                    elif vertex.module_dependency:
                        dest_node_index = self._find_vertex_index_relative_to_path(
                            vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency,
                            vertex.module_dependency_num
                        )
                        if dest_node_index == -1:
                            dest_node_index = self._find_vertex_index_relative_to_path(
                                vertex_reference.block_type, reference_name, vertex.path, vertex.path,
                                vertex.module_dependency_num
                            )
                    else:
                        dest_node_index = self._find_vertex_index_relative_to_path(
                            vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency,
                            vertex.module_dependency_num
                        )
                    if dest_node_index > -1 and origin_node_index > -1:
                        if vertex_reference.block_type == BlockType.MODULE:
                            try:
                                self._connect_module(
                                    sub_values, attribute_key, self.vertices[dest_node_index], origin_node_index,
                                    cross_variable_edges
                                )
                            except Exception:
                                logging.warning(
                                    f"Module {self.vertices[dest_node_index]} does not have source attribute, skipping"
                                )
                        else:
                            self._create_edge(origin_node_index, dest_node_index, attribute_key, cross_variable_edges)
                        break

        if vertex.block_type == BlockType.MODULE and vertex.attributes.get('source') \
                and isinstance(vertex.attributes.get('source')[0], str):
            target_path = get_path_with_nested_modules(vertex)
            dest_module_path = self._get_dest_module_path(
                curr_module_dir=self.get_dirname(vertex.path),
                dest_module_source=vertex.attributes["source"][0],
                dest_module_version=vertex.attributes.get("version", ["latest"])[0]
            )
            target_variables = [
                index
                for index in self.vertices_by_module_dependency.get(
                    (target_path, self.module.module_address_map.get((vertex.path, vertex.name))), {}).get(
                    BlockType.VARIABLE, [])
                if self.get_dirname(self.vertices[index].path) == dest_module_path
            ]
            for attribute in vertex.attributes.keys():
                if attribute in MODULE_RESERVED_ATTRIBUTES:
                    continue
                target_variable = next((v for v in target_variables if self.vertices[v].name == attribute), None)
                if target_variable is not None:
                    self._create_edge(target_variable, origin_node_index, "default", cross_variable_edges)
        elif vertex.block_type == BlockType.TF_VARIABLE:
            # Assuming the tfvars file is in the same directory as the variables file (best practice)
            target_variables = [
                index
                for index in self.vertices_block_name_map.get(BlockType.VARIABLE, {}).get(vertex.name, [])
                if self.get_dirname(self.vertices[index].path) == self.get_dirname(vertex.path)
            ]
            if len(target_variables) == 1:
                self._create_edge(target_variables[0], origin_node_index, "default", cross_variable_edges)

    def _build_cross_variable_edges(self):
        target_nodes_indexes = [v for v, referenced_vertices in self.out_edges.items() if
                                self.vertices[v].block_type == BlockType.RESOURCE and any(
            self.vertices[e.dest].block_type != BlockType.RESOURCE for e in referenced_vertices)]
        aliases = self._get_aliases()
        resources_types = self.get_resources_types_in_graph()
        for origin_node_index in target_nodes_indexes:
            vertex = self.vertices[origin_node_index]
            self._build_edges_for_vertex(origin_node_index, vertex, aliases, resources_types, True)
            modules = vertex.breadcrumbs.get(CustomAttributes.SOURCE_MODULE, [])
            for module in modules:
                self._build_edges_for_vertex(origin_node_index, vertex, aliases, resources_types, True, module)

    def _create_edge(self, origin_vertex_index: int, dest_vertex_index: int, label: str,
                     cross_variable_edges: bool = False) -> bool:
        if origin_vertex_index == dest_vertex_index:
            return False
        edge = Edge(origin_vertex_index, dest_vertex_index, label)
        if cross_variable_edges:
            if self.vertices[dest_vertex_index].block_type != BlockType.RESOURCE or \
                    self.vertices[origin_vertex_index].block_type != BlockType.RESOURCE:
                return False
            if edge in self.out_edges[origin_vertex_index]:
                return False
            edge.label = CROSS_VARIABLE_EDGE_PREFIX + edge.label
            if edge in self.out_edges[origin_vertex_index]:
                return False
        self.edges.append(edge)
        self.out_edges[origin_vertex_index].append(edge)
        self.in_edges[dest_vertex_index].append(edge)
        return True

    def _connect_module(
        self, sub_values: List[str], attribute_key: str, module_node: TerraformBlock, origin_node_index: int, cross_variable_edges: bool = False
    ) -> None:
        """
        :param sub_values: list of sub values of the attribute value.
                            example: given 'instance_type = module.child.myoutput',
                                        then attribute_key = instance_type, sub_values = ['child', 'myoutput']
        :param attribute_key: the name of the attribute that has module block as value
        :param module_node: the graph node of the module

        The function receives a node of a block of type BlockType.Module, and finds all the nodes of blocks that belong to this
        module, and creates edges between them.
        """
        curr_module_dir = self.get_dirname(module_node.path)
        dest_module_source = module_node.attributes["source"][0]
        dest_module_version = module_node.attributes.get("version", ["latest"])[0]
        dest_module_path = self._get_dest_module_path(
            curr_module_dir=curr_module_dir,
            dest_module_source=dest_module_source,
            dest_module_version=dest_module_version
        )

        if len(sub_values) > 1:
            block_name_in_other_module = sub_values[1]
            output_blocks_with_name = self.vertices_block_name_map.get(BlockType.OUTPUT, {}).get(
                block_name_in_other_module, []
            )
            for vertex_index in output_blocks_with_name:
                vertex = self.vertices[vertex_index]
                if (self.get_dirname(vertex.path) == dest_module_path) and (
                    vertex.module_dependency == module_node.module_dependency  # The vertex is in the same file
                    or self.get_abspath(vertex.module_dependency)
                    == self.get_abspath(module_node.path)  # The vertex is in the correct dependency path
                ):
                    added_edge = self._create_edge(origin_node_index, vertex_index, attribute_key, cross_variable_edges)
                    if added_edge:
                        self.vertices[origin_node_index].add_module_connection(attribute_key, vertex_index)
                    break

    def _get_dest_module_path(self, curr_module_dir: str, dest_module_source: str, dest_module_version: str) -> str:
        """
        :param curr_module_dir: current source directory
        :param dest_module_source: the value of module.source
        :return: the real path in the local file system of the dest module
        """
        dest_module_path = Path()
        if is_local_path(curr_module_dir, dest_module_source):
            dest_module_path = Path(curr_module_dir) / dest_module_source
        elif (dest_module_source, dest_module_version) in self.module.external_modules_source_map:
            return self.module.external_modules_source_map[(dest_module_source, dest_module_version)]
        else:
            try:
                if dest_module_source not in self.relative_paths_cache:
                    self.relative_paths_cache[dest_module_source] = list(Path(self.module.source_dir).rglob(dest_module_source))
                dest_module_path = next(
                    (path for path in self.relative_paths_cache.get(dest_module_source)), dest_module_path
                )
            except (OSError, ValueError):
                logging.debug(f"Error to get dest_module_path {dest_module_source}", exc_info=True)
                return ""
            except NotImplementedError as e:
                if 'Non-relative patterns are unsupported' in str(e):
                    return ""
                raise e
        return os.path.abspath(dest_module_path)

    def _find_vertex_index_relative_to_path(
        self, block_type: BlockType, name: str, block_path: str, module_path: str, module_num: str, relative_module_idx: Optional[int] = None
    ) -> int:
        relative_vertices = []
        if relative_module_idx is not None:
            module_dependency_by_name_key = next(k for k, v in self.vertices_by_module_dependency.items() if v.get(BlockType.MODULE, []).__contains__(relative_module_idx))
        else:
            module_dependency_by_name_key = (module_path, module_num)
        possible_vertices = self.vertices_by_module_dependency_by_name.get(module_dependency_by_name_key, {}).get(block_type, {}).get(name, [])
        for vertex_index in possible_vertices:
            vertex = self.vertices[vertex_index]
            if self.get_dirname(vertex.path) == self.get_dirname(block_path):
                relative_vertices.append(vertex_index)

        if len(relative_vertices) == 1:
            relative_vertex = relative_vertices[0]
        else:
            relative_vertex = self._find_vertex_with_longest_path_match(relative_vertices, block_path)
        return relative_vertex

    def _find_vertex_with_longest_path_match(self, relevant_vertices_indexes: List[int], origin_path: str) -> int:
        vertex_index_with_longest_common_prefix = -1
        longest_common_prefix = ""
        for vertex_index in relevant_vertices_indexes:
            vertex = self.vertices[vertex_index]
            common_prefix = os.path.commonpath([os.path.realpath(vertex.path), os.path.realpath(origin_path)])
            if len(common_prefix) > len(longest_common_prefix):
                vertex_index_with_longest_common_prefix = vertex_index
                longest_common_prefix = common_prefix
        return vertex_index_with_longest_common_prefix

    def get_vertices_hash_codes_to_attributes_map(self) -> Dict[str, Dict[str, Any]]:
        return {vertex.get_hash(): vertex.get_attribute_dict() for vertex in self.vertices}

    def order_edges_by_hash_codes(self) -> Dict[str, Edge]:
        edges = {}
        for edge in self.edges:
            edge_data = {
                "edge_label": edge.label,
                "from_vertex_hash": self.get_vertex_hash_by_index(vertex_index=edge.origin),
                "to_vertex_hash": self.get_vertex_hash_by_index(vertex_index=edge.dest),
            }
            edge_hash = calculate_hash(edge_data)
            edges[edge_hash] = edge
        return edges

    def get_vertex_hash_by_index(self, vertex_index: int) -> str:
        return self.vertex_hash_cache.setdefault(vertex_index, self.vertices[vertex_index].get_hash())

    def update_vertex_attribute(
        self,
        vertex_index: int,
        attribute_key: str,
        attribute_value: Any,
        change_origin_id: int,
        attribute_at_dest: Optional[Union[str, List[str]]],
    ) -> None:
        previous_breadcrumbs = []
        attribute_at_dest = self.vertices[change_origin_id].find_attribute(attribute_at_dest)
        if attribute_at_dest:
            previous_breadcrumbs = self.vertices[change_origin_id].changed_attributes.get(attribute_at_dest, [])
        self.vertices[vertex_index].update_attribute(
            attribute_key, attribute_value, change_origin_id, previous_breadcrumbs, attribute_at_dest
        )

    def update_vertices_configs(self) -> None:
        for vertex in self.vertices:
            changed_attributes = list(vertex.changed_attributes.keys())
            changed_attributes = filter_sub_keys(changed_attributes)
            self.update_vertex_config(vertex, changed_attributes)

    def update_vertex_config(self, vertex: TerraformBlock, changed_attributes: Union[List[str], Dict[str, Any]], dynamic_blocks: bool = False) -> None:
        if not changed_attributes:
            # skip, if there is no change
            return

        updated_config = deepcopy(vertex.config)
        if vertex.block_type != BlockType.LOCALS:
            parts = vertex.name.split(".")
            start = 0
            end = 1
            while end <= len(parts):
                cur_key = ".".join(parts[start:end])
                if cur_key in updated_config:
                    updated_config = updated_config[cur_key]
                    start = end
                end += 1
        for changed_attribute in changed_attributes:
            new_value = vertex.attributes.get(changed_attribute, None)
            if new_value is not None:
                if vertex.block_type == BlockType.LOCALS:
                    changed_attribute = changed_attribute.replace(vertex.name + ".", "")
                updated_config = update_dictionary_attribute(updated_config, changed_attribute, new_value, dynamic_blocks)

        if len(changed_attributes) > 0:
            if vertex.block_type == BlockType.LOCALS:
                updated_config = updated_config.get(vertex.name)
            update_dictionary_attribute(vertex.config, vertex.name, updated_config, dynamic_blocks)

    def get_resources_types_in_graph(self) -> List[str]:
        return self.module.get_resources_types()

    def update_vertices_breadcrumbs_and_module_connections(self) -> None:
        """
        The function processes each vertex's breadcrumbs:
        1. Get more data to each vertex in breadcrumb (name, path, hash and type)
        2. If a breadcrumb is originated in a different module, it will have 'module_connection'=True
        3. If a vertex has a 'source module' we will add a special breadcrumb for it
        """
        for vertex in self.vertices:
            for attribute_key, breadcrumbs_list in vertex.changed_attributes.items():
                hash_breadcrumbs = []
                for breadcrumb in breadcrumbs_list:
                    v = self.vertices[breadcrumb.vertex_id]
                    breadcrumb = v.get_export_data()
                    breadcrumb["module_connection"] = self._determine_if_module_connection(breadcrumbs_list, v)
                    hash_breadcrumbs.append(breadcrumb)
                vertex.breadcrumbs[attribute_key] = hash_breadcrumbs
            if len(vertex.source_module) == 1:
                v = vertex
                source_module_data = []
                while len(v.source_module) == 1:
                    idx = list(v.source_module)[0]
                    v = self.vertices[idx]
                    module_data = v.get_export_data()
                    module_data["idx"] = idx
                    source_module_data.append(module_data)
                source_module_data.reverse()
                vertex.breadcrumbs[CustomAttributes.SOURCE_MODULE] = source_module_data

    @staticmethod
    def _determine_if_module_connection(breadcrumbs_list: List[int], vertex_in_breadcrumbs: TerraformBlock) -> bool:
        """
        :param breadcrumbs_list: list of vertex's breadcrumbs
        :param vertex_in_breadcrumbs: one of the vertices in the breadcrumb list
        :return: True if vertex_in_breadcrumbs has in its module_connections at least one of the vertices in breadcrumbs_list
        """
        if not vertex_in_breadcrumbs.module_connections:
            return False
        for connection_list in vertex_in_breadcrumbs.module_connections.values():
            if any(i in breadcrumbs_list for i in connection_list):
                return True
        return False

    def get_dirname(self, path: str) -> str:
        dir_name = self.dirname_cache.get(path)
        if not dir_name:
            dir_name = os.path.dirname(path)
            self.dirname_cache[path] = dir_name
        return dir_name

    def get_abspath(self, path: str) -> str:
        dir_name = self.abspath_cache.get(path)
        if not dir_name:
            dir_name = os.path.abspath(path)
            self.abspath_cache[path] = dir_name
        return dir_name

    def update_nested_modules_address(self) -> None:
        for vertex in self.vertices:
            if vertex.block_type not in parser_registry.context_parsers:
                continue
            source_module = vertex.breadcrumbs.get(CustomAttributes.SOURCE_MODULE)

            address_prefix = ''
            if source_module:
                for module in source_module:
                    address_prefix += f"{module.get('type')}.{module.get('name')}."

            context_parser = parser_registry.context_parsers[vertex.block_type]
            entity_context_path = context_parser.get_entity_context_path(vertex.config)
            resource_id = '.'.join(entity_context_path) if entity_context_path else vertex.name
            address = f'{address_prefix}{resource_id}'
            vertex.attributes[CustomAttributes.TF_RESOURCE_ADDRESS] = address

            vertex_context = vertex.config
            definition_path = context_parser.get_entity_definition_path(vertex.config)
            for path in definition_path:
                vertex_context = vertex_context.get(path, vertex_context)
            vertex_context[CustomAttributes.TF_RESOURCE_ADDRESS] = address

def to_list(data):
    if isinstance(data, list) and len(data) == 1 and (isinstance(data[0], str) or isinstance(data[0], int)):
        return data
    elif isinstance(data, list):
        return [to_list(x) for x in data]
    elif isinstance(data, dict):
        return {key: to_list(val) for key, val in data.items()}
    else:
        return [data]

def update_dictionary_attribute(
        config: Union[List[Any], Dict[str, Any]], key_to_update: str, new_value: Any, dynamic_blocks: bool = False
) -> Union[List[Any], Dict[str, Any]]:
    key_parts = key_to_update.split(".")

    if isinstance(config, dict) and isinstance(key_parts, list):
        key = key_parts[0]
        inner_config = config.get(key)

        if inner_config is not None:
            if len(key_parts) == 1:
                if isinstance(inner_config, list) and not isinstance(new_value, list):
                    new_value = [new_value]
                config[key] = to_list(new_value) if dynamic_blocks else new_value
                return config
            else:
                config[key] = update_dictionary_attribute(inner_config, ".".join(key_parts[1:]), new_value, dynamic_blocks=dynamic_blocks)
        else:
            for key in config:
                config[key] = update_dictionary_attribute(config[key], key_to_update, new_value, dynamic_blocks=dynamic_blocks)
    if isinstance(config, list):
        return update_list_attribute(
            config=config,
            key_parts=key_parts,
            key_to_update=key_to_update,
            new_value=new_value,
            dynamic_blocks=dynamic_blocks
        )
    return config

def update_list_attribute(
    config: list[Any], key_parts: list[str], key_to_update: str, new_value: Any, dynamic_blocks: bool = False
) -> list[Any] | dict[str, Any]:
    """Updates a list attribute in the given config"""

    if not config:
        # happens when we can't correctly evaluate something, because of strange defaults or 'for_each' blocks
        return config

    if len(key_parts) == 1:
        idx = force_int(key_parts[0])
        inner_config = config[0]

        if idx is not None and isinstance(inner_config, list):
            if not inner_config:
                # happens when config = [[]]
                return config

            inner_config[idx] = new_value
            return config
    entry_to_update = int(key_parts[0]) if key_parts[0].isnumeric() else -1
    for i, config_value in enumerate(config):
        if entry_to_update == -1:
            config[i] = update_dictionary_attribute(config=config_value, key_to_update=key_to_update, new_value=new_value, dynamic_blocks=dynamic_blocks)
        elif entry_to_update == i:
            config[i] = update_dictionary_attribute(config=config_value, key_to_update=".".join(key_parts[1:]), new_value=new_value, dynamic_blocks=dynamic_blocks)

    return config

def get_path_with_nested_modules(block: TerraformBlock) -> str:
    if not block.module_dependency:
        return block.path
    if not strtobool(os.getenv('CHECKOV_ENABLE_NESTED_MODULES', 'True')):
        return unify_dependency_path([block.module_dependency, block.path])
    return get_tf_definition_key_from_module_dependency(block.path, block.module_dependency, block.module_dependency_num)


#!/usr/bin/env python2

# transpiled with BefunCompile v1.3.0 (c) 2017
import sys
def td(a,b):
    return ((0)if(b==0)else(a//b))
def tm(a,b):
    return ((0)if(b==0)else(a%b))
s=[]
def sp():
    global s
    if (len(s) == 0):
        return 0
    return s.pop()
def sa(v):
    global s
    s.append(v)
def sr():
    global s
    if (len(s) == 0):
        return 0
    return s[-1]
x0=1
x1=2
x2=2
def _0():
    sa(0)
    return 1
def _1():
    global x0
    global x1
    global t0
    sp();
    sa(x0+x1)
    t0=x0+x1
    x0=x1
    x1=t0

    return (4)if(sr()>10240000)else(2)
def _2():
    global t0
    global t1
    sa(sr());
    t0=(sr()/2)*2
    sa(sp()-t0)

    t1=sp()

    return (1)if((t1)!=0)else(3)
def _3():
    global x2
    sa(sp()+x2)

    x2=sp()
    sa(0)
    return 1
def _4():
    global x2
    sys.stdout.write(str(x2)+" ")
    sys.stdout.flush()

    sp();
    return 5
m=[_0,_1,_2,_3,_4]
c=0
while c<5:
    c=m[c]()


# Group: Daniel Martín Cruz, Ion Bueno Ulacia

import pmdarima as pm
import pandas as pd
import matplotlib.pyplot as plt

def auto_arima(train, start_p=1, start_q=1, test='adf'):
    automodel = pm.auto_arima(train,
                              start_p=start_p,
                              start_q=start_q,
                              test=test,
                              seasonal=False,
                              trace=False,
                              error_action="ignore")
    return automodel

def plot_auto_arima(train, test, automodel, steps, location):
    if steps is None:
        steps = len(test)
    # Forecast
    fc, _ = automodel.predict(n_periods=steps, return_conf_int=True)
    # Weekly index
    fc_ind = pd.date_range(train.index[train.shape[0]-1], periods=steps)
    # Forecast series
    fc_series = pd.Series(fc, index=fc_ind)
    aux_df = pd.DataFrame({'Training': train, 'Test': test, 'Forecast': fc_series})
    # Create plot
    ax = aux_df.plot(figsize=(10, 6))
    fig = ax.get_figure()
    plt.xlabel('Date')
    plt.ylabel(train.name)
    plt.legend(("Training", "Test", "Forecast"), loc="upper left")
    plt.title(f'AUTO-ARIMA model in {location} for {train.name} cases')
    return ax

def diagnostics_auto_arima(automodel, location, target):
    fig = automodel.plot_diagnostics(figsize=(10,6))
    plt.suptitle(f'Residual plots for AUTO-ARIMA model in {location} for {target} cases')
    fig.savefig('../plots/' + location + "_" + target + '_diag.png')
    return fig


# coding: utf-8

from __future__ import absolute_import
from datetime import date, datetime  # noqa: F401

from typing import List, Dict  # noqa: F401

from openapi_server.models.base_model_ import Model
from openapi_server import util

class ResponseTxModel(Model):
    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).

    Do not edit the class manually.
    """

    def __init__(self, code=None, textdata_list=None):  # noqa: E501
        """ResponseTxModel - a model defined in OpenAPI

        :param code: The code of this ResponseTxModel.  # noqa: E501
        :type code: int
        :param textdata_list: The textdata_list of this ResponseTxModel.  # noqa: E501
        :type textdata_list: List[str]
        """
        self.openapi_types = {
            'code': int,
            'textdata_list': List[str]
        }

        self.attribute_map = {
            'code': 'code',
            'textdata_list': 'textdata_list'
        }

        self._code = code
        self._textdata_list = textdata_list

    @classmethod
    def from_dict(cls, dikt) -> 'ResponseTxModel':
        """Returns the dict as a model

        :param dikt: A dict.
        :type: dict
        :return: The ResponseTxModel of this ResponseTxModel.  # noqa: E501
        :rtype: ResponseTxModel
        """
        return util.deserialize_model(dikt, cls)

    @property
    def code(self):
        """Gets the code of this ResponseTxModel.

        :return: The code of this ResponseTxModel.
        :rtype: int
        """
        return self._code

    @code.setter
    def code(self, code):
        """Sets the code of this ResponseTxModel.

        :param code: The code of this ResponseTxModel.
        :type code: int
        """

        self._code = code

    @property
    def textdata_list(self):
        """Gets the textdata_list of this ResponseTxModel.

        :return: The textdata_list of this ResponseTxModel.
        :rtype: List[str]
        """
        return self._textdata_list

    @textdata_list.setter
    def textdata_list(self, textdata_list):
        """Sets the textdata_list of this ResponseTxModel.

        :param textdata_list: The textdata_list of this ResponseTxModel.
        :type textdata_list: List[str]
        """

        self._textdata_list = textdata_list


import pandas as pd
import numpy as np
import json
from flask import Flask
from flask_cors import CORS

import datetime

df = pd.read_csv('train-Copy.csv')
df.plot()
ds1 = df['ds'].tail(7)
y1 = df['y'].tail(7)
ds = np.array(ds1)
y = np.array(np.float64(y1))

ds0 = pd.to_datetime(ds[0])
ds1 = pd.to_datetime(ds[1])
ds2 = pd.to_datetime(ds[2])
ds3 = pd.to_datetime(ds[3])
ds4 = pd.to_datetime(ds[4])
ds5 = pd.to_datetime(ds[5])
ds6 = pd.to_datetime(ds[6])

def myconverter(o):
    if isinstance(o, datetime.datetime):
        return o.__str__()

ds0_json = json.dumps(ds0, default = myconverter)
ds1_json = json.dumps(ds1, default = myconverter)
ds2_json = json.dumps(ds2, default = myconverter)
ds3_json = json.dumps(ds3, default = myconverter)
ds4_json = json.dumps(ds4, default = myconverter)
ds5_json = json.dumps(ds5, default = myconverter)
ds6_json = json.dumps(ds5, default = myconverter)

class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return json.JSONEncoder.default(self, obj)

json_dump = json.dumps({'ds0':ds0_json,'y0':y[0],'ds1':ds1_json,'y1':y[1],'ds2':ds2_json,'y2':y[2],'ds3':ds3_json,'y3':y[3],'ds4':ds4_json,'y4':y[4],'ds5':ds5_json,'y5':y[5],
                      'ds6':ds6_json,'y6':y[6]}, cls=NumpyEncoder)
print(json_dump)

app = Flask(__name__)
#if __name__ == "__main__":
#    app.run(host='132.186.199.33 ',port='80')
app.config["DEBUG"] = False
cors = CORS(app, resources={r"/*": {"origins": "*"}})
@app.route("/historical", methods=['GET'])
def hello():
    return json_dump


import sys
mn = input('Bem vindo a torre de hanoi de Vitor e Bet voce deseja jogar?{S/N}:').upper()
if mn == 'N':
    print ('tchau'), sys.exit()

else:
    print ('''Regras:A Torre de Hanói é um quebra-cabeça que consiste em três hastes e discos de tamanhos diferentes
            que são posicionados na primeira haste de forma decrescente. O objetivo do quebra-cabeça
            é mover todos os discos para última haste.
            Para isso pode-se mover apenas um disco de cada vez para uma das hastes.
            Os discos podem ser empilhados, no entanto um disco nunca pode ser empilhado sobre outro disco menor que ele.''')
try:
    block =int(input ('Voce deseja jogar com quantos bloquinhos: '))

except ValueError:
    block = int(input('Voce deseja jogar com quantos bloquinhos: '))

a = list(range(1,block+1)[::-1])
b = []
c = []
marcio = 0
while marcio == 0:
    print("A -",a)
    print("B -",b)
    print("C -",c)
    if c == list(range(1,block+1)[::-1]):
        print("Parabés você GANHOU!!!")
        marcio+=1
        sys.exit()
    var = 0
    tira= input("De qual pilha você deseja retirar o objeto:").upper()
    coloca = input("Em qual pilha você deseja colocar esse objeto:").upper()
    if tira == "A":
        var = a.pop()
    elif tira == "B":
        var = b.pop()
    elif tira == "C":
        var= c.pop()
    else:
        print ("valor inválido")
        continue
    jooj= 0
    while jooj == 0:
        if coloca == "A":
            if len(a)==0:
                a.append(var)
                jooj+=1
            elif a[len(a)-1] < var:
                coloca= input("Digite uma pilha válida").upper()
            else:
                a.append(var)
                jooj+=1

        elif coloca == "B":
            if len(b)==0:
                b.append(var)
                jooj+=1
            elif b[len(b)-1] < var:
                coloca= input("Digite uma pilha válida:").upper()
            else:
                b.append(var)
                jooj+=1

        elif coloca == "C":
            if len(c) == 0:
                c.append(var)
                jooj+=1
            elif c[len(c)-1] < var:
                coloca=input("Digite uma pilha válida:").upper()
            else:
                c.append(var)
                jooj+=1


# encoding: utf-8
"""
@author : zhirui zhou
@contact: evilpsycho42@gmail.com
@time   : 2019/11/7 14:14
"""

def walk_forward_split(series_idxes, encode_lens, decode_lens, test_size):
    n = len(series_idxes) - encode_lens - decode_lens + 1
    if test_size < 1:
        test_size = int(test_size * n)
    train_index = series_idxes[: -test_size]
    valid_index = series_idxes[-(decode_lens + test_size - 1 + encode_lens):]
    return train_index, valid_index


from Load import *
from GeneratorBSP import *
from random import randint
import math

class Goblin(pygame.sprite.Sprite):
    def __init__(self, game, x, y, target): #weapon too
        self.game = game
        self.groups = game.all_sprites, game.mobs
        pygame.sprite.Sprite.__init__(self, self.groups)

        self.image = pygame.Surface((TILESIZE - int(TILESIZE/2), TILESIZE - int(TILESIZE/2)))
        self.image.fill(GREEN)
        self.rect = self.image.get_rect()
        self.target = target
        #self.x = x
        #self.y = y
        #self.rect.x = x * TILESIZE
        #self.rect.y = y * TILESIZE

        self.pos = vec(x, y) * TILESIZE
        self.vel = vec(0, 0)
        self.acc = vec(0, 0)
        self.rect.center = self.pos
        self.rot = 0
        self.speed = 30

    def stuff(self):
            # self.speedx, self.speedy = 1, 1
            # self.vx, self.vy = 0, 0
            # self.movetime = 100
            # self.attacktime = 0

            # Find the center of their original room
            # This will be used to find the radius it wanders in (to prevent ai from moving into corners)
            for u in range(len(dungeon.rooms)):
                if y > dungeon.rooms[u].row and y < dungeon.rooms[u].row + dungeon.rooms[u].height:
                    if x > dungeon.rooms[u].col and y < dungeon.rooms[u].col + dungeon.rooms[u].width:
                        self.spawnx = (dungeon.rooms[u].col + dungeon.rooms[u].width//2) * TILESIZE
                        self.spawny = (dungeon.rooms[u].row + dungeon.rooms[u].height//2) * TILESIZE
                        self.spawnw = (dungeon.rooms[u].width - dungeon.rooms[u].width/1.9) * TILESIZE
                        self.spawnh = (dungeon.rooms[u].height - dungeon.rooms[u].height/1.9) * TILESIZE
    def move(self):
        # If close to player, chase him
        if math.hypot(self.target.rect.x - self.rect.x, self.target.rect.y - self.rect.y) < 300:
            self.attack()
            if self.rect.x < self.target.rect.x:
                self.vx = self.speedx
            elif self.rect.x > self.target.rect.x:
                self.vx = -self.speedx
            elif self.rect.x == self.target.rect.x:
                self.vx = 0 # to prevent wobbling

            if self.rect.y < self.target.rect.y:
                self.vy = self.speedy
            elif self.rect.y > self.target.rect.y:
                self.vy = -self.speedy
            elif self.rect.y == self.target.rect.y:
                self.vy = 0 # to prevent wobbling

        # If not chasing and in moving radius (slightly smaller than the room), move randomly
        elif math.hypot(self.spawnx - self.rect.x, self.spawny - self.rect.y) < min(self.spawnw, self.spawnh):
            rand_move = randint(1,5)
            self.movetime += 1
            if self.movetime > 100:
                self.movetime = 0
                if rand_move == 1:
                    self.vx = self.speedx
                if rand_move == 2:
                    self.vx = -self.speedx
                if rand_move == 3:
                    self.vy = self.speedy
                if rand_move == 4:
                    self.vy = -self.speedy
                if rand_move == 5:
                    self.vx, self.vy = 0, 0

        # If not chasing and out of spawn radius, go back near spawn
        else:
            if self.rect.x < self.spawnx:
                self.vx = self.speedx
            elif self.rect.x > self.spawnx:
                self.vx = -self.speedx
            elif self.rect.x == self.spawnx:
                self.vx = 0 # to prevent wobbling

            if self.rect.y < self.spawny:
                self.vy = self.speedy
            elif self.rect.y > self.spawny:
                self.vy = -self.speedy
            elif self.rect.y == self.spawny:
                self.vy = 0 # to prevent wobbling

    def attack(self):
        self.attacktime += 1
        if self.attacktime > 50:
            self.attacktime = 0
            bullet = EnemyBullets(self.game, self.rect.center[0], self.rect.center[1], self.target)

    def wallcollision(self):
        hits = pygame.sprite.groupcollide(self.game.mobs, self.game.walls, False, False)
        if hits:
            if self.vx > 0:
                self.x = hits[0].rect.left - self.rect.width
            if self.vx < 0:
                self.x = hits[0].rect.right
            self.vx = 0
            self.rect.x = self.x

            if self.vy > 0:
                self.y = hits[0].rect.top - self.rect.height
            if self.vy < 0:
                self.y = hits[0].rect.bottom
            self.vy = 0
            self.rect.y = self.y

    def avoid_mobs(self):
        for mob in self.game.mobs:
            if mob != self:
                dist = self.pos - mob.pos
                if 0 < dist.length() < TILESIZE*2:
                    self.acc += dist.normalize()

    def update(self):
        self.rot = (self.target.rect.center - self.pos).angle_to(vec(1, 0))
        #self.image = pygame.transform.rotate(pygame.Surface((TILESIZE - int(TILESIZE/2), TILESIZE - int(TILESIZE/2))), self.rot)
        #self.rect = self.image.get_rect()
        self.rect.center = self.pos
        self.acc = vec(self.speed, 0).rotate(-self.rot)
        self.avoid_mobs()
        self.acc.scale_to_length(self.speed)
        self.acc += self.vel * -1
        self.vel += self.acc * 0.06
        self.pos += self.vel * 0.06 + 0.5 * self.acc * 0.06 ** 2
        self.rect.centerx = self.pos.x
        self.rect.centery = self.pos.y
        self.wallcollision()
        print(self.rect.center, self.target.rect.center)

        # self.move()
        # self.rect.x += self.vx
        # self.rect.y += self.vy
        # self.collision()

class EnemyBullets(pygame.sprite.Sprite):
    def __init__(self, game, x, y, target):
        self.game = game
        self.groups = game.all_sprites, game.enemybullets
        pygame.sprite.Sprite.__init__(self, self.groups)

        self.image = pygame.Surface((5,5))
        self.image.fill(RED)
        self.rect = self.image.get_rect()
        self.rect.x = x
        self.rect.y = y

        self.posx = self.rect.x
        self.posy = self.rect.y
        self.changex = 0
        self.changey = 0
        self.dx = target.rect.x - self.rect.x # X-Distance
        self.dy = target.rect.y - self.rect.y # Y-Distance
        self.dist = math.sqrt(self.dx ** 2 + self.dy ** 2) # Total Distance between Enemy and Player
        self.dx = (self.dx / self.dist) * 10
        self.dy = (self.dy / self.dist) * 10
        self.changex += self.dx
        self.changey += self.dy

        self.lifespan = 0

    def update(self):
        self.posx += self.changex
        self.posy += self.changey
        self.rect.x = self.posx
        self.rect.y = self.posy

        if pygame.sprite.spritecollideany(self, self.game.walls):
            self.kill()
        if self.lifespan > 50:
            self.kill()
        self.lifespan += 1


from lib2to3.pytree import Leaf

from django.forms import ModelForm,ModelMultipleChoiceField,Form,ChoiceField
from Studierendenverwaltung.models import *

class LehveranstaltungForm(ModelForm):

    class Meta:
        model = Lehveranstaltungen
        fields = ['Kurzname','Title']

class EinschreibungForm(ModelForm):

    class Meta:
        model = Lehveranstaltungen
        exclude = ['Kurzname','Title']

def get_my_choice ():
    Select_CHOICES = [[lehver.id, lehver.Title] for lehver in Lehveranstaltungen.objects.all()]
    return  Select_CHOICES

class SelectForm(Form):

    lehveranstaltung = ChoiceField(choices=get_my_choice())

    # def __init__(self, *args, **kwargs):
    #     super(SelectForm,self).__init__(*args,**kwargs)
    #     self.fields['Title'] = ChoiceField(choices = get_my_choice())

    #select = ModelMultipleChoiceField(choices=Select_CHOICES, widget=ModelMultipleChoiceField,queryset=Lehveranstaltungen.objects.all() ,required=False)


'''
    Usage: python threaded_object_detection.py
'''
import os
import cv2
import numpy as np
from imutils.video import FPS
from imutils.video import WebcamVideoStream
from config import PATH_TO_LABELS, path_to_cfg, path_to_ckpt

# Suppress TensorFlow logging
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow as tf
from object_detection.utils import label_map_util, config_util
from object_detection.builders import model_builder
from object_detection.utils import visualization_utils as viz_utils

# Suppress TensorFlow logging (2)
tf.get_logger().setLevel('ERROR')

print("[INFO] Building model pipeline and detector...")
configs = config_util.get_configs_from_pipeline_file(path_to_cfg(model='ssdmobilenet_v2'))
model_config = configs['model']
detector = model_builder.build(model_config=model_config, is_training=False)

print("[INFO] Restoring model checkpoint...")
PATH_TO_RESTORE = os.path.join(path_to_ckpt(model='ssdmobilenet_v2'), 'ckpt-0')
ckpt = tf.compat.v2.train.Checkpoint(model=detector)
ckpt.restore(PATH_TO_RESTORE).expect_partial()

@tf.function
def detect(img):
    img, shapes = detector.preprocess(img)
    prediction_dict = detector.predict(img, shapes)
    detections = detector.postprocess(prediction_dict, shapes)

    return (detections, prediction_dict, tf.reshape(shapes, [-1]))

#Load label map
category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)

# Create the file video stream object and start reading the FPS
vs = WebcamVideoStream().start()
fps = FPS().start()
print("[INFO] Starting video stream...")

while fps._numFrames<800:
    frame = vs.read()

    frame2 = frame
    frame = np.expand_dims(frame, axis=0)

    input_tensor = tf.convert_to_tensor(frame, dtype=tf.float32)
    (detections, predictions_dict, shapes) = detect(input_tensor)

    label_id_offset = 1
    frame2 = frame2.copy()

    viz_utils.visualize_boxes_and_labels_on_image_array(
          frame2,
          detections['detection_boxes'][0].numpy(),
          (detections['detection_classes'][0].numpy() + label_id_offset).astype(int),
          detections['detection_scores'][0].numpy(),
          category_index,
          use_normalized_coordinates=True,
          max_boxes_to_draw=200,
          min_score_thresh=.50,
          agnostic_mode=False)

    # Show the frame and update the FPA
    cv2.imshow('Webcam', frame2)
    fps.update()

    key = cv2.waitKey(25) & 0xFF
    if key == ord('q'):
        print("[INFO] Ending video stream...")
        fps.stop()
        break

fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approximate fps: {:.2f}".format(fps.fps()))

cv2.destroyAllWindows()
vs.stop()


# Copyright (c) 2016, Daniele Venzano
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from zoe_lib.exceptions import InvalidApplicationDescription
from zoe_scheduler.state.base import BaseState

class Application(BaseState):

    api_out_attrs = ['name', 'version', 'will_end', 'priority', 'requires_binary']
    api_in_attrs = ['name', 'version', 'will_end', 'priority', 'requires_binary']

    def __init__(self, state):
        super().__init__(state)

        self.user = None
        self.executions = []

        self.name = ''
        self.version = 0
        self.will_end = True
        self.priority = 512
        self.requires_binary = False
        self.processes = []

    def to_dict(self, checkpoint):
        d = super().to_dict(checkpoint)
        d['processes'] = []
        for p in self.processes:
            d['processes'].append(p.to_dict())

        if checkpoint:
            d['user_id'] = self.user.id
        return d

    def from_dict(self, data, checkpoint):
        super().from_dict(data, checkpoint)

        try:
            self.version = int(self.version)
        except ValueError:
            raise InvalidApplicationDescription(msg="version field should be an int")
        except KeyError:
            raise InvalidApplicationDescription(msg="Missing required key: version")

        try:
            self.will_end = bool(self.will_end)
        except ValueError:
            raise InvalidApplicationDescription(msg="will_end field must be a boolean")

        try:
            self.requires_binary = bool(self.requires_binary)
        except ValueError:
            raise InvalidApplicationDescription(msg="requires_binary field must be a boolean")

        try:
            self.priority = int(self.priority)
        except ValueError:
            raise InvalidApplicationDescription("priority field must be an int")
        if self.priority < 0 or self.priority > 1024:
            raise InvalidApplicationDescription("priority must be between 0 and 1024")

        for p in data['processes']:
            aux = Process()
            aux.from_dict(p)
            self.processes.append(aux)

        found_monitor = False
        for p in self.processes:
            if p.monitor:
                found_monitor = True
                break
        if not found_monitor:
            raise InvalidApplicationDescription("at least one process should have monitor set to True")

        user = self.state_manger.get_one('user', id=data['user_id'])
        if user is None:
            raise InvalidApplicationDescription('Deserialized application points to a non-existent user')
        self.user = user
        user.applications.append(self)

    @property
    def owner(self):
        return self.user

    def total_memory(self) -> int:
        memory = 0
        for p in self.processes:
            memory += p.required_resources['memory']
        return memory

    def container_count(self) -> int:
        return len(self.processes)

    def add_execution(self, execution):
        execution.application = self
        self.executions.append(execution)

class ProcessEndpoint:
    def __init__(self):
        self.name = ''
        self.protocol = ''
        self.port_number = 0
        self.path = ''
        self.is_main_endpoint = False

    def to_dict(self):
        d = {
            'name': self.name,
            'protocol': self.protocol,
            'port_number': self.port_number,
            'path': self.path,
            'is_main_endpoint': self.is_main_endpoint
        }
        return d

    def from_dict(self, data):
        required_keys = ['name', 'protocol', 'port_number', 'is_main_endpoint']
        for k in required_keys:
            try:
                setattr(self, k, data[k])
            except KeyError:
                raise InvalidApplicationDescription(msg="Missing required key: %s" % k)

        try:
            self.port_number = int(self.port_number)
        except ValueError:
            raise InvalidApplicationDescription(msg="port_number field should be an integer")

        try:
            self.is_main_endpoint = bool(self.is_main_endpoint)
        except ValueError:
            raise InvalidApplicationDescription(msg="is_main_endpoint field should be a boolean")

        if 'path' in data:
            self.path = data['path']

    def get_url(self, address):
        return self.protocol + "://" + address + ":{}".format(self.port_number) + self.path

class Process:
    def __init__(self):
        self.name = ''
        self.version = 0
        self.docker_image = ''
        self.monitor = False  # if this process dies, the whole application is considered as complete and the execution is terminated
        self.ports = []  # A list of ProcessEndpoints
        self.required_resources = {}
        self.environment = []  # Environment variables to pass to Docker
        self.command = None  # Commandline to pass to the Docker container

    def to_dict(self):
        d = {
            'name': self.name,
            'docker_image': self.docker_image,
            'monitor': self.monitor,
            'required_resources': self.required_resources,
            'environment': self.environment,
            'command': self.command,
            'ports': []
        }
        for p in self.ports:
            d['ports'].append(p.to_dict())
        return d

    def from_dict(self, data):
        required_keys = ['name', 'docker_image', 'monitor']
        for k in required_keys:
            try:
                setattr(self, k, data[k])
            except KeyError:
                raise InvalidApplicationDescription(msg="Missing required key: %s" % k)

        try:
            self.monitor = bool(self.monitor)
        except ValueError:
            raise InvalidApplicationDescription(msg="monitor field should be a boolean")

        if 'ports' not in data:
            raise InvalidApplicationDescription(msg="Missing required key: ports")
        if not hasattr(data['ports'], '__iter__'):
            raise InvalidApplicationDescription(msg='ports should be an iterable')
        for pp in data['ports']:
            aux = ProcessEndpoint()
            aux.from_dict(pp)
            self.ports.append(aux)

        if 'required_resources' not in data:
            raise InvalidApplicationDescription(msg="Missing required key: required_resources")
        if not isinstance(data['required_resources'], dict):
            raise InvalidApplicationDescription(msg="required_resources should be a dictionary")
        if 'memory' not in data['required_resources']:
            raise InvalidApplicationDescription(msg="Missing required key: required_resources -> memory")

        self.required_resources = data['required_resources'].copy()
        try:
            self.required_resources['memory'] = int(self.required_resources['memory'])
        except ValueError:
            raise InvalidApplicationDescription(msg="required_resources -> memory field should be an int")

        if 'environment' in data:
            if not hasattr(data['environment'], '__iter__'):
                raise InvalidApplicationDescription(msg='environment should be an iterable')
            self.environment = data['environment'].copy()
            for e in self.environment:
                if len(e) != 2:
                    raise InvalidApplicationDescription(msg='environment variable should have a name and a value')
                if not isinstance(e[0], str):
                    raise InvalidApplicationDescription(msg='environment variable names must be strings: {}'.format(e[0]))
                if not isinstance(e[1], str):
                    raise InvalidApplicationDescription(msg='environment variable values must be strings: {}'.format(e[1]))

        if 'command' in data:
            self.command = data['command']


import RPi.GPIO as GPIO
GPIO.setmode(GPIO.BCM)
from signalrManager import *
from displayManager import *
from cameraManager import *
import threading
from loggingManager import *

def log(message, withTimestamp=False):
    logInfo('buttonEventDetectSample', message, withTimestamp)

inputPin = 23
GPIO.setup(inputPin,GPIO.IN,pull_up_down=GPIO.PUD_UP)

## Define button status variables
firstButtonClickRegistered = False
firstButtonClickRegisteredAt = time.time()
secondButtonClickRegistered = False
## Define button status variables

def expectSecondClickFor(secondsToWait):
    log("Started waiting for {0} seconds...".format(secondsToWait))
    t_end = time.time() + secondsToWait
    while time.time() < t_end:
        continue
    log("Waiting for second click finished")

    if not secondButtonClickRegistered:
        log("Showing home screen..")
        showHomeScreen()
        log("Home screen shown!")
        resetButtonStates()

def resetButtonStates():
    global firstButtonClickRegistered
    global secondButtonClickRegistered

    firstButtonClickRegistered = False
    secondButtonClickRegistered = False

def buttonPressHandler(channel):
    global firstButtonClickRegistered
    global secondButtonClickRegistered
    global firstButtonClickRegisteredAt

    log("BUTTON CLICKED", True)

    if firstButtonClickRegistered and secondButtonClickRegistered:
        resetButtonStates()

    if not firstButtonClickRegistered:
        firstButtonClickRegistered = True
        log("Showing the 'Please stepback and push' message...")
        goOnMarkerAndPushButton()
        log("Message shown!")
        # Start counting the seconds till the second click. If no click was registered
        t1 = threading.Thread(target=expectSecondClickFor, args=[15])
        #Background thread will finish with the main program
        t1.setDaemon(True)
        #Start expectSecondClickFor(n) in a separate thread
        t1.start()
        log("Update datetime first time button was clicked")
        firstButtonClickRegisteredAt = time.time()

    elif (firstButtonClickRegistered) and (not secondButtonClickRegistered) and (time.time()-firstButtonClickRegisteredAt > 3):
        log(time.time()-firstButtonClickRegisteredAt)
        secondButtonClickRegistered = True

        log("Showing the 'We are taking picture' message...")
        takingPicture()
        log("Message shown!")
        takePictureWithTheCamera()
        sendNewMailRequest()
        countDownFrom(9)

GPIO.add_event_detect(23,GPIO.RISING,callback=buttonPressHandler,bouncetime= 6000)

while(1):
      log("Showing home screen..")
      showHomeScreen()
      log("Home screen shown!")

      keypressed = raw_input('\n\n\n\n***** MENU ***** \n1. Press \'q\' to quit\n2. Press \'r\' to reset (NOT implemented yet)\n***** MENU *****\n\n\n ')

      if keypressed == 'q':
           GPIO.cleanup()
           break
      elif keypressed == 'r':
           GPIO.cleanup()
           log("Here we should [r]eset the script.")
      elif keypressed != 'q' and keypressed != 'r':
           log("You pressed other key:" + keypressed)
      else:
           log("Unknown input")


from __future__ import unicode_literals

from django.db import models
import re
from . import utils

class Input(models.Model):

	branch_url1 = models.CharField(
		verbose_name='Branch URL 1',
		max_length=255,
		)

	branch_url2 = models.CharField(
		verbose_name='Branch URL 2',
		max_length=255,
		)

	start_rev = models.IntegerField(
		verbose_name='Start Revision ',
		)

	end_rev = models.IntegerField(
		verbose_name='Stop Revision ',
		default=0,
		)

	head_rev = models.BooleanField(
		verbose_name='Head Revision ',
		default=False,
		)


import pandas as pd
import numpy as np
import re
import xlrd
import sys, getopt
import json
from pathlib import Path

TRANSLATION = {
    1:
    {'G':'Goal by {0}.',
    'GD':'Goal on the drive by {0}.',
    'GS':'Goal on the shot by {0}.',
    'OG':'Own Goal.',
    'E': 'Error by {0}.',
    'ET': 'Turnover penalty on {0}.',
    'EB': 'Blue card on {0} forces turnover.',
    'EY': 'Yellow card on {0} forces turnover.',
    'ER':'Red card on {0} forces turnover.',
    'EM':'Missed shot by {0}. Turnover.',
    'EP':'Errant pass by {0}. Turnover.',
    'ED':'Drop by {0}. Turnover.',
    'RCA':'Snitch catch by {0} is GOOD.',
    'OCA':'Snitch catch by {0} is GOOD.',
    '2CA':'Snitch catch by {0} is GOOD.',
    'RCB':'Snitch catch by {0} is GOOD.',
    'OCB':'Snitch catch by {0} is GOOD.',
    '2CB':'Snitch catch by {0} is GOOD.',
    'B':'blue card on {0}',
    'R': 'reset forced by {0}',
    'Y': 'yellow card on {0}',
    '2Y': 'second yellow card on {0}, {0} is ejected from the game',
    '1R': 'red card on {0},{0} is ejected from the game',
    'SOP':'SNITCH ON PITCH begins',
    'TIMEOUT':'TIMEOUT by {0}'
    },
    2:{
    'BU':'BROOMS UP! Quaffle Possession by {0}, Bludger Control by {1}',
    'G':'Goal by {0}, assist by {1}',
    'GD':'Goal on the drive by {0}, assist by {1}',
    'GS':'Goal on the shot by {0}, assist by {1}',
    'OG':'Own Goal',
    'GP':'{1} passes to {0} at the hoops, GOAL.',
    'T' :'Turnover by {0}',
    'TB':'Beat by {0} on {1} forces a TURNOVER.',
    'TBP':'Beater pressure by {0} on {1} forces a TURNOVER.',
    'TLB':'Bludger throw by {0} blocks shot/pass by {1}. Turnover',
    'TC':'Turnover forced by physical contact by {0} on {1}',
    'TL':'Shot by {1} blocked by {0}. Turnover.',
    'TD':'Pass between {1} defended by {0}. Turnover',
    'EP':'Errant pass from {0} in the direction of {1} . Turnover',
    'ED':'Pass by {1} dropped by {0}. Turnover'
    }
}

def get_info(fname):
    xl_workbook = xlrd.open_workbook(fname)
    data_sheet = xl_workbook.sheet_by_index(0)
    roster_sheet = xl_workbook.sheet_by_index(1)
    return data_sheet,roster_sheet

def get_roster(roster_sheet):
    datuu = [[s.value for s in r] for r in roster_sheet.get_rows()]
    coln = datuu.pop(0)
    df = pd.DataFrame(datuu,columns=coln)
    df[''] = df[''].apply(lambda x: int(x))
    return df.replace(r'^\s*$', np.NaN, regex=True).set_index('').to_dict()
def get_info_from_data(data_sheet):
    raw = [[s.value for s in r] for r in data_sheet.get_rows()]
    header = [[s for s in row if s] for row in raw[1:4]]
    data = raw[6:]
    return header,data

def get_teams_from_header(header):
    return {header[0][0]:header[0][1],header[2][0]:header[2][1]}

# Brooms Up
def get_brooms_up(header,roster,teams):
    vala,valb = ['-'.join([str(x) for x in h]) for h in header if h]
    qa = re.search(r'[Qq](\d+)',vala)
    qb = re.search(r'[Qq](\d+)',valb)
    ba = re.search(r'[Bb](\d+)',vala)
    bb = re.search(r'[Bb](\d+)',valb)
    q_params, b_params = [None,None], [None,None]
    if qa or qb:
        q_params = (int(qa.groups()[0]),'A')if qa else (int(qb.groups()[0]),'B')
    if ba or bb:
        b_params = (int(ba.groups()[0]),'A')if ba else (int(bb.groups()[0]),'B')
    extras,offense,time = [],'Brooms Up','0000'
    result,primary,secondary ='BU',get_name(roster,teams,*q_params),get_name(roster,teams,*b_params)
    return {'extras':extras,'offense':offense, 'defense':'','length':-1,'time':time,'result':result,'primary':[primary],'secondary':[secondary],'period':'FLOOR'}

# Can take two forms of player_number, team
# E.g to look up A-34
# Either player_number = 'A34', team = None or
# player_number = 34, team = 'A'
# Can take a list of player_numbers (of one format only)
def get_name(roster,teams,player_number,team=None):

    if player_number is None:    #Nonexistent player
        return None
    if type(player_number)==list:
        return ' and '.join([get_name(roster,teams,num,team) for num in player_number])

    #Convert
    team,number = (team,player_number) if team else (player_number[0],player_number[1:])
    team_name = teams[team]
    team_roster = roster[team]
    if number =='?':
        return '{}-UNK'.format(team_name)
    else:
        n = int(float(number))
    if n in team_roster:
        if team_roster[n]!=team_roster[n]: #check for np.nan
            return '{}-{}'.format(team_name,n) #insert TEAM-# if not in roster or TEAM-NAME if in roster
        else:
            return '{}-{}'.format(team_name,team_roster[n])
    return '{}-UNK'.format(team_name) #if number is not known, use TEAM-UNK

def get_possessions(data_rows):
    possessions = []
    for j in np.arange(0,len(data_rows)-2,3):
        head = data_rows[j]
        vals = data_rows[j+1]
        for i in np.arange(0,len(vals)-2,3):
            extras = head[i]
            team = head[i+1]
            time = head[i+2]
            result = vals[i]
            if vals[i+1]!='':
                primary = [x for x in vals[i+1].split(',')] if type(vals[i+1])==str  else vals[i+1]
            else:
                primary = None
            if vals[i+2]!='':
                secondary = [x for x in vals[i+2].split(',')] if type(vals[i+2])==str else vals[i+2]
            else:
                secondary = []
            pos = (extras,team,time,result,primary,secondary)
            if len([x for x in pos if x])>1:
                possessions.append(pos)
    return possessions
def get_extras(extra):
    if not extra:
        return  []
    else:
        extra_list = []
        for x in extra.split(','):
            if x[0].isdigit():
                extra_list.append( (x[:2],x[2:]))
            else:
                if len(x)>1:
                    extra_list.append((x[0],x[1:]))
                else:
                    extra_list.append((x[0],None))
        return extra_list
def process_extra(ex, roster,teams,offense,defense):
    etype,eplayer = ex
    if etype == 'S':
        return ('SOP',1)
    elif etype =='R':
        return ('R',get_name(roster,teams,eplayer,defense))
    elif etype =='B' or etype =='Y' or etype=='1R' or etype=='2Y':
        return (etype,get_name(roster,teams,eplayer))
    elif etype =='T':
        return ('TIMEOUT',teams[eplayer[0]])
    else:
        print(ex)
    return None
def interpret(pos,roster,teams):
    ex,team,time,result,primary,secondary = pos
    offense = team
    defense = 'A' if team=='B' else 'B'
    primary_team = ''
    if result=='RCA':
        primary_team = 'A'
    elif result == 'RCB':
        primary_team = 'B'
    elif result[0]=='T':
        primary_team = defense
    else:
        primary_team = offense
    secondary_team = team
    primary_name = get_name(roster,teams,primary,primary_team)
    secondary_name = get_name(roster,teams,secondary,secondary_team)
    extras = [process_extra(extra,roster,teams,offense,defense) for extra in get_extras(ex)]
    secondary_name = secondary_name.split(' and ') if secondary_name else []
    primary_name = primary_name.split(' and ') if primary_name else []
    period = ''
    if len(time)>=3:
        if time[0:2]=='SD':
            period ='2OT'
        elif time[0:2] =='OT':
            period ='OT'
        period ='SOP' if time >'1800' else 'FLOOR'
    return {'extras':extras,'offense':teams[offense], 'defense':teams[defense],'time':time, 'result':result, 'primary':primary_name,'secondary':secondary_name,'period':period}

def gen_pbp(interpreted):
    result = interpreted['result']
    time = interpreted['time']
    offense = interpreted['offense']
    extras = interpreted['extras']
    p = [x for x in [interpreted['primary'],interpreted['secondary']] if x]
    l = len(p)

    try:
        itime = int(time)
        time_str = '({0:02d}:{1:02d})'.format(itime//100,itime%100)
    except:
        time_str = '({})'.format(time) if time else ''
    base = '{}{} possession: {}'.format(time_str,offense, TRANSLATION[l][result].format(*p))
    if extras:
        base +=' During the play, '
        base +=','.join([TRANSLATION[1][extra[0]].format(extra[1]) for extra in extras])
    return base

def ind_stats(interpreted_list):
    columns = ['Goals','Assists','Ball TO Forced','Contact TO Forced','Beat TO Forced','Resets Forced','Errors','ISR Catches','OSR Catches','Blues','Yellows','Second Yellows','Straight Reds']
    stats = {c:{} for c in columns}
    qpd = 0
    t1 = None
    for pos in interpreted_list:
        res = pos['result']
        primary = pos['primary']
        secondary = pos['secondary']
        if res[0]=='G':
            if not t1:
                t1= pos['offense']
                qpd = 10
            elif t1==pos['offense']:
                qpd+=10
            else:
                qpd-=10
        elif res in ['RCA','RCB','OCA','OCB','2CA','2CB']:
            if abs(qpd)<=30:
                if primary[0] in stats['ISR Catches']:
                    stats['ISR Catches'][primary[0]]+=1
                else:
                    stats['ISR Catches'][primary[0]]=1
            else:
                if primary[0] in stats['OSR Catches']:
                    stats['OSR Catches'][primary[0]]+=1
                else:
                    stats['OSR Catches'][primary[0]]=1
            qpd= qpd+30 if p.split('-')[0]==t1 else qpd-30
        if res=='TB':
            for p in primary:
                if p in stats['Beat TO Forced']:
                    stats['Beat TO Forced'][p]+=1/len(primary)
                else:
                    stats['Beat TO Forced'][p]=1/len(primary)
        elif res in ('TD','TL'):
            for p in primary:
                if p in stats['Ball TO Forced']:
                    stats['Ball TO Forced'][p]+=1/len(primary)
                else:
                    stats['Ball TO Forced'][p]=1/len(primary)
        elif res=='TC':
            for p in primary:
                if p in stats['Contact TO Forced']:
                    stats['Contact TO Forced'][p]+=1/len(primary)
                else:
                    stats['Contact TO Forced'][p]=1/len(primary)
        elif res[0]=='G':
            for p in primary:
                if p in stats['Goals']:
                    stats['Goals'][p]+=1
                else:
                    stats['Goals'][p]=1
            for s in secondary:
                if s in stats['Assists']:
                    stats['Assists'][s]+=1/len(secondary)
                else:
                    stats['Assists'][s]=1/len(secondary)
        elif res[0]=='E':
            for p in primary:
                if p in stats['Errors']:
                    stats['Errors'][p]+=1/len(primary)
                else:
                    stats['Errors'][p]=1/len(primary)

        for extra in pos['extras']:
            extra_type, extra_player = extra
            if extra_type == 'B':
                if extra_player in stats['Blues']:
                    stats['Blues'][extra_player]+=1
                else:
                    stats['Blues'][extra_player]=1
            elif extra_type == 'Y':
                if extra_player in stats['Yellows']:
                    stats['Yellows'][extra_player]+=1
                else:
                    stats['Yellows'][extra_player]=1
            elif extra_type == '2Y':
                if extra_player in stats['Second Yellows']:
                    stats['Second Yellows'][extra_player]+=1
                else:
                    stats['Second Yellows'][extra_player]=1
            if extra_type == '1R':
                if extra_player in stats['Straight Reds']:
                    stats['Straight Reds'][extra_player]+=1
                else:
                    stats['Straight Reds'][extra_player]=1
            if extra_type == 'R':
                if extra_player in stats['Resets Forced']:
                    stats['Resets Forced'][extra_player]+=1
                else:
                    stats['Resets Forced'][extra_player]=1
    df = pd.DataFrame(stats).dropna(how='all').fillna(0).astype(int)[columns].sort_values(by=['Goals','Assists','Errors'],ascending=[False,False,True])
    return df

def add_times(play_list):
    first_play = True
    last_play = None
    period = 'FLOOR'
    for play in play_list:
        if play['extras'] and 'SOP' in [e[0] for e in play['extras']]:
            period='SOP'
        if not play['period']:
            play['period']=period
        if first_play:
            first_play = False
            last_play = play
            continue
        else:
            is_snitch_play = lambda x: x['result'] and x['result'] in ['RCA','RCB','OCA','OCB','2CA','2CB']
            has_time = lambda x: 'time' in x and len(x['time'])>3 and x['time'].isnumeric()

            if play and not is_snitch_play(play) and has_time(play) and has_time(last_play):
                play['length'] = int(play['time']) -int(last_play['time'])
            else:
                play['length']=-1
            last_play = play

def process_file(ifile):
    stem = ifile.split('.xls')[0]
    print('Processing Game: {}'.format(stem),end='')
    pbp_output_file = stem+'_pbp.txt'
    stats_output_file = stem+'_stats.csv'
    json_output_file = stem+'_data.json'
    dat, ros = get_info(ifile)
    roster = get_roster(ros)
    header,data = get_info_from_data(dat)
    teams = get_teams_from_header(header)
    get_brooms_up(header,roster,teams)
    possessions = get_possessions(data)
    interpreted = [get_brooms_up(header,roster,teams)]+[interpret(pos,roster,teams) for pos in possessions]
    add_times(interpreted)
    print(': {} plays'.format(len(interpreted)))
    with open(json_output_file,'w+') as f:
        json.dump({i:v for i,v in enumerate(interpreted)},f,indent=2)
    print('.',end='')
    play_by_play = [gen_pbp(i)+'\n' for i in interpreted]
    with open(pbp_output_file,'w+') as f:
        f.writelines(play_by_play)
    print('.',end='')
    stats = ind_stats(interpreted)
    print('.',end='')
    stats.to_csv(stats_output_file)
    print('Complete')

def main(argv):
    inputfile = ''
    recursive = False
    try:
      opts, args = getopt.getopt(argv,"ai:",['all','inputfile='])
    except getopt.GetoptError:
      sys.exit(2)
    for opt,arg in opts:
        if opt in ('-i','--inputfile'):
            inputfile = arg
        if opt in ('-a','--all'):
            recursive = True
    if recursive:
        for path in Path('').rglob('*.xlsx'):
            try:
                process_file(str(path))
            except xlrd.biffh.XLRDError as e:
                print('Unable to read {}'.format(path))
    elif not inputfile:
        print('Please include an inputfile')
    else:
        process_file(inputfile)

if __name__ == "__main__":
    main(sys.argv[1:])


# Copyright 2023, The TensorFlow Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fast clipping function for `tf.keras.layers.Embedding`."""

from typing import Any, Mapping, Tuple, Union
import tensorflow as tf
from tensorflow_privacy.privacy.fast_gradient_clipping import type_aliases

def embedding_layer_computation(
    layer_instance: tf.keras.layers.Embedding,
    input_args: Tuple[Any, ...],
    input_kwargs: Mapping[str, Any],
    tape: tf.GradientTape,
    num_microbatches: Union[tf.Tensor, None] = None,
) -> type_aliases.RegistryFunctionOutput:
  """Registry function for `tf.keras.layers.Embedding`.

  The logic of this computation is based on the `tf.keras.layers.Dense`
  computation and the fact that an embedding layer is just a dense layer
  with no activation function and an output vector of the form X*W for input
  X, where the i-th row of W is the i-th embedding vector and the j-th row of
  X is a one-hot vector representing the input of example j.

  Args:
    layer_instance: A `tf.keras.layers.Embedding` instance.
    input_args: See `dense_layer_computation()` in `dense.py`.
    input_kwargs: See `dense_layer_computation()` in `dense.py`.
    tape: See `dense_layer_computation()` in `dense.py`.
    num_microbatches: See `dense_layer_computation()` in `dense.py`.

  Returns:
    See `dense_layer_computation()` in `dense.py`.
  """
  if input_kwargs:
    raise ValueError("Embedding layer calls should not receive kwargs.")
  del input_kwargs  # Unused in embedding layer calls.
  if len(input_args) != 1:
    raise ValueError("Only layer inputs of length 1 are permitted.")
  if hasattr(layer_instance, "sparse"):  # for backwards compatibility
    if layer_instance.sparse:
      raise NotImplementedError("Sparse output tensors are not supported.")
  if isinstance(input_args[0], tf.SparseTensor):
    raise NotImplementedError("Sparse input tensors are not supported.")

  # Disable experimental features.
  if hasattr(layer_instance, "_use_one_hot_matmul"):
    if layer_instance._use_one_hot_matmul:  # pylint: disable=protected-access
      raise NotImplementedError(
          "The experimental embedding feature"
          "'_use_one_hot_matmul' is not supported."
      )
  input_ids = tf.cast(*input_args, tf.int32)
  if len(layer_instance.trainable_variables) != 1:
    raise ValueError(
        "Only layer instances with only one set of trainable variables"
        "are permitted."
    )
  base_vars = layer_instance.trainable_variables[0]
  tape.watch(base_vars)
  outputs = tf.nn.embedding_lookup(base_vars, input_ids)

  def sqr_norm_fn(base_vars_grads):
    """Fast square norm function for Keras embedding layers.

    Args:
      base_vars_grads: A list of batched embedding gradients.

    Returns:
      A 1D `tf.Tensor` of squared gradient norms.

    NOTE: to help understand the code, we document in the function body what
    the expected intermediate variables are for the below running example:

      input_ids = [[1, 1, 2], [0], [2, 0]]
      base_vars_grads.indices = [1, 1, 2, 0, 2, 0]
      base_vars_grads.values = [[0.2], [0.2], [0.3], [0.1], [0.3], [0.1]]

    For ease of reference, we also list these variables below:

      row_indices = [[0], [0], [0], [1], [2], [2]]
      slice_indices = [[1], [1], [2], [0], [2], [0]]
      paired_indices = [[0, 1], [0, 1], [0, 2], [1, 0], [2, 2], [2, 0]]
      unique_paired_indices = [[0, 1], [0, 2], [1, 0], [2, 2], [2, 0]]
      new_index_positions = [0, 0, 1, 2, 3, 4]
      num_unique_paired_indices = 5
      unique_batch_ids = [0, 0, 1, 2, 2]
      summed_gradients
        = [0.2 + 0.2, 0.3, 0.1, 0.3, 0.1]
        = [[0.4], [0.3], [0.1], [0.3], [0.1]]
      sqr_gradient_sum = [0.16, 0.09, 0.01, 0.09, 0.01]
    """
    # We first get a 1D tensor of the row indices.
    nrows = tf.shape(input_ids)[0]
    if isinstance(input_ids, tf.RaggedTensor):
      row_indices = tf.expand_dims(
          input_ids.merge_dims(1, -1).value_rowids(), axis=-1
      )
    elif isinstance(input_ids, tf.Tensor):
      ncols = tf.reduce_prod(tf.shape(input_ids)[1:])
      repeats = tf.repeat(ncols, nrows)
      row_indices = tf.reshape(tf.repeat(tf.range(nrows), repeats), [-1, 1])
    else:
      raise NotImplementedError(
          "Cannot parse input_ids of type %s" % input_ids.__class__.__name__
      )
    row_indices = tf.cast(row_indices, tf.int64)
    if num_microbatches is not None:
      microbatch_size = tf.cast(nrows / num_microbatches, tf.int64)
      nrows = num_microbatches
      row_indices = tf.cast(
          tf.math.floordiv(row_indices, microbatch_size), tf.int64
      )
    # NOTE: expected values for the running example above are
    #   row_indices = [[0], [0], [0], [1], [2], [2]]

    # Now, sum-reduce the `IndexedSlices` that is the result of a
    # `tape.gradient()` call. The sum is reduced by the repeated embedding
    # indices and batch index. It is adapted from the logic in:
    #   tf.keras.optimizers.legacy.optimizer_v2._deduplicate_indexed_slices
    if not isinstance(base_vars_grads, tf.IndexedSlices):
      raise NotImplementedError(
          "Cannot parse embedding gradients of type: %s"
          % base_vars_grads.__class__.__name__
      )
    slice_indices = tf.expand_dims(base_vars_grads.indices, axis=-1)
    paired_indices = tf.concat(
        [tf.cast(row_indices, tf.int64), tf.cast(slice_indices, tf.int64)],
        axis=1,
    )
    (unique_paired_indices, new_index_positions) = tf.raw_ops.UniqueV2(
        x=paired_indices, axis=[0]
    )
    # NOTE: expected values for the running example above are
    #   slice_indices = [[1], [1], [2], [0], [2], [0]]
    #   paired_indices = [[0, 1], [0, 1], [0, 2], [1, 0], [2, 2], [2, 0]]
    #   unique_paired_indices = [[0, 1], [0, 2], [1, 0], [2, 2], [2, 0]]
    #   new_index_positions = [0, 0, 1, 2, 3, 4]

    # Next, sum according to the new positions and compute the squared
    # gradient norms. Oddly enough, not sorting
    # these indices will break tensor shape inference logic on TPUs.
    num_unique_paired_indices = tf.shape(unique_paired_indices)[0]
    unique_batch_ids = unique_paired_indices[:, 0]
    summed_gradients = tf.math.unsorted_segment_sum(
        base_vars_grads.values,
        new_index_positions,
        num_unique_paired_indices,
    )
    sqr_gradient_sum = tf.reduce_sum(tf.square(summed_gradients), axis=1)
    # NOTE: expected values for the running example above are
    #   num_unique_paired_indices = 5
    #   unique_batch_ids = [0, 0, 1, 2, 2]
    #   summed_gradients
    #     = [0.2 + 0.2, 0.3, 0.1, 0.3, 0.1]
    #     = [[0.4], [0.3], [0.1], [0.3], [0.1]]
    #   sqr_gradient_sum = [0.16, 0.09, 0.01, 0.09, 0.01]

    # Use a scatter-add strategy to ensure TPU compatibility.
    result = tf.zeros([nrows])
    return tf.tensor_scatter_nd_add(
        result,
        tf.expand_dims(unique_batch_ids, axis=-1),
        sqr_gradient_sum,
    )
    # NOTE: the expected output for the running example is
    #   [0.16 + 0.09, 0.01, 0.09 + 0.01] = [0.25, 0.01, 0.1]

  return base_vars, outputs, sqr_norm_fn


#
# Copyright © 2016 Jeremy Gibson <jeremy.gibson@ncdcr.gov>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import json
import subprocess
import sys
from queue import Queue, Empty
from threading import Thread
from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory, listenWS
from twisted.python import log
from twisted.internet import reactor

class DarcMailConvert(WebSocketServerProtocol):
    """DarcMailConvert manages communication with the tomes-docker web client for the tomes-darcmail application. It
    implements a Websocket server for communication with a Flask client.

    Attributes:
        server_name: A string identifying the server.
        on_posix: A boolean that indicates whether this is a posix system.  Used by queue.
        cur_folder: A string populated from the web-client.
        cur_folder: A boolean populated from the web-client.
        chunk: A boolean populated from the web-client.
        chunk-size: An integer that indicates the number of messages in a chunk.
        transfer-name: A String indicating the name of the account.
        build_opts: A list to hold the options needed for the subprocess.
        file_tree: A list to hold the folder names found in mime-sources.
        production: A boolean to help switch between production and development paths.
    """
    WS_FOLDER_LIST = 1
    WS_RECV_PACKAGE = 2
    WS_PROG_ON = 4
    WS_PROG_OFF = 5
    WS_SEND = 3

    def __init__(self):
        super().__init__()
        self.server_name = "DarcMailConverter"
        self.on_posix = 'posix' in sys.builtin_module_names
        self.cur_folder = None
        self.stitch = None
        self.chunk = None
        self.chunk_size = None
        self.from_eml = None
        self.transfer_name = None
        self.build_opts = []
        self.production = True
        self.file_tree = []
        if self.production:
            self.dm_exec = '/usr/src/app/docker_dmc/DarcMailCLI.py'
            self.base_package_location = "/home/tomes/data"
        else:
            self.dm_exec = 'D:\\Development\\Python\\DockerComposeStruct\\devel\\tomes_docker_app\\Servers\\darcm_server\\docker_dmc\\DarcMailCLI.py'
            self.base_package_location = 'E:\\RESOURCES\\TEST_RESOURCES\\tomes\\data'

    def onConnect(self, request) -> tuple:
        print("{}: Connected".format(self.server_name))
        headers = {'Access-Control-Allow-Origin': '*'}
        return None, headers

    def onOpen(self):
        s = self._get_folder_tree()
        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_FOLDER_LIST, s))
        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_SEND, 'Client connected!'))

    def _get_folder_tree(self) -> str:
        """Gets mime-sources folder names from the tomes directory structure, and returns to the Flask client
        TODO: change mboxes to mime-sources"""
        children = os.listdir(os.path.join(self.base_package_location, 'mboxes'))
        nc = []
        for c in children:
            nc.append({'name': c})
        self.file_tree.append({'name': "Mime Sources", 'children': nc})
        return json.dumps(self.file_tree)

    def onMessage(self, payload, isBinary):
        payload = json.loads(bytes.decode(payload, encoding='utf-8'))
        if isBinary:
            pass
        else:
            if payload['o'] == DarcMailConvert.WS_RECV_PACKAGE:
                self.cur_folder = payload['data']['fldr']
                self.stitch = payload['data']['stitch']
                self.chunk = payload['data']['chunk']
                self.chunk_size = payload['data']['chunk_size']
                self.from_eml = payload['data']['from_eml']
                self.transfer_name = payload['data']['trans_name']
                self._build_convert_opts()
                # callInThread necessary to prevent blocking
                reactor.callInThread(self._convert)
            if payload['o'] == 5:
                self._get_suggested_name(payload['data']['fldr'])

    def _find_folder(self, folder) -> str:
        for root, dirs, files in os.walk(self.base_package_location):
            for d in dirs:
                #self.sendMessage(self.get_message_for_sending(2, "Fldrs: {}".format(d)))
                if d == folder:
                    return os.path.join(root, d)

    def onClose(self, wasClean, code, reason):
        print("WebSocket connection closed: {0}".format(reason))

    def _get_suggested_name(self, fldr):
        """Suggests a name for the transfer."""
        for root, dirs, files in os.walk(self.base_package_location):
            for d in dirs:
                if d == fldr:
                    root_split = root.split(os.path.sep)
                    i = root_split.index('data') + 2
                    final = root_split[i:] + [d]
                    final = '_'.join(final)
                    final = final.replace(" ", "_")
                    self.sendMessage(self._get_message_for_sending(5, final))

    def _build_convert_opts(self):
        """Builds the list of arguments to be passed to the subprocess call."""
        self.build_opts = ['python', self.dm_exec]
        package_location = self._find_folder(self.cur_folder)

        # BEGIN REQUIRED These are required
        self.build_opts.append('-a')
        self.build_opts.append(self.transfer_name)
        self.build_opts.append("-d")
        self.build_opts.append(package_location)
        self.build_opts.append("-tt")
        # End REQUIRED

        # BEGIN NOT REQUIRED These are NOT Required but recommended
        self.build_opts.append('-c')
        if self.chunk:
            self.build_opts.append('{}'.format(self.chunk_size))
        else:
            self.build_opts.append('{}'.format(0))

        if self.stitch:
            self.build_opts.append('-st')
        # END NOT REQUIRED

        # BEGIN OPTIONS: These are options
        if self.from_eml:
            # Requested Structure is EML
            self.build_opts.append('-fe')

        # END OPTIONS

        for i in range(len(self.build_opts)):
            self.build_opts[i] = str(self.build_opts[i])

    def enqueue_out(self, out, queue):
        """Where the worker thread sends logging messages from the subprocess.
        Keeps the twisted process from blocking"""
        for line in iter(out.readline, b''):
            queue.put(line)
        out.close()

    def _get_message_for_sending(self, router, message) -> str:
        """Helper function to build WebSocket messages

        Args:
            router: An integer that indicates the callback function to run on the client.
            message: A string with the message to send to the client function.
        """
        l = json.dumps({'router': router, 'data': message})
        return l.encode('utf-8')

    def _convert(self):
        """The worker function for the class.  Calls the tomes-darcmail program and returns its logging messages back
        to the WebSocket client.
        """
        p = subprocess.Popen(self.build_opts, stdout=subprocess.PIPE, bufsize=1, close_fds=self.on_posix)
        q = Queue()
        t = Thread(target=self.enqueue_out, args=(p.stdout, q))
        t.daemon = True
        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_SEND, "Processing: {}".format(self.cur_folder)))
        t.start()
        line = None
        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_PROG_ON, ''))
        while t.is_alive():
            try:
                # Wait for a line to be generated.
                line = q.get_nowait()
            except Empty:
                # if the queue is empty but not complete continue
                pass
            else:
                l = bytes.decode(line, encoding='utf-8')
                sender = json.dumps({'router': DarcMailConvert.WS_SEND, 'data': l.strip()})
                self.sendMessage(sender.encode('utf-8'))

        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_SEND, "Complete"))
        self.sendMessage(self._get_message_for_sending(DarcMailConvert.WS_PROG_OFF, ''))

if __name__ == "__main__":
    log.startLogging(sys.stdout)
    factory = WebSocketServerFactory()
    factory.protocol = DarcMailConvert
    reactor.listenTCP(9001, factory)
    reactor.run()


""" ptah.form fields """
from ptah import view, config, manage
from ptah.form.field import FIELD_ID, PREVIEW_ID

class FieldsModule(manage.PtahModule):
    __doc__ = u'A list of all registered fields with your application.'

    title = 'Field types'
    manage.module('fields')

class FieldsView(view.View):
    view.pview(
        context = FieldsModule,
        template = view.template('ptah.manage:templates/fields.pt'))

    def update(self):
        data = []

        fields = config.registry.storage[FIELD_ID]
        previews = config.registry.storage[PREVIEW_ID]

        for name, cls in fields.items():
            data.append({'name': name,
                         'doc': cls.__doc__,
                         'preview': previews.get(cls)})

        data.sort()
        self.fields = data