#!/usr/bin/env python3
# Copyright 2016 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import os.path
import sys
import argparse
import collections
import functools
import re
import copy
try:
  import json
except ImportError:
  import simplejson as json

import pdl

try:
  unicode
except NameError:
  # Define unicode for Py3
  def unicode(s, *_):
    return s

# Path handling for libraries and templates
# Paths have to be normalized because Jinja uses the exact template path to
# determine the hash used in the cache filename, and we need a pre-caching step
# to be concurrency-safe. Use absolute path because __file__ is absolute if
# module is imported, and relative if executed directly.
# If paths differ between pre-caching and individual file compilation, the cache
# is regenerated, which causes a race condition and breaks concurrent build,
# since some compile processes will try to read the partially written cache.
module_path, module_filename = os.path.split(os.path.realpath(__file__))

def read_config():
  # pylint: disable=W0703
  def json_to_object(data, output_base, config_base):
    def json_object_hook(object_dict):
      items = [(k, os.path.join(config_base, v) if k == "path" else v)
               for (k, v) in object_dict.items()]
      items = [(k, os.path.join(output_base, v) if k == "output" else v)
               for (k, v) in items]
      keys, values = list(zip(*items))
      # 'async' is a keyword since Python 3.7.
      # Avoid namedtuple(rename=True) for compatibility with Python 2.X.
      keys = tuple('async_' if k == 'async' else k for k in keys)
      return collections.namedtuple('X', keys)(*values)
    return json.loads(data, object_hook=json_object_hook)

  def init_defaults(config_tuple, path, defaults):
    keys = list(config_tuple._fields)  # pylint: disable=E1101
    values = [getattr(config_tuple, k) for k in keys]
    for i in range(len(keys)):
      if hasattr(values[i], "_fields"):
        values[i] = init_defaults(values[i], path + "." + keys[i], defaults)
    for optional in defaults:
      if optional.find(path + ".") != 0:
        continue
      optional_key = optional[len(path) + 1:]
      if optional_key.find(".") == -1 and optional_key not in keys:
        keys.append(optional_key)
        values.append(defaults[optional])
    return collections.namedtuple('X', keys)(*values)

  try:
    cmdline_parser = argparse.ArgumentParser()
    cmdline_parser.add_argument("--output_base", type=unicode, required=True)
    cmdline_parser.add_argument("--jinja_dir", type=unicode, required=True)
    cmdline_parser.add_argument("--config", type=unicode, required=True)
    cmdline_parser.add_argument("--config_value", default=[], action="append")
    cmdline_parser.add_argument(
      "--inspector_protocol_dir", type=unicode, required=True,
      help=("directory with code_generator.py and C++ encoding / binding "
          "libraries, relative to the root of the source tree."))
    arg_options = cmdline_parser.parse_args()
    jinja_dir = arg_options.jinja_dir
    output_base = arg_options.output_base
    config_file = arg_options.config
    config_base = os.path.dirname(config_file)
    config_values = arg_options.config_value
    inspector_protocol_dir = arg_options.inspector_protocol_dir.lstrip('/')
  except Exception:
    # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html
    exc = sys.exc_info()[1]
    sys.stderr.write("Failed to parse command-line arguments: %s\n\n" % exc)
    exit(1)

  try:
    config_json_file = open(config_file, "r")
    config_json_string = config_json_file.read()
    config_partial = json_to_object(config_json_string, output_base,
                                    config_base)
    config_json_file.close()
    defaults = {
      ".use_snake_file_names": False,
      ".use_title_case_methods": False,
      ".use_embedder_types": False,
      ".imported": False,
      ".imported.export_macro": "",
      ".imported.export_header": False,
      ".imported.header": False,
      ".imported.package": False,
      ".imported.options": False,
      ".protocol.export_macro": "",
      ".protocol.export_header": False,
      ".protocol.options": False,
      ".protocol.file_name_prefix": "",
      ".exported": False,
      ".exported.export_macro": "",
      ".exported.export_header": False,
      ".lib": False,
      ".lib.export_macro": "",
      ".lib.export_header": False,
      ".crdtp": False,
      ".crdtp.dir": os.path.join(inspector_protocol_dir, "crdtp"),
      ".crdtp.namespace": "crdtp",
    }
    for key_value in config_values:
      parts = key_value.split("=")
      if len(parts) == 2:
        defaults["." + parts[0]] = parts[1]
    return (jinja_dir, config_file, init_defaults(config_partial, "", defaults))
  except Exception:
    # Work with python 2 and 3 http://docs.python.org/py3k/howto/pyporting.html
    exc = sys.exc_info()[1]
    sys.stderr.write("Failed to parse config file: %s\n\n" % exc)
    exit(1)

# ---- Begin of utilities exposed to generator ----

def to_title_case(name):
  return name[:1].upper() + name[1:]

def dash_to_camelcase(word):
  prefix = ""
  if word[0] == "-":
    prefix = "Negative"
    word = word[1:]
  return prefix + "".join(to_title_case(x) or "-" for x in word.split("-"))

def to_snake_case(name):
  name = re.sub(r"([A-Z]{2,})([A-Z][a-z])", r"\1_\2", name)
  return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name, sys.maxsize).lower()

def to_method_case(config, name):
  if config.use_title_case_methods:
    return to_title_case(name)
  return name

def join_arrays(dict, keys):
  result = []
  for key in keys:
    if key in dict:
      result += dict[key]
  return result

def format_include(config, header, file_name=None):
  if file_name is not None:
    header = header + "/" + file_name + ".h"
  header = "\"" + header + "\"" if header[0] not in "<\"" else header
  if config.use_snake_file_names:
    header = to_snake_case(header)
  return header

def format_domain_include(config, header, file_name):
  return format_include(config, header,
                        config.protocol.file_name_prefix + file_name)

def to_file_name(config, file_name):
  if config.use_snake_file_names:
    return to_snake_case(file_name).replace(".cpp", ".cc")
  return file_name

# ---- End of utilities exposed to generator ----

def initialize_jinja_env(jinja_dir, cache_dir, config):
  # pylint: disable=F0401
  sys.path.insert(1, os.path.abspath(jinja_dir))
  import jinja2

  jinja_env = jinja2.Environment(
    loader=jinja2.FileSystemLoader(module_path),
    # Bytecode cache is not concurrency-safe unless pre-cached:
    # if pre-cached this is read-only, but writing creates a race condition.
    bytecode_cache=jinja2.FileSystemBytecodeCache(cache_dir),
    keep_trailing_newline=True,  # newline-terminate generated files
    lstrip_blocks=True,  # so can indent control flow tags
    trim_blocks=True)
  jinja_env.filters.update({
      "to_title_case": to_title_case,
      "dash_to_camelcase": dash_to_camelcase,
      "to_method_case": functools.partial(to_method_case, config)})
  jinja_env.add_extension("jinja2.ext.loopcontrols")
  return jinja_env

def create_imported_type_definition(domain_name, type, imported_namespace):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "pass_type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<%s::%s::API::%s>" % (
        imported_namespace, domain_name, type["id"]),
    "raw_type": "%s::%s::API::%s" % (
        imported_namespace, domain_name, type["id"]),
    "raw_pass_type": "%s::%s::API::%s*" % (
        imported_namespace, domain_name, type["id"]),
    "raw_return_type": "%s::%s::API::%s*" % (
        imported_namespace, domain_name, type["id"]),
  }

def create_user_type_definition(domain_name, type):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::%s::%s>" % (
        domain_name, type["id"]),
    "pass_type": "std::unique_ptr<protocol::%s::%s>" % (
        domain_name, type["id"]),
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::%s::%s>" % (domain_name, type["id"]),
    "raw_type": "protocol::%s::%s" % (domain_name, type["id"]),
    "raw_pass_type": "protocol::%s::%s*" % (domain_name, type["id"]),
    "raw_return_type": "protocol::%s::%s*" % (domain_name, type["id"]),
  }

def create_object_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::DictionaryValue>",
    "pass_type": "std::unique_ptr<protocol::DictionaryValue>",
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::DictionaryValue>",
    "raw_type": "protocol::DictionaryValue",
    "raw_pass_type": "protocol::DictionaryValue*",
    "raw_return_type": "protocol::DictionaryValue*",
  }

def create_any_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::Value>",
    "pass_type": "std::unique_ptr<protocol::Value>",
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::Value>",
    "raw_type": "protocol::Value",
    "raw_pass_type": "protocol::Value*",
    "raw_return_type": "protocol::Value*",
  }

def create_string_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "String",
    "pass_type": "const String&",
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": "String",
    "raw_type": "String",
    "raw_pass_type": "const String&",
    "raw_return_type": "String",
    "is_primitive": True
  }

def create_binary_type_definition():
  # pylint: disable=W0622
  return {
    "return_type": "Binary",
    "pass_type": "const Binary&",
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": "Binary",
    "raw_type": "Binary",
    "raw_pass_type": "const Binary&",
    "raw_return_type": "Binary",
    "is_primitive": True
  }

def create_primitive_type_definition(type):
  # pylint: disable=W0622
  typedefs = {
    "number": "double",
    "integer": "int",
    "boolean": "bool"
  }
  defaults = {
    "number": "0",
    "integer": "0",
    "boolean": "false"
  }
  jsontypes = {
    "number": "TypeDouble",
    "integer": "TypeInteger",
    "boolean": "TypeBoolean",
  }
  return {
    "return_type": typedefs[type],
    "pass_type": typedefs[type],
    "to_pass_type": "%s",
    "to_raw_type": "%s",
    "to_rvalue": "%s",
    "type": typedefs[type],
    "raw_type": typedefs[type],
    "raw_pass_type": typedefs[type],
    "raw_return_type": typedefs[type],
    "default_value": defaults[type],
    "is_primitive": True
  }

def wrap_array_definition(type):
  # pylint: disable=W0622
  return {
    "return_type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "pass_type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "to_raw_type": "%s.get()",
    "to_pass_type": "std::move(%s)",
    "to_rvalue": "std::move(%s)",
    "type": "std::unique_ptr<protocol::Array<%s>>" % type["raw_type"],
    "raw_type": "protocol::Array<%s>" % type["raw_type"],
    "raw_pass_type": "protocol::Array<%s>*" % type["raw_type"],
    "raw_return_type": "protocol::Array<%s>*" % type["raw_type"],
    "out_type": "protocol::Array<%s>&" % type["raw_type"],
  }

class Protocol(object):

  def __init__(self, config):
    self.config = config
    self.json_api = {"domains": []}
    self.imported_domains = []
    self.exported_domains = []
    self.generate_domains = self.read_protocol_file(config.protocol.path)

    if config.protocol.options:
      self.generate_domains = [rule.domain for rule in config.protocol.options]
      self.exported_domains = [rule.domain for rule in config.protocol.options
                               if hasattr(rule, "exported")]

    if config.imported:
      self.imported_domains = self.read_protocol_file(config.imported.path)
      if config.imported.options:
        self.imported_domains = [rule.domain
                                 for rule in config.imported.options]

    self.patch_full_qualified_refs()
    self.create_type_definitions()
    self.generate_used_types()

  def read_protocol_file(self, file_name):
    input_file = open(file_name, "r")
    parsed_json = pdl.loads(input_file.read(), file_name)
    input_file.close()
    version = '%s.%s' % (parsed_json["version"]["major"],
                         parsed_json["version"]["minor"])
    domains = []
    for domain in parsed_json["domains"]:
      domains.append(domain["domain"])
      domain["version"] = version
    self.json_api["domains"] += parsed_json["domains"]
    return domains

  def patch_full_qualified_refs(self):
    def patch_full_qualified_refs_in_domain(json, domain_name):
      if isinstance(json, list):
        for item in json:
          patch_full_qualified_refs_in_domain(item, domain_name)
      if not isinstance(json, dict):
        return
      for key in json:
        if key == "type" and json[key] == "string":
          json[key] = domain_name + ".string"
        if key != "$ref":
          patch_full_qualified_refs_in_domain(json[key], domain_name)
          continue
        if json["$ref"].find(".") == -1:
          json["$ref"] = domain_name + "." + json["$ref"]
      return

    for domain in self.json_api["domains"]:
      patch_full_qualified_refs_in_domain(domain, domain["domain"])

  def all_references(self, json):
    refs = set()
    if isinstance(json, list):
      for item in json:
        refs |= self.all_references(item)
    if not isinstance(json, dict):
      return refs
    for key in json:
      if key != "$ref":
        refs |= self.all_references(json[key])
      else:
        refs.add(json["$ref"])
    return refs

  def check_if_dependency_declared(self, domain, refs):
    dependencies = domain.get('dependencies', set())
    for ref in refs:
      type_definition = self.type_definitions[ref]
      if type_definition.get('is_primitive', False):
        continue
      domain_match = re.match(r'^(.*)[.]', ref)
      if domain_match:
        referenced_domain_name = domain_match.group(1)
        if referenced_domain_name != domain['domain'] and not referenced_domain_name in dependencies:
            sys.stderr.write(("Domains [%s] uses type [%s] from domain [%s], but did not declare the dependency\n\n"
                    ) % (domain["domain"], ref, referenced_domain_name))
            exit(1)

  def generate_used_types(self):
    all_refs = set()
    for domain in self.json_api["domains"]:
      domain_name = domain["domain"]
      if "commands" in domain:
        for command in domain["commands"]:
          if self.generate_command(domain_name, command["name"]):
            all_refs_command = self.all_references(command)
            # If the command has a redirect, it is as if it didn't exist on this domain.
            if not command.get('redirect', False):
              self.check_if_dependency_declared(domain, all_refs_command)
            all_refs |= all_refs_command

      if "events" in domain:
        for event in domain["events"]:
          if self.generate_event(domain_name, event["name"]):
            all_refs_event = self.all_references(event)
            self.check_if_dependency_declared(domain, all_refs_event)
            all_refs |= all_refs_event

    dependencies = self.generate_type_dependencies()
    queue = set(all_refs)
    while len(queue):
      ref = queue.pop()
      if ref in dependencies:
        queue |= dependencies[ref] - all_refs
        all_refs |= dependencies[ref]
    self.used_types = all_refs

  def generate_type_dependencies(self):
    dependencies = dict()
    domains_with_types = (x for x in self.json_api["domains"] if "types" in x)
    for domain in domains_with_types:
      domain_name = domain["domain"]
      for type in domain["types"]:
        related_types = self.all_references(type)
        if len(related_types):
          dependencies[domain_name + "." + type["id"]] = related_types
    return dependencies

  def create_type_definitions(self):
    imported_namespace = ""
    if self.config.imported:
      imported_namespace = "::".join(self.config.imported.namespace)
    self.type_definitions = {}
    self.type_definitions["number"] = create_primitive_type_definition("number")
    self.type_definitions["integer"] = create_primitive_type_definition("integer")
    self.type_definitions["boolean"] = create_primitive_type_definition("boolean")
    self.type_definitions["object"] = create_object_type_definition()
    self.type_definitions["any"] = create_any_type_definition()
    self.type_definitions["binary"] = create_binary_type_definition()
    for domain in self.json_api["domains"]:
      self.type_definitions[domain["domain"] + ".string"] = (
          create_string_type_definition())
      self.type_definitions[domain["domain"] + ".binary"] = (
          create_binary_type_definition())
      if not ("types" in domain):
        continue
      for type in domain["types"]:
        type_name = domain["domain"] + "." + type["id"]
        if type["type"] == "object" and domain["domain"] in self.imported_domains:
          self.type_definitions[type_name] = create_imported_type_definition(
              domain["domain"], type, imported_namespace)
        elif type["type"] == "object":
          self.type_definitions[type_name] = create_user_type_definition(
              domain["domain"], type)
        elif type["type"] == "array":
          self.type_definitions[type_name] = self.resolve_type(type)
        elif type["type"] == domain["domain"] + ".string":
          self.type_definitions[type_name] = create_string_type_definition()
        elif type["type"] == domain["domain"] + ".binary":
          self.type_definitions[type_name] = create_binary_type_definition()
        else:
          self.type_definitions[type_name] = create_primitive_type_definition(
              type["type"])

  def check_options(self, options, domain, name, include_attr, exclude_attr,
                    default):
    for rule in options:
      if rule.domain != domain:
        continue
      if include_attr and hasattr(rule, include_attr):
        return name in getattr(rule, include_attr)
      if exclude_attr and hasattr(rule, exclude_attr):
        return name not in getattr(rule, exclude_attr)
      return default
    return False

  # ---- Begin of methods exposed to generator

  def type_definition(self, name):
    return self.type_definitions[name]

  def resolve_type(self, prop):
    if "$ref" in prop:
      return self.type_definitions[prop["$ref"]]
    if prop["type"] == "array":
      return wrap_array_definition(self.resolve_type(prop["items"]))
    return self.type_definitions[prop["type"]]

  def generate_command(self, domain, command):
    if not self.config.protocol.options:
      return domain in self.generate_domains
    return self.check_options(self.config.protocol.options, domain, command,
                              "include", "exclude", True)

  def generate_event(self, domain, event):
    if not self.config.protocol.options:
      return domain in self.generate_domains
    return self.check_options(self.config.protocol.options, domain, event,
                              "include_events", "exclude_events", True)

  def generate_type(self, domain, typename):
    return domain + "." + typename in self.used_types

  def is_async_command(self, domain, command):
    if not self.config.protocol.options:
      return False
    return self.check_options(self.config.protocol.options, domain, command,
                              "async_", None, False)

  def is_exported(self, domain, name):
    if not self.config.protocol.options:
      return False
    return self.check_options(self.config.protocol.options, domain, name,
                              "exported", None, False)

  def is_imported(self, domain, name):
    if not self.config.imported:
      return False
    if not self.config.imported.options:
      return domain in self.imported_domains
    return self.check_options(self.config.imported.options, domain, name,
                              "imported", None, False)

  def is_exported_domain(self, domain):
    return domain in self.exported_domains

  def generate_disable(self, domain):
    if "commands" not in domain:
      return True
    for command in domain["commands"]:
      if command["name"] == "disable" and self.generate_command(
          domain["domain"], "disable"):
        return False
    return True

  def is_imported_dependency(self, domain):
    return domain in self.generate_domains or domain in self.imported_domains

def main():
  jinja_dir, config_file, config = read_config()

  protocol = Protocol(config)

  if not config.exported and len(protocol.exported_domains):
    sys.stderr.write(("Domains [%s] are exported, but config is missing export "
                      "entry\n\n") % ", ".join(protocol.exported_domains))
    exit(1)

  if not os.path.exists(config.protocol.output):
    os.mkdir(config.protocol.output)
  if len(protocol.exported_domains) and not os.path.exists(
      config.exported.output):
    os.mkdir(config.exported.output)
  jinja_env = initialize_jinja_env(jinja_dir, config.protocol.output, config)

  inputs = []
  inputs.append(__file__)
  inputs.append(config_file)
  inputs.append(config.protocol.path)
  if config.imported:
    inputs.append(config.imported.path)
  templates_dir = os.path.join(module_path, "templates")
  inputs.append(os.path.join(templates_dir, "TypeBuilder_h.template"))
  inputs.append(os.path.join(templates_dir, "TypeBuilder_cpp.template"))
  inputs.append(os.path.join(templates_dir, "Exported_h.template"))
  inputs.append(os.path.join(templates_dir, "Imported_h.template"))

  h_template = jinja_env.get_template("templates/TypeBuilder_h.template")
  cpp_template = jinja_env.get_template("templates/TypeBuilder_cpp.template")
  exported_template = jinja_env.get_template("templates/Exported_h.template")
  imported_template = jinja_env.get_template("templates/Imported_h.template")

  outputs = dict()

  for domain in protocol.json_api["domains"]:
    class_name = domain["domain"]
    file_name = config.protocol.file_name_prefix + class_name
    template_context = {
      "protocol": protocol,
      "config": config,
      "domain": domain,
      "join_arrays": join_arrays,
      "format_include": functools.partial(format_include, config),
      "format_domain_include": functools.partial(format_domain_include, config),
    }

    if domain["domain"] in protocol.generate_domains:
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".h"))] = h_template.render(template_context)
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".cpp"))] = cpp_template.render(template_context)
      if domain["domain"] in protocol.exported_domains:
        outputs[os.path.join(config.exported.output, to_file_name(
            config, file_name + ".h"))] = exported_template.render(
                template_context)
    if domain["domain"] in protocol.imported_domains:
      outputs[os.path.join(config.protocol.output, to_file_name(
          config, file_name + ".h"))] = imported_template.render(
              template_context)

  if config.lib:
    template_context = {
      "config": config,
      "format_include": functools.partial(format_include, config),
    }

    lib_templates_dir = os.path.join(module_path, "lib")
    # Note these should be sorted in the right order.

    # TODO(dgozman): sort them programmatically based on commented includes.

    forward_h_templates = [
      "Forward_h.template",
    ]

    protocol_h_templates = []
    protocol_cpp_templates = []

    if not config.use_embedder_types:
      protocol_h_templates += [
        "Values_h.template",
        "Object_h.template",
        "ValueConversions_h.template",
      ]
      protocol_cpp_templates += [
        "Protocol_cpp.template",
        "Values_cpp.template",
        "Object_cpp.template",
        "ValueConversions_cpp.template",
      ]
    else:
      protocol_h_templates += [
        "Forward_h.template",
      ]

    def generate_lib_file(file_name, template_files):
      parts = []
      for template_file in template_files:
        inputs.append(os.path.join(lib_templates_dir, template_file))
        template = jinja_env.get_template("lib/" + template_file)
        parts.append(template.render(template_context))
      outputs[file_name] = "\n\n".join(parts)

    generate_lib_file(os.path.join(config.lib.output, to_file_name(
        config, "Forward.h")), forward_h_templates)
    generate_lib_file(os.path.join(config.lib.output, to_file_name(
        config, "Protocol.h")), protocol_h_templates)

    if not config.use_embedder_types:
      generate_lib_file(os.path.join(config.lib.output, to_file_name(
          config, "Protocol.cpp")), protocol_cpp_templates)

  # Make gyp / make generatos happy, otherwise make rebuilds world.
  inputs_ts = max(map(os.path.getmtime, inputs))
  up_to_date = True
  for output_file in outputs.keys():
    if (not os.path.exists(output_file)
        or os.path.getmtime(output_file) < inputs_ts):
      up_to_date = False
      break
  if up_to_date:
    sys.exit()

  for file_name, content in outputs.items():
    # Remove output file first to account for potential case changes.
    try:
      os.remove(file_name)
    except OSError:
      pass
    out_file = open(file_name, "w")
    out_file.write(content)
    out_file.close()

if __name__ == "__main__":
  main()


import os
import numpy as np
import tensorflow as tf
import pandas as pd
import matplotlib.pyplot as plt
import sys
#You have freedom of using eager execution in tensorflow

plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

'''

IST 597: Foundations of Deep Learning

Problem 1d: MLPs \& the Spiral Problem

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    /////////******************/////////
    //// Umar Farooq Mohammad  /////////
     ////  ubm5020@psu.edu     /////////
    /////////******************/////////

'''
def computeCost(X,y,theta,reg):
    scores,probs = predict(X,theta)
    probs = probs.numpy()
    size = X.shape[0]
    probs = -np.log(probs)
    cost = 0  # initialization to zero at first
    for i in range(len(y)):
        temp = y[i]
        cost = cost + probs[i][temp]
    cost = cost/size
    W = theta[0]
    W2 = theta[2]
    res =((reg/2)*np.sum(np.power(W.numpy(),2))) +  ((reg/2)*np.sum(np.power(W2.numpy(),2))) + cost
    return res

def computeGrad(X,y,theta,reg): # returns nabla
    scores,probs = predict(X,theta)
    probs = probs.numpy()
    size = X.shape[0]
    for i in range(len(y)):
        temp = y[i]
        probs[i][temp] -= 1

    W = theta[0].numpy()
    W2 = theta[2].numpy()
    Pred = np.matmul(X,W) + theta[1].numpy() # b
    res = np.multiply(Pred,(Pred>0))
    dLdh = (1/size)*np.matmul(probs,W2.transpose())
    dLdPred = np.multiply(dLdh,Pred>0)
    dLdf = probs

    dW2 = np.matmul(res.transpose(),(1/size)*dLdf) + (reg*W2)
    db2 = np.sum((1/size)*dLdf,axis=0)
    dW = np.matmul(X.transpose(),dLdPred) + (reg*W)
    db = np.sum(dLdPred,axis=0)
    return (dW,db,dW2,db2)

def predict(X,theta):
    W = theta[0].numpy()
    b = theta[1].numpy()
    W2 = theta[2].numpy()
    b2 = theta[3].numpy()
    Pred = np.matmul(X,W) + b
    res = np.multiply(Pred,(Pred>0))
    #scores and probability
    scores = np.matmul(res,W2) + b2
    A = tf.exp(scores)
    B = tf.math.reduce_sum(A,axis=1)
    probs = tf.transpose((tf.transpose(A)/B))
    return (scores,probs)

np.random.seed(21131186)
# Load in the data from disk
path = os.getcwd() + '/data/spiral_train.dat'
data = pd.read_csv(path, header=None)

# set X (training data) and y (target variable)
cols = data.shape[1]
X = data.iloc[:,0:cols-1]
y = data.iloc[:,cols-1:cols]

# convert from data frames to numpy matrices
X = np.array(X.values)
y = np.array(y.values)
y = y.flatten()
X_tf = tf.constant(X)
Y_tf = tf.constant(y)
# initialize parameters randomly
D = X.shape[1]
K = np.amax(y) + 1

# initialize parameters randomly
h = 100 # size of hidden layer
initializer = tf.random_normal_initializer(mean=0.0, stddev=1.5, seed=21131186)
W = tf.Variable(initializer([D, h]))
b = tf.Variable(tf.random.normal([h]))
W2 = tf.Variable(initializer([h, K]))
b2 = tf.Variable(tf.random.normal([K]))
theta = (W,b,W2,b2)

# some hyperparameters
n_e = 4000
check = 10 # every so many pass/epochs, print loss/error to terminal
step_size = 0.01
reg = 0.001 # regularization strength

# gradient descent loop

losses = []
for i in range(n_e):
    # WRITEME: write your code here to perform a step of gradient descent & record anything else desired for later
    theta = (W,b,W2,b2)
    dW,db,dW2,db2 = computeGrad(X,y,theta,reg)
    loss = computeCost(X,y,theta,reg)
    losses.append(loss)
    if i % check == 0:
        print ("iteration %d: loss %f" % (i, loss))
    W = W - (step_size*dW)
    b = b - (step_size*db)
    W2 = W2 - (step_size*dW2)
    b2 = b2 - (step_size*db2)
# perform a parameter update
# WRITEME: write your update rule(s) here

# TODO: remove this line below once you have correctly implemented/gradient-checked your various sub-routines
#sys.exit(0)

scores, probs = predict(X,theta)
predicted_class = (tf.argmax(scores, axis=1))
print ('training accuracy: %.2f' % ((np.mean(predicted_class == y))))

# plot

plt.title("Training and Validation cost per epoch")
plt.xlabel("epoch number")
plt.ylabel("cost for this epoch")
plt.plot(losses)
# plot the resulting classifier
h = 0.02
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

Z, P = predict(np.concatenate((xx.ravel().reshape(-1,1), yy.ravel().reshape(-1,1)),axis=1), theta)

Z = np.argmax(Z, axis=1)
Z = Z.reshape(xx.shape)
fig = plt.figure()
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral, alpha=0.8)
plt.scatter(X[:, 0], X[:, 1], c=y, s=40, cmap=plt.cm.Spectral)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
#fig.savefig('spiral_net.png')

plt.show()


#!/usr/bin/env python
#-*- coding: utf-8 -*
import json
import numpy as np
import nltk
from nltk.stem.lancaster import LancasterStemmer

words = []  #to store the single words.
stemmer = LancasterStemmer()
classes = []
ERROR_THRESHOLD = 0.2
synapse_file = "/home/pi/ros/src/nlp/src/synapses.json"

with open(synapse_file,'r') as data_file:
      synapse = json.load(data_file)
      synapse_0 = np.asarray(synapse['synapse0'])
      synapse_1 = np.asarray(synapse['synapse1'])
      words = np.asarray(synapse['words'])
      classes = np.asarray(synapse['classes'])

#define some related functions
# compute sigmoid nonlinearity
def sigmoid(x):
    output = 1/(1+np.exp(-x))
    return output

# convert output of sigmoid function to its derivative
def sigmoid_output_to_derivative(output):
    return output*(1-output)

def clean_up_sentence(sentence):
    sentence_words = nltk.word_tokenize(sentence)
    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=False):
    sentence_words = clean_up_sentence(sentence)
    bag = [0]*len(words)
    for s in sentence_words:
        for i,w in enumerate(words):
            if w == s:
                bag[i] = 1
                if show_details:
                    print ("found in bag: %s" % w)

    return(np.array(bag))

def think(sentence, show_details=False):
    x = bow(sentence.lower(), words, show_details)
    if show_details:
        print ("sentence:", sentence, "bow:", x)
    l0 = x
    l1 = sigmoid(np.dot(l0, synapse_0))
    l2 = sigmoid(np.dot(l1, synapse_1))
    return l2

def classify(sentence, show_details=False):

    results = think(sentence, show_details)

    results = [[i,r] for i,r in enumerate(results) if r>ERROR_THRESHOLD ]
    results.sort(key=lambda x: x[1], reverse=True)
    return_results =[[classes[r[0]],r[1]] for r in results]
    return return_results


import os

import pytest

import ngage.cli

this_dir = os.path.dirname(__file__)
data_dir = os.path.join(this_dir, "data")

@pytest.fixture()
def ctx():
    return ngage.cli.Context(home=os.path.join(data_dir, "config", "tst0"))

host_tst0 = "tst0.example.com"
host_tst1 = "tst1.example.com"
host_tst2 = "tst2.example.com"

def hkwa(host):
    """make kwargs for host"""
    return dict(
        host=host,
        #        user='test'
    )

def test_default():
    ctx = ngage.cli.Context()
    config = ctx.get_connect_config({})
    assert config
    assert len(config)
    assert config["type"]

    config = ctx.get_connect_config(hkwa(host_tst0))
    assert host_tst0 == config["host"]

def test_host_exactmatch(ctx):
    config = ctx.get_connect_config(hkwa(host_tst0))
    assert "exactmatch" == config["type"]
    assert host_tst0 == config["host"]

def test_host_match(ctx):
    config = ctx.get_connect_config(hkwa(host_tst2))
    assert "match" == config["type"]
    assert host_tst2 == config["host"]

def test_host_fallthrough(ctx):
    config = ctx.get_connect_config(hkwa("notfound"))
    assert "fallthrough" == config["type"]
    assert "notfound" == config["host"]

def test_host_override(ctx):
    config = ctx.get_connect_config(hkwa(host_tst1))
    assert "override" == config["type"]
    assert "newhost" == config["host"]
    assert "newuser" == config["user"]

def test_copy_notref(ctx):
    config = ctx.get_connect_config(hkwa(host_tst1))
    config = ctx.get_connect_config(hkwa(host_tst1))
    assert "override" == config["type"]
    assert "newhost" == config["host"]


# idice_mp.py: idiomatic python parallelized with multiprocessing
import multiprocessing as mp
from idice import idice

cpu_cores = mp.cpu_count()
pool = mp.Pool(processes=cpu_cores)

def idice_mp(nsim):
    lst = [nsim/cpu_cores for n in range(cpu_cores)]
    res = pool.map(idice, lst)

    return 1.0 * sum(res) / cpu_cores


import sqlite3

conexion=sqlite3.connect('usuarios.db')
cursor=conexion.cursor()

#cursor.execute('''
#    CREATE TABLE usuarios(
#        dni VARCHAR(9) PRIMARY KEY,
#        nombre VARCHAR(100),
#        edad INTEGER,
#        correo VARCHAR(100)
#    )
#    ''')
usuarios=[
    ('dsjjkcn','Sebastian',19,'s@g.com'),
    ('kjjfn','Miguel',13,'m@g.com'),
    ('kasnd','Luis',11,'l@g.com'),
    ('asda','Isabel',11,'i@g.com'),
]

cursor.executemany('INSERT INTO usuarios VALUES (?,?,?,?)', usuarios)

conexion.commit()
conexion.close()

Fase 4 - Temas avanzados/Tema 14 - Bases de datos con SQLite/leccion2.py


from __future__ import annotations
from typing import Any, Optional, TypeVar, cast, TYPE_CHECKING
from datetime import date
from jsonclasses.types import Types
from jsonclasses.fdef import FStore, FType
from jsonclasses.mgraph import MGraph
from inflection import underscore, camelize
from .utils import (ref_field_key, ref_field_keys, ref_db_field_key,
                    ref_db_field_keys)
from .coder import Coder
from .pconf import PConf
if TYPE_CHECKING:
    from .pymongo_object import PymongoObject
    T = TypeVar('T', bound=PymongoObject)

class Decoder(Coder):

    def decode_list(self,
                    value: list[Any],
                    cls: type[T],
                    types: Types,
                    graph: MGraph) -> Optional[list[Any]]:
        if value is None:
            return None
        item_types = types.fdef.item_types
        new_cls = item_types.fdef.raw_inst_types
        return ([self.decode_item(value=item, cls=new_cls,
                                  types=item_types,
                                  graph=graph)
                for item in value])

    def decode_dict(self,
                    value: dict[str, Any],
                    cls: type[T],
                    types: Types,
                    graph: MGraph) -> Optional[dict[str, Any]]:
        if value is None:
            return None
        item_types = types.fdef.item_types
        return {k: self.decode_item(value=v, cls=cls, types=item_types, graph=graph) for k, v in value.items()}

    def decode_shape(self,
                     value: dict[str, Any],
                     cls: type[T],
                     types: Types,
                     graph: MGraph) -> dict[str, Any]:
        config: PConf = cls.pconf
        shape_types = cast(dict[str, Any], types.fdef.raw_shape_types)
        retval = {}
        for k, item_types in shape_types.items():
            retval[k] = self.decode_item(
                value=value[(camelize(k, False)
                             if config.camelize_db_keys else k)],
                cls=cls,
                types=item_types,
                graph=graph)
        return retval

    def decode_item(self,
                    value: Any,
                    cls: type[T],
                    types: Types,
                    graph: MGraph) -> Any:
        if value is None:
            return value
        if types.fdef.ftype == FType.DATE:
            return date.fromisoformat(value.isoformat()[:10])
        elif types.fdef.ftype == FType.ENUM:
            if isinstance(types.fdef.enum_class, str):
                enum_cls = cast(type, types.fdef.enum_class)
                return enum_cls(value)
            else:
                enum_cls = cast(type, types.fdef.enum_class)
                return enum_cls(value)
        elif types.fdef.ftype == FType.LIST:
            return self.decode_list(value=value, cls=cls, types=types,
                                    graph=graph)
        elif types.fdef.ftype == FType.DICT:
            return self.decode_dict(value=value, cls=cls, types=types,
                                    graph=graph)
        elif types.fdef.ftype == FType.SHAPE:
            return self.decode_shape(value=value, cls=cls, types=types,
                                     graph=graph)
        elif types.fdef.ftype == FType.INSTANCE:
            return self.decode_instance(value=value, cls=cls, types=types,
                                        graph=graph)
        else:
            return value

    def decode_instance(self,
                        value: dict[str, Any],
                        cls: type[T],
                        types: Types,
                        graph: MGraph) -> Any:
        inst_id = str(value.get('_id'))
        dest = graph.getp(cls, inst_id)
        exist = True
        if dest is None:
            dest = cls()
            exist = False
        for field in cls.cdef.fields:
            if cls.pconf.camelize_db_keys:
                key = camelize(field.name, False)
            else:
                key = field.name
            if self.is_id_field(field):
                if not exist:
                    setattr(dest, field.name, inst_id)
                    graph.put(dest)
            elif self.is_foreign_key_storage(field):
                if value.get(key) is not None:
                    if isinstance(value.get(key), list):
                        t = field.fdef.item_types
                        new_cls = t.fdef.raw_inst_types
                        setattr(dest, field.name,
                                self.decode_list(
                                    value[key], new_cls, field.types, graph))
                    else:
                        new_cls = field.fdef.inst_cls
                        setattr(dest, field.name,
                                self.decode_instance(
                                    cast(dict[str, Any], value.get(key)),
                                    new_cls, field.types, graph))
            elif self.is_local_key_reference_field(field):
                if value.get(key) is not None:
                    new_cls = field.fdef.inst_cls
                    inst = self.decode_item(
                        value=value.get(key), types=field.types, cls=new_cls,
                        graph=graph)
                    setattr(dest, field.name, inst)
                ref_id = value.get(ref_db_field_key(field.name, cls=cls))
                setattr(dest, ref_field_key(field.name), str(ref_id))
            elif self.is_local_keys_reference_field(field):
                if value.get(key) is not None:
                    new_cls = field.fdef.inst_cls
                    setattr(dest, field.name,
                            self.decode_list(
                                value[key], new_cls, field.types, graph))

                saved_keys = value.get(ref_db_field_keys(field.name, cls))
                setattr(dest, ref_field_keys(field.name),
                        [str(k) for k in saved_keys])
            elif self.is_instance_field(field):
                new_cls = field.fdef.inst_cls
                setattr(dest, field.name, self.decode_item(
                    value=value.get(key), types=field.types, cls=new_cls,
                    graph=graph))
            else:
                if not exist:
                    setattr(
                        dest,
                        field.name,
                        self.decode_item(value=value.get(key),
                                         types=field.types,
                                         cls=cls, graph=graph))
        return dest

    def apply_initial_status(self, root: T,
                             graph: Optional[MGraph] = None) -> None:
        if graph is None:
            graph = MGraph()
        if graph.get(root) is not None:
            return
        graph.put(root)
        for field in root.__class__.cdef.fields:
            if self.is_instance_field(field):
                if getattr(root, field.name) is not None:
                    self.apply_initial_status(getattr(root, field.name), graph)
            elif self.is_list_instance_field(field, root.__class__):
                if getattr(root, field.name) is not None:
                    for item in getattr(root, field.name):
                        self.apply_initial_status(item, graph)
        root._mark_unmodified()

    def decode_root(self,
                    root: dict[str, Any],
                    cls: type[T],
                    graph: Optional[MGraph] = None) -> T:
        if graph is None:
            graph = MGraph()
        types = Types().instanceof(cls)
        decoded = self.decode_instance(root, cls, types, graph)
        self.apply_initial_status(decoded)
        return decoded

    def decode_root_list(self,
                         root_list: list[dict[str, Any]],
                         cls: type[T],
                         graph: Optional[MGraph] = None) -> list[T]:
        if graph is None:
            graph = MGraph()
        types = Types().instanceof(cls)
        results: list[T] = []
        for root in root_list:
            decoded = self.decode_instance(root, cls, types, graph)
            results.append(decoded)
        for result in results:
            self.apply_initial_status(result)
        return results


import pyperclip,re
raw_data=pyperclip.paste()
phone_regex_mobile=re.compile(r'\d{10}')
phone_regex_landline=re.compile(r'\d+-\d+')
email_regex=re.compile(r'\S*@\S*')
phone_book_mobile = phone_regex_mobile.findall(raw_data)
phone_book_landline = phone_regex_landline.findall(raw_data)
email_book=email_regex.findall(raw_data)
print("The Landline Numbers Are:-")
for ph in phone_book_landline:
    print(ph)
print("The Mobile Numbers Are:-")
for ph in phone_book_mobile:
    print(ph)
print("The Email IDs Are:-")
for em in email_book:
    print(em)


import json
import csv

def WriteSystems(systems, stations):
    systems_with_station = {x['system_id'] for x in stations}

    with open('data/csv/System.csv', 'w') as csv_file:
        writer = csv.writer(csv_file)
        for system in systems:
            if system['id'] in systems_with_station:
                writer.writerow([system['id'], system['name'], system['x'], system['y'], system['z'], ''])

def WriteStations(stations, systems_lookup, commodity_lookup):
    with open('data/csv/SC.csv', 'w') as csv_file:
        writer = csv.writer(csv_file)
        for station in stations:
            station_name = station['name']

            station_system_name = systems_lookup[station['system_id']]

            for listing in station['listings']:
                commodity = commodity_lookup[listing['commodity_id']]

                writer.writerow([
                    listing['id'],
                    station_system_name,
                    '',
                    station_name,
                    commodity['category']['name'],
                    commodity['name'],
                    commodity['average_price'],
                    listing['buy_price'],
                    listing['sell_price'],
                    '',
                    listing['demand'],
                    '',
                    listing['supply'],
                    '',
                    listing['supply'],
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    ''
                    ])

# SCId,SCStationSystem,SCStationLocal,SCStationName,SCStationCommodGroup,SCStationCommod,SCStationCommodAvg,
# SCStationPrice,SCStationSell,SCStationFence,SCStationDemand,SCStationDemandCode,SCStationSupply,SCStationSupplyCode,
# SCStationStock,SCStationIllegal,SCStationConsumer,SCStationProducer,SCStationLastUpdate,SCStationForSale,
# S1,S2,S3,S4,N1,N2,N3,N4,B1,B2,SCUNIQUE

if __name__ == '__main__':
    commodities = json.loads(open('data/commodities.json', 'r').read())
    stations = json.loads(open('data/stations.json', 'r').read())
    systems = json.loads(open('data/systems.json', 'r').read())

    WriteSystems(systems, stations)

    commodity_lookup = {commodity['id']: commodity for commodity in commodities}
    systems_lookup = {system['id']: system['name'] for system in systems}

    WriteStations(stations, systems_lookup, commodity_lookup)


"""empty message

Revision ID: 0f2daa5d0ce3
Revises:
Create Date: 2019-03-17 20:26:45.665083

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '0f2daa5d0ce3'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('given_name',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(), nullable=True),
    sa.Column('male', sa.BOOLEAN(), nullable=True),
    sa.Column('female', sa.BOOLEAN(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('surname',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(), nullable=True),
    sa.Column('lan', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('surname')
    op.drop_table('given_name')
    # ### end Alembic commands ###


import torch
from torch import nn
from torch.nn.utils.weight_norm import weight_norm

from visdialch.utils import DynamicRNN

# Attend to bottom up features using question and history
class BottomUpLateFusionEncoder(nn.Module):
    def __init__(self, config, vocabulary):
        super().__init__()
        self.config = config

        self.word_embed = nn.Embedding(
            len(vocabulary), config["word_embedding_size"], padding_idx=vocabulary.PAD_INDEX
        )
        self.hist_rnn = nn.LSTM(
            config["word_embedding_size"],
            config["lstm_hidden_size"],
            config["lstm_num_layers"],
            batch_first=True,
            dropout=config["dropout"]
        )
        self.ques_rnn = nn.LSTM(
            config["word_embedding_size"],
            config["lstm_hidden_size"],
            config["lstm_num_layers"],
            batch_first=True,
            dropout=config["dropout"]
        )
        self.dropout = nn.Dropout(p=config["dropout"])

        # questions and history are right padded sequences of variable length
        # use the DynamicRNN utility module to handle them properly
        self.hist_rnn = DynamicRNN(self.hist_rnn)
        self.ques_rnn = DynamicRNN(self.ques_rnn)

        att_hidden_size = 512
        lstm_size = self.config["lstm_hidden_size"]

        # Ignoring weight_norm for now, which changes .weights key to .weights_g, .weights_v
        # Attention
        self.att_fc = nn.Sequential(weight_norm(nn.Linear(lstm_size * 2 + self.config["img_feature_size"], att_hidden_size), dim=None), nn.ReLU())
        self.att_weights = weight_norm(nn.Linear(att_hidden_size, 1))
        # self.att_fc = nn.Sequential(nn.Linear(lstm_size * 2 + self.config["img_feature_size"], att_hidden_size), nn.ReLU())
        # self.att_weights = nn.Linear(att_hidden_size, 1)

        # Embedders
        self.ques_net = nn.Sequential(weight_norm(nn.Linear(lstm_size, lstm_size), dim=None), nn.ReLU())
        self.img_net = nn.Sequential(weight_norm(nn.Linear(self.config["img_feature_size"], lstm_size), dim=None), nn.ReLU())
        # self.ques_net = nn.Sequential(nn.Linear(lstm_size, lstm_size), nn.ReLU())
        # self.img_net = nn.Sequential(nn.Linear(self.config["img_feature_size"], lstm_size), nn.ReLU())

        # Now the fusion for embeddings (NOT for context)
        self.fusion = nn.Linear(lstm_size, lstm_size)

        nn.init.kaiming_uniform_(self.fusion.weight)
        nn.init.constant_(self.fusion.bias, 0)

    def forward(self, batch):
        # shape: (batch_size, num_proposals, img_feature_size) - RCNN bottom-up features
        img = batch["img_feat"]
        # shape: (batch_size, 10, max_sequence_length)
        ques = batch["ques"]
        # shape: (batch_size, 10, max_sequence_length * 2 * 10), concatenated qa * 10 rounds
        hist = batch["hist"]
        # num_rounds = 10, even for test split (padded dialog rounds at the end)
        batch_size, num_rounds, max_sequence_length = ques.size()

        if img.dim() != 3:
            raise Exception("Expected bottom up image features with 3 dimensions, got {}".format(img.dim())) # Probably raise this outside of forward

        num_bu_feat = img.shape[1]

        # Create attention for each image feature

        # repeat image feature vectors to be provided for every round
        img = img.view(batch_size, 1, num_bu_feat, self.config["img_feature_size"])
        img = img.repeat(1, num_rounds, 1, 1)
        img = img.view(batch_size * num_rounds, num_bu_feat, self.config["img_feature_size"])

        # embed questions
        ques = ques.view(batch_size * num_rounds, max_sequence_length)
        ques_embed = self.word_embed(ques)
        ques_embed = self.ques_rnn(ques_embed, batch["ques_len"])
        stacked_ques = ques_embed.unsqueeze(1).repeat(1, num_bu_feat, 1)

        # embed history
        hist = hist.view(batch_size * num_rounds, max_sequence_length * 20)
        hist_embed = self.word_embed(hist)
        hist_embed = self.hist_rnn(hist_embed, batch["hist_len"])
        stacked_hist = hist_embed.unsqueeze(1).repeat(1, num_bu_feat, 1)

        fused_context = torch.cat((img, stacked_ques, stacked_hist), 2)
        fused_context = self.dropout(fused_context)

        # Convert fused_context to attention
        # ReLU for nonlinearity, as per modified VQA implementation
        att_vector = self.att_fc(fused_context)
        att_logits = self.att_weights(att_vector)
        att_weights = nn.functional.softmax(att_logits, 1)
        att_img = (att_weights * img).sum(1)

        # Match question and attended image dimensions with FC
        ques_repr = self.ques_net(ques_embed)
        img_repr = self.img_net(att_img)

        att_question = img_repr * ques_repr
        bu_embedding = torch.tanh(self.fusion(att_question))
        bu_embedding = bu_embedding.view(batch_size, num_rounds, -1)
        return bu_embedding


# -- coding:UTF-8 --

# 数据探测

import numpy as np
import os
import time

print(os.listdir('../data/40_garbage_classify-for-pythorch'))

from os import walk

# for(dirpath,dirname,filenames) in walk('../data/40_garbage_classify-for-pythorch'):
#     print("* " * 60)
#     print("Director path = ",dirpath)
#     print("total examples = ",len(filenames))
#     print("File name Examples = ",filenames[:5])

#2数据预处理

label_dict = {
    "0": "其他垃圾/一次性快餐盒",
    "1": "其他垃圾/污损塑料",
    "2": "其他垃圾/烟蒂",
    "3": "其他垃圾/牙签",
    "4": "其他垃圾/破碎花盆及碟碗",
    "5": "其他垃圾/竹筷",
    "6": "厨余垃圾/剩饭剩菜",
    "7": "厨余垃圾/大骨头",
    "8": "厨余垃圾/水果果皮",
    "9": "厨余垃圾/水果果肉",
    "10": "厨余垃圾/茶叶渣",
    "11": "厨余垃圾/菜叶菜根",
    "12": "厨余垃圾/蛋壳",
    "13": "厨余垃圾/鱼骨",
    "14": "可回收物/充电宝",
    "15": "可回收物/包",
    "16": "可回收物/化妆品瓶",
    "17": "可回收物/塑料玩具",
    "18": "可回收物/塑料碗盆",
    "19": "可回收物/塑料衣架",
    "20": "可回收物/快递纸袋",
    "21": "可回收物/插头电线",
    "22": "可回收物/旧衣服",
    "23": "可回收物/易拉罐",
    "24": "可回收物/枕头",
    "25": "可回收物/毛绒玩具",
    "26": "可回收物/洗发水瓶",
    "27": "可回收物/玻璃杯",
    "28": "可回收物/皮鞋",
    "29": "可回收物/砧板",
    "30": "可回收物/纸板箱",
    "31": "可回收物/调料瓶",
    "32": "可回收物/酒瓶",
    "33": "可回收物/金属食品罐",
    "34": "可回收物/锅",
    "35": "可回收物/食用油桶",
    "36": "可回收物/饮料瓶",
    "37": "有害垃圾/干电池",
    "38": "有害垃圾/软膏",
    "39": "有害垃圾/过期药物"
}
# 2.1导入图像分类库
import torch
from torchvision import datasets,models,transforms
from matplotlib import  pyplot as plt
# 解决中文乱码的问题
plt.rcParams['font.family'] = ['sans-serif']
plt.rcParams['font.sans-serif'] = ['SimHei']

# 2.2定义数据输入

# 数据tensor运行方式：cpu or  cuda
device = torch.device('coda' if torch.cuda.is_available() else 'cpu')

print('device = ',device)

# 定义数据路径
TRAIN = '../data/40_garbage_classify-for-pythorch/tarin'
VAL = '../data/40_garbage_classify-for-pythorch/val'

print('train_path = ',TRAIN)
print('val_path = ',VAL)

# device =  cpu
# train_path =  ../data/40_garbage_classify-for-pythorch/train
# val_path =  ../data/40_garbage_classify-for-pythorch/val

# 2.3原始垃圾数据展示
train_data = datasets.ImageFolder(TRAIN)
val_data = datasets.ImageFolder(VAL)

from PIL import Image
fig = plt.figure(figsize=(25,8))
print('train_data.imgs = ',train_data.imgs[:4])

# train_data.imgs中的每张图片，进行可视化展示，看看原始数据的样子
# for idx,img in enumerate(train_data.imgs[:18]):
#     # ax 图表示例
#     ax = fig.add_subplot(3,18/3,idx+1,xticks = [],yticks = [])
#
#     file_name_path = img[0]
#     target_idx = img[1]
#     print(idx,img,file_name_path,target_idx)
#     target_name = label_dict.get(str(target_idx),"")
#
# #     绘制图表
#     image = Image.open(file_name_path)
#
#     ax.set_title('{}-{}'.format(target_idx,target_name))
#     plt.imshow(image)
# plt.show()

# 2.4定义数据预处理方法

# 数据预处理过程
train_transforms = transforms.Compose([
    # 图像变了：对我们的图片进行缩放
    transforms.Resize((256,256)),
    # 裁剪：中心裁剪
    transforms.CenterCrop(224),
    # 数据归一化操作[0,1]
    transforms.ToTensor(),
    #图像数据标准化
    transforms.Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225])
])

val_transforms = transforms.Compose([
    # 图像变了：对我们的图片进行缩放
    transforms.Resize((256,256)),
    # 裁剪：中心裁剪
    transforms.CenterCrop(224),
    # 数据归一化操作[0,1]
    transforms.ToTensor(),
    #图像数据标准化
    transforms.Normalize(mean=[0.485,0.456,0.406],std=[0.229,0.224,0.225])
])

# 数据格式转换
train_data =datasets.ImageFolder(TRAIN,transform = train_transforms)
val_data =datasets.ImageFolder(VAL,transform = val_transforms)

print('train_data example = ',len(train_data))
print('val_data example = ',len(val_data))
# train_data example =  11841
# val_data example =  2961
# 那么转换后的数据格式是什么样的？我们重点关注imgs 和class_to_idx属性

print(train_data.imgs[:2])
print(train_data.class_to_idx)

assert  train_data.class_to_idx.keys() ==val_data.class_to_idx.keys()

class_list = [str(i) for i in list(range(40))]

# 2.5 数据加载
# 定义加载数据的常量
from torch.utils.data.dataset import Dataset
from torch.utils.data.dataloader import DataLoader

batch_size = 32
num_workers = 2
train_loader = DataLoader(
    train_data,batch_size = batch_size,
    num_workers = num_workers,
    shuffle = True
)

val_loader = DataLoader(
    val_data,batch_size = batch_size,
    num_workers = num_workers,
    shuffle = False
)
print("* "*60)

print('train_loader = ',train_loader.dataset)
print("* "*60)
print('val_loader = ',val_loader.dataset)

# 3 数据展示

images,labels = next(iter(train_loader))
print('images.shape = ',images.shape)
# images.shape =  torch.Size([32, 3, 224, 224])

# 取的batch_size 获取第一个图片
print(images[0].shape)
# torch.Size([3, 224, 224])

# 图片的展示
import  matplotlib.pyplot as plt
# 解决中文乱码的问题
# plt.rcParams['font.sans-serif'] = 'SimHei'

def imgshow(img):
    image = img.permute(1,2,0)#3,244,244->224,224,3
    image = torch.clamp(image,0,1)
    plt.imshow(image)
    # plt.show()

# 单个展示数据
# imgshow(images[1])

# 接下来，我们对batch_size大小的数据进行可视化展示
classes = [str(i) for i in list(range(40))]
fig = plt.figure(figsize=(25,8))

for idx in np.arange(18):
    # 生成一个图片 3*6 大小
    ax=fig.add_subplot(3,18/3,idx+1,xticks = [],yticks = [])
    # 展示
    imgshow(images[idx])

    # 增加一些标题信息
    target_idx = classes[labels[idx]]
    target_name = label_dict[target_idx]
    ax.set_title('{}-{}'.format(target_idx,target_name))
plt.show()

process/04垃圾分类数据预处理与可视化.py

04垃圾分类数据预处理与可视化.py


from pytest import raises
from . import (
    HttpJsonResponse,
    check_github_api_response_for_errors,
    sut_module_target,
)

exp_owner = "swellaby"
exp_repository = "letra"
raise_github_repository_not_found_error_mock_target = (
    f"{sut_module_target}.raise_github_repository_not_found_error"
)
raise_github_unauthorized_error_mock_target = (
    f"{sut_module_target}.raise_github_unauthorized_error"
)

def call_sut_function(resp):
    return check_github_api_response_for_errors(
        response=resp,
        owner=exp_owner,
        repository=exp_repository,
        request_headers={},
    )

def test_returns_on_200_status_code():
    resp = HttpJsonResponse(status=200, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_201_status_code():
    resp = HttpJsonResponse(status=201, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_202_status_code():
    resp = HttpJsonResponse(status=202, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_203_status_code():
    resp = HttpJsonResponse(status=203, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_204_status_code():
    resp = HttpJsonResponse(status=204, headers={}, data="")
    assert call_sut_function(resp) is None

def test_raises_correct_err_on_404_status_code(monkeypatch):
    resp = HttpJsonResponse(status=404, headers={}, data="")
    exp_err = "repository not found"
    exp_headers = {"Authorization": "token 9999999"}
    act_owner = ""
    act_repository = ""
    act_req_headers = {}

    def mock_not_found(owner: str, repository: str, request_headers: dict):
        nonlocal act_owner, act_repository, act_req_headers
        act_owner = owner
        act_repository = repository
        act_req_headers = request_headers
        raise ValueError(exp_err)

    monkeypatch.setattr(
        f"{sut_module_target}.raise_github_repository_not_found_error",
        mock_not_found,
    )

    with raises(ValueError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers=exp_headers,
        )

    assert str(err.value) == exp_err
    assert act_owner == exp_owner
    assert act_repository == exp_repository
    assert act_req_headers == exp_headers

def test_raises_correct_err_on_401_status_code():
    exp_err = (
        f"Invalid authentication. Unable "
        f"to access GitHub repository: {exp_owner}/{exp_repository}"
    )
    resp = HttpJsonResponse(status=401, headers={}, data="")

    with raises(ValueError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers={},
        )

    assert str(err.value) == exp_err

def test_raises_correct_err_on_403_status_code(monkeypatch):
    resp = HttpJsonResponse(status=403, headers={}, data="")
    exp_err = "not authorized"
    exp_headers = {"Authorization": "token 123987"}
    act_response = {}
    act_owner = ""
    act_repository = ""
    act_req_headers = {}

    def mock_not_authorized(response, owner, repository, request_headers):
        nonlocal act_response, act_owner, act_repository, act_req_headers
        act_response = response
        act_owner = owner
        act_repository = repository
        act_req_headers = request_headers
        raise IOError(exp_err)

    mock_target = f"{sut_module_target}.raise_github_unauthorized_error"
    monkeypatch.setattr(mock_target, mock_not_authorized)

    with raises(IOError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers=exp_headers,
        )

    assert str(err.value) == exp_err
    assert act_response == resp
    assert act_owner == exp_owner
    assert act_repository == exp_repository
    assert act_req_headers == exp_headers

def test_raises_correct_err_on_unknown_status_code():
    status_code = 500
    exp_err = (
        "Unexpected HTTP error encountered from GitHub API. "
        f"HTTP response status code: {status_code}"
    )
    resp = HttpJsonResponse(status=status_code, headers={}, data="")

    with raises(IOError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers={},
        )

    assert str(err.value) == exp_err


import os
import sys  # for PyQt5.5 debugging
from PyQt5.QtWidgets import *
from PyQt5 import uic
from PyQt5 import QtCore
import pickle

form_entry = uic.loadUiType("GUI/EntryForm.ui")[0]
form_main_suite = uic.loadUiType('GUI/MainSuiteForm.ui')[0]

class DataDNAIdentification:
    def __init__(self, location_save='', analyst='', date='', optional=''):
        self.location_save = location_save
        self.analyst = analyst
        self.date = date
        self.optional = optional

    def get_dir(self):
        new_dir = '%s/%s_%s' % (self.location_save,self.date, self.analyst) if self.optional=='' \
            else '%s/%s_%s_%s' % (self.location_save,self.date, self.analyst, self.optional)
        return new_dir

class EntryForm(QDialog, form_entry):
    def __init__(self):
        super().__init__()
        self.setupUi(self)
        readfile_setting = open(os.getcwd() + '/Settings/Settings.ini', mode='r')  # File I/O Error 가능성
        location_save = readfile_setting.readline().rstrip('\n').split('=')[1]
        analyst = readfile_setting.readline().rstrip('\n').split('=')[1]
        readfile_setting.close()
        self.set_line_texts(location_save=location_save, analyst=analyst,
                            date=QtCore.QDate.currentDate())
        self.GUI_main_suite = None

    def set_line_texts(self, location_save="", analyst="", date=QtCore.QDate.currentDate(), optional=""):
        self.line_savelocation.setText(location_save)
        self.line_analyst.setText(analyst)
        self.line_date.setDate(date)
        self.line_optional.setText(optional)

    def set_line_ro(self, condition=True):
        self.line_analyst.setReadOnly(condition)
        self.line_date.setReadOnly(condition)
        self.line_optional.setReadOnly(condition)

    def click_btn_change_savelocation(self):
        foldername = QFileDialog.getExistingDirectory(self, 'Open folder')
        if foldername is not "":
            self.line_savelocation.setText(foldername)
        else:
            return

    def click_btn_new(self):
        QMessageBox.information(self, "Notice", "Input new profile and press OK button.")
        self.set_line_ro(False)

    def click_btn_ok(self):
        new_dir = '%s/%s_%s' % (self.line_savelocation.text(),
                                self.line_date.date().toString('yyyyMMdd'),
                                self.line_analyst.text()) if self.line_optional.text() == '' \
                                else '%s/%s_%s_%s' % (self.line_savelocation.text(),
                                                      self.line_date.date().toString('yyyyMMdd'),
                                                      self.line_analyst.text(),
                                                      self.line_optional.text())
        print(new_dir)
        os.chdir(self.line_savelocation.text())
        if not os.path.exists(new_dir):  # 덮어쓰기 허용
            os.mkdir(new_dir)
            os.chdir(new_dir)
        else:
            QMessageBox.information(self, "Error", "Same folder exists")
            return
        new_ddi = DataDNAIdentification(location_save=new_dir, analyst=self.line_analyst.text(),
                                        date=self.line_date.date().toString('yyyyMMdd'),
                                        optional=self.line_optional.text())
        self.GUI_main_suite = MainSuiteForm(new_ddi)
        self.GUI_main_suite.show()
        self.close()

    def click_btn_load(self):
        location_load = QFileDialog.getExistingDirectory(self, 'Open folder')
        if location_load is not "":
            if os.path.exists('%s/%s' % (location_load, 'data.pickle')):
                os.chdir(location_load)
                with open('data.pickle', 'rb') as f:
                    load_ddi = pickle.load(f)
                    print('loaded')
                self.GUI_main_suite = MainSuiteForm(load_ddi)
                self.GUI_main_suite.show()
                self.close()
            else:
                QMessageBox.information(self, 'Error', 'No information file in the folder.')
                return
        else:
            QMessageBox.information(self, "Error", "Select a folder.")
            return

    def click_btn_exit(self):
        sys.exit()

class MainSuiteForm(QMainWindow, form_main_suite):
    def __init__(self, ddi):
        super().__init__()
        self.setupUi(self)
        self.ddi_present = ddi
        print(self.ddi_present.get_dir())
        self.save()

    def save(self):
        with open('data.pickle', 'wb') as f:
            pickle.dump(self.ddi_present, f, pickle.HIGHEST_PROTOCOL)
            print('saved')

def except_hook(cls, exception, traceback): # for PyQt5.5 debugging
    sys.__excepthook__(cls, exception, traceback)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    GUI_entry = EntryForm()
    GUI_entry.setFixedSize(GUI_entry.size())  # 창 크기 고정
    GUI_entry.show()
    sys.excepthook = except_hook    # for PyQt5.5 debugging
    sys.exit(app.exec_())


from setuptools import setup

entrypoints = {
        'console_scripts': ['HydXS=HydXS.cli_interface:app'],
    }

setup(name='HydXS',
      version='0.1a',
      description='A tool for automatic detection of bankfull boundaries from river cross section data',
      url='http://github.com/sailngarbwm/HydXS',
      author='kathryn russell',
      author_email='kathryn.russel@unimelb.edu.au',
      license='CC',
      packages=['HydXS'],
      entry_points = entrypoints,
      zip_safe=False)


import cv2 as ocv
import time
from bluedot import BlueDot
from signal import pause

camera = ocv.VideoCapture(0)
webcam = 0
picture = 0

camera.set(3, 640) #Largeur
camera.set(4, 480) #Hauteur
camera.set(10, 200) #Luminosite

fermion = ocv.imread("Ressources/Fermion.jpg")

def trigger(position):

    if position.top:

        print("Starting Webcam")
        webcam = 1

    elif position.bottom:

        print("Picture Display")
        ocv.imshow("Fermion", fermion)
        time.sleep(5)

    elif position.right:

        print("STOP")
        exit()

bd = BlueDot()

bd.when_pressed = trigger

pause()

Programmes et ressouces de tests/TESTOCV.py


from model import MixMatchVAE
import torch
from scipy.interpolate import interp1d
import numpy as np
import torchvision

def sample():
    n_sample = 20
    x, y = next(iter(testloader))
    result = torch.zeros(x.shape[0] * n_sample, 1, 28, 28).to(device)
    C = torch.zeros(x.shape[0] * n_sample, 1, 28, 28).to(device)
    z_val = [z for z in np.arange(-2, 2, 0.1)]
    with torch.no_grad():
        cnt = 0
        for i in range(x.shape[0]):
            reshaped_image = x[i:i+1].view(1, -1)
            data, condition = reshaped_image[:, :500], \
                              reshaped_image[:, 500:]
            data, condition = data.to(device), condition.to(device)
            result[cnt] = x[i]
            cnt += 1
            C_image = torch.zeros(reshaped_image.shape).to(device)
            C_image[:, 500:] = condition
            C_image = C_image.view(x[i:i + 1].shape)
            result[cnt] = C_image
            cnt += 1
            for j in range(2, n_sample):
                z = torch.ones(1, 32).to(device)
                z *= z_val[j]
                reconstructed = model.sample(condition, 64)
                rec_image = torch.zeros(reshaped_image.shape).to(device)

                rec_image[:, :500] = reconstructed
                rec_image[:, 500:] = condition

                rec_image = rec_image.view(x[i:i+1].shape)
                result[cnt] = rec_image
                cnt += 1
    torchvision.utils.save_image(result, nrow=20, fp="results/sampled.png")

if __name__ == "__main__":
    device = "cuda"

    class MixMatchArgs(object):
        def __init__(self):
            self.input_dim = 500  # (28 * 28) // 2
            self.output_dim = 500  # (28 * 28) // 2
            self.condition_dim = (28 * 28) - 500
            self.hidden_dim = 128
            self.latent_dim = 32
            self.device = device

    model = MixMatchVAE(MixMatchArgs()).to(device)

    transform = torchvision.transforms.Compose([
        torchvision.transforms.ToTensor(),
    ])

    testset = torchvision.datasets.MNIST(root='./data', train=False,
                                         download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=20,
                                             shuffle=False, num_workers=0)

    model.load_state_dict(torch.load("checkpoint/mix_match_48.pth"))
    model.eval()
    sample()


#!/usr/bin/python3

import configparser # Python 3 only
import sys, os
import glob
import time
import shutil

# KDE rc files differences:
#     Keys are cAsE sensitive
#     No spaces around the =
#     [Sections can have spaces and : colons]
# Also (NOT FIXED):
#     Can have [Sub][Sections]
class KdeConfig(configparser.ConfigParser):
	def __init__(self, filename):
		super().__init__()

		# Keep case sensitive keys
		# http://stackoverflow.com/questions/19359556/configparser-reads-capital-keys-and-make-them-lower-case
		self.optionxform = str

		self.filename = filename
		self.read(self.filename)

	def set(self, section, option, value):
		if not self.has_section(section):
			self.add_section(section)
		super().set(section, option, str(value))

	def setProp(self, key, value):
		section, option = key.split('.', 1)
		return self.set(section, option, value)

	def getProp(self, key):
		section, option = key.split('.', 1)
		return self.get(section, option)

	def default(self, section, option, value):
		if not self.has_option(section, option):
			self.set(section, option, value)

	def save(self):
		with open(self.filename, 'w') as fp:
			self.write(fp, space_around_delimiters=False)

def limit(minVal, val, maxVal):
	return min(max(minVal, val), maxVal)

def limitColor(val):
	return limit(0, val, 255)

def deltaColor(colorStr, delta):
	r, g, b = map(int, colorStr.split(','))
	if max(r, g, b) >= 128:
		d = -delta
	else:
		d = delta
	return '{},{},{}'.format(limitColor(r+d), limitColor(g+d), limitColor(b+d))

def hoverEffect(colorStr, rd, gd, bd):
	def f(a, b):
		if a + b > 255:
			return limitColor(a - b)
		else:
			return limitColor(a + b)
	r, g, b = map(int, colorStr.split(','))
	return '{},{},{}'.format(f(r, rd), f(g, gd), f(b, bd))

def setTitlebarColors(backgroundColor='0,0,0', textColor='255,255,255'):
	filename = os.path.abspath(os.path.expanduser('~/.config/kdeglobals'))
	kdeglobals = KdeConfig(filename)
	kdeglobals['WM']['activeBackground'] = backgroundColor
	kdeglobals['WM']['inactiveBackground'] = backgroundColor
	kdeglobals['WM']['frame'] = backgroundColor
	kdeglobals['WM']['inactiveFrame'] = backgroundColor
	kdeglobals['WM']['activeForeground'] = textColor
	kdeglobals['WM']['inactiveForeground'] = textColor
	kdeglobals.save()

def applyColorSchemeTitlebarColors(kdeglobals, colorScheme):
	# Default to Breeze.colors if the file leaves it undefined.
	activeBackgroundColor = colorScheme['Colors:Window'].get('BackgroundNormal', '239,240,241')
	kdeglobals['WM']['activeBackground'] = colorScheme['WM'].get('activeBackground', '71,80,87')
	kdeglobals['WM']['inactiveBackground'] = colorScheme['WM'].get('inactiveBackground', '239,240,241')
	kdeglobals['WM']['frame'] = colorScheme['WM'].get('frame', activeBackgroundColor)
	kdeglobals['WM']['inactiveFrame'] = colorScheme['WM'].get('inactiveFrame', kdeglobals['WM']['frame'])
	kdeglobals['WM']['activeForeground'] = colorScheme['WM'].get('activeForeground', '252,252,252')
	kdeglobals['WM']['inactiveForeground'] = colorScheme['WM'].get('inactiveForeground', '189,195,199')
	kdeglobals.save()

def scanForColorScheme(colorSchemeName):
	def testColorSchemeName(colorScheme, colorSchemeName):
		return colorScheme['General'].get('Name', '') == colorSchemeName

	def scanColorSchemeDir(colorSchemeDir):
		colorSchemeFilename = colorSchemeName + '.colors'
		dirFilenames = os.listdir(colorSchemeDir)

		# First check to see if the obvious filename exists.
		if colorSchemeFilename in dirFilenames:
			# If so, move it to the front.
			dirFilenames.insert(0, dirFilenames.pop(dirFilenames.index(colorSchemeFilename)))

		# Scan folder for the correct color scheme
		for filename in os.listdir(colorSchemeDir):
			colorSchemeFilepath = os.path.join(colorSchemeDir, filename)
			# print(colorSchemeFilepath)

			if not filename.endswith('.colors'):
				continue

			if not os.path.isfile(colorSchemeFilepath):
				continue

			colorScheme = KdeConfig(colorSchemeFilepath)
			if testColorSchemeName(colorScheme, colorSchemeName):
				return colorScheme

		# Color scheme was not found in this dir
		return None

	# 1st: Check the home color-scheme dir (downloads by the user).
	homeColorSchemeFilepath = os.path.abspath(os.path.expanduser('~/.local/share/color-schemes/'))
	colorScheme = scanColorSchemeDir(homeColorSchemeFilepath)
	if colorScheme:
		return colorScheme

	# 2nd: Check the root color-scheme dir.
	colorScheme = scanColorSchemeDir('/usr/share/color-schemes/')
	if colorScheme:
		return colorScheme

	# No color scheme fulfulled the conditions anywhere
	return None

def resetTitlebarColors():
	kdeglobalsFilename = os.path.abspath(os.path.expanduser('~/.config/kdeglobals'))
	kdeglobals = KdeConfig(kdeglobalsFilename)

	colorSchemeName = kdeglobals['General'].get('ColorScheme', 'Breeze')

	colorScheme = scanForColorScheme(colorSchemeName)
	if colorScheme:
		applyColorSchemeTitlebarColors(kdeglobals, colorScheme)

	# elif os.path.isfile(localDesktopThemeFilename):
	# elif os.path.isfile(rootDesktopThemeFilename):
		# The color scheme is probably in a desktop theme folder...
		# Whatever, user can reapply it.
	else:
		# Deleting the keys will use white + highlight color,
		# which isn't the same as the default Breeze colors.
		del kdeglobals['WM']['activeBackground']
		del kdeglobals['WM']['inactiveBackground']
		del kdeglobals['WM']['frame']
		del kdeglobals['WM']['inactiveFrame']
		del kdeglobals['WM']['activeForeground']
		del kdeglobals['WM']['inactiveForeground']
		kdeglobals.save()

class DesktopTheme:
	def __init__(self, themeName):
		self.themeName = themeName
		themeDir = os.path.abspath(os.path.expanduser('~/.local/share/plasma/desktoptheme'))
		self.themeDir = os.path.join(themeDir, themeName)
		self.dontReload = False

	def colorsConfig(self):
		filename = os.path.join(self.themeDir, 'colors')
		return KdeConfig(filename)

	def reloadTheme(self):
		if self.dontReload:
			# We're setting multiple properties at once,
			# and want to skip automatically reloading
			# after each change.
			return

		# Switch to another theme and back to apply the changes to breeze-alphablack.
		filename = os.path.abspath(os.path.expanduser('~/.config/plasmarc'))
		plasmarc = KdeConfig(filename)

		# Make sure we're not already in the process of changing the theme.
		if plasmarc['Theme']['name'] == self.themeName:
			plasmarc['Theme']['name'] = 'breeze-dark'
			plasmarc.save()
			time.sleep(1)
		plasmarc['Theme']['name'] = self.themeName
		plasmarc.save()

	def clearCache(self):
		wildcardPath='plasma-svgelements-{}_*'.format(self.themeName)
		cacheDir = os.path.realpath(os.path.expanduser('~/.cache/'))
		for file in glob.glob(os.path.join(cacheDir, wildcardPath)):
			print('Deleting cached theme file "{}"'.format(file))
			os.remove(file)

		wildcardPath='plasma_theme_{}_*.kcache'.format(self.themeName)
		for file in glob.glob(os.path.join(cacheDir, wildcardPath)):
			print('Deleting cached theme file "{}"'.format(file))
			os.remove(file)

class BreezeAlphaBlack(DesktopTheme):
	def __init__(self, themeName='breeze-alphablack'):
		super().__init__(themeName=themeName)
		self.configProps = [
			# (group, key, defaultValue, setterKey)
			('dialog', 'opacity', 0.9, 'setDialogOpacity'),
			('dialog', 'padding', 6, 'setDialogPadding'),
			('panel', 'opacity', 0.9, 'setPanelOpacity'),
			('panel', 'padding', 2, 'setPanelPadding'),
			('panel', 'taskStyle', 'inside', 'setTasksSvg'),
			('theme', 'accentColor', '0,0,0', 'setAccentColor'),
			('theme', 'highlightColor', '61,174,230', 'setHighlightColor'),
			('theme', 'textColor', '239,240,241', 'setTextColor'),
			('widget', 'opacity', 0.9, 'setWidgetOpacity'),
		]

	def themeConfig(self):
		filename = os.path.join(self.themeDir, 'config.ini')
		config = KdeConfig(filename)
		for (group, key, value, setterKey) in self.configProps:
			config.default(group, key, value)
		return config

	def configGet(self, propPath):
		config = self.themeConfig()
		value = config.getProp(propPath)
		return value

	def configSet(self, propPath, value):
		group1, key1 = propPath.split('.')
		for (group2, key2, defaultValue, setterKey) in self.configProps:
			if group1 == group2 and key1 == key2:
				setter = getattr(self, setterKey)
				setter(value)
				break

	def useTemplate(self, inPath, outPath):
		templatePath = os.path.join(self.themeDir, '_templates', inPath)
		outPath = os.path.join(self.themeDir, outPath)
		shutil.copyfile(templatePath, outPath)
		self.reloadTheme()

	def renderTemplate(self, templatePath, **kwargs):
		inFilename = os.path.join(self.themeDir, '_templates', templatePath)
		outFilename = os.path.join(self.themeDir, templatePath)
		outDir = os.path.dirname(outFilename)
		os.makedirs(outDir, exist_ok=True)
		with open(inFilename, 'r') as fin:
			with open(outFilename, 'w') as fout:
				for line in fin:
					for a, b in kwargs.items():
						line = line.replace(a, b)
					fout.write(line)

	def getDialogPadding(self, config):
		# Plasma doesn't like a padding of 0, so just use a really small number (which is rounded to 0).
		dialogPadding = float(config.getProp('dialog.padding'))
		dialogPadding = max(0.001, dialogPadding)
		return dialogPadding

	def renderDialogBackground(self, config):
		dialogPadding = self.getDialogPadding(config)
		self.renderTemplate('dialogs/background.svg', **{
			'{{fillOpacity}}': str(config.getProp('dialog.opacity')),
			'{{padding}}': str(dialogPadding),
		})

	def renderPlasmoidHeading(self, config):
		dialogPadding = self.getDialogPadding(config)
		self.renderTemplate('widgets/plasmoidheading.svg', **{
			'{{padding}}': str(dialogPadding),
		})

	def setDialogProperty(self, key, newValue):
		config = self.themeConfig()
		config.set('dialog', key, newValue)
		self.renderDialogBackground(config)
		self.renderPlasmoidHeading(config)
		config.save()
		self.clearCache() # Not really necessary
		self.reloadTheme()

	def setDialogOpacity(self, newOpacity=0.9):
		self.setDialogProperty('opacity', newOpacity)

	def setDialogPadding(self, newPadding=6):
		self.setDialogProperty('padding', newPadding)

	def renderPanel(self, config):
		# inFilename = os.path.join(self.themeDir, '_templates/panel-background.svg')
		# outFilename = os.path.join(self.themeDir, 'widgets/panel-background.svgz')
		# buildFromTemplate(inFilename, outFilename, **{
		panelOpacity = config.getProp('panel.opacity')
		noPanelPadding = config.getint('panel', 'padding') == 0
		shadowOpacity = 1 if float(panelOpacity) >= 0.3 else 0
		self.renderTemplate('widgets/panel-background.svg', **{
			'{{panelOpacity}}': str(panelOpacity),
			'{{panelPadding}}': str(config.getProp('panel.padding')),
			'{{noPanelPadding}}': '<rect id="hint-no-border-padding"></rect>' if noPanelPadding else '',
			'{{shadowOpacity}}': str(shadowOpacity),
		})
		print('noPanelPadding', noPanelPadding)
		print('shadowOpacity', shadowOpacity)

	def setPanelOpacity(self, newOpacity=0.9):
		config = self.themeConfig()
		config.set('panel', 'opacity', newOpacity)
		self.renderPanel(config)
		config.save()
		self.clearCache() # Not really necessary
		self.reloadTheme()

	def setPanelPadding(self, newPadding=2):
		config = self.themeConfig()
		config.set('panel', 'padding', newPadding)
		self.renderPanel(config)
		config.save()
		self.clearCache() # Necessary
		self.reloadTheme()

	def renderWidgetBackground(self, config):
		# Breeze's widget/background.svg
		# fill: 0.9
		# shadows: 1.0 (Linear gradients)
		# corner shadows: 0.6 (Radial gradients)
		fillOpacity = float(config.getProp('widget.opacity'))
		if fillOpacity == 0:
			# Hide shadows completely if there's no fill
			shadowOpacity = 0
			cornerOpacity = 0
		else:
			shadowOpacity = min(fillOpacity + 0.1, 1)
			cornerOpacity = 0.6 * shadowOpacity

		self.renderTemplate('widgets/background.svg', **{
			'{{fillOpacity}}': str(fillOpacity),
			'{{shadowOpacity}}': str(shadowOpacity),
			'{{cornerOpacity}}': str(cornerOpacity),
		})

	def setWidgetOpacity(self, newOpacity=0.9):
		config = self.themeConfig()
		config.set('widget', 'opacity', newOpacity)
		self.renderWidgetBackground(config)
		config.save()
		self.clearCache() # Not really necessary
		self.reloadTheme()

	def _applyColors(self, accentColor='0,0,0', textColor='239,240,241', highlightColor='61,174,230'):
		altColor = deltaColor(accentColor, 23)
		compColor = deltaColor(accentColor, 17)
		focusColor = hoverEffect(highlightColor, -31, -28, 25) # 61,174,230 => 30,146,255
		hoverColor = highlightColor
		selectionColor = highlightColor
		selectionAltColor = hoverEffect(selectionColor, -13, -36, -47) # 61,174,230 => 48,138,183

		print('BackgroundNormal: {}'.format(accentColor))
		print('BackgroundAlternate: {}'.format(altColor))
		print('Complementary.BackgroundNormal: {}'.format(compColor))
		print('DecorationFocus: {}'.format(focusColor))
		print('DecorationHover: {}'.format(hoverColor))
		print('Selection.BackgroundNormal: {}'.format(selectionColor))
		print('Selection.BackgroundAlternate: {}'.format(selectionAltColor))

		setTitlebarColors(accentColor, textColor)

		colors = self.colorsConfig()
		allColorGroups = [
			'Colors:Button',
			'Colors:Selection',
			'Colors:Tooltip',
			'Colors:View',
			'Colors:Window',
			'Colors:Complementary',
		]
		def applyToGroups(groups, key, value):
			for group in groups:
				colors[group][key] = value

		colors['Colors:Button']['BackgroundNormal'] = compColor
		colors['Colors:Button']['BackgroundAlternate'] = altColor
		colors['Colors:Button']['ForegroundNormal'] = textColor
		colors['Colors:View']['BackgroundNormal'] = accentColor
		colors['Colors:View']['BackgroundAlternate'] = altColor
		colors['Colors:View']['ForegroundNormal'] = textColor
		colors['Colors:Window']['BackgroundNormal'] = accentColor
		colors['Colors:Window']['BackgroundAlternate'] = altColor
		colors['Colors:Window']['ForegroundNormal'] = textColor
		colors['Colors:Complementary']['BackgroundNormal'] = compColor
		colors['Colors:Complementary']['BackgroundAlternate'] = altColor
		colors['Colors:Complementary']['ForegroundNormal'] = textColor

		# Focus
		applyToGroups(allColorGroups, 'DecorationFocus', focusColor)

		# Hover
		applyToGroups(allColorGroups, 'DecorationHover', hoverColor)
		colors['Colors:Selection']['BackgroundNormal'] = selectionColor # Note this variable controls `theme.highlightColor`
		colors['Colors:Selection']['BackgroundAlternate'] = selectionAltColor

		colors.save()

		config = self.themeConfig()
		config.set('theme', 'accentColor', accentColor)
		config.set('theme', 'highlightColor', highlightColor)
		config.set('theme', 'textColor', textColor)
		config.save()

		self.reloadTheme()

	def applyColors(self, accentColor=None, textColor=None, highlightColor=None):
		config = self.themeConfig()
		if accentColor is None:
			accentColor = config.get('theme', 'accentColor')
		if textColor is None:
			textColor = config.get('theme', 'textColor')
		if highlightColor is None:
			highlightColor = config.get('theme', 'highlightColor')
		self._applyColors(accentColor, textColor, highlightColor)

	def setAccentColor(self, accentColor):
		self.applyColors(accentColor=accentColor)

	def setHighlightColor(self, highlightColor):
		self.applyColors(highlightColor=highlightColor)

	def setTextColor(self, textColor):
		self.applyColors(textColor=textColor)

	def setTasksSvg(self, taskTheme):
		templatePath = "tasks-{}.svg".format(taskTheme)
		self.useTemplate(templatePath, 'widgets/tasks.svg')

		config = self.themeConfig()
		config.set('panel', 'taskStyle', taskTheme)
		config.save()

		self.reloadTheme()

	def resetToDefaults(self):
		self.dontReload = True
		for (group, key, value, setterKey) in self.configProps:
			print("===[ {}.{}: {} ]===".format(group, key, value))
			setter = getattr(self, setterKey)
			setter(value)

		self.dontReload = False
		self.reloadTheme()

#--- Main
def theme_getall(args):
	import json

	desktoptheme = BreezeAlphaBlack()
	config = desktoptheme.themeConfig()

	if args.json:
		out = {}
		for section in config.sections():
			out[section] = {}
			for name, value in config.items(section):
				out[section][name] = value
		print(json.dumps(out, indent="\t"))
	else:
		for section in config.sections():
			for name, value in config.items(section):
				print("{}.{}: {}".format(section, name, value))

def theme_get(args):
	desktopTheme = BreezeAlphaBlack()
	argsVars = vars(args)
	propPath = argsVars['section.property']
	value = desktopTheme.configGet(propPath)
	print(value)

def theme_set(args):
	desktopTheme = BreezeAlphaBlack()
	config = desktopTheme.themeConfig()
	argsVars = vars(args)
	propPath = argsVars['section.property']
	desktopTheme.configSet(propPath, args.value)

def theme_reset(args):
	desktopTheme = BreezeAlphaBlack()
	desktopTheme.resetToDefaults()

def theme_resetTitlebarColors(args):
	resetTitlebarColors()

def main():
	import argparse

	parser = argparse.ArgumentParser(prog='desktoptheme', description='Python script to modify a desktop theme.')
	subparsers = parser.add_subparsers()

	def add_subcommand(name, func, *args):
		tokens = ['[{}]'.format(arg) for arg in args]
		tokens = ['python3', 'desktoptheme.py', name] + tokens
		cmdstr = ' '.join(tokens)
		parser_subcommand = subparsers.add_parser(name, help=cmdstr)
		parser_subcommand.set_defaults(func=func)
		for arg in args:
			parser_subcommand.add_argument(arg)
		return parser_subcommand

	parser_getall = add_subcommand('getall', theme_getall)
	parser_getall.add_argument('--json', default=False, action='store_true')
	add_subcommand('get', theme_get, 'section.property')
	add_subcommand('set', theme_set, 'section.property', 'value')
	add_subcommand('reset', theme_reset)
	add_subcommand('resettitlebarcolors', theme_resetTitlebarColors)

	args = parser.parse_args()

	if 'func' in args:
		try:
			args.func(args)
		except KeyboardInterrupt:
			pass
	else:
		parser.print_help()

if __name__ == '__main__':
	main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (c) 2015, utamaro
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import unittest
import json
import filecmp
import storjswift
import time
import sys

class TestStorjswift(unittest.TestCase):
    def speedCheck(self,status,hash,rate=1):
        j=json.loads(status)
        value=j[hash]['uploadSpeed']
        self.assertTrue(value>=0,"upload speed limit test")
#        self.assertTrue(value>=0 and value<1024*1024*rate,"upload speed limit test")
        value=j[hash]['downloadSpeed']
        self.assertTrue(value>=0,"download speed limit test")
#        self.assertTrue(value>=0 and value<1024*1024*rate,"download speed limit test")

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def leecherTest(self):
        self.assertTrue(storjswift.getPort()==0,"before start status test")
        port=storjswift.startDaemon(20000)
        self.assertTrue(port==-1,"client bind fail test")
        port=storjswift.startDaemon()
        self.assertTrue(port>0,"client bind test")
        hash=storjswift.getHash("tests/bigBuckBunny_avchd_720p.ogv")
        self.assertTrue(hash=="5e9d1f02e8a2ff5d2783b6aefe1d3c570d6b54c8","hash test")
        storjswift.addFile("result.ogv",hash)
        storjswift.addPeer("127.0.0.1:20000",hash)
        self.assertTrue(not storjswift.isComplete(hash),"not completion test")
        status=storjswift.reportStatus()
        print("status:"+status)
        j=json.loads(status)
        f=j[hash]['filename']
        self.assertTrue(f=="result.ogv","file name test")
        value=j[hash]['seeders']
        self.assertTrue(value==1,"seeders test")
        value=j[hash]['leechers']
        self.assertTrue(value==0,"leechers test")
        self.speedCheck(status,hash,4)

        print("sleeping 100s")
        time.sleep(100)
        status=storjswift.reportStatus()
        print("status:"+status)
        j=json.loads(status)
        value=j[hash]['fileSize']
        self.assertTrue(value>46935927 and value<46935929,"file size test")
        value=j[hash]['downloadedBytes']
        self.assertTrue(value>46935928,"downloaded bytes size test")
        value=j[hash]['downloadedFileSize']
        self.assertTrue(value>46935927 and value<46935929,"downloaded file size test")
        self.assertTrue(filecmp.cmp("result.ogv","tests/bigBuckBunny_avchd_720p.ogv"),"downloaded file correctness test")
        self.assertTrue(storjswift.isComplete(hash),"completion test")
        self.assertTrue(not storjswift.isComplete("aaaaaa"),"completion check to non-existed file test")
        self.speedCheck(status,hash)

        rr=storjswift.removeFile("aaa",True,True);
        self.assertTrue(rr==-1,"remove non existed file test")
        storjswift.removeFile(hash,1,1)
        status=storjswift.reportStatus()
        print("status:"+status)
        j=json.loads(status)
        self.assertTrue(hash not in j ,"file hash after removal test")
        self.assertTrue(not os.path.exists("result.ogv"),"file removal test")
        self.assertTrue(not os.path.exists("result.ogv.mhash"),"mhash file removal test")
        self.assertTrue(not os.path.exists("result.ogv.mbinmap"),"mbinmap removal test")
        rr=storjswift.removeFile(hash,True,True)
        self.assertTrue(rr==-1,"remove file from empty list test")

        storjswift.shutdown()

    def seederTest(self):
        port=storjswift.startDaemon(20001)
        self.assertTrue(port==20001,"server bind test")
        hash=storjswift.addFile("tests/bigBuckBunny_avchd_720p.ogv")
        storjswift.setMaxSpeed(hash,1024*1024,1024*1024)
        r=storjswift.setMaxSpeed("aaa",1024*1024,1024*1024);
        self.assertTrue(r==-1,"set speed to non-existed file test");
        status=storjswift.reportStatus()
        print("status:"+status)
        j=json.loads(status)
        value=j[hash]['seeders']
        self.assertTrue(value==0,"seeders test")
        value=j[hash]['leechers']
        self.assertTrue(value==0,"leechers test")
        self.speedCheck(status,hash,2)

        print("sleeping 100s")
        time.sleep(100)
        status=storjswift.reportStatus()
        print("status:"+status)
        j=json.loads(status)
        value=j[hash]['leechers']
        self.assertTrue(value==0,"leechers test")
        value=j[hash]['uploadedBytes']
        self.assertTrue(value>46935928,"uploaded bytes size test")
        self.speedCheck(status,hash)
        storjswift.shutdown()

#    def test_storjfs(self):
#        if sys.argv[1]=="seederTest":
#            self.seeder()
#        if sys.argv[1]=="leecherTest":
#            self.leecher()

if __name__ == '__main__':
    unittest.main()


import pandas as pn
from sklearn.linear_model import Perceptron
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler

data_train = pn.read_csv('/Users/oleg/PycharmProjects/ML-Found-Yandex/DATA/train.csv', header = None)
data_test = pn.read_csv('/Users/oleg/PycharmProjects/ML-Found-Yandex/DATA/test.csv', header = None)

clf = Perceptron(random_state=241)
clf.fit(data_train.loc[:, 1:], data_train.loc[:, 0])
predictions = clf.predict(data_test.loc[:, 1:])
accuracy_before = accuracy_score(data_test.loc[:, 0], predictions)
# print('accuracy before normalize: ', accuracy)

#Normalize data sets
scaler = StandardScaler()
data_train_scaled = scaler.fit_transform(data_train.loc[:, 1:])
data_test_scaled = scaler.transform(data_test.loc[:, 1:])

y_train_scaled = data_train.loc[:, 0]
y_test_scaled = data_test.loc[:, 0]

clf.fit(data_train_scaled, y_train_scaled)
predictions = clf.predict(data_test_scaled)
accuracy_after = accuracy_score(y_test_scaled, predictions)
print(accuracy_after - accuracy_before)
print(accuracy_after)
# print(data_test)