import sys
import nibabel as nib
import numpy
import argparse

def main(*args):

  helpText = "   Match the histogram of an image to the histogram of a reference image using information " \
             "from their segmentations."
  parser = argparse.ArgumentParser(description=helpText)

  helpText = "Reference image"
  parser.add_argument("ref", type=str, help=helpText)

  helpText = "Source image"
  parser.add_argument("src", type=str, help=helpText)

  helpText = "Reference segmentation"
  parser.add_argument("refSeg", type=str, help=helpText)

  helpText = "Source segmentation"
  parser.add_argument("srcSeg", type=str, help=helpText)

  helpText = "Output image"
  parser.add_argument("output", type=str, help=helpText)

  helpText = "ventricle label (default 5)"
  parser.add_argument("-ventLabel", type=int, help=helpText)

  helpText = "GM label (default 3)"
  parser.add_argument("-gmLabel", type=int, help=helpText)

  helpText = "WM label (default 2)"
  parser.add_argument("-wmLabel", type=int, help=helpText)

  helpText = "CSF label (default 1)"
  parser.add_argument("-csfLabel", type=int, help=helpText)

  #############################################################

  args = parser.parse_args()

  refImgName = args.ref
  srcImgName = args.src

  refSegName = args.refSeg
  srcSegName = args.srcSeg

  outImgName = args.output

  ventLabel = args.ventLabel
  gmLabel = args.gmLabel
  wmLabel = args.wmLabel
  csfLabel = args.csfLabel

  if ventLabel == None:
    ventLabel = 5
  if gmLabel == None:
    gmLabel = 3
  if wmLabel == None:
    wmLabel = 2
  if csfLabel == None:
    csfLabel = 1

  refImg = nib.load(refImgName)
  srcImg = nib.load(srcImgName)

  refImgData = refImg.get_data().squeeze()
  srcImgData = srcImg.get_data().squeeze()

  srcMin = numpy.min(srcImgData)
  srcMax = numpy.max(srcImgData)

  refMin = numpy.min(refImgData)
  refMax = numpy.max(refImgData)

  outImgData = srcImgData.copy()

  refSegImg = nib.load(refSegName)
  refSeg = refSegImg.get_data().squeeze()
  if not (refSeg.shape == refImgData.shape):
    raise Exception('Reference and its segmentation have different shapes')

  srcSegImg = nib.load(srcSegName)
  srcSeg = srcSegImg.get_data().squeeze()
  if not (srcSeg.shape == srcImgData.shape):
    raise Exception('Source image and its segmentation have different shapes')

  srcPts = [srcMin, srcMax]
  refPts = [refMin, refMax]

  # low and high percentiles
  vs = numpy.percentile(srcImgData, [1,99])
  srcPts = srcPts + list(vs)
  vs = numpy.percentile(refImgData, [1,99])
  refPts = refPts + list(vs)

  # CSF
  vs = srcImgData[ numpy.logical_or(srcSeg == csfLabel, srcSeg == ventLabel)]
  srcPts.append(numpy.mean(vs))
  vs = refImgData[numpy.logical_or(refSeg == csfLabel, refSeg == ventLabel)]
  refPts.append(numpy.mean(vs))

  # GM
  vs = srcImgData[srcSeg == gmLabel]
  srcPts.append(numpy.mean(vs))
  vs = refImgData[refSeg == gmLabel]
  refPts.append(numpy.mean(vs))

  # WM
  vs = srcImgData[srcSeg == wmLabel]
  srcPts.append(numpy.mean(vs))
  vs = refImgData[refSeg == wmLabel]
  refPts.append(numpy.mean(vs))

  ix = numpy.argsort(srcPts)
  srcPts = numpy.asarray(srcPts)[ix]
  refPts = numpy.asarray(refPts)[ix]

  x = numpy.ravel(srcImgData)

  y = numpy.interp(x, srcPts, refPts)

  y = numpy.reshape(y, srcImgData.shape)

  aff = srcImg.get_affine()
  outImg = nib.Nifti1Image(y, aff)
  nib.save(outImg, outImgName)

###############################################################

if __name__ == '__main__':
  sys.exit(main(*sys.argv))


#!/usr/bin/env python

from itertools import takewhile
import logging
import os
from random import randint
import re
import sys
from urllib.parse import urlencode
from urllib.request import urlopen

from bs4 import BeautifulSoup
import yaml

import typos

# http://www.foia.gov/foiareport.js
# Excludes "ALL" (All agencies, though it's not, really).
# Excludes " ", which in `agenciesAb` is "SIGIR", the Special Inspector
# General for Iraq Reconstruction, which no longer accepts FOIA requests.
AGENCIES = [
    'USDA', 'DOC', 'DoD', 'ED', 'DOE', 'HHS', 'DHS', 'HUD', 'DOI', 'DOJ',
    'U.S. DOL', 'State', 'DOT', 'Treasury', 'VA', 'ACUS', 'USAID', 'ABMC',
    'NRPC', 'AFRH', 'BBG', 'CIA', 'CSB', 'USCCR', 'CPPBSD', 'CFTC', 'CFPB',
    'U.S. CPSC', 'CNCS', 'CIGIE', 'CSOSA', 'DNFSB', 'EPA', 'EEOC', 'CEQ',
    'OMB', 'ONDCP', 'OSTP', 'USTR', 'Ex-Im Bank', 'FCA', 'FCSIC', 'FCC',
    'FDIC', 'FEC', 'FERC', 'FFIEC', 'FHFA', 'FLRA', 'FMC', 'FMCS', 'FMSHRC',
    'FOMC', 'FRB', 'FRTIB', 'FTC', 'GSA', 'IMLS', 'IAF', 'LSC', 'MSPB', 'MCC',
    'NASA', 'NARA', 'NCPC', 'NCUA', 'NEA', 'NEH', 'NIGC', 'NLRB', 'NMB',
    'NSF', 'NTSB', 'USNRC', 'OSHRC', 'OGE', 'ONHIR', 'OPM', 'OSC', 'ODNI',
    'OPIC', 'PC', 'PCLOB', 'PBGC', 'PRC', 'RATB', 'US RRB', 'SEC', 'SSS',
    'SBA', 'SSA', 'SIGAR', 'STB', 'TVA', 'US ADF', 'CO', 'USIBWC', 'USITC',
    'USPS', 'USTDA']

PHONE_RE = re.compile(
    r"""(?P<prefix>\+?[\d\s\(\)\-]*)"""
    r"""(?P<area_code>\(?\d{3}\)?[\s\-\(\)]*)"""
    r"""(?P<first_three>\d{3}[\-\s\(\)]*)"""
    r"""(?P<last_four>\d{4}[\-\s]*)"""
    r"""(?P<extension>[\s\(,]*?ext[ .]*?\d{3,5})?"""
    r"""(?P<tty>\s*\(tty)?""", re.IGNORECASE)

ADDY_RE = re.compile(
    r"""(?P<city>.*)"""
    r"""[\s]*?,[\s]*?(?P<state>[A-Z\.]{2,4})"""
    r"""\s*(?P<zip>[0-9-]+)""")

def agency_description(doc):
    """Account for BRs and such while finding the description."""
    description = ""
    after_h2 = doc("h2")[-1].next_elements
    next(after_h2)      # skip the text *within* the h2
    while_text = takewhile(lambda el: el.name is None or el.name == 'br',
                           after_h2)    # stop when we a different type
    for el in while_text:
        if el.name is None:
            description += el.string.strip()
        # Only want one new line when there are two BRs
        elif el.name == "br" and not description.endswith("\n"):
            description += "\n"
    return description.strip()

def clean_paragraphs(doc):
    """Find all paragraphs with content. Return paragraph els + content
    strings. Beautiful Soup doesn't handle unclosed tags very graciously, so
    account for paragraphs within paragraphs."""
    lines, ps = [], []
    for p in doc("p"):
        text = ""
        for child in p.contents:
            if child.name != 'p' and child.string and child.string.strip():
                text += child.string.strip()
        if text:
            lines.append(text)
            ps.append(p)
    return lines, ps

def clean_phone_number(line):
    """
    Given "(123) 456-7890 (Telephone)", extract the number and format
    """
    match = PHONE_RE.search(line)
    if match:
        # kill all non-numbers
        prefix = "".join(ch for ch in match.group("prefix") if ch.isdigit())
        area_code = "".join(ch for ch in match.group("area_code")
                            if ch.isdigit())
        first_three = "".join(ch for ch in match.group("first_three")
                              if ch.isdigit())
        last_four = "".join(ch for ch in match.group("last_four")
                            if ch.isdigit())
        number = "-".join([area_code, first_three, last_four])
        if prefix:
            number = "+" + prefix + " " + number
        extension = match.group("extension")
        if extension:
            extension = re.sub("\D", "", extension)
            number = number + " x" + extension
        if match.group("tty"):
            number += " (TTY)"
        return number

    else:
        raise Exception("Error extracting phone number",
                        "phone line: " + line)

def address_list_to_dict(address_list):
    """
    Converts a list containing address elements into a dictionary

    Address List
    ["Martha R. Sell", "FOIA Assistant", "2300 Clarendon Boulevard",
        "Arlington, VA 22201"]

    Address Dict
    {'state': 'VA', 'address_lines': ['Martha R. Sell', 'FOIA Assistant'],
        'city': 'Arlington', 'street': '2300 Clarendon Boulevard',
        'zip': '22201'}

    Only returns an address dict if it contains a valid street, zip,
    state, and city.
    """
    address_dict = {}

    if len(address_list) > 1:
        address_dict['street'] = address_list[-2]
    if len(address_list) > 2:
        address_dict['address_lines'] = address_list[0:-2]

    match = ADDY_RE.match(address_list[-1])
    if match:
        address_dict['zip'] = match.group('zip').strip()
        address_dict['state'] = re.sub("\W", "", match.group('state'))
        address_dict['city'] = match.group('city').strip()

    if re.search('\d', address_dict['street']) \
            and address_dict.get('zip') \
            and address_dict.get('state') \
            and address_dict.get('city'):
        return address_dict

def split_address_from(lines):
    """ Extracts address from lines into a list like the one below:
    Address list
    ["Martha R. Sell", "FOIA Assistant", "2300 Clarendon Boulevard",
        "Arlington, VA 22201"]"""

    address_list, remaining = [], []
    cues = ("phone", "fax", "service center")
    for line in lines:
        if remaining:   # already switched over
            remaining.append(line)
        elif PHONE_RE.search(line) and any(q in line.lower() for q in cues):
            remaining.append(line)
        else:
            # Separate line breaks
            address_list.extend(re.split(r"[\n\r]+", line))
    if not remaining:
        raise Exception("error finding address end", lines)
    else:
        return address_list, remaining

def find_emails(lines, ps):
    """Find email address, then associated mailto"""
    email_re = re.compile(r"\be\-?mail", re.IGNORECASE)
    emails = []
    for idx, line in enumerate(lines):
        if email_re.search(line):
            a = ps[idx].a
            if a:
                emails_str = a["href"].replace("mailto:", "").strip()
                if "http://" not in emails_str:
                    emails.extend(re.split(r";\s*", emails_str))
            else:
                raise Exception("Error extracting email", line, idx,
                                ps[idx].prettify())
    return emails

def extract_numbers(phones):
    """
    Extracts all phone numbers from a line and adds them to a list
    """

    clean_numbers = []
    phones = re.sub('[,\s(]+ext', ' ext', phones)
    phones = phones.split(",")
    for phone in phones:
        phone = phone.strip()
        if PHONE_RE.match(phone):
            clean_numbers.append(clean_phone_number(phone))
    return clean_numbers

def organize_contact(contact_info):
    """
    Organize contact info into a dictionary to facilitate extraction
    """
    contact_info = contact_info.split("Phone: ")
    name_str = contact_info[0].strip(" ,'")
    phone_str = contact_info[-1]
    clean_numbers = extract_numbers(phone_str)

    cleaned_contact_info = {}
    if name_str:
        cleaned_contact_info['name'] = name_str
    if clean_numbers:
        cleaned_contact_info['phone'] = clean_numbers

    if cleaned_contact_info:
        return cleaned_contact_info

def find_bold_fields(ps):
    """Remaining fields: website, request form, anything else"""
    simple_search = ["service center", "public liaison", "notes",
                     "foia officer"]
    link_search = ["website", "request form"]
    for p in ps:
        strong = p.strong
        text = strong.string.replace(":", "").strip() if strong else ""
        lower = text.lower()
        if strong and text != "FOIA Contact":
            try:
                value = strong.next_sibling.string
            except:
                value = p.next_sibling.string
            if value:
                value = value.strip()
            yielded = False
            for term in simple_search:
                if term in lower:
                    yield term.replace(" ", "_"), value
                    yielded = True

            for term in link_search:
                if term in lower:
                    if p.a:
                        yield term.replace(" ", "_"), p.a["href"].strip()
                        yielded = True
                    else:
                        raise Exception("error extracting " + term,
                                        p.prettify())
            if not yielded:
                # FTC: FOIA Hotline
                # GSA: Program Manager
                yield "misc", (text, value)

def parse_department(elem, name):
    """Get data from a 'div' (elem) associated with a single department"""
    data = {"name": name, "top_level": False}
    lines, ps = clean_paragraphs(elem)
    # remove first el (which introduces the section)
    lines, ps = lines[1:], ps[1:]

    address_list, lines = split_address_from(lines)
    address_dict = address_list_to_dict(address_list=address_list)
    if address_dict:
        data['address'] = address_dict
    else:
        logging.warning("Valid address not found for %s", name)

    ps = ps[-len(lines):]   # Also throw away associated paragraphs
    for line in lines:
        lower = line.lower()
        if ('phone' in lower and 'public liaison' not in lower
                and 'service center' not in lower and 'phone' not in data):
            data['phone'] = clean_phone_number(line)
        elif 'fax' in lower and 'fax' not in data and PHONE_RE.match(line):
            data['fax'] = clean_phone_number(line)
    emails = find_emails(lines, ps)
    if emails:
        data['emails'] = emails
    for key, value in find_bold_fields(ps):
        if key == 'misc':
            misc_key, misc_value = value
            if 'misc' not in data:
                data['misc'] = {}
            misc_value = organize_contact(misc_value)
            if misc_value:
                data['misc'][misc_key] = misc_value
        else:
            if key in ['service_center', 'public_liaison', 'foia_officer']:
                value = organize_contact(value)

            if value:
                data[key] = value
    return data

def parse_agency(abb, doc):
    """Make sense of a block of HTML from FOIA.gov"""
    agency_name = doc.h1.text.strip()
    description = agency_description(doc)

    # get each dept id and name, parse department from its div. Skip the first
    # as it is always a 'please select'
    departments = []
    for option in doc("option")[1:]:
        opt_id = option['value']
        elem = doc(id=opt_id)[0]
        # Needed to replace the ? with - in order to
        # accomate Carlsbad Field Office
        dept_name = option.string.strip().replace('?', '–')
        departments.append(parse_department(elem, dept_name))

    agency = {"abbreviation": abb,
              "name": agency_name,
              "description": description,
              "departments": departments}
    return agency

def fix_known_typos(text):
    """Account for faulty data"""
    for error, fix in typos.REPLACEMENTS.items():
        text = text.replace(error, fix)
    return text

def agency_yaml_filename(data_directory, agency_abbr):
    return os.path.join(data_directory, '%s.yaml' % agency_abbr)

def read_manual_data(agency_abbr, manual_data_dir='manual_data'):
    if os.path.isdir(manual_data_dir):
        filename = agency_yaml_filename(manual_data_dir, agency_abbr)
        if os.path.exists(filename):
            with open(filename, 'r') as yaml_file:
                manual_data = yaml.load(yaml_file)
            return manual_data

def update_list_in_dict(data, field, new_values_list):
    original_values = set(data.get(field, []))
    data[field] = sorted(list(original_values | set(new_values_list)))

def update_non_departments(agency_data, manual_data):
    """ Apply all the non-department changes from manual_data to agency_data.
    """

    agency_data = dict(agency_data)

    list_fields = ['common_requests', 'keywords']

    for field in manual_data.keys():
        if field not in list_fields + ['departments']:
            agency_data[field] = manual_data[field]

    for field in list_fields:
        if field in manual_data and field not in ['departments']:
            update_list_in_dict(agency_data, field, manual_data[field])
    return agency_data

def actual_apply(agency_data, manual_data):
    """ Actually apply the changes in manual_data to agency_data. This handles
    the departments. """

    agency_data = update_non_departments(agency_data, manual_data)
    manual_depts = {}
    if 'departments' in manual_data:
        for dept in manual_data['departments']:
            manual_depts[dept['name']] = dept

        departments = []
        if 'departments' in agency_data:
            for dept in agency_data['departments']:
                if dept['name'] in manual_depts:
                    new_department = update_non_departments(
                        dept, manual_depts[dept['name']])
                else:
                    new_department = dict(dept)
                departments.append(new_department)
            agency_data['departments'] = departments
    return agency_data

def apply_manual_data(agency_abbr, agency_data):
    """ In the manual data directory, we have all the manual over-rides for
    various contact fields. Apply those here. """
    manual_data = read_manual_data(agency_abbr)
    if manual_data:
        return actual_apply(agency_data, manual_data)
    else:
        return agency_data

def get_unknown_office_details(agency_data):
    for dept in agency_data['departments']:
        if dept['name'] == "I don't know which office":
            return dict(dept)

def all_but_unknown(agency_data):
    departments = [d for d in agency_data['departments']
                   if d['name'] != "I don't know which office"]
    return departments

def populate_parent(agency_data):
    unknown_office = get_unknown_office_details(agency_data)
    if unknown_office:
        departments = all_but_unknown(agency_data)
        agency_data = dict(agency_data, departments=departments)
        for field, value in unknown_office.items():
            if field not in ['name']:
                agency_data[field] = value
    return agency_data

def save_agency(abb):
    """For a given agency, download (if not already present) their HTML,
    process it, and save the resulting YAML"""
    os.makedirs('html', exist_ok=True)
    html_path = "html" + os.sep + "%s.html" % abb
    if not os.path.isfile(html_path):
        body = ""
        body = download_agency(abb)
        if body:
            with open(html_path, 'w') as f:
                f.write(body)
            logging.info("[%s] Downloaded.", abb)
        else:
            logging.warning("[%s] DID NOT DOWNLOAD, NO.", abb)
            return
    else:
        logging.info("[%s] Already downloaded.", abb)

    with open(html_path, 'r') as f:
        text = f.read()
    text = fix_known_typos(text)
    data = parse_agency(abb, BeautifulSoup(text))
    data = populate_parent(data)
    data = apply_manual_data(abb, data)
    save_agency_data(abb, data)

def save_agency_data(agency_abbr, data, data_directory='data'):
    """ Actually do the save. """
    os.makedirs(data_directory, exist_ok=True)

    if data:
        with open(data_directory + os.sep + "%s.yaml" % agency_abbr, 'w') as f:
            f.write(yaml.dump(data, default_flow_style=False,
                    allow_unicode=True))
            logging.info("[%s] Parsed.", agency_abbr)
    else:
        logging.warning("[%s] DID NOT PARSE, NO.", agency_abbr)

def save_agencies():
    """Save all agencies"""
    for agency in AGENCIES:
        save_agency(agency)

def agency_url(abb):
    """Construct download url, add cache busting -- the site does this too"""
    params = {"agency": abb, "Random": randint(1, 1000)}
    return "http://www.foia.gov/foia/FoiaMakeRequest?" + urlencode(params)

def download_agency(abb):
    """Agency HTML files"""
    url = agency_url(abb)
    return urlopen(url).read().decode("utf-8")

if __name__ == "__main__":
    """
        python scraper.py <<agency_abbreviation>>
        will only scrape and save the data for the provided agency.

        python scraper.py will scrape and save data for all the agencies.
    """
    logging.basicConfig(level=logging.INFO)

    agency_abbr = None
    if len(sys.argv) > 1:
        agency_abbr = sys.argv[1]

    if agency_abbr:
        save_agency(agency_abbr)
    else:
        save_agencies()


#===============================================================================
# Copyright 2012 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#===============================================================================

#============= enthought library imports =======================
from traits.api import HasTraits, Instance
from traitsui.api import View, Group
#============= standard library imports ========================
import numpy as np
import math
#============= local library imports  ==========================

from src.helpers.traitsui_shortcuts import instance_item
from src.graph.graph import Graph
from src.graph.error_bar_overlay import ErrorBarOverlay
from chaco.array_data_source import ArrayDataSource
from matplotlib.figure import Figure
from src.graph.mpl_editor import MPLFigureEditor
from matplotlib.cm import coolwarm

class FluxView(HasTraits):
    flux_2D = Instance(Graph)
    flux_3D = Instance(Figure, ())
    def replot(self, monitors, unknowns, regressor2D, regressor3D):

        self.replot2D(monitors, unknowns, regressor2D)
        self.replot3D(monitors, unknowns, regressor3D)

    def replot3D(self, monitors, unknowns, regressor):
        from mpl_toolkits.mplot3d import Axes3D

        fig = self.flux_3D
        ax = fig.gca(projection='3d')

        mx, my, mys, mye = zip(*[(pi.x, pi.y, pi.pred_j, pi.pred_j_err)
                                  for pi in monitors])
        x, y, ys, ye = zip(*[(pi.x, pi.y, pi.pred_j, pi.pred_j_err)
                             for pi in unknowns])

        x = np.array(x)
        y = np.array(y)

        ys = np.array(ys)
        mys = np.array(mys)

        w = max(x) - min(x)
        w = w * 1.25
        c = len(x)
        r = len(y)
        n = r * 2 - 1

        xx, yy = np.mgrid[0:n + 1, 0:n + 1]
        xx = xx / float(n) * w - w / 2.
        yy = yy / float(n) * w - w / 2.

        z = np.zeros((r * 2, c * 2))

        for i in range(r * 2):
            for j in range(c * 2):
                pt = (xx[i, j],
                      yy[i, j])
                v, _ = regressor.predict([pt])[0]
                z[i, j] = v

        ax.plot_surface(xx, yy, z, alpha=0.3,
                        rstride=1, cstride=1,
                        cmap=coolwarm,
                        linewidths=0
                        )

        ax.set_xlabel('X (mm)')
        ax.set_ylabel('Y (mm)')
        ax.set_zlabel('Z (%change)')
        self.flux_3D.canvas.draw()

    def replot2D(self, monitors, unknowns, regressor):
        g = self.flux_2D
        g.clear()
        g.new_plot()
        self._plot(g, monitors, use_predicted_value=False)

        self._plot(g, unknowns)
        if regressor:
            low, high = g.get_x_limits()
            fx = np.linspace(low, high, 200)
            fy = regressor.predict(fx)
            g.new_series(fx, fy, color='black')

            ly, uy = regressor.calculate_ci(fx)
            g.new_series(fx, ly, color='red', line_style='dash')
            g.new_series(fx, uy, color='red', line_style='dash')

    def _plot(self, graph, positions, use_predicted_value=True):
        xs = []
        ys = []
        es = []
        for pi in positions:
            if pi.pred_j:
#                #convert pi.x,pi.y to radians
                r = math.atan2(pi.x, pi.y)
                xs.append(r)
                if use_predicted_value:
                    ys.append(pi.pred_j)
                    es.append(pi.pred_j_err)
                else:
                    ys.append(pi.j)
                    es.append(pi.j_err)

        s, _ = graph.new_series(xs, ys, yerror=ArrayDataSource(data=es),
                                type='scatter', marker='circle', marker_size=2)
        ebo = ErrorBarOverlay(component=s, orientation='y')
        s.overlays.append(ebo)

        s.index_range.tight_bounds = False
        s.value_range.tight_bounds = False

    def traits_view(self):
        v = View(Group(
                       Group(instance_item('flux_2D'),
                             label='2D'
                             ),
                       Group(instance_item('flux_3D', editor=MPLFigureEditor()),
                             label='3D'
                             ),
                       layout='tabbed'
                       )

                 )
        return v

    def _flux_2D_default(self):
#        g = RegressionGraph(container_dict=dict(padding=5))
        g = Graph(container_dict=dict(padding=5))
        return g
#============= EOF =============================================


#!/usr/bin/env python

# encoding: utf-8

import csv

import time

import Levenshtein

threshold_similarity_co_name = 0.7

#threshold_similarity_people_name = 0.6 # 0.55 the threshold on the distance of two strings, if lower then the two strings are identical

def name_match(match_file1, match_file2, output_file_name, colume_name):

        # read file 'input_file_name' to obtain information about the user, such as fullname, twitter account, etc.

        match_dataset1 = [] # information read from match_dataset1.csv

        match_dataset2 = [] # information read from match_dataset2.csv

        first_row = True

        with open(match_file1, 'r') as inputFile :

                reader = csv.reader(inputFile)

                for row in reader:

                        if first_row:

                                first_row = False

                                continue

                        match_dataset1.append(row)

        first_row = True

        with open(match_file2, 'r') as inputFile :

                reader = csv.reader(inputFile)

                for row in reader:

                        if first_row:

                                first_row = False

                                continue

                        match_dataset2.append(row)

        output = []

        for row in match_dataset1:

                #row = match_dataset1[row_index, :]

                asx_code = row[0]

                company_name = row[1]

                print (row)

                matched = None

                for row2 in match_dataset2:

                        controllering_corproation = row2[0]

                        similarity_ratio = Levenshtein.ratio(company_name, controllering_corproation)

                        if similarity_ratio > threshold_similarity_co_name:

                                matched = True

                                print ("matched")

                                row.append(controllering_corproation)

                                output.append(row)

                                break

                        pass

                if not matched:

                        row.append("-")

                        output.append(row)

        with open(output_file_name, 'w', newline='') as outputFile:

                writer = csv.writer(outputFile)

                write_row_string = ["ASX code", "Controlled entities"]

                write_row_string.extend(colume_name)

                print (write_row_string)
                writer.writerow(write_row_string)

                writer.writerows(output)

if __name__ == '__main__':

        output_file_name = "./DataMatched0.csv"

        input1 = "./DataMatched.csv"

        input2 = "./Data_2014_2015.csv"

        colume_names = ["Match with 2014_2015"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched1.csv"

        input1 = "./DataMatched0.csv"

        input2 = "./Data_2013_2014.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched2.csv"

        input1 = "./DataMatched1.csv"

        input2 = "./Data_2012_2013.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014", "Match with 2012_2013"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched3.csv"

        input1 = "./DataMatched2.csv"

        input2 = "./Data_2011_2012.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014", "Match with 2012_2013", "Match with 2011_2012"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched4.csv"

        input1 = "./DataMatched3.csv"

        input2 = "./Data_2010_2011.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014", "Match with 2012_2013", "Match with 2011_2012", "Match with 2010_2011"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched5.csv"

        input1 = "./DataMatched4.csv"

        input2 = "./Data_2009_2010.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014", "Match with 2012_2013", "Match with 2011_2012", "Match with 2010_2011", "Match with 2009_2010"]

        name_match(input1, input2, output_file_name, colume_names)

        output_file_name = "./DataMatched6.csv"

        input1 = "./DataMatched5.csv"

        input2 = "./Data_2008_2009.csv"

        colume_names = ["Match with 2014_2015", "Match with 2013_2014", "Match with 2012_2013", "Match with 2011_2012", "Match with 2010_2011", "Match with 2009_2010", "Match with 2008_2009"]

        name_match(input1, input2, output_file_name, colume_names)

        print (threshold_similarity_co_name)


'''
Copyright (C) 2015 Pistiwique, Pitiwazou

Created by Pistiwique, Pitiwazou

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

import bpy, subprocess
from os import listdir
from os.path import join, isfile
from ..function_utils.get_path import (get_addon_path,
                                       get_directory,
                                       get_library_path,
                                       )

def check_existing_IBL_Tool(world):
    """ Check if the AM_IBL_Worl setup exist """

    if world in bpy.data.worlds:
        return True
    return False

#------------------------------------------------------------------
#
#------------------------------------------------------------------

def update_projection(sef, context):
    AM = bpy.context.window_manager.asset_m
    IBL_world = bpy.data.worlds["AM_IBL_WORLD"]
    nodes = IBL_world.node_tree.nodes
    nodes['Environment'].projection = AM.projection
    nodes['Reflexion'].projection = AM.projection

#------------------------------------------------------------------
#
#------------------------------------------------------------------

def setup_node_editor():
    """ Setup the node editor """

    bpy.context.area.type = 'NODE_EDITOR'
    bpy.context.scene.render.engine = 'CYCLES'
    bpy.context.space_data.tree_type = 'ShaderNodeTree'
    bpy.context.space_data.shader_type = 'WORLD'

#------------------------------------------------------------------
#
#------------------------------------------------------------------

def get_ibl_from_thumbnail(ibl, dir):
    """ Get the ibl full path from the selected preview """

    extentions = (".hdr", ".exr", ".jpg", ".jpeg", ".png", ".tif")
    ibl_name = [f for f in listdir(dir) if f.endswith(extentions) and f.rsplit(".", 1)[0] == ibl.rsplit(".", 1)[0]]

    return ibl_name[0]

#------------------------------------------------------------------
#
#------------------------------------------------------------------

def generate_thumbnail_ibl(thumbnail_directory):
    """ Generate the thumbnails of the IBL """

    AM = bpy.context.window_manager.asset_m
    addon_path = get_addon_path()
    IBL_dir = get_directory("IBL")
    thumb_list = ";".join(AM.ibl_to_thumb)
    script_thumbnailer = join(addon_path, "background_tools", 'generate_ibl_thumbnail.py')
    blend_thumbnailer = join(addon_path, 'blend_tools', 'asset', 'thumbnailer.blend')
    subprocess.Popen([bpy.app.binary_path, blend_thumbnailer, '-b', '--python', script_thumbnailer, IBL_dir, thumb_list, thumbnail_directory, AM.thumb_ext])

#------------------------------------------------------------------
#
#------------------------------------------------------------------

def import_ibl_from_asset_management():
    """ Import and setup ibl """

    WM = bpy.context.window_manager
    AM = WM.asset_m
    addon_path = get_addon_path()
    library_path = get_library_path()
    ibl_node_group = "AM_IBL_Tool"
    blur_node_group = "ImageBlur"
    directory_path = get_directory("IBL")

    setup_node_editor()

    if not check_existing_IBL_Tool("AM_IBL_WORLD"):
        blendfile = join(addon_path, "blend_tools", "IBL", "IBL_Tool.blend")
        directory = join(blendfile, "NodeTree")

        IBL_world = bpy.data.worlds.new("AM_IBL_WORLD")
        IBL_world.use_nodes = True
        bpy.context.scene.world = IBL_world
        nodes = IBL_world.node_tree.nodes

        nodes.remove(nodes["Background"])

        with bpy.data.libraries.load(blendfile) as (data_from, data_to):
            if data_from.node_groups:
                bpy.ops.wm.append(filename=ibl_node_group, directory=directory)
                bpy.ops.wm.append(filename=blur_node_group, directory=directory)

        tex_coord_node = nodes.new("ShaderNodeTexCoord")
        tex_coord_node.location = -810, 260

        mapping_node = nodes.new("ShaderNodeMapping")
        mapping_node.location = -560, 280

        blur_node = nodes.new("ShaderNodeGroup")
        blur_node.location = -140, 200
        blur_node.name = blur_node_group
        blur_node.node_tree = bpy.data.node_groups[blur_node_group]

        env_node = nodes.new("ShaderNodeTexEnvironment")
        env_node.location = 100, 360
        env_node.name = "Environment"

        reflexion_node = nodes.new("ShaderNodeTexEnvironment")
        reflexion_node.location = 100, 120
        reflexion_node.name = "Reflexion"

        ibl_name = get_ibl_from_thumbnail(WM.AssetM_previews, directory_path)
        img = bpy.data.images.load(join(directory_path, ibl_name))
        env_node.image = img
        reflexion_node.image = img

        ibl_node = nodes.new("ShaderNodeGroup")
        ibl_node.location = 320, 300
        ibl_node.name = ibl_node_group
        ibl_node.node_tree = bpy.data.node_groups[ibl_node_group]

        output_node = nodes["World Output"]
        output_node.location = 550, 300

        tree = IBL_world.node_tree
        tree.links.new(tex_coord_node.outputs[3], mapping_node.inputs[0])
        tree.links.new(mapping_node.outputs[0], blur_node.inputs[0])
        tree.links.new(mapping_node.outputs[0], env_node.inputs[0])
        tree.links.new(blur_node.outputs[0], reflexion_node.inputs[0])
        tree.links.new(env_node.outputs[0], ibl_node.inputs[1])
        tree.links.new(reflexion_node.outputs[0], ibl_node.inputs[6])
        tree.links.new(ibl_node.outputs[0], output_node.inputs[0])

    else:
        IBL_world = bpy.data.worlds["AM_IBL_WORLD"]
        IBL_world.use_nodes = True
        bpy.context.scene.world = IBL_world
        nodes = IBL_world.node_tree.nodes
        ibl_name = get_ibl_from_thumbnail(WM.AssetM_previews, directory_path)

        if not "Reflexion" in IBL_world.node_tree.nodes:
            reflexion_node = nodes.new("ShaderNodeTexEnvironment")
            reflexion_node.location = 100, 120
            reflexion_node.name = "Reflexion"

            tree = IBL_world.node_tree
            tree.links.new(nodes['Mapping'].outputs[0], nodes['Environment'].inputs[0])
            tree.links.new(nodes['ImageBlur'].outputs[0], reflexion_node.inputs[0])
            tree.links.new(reflexion_node.outputs[0], nodes['AM_IBL_Tool'].inputs[6])

        if ibl_name in bpy.data.images:
            nodes["Environment"].image = bpy.data.images[ibl_name]
            nodes["Reflexion"].image = bpy.data.images[ibl_name]
        else:
            img = bpy.data.images.load(join(directory_path, ibl_name))
            nodes["Environment"].image = img
            nodes["Reflexion"].image = img

    if isfile(join(directory_path, WM.AssetM_previews.rsplit(".", 1)[0] + "_settings")):
        bpy.ops.wm.load_ibl_settings()

    else:
        nodes['Mapping'].rotation = (0.0, 0.0, 0.0)
        AM.projection = 'EQUIRECTANGULAR'
        nodes['ImageBlur'].inputs[1].default_value = 0.0
        IBL_world.cycles_visibility.camera = True
        bpy.context.scene.cycles.film_transparent = False
        nodes['AM_IBL_Tool'].inputs[0].default_value = 1.0
        nodes['AM_IBL_Tool'].inputs[2].default_value = 1.0
        nodes['AM_IBL_Tool'].inputs[3].default_value = 1.0
        nodes['AM_IBL_Tool'].inputs[4].default_value = (0.5, 0.5, 0.5, 1.0)
        nodes['AM_IBL_Tool'].inputs[5].default_value = 0.5
        nodes['AM_IBL_Tool'].inputs[7].default_value = 1.0
        nodes['AM_IBL_Tool'].inputs[8].default_value = 1.0
        nodes['AM_IBL_Tool'].inputs[9].default_value = (0.5, 0.5, 0.5, 1.0)
        nodes['AM_IBL_Tool'].inputs[10].default_value = 0.5

    bpy.context.area.type = 'VIEW_3D'
    cycles = bpy.context.scene.cycles
    if cycles.sample_clamp_direct == 0 and cycles.sample_clamp_indirect == 0:
        bpy.context.scene.cycles.sample_clamp_direct = 3
        bpy.context.scene.cycles.sample_clamp_indirect = 3


import sublime, json
from urllib.request import urlopen
from collections import deque

def load_completions(plugin_path):
	completions_data = {}
	for filename in ["lucee_tags","lucee_functions","lucee_member_functions","cfml_tags","cfml_functions","cfml_member_functions"]:
		completions_data[filename] = load_json_data(plugin_path, filename)

	# setup completion lists
	completions = {}
	for dialect in ["lucee","cfml"]:
		# tags
		completions[dialect + "_tags"] = []
		completions[dialect + "_tags_in_script"] = []
		completions[dialect + "_tag_attributes"] = {}
		for tag_name in sorted(completions_data[dialect + "_tags"].keys()):
			tag_attributes = completions_data[dialect + "_tags"][tag_name]
			completions[dialect + "_tags"].append(make_tag_completion(tag_name, dialect, tag_attributes[0]))
			completions[dialect + "_tags_in_script"].append(make_tag_completion(tag_name[(1 if dialect == "lucee" else 2):], dialect, tag_attributes[0]))
			completions[dialect + "_tag_attributes"][tag_name] = [(a + '\trequired', a + '="$1"') for a in tag_attributes[0]]
			completions[dialect + "_tag_attributes"][tag_name].extend([(a + '\toptional', a + '="$1"') for a in tag_attributes[1]])

		# functions
		completions[dialect + "_functions"] = [(funct + '\tfn (' + dialect + ')', funct + completions_data[dialect + "_functions"][funct]) for funct in sorted(completions_data[dialect + "_functions"].keys())]

		# member functions
		mem_func_comp = []
		for member_function_type in sorted(completions_data[dialect + "_member_functions"].keys()):
			for funct in sorted(completions_data[dialect + "_member_functions"][member_function_type].keys()):
				mem_func_comp.append( (funct + '\t' + member_function_type + '.fn (' + dialect + ')', funct + completions_data[dialect + "_member_functions"][member_function_type][funct]))
		completions[dialect + "_member_functions"] = mem_func_comp

	return completions

def load_json_data(plugin_path, filename):
	with open(plugin_path + '/json/' + filename + '.json', 'r') as f:
		json_data = f.read()
	return json.loads(json_data)

def make_tag_completion(tag, type, required_attrs):
	attrs = ''
	for index, attr in enumerate(required_attrs, 1):
		attrs += ' ' + attr + '="$' + str(index) + '"'
	return (tag + '\ttag (' + type + ')', tag + attrs)

def get_previous_word(view, pos):
	previous_word_start = view.find_by_class(pos, False, sublime.CLASS_WORD_START)
	previous_word = view.substr(sublime.Region(previous_word_start, pos)).strip().lower()
	return previous_word

def get_tag_name(view, pos):
	# walk backwards from cursor, looking for tag name scope
	for index in range(500):
		if view.match_selector(pos - index,"entity.name.tag"):
			tag_name_region = view.expand_by_class(pos - index, sublime.CLASS_WORD_START | sublime.CLASS_WORD_END, "</>;{}()")
			tag_name = view.substr(tag_name_region).lower()
			return tag_name
		if view.match_selector(pos - index,"punctuation.definition.tag.begin"):
			return None

def get_last_open_tag(view, pos):
	open_tags = deque()
	tag_starts = [r for r in view.find_by_selector("punctuation.definition.tag.begin") if r.end() <= pos]
	tag_ends = [r for r in view.find_by_selector("punctuation.definition.tag.end") if r.end() <= pos]

	# if lengths don't match don't bother trying to find last open tag
	if len(tag_starts) != len(tag_ends):
		return None

	for tag_start, tag_end in zip(tag_starts, tag_ends):
		tag_name_region = sublime.Region(tag_start.end(), view.find_by_class(tag_start.end(), True, sublime.CLASS_WORD_END, "/>"))
		tag_name = view.substr(tag_name_region)

		# if length is 1 then this is a tag opening punctuation
		if tag_start.size() == 1:

			if tag_end.size() > 1:
				# self closing tag has tag end of size 2 "/>" - skip these
				continue

			# check to exclude cfml tags that should not have a closing tag
			if tag_name in ["cfset","cfelse","cfelseif",":set",":else",":elseif"]:
				continue
			# check to exclude html tags that should not have a closing tag
			if tag_name in ["area","base","br","col","command","embed","hr","img","input","link","meta","param","source"]:
				continue

			open_tags.appendleft(tag_name)

		# if length is 2 then this is a tag closing punctuation
		if tag_start.size() == 2 and tag_name in open_tags:
			open_tags.remove(tag_name)

	return open_tags.popleft() if len(open_tags) > 0 else None

def get_support_function_name(view, pt):
	args_region = None
	function_call_arguments_scope = "meta.support.function-call.arguments.cfml"
	scope_count = view.scope_name(pt).count(function_call_arguments_scope)
	scope_to_find = " ".join([function_call_arguments_scope] * scope_count)
	for r in view.find_by_selector(scope_to_find):
		if r.contains(pt):
			args_region = r
			break
	if args_region:
		return view.substr(view.word(args_region.begin()-1)).lower()
	return None


from django.views import View
from articles.models import Article
from comments.models import Comment
from django.http import JsonResponse
from utils.decorators import fail_safe_api
from utils.models import nested_model_to_dict
from utils.request import parse_body, set_user
from utils.koora import deleteImageFor, generate_url_for, uploadImageFor, setTagsFor

class DetailAPIView(View):

    def dispatch(self, request, *args, **kwargs):
        set_user(request)
        if request.user.is_authenticated:
            parse_body(request, for_method=request.method)
        return super(DetailAPIView, self).dispatch(request, *args, **kwargs)

    @fail_safe_api(for_model=Article)
    def get(self, request, slug):

        article = Article.objects.get(slug=slug)

        content = {
            "status": 200,
            "data": {
                "article": nested_model_to_dict(article)
            },
            "meta": {
                "article_count": 1
            }
        }

        return JsonResponse(content)

    @fail_safe_api(for_model=Article, needs_authentication=True)
    def put(self, request, slug):

        article = Article.objects.get(slug=slug)
        article.title = request.PUT.get('title', '')
        article.content = request.PUT.get('content', '')
        article.category = request.PUT.get('category', '')

        image = request.FILES.get('article_image', False)

        post_type = request.PUT.get('post_type', 'public')
        article.is_private = post_type == 'private'

        if image:
            deleteImageFor(article)
            uploadImageFor(article, image, article.slug)

        article.remove_tags()

        tags = request.PUT.get('tags', '').strip().split(",")

        setTagsFor(article, tags)

        article.save()

        return JsonResponse({
          "status" : 200,
          "data" : {
              "article" : nested_model_to_dict(article)
          },
          "message" : "article updated"
        })

    @fail_safe_api(for_model=Article, needs_authentication=True)
    def post(self, request, slug):
        article = Article.objects.get(slug=slug)

        user = request.user

        content = request.POST.get('content', '')
        object_id = request.POST.get('object_id', 1)
        content_type = article.content_type

        created_comment = Comment.objects.create(
            user=user, content=content, object_id=object_id, content_type=content_type)

        return JsonResponse({
            "status" : 200,
            "data" : nested_model_to_dict(created_comment),
            "message" : "comment added",
            "meta" : {
                "article_count" : 1,
                "comment_count" : 1
            }
        })

    @fail_safe_api(for_model=Article, needs_authentication=True)
    def delete(self, request, slug):

        article = Article.objects.get(slug=slug)

        if article.image_url:
            deleteImageFor(article)
        article.delete()

        return JsonResponse({
            "status" : 200,
            "message" : 'article deleted'
        })


from setuptools import setup

long_description = """
Implementations of classic data structures in Python.
"""

setup(
    name="data-structures",
    version="0.1-dev",
    description="Classic Data Structures",
    long_description=long_description,
    # The project URL.
    url='http://github.com/geekofalltrades/data-structures',
    # Author details
    author='William Dougherty',
    author_email='thegeekofalltrades@gmail.com',
    # Choose your license
    #   and remember to include the license text in a 'docs' directory.
    # license='MIT',
    packages=['data_structures'],
    install_requires=['setuptools', ]
)


# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ==============================================================================
################################# AUTHOR: ######################################
############################# DIEGO PERGOLINI ##################################
# ==============================================================================
"""Script to train the DNC on a sentiment classification task."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import random
import json
import tensorflow as tf
from Utility import getConfiguration
from Utility import  memory_usage
import dnc
import argparse
import math
import time

parser = argparse.ArgumentParser()
parser.add_argument("--configuration", help="The configuration file")
args = parser.parse_args()
if args.configuration:
    print(args.configuration)
    FLAGS,numLines = getConfiguration(args.configuration)
else:
    FLAGS, numLines = getConfiguration("configuration.json")

if FLAGS.num_classes == 5:
    from ReviewReaderWithTitleFined import balancedGetDataset
    from ReviewReaderWithTitleFined import getBalancedReviewsAndOverall
elif FLAGS.num_classes == 2:
    from ReviewReaderTitle import balancedGetDataset
    from ReviewReaderTitle import getBalancedReviewsAndOverall

def run_model(input_sequence, output_size):
  """Runs model on input sequence."""

  access_config = {
      "memory_size": FLAGS.memory_size,
      "word_size": FLAGS.word_size,
      "num_reads": FLAGS.num_read_heads,
      "num_writes": FLAGS.num_write_heads,
  }
  controller_config = {
      "hidden_size": FLAGS.hidden_size,
  }
  clip_value = FLAGS.clip_value
  #Creo la cella dnc
  dnc_core = dnc.DNC(access_config, controller_config, output_size, clip_value)
  initial_state = dnc_core.initial_state(FLAGS.batch_size)

  #Funzione che ritorna una coppia (output,state), dove output in questo caso sara un tensore
  #di forma  [batch_size,max_time,cell.output_size] perche il flag time_major e impostato a False
  #se lo si setta a True invece la forma dell'output diventa [max_time,batch_size,cell.output_size].
  output_sequence, _ = tf.nn.dynamic_rnn(
      cell=dnc_core,
      inputs=input_sequence,
      time_major=False,
      initial_state=initial_state)

  return output_sequence

"""Addestra la DNC ed al termine di ogni epoca di addestramento viene effettuata una fase di test"""
def execute(dictSize,num_training_iterations, report_interval):
  max_lenght = FLAGS.max_lenght
    #Placeholder che andra' a contenere il batch di label relativa alle recensioni
  y_= tf.placeholder(tf.float32,shape=[FLAGS.batch_size,max_lenght,FLAGS.num_classes])

  #Placeholder che andra' a contenere il batch di recensioni opportunamente codificate
  x = tf.placeholder(tf.float32, [FLAGS.batch_size, max_lenght, dictSize])

  #Placeholder che andra' a contenere il batch di maschere da applicare per il calcolo della cross-entropy
  mask = tf.placeholder(tf.float32,shape=[FLAGS.batch_size,max_lenght])

  #Richiamando il metodo run_model ottengo la sequenza prodotta dalla rete
  output_logits = run_model(x, FLAGS.num_classes)

  #Calcolo della cross entropy totale tra le labels e gli output prodotti dalla rete
  cross = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=output_logits)

  #Calcolo l'errore relativo ai singoli batch, applicando una maschera che considera gli output prodotti dalla rete
  #solo negli unrolling corrispondenti a delle parole della recensione e non considerando quelli prodotti in corrispondenza
  #di padding. La maschera applicata fornisce peso via via crescente mano a mano che si procede verso le ultime parole
  #della recensione.
  batch_error = tf.reduce_sum(cross * mask, 1)

  #Faccio la media degli errori dei singoli batch
  total_error = tf.reduce_mean(batch_error)

  #Ricavo la polarita' che la rete ha indicato all'ultima parola di ogni recensione,
  prediction = tf.arg_max(output_logits[:,max_lenght-1], 1)

  #Ricavo la polarita' indicata dalle label
  expected = tf.arg_max(y_[:,max_lenght-1], 1)

  #Ricavo quante predizioni della polarita' sono state fatte correttamente
  correct_prediction = tf.equal(prediction, expected)

  #Ricavo cosi' l'accuratezza ottenuta in questo batch
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

  #Set up optimizer with global norm clipping.
  trainable_variables = tf.trainable_variables()
  grads, _ = tf.clip_by_global_norm(
      tf.gradients(total_error, trainable_variables), FLAGS.max_grad_norm)

  global_step = tf.get_variable(
      name="global_step",
      shape=[],
      dtype=tf.int64,
      initializer=tf.random_uniform_initializer(-1, 1),
      trainable=False,
      collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP])

  #Placeholder che conterra' il learning rate
  learning_rate = tf.placeholder(tf.float32)

  optimizer = tf.train.RMSPropOptimizer(
      learning_rate, epsilon=FLAGS.optimizer_epsilon)

  #Passo di addestramento da eseguire per addestrare la rete
  train_step = optimizer.apply_gradients(
      zip(grads, trainable_variables), global_step=global_step)

  #Oggetto per salvare lo stato della rete.
  saver = tf.train.Saver()

  #Impostazione dei parametri relativi al salvataggio dello stato della rete.
  if FLAGS.checkpoint_interval > 0:
    hooks = [
        tf.train.CheckpointSaverHook(
            checkpoint_dir=FLAGS.checkpoint_dir,
            save_steps=FLAGS.checkpoint_interval,
            saver=saver)
    ]
  else:
    hooks = []

  #Viene scritto un file di log dell'esecuzione corrente.
  date = time.strftime("%b%d%H:%M:%S")
  outputFile = open("Experiment"+date+".txt",'w')

  with open(args.configuration) as data_file:
    configuration = json.load(data_file)
    outputFile.write(json.dumps(configuration))

  config = tf.ConfigProto()
  config.gpu_options.per_process_gpu_memory_fraction = 0.4
  #Viene inizializzata una sessione
  with tf.train.SingularMonitoredSession(
      hooks=hooks, checkpoint_dir=FLAGS.checkpoint_dir,config=config) as sess:

        #Il numero di istanze di training  e testing viene diviso per due perche' indica che il numero di recensioni
        # positive e negative siano uguali

        if FLAGS.num_classes ==5:
            numTraining = (FLAGS.num_training_iterations)
            numTesting = FLAGS.num_testing_iterations
            allLines = getBalancedReviewsAndOverall(numTraining,numTesting,FLAGS.dataset, FLAGS.max_lenght, FLAGS.random, FLAGS.seed)
        elif FLAGS.num_classes ==2:
            numTraining = (FLAGS.num_training_iterations // 2)
            numTesting = (FLAGS.num_testing_iterations // 2)
            #Vengono ottenute tutte le recensioni necessarie per training e testing
            allLines = getBalancedReviewsAndOverall(numTraining,numTraining,
                                                numTesting,numTesting,
                                                FLAGS.dataset,FLAGS.max_lenght,FLAGS.random,FLAGS.seed)

        #Viene inizializzato un generatore attraverso il quale ottenere mano a mano i vari batch di training e testing,
        # viene inoltre indicato quale modello word2vec debba essere utilizzato
        generator = balancedGetDataset(allLines,FLAGS.w2v_model,FLAGS)

        randomizer = random.Random(1)

        best_test_accuracy = 0

        date = time.strftime("%H:%M:%S")

#################TRAINING########################################################
        # Testing
        test_accuracy = 0
        total_test_accuracy = 0

        #################################################TESTING########################################################
        for test_iteration in range(0,(FLAGS.num_testing_iterations//FLAGS.batch_size)):

            datasetTest = next(generator)
            act_accuracy = sess.run(accuracy,
                                    {x: datasetTest[0],
                                     y_: datasetTest[1],
                                     mask: datasetTest[2]})
            test_accuracy += act_accuracy
            total_test_accuracy += act_accuracy
            if (test_iteration + 1) % report_interval == 0:

                tf.logging.info("%d: Avg testing accuracy %f.\n",
                                test_iteration+1, test_accuracy / report_interval
                                )
                info4 = str(test_iteration+1)+ ": Avg testing accuracy: "+str(test_accuracy / report_interval)+"\n"
                outputFile.write(info4)
                test_accuracy = 0
        tf.logging.info("Epoch: %d,Iteration: %d, Average Testing accuracy: %f\n",1, test_iteration+1,
                        total_test_accuracy /( FLAGS.num_testing_iterations//FLAGS.batch_size))
        info5 = "Epoch: "+str(1)+",Iteration: "+ str(test_iteration+1)+", Average Testing accuracy: "+str(total_test_accuracy /( FLAGS.num_testing_iterations//FLAGS.batch_size))+"\n"
        outputFile.write(info5)

        epoch_test_accuracy = total_test_accuracy /( FLAGS.num_testing_iterations//FLAGS.batch_size)
        if epoch_test_accuracy > best_test_accuracy:
            best_test_accuracy = epoch_test_accuracy

  date = time.strftime("%H:%M:%S")
  outputFile.close()

def main(unused_argv):
  tf.logging.set_verbosity(3)  # Print INFO log messages.

  execute(FLAGS.word_dimension,FLAGS.num_training_iterations, FLAGS.report_interval)

if __name__ == "__main__":
  tf.app.run()


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 27 15:02:19 2018

@author: SCUTYJ
"""

import numpy as np
import pandas as pd
import pywt
import matplotlib.pyplot as plt
import os
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

#传入data必须为一维数组
def wpt1D(data,wavelet='db1',mode='symmetric',maxlevel=2):

    #一维小波包分解
    wp = pywt.WaveletPacket(data=data, wavelet=wavelet,mode=mode,maxlevel = maxlevel)

    #列出所有树节点路径
    paths = [node.path for node in wp.get_level(maxlevel)]

    #合并所有四级子节点
    num = int(data.shape[0]/(2**maxlevel))
    df_wpt = pd.DataFrame(index=range(num))
    for a in paths:
        df_wpt[a] = wp[a].data

    return df_wpt

# df_yes = pd.read_table(r'D:\Jupyter Notebook\Research_Ultrasound\质量检测\yes_no\yes.zwav',
#                        engine='python',sep='\t').set_index('TIME(us)')
# df_no = pd.read_table(r'D:\Jupyter Notebook\Research_Ultrasound\质量检测\yes_no\no.zwav',
#                     engine='python',sep='\t').set_index('TIME(us)')
# df = pd.read_pickle('ut_example.pkl')
# df.columns = ['DATA']

df1 = pd.read_csv('WPT/HDPE_WOOD_0.csv',engine='python',header=None)
df2 = pd.read_csv('WPT/HDPE_WOOD_1.csv',engine='python',header=None)
df3 = pd.read_csv('WPT/HDPE_WOOD_2.csv',engine='python',header=None)
df1.index = df2.index = df3.index = np.linspace(0,163.8,4096)

def plot_wpt(df1, df2, df3, maxlevel, title):

    df_wpt1 = wpt1D(df1[70:90.44][0],'db1','symmetric',maxlevel)
    df_wpt2 = wpt1D(df2[70:90.44][0],'db1','symmetric',maxlevel)
    df_wpt3 = wpt1D(df3[70:90.44][0],'db1','symmetric',maxlevel)

    if maxlevel == 1:
        plt.figure()
        for i in  range(1,2**maxlevel+1):
            plt.subplot(2,1,i)
            plt.plot(df_wpt1.index, df_wpt1.iloc[:,i-1])
            plt.plot(df_wpt2.index, df_wpt2.iloc[:,i-1])
            plt.plot(df_wpt3.index, df_wpt3.iloc[:,i-1])            #plt.ylim((-35,35))
            plt.title('{}'.format(df_wpt1.columns[i-1]))
        plt.subplots_adjust(left=None, bottom=None, right=None, top=None,
        wspace=0.2, hspace=0.4)
        plt.suptitle(title)
        plt.show()

    if maxlevel == 2:
        plt.figure()
        for i in  range(1,2**maxlevel+1):
            plt.subplot(2,2,i)
            plt.plot(df_wpt1.index, df_wpt1.iloc[:,i-1])
            plt.plot(df_wpt2.index, df_wpt2.iloc[:,i-1])
            plt.plot(df_wpt3.index, df_wpt3.iloc[:,i-1])            #plt.ylim((-35,35))
            plt.title('{}'.format(df_wpt1.columns[i-1]))
        plt.subplots_adjust(left=None, bottom=None, right=None, top=None,
        wspace=0.2, hspace=0.4)
        plt.suptitle(title)
        plt.show()

    if maxlevel == 3:
        plt.figure()
        for i in  range(1,2**maxlevel+1):
            plt.subplot(4,2,i)
            plt.plot(df_wpt1.index, df_wpt1.iloc[:,i-1])
            plt.plot(df_wpt2.index, df_wpt2.iloc[:,i-1])
            plt.plot(df_wpt3.index, df_wpt3.iloc[:,i-1])            #plt.ylim((-30,30))
            plt.title('{}'.format(df_wpt1.columns[i-1]))
        plt.subplots_adjust(left=None, bottom=None, right=None, top=None,
        wspace=0.4, hspace=0.6)
        plt.suptitle(title)
        plt.show()

    if maxlevel == 4:
        plt.figure()
        for i in  range(1,2**maxlevel+1):
            plt.subplot(4,4,i)
            plt.plot(df_wpt1.index, df_wpt1.iloc[:,i-1])
            plt.plot(df_wpt2.index, df_wpt2.iloc[:,i-1])
            plt.plot(df_wpt3.index, df_wpt3.iloc[:,i-1])
            #plt.ylim((-35,35))
            plt.title('{}'.format(df_wpt1.columns[i-1]))
        plt.subplots_adjust(left=None, bottom=None, right=None, top=None,
        wspace=0.6, hspace=0.8)
        plt.suptitle(title)
        plt.show()
    return df_wpt1,df_wpt2,df_wpt3

HDPE_WOOD = pd.concat([df1,df2,df3],axis=1)[70:90.44]
HDPE_WOOD_2 = pd.concat([df1,df2,df3],axis=1)[73:76]
HDPE_WOOD.columns = [0,1,2]
HDPE_WOOD.plot()
HDPE_WOOD_2.columns = [0,1,2]
HDPE_WOOD_2.plot()
plt.title('原始信号')
df_wpt1,df_wpt2,df_wpt3 = plot_wpt(df1,df2,df3,4, ' ')
pd.concat(([df_wpt1['adad'],df_wpt2['adad'],df_wpt3['adad']]),axis=1).plot()
plt.legend([0,1,2])
plt.title('7.8125-9.375MHz频段')
plt.show()
# =============================================================================
# #导入示例数据测试(DF结构，索引为Time，列为幅值)
# df = pd.read_pickle('ut_example.pkl')
# data1 = df[70:90.44]['example'].values
# data2 = df[140:160.44]['example'].values
#
# #截取两批512个点，合并成32X32二维数组
# ut_wpt = pd.concat([wpt1D(data1),wpt1D(data2)],axis=1).values
# =============================================================================


import numpy as np, time, h5py, copy, argparse, os, mpi4py.MPI, sys, pipes, shutil
from enlib import enmap, utils, pmat, fft, config, array_ops, map_equation, nmat, errors
from enlib import log, bench, scan
from enlib.cg import CG
from enact import data, nmat_measure, filedb, todinfo

config.default("filedb", "filedb.txt", "File describing the location of the TOD and their metadata")
config.default("todinfo", "todinfo.txt", "File describing location of the TOD id lists")
config.default("map_bits", 32, "Bit-depth to use for maps and TOD")
config.default("downsample", 1, "Factor with which to downsample the TOD")
config.default("map_precon", "bin", "Preconditioner to use for map-making")
config.default("map_cg_nmax", 1000, "Max number of CG steps to perform in map-making")
config.default("verbosity", 1, "Verbosity for output. Higher means more verbose. 0 outputs only errors etc. 1 outputs INFO-level and 2 outputs DEBUG-level messages.")
config.default("task_dist", "size", "How to assign scans to each mpi task. Can be 'plain' for myid:n:nproc-type assignment, 'size' for equal-total-size assignment. The optimal would be 'time', for equal total time for each, but that's not implemented currently.")

parser = config.ArgumentParser(os.environ["HOME"] + "/.enkirc")
parser.add_argument("filelist")
parser.add_argument("area")
parser.add_argument("odir")
parser.add_argument("prefix",nargs="?")
parser.add_argument("-d", "--dump", type=int, default=10)
parser.add_argument("--ncomp",      type=int, default=3)
parser.add_argument("--ndet",       type=int, default=0)
args = parser.parse_args()

precon= config.get("map_precon")
dtype = np.float32 if config.get("map_bits") == 32 else np.float64
comm  = mpi4py.MPI.COMM_WORLD
myid  = comm.rank
nproc = comm.size
nmax  = config.get("map_cg_nmax")

db       = filedb.ACTFiles(config.get("filedb"))
filelist = todinfo.get_tods(args.filelist, config.get("todinfo"))

area = enmap.read_map(args.area)
area = enmap.zeros((args.ncomp,)+area.shape[-2:], area.wcs, dtype)
utils.mkdir(args.odir)
root = args.odir + "/" + (args.prefix + "_" if args.prefix else "")

# Dump our settings
if myid == 0:
	config.save(root + "config.txt")
	with open(root + "args.txt","w") as f:
		f.write(" ".join([pipes.quote(a) for a in sys.argv[1:]]) + "\n")
	with open(root + "env.txt","w") as f:
		for k,v in os.environ.items():
			f.write("%s: %s\n" %(k,v))
	with open(root + "ids.txt","w") as f:
		for id in filelist:
			f.write("%s\n" % id)
	shutil.copyfile(config.get("filedb"), root + "filedb.txt")
# Set up logging
utils.mkdir(root + "log")
logfile   = root + "log/log%03d.txt" % myid
log_level = log.verbosity2level(config.get("verbosity"))
L = log.init(level=log_level, file=logfile, rank=myid)
# And benchmarking
utils.mkdir(root + "bench")
benchfile = root + "bench/bench%03d.txt" % myid

# Read in all our scans
L.info("Reading scans")
tmpinds    = np.arange(len(filelist))[myid::nproc]
myscans, myinds  = [], []
for ind in tmpinds:
	try:
		d = scan.read_scan(filelist[ind])
	except IOError:
		try:
			d = data.ACTScan(db[filelist[ind]])
		except errors.DataMissing as e:
			L.debug("Skipped %s (%s)" % (filelist[ind], e.message))
			continue
	d = d[:,::config.get("downsample")]
	if args.ndet > 0: d = d[:args.ndet]
	myscans.append(d)
	myinds.append(ind)
	L.debug("Read %s" % filelist[ind])

nread = comm.allreduce(len(myscans))
L.info("Found %d tods" % nread)
if nread == 0:
	L.info("Giving up")
	sys.exit(1)

if config.get("task_dist") == "size":
	# Try to get about the same amount of data for each mpi task.
	mycosts = [s.nsamp*s.ndet for s in myscans]
	all_costs = comm.allreduce(mycosts)
	all_inds  = comm.allreduce(myinds)
	myinds_old = myinds
	myinds = [all_inds[i] for i in utils.equal_split(all_costs, nproc)[myid]]
	# And reread the correct files this time. Ideally we would
	# transfer this with an mpi all-to-all, but then we would
	# need to serialize and unserialize lots of data, which
	# would require lots of code.
	if sorted(myinds_old) != sorted(myinds):
		L.info("Rereading shuffled scans")
		myscans = []
		for ind in myinds:
			d = data.ACTScan(db[filelist[ind]])[:,::config.get("downsample")]
			if args.ndet > 0: d = d[:args.ndet]
			myscans.append(d)
			L.debug("Read %s" % filelist[ind])
	else:
		myinds = myinds_old

L.info("Building equation system")
eq  = map_equation.LinearSystemMap(myscans, area, precon=precon)
eq.write(root)
bench.stats.write(benchfile)

L.info("Computing approximate map")
x  = eq.M(eq.b)
bmap = eq.dof.unzip(x)[0]
if myid == 0: enmap.write_map(root + "bin.fits", bmap)

def solve_cg(eq, nmax=1000, ofmt=None, dump_interval=10):
	cg = CG(eq.A, eq.b, M=eq.M, dot=eq.dof.dot)
	while cg.i < nmax:
		with bench.mark("cg_step"):
			cg.step()
		dt = bench.stats["cg_step"]["time"].last
		L.info("CG step %5d %15.7e %6.1f %6.3f" % (cg.i, cg.err, dt, dt/len(eq.scans)))
		xmap, xjunk = eq.dof.unzip(cg.x)
		if ofmt and cg.i % dump_interval == 0 and myid == 0:
			enmap.write_map(ofmt % cg.i, eq.dof.unzip(cg.x)[0])
		# Output benchmarking information
		bench.stats.write(benchfile)
	return cg.x

if nmax > 0:
	L.info("Solving equation")
	x = solve_cg(eq, nmax=nmax, ofmt=root + "map%04d.fits", dump_interval=args.dump)


import sys,datetime
import sqlite3
import tsp_ga

tour=sqlite3.connect('stadiumTour.db')
tr=tour.cursor()
dict1={0:"Brighton",1:"Bristol",2:"Cambridge",3:"Glasgow",4:"Liverpool",5:"London",6:"Manchester",7:"Oxford"}
#stadium starts
def displaycities():
      sql="SELECT * from stadium;"
      tr.execute(sql)
      result=tr.fetchall();
      #print(result)
      #for record in result:
            #print (record)
      print("Sno\tStadium Name\tCity\tZipcode\tCapacity")
      for record in result:
            print(record[0],"\t",record[1],"\t",record[2],"\t",record[3],"\t",
                  record[4],"\t")

def searchcities():
      name=input('Enter the name')
      sql="SELECT sname,city from stadium WHERE city='"+name+"';"
      tr.execute(sql)
      result=tr.fetchone()
      if result==None:
            print("no record found")
      else:
            print(result[0],",",result[1],"\n")

def shrtst_path():
      print('shortest path is:')
      testList=tsp_ga.ga()
      cost=testList[0][0]
      s=testList[0][5]
      s=s.replace(',','')
      s=s.replace(' ','')
      s=s.replace('[','')
      s=s.replace(']','')
      l1=[int(y) for y in list(s)]

      for i in range(len(l1)):
            print(dict1[l1[i]],"->",end=" ")
      print('\ncost of the shortest path is: ',cost)

#stadium ends

#staff starts
def displaystaff():
      sql="SELECT * from staff;"
      tr.execute(sql)
      result=tr.fetchall();
      #print(result)
      #for record in result:
            #print (record)
      print("Stid\tStaff name\tJob")
      for record in result:
            print(record[0],"\t",record[1],"\t",record[2])

def searchstaff():
      name=input('Enter the name: ')
      sql="SELECT stid,sname,job from staff WHERE Sname='"+name+"';"
      tr.execute(sql)
      result=tr.fetchone()
      if result==None:
            print("no record found")
      else:
            print(result[0],",",result[1],",",result[2],"\n")

def updatestaff():
      print("""
                  1.Insert
                  2.Delete
                  3.Update
            """)
      c=int(input("enter c hoice"))
      if c==1:
            try:
                  x=int(input('Enter stid:'))
                  y=input('Enter sname')
                  z=input('enter job')
                  tr.execute("insert into staff values(?,?,?)",(x,y,z))
                  tour.commit()
                  print ("One record added successfully.")
            except:
                  print ("Error in operation.")

                  tour.rollback()

      elif c==2:
            try:
                  x=input('Enter the staff name to be deleted:')
                  tr.execute("delete from staff where sname='"+x+"';")
                  print('delete sucessfull')
                  tour.commit()
            except:
                  print('error in Operation')

                  tour.rollback()

      elif c==3:
            try:
                  w=input('Enter the staff name to be update')
                  x=int(input('Enter stid:'))
                  y=input('Enter sname')
                  z=input('enter job')
                  tr.execute('update staff set stid=?,sname=?,job=? where sname=?;',(w,x,y,z))
                  tour.commit()
                  print("record updated sucessfully")
            except:
                  print ("Error in operation.")

                  tour.rollback()

#staff ends

#equpments starts

def displayequipments():
      sql="SELECT * from equipments;"
      tr.execute(sql)
      result=tr.fetchall();
      #print(result)
      #for record in result:
            #print (record)
      print("equipID\tEName\tQty")
      for record in result:
            print(record[0],"\t",record[1],"\t",record[2])

def searchequipments():
      name=input('Enter the equipment: ')
      sql="SELECT equipID,Ename,Qty from Equipments WHERE Ename='"+name+"';"
      tr.execute(sql)
      result=tr.fetchone()
      if result==None:
            print("no record found")
      else:
            #
            print(result[0],",",result[1],",",result[2],"\n")

def updateequipments():
      print("""
                  1.Insert
                  2.Delete
                  3.Update
            """)
      c=int(input("enter c hoice"))
      if c==1:
            try:
                  x=int(input('Enter equipid:'))
                  y=input('Enter ename')
                  z=input('enter qty')
                  tr.execute("insert into equipments values(?,?,?)",(x,y,z))
                  tour.commit()
                  print ("One record added successfully.")
            except:
                  print ("Error in operation.")

                  tour.rollback()

      elif c==2:
            try:
                  x=input('Enter the ename name to be deleted:')
                  tr.execute("delete from equipments where Ename='"+x+"';")
                  print('delete sucessfull')
                  tour.commit()
            except:
                  print('error in Operation')

                  tour.rollback()

      elif c==3:
            try:
                  w=input('Enter the ename name to be update')
                  x=int(input('Enter equipid:'))
                  y=input('Enter ename')
                  z=input('enter qty')
                  tr.execute('update equipments set equipID=?,Ename=?,Qty=? where Ename=?;',(w,x,y,z))
                  tour.commit()
                  print("record updated sucessfully")
            except:
                  print ("Error in operation.")

                  tour.rollback()

#equipment ends

#merchandise starts
def displaymerchandise():
      sql="SELECT * from merchandise;"
      tr.execute(sql)
      result=tr.fetchall();
      #print(result)
      #for record in result:
            #print (record)
      print("Type\tHoodie\tQty\tprice")
      for record in result:
            print(record[0],"\t",record[1],"\t",record[2],"\t",record[3])

def searchmerchandise():
      name=input('Enter the type of merchandise: ')
      sql="SELECT * from merchandise WHERE Type='"+name+"';"
      tr.execute(sql)
      result=tr.fetchone()
      if result==None:
            print("no record found")
      else:
            #
            print(result[0],",",result[1],",",result[2],",",result[3],"\n")

def updatemerchandise():
      print("""
                  1.Insert
                  2.Delete
                  3.Update
            """)
      c=int(input("enter c hoice"))
      if c==1:
            try:
                  x=int(input('Enter type:'))
                  y=input('Enter merch_id')
                  z=input('enter quantity')
                  v=input('enter price')
                  tr.execute("insert into merchandise values(?,?,?)",(x,y,z,v))
                  tour.commit()
                  print ("One record added successfully.")
            except:
                  print ("Error in operation.")

                  tour.rollback()

      elif c==2:
            try:
                  x=input('Enter the type of merch to be deleted:')
                  tr.execute("delete from merchandise where type='"+x+"';")
                  print('delete sucessfull')
                  tour.commit()
            except:
                  print('error in Operation')

                  tour.rollback()

      elif c==3:
            try:
                  w=input('Enter the ename name to be update')
                  x=int(input('Enter type:'))
                  y=input('Enter merch_id')
                  z=input('enter quantity')
                  v=input('enter price')

                  tr.execute('update merchandise set type=?,merch_id=?,quantity=?,price=? where =?;',(w,x,y,z,v))
                  tour.commit()
                  print("record updated sucessfully")
            except:
                  print ("Error in operation.")

                  tour.rollback()

#merchandise ends

def cities():
      done=False
      while done==False:
            print("""==========Touring Cities=========
                  1.Display the cities
                  2.Search a city
                  3.Generate shortest path
                  4.Go to main menu
                  """)
            choice=int(input("Enter Choice:"))
            if choice==1:
                  displaycities()
            elif choice==2:
                  searchcities()
            elif choice==3:
                  shrtst_path()
            elif choice==4:
                  main()

def staff():
      done=False
      while done==False:
            print("""==========Staff module==========
                  1.Display Staff
                  2.Search a Staff
                  3.Update staff table
                  4.Go to main menu
                  """)
            choice=int(input("Enter Choice:"))
            if choice==1:
                  displaystaff()
            elif choice==2:
                  searchstaff()
            elif choice==3:
                  updatestaff()
            elif choice==4:
                  main()

def equipments():
      done=False
      while done==False:
            print("""==========Staff module==========
                  1.Display Equpments
                  2.Search a Equipment
                  3.Update Equipment table
                  4.Go to main menu
                  """)
            choice=int(input("Enter Choice:"))
            if choice==1:
                  displayequipments()
            elif choice==2:
                  searchequipments()
            elif choice==3:
                  updateequipments()
            elif choice==4:
                  main()

def merchandise():
      done=False
      while done==False:
            print("""==========Staff module==========
                  1.Display Merchandise
                  2.Search a Merchandise
                  3.Update Merchandise table
                  4.Go to main menu
                  """)
            choice=int(input("Enter Choice:"))
            if choice==1:
                  displaymerchandise()
            elif choice==2:
                  searchmerchandise()
            elif choice==3:
                  updatemerchandise()
            elif choice==4:
                  main()

def main():
      done=False
      while done==False:
            print(""" ======STADIUM TOUR MANAGEMENT MENU=======
                  1. Touring cities
                  2. Staff
                  3. Equipments
                  4.Merchandise
                  5. Exit
                  """)
            choice=int(input("Enter Choice:"))
            if choice==1:
                        cities()
            elif choice==2:
                        staff()
            elif choice==3:
                        equipments()
            elif choice==4:
                        merchandise()
            elif choice==5:
                  sys.exit()
            else:
                  print('invalid choice')

main()


"""cmdb URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf.urls import url ,include

#引入myapp应用的渲染
from myapp import views
from myapp import api
from myapp import api_task,api_user,api_es,api_ssh,api_env

urlpatterns = [
    #前后端未分离使用的路由
    path('admin/', admin.site.urls),
    #path('login/index/', views.login, name='index'),
    url(r'^login/index/', views.login, name='login'),
    url(r'^login/logout/', views.logout, name='logout'),
    url(r'^login/login/', views.session_check, name='login_check'),

    url(r'^cmdb/index/', views.login_auth, name='login_auth'),
    url(r'^cmdb/index_return/', views.index_return, name='index_return'),
    url(r'^cmdb/data_add/', views.data_add, name='data_add'),
    url(r'^cmdb/data_look/', views.data_look, name='data_look'),
    url(r'^cmdb/data_alter/', views.data_alter, name='data_alter'),
    url(r'^cmdb/data_delete/', views.data_delete, name='data_delete'),
    url(r'^cmdb/add_message/', views.add_message, name='add_message'),
    url(r'^cmdb/data_search/', views.data_search, name='data_search'),

    url(r'^cmdb/user/', views.user_manage, name='user_manage'),
    url(r'^cmdb/user_add/', views.user_add, name='user_add'),
    url(r'^cmdb/user_alter/', views.user_alter, name='user_alter'),
    url(r'^cmdb/user_add_get/', views.user_add_get, name='user_add_get'),
    url(r'^cmdb/user_add_alter/', views.user_add_alter, name='user_add_alter'),
    url(r'^cmdb/user_delete/', views.user_delete, name='user_delete'),
    url(r'^cmdb/user_add_delete/', views.user_add_delete, name='user_add_delete'),
    url(r'^cmdb/test/', views.test, name='test'),

    #API接口测试路由
    url(r'^api/test/$', api.GetTestView.as_view(), name='GetMessageView'),

    #服务器资源相关路由
    url(r'^api/es-select/$',api_es.PostEsSelectView.as_view(), name='PostEsSelectView'),
    url(r'^api/es-search/$',api_es.GetEsSearchView.as_view(), name='GetEsSearchView'),
    url(r'^api/es-add/$',api_es.PostEsAddView.as_view(), name='PostEsAddView'),
    url(r'^api/es-delete/$',api_es.PostEsDeleteView.as_view(), name='PostEsDeleteView'),
    url(r'^api/es-delete-id/$',api_es.PostEsRmidView.as_view(), name='PostEsRmidView'),
    url(r'^api/es-update/$',api_es.PostEsEditView.as_view(), name='PostEsEditView'),
    url(r'^api/es-update-id/$',api_es.PostEsEditidView.as_view(), name='PostEsEditidView'),
    url(r'^api/es-update-auth/$',api_es.PostEdiAuthView.as_view(), name='PostEdiAuthView'),

    #用户操作相关路由
    url(r'^api/login/$',api_user.GetLoginView.as_view(), name='GetLoginViewView'),
    url(r'^api/logout/$',api_user.GetLogoutView.as_view(), name='GetLogoutView'),
    url(r'^api/user-add/$',api_user.PostUserAddView.as_view(), name='PostUserAddView'),
    url(r'^api/user-alter/$',api_user.PostUserAlterView.as_view(), name='PostUserAlterView'),
    url(r'^api/user-alter-passwd/$',api_user.PostUserAlterPasswdView.as_view(), name='PostUserAlterPasswdView'),
    url(r'^api/user-delete/$',api_user.PostUserDeleteView.as_view(), name='PostUserDeleteView'),
    url(r'^api/user-gain/$',api_user.PostUserGainView.as_view(), name='PostUserGainView'),

    #待办事项功能相关路由
    url(r'^api/es-task-add/$', api_task.PostTaskAddView.as_view(), name='PostTaskAddView'),
    url(r'^api/es-task-select/$', api_task.PostTaskSelectView.as_view(), name='PostTaskSelectView'),
    url(r'^api/es-task-delete-id/$', api_task.PostTaskRmidView.as_view(), name='PostTaskRmidView'),

    #ssh相关路由
    url(r'^api/ssh-extract/$', api_ssh.PostSSHView.as_view(), name='PostSSHView'),
    url(r'^api/ssh-detect/$', api_ssh.PostDetectView.as_view(), name='PostDetectView'),

    #批处理相关路由
    url(r'^api/auth-select/$', api_env.PostAutoSelectView.as_view(), name='PostAutoSelectView'),
    url(r'^api/script-select/$', api_env.PostSelectScriptView.as_view(), name='PostSelectScriptView'),
    url(r'^api/script-add/$', api_env.PostAddScriptView.as_view(), name='PostAddScriptView'),
    url(r'^api/script-del/$', api_env.PostDelScriptView.as_view(), name='PostDelScriptView'),
    url(r'^api/execute-script/$', api_env.PostExecuteView.as_view(), name='PostExecuteView'),

]


import matplotlib.pyplot as plt
import numpy as np
import time
from time import strftime
from edempy import BoxBin, Deck
import edempy
import matplotlib; matplotlib.use("TkAgg")

def maintance_time(interval_table: list, L_boxbin, R_boxbin, deck, is_draw: bool, is_save: bool):
    #'ResidenceTime_Example.dem')
    #deck = Deck(deck_name)
    nTimesteps = deck.numTimesteps    # liczba krokow czasowych
    print("liczba krokow czasowych: ", nTimesteps)
    binbox_name = "123"

    # Step 1: Define size of a single timestep and bin x, y, z max/min
    tstep = deck.timestep[1].timestep  # Size of a single timestep

    # Step 2: Create empty dict to count pID residence time
    insideBin = {}  # Use dict for speed {pID:count}

    for i in range(interval_table[0], interval_table[1]):
        print("krok czasowy: ", i)
        # Step 3: Loop ids per timestep, if particle is inside bin add to the count in "insideBin"
        for j in range(3):  # iteracja po wszystkich skalach
            ids = deck.timestep[i].particle[j].getIds()
            pos = deck.timestep[i].particle[j].getPositions()
            p_ids_first_bin = L_boxbin.getBinnedObjects(ids, pos)
            p_ids_second_bin = R_boxbin.getBinnedObjects(ids, pos)

            for p_id in p_ids_first_bin:
                if p_id in insideBin:
                    insideBin[p_id] = insideBin[p_id] + 1  # If already in dict add 1 to the count
                else:
                    insideBin.update({p_id: 1})  # If not in the dict, add it, and add 1 to the count

            for p_id in p_ids_second_bin:
                if p_id in insideBin:
                    insideBin[p_id] = insideBin[p_id] + 1  # If already in dict add 1 to the count
                else:
                    insideBin.update({p_id: 1})  # If not in the dict, add it, and add 1 to the count

    # Step 5: Plot data as a histogram
    residenceTime = np.array(list(insideBin.values())) * tstep
    plt.hist(residenceTime, 11)
    plt.xlabel('Residence Time [s]')
    plt.ylabel('Frequency')
    plt.title(f'Residence Time Histogram where particle in {binbox_name} BinBox')
    if is_save:
        plt.savefig(f"Residence_Time_{binbox_name}_BinBox_{strftime('%m_%d_%Y-%H_%M_%S')}.png")
    if is_draw:
        plt.show()

def main():
    """
    Main function

    parametry wejsciowe:
    interval_table  = [poczatkowy krok, koncowy krok czasowy],
    list_section    = [20, 30, 40, 50, 60, 80] - od 5 do 10 elementow,
    filepath        = sciezka
    L_boxbin            = wymiary boxa np. BoxBin([0, -0.8, -0.75], 3, 0.25, 1.5)
    R_boxbin            = -||-
    id_draw         = czy rysowac (True / False)
    is_save         = czy zapisac (True / False)
    """

    interval_table = [1, 260]
    filepath = "C:\\Users\\Jakub\\PycharmProjects\\test2\\testownik11_prof_Robert_Krol\\projekt_2\\POLKOWICE_etap_2\\simulation_0\\simulation_0.dem"
    L_boxbin = BoxBin([0, -0.8, -0.75], 3, 0.25, 1.5)
    R_boxbin = BoxBin([0, 0.8, -0.75], 3, 0.25, 1.5)
    is_draw = True
    is_save = False

    deck = Deck(filepath)

    maintance_time(interval_table=interval_table, L_boxbin=L_boxbin, R_boxbin=R_boxbin,
                   deck=deck, is_draw=is_draw, is_save=is_save)

if __name__ == '__main__':
    import sys

    sys.exit(main())


#!/usr/bin/env python

# pip install tensorflow
# pip install keras

import cv2   as cv
import numpy as np
from threading import Thread
import time
from umucv.util import putText
from umucv.stream import autoStream

import keras
import keras.backend as K
from keras.applications.inception_v3 import InceptionV3, preprocess_input, decode_predictions
from keras.preprocessing import image

model = InceptionV3(weights='imagenet')

def classify(img):
    h,w,_ = img.shape
    dw = (w-299)//2
    dh = (h-299)//2
    win = img[dh:299+dh,dw:299+dw]
    arr = preprocess_input(np.expand_dims(win.astype(np.float32), axis=0))
    preds = model.predict(arr)
    _,lab,p = decode_predictions(preds, top=3)[0][0]
    if p < 0.5: lab = ''
    return win, lab

# la primera llamada debe hacerse fuera del hilo
classify(np.zeros((480,640,3), np.uint8))

frame = None
goon = True
win  = None

def work():
    global win,lab
    while goon:
        if frame is not None:
            t0 = time.time()
            win,lab = classify(frame)
            t1 = time.time()
            putText(win, '{:.0f}ms  {}'.format(1000*(t1-t0), lab))

t = Thread(target=work,args=())
t.start()

for key, frame in autoStream():
    cv.imshow('cam',frame)
    if win is not None:
            cv.imshow('inception', win)
            win = None

goon = False


#Author: Sikder Tahsin Al Amin
#Project: incr-gamma
#Description: Compute incremental SGDregressor of a data set.
#Run: Python3 filename.py dataset=file.py

import numpy as np
import pandas as pd
from sklearn.linear_model import SGDRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn import linear_model
import sys
import time

sys.argv = ["incrementalPCA.py","dataset=C:/My Computer/study/R/gammaPartition/test1.csv"]

if len(sys.argv) != 2:
    print("Not correct arguments. call python3 filename.py dataset=file.csv");
    sys.exit()

arg2=sys.argv[1]
arg2=arg2.split('=')
input_dataset=arg2[1] ##dataset name
print(input_dataset)

start = time.time() #time starts

"""
train = pd.read_csv(input_dataset)
X = train.iloc[:,0:-1].to_numpy() #extract X variables and convert it to numpy array
y = train.iloc[:,-1].to_numpy() #extract y and convert and convert it to numpy array

model = make_pipeline(StandardScaler(),SGDRegressor(max_iter=1000, tol=1e-3))
model.fit(X,y)
print("coefficients: ", model.steps[1][1].coef_)
print("intercepts: ",model.steps[1][1].intercept_)
#"""
###with linearregression##
"""
data = pd.read_csv(input_dataset)
X = data.iloc[:,0:-1].to_numpy() #extract X variables and convert it to numpy array
y = data.iloc[:,-1].to_numpy() #extract y and convert and convert it to numpy array
lr = linear_model.LinearRegression()
mlr=lr.fit(X,y)
"""
#####
#trains = pd.read_csv(input_dataset, chunksize=20)
#model = SGDRegressor() #incremental SGDregressor from sklearn
chunkSize = 20

for train in pd.read_csv(input_dataset, chunksize=chunkSize):
    model = make_pipeline(StandardScaler(),SGDRegressor(max_iter=1000, tol=1e-3))
    X = model.named_steps['standardscaler'].fit_transform(train.iloc[:,0:-1])
    y = train.iloc[:,-1].to_numpy()
    model.named_steps['sgdregressor'].partial_fit(X,y)

end = time.time()

#print("LR")
#print("coefficients: ", mlr.coef_)
#print("intercepts: ",mlr.intercept_)
###
print("SGD")
print("coefficients: ", model.steps[1][1].coef_)
print("intercepts: ",model.steps[1][1].intercept_)
#print("coefficients: ", model.coef_)
#print("intercepts: ",model.intercept_)
print("Total time: ",end-start)


import dataset
import framework
from .. import args

@args
def a(parser: framework.helpers.ArgumentParser):
    parser.add_argument("-ctl.reversed", default=True)
    parser.add_argument("-ctl.max_depth", default=5)
    parser.add_argument("-ctl.n_more_depth", default=5)
    parser.add_argument("-ctl.n_more_depth_valid", default=3)
    parser.add_argument("-ctl.detailed_output", default=True)

class CTLDataMixin:
    VALID_NUM_WORKERS = 0

    def get_ctl_loader(self, split: str) -> dataset.CompositionalTableLookup:
        return dataset.CompositionalTableLookup(split, self.helper.args.ctl.max_depth, 11,
                                                reversed=self.helper.args.ctl.reversed,
                                                n_more_depth=self.helper.args.ctl.n_more_depth,
                                                atomic_input=True, copy_input=True, max_n_sampes=1000000,
                                                n_more_depth_valid=self.helper.args.ctl.n_more_depth_valid,
                                                detailed_output=self.helper.args.ctl.detailed_output)

    def create_datasets(self):
        self.batch_dim = 1
        self.train_set = self.get_ctl_loader("train")
        self.valid_sets.valid = self.get_ctl_loader("valid")
        self.valid_sets.test = self.get_ctl_loader("test")

class CTLClassifierDataMixin(CTLDataMixin):
    def get_ctl_loader(self, split: str) -> dataset.CompositionalTableLookup:
        return dataset.CompositionalTableLookupClassification(split, self.helper.args.ctl.max_depth, 11,
                                                              reversed=self.helper.args.ctl.reversed,
                                                              n_more_depth=self.helper.args.ctl.n_more_depth,
                                                              n_more_depth_valid=self.helper.args.ctl.n_more_depth_valid,
                                                              atomic_input=True, max_n_sampes=1000000)


import base64
from urllib.request import Request, urlopen

from odoo import models, fields, api, _
from odoo.exceptions import UserError
from ...utils.magento.product import Client

class MagentoProductProduct(models.Model):
    _name = 'magento.product.product'
    # current ignore this because parent products can have same child products
    # _inherit = 'magento.binding'
    backend_id = fields.Many2one(
        comodel_name='magento.backend',
        string='Magento Backend',
        required=True,
        ondelete='restrict',
    )
    external_id = fields.Integer(string='ID on Magento')

    _inherits = {'product.product': 'odoo_id'}
    _description = 'Magento Product'

    odoo_id = fields.Many2one('product.product',
                              string='Product',
                              required=True,
                              ondelete="cascade")

    website_ids = fields.Many2many(comodel_name='magento.website',
                                   string='Websites',
                                   readonly=True)
    created_at = fields.Date('Created At (on Magento)')
    updated_at = fields.Date('Updated At (on Magento)')

    @api.model
    def import_image(self):
        for rec in self:
            backend_id = rec.backend_id
            url = backend_id.web_url
            access_token = backend_id.access_token
            sku = rec.default_code
            client = Client(url, access_token)
            if str(sku) != '':
                gallery = client.call('rest/V1/products/' + str(sku) + '/media', '')
                if len(gallery) > 0:
                    file = gallery[0]['file']
                    link = url + '/media/catalog/product' + file
                    try:
                        if link:
                            req = Request(link, headers={'User-Agent': 'Mozilla/5.0'})
                            profile_image = base64.encodebytes(urlopen(req).read())
                            val = {
                                'image_medium': profile_image,
                            }
                            rec.image_small = profile_image
                    except:
                        raise UserError(_('Please provide correct URL or check your image size.!'))


# Importing essential libraries and modules

from flask import Flask, render_template, request, Markup
import numpy as np
import pandas as pd
from utils.disease import disease_dic
import requests
import io
import torch
from torchvision import transforms
from PIL import Image
from utils.model import ResNet9

# ==============================================================================================

# -------------------------LOADING THE TRAINED MODELS -----------------------------------------------

# Loading plant disease classification model

disease_classes = ['Apple___Apple_scab',
                   'Apple___Black_rot',
                   'Apple___Cedar_apple_rust',
                   'Apple___healthy',
                   'Blueberry___healthy',
                   'Cherry_(including_sour)___Powdery_mildew',
                   'Cherry_(including_sour)___healthy',
                   'Corn_(maize)___Cercospora_leaf_spot Gray_leaf_spot',
                   'Corn_(maize)___Common_rust_',
                   'Corn_(maize)___Northern_Leaf_Blight',
                   'Corn_(maize)___healthy',
                   'Grape___Black_rot',
                   'Grape___Esca_(Black_Measles)',
                   'Grape___Leaf_blight_(Isariopsis_Leaf_Spot)',
                   'Grape___healthy',
                   'Orange___Haunglongbing_(Citrus_greening)',
                   'Peach___Bacterial_spot',
                   'Peach___healthy',
                   'Pepper,_bell___Bacterial_spot',
                   'Pepper,_bell___healthy',
                   'Potato___Early_blight',
                   'Potato___Late_blight',
                   'Potato___healthy',
                   'Raspberry___healthy',
                   'Soybean___healthy',
                   'Squash___Powdery_mildew',
                   'Strawberry___Leaf_scorch',
                   'Strawberry___healthy',
                   'Tomato___Bacterial_spot',
                   'Tomato___Early_blight',
                   'Tomato___Late_blight',
                   'Tomato___Leaf_Mold',
                   'Tomato___Septoria_leaf_spot',
                   'Tomato___Spider_mites Two-spotted_spider_mite',
                   'Tomato___Target_Spot',
                   'Tomato___Tomato_Yellow_Leaf_Curl_Virus',
                   'Tomato___Tomato_mosaic_virus',
                   'Tomato___healthy']

disease_model_path = 'models/plant_disease_model.pth'
disease_model = ResNet9(3, len(disease_classes))
disease_model.load_state_dict(torch.load(
    disease_model_path, map_location=torch.device('cpu')))
disease_model.eval()

# Loading crop recommendation model

# =========================================================================================

# Custom functions for calculations

def predict_image(img, model=disease_model):
    """
    Transforms image to tensor and predicts disease label
    :params: image
    :return: prediction (string)
    """
    transform = transforms.Compose([
        transforms.Resize(256),
        transforms.ToTensor(),
    ])
    image = Image.open(io.BytesIO(img))
    img_t = transform(image)
    img_u = torch.unsqueeze(img_t, 0)

    # Get predictions from model
    yb = model(img_u)
    # Pick index with highest probability
    _, preds = torch.max(yb, dim=1)
    prediction = disease_classes[preds[0].item()]
    # Retrieve the class label
    return prediction

# ===============================================================================================
# ------------------------------------ FLASK APP -------------------------------------------------

app = Flask(__name__)

# render home page

@ app.route('/')
def home():
    title = 'Harvestify - Home'
    #return render_template('index1.html', title=title)
    return render_template('index1_1_1.html', title=title)

# render crop recommendation form page

@ app.route('/crop-recommend')
def crop_recommend():
    title = 'Harvestify - Crop Recommendation'
    return render_template('./templates/crop.html', title=title)

# render fertilizer recommendation form page

@ app.route('/fertilizer')
def fertilizer_recommendation():
    title = 'Harvestify - Fertilizer Suggestion'

    return render_template('mytemplate.html', title=title)

# render disease prediction input page

# ===============================================================================================

# RENDER PREDICTION PAGES

# render crop recommendation result page

# render fertilizer recommendation result page

@app.route('/disease-predict', methods=['GET', 'POST'])
def disease_prediction():
    title = 'Harvestify - Disease Detection'

    if request.method == 'POST':
        if 'file' not in request.files:
            return redirect(request.url)
        file = request.files.get('file')
        if not file:
            #return render_template('index.html', title=title)
            return "image not received"
        try:
            img = file.read()

            prediction = predict_image(img)
            print(prediction)
            prediction = Markup(str(disease_dic[prediction]))
            return render_template('page-1_1.html', prediction=prediction, title=title)
            #return render_template('index.html', prediction=prediction, title=title)
            #return prediction
        except:
            pass
    return render_template('mytemplate.html', title=title)

# ===============================================================================================
if __name__ == '__main__':
    app.run(debug=True, host='localhost',port=8000)