# -*- coding: utf-8 -*-
import scrapy
import json
import re
from locations.items import GeojsonPointItem

class AldiUSSpider(scrapy.Spider):
    name = 'aldi_us'
    item_attributes = { 'brand': "Aldi", 'brand_wikidata': "Q125054" }
    allowed_domains = ['www.yellowmap.de']
    start_urls = (
        'https://www.yellowmap.de/Presentation/AldiSued/en-US/ResultList?LocX=&LocY=&Lux=-115.224609375&Luy=58.17070248348609&Rlx=-62.666015625&Rly=17.14079039331665&ZoomLevel=4',
    )

    def parse(self, response):
        data = json.loads(response.body_as_unicode())
        container = data['Container']
        stores = self.parse_data(container)

        for store in stores:
            json_data = json.loads(store.css('.resultItem::attr(data-json)').extract_first())
            ref = json_data['id']
            lat = json_data['locY']
            lon = json_data['locX']
            name = store.css('.resultItem-CompanyName::text').extract_first()
            street = store.css('.resultItem-Street::text').extract_first()
            address1 = store.css('.resultItem-City::text').extract_first().split(',')
            hours_data = store.css('.openingHoursTable > tr')

            pattern = r'(.*?)(\s|,\s)([0-9]{1,5})'
            if len(address1) == 2:
                city = address1[0]
                match = re.search(pattern, address1[1].strip())
                state = match.groups()[0]
                zipcode = match.groups()[2]

            elif len(address1) == 1:
                match = re.search(pattern, address1[0].strip())
                city, state, zipcode = match.groups()

            properties = {
                'ref': ref,
                'name': name,
                'opening_hours': self.hours(hours_data),
                'lat': lat,
                'lon': lon,
                'street': street,
                'city': city,
                'state': state,
                'postcode': zipcode
            }

            yield GeojsonPointItem(**properties)

        if 'zoom' in response.meta.keys():
            zoom = response.meta["zoom"]
        else:
            zoom = 5

        for child in self.children(container):
            yield scrapy.Request(
                'https://www.yellowmap.de/Presentation/AldiSued/en-US/ResultList?LocX={}&LocY={}&ZoomLevel={}'.format(child['locX'], child['locY'], zoom),
                meta = {'zoom' : zoom + 1},
                callback = self.parse
            )

    def children(self, data):
        json_data = re.search(r'Tiles:\s+(\[{.*}\])', data)
        if json_data:
            return json.loads(json_data.groups()[0])
        else:
            return []

    def parse_data(self, data):
        data = scrapy.http.HtmlResponse(url='', body=data, encoding='utf-8')
        stores = data.css('.resultItem')
        if stores:
            return stores
        else:
            return []

    def hours(self, data):
        opening_hours = ''
        for day_group in data:
            time = day_group.css('td::text').extract()
            day = time[0].strip()
            hours = time[1].strip()

            if '-' in day:
                f_day = day.split('-')[0].strip()[:2]
                t_day = day.split('-')[1].strip()[:2]
                day = '{}-{}'.format(f_day, t_day)
            else:
                day = day[:2]
            if hours == 'closed' :
                opening_hours += '{} {};'.format(
                    day,
                    hours
                )
            else :
                f_hour_text = hours.split('-')[0].strip()
                t_hour_text = hours.split('-')[1].strip()
                f_hour = self.normalize_time(f_hour_text)
                t_hour = self.normalize_time(t_hour_text)
                opening_hours += '{} {}; '.format(
                    day,
                    '{}-{}'.format(f_hour, t_hour)
                )

        return opening_hours

    def normalize_time(self, time_str):
        match = re.search(r'([0-9]{1,2}):([0-9]{1,2}) (A|P)M$', time_str)
        if not match:
            match = re.search(r'([0-9]{1,2}) (A|P)M$', time_str)
            h, am_pm = match.groups()
            m = "0"
        else:
            h, m, am_pm = match.groups()

        return '%02d:%02d' % (
            int(h) + 12 if am_pm == 'P' else int(h),
            int(m),
        )


#! /usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2018 Kyoto University (Hirofumi Inaguma)
#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)

"""Base class for all dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import codecs
import logging
import random
import six
import time
from torch.multiprocessing import Process
from torch.multiprocessing import Queue

random.seed(1)

logger = logging.getLogger('training')

class Base(object):

    def __init__(self):
        self.epoch = 0
        self.iteration = 0
        self.offset = 0

        # for multiprocessing
        self._epoch = 0

        # Setting for multiprocessing
        self.preloading_process = None
        self.queue = Queue()
        self.queue_size = 0

    def count_vocab_size(self, dict_path):
        vocab_count = 1  # for <blank>
        with codecs.open(dict_path, 'r', 'utf-8') as f:
            for line in f:
                if line.strip() != '':
                    vocab_count += 1
        return vocab_count

    def __len__(self):
        return len(self.df)

    def __getitem__(self, index):
        raise NotImplementedError()

    def __iter__(self):
        """Returns self."""
        return self

    @property
    def epoch_detail(self):
        # Floating point version of epoch
        return self.epoch + (self.offset / len(self))

    def __next__(self, batch_size=None):
        """Generate each mini-batch.

        Args:
            batch_size (int): the size of mini-batch
        Returns:
            batch (tuple):
            is_new_epoch (bool): If true, 1 epoch is finished

        """
        if batch_size is None:
            batch_size = self.batch_size

        if self.n_ques is None:
            if self.max_epoch is not None and self.epoch >= self.max_epoch:
                raise StopIteration()
            # NOTE: max_epoch == None means infinite loop

            data_indices, is_new_epoch = self.sample_index(batch_size)
            batch = self.make_batch(data_indices)
            self.iteration += len(data_indices)
        else:
            # Clean up multiprocessing
            if self.preloading_process is not None and self.queue_size == 0:
                self.preloading_process.terminate()
                self.preloading_process.join()

            if self.max_epoch is not None and self.epoch >= self.max_epoch:
                # Clean up multiprocessing
                self.preloading_process.terminate()
                self.preloading_process.join()
                raise StopIteration()
            # NOTE: max_epoch == None means infinite loop

            # Enqueue mini-batches
            if self.queue_size == 0:
                self.df_indices_list = []
                self.is_new_epoch_list = []
                for _ in six.moves.range(self.n_ques):
                    data_indices, is_new_epoch = self.sample_index(batch_size)
                    self.df_indices_list.append(data_indices)
                    self.is_new_epoch_list.append(is_new_epoch)
                self.preloading_process = Process(self.preloading_loop,
                                                  args=(self.queue, self.df_indices_list))
                self.preloading_process.start()
                self.queue_size += self.n_ques
                time.sleep(3)

            # print(self.queue.qsize())
            # print(self.queue_size)

            self.iteration += len(self.df_indices_list[self.n_ques - self.queue_size])
            self.queue_size -= 1
            batch = self.queue.get()
            is_new_epoch = self.is_new_epoch_list.pop(0)

        if is_new_epoch:
            self.epoch += 1

        return batch, is_new_epoch

    def next(self, batch_size=None):
        # For python2
        return self.__next__(batch_size)

    def sample_index(self, batch_size):
        """Sample data indices of mini-batch.

        Args:
            batch_size (int): the size of mini-batch
        Returns:
            data_indices (np.ndarray):
            is_new_epoch (bool):

        """
        is_new_epoch = False

        if self.sort_by_input_length or not self.shuffle:
            if self.sort_by_input_length:
                # Change batch size dynamically
                min_n_frames_batch = self.df[self.offset:self.offset + 1]['xlen'].values[0]
                batch_size_tmp = self.select_batch_size(batch_size, min_n_frames_batch)
            else:
                batch_size_tmp = batch_size

            if len(self.rest) > batch_size_tmp:
                data_indices = list(self.df[self.offset:self.offset + batch_size_tmp].index)
                self.rest -= set(data_indices)
                # NOTE: rest is in uttrance length order when sort_by_input_length == True
                # NOTE: otherwise in name length order when shuffle == False
                self.offset += len(data_indices)
            else:
                # Last mini-batch
                data_indices = list(self.df[self.offset: self.offset + len(self.rest)].index)
                self._reset()
                is_new_epoch = True
                self._epoch += 1
                if self._epoch == self.sort_stop_epoch:
                    self.sort_by_input_length = False
                    self.shuffle = True

            # Sort in the descending order for pytorch
            data_indices = data_indices[::-1]
        else:
            # Randomly sample uttrances
            if len(self.rest) > batch_size:
                data_indices = random.sample(list(self.rest), batch_size)
                self.rest -= set(data_indices)
            else:
                # Last mini-batch
                data_indices = list(self.rest)
                self._reset()
                is_new_epoch = True
                self._epoch += 1

            self.offset += len(data_indices)

        return data_indices, is_new_epoch

    def select_batch_size(self, batch_size, min_n_frames_batch):
        if not self.dynamic_batching:
            return batch_size

        if min_n_frames_batch <= 800:
            pass
        elif min_n_frames_batch <= 1600:
            batch_size = int(batch_size / 2)
        else:
            batch_size = int(batch_size / 4)

        if batch_size < 1:
            batch_size = 1

        return batch_size

    def reset(self):
        self._reset()

        self.queue = Queue()
        self.queue_size = 0

        # Clean up multiprocessing
        if self.preloading_process is not None:
            self.preloading_process.terminate()
            self.preloading_process.join()

    def _reset(self):
        """Reset data counter and offset."""
        self.rest = set(list(self.df.index))
        self.offset = 0

    def preloading_loop(self, queue, df_indices_list):
        """.

        Args:
            queue ():
            df_indices_list (np.ndarray):

        """
        # print("Pre-loading started.")
        for i in six.moves.range(len(df_indices_list)):
            queue.put(self.make_batch(df_indices_list[i]))
        # print("Pre-loading done.")


from flask_jwt_extended import get_jwt_identity

import pandas as pd
from xlrd.biffh import XLRDError
import json
from pwa.blueprints.auth.models import User
from pwa.blueprints.precise.models import Patient, Filter
from pwa import db_
import re
import logging

logger = logging.getLogger("Flask.janine_utils")

# step one open file and create dataframe
def openFileAndParse(inputFile):
    try:
        sess = db_.session()
    except Exception as e:
        logger.error(f"Can't get db session because: {e}")
    Bytes = pd.io.common.BytesIO
    input_data = Bytes(inputFile)
    try:
        data = pd.read_excel(input_data, "Feuil1", engine='xlrd')
    except XLRDError:
        data = pd.read_excel(input_data, "Sheet1", engine='xlrd')
    data.fillna("")
    filters = [filter for filter in data.to_dict(orient='records') if type(filter['Scan Number']) != float]
    for iii, filter in enumerate(filters):
        try:
            filter['Date'] = enforce_dates(filter['Date'], filters[iii-1]['Date'])
        except Exception as e:
            logger.error(f"Couldn't enforce date because: {e}")
        filter['# Patients'] = enforce_patient_number_format(filter['# Patients'])
        filter['Scan Number'] = enforce_filter_number_format(filter['Scan Number'])
        filter['# Patients'] = fill_in_patient_number(filter['# Patients'], filter['Scan Number'])
        filter['Scan Number'] = fill_in_filter_number(filter['Scan Number'], filter['# Patients'])

        for item in filter.keys():
            if str(filter[item]) == "nan":
                filter[item] == str(filter[item])
        update_filter = Filter.query.filter_by(_filter_id=filter['Scan Number']).first()
        try:
            update_filter._janine_data = json.dumps(filter)
        except AttributeError as e:
            logger.error(f"No filter found to update for {filter['Scan Number']}")
            continue
    sess.commit()
    return filters

def enforce_patient_number_format(patient_number:str) -> str:
    # compensate for the fact that janine can't seem to keep her filter numbering schema consistent
    if len(str(patient_number)) == 5 or str(patient_number) == "nan":
        return patient_number
    else:
        patient_re = re.compile(r"P(\d+)")
        result = patient_re.search(str(patient_number))
        patient_number = "P0" + result.group(1)
    return patient_number

def fill_in_patient_number(patient_number:str, filter_number:str) -> str:
    if str(patient_number) == "nan":
        return get_patient_by_filter(filter_number)
    else:
        return patient_number

def enforce_filter_number_format(filter_number:str) -> str:
    # the same deal, but for filter numbers
    if len(str(filter_number)) == 8 or str(filter_number) == "nan":
        return filter_number
    else:
        filter_re = re.compile(r"1\dAA(\d+)")
        result = filter_re.search(str(filter_number))
        if len(str(filter_number)) < 8:
            filter_number = "18AA0" + result.group(1)
        else:
            logger.debug(f"Not sure what to do with {str(filter_number)}")
    print(filter_number)
    return filter_number

def fill_in_filter_number(filter_number:str, patient_number:str) -> str:
    if str(filter_number) == "nan":
        return get_filter_by_patient(patient_number)
    else:
        return filter_number

def enforce_dates(cur_date:str, prev_date:str) -> str:
    if str(cur_date) == 'nan':
        logger.debug(f"No date for sample.")
        return prev_date
    else:
        return cur_date

def get_patient_by_filter(filter_num:str) -> str:
    try:
        qFilter = Filter.query.filter_by(_filter_id=filter_num).first()._parent_patient_id
    except:
        logger.error(f"No filter found for {filter_num}")
        return None
    return qFilter

def get_filter_by_patient(patient_num:str) -> str:
    try:
        qPatient = Patient.query.filter_by(_patient_id=patient_num).first().filters[0]._filter_id
    except:
        logger.error(f"No patient found for {patient_num}")
        return None
    return qPatient

# def makeLists(filters: list):
#     # split filters into two lists, ones that have exact matches in DB and ones that don't.
#     filterCheck = []
#     matchesDB = []
#     notinDB = []
#     allFilters = mng.getAllFilters()
#     for filter in filters:
#         filter.pop("# Patients")
#         try:
#             filter.pop("Date ")
#         except KeyError:
#             filter.pop("Date")
#         try:
#             filter['Scan Number'] = namer.parseFilter(filter['Scan Number'])
#             filterCheck.append(filter)
#         except:
#             notinDB.append(filter)
#     for filter in filterCheck:
#         if filter['Scan Number'] in allFilters:
#             matchesDB.append(filter)
#         else:
#             notinDB.append(filter)
#     return matchesDB, notinDB
#
# def addJanineData(filter: dict):
#
#     if type(filter[' Comments']) == float:
#         filter[' Comments'] = ""
#     filterNum = filter.pop('Scan Number')
#     #print(filterNum)
#     patient = mng.getPatientByFilter(filterNum)[0]
#     #print(patient)
#     doc = mng.retrieveDoc(patient)
#     doc['filters'][filterNum]['janine'] = filter
#     mng.shoveDoc(doc)
#
# def addChoicesToDict(notInDB: dict):
#     allFilts = mng.getAllFilters()
#     notInDB['choices'] = [""] + difflib.get_close_matches(notInDB['Scan Number'], allFilts, 3)
#     return notInDB

def uploadFile(file):
    user = User.query.filter_by(email=get_jwt_identity()).first()
    upload_file = file.read()
    upsize = len(upload_file) / 1024
    logger.info(f"{user.first_name} {user.last_name} uploaded a {upsize}kb file")
    return upload_file


from __future__ import print_function

import networkx as nx

"""
Python implementation of graphs, reduced graphs, directed graphs, fat
graphs (graphs with ordered adjacency lists) and fat directed graphs.

Vertices are arbitrary hashable python objects. Graph methods must
not change vertices.

To instantiate a graph, provide a list of edges, as pairs of vertices,
and an optional list of additional vertices, which may be isolated.
Vertices are saved as a set, so redundancies in the vertex list will
be ignored.  Edges are saved as a list, so multiple edges are allowed.

If G is a graph and v a vertex of G then G[v] is a list of vertices
adjacent to v.  If there are n edges joining v to w then w will
appear in this list n times.  If G is a digraph, then G[v] returns
a list of all endpoints of edges having either head or tail at v.
There additional Digraph methods to get lists of incoming or outgoing
edges.

For most graphs G, G(v) returns the set of edges incident to G.
In the case of a FatGraph, the return value is the ordered list
of incident edges.  (Loops will appear twice).

When called as a function, an edge is the non-trivial involution of
its endpoints; calling it with one endpoint returns the other one.

One can iterate over edges as:

for x, y in G.edges:
  ...

where x and y will be the two endpoints of the edge (ordered as
tail, head in the case of directed edges).
"""

_within_sage = False
try:
    import sage.all
    import sage.graphs.graph
    _within_sage = True
except ImportError:
    try:
        from .planarity import planar
        from spherogram.planarity import planar
    except ValueError: # Allow importing from source directory
        pass

from collections import deque, defaultdict
import operator

class CyclicList(list):
    def __getitem__(self, n):
        return list.__getitem__(self, n%len(self))
    def succ(self, x):
        return self[(self.index(x)+1)%len(self)]
    def pred(self, x):
        return self[(self.index(x)-1)%len(self)]

class BaseEdge(tuple):
    """
    Base class for edges: a 2-tuple of vertices with extra methods.
    Calling a BaseEdge with one of its vertices returns the other one.
    """

    def __new__(cls, x, y, **kwargs):
        return tuple.__new__(cls, (x,y))

    def __call__(self, end):
        """
        Calling an edge with one endpoint returns the other one.
        """
        if end is self[0]:
            return self[1]
        elif end is self[1]:
            return self[0]
        else:
            raise ValueError('Vertex is not an endpoint')

    def __hash__(self):
        return id(self)

    def incident_to(self):
        return list(self)

    def is_loop(self):
        return self[0] is self[1]

class Edge(BaseEdge):
    """
    An undirected edge.  We allow multiple Edges between the same
    vertices, so the __eq__ operator is overloaded.
    """

    def __repr__(self):
        return '%s --- %s'%self

    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return self is other

class MultiEdge(BaseEdge):
    """
    An undirected edge.  MultiEdges are equal if they have the
    same vertices.  The multiplicity is initialized to 1.
    """

    def __init__(self, x, y):
        self.multiplicity = 1

    def __repr__(self):
        return '%s --%d-- %s'%(self[0], self.multiplicity, self[1])

    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return set(self) == set(other)

class DirectedEdge(BaseEdge):
    """
    An Edge with a tail and a head.  The two vertices can be accessed as
    E.tail and E.head
    """

    def __repr__(self):
        return '%s --> %s'%self

    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return self is other

    @property
    def head(self):
        return self[1]

    @property
    def tail(self):
        return self[0]

class DirectedMultiEdge(DirectedEdge):
    """
    A DirectedEdge with multiplicity.  DirectedMultiEdges are equal if
    they have the same head and tail.  The multiplicity is initialized to 1.
    """

    def __init__(self, v, w):
        self.multiplicity = 1

    def __repr__(self):
        return '%s --%d-> %s'%(self[0], self.multiplicity, self[1])

    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return tuple(self) == tuple(other)

class FatEdge(Edge):
    """
    An Edge that knows its place among the edges incident to each
    of its vertices.  Initialize with two pairs (v,n) and (w,m) meaning
    that this edge joins v to w and has index n at v and m at w.
    The parity of the optional integer argument twists determines
    whether the edge is twisted or not.
    """
    def __new__(cls, x, y, twists=0):
        return tuple.__new__(cls, (x[0],y[0]))

    def __init__(self, x, y, twists=0):
        self.slots = [x[1],y[1]]
        self.twisted = True if twists%2 != 0 else False

    def __repr__(self):
        return '%s[%d] -%s- %s[%d]'%(self[0], self.slots[0],
                                     'x' if self.twisted else '-',
                                     self[1], self.slots[1])

    def __hash__(self):
        return id(self)

    def slot(self, vertex):
        try:
            return self.slots[self.index(vertex)]
        except ValueError:
            raise ValueError('Vertex is not an end of this edge.')

    def set_slot(self, vertex, n):
        try:
            self.slots[self.index(vertex)] = n
        except ValueError:
            raise ValueError('Vertex is not an end of this edge.')

class Graph(object):
    """
    A set of vertices and a set of edges joining pairs of vertices.
    Vertices are arbitrary hashable objects and should not be
    modified by Graph methods.
    """
    edge_class = Edge

    def __init__(self, pairs=[], singles=[]):
        self.vertices = set()
        self.edges = set()
        self.incidence_dict = {}
        self.Edge = self.__class__.edge_class
        for pair in pairs:
            self.add_edge(*pair)
        for vertex in singles:
            self.add_vertex(vertex)
        self._validate()

    def _validate(self):
        pass

    def __repr__(self):
        V = 'Vertices:\n  ' + '\n  '.join([str(v) for v in self.vertices])
        E = 'Edges:\n  ' + '\n  '.join([str(e) for e in self.edges])
        return '%s\n%s'%(V,E)

    def __call__(self, vertex):
        """
        Return a list of incident edges, including loops, which appear
        twice.
        """
        return self.incidence_dict[vertex]

    def __getitem__(self, vertex):
        """
        Return a list of adjacent vertices, one per incident edge.
        """
        return [e(vertex) for e in self.incidence_dict[vertex]]

    def incident(self, vertex):
        """
        Return the set of non-loops incident to the vertex.
        """
        return set(e for e in self.incidence_dict[vertex] if not e.is_loop())

    # Allow flows to go in either direction across an edge.
    flow_incident = incident

    def children(self, vertex):
        """
        Return the set of distinct adjacent vertices.
        """
        return set(e(vertex) for e in self.incidence_dict[vertex])

    def add_edge(self, *args, **kwargs):
        edge = self.Edge(*args, **kwargs)
        self.edges.add(edge)
        self.vertices.update(edge)
        for v in edge:
            try:
                self.incidence_dict[v].append(edge)
            except KeyError:
                self.incidence_dict[v] = [edge]
        return edge

    def remove_edge(self, edge):
        for vertex in edge:
            try:
                self.incidence_dict[vertex].remove(edge)
            except (KeyError, ValueError):
                pass
        self.edges.remove(edge)

    def add_vertex(self, hashable):
        self.vertices.add(hashable)
        if not self.incidence_dict.has_key(hashable):
            self.incidence_dict[hashable] = []

    def remove_vertex(self, hashable):
        incident = list(self.incidence_dict[hashable])
        for edge in incident:
            self.remove_edge(edge)
        self.vertices.remove(hashable)
        self.incidence_dict.pop(hashable)

    def valence(self, vertex):
        """
        Return the valence of a vertex.
        """
        return len(self.incidence_dict[vertex])

    def depth_first_search(self, start):
        """
        Generator which yields the vertices in the same component as
        start, in depth-first order beginning with start.
        """
        if start not in self.vertices:
            raise ValueError('That starting vertex is not in the graph.')
        stack = [start]
        seen = set(stack)
        while stack:
            current = stack.pop()
            for v in self.children(current):
                if v not in seen:
                    stack.append(v)
                    seen.add(v)
            yield current

    def breadth_first_edges(self, source, forbidden=set(), for_flow=False):
        """
        Generator for non-loop edges of a graph in the complement of
        a forbidden set, ordered by distance from the source.  Used for
        generating paths in the Ford-Fulkerson method.

        Yields triples (e, v, f) where e and f are edges containing v
        and e precedes f in the breadth-first ordering.

        The optional flag "for_flow" specifies that the flow_incident
        method should be used in place of the incident method for
        extending paths.  For example, Digraphs set flow_incident =
        outgoing.
        """
        incident = self.flow_incident if for_flow else self.incident
        initial = incident(source) - forbidden
        seen = forbidden | initial
        # Use a deque since we need to popleft.
        fifo = deque([ (None, source, e) for e in initial ])
        while fifo:
            parent, vertex, child = fifo.popleft()
            new_edges = incident(child(vertex)) - seen
            seen |= new_edges
            fifo.extend([(child, child(vertex), edge) for edge in new_edges])
            yield parent, vertex, child

    def next(self):  #For Python 2 compatibility
        return self.__next__()

    def components(self, deleted_vertices=[]):
        """
        Return the vertex sets of the connected components of the
        graph obtained by removing the deleted_vertices and any edges
        incident to them.

        >>> G = Graph([(0,1),(1,2),(2,0),(2,3),(3,4),(4,2)])
        >>> G.components()
        [frozenset([0, 1, 2, 3, 4])]
        >>> sorted(G.components(deleted_vertices=[2]))
        [frozenset([0, 1]), frozenset([3, 4])]
        >>> G.components(deleted_vertices=[0])
        [frozenset([1, 2, 3, 4])]
        """
        forbidden = set()
        for vertex in deleted_vertices:
            forbidden |= self.incident(vertex)
        vertices, result = self.vertices - set(deleted_vertices), []
        while vertices:
            component, start = set(), vertices.pop()
            component.add(start)
            generator = self.breadth_first_edges(
                source=start, forbidden=forbidden)
            for parent, vertex, child in generator:
                new_vertex = child(vertex)
                component.add(new_vertex)
                if new_vertex in vertices:
                    vertices.remove(new_vertex)
            result.append(frozenset(component))
        return result

    def is_connected(self, deleted_vertices=[]):
        """
        Determine whether the graph obtained by removing the
        deleted_vertices and incident edges is connected.

        >>> G = Graph([(0,1),(1,2),(2,0),(2,3),(3,4),(4,2)])
        >>> G.is_connected(deleted_vertices=[2])
        False
        """
        return len(self.components(deleted_vertices)) <= 1

    def one_min_cut(self, source, sink, capacity=None):
        """
        Find one minimal cut which separates source from sink, using
        the classical Ford-Fulkerson algorithm.te

        Returns a dict containing the set of vertices on the source
        side of the cut, the set of edges that cross the cut, a
        maximal family of weighted edge-disjoint paths from source to
        sink, the set of edges with non-zero residual, and the
        associated maximum flow.

        The edge capacities are supplied as a dictionary, with
        edges as keys and the capacity of the edge as value.  If
        no capacity dict is supplied, every edge is given capacity
        1.  Edges omitted from the capacity dict have infinite
        capacity.

        When called as a Graph method, the flow is relative to the
        implicit orientation of an undirected edge e from e[0] to
        e[1].  (This is determined when the edge is first constructed
        from a pair of vertices.) A negative flow value means the flow
        direction is from e[1] to e[0].  When called as a DiGraph
        method, paths are directed and flows go in the direction of
        the directed edge.

        >>> caps = {('s',0):3,('s',1):2,(0,1):2,(0,'t'):4,(1,'t'):1}
        >>> G = Graph(caps.keys())
        >>> cap_dict = dict((e, caps[tuple(e)]) for e in G.edges)
        >>> flow = G.one_min_cut('s', 't', cap_dict)['flow']
        >>> [flow[e] for e in sorted(G.edges)]
        [-1, 4, 1, 3, 2]
        >>> G = Digraph(caps.keys())
        >>> cap_dict = dict((e, caps[tuple(e)]) for e in G.edges)
        >>> flow = G.one_min_cut('s', 't', cap_dict)['flow']
        >>> [flow[e] for e in sorted(G.edges)]
        [0, 3, 1, 3, 1]
        """
        if sink == source:
            return None
        if capacity is None:
            residual = dict.fromkeys(self.edges, 1)
        else:
            residual = dict.copy(capacity)
            for e in self.edges:
                if e not in residual:
                    residual[e] = float('inf')
        full_edges = set([e for e in self.edges if residual[e] == 0])
        children = {}
        for vertex in self.vertices:
            children[vertex] = set()
        cut_set = set()
        path_list = []
        while True:
            # Try to find a new path from source to sink
            parents, cut_set, reached_sink = {}, set([source]), False
            generator = self.breadth_first_edges(
                source=source,
                forbidden=full_edges,
                for_flow=True)
            for parent, vertex, child in generator:
                 parents[child] = (parent, vertex)
                 cut_set.add(child(vertex))
                 if child(vertex) == sink:
                     reached_sink = True
                     break
            # If we did not get to the sink, we visited every vertex
            # reachable from the source, thereby providing the cut
            # set.
            if not reached_sink:
                break
            # If we got to the sink, do the bookkeeping and continue.
            path = deque()
            flow = residual[child]
            while True:
                path.appendleft( (vertex, child) )
                children[vertex].add(child)
                if vertex == source:
                    break
                child = parent
                parent, vertex = parents[child]
                flow = min(flow, residual[child])
            for vertex, edge in path:
                residual[edge] -= flow
                if residual[edge] == 0:
                    full_edges.add(edge)
            path_list.append( (flow, path) )
        # Find the cut edges.
        cut_edges = set()
        for vertex in cut_set:
            cut_edges |= set([edge for edge in self.edges
                              if vertex in edge
                              and edge(vertex) not in cut_set])
        # Find the unsaturated edges.
        unsaturated = [ e for e in self.edges if residual[e] > 0 ]
        flow_dict = dict.fromkeys(self.edges, 0)
        # Compute the flow.
        for flow, edges in path_list:
            for vertex, edge in edges:
                if vertex == edge[0]:
                    flow_dict[edge] += flow
                else:
                    flow_dict[edge] -= flow
        return {'set': cut_set, 'edges': cut_edges, 'paths': path_list,
                'residuals': residual, 'unsaturated': unsaturated,
                'flow': flow_dict}

    def reduced(self):
        R = ReducedGraph()
        for e in self.edges:
            R.add_edge(*e)
        return R

    def is_planar(self):
        return self.reduced().is_planar()

    def merge(self, V1, V2):
        """
        Merge two vertices and remove all edges between them.  The
        vertices must support the __or__ method (e.g. all vertices
        might be frozensets).  The two vertices V1 and V2 are replaced
        by V1|V2.

        >>> G = Graph([(0,1),(1,2),(2,0)]).mergeable()
        >>> F = lambda x: frozenset([x])
        >>> G.merge(F(1),F(2))
        >>> sorted(G.vertices)
        [frozenset([1, 2]), frozenset([0])]
        >>> sorted(G.edges)[0]
        frozenset([0]) --- frozenset([1, 2])
        """
        new_vertex = V1|V2
        if new_vertex in self.vertices:
            raise ValueError('Merged vertex already exists!')
        self.edges -= set([e for e in self.edges if V1 in e and V2 in e])
        self.vertices.remove(V1)
        self.vertices.remove(V2)
        old_vertices = (V1, V2)
        self.vertices.add(new_vertex)
        for edge in [e for e in self.edges if V1 in e or V2 in e]:
            self.edges.remove(edge)
            x, y = edge
            if x in old_vertices:
                self.edges.add( self.Edge(new_vertex, y) )
            if y in old_vertices:
                self.edges.add( self.Edge(x, new_vertex) )

    def mergeable(self):
        """
        Return a new graph whose vertices are singleton frozensets,
        each containing a vertex of this graph, and with edges
        corresponding to those of this graph.  The new graph
        will support merging vertices.
        """
        V = [frozenset([v]) for v in self.vertices]
        E = [self.Edge(frozenset([x]), frozenset([y])) for x, y in self.edges]
        return self.__class__(E,V)

    def to_networkx(self):
        """
        Return a copy of the graph in the networkx format
        """
        G = nx.Graph()
        G.add_nodes_from(self.vertices)
        G.add_edges_from(self.edges)
        return G

class ReducedGraph(Graph):
    """
    A graph with at most one edge between any two vertices,
    but having edges with multiplicities.
    """
    edge_class = MultiEdge

    def __init__(self, pairs=[], singles=[]):
        self.Edge = self.__class__.edge_class
        self.find_edge = dict()
        Graph.__init__(self, pairs, singles)

    def add_edge(self, x, y):
        if self.find_edge.has_key((x,y)):
            edge = self.find_edge[(x, y)]
            edge.multiplicity += 1
        else:
            edge = self.Edge(x, y)
            self.vertices.update([x,y])
            self.edges.add(edge)
            self.find_edge[(x,y)] = edge
            for v in edge:
                try:
                    self.incidence_dict[v].append(edge)
                except KeyError:
                    self.incidence_dict[v] = [edge]
        return edge

    def multi_valence(self, vertex):
        """
        Return the valence of a vertex, counting edge multiplicities.
        """
        valence = 0
        for e in self.incidence_dict[vertex]:
                valence += e.multiplicity
        return valence

    def is_planar(self):
        """
        Return the planarity.
        """

        verts_with_loops = set()
        non_loop_edges = set()
        for e in self.edges:
            v, w = e
            if v != w:
                non_loop_edges.add( (v, w) )
            else:
                verts_with_loops.add(v)

        sans_loops = ReducedGraph(non_loop_edges)
        if _within_sage:
            S = sans_loops.sage(loops=False, multiedges=False)
            is_planar = S.is_planar(set_embedding=True)
            embedding = S.get_embedding() if is_planar else None
        else:
            is_planar, embedding = planar(sans_loops)

        if is_planar:
            for v in verts_with_loops:
                embedding[v].append(v)
            self._embedding = embedding

        return is_planar

    def embedding(self):
        if self.is_planar():
            return self._embedding

    def one_min_cut(self, source, sink):
        capacity = dict((e, e.multiplicity) for e in self.edges)
        cut = Graph.one_min_cut(self, source, sink, capacity)
        cut['size'] = sum([e.multiplicity for e in cut['edges'] ])
        return cut

    def cut_pairs(self):
        """
        Return a list of cut_pairs.  The graph is assumed to be
        connected and to have no cut vertices.
        """
        pairs = []
        majors = [v for v in self.vertices if self.valence(v) > 2]
        if len(majors) == 2:
            v, V = majors
            if self.valence(v) == 3:
                return []
            edge = self.find_edge[(v,V)]
            if not edge or edge.multiplicity < 2:
                return []
            return majors
        major_set = set(majors)
        for n in xrange(1,len(majors)):
            for v in majors[:n]:
                pair = (v, majors[n])
                components = self.components(deleted_vertices=pair)
                if len(components) > 2:
                    pairs.append(pair)
                elif len(components) == 2:
                    M0 = len(major_set & components[0])
                    M1 = len(major_set & components[1])
                    edge = self.find_edge[pair]
                    if edge:
                        if M0 or M1:
                            pairs.append(pair)
                            continue
                    else:
                        if M0 and M1:
                            pairs.append(pair)
        return pairs

class FatGraph(Graph):
    """
    A FatGraph is a Graph which maintains a CyclicList of incident
    edges for each vertex.  The edges are FatEdges, which come in two
    flavors: twisted and untwisted.  Since the incident edges are in
    particulary cyclically ordered, a FatGraph has a canonical
    embedding as a spine of a surface with boundary.  However, a
    CyclicList has a first element.  This extra data, namely a choice
    of distinguished edge for each vertex, is used to give a canonical
    mapping from FatGraphs to link or tangle diagrams.

    A FatGraph should be initialized with a sequence of either double
    pairs ((v,n), (w,m)) or triples ((v,n), (w,m), T).  These indicate
    that there is an edge from v to w which has index n at v and m at
    w.  If T is supplied then the parity of T determines the twist.
    """
    edge_class = FatEdge

    def __call__(self, vertex):
        return self.incidence_dict[vertex]

    def add_edge(self, *args):
        incidences = self.incidence_dict
        edge = self.Edge(*args)
        self.edges.add(edge)
        self.vertices.update(edge)
        for v in edge:
            # the values of incidence_dict should be objects that keep
            # themselves sorted.
            if v in incidences:
                incidences[v].append(edge)
                incidences[v].sort(key=lambda e : e.slot(v))
            else:
                incidences[v] = CyclicList([edge])
        return edge

    def _validate(self):
        for v in self.vertices:
            slots = [e.slot(v) for e in self(v)]
            assert slots == range(len(slots))

    def reorder(self, vertex, cyclist):
        for e, n in zip(self(vertex), cyclist):
            e.set_slot(vertex, n)
        self.incidence_dict[vertex].sort(key=lambda e : e.slot(vertex))

    def boundary_cycles(self):
        left  = [(e[0], e, 'L') for e in self.edges]
        right = [(e[0], e, 'R') for e in self.edges]
        sides = left + right
        cycles = []
        while sides:
            cycle = []
            v, e, s = start = sides.pop()
            #print('start:', start)
            while True:
                cycle.append(e)
                # cross the edge
                v = e(v)
                if (
                    ( e.twisted and s=='L') or
                    ( not e.twisted and (s=='L')==(v==e[0]) )
                    ): # go counter-clockwise
                    e = self(v).succ(e)
                    s = 'R' if e.twisted or v == e[0] else 'L'
                else: # go clockwise
                    e = self(v).pred(e)
                    s = 'L' if e.twisted or v == e[0] else 'R'
                if (e[0],e,s) == start:
                    cycles.append(cycle)
                    break
                #print('next', (e[0],e,s))
                sides.remove( (e[0],e,s) )
        return cycles

    def filled_euler(self):
        return len(self.vertices) - len(self.edges) + len(self.boundary_cycles())

class Digraph(Graph):
    """
    A directed graph.  This subclass adds methods for finding incoming
    and outgoing edges, indegree and outdegree, and deals with all
    three notions of connectedness for digraphs.

    >>> G = Digraph([(0,1),(1,2),(2,3),(3,1),(4,2)])
    >>> G.indegree(1), G.outdegree(1)
    (2, 1)
    >>> G.is_weakly_connected()
    True
    >>> G.is_connected()
    False
    >>> G.is_strongly_connected()
    False
    >>> G.components()
    Traceback (most recent call last):
        ...
    ValueError: Not meaningful for Digraphs.
    Use weak_components() or strong_components()
    >>> G.weak_components()
    [frozenset([0, 1, 2, 3, 4])]
    >>> G.strong_components()
    [frozenset([1, 2, 3]), frozenset([0]), frozenset([4])]
    """

    edge_class = DirectedEdge

    def outgoing(self, vertex):
        """
        Return the set of non-loop edges which *begin* at the vertex.
        """
        return set(e for e in self(vertex)
                     if e.tail is vertex and not e.head is vertex)

    # Force flows to go in the direction of the edge.
    flow_incident = outgoing

    def incoming(self, vertex):
        """
        Return the set of non-loop edges which *end* at the vertex.
        """
        return set(e for e in self(vertex)
                     if e.head is vertex and not e.tail is vertex)

    def children(self, vertex):
        """
        Return the set of distinct vertices which are endpoints of
        outgoing non-loop edges.
        """
        return set(e(vertex) for e in self.outgoing(vertex))

    def indegree(self, vertex):
        return len([e for e in self.incidence_dict[vertex] if e.head is vertex])

    def outdegree(self, vertex):
        return len([e for e in self.incidence_dict[vertex] if e.tail is vertex])

    # For Digraphs we call them weak components.
    weak_components = Graph.components

    def components(self):
        raise ValueError('Not meaningful for Digraphs.\n'
                         'Use weak_components() or strong_components()')

    def is_weakly_connected(self):
        """
        A digraph is weakly connected if the associated undirected graph
        is connected.
        """
        return len(self.weak_components()) <= 1

    def is_connected(self):
        """
        A digraph is connected if, for every pair of vertices v, w,
        there is either a directed path from v to w or a directed path
        from w to v.  This is equivalent to the condition that the
        strong component DAG be a linear order.  Note that
        reachability is not a symmetric relation, so there is no
        associated notion of a connected component.
        """
        DAG = self.component_DAG()
        if not DAG.is_weakly_connected():
            return False
        elif [v for v in DAG.vertices
              if DAG.indegree(v)>1 or DAG.outdegree(v)>1]:
            return False
        else:
            return True

    def strong_components(self):
        """
        Return the vertex sets of the strongly connected components.

        >>> G = Digraph([(0,1),(0,2),(1,2),(2,3),(3,1)])
        >>> G.strong_components()
        [frozenset([1, 2, 3]), frozenset([0])]
        >>> G = Digraph([(0,1),(0,2),(1,2),(2,3),(1,3)])
        >>> G.strong_components()
        [frozenset([3]), frozenset([2]), frozenset([1]), frozenset([0])]
        """
        return StrongConnector(self).components

    def is_strongly_connected(self):
        """
        A digraph is strongly connected if, for every pair of vertices
        v, w, there is a directed path from v to w and a directed path
        from w to v.
        """
        return len(self.strong_components()) <= 1

    def component_DAG(self):
        """
        Return the acyclic digraph whose vertices are the strong
        components of this digraph.  Two components are joined by an
        edge if this digraph has an edge from one component to the
        other.

        >>> G = Digraph([(0,1),(0,2),(1,2),(2,3),(3,1)])
        >>> G.component_DAG()
        Vertices:
          frozenset([1, 2, 3])
          frozenset([0])
        Edges:
          frozenset([0]) --> frozenset([1, 2, 3])
        """
        return StrongConnector(self).DAG()

class StrongConnector(object):
    """
    Finds strong components of a digraph using Tarjan's algorithm;
    see http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
    """
    def __init__(self, digraph):
        self.digraph = digraph
        self.seen = []
        self.unclassified = []
        self.components = []
        self.root = {}
        self.which_component = {}
        self.links = set()
        for vertex in self.digraph.vertices:
            if vertex not in self.seen:
                self.search(vertex)

    def search(self, vertex):
        self.root[vertex] = len(self.seen)
        self.seen.append(vertex)
        self.unclassified.append(vertex)
        for child in self.digraph.children(vertex):
            if child not in self.seen:
                self.search(child)
                self.root[vertex] = min(self.root[child],
                                        self.root[vertex])
            elif child in self.unclassified:
                self.root[vertex] = min(self.seen.index(child),
                                        self.root[vertex])
            else:
                self.links.add((vertex, child))
        if self.root[vertex] == self.seen.index(vertex):
            component = []
            while True:
                child = self.unclassified.pop()
                component.append(child)
                if child == vertex:
                    break
            if self.unclassified:
                self.links.add((self.unclassified[-1], vertex))
            component = frozenset(component)
            self.components.append(component)
            for child in component:
                self.which_component[child] = component

    def DAG(self):
        """
        Return the directed acyclic graph whose vertices are the
        strong components of the underlying digraph.  There is an edge
        joining two components if and only if there is an edge of the
        underlying digraph having an endpoint in each component.
        Using self.links, rather than self.digraph.edges, is a slight
        optimization.
        """
        edges = set()
        for tail, head in self.links:
            dag_tail= self.which_component[tail]
            dag_head = self.which_component[head]
            if dag_head != dag_tail:
                edges.add( (dag_tail, dag_head) )
        return Digraph(edges, self.components)

class Poset(set):
    """
    A partially ordered set, generated from a directed acyclic graph.
    Instantiate with a Digraph.  A ValueError exception is raised if the
    Digraph contains a cycle.
    """
    def __init__(self, digraph):
        self.elements = set(digraph.vertices)
        self.larger = {}
        self.smaller = {}
        self.successors = {}
        self.closed = set()
        for vertex in self:
            self.larger[vertex] = set()
            self.smaller[vertex] = set()
            self.successors[vertex] = set(digraph[vertex])
        seen = []
        for vertex in self:
            if vertex not in seen:
                self.search(vertex, seen, digraph)

    def __iter__(self):
        return self.elements.__iter__()

    def __len__(self):
        return len(self.elements)

    def search(self, vertex, seen, digraph):
        seen.append(vertex)
        for child in digraph.children(vertex):
            if child in self.smaller[vertex]:
                raise ValueError('Digraph is not acyclic.')
            self.smaller[child].add(vertex)
            self.smaller[child] |= self.smaller[vertex]
            self.search(child, seen, digraph)
            self.larger[vertex].add(child)
            self.larger[vertex] |= self.larger[child]

    def compare(self, x, y):
        if x == y:
            return 0
        elif x in self.smaller[y]:
            return 1
        elif y in self.smaller[x]:
            return -1
        else:
            return None

    def incomparable(self, x):
        """
        Return the elements which are not comparable to x.
        """
        return self.elements - self.smaller[x] - self.larger[x] - set([x])

    def smallest(self):
        """
        Return the subset of minimal elements.
        """
        return frozenset( [ x for x in self if not self.smaller[x] ] )

    def largest(self):
        """
        Return the subset of maximal elements.
        """
        return frozenset( [ x for x in self if not self.larger[x] ] )

    def closure(self, A):
        """
        Return the smallest set X containing A which is is closed
        under < , i.e. such that (x in X and y < x) => y in X.
        """
        result = frozenset(A)
        for a in A:
            result |= self.smaller[a]
        if len(result) == len(A):
            return result
        else:
            return self.closure(result)

    def XXclosed_subsets(self, start=None):
        """
        Generator for all transitively closed subsets.
        """
        if start is None:
            if self.closed:
                for subset in self.closed:
                    yield subset
                return
            else:
                start = self.smallest()
        complement = self.elements - start
        if start not in self.closed:
            self.closed.add(start)
            yield start
        for element in complement:
           # print( 'adding ', element)
            extended = self.closure(start | set([element]))
            for subset in self.XXclosed_subsets(extended):
                yield subset

    def XXXclosed_subsets(self, start=None):
        """
        Generator for all transitively closed subsets.  The subsets
        are computed once, then cached for use in subsequent calls.
        """
        if start is None:
            if self.closed:
                for subset in self.closed:
                    yield subset
                return
            else:
                start = self.smallest()
        if start not in self.closed:
            self.closed.add(start)
            yield start
        children = set()
        for element in start:
            children.update(self.successors[element] - start)
        for child in children:
            extended = self.closure(start | set([child]))
            for subset in self.XXXclosed_subsets(extended):
                yield subset

    def closed_subsets(self):
        """
        Generator for all nonempty transitively closed subsets.
        """
        # A closed subset is the closure of its set of maximal
        # elements, which is an arbitrary subset of pairwise
        # incomparable elements.  For the empty relation, every subset
        # is pairwise incomparable.  So, in general, this can easily
        # generate exponentially many subsets.
        #
        # This is used for finding level transformations of group
        # presentations.  We should think about taking advantage of
        # the special features of that situation.  For now, though, we
        # just iterate through all subsets and check each one.

        for X in powerset(self.elements):
            if 0 < len(X):
                pairwise_incomparable = True
                for x in X:
                    if ( X.intersection(self.smaller[x]) or
                         X.intersection(self.larger[x]) ):
                        pairwise_incomparable = False
                        break
                if pairwise_incomparable:
                    yield self.closure(X)

def powerset(S):
    """Recursive generator for all subsets of a set."""
    X = S.copy()
    while X:
        for x in X: break
        X.remove(x)
        singleton = set([x])
        for Y in powerset(X):
            yield(singleton | Y)
        yield X

if _within_sage:
    def _to_sage(self, loops=True, multiedges=True):
        S = sage.graphs.graph.Graph(loops=loops, multiedges=multiedges)
        S.add_vertices(self.vertices)
        for e in self.edges:
            v, w = e
            S.add_edge(v, w, repr(e))
        return S
    Graph.sage = _to_sage

    def _to_sage_digraph(self):
        S = sage.graphs.graph.DiGraph(loops=True, multiedges=True)
        S.add_vertices(self.vertices)
        for e in self.edges:
            v, w = e
            S.add_edge(v, w, repr(e))
        return S
    Digraph.sage = _to_sage_digraph


"""
This script displays the flat images in an RC Ginga window.
"""

def parse_args(options=None, return_parser=False):

    import argparse

    parser = argparse.ArgumentParser(description='Display MasterFlat images in a previously '
                                                 'launched RC Ginga viewer',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--type", type=str, default='all', help="Which flats to display. Must be one of: pixel, illum, all")
    parser.add_argument('master_file', type=str,
                        help='PypeIt MasterFlat file [e.g. MasterFlat_A_1_01.fits]')
    parser.add_argument('--try_old', default=False, action='store_true',
                        help='Attempt to load old datamodel versions.  A crash may ensue..')

    if return_parser:
        return parser

    return parser.parse_args() if options is None else parser.parse_args(options)

def main(args):

    from pypeit import flatfield

    # Load
    flatImages = flatfield.FlatImages.from_file(args.master_file, chk_version=(not args.try_old))
    flatImages.show(args.type)


import boto3
import logging
import os

def get_log_level():
    log_level = os.environ.get("log_level", "INFO")
    return log_level.upper() if isinstance(log_level, str) else log_level

def get_ecs_cluster_name():
    return os.environ.get("ecs_cluster_name", "telemetry")

def get_lambda_name():
    os.environ.get("lambda_name", "ecs-riemann-reload")

def get_riemann_consumer_ecs_service_name():
    return os.environ.get("riemann_consumer_ecs_service_name", "riemann-consumer")

def create_logger(level=logging.INFO):
    logger = logging.getLogger()
    logger.setLevel(level)

    return logger

def lambda_handler(event, context):
    logger = create_logger(get_log_level())
    try:
        logger.info(f"Lambda Request ID: {context.aws_request_id}")
    except AttributeError:
        logger.debug(f"No context object available")

    logger.info(f"Event received from SNS: \"{event}\"")

    ecs_client = boto3.client("ecs")

    ecs_service_name = get_riemann_consumer_ecs_service_name()
    logger.info(f"Requesting a new deployment of the ECS {ecs_service_name} service")
    try:
        response = ecs_client.update_service(
            cluster=get_ecs_cluster_name(), service=ecs_service_name, forceNewDeployment=True
        )
        logger.info(f"Deployment request completed: \"{response}\"")

        return {
            'success': True,
            'serviceName': response['service']['serviceName'],
            'status': response['service']['status'],
            'desiredCount': response['service']['desiredCount'],
            'runningCount': response['service']['runningCount'],
            'pendingCount': response['service']['pendingCount'],
        }
    except Exception as e:
        logger.error(f"Deployment action failed: {e}")

        return {
            'success': False,
            'errorMessage': str(e)
        }


from discord.ext import commands
from helpers.funcs import create_embed, get_reacts, check
from models.classModels import Class, Task, Exam, User

import datetime

c_daysList = ["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"]

class Setup(commands.Cog, name="setup", description="Set up the bot and personalize it to your schedule!"):
    def __init__(self, client):
        self.client = client

    @commands.command(name="new-class", brief="Create a class", description="Create a class to add to your schedule, or to publish and share to your students!")
    async def newclass(self, context):
        embed = create_embed(
            "Create a class!",
            description="To create a class, first choose whether you are creating this class for yourself or as a teacher. If you are creating this as a teacher, you will be provided with a class code to share with your students.",
            fields=[
                ["As a Teacher", "React with 👨🏫", True],
                ["For Yourself", "React with 👨🎓", True]
            ]
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["👨🏫", "👨🎓"])
        current = Class()
        if react.emoji == "👨🏫":
            current.teacher = context.author.id
            current.subject = current.teacher
        elif react.emoji == "👨🎓":
            embed = create_embed(
                "Moving on...",
                description="Now, tell me the name of the teacher who is teaching this class. Just type it in below:"
            )
            message = await context.send(embed=embed)
            resp = await self.client.wait_for("message", check=check(context.author, context.channel))
            current.teacher = resp.content
            current.subject = current.teacher
        else:
            raise Exception("You've broken Python!")

        embed = create_embed(
            "Perfect! Let's keep going...",
            description="Now, set your class name. This is what you will use to add Tasks and Exams, and to reference it later. Just type it in below:"
        )

        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.name = resp.content
        embed = create_embed(
          f'{current.name} is your class',
          description = "Is this correct?"
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["❌", "✅"])
        while(react.emoji=="❌"):
          embed = create_embed(
            #Maybe 'your' name? If its the teacher??
            "Please enter the class's name: "
          )
          message = await context.send(embed=embed)
          resp = await self.client.wait_for("message", check=check(context.author, context.channel))
          current.name = resp.content
          embed = create_embed(
            f'{current.name} is your class name',
            description = "Is this correct?"
          )
          message = await context.send(embed=embed)
          react = await get_reacts(context, self.client, message, ["❌", "✅"])

        embed = create_embed(
          f"Now, we need the information for {current.name}'s schedule",
          description=f"Please enter the hours that {current.name} will meet"
        )
        message = await context.send(embed=embed)
        index = 0
        for item in current.days:
            message = await context.send(f"Please enter times for {c_daysList[index]}! If the class doesn't meet on that day, enter anything that's not two numbers separated by a `-`.")
            resp = await self.client.wait_for("message", check=check(context.author, context.channel))
            try:
                resp = resp.content
                if not "-" in resp:
                    raise Exception("Nope")
                current.days[index]=str(resp)
                index += 1
            except Exception as e:
                index += 1
        current.at = current.days
        embed = create_embed(
          "Almost done! The last thing to do is to enter a description of the class"
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.description = resp.content
        embed = create_embed(
            "Is this correct?",
            fields=[
                ["Teacher", "You" if isinstance(current.teacher, int) else current.teacher, True],
                ["Name", current.name, True],
                ["Description", current.description, True],
                ["Meeting Times", ", ".join([c_daysList[i] + ": " + current.days[i] for i in range(7)])]
            ]
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["✅", "❌"])
        if react.emoji == "✅":
            await context.send("Successfully added class!")
        elif react.emoji == "❌":
            await context.send("Cancelled adding class!")
        else:
            raise Exception("You've broken Python!")

    @commands.command(name="join-class", brief="Join a class", description="Join a class with a join code")
    async def joinclass(self, context):
        await context.send("Enter the join code provided by your teacher! (Example: `783465`)")
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        class React:
            emoji = "❌"
        react = React()
        while(react.emoji=="❌"):
            code = int(resp.content)
            embed = create_embed(
                "Are you sure?",
                description="Are you sure you want to join this class? ✅ or ❌",
                fields=[
                    ["Class", "Biology Honors"],
                    ["Description:", "Bio Honors Period 4! Room 68, Second Floor. Have fun learning Biology!"],
                    ["Teacher", "Mr. Jeff", True],
                    ["Meeting Times", "Monday: 2-3:30, Tuesday: 4-5, Wednesday: 2-3:30, Thursday: 4-5, Friday: 2-3:30"]
                ]
            )
            message = await context.send(embed=embed)
            react = await get_reacts(context, self.client, message, ["✅", "❌"])
            if react.emoji == "✅":
                await context.send("Successfully joined class!")
            elif react.emoji == "❌":
                await context.send("Cancelled joining class!")
            else:
                raise Exception("You've broken Python!")
            #await context.send("Invalid code! Please enter a valid code")#do we want to keep this?

    @commands.command(name='new-task', brief="Create a new task", description="Create a new task for your class!")
    async def newtask(self, context):
        embed = create_embed(
            "Time to make a new Task!",
            description="Which class is this task for?\n\nYour classes: `Science`, `History`, `English`, `Math`, `Tech`, `Spanish`, and `Computer Science`"
        )
        await context.send(embed=embed)
        current = Task()
        subject = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.subject = subject.content
        embed=create_embed(
            f"New Task for {current.subject}",
            description="Please enter the task name!"
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.name = resp.content
        embed=create_embed(
            f"New Task: {current.name}",
            description="A description of this task (optional)"
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.description = resp.content
        while True:
            embed=create_embed(
                f"New Task: {current.name}",
                description="When is this task at? Format: mm/dd/yy\nExample: `03/09/21`."
            )
            message = await context.send(embed=embed)
            resp = await self.client.wait_for("message", check=check(context.author, context.channel))
            resp = resp.content.split("/")
            if len(resp) == 3 and all(len(x) == 2 for x in resp):
                try:
                    for x in range(len(resp)):
                        resp[x] = int(resp[x])
                    current.dueDate = datetime.date(resp[2], resp[0], resp[1])
                    current.at = current.dueDate
                    break
                except Exception as e:
                    print(e)
                    await context.send("Invalid Date!")
            else:
                await context.send("Invalid Date!")
        embed = create_embed(
            f"Task: {current.name}",
            description="Is this correct? ✅ or ❌",
            fields=[
                ["Description", current.description],
                ["Subject", current.subject, True],
                ["At", current.dueDate.strftime("%A %B %d, %Y (%m/%d/%y)"), True]
            ]
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["✅", "❌"])
        if react.emoji == "✅":
            await context.send("Successfully added task!")
        elif react.emoji == "❌":
            await context.send("Cancelled adding task!")
        else:
            raise Exception("You've broken Python!")

    @commands.command(name='new-exam', brief="Create a new exam", description="Create a new exam for your class!")
    async def newexam(self, context):
        embed = create_embed(
            "Time to make a new Exam!",
            description="Which class is this exam for?\n\nYour classes: `Science`, `History`, `English`, `Math`, `Tech`, `Spanish`, and `Computer Science`"
        )
        await context.send(embed=embed)
        current = Exam()
        subject = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.subject = subject.content
        embed=create_embed(
            f"New Exam for {current.subject}",
            description="Please enter the exam name!"
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.name = resp.content
        embed=create_embed(
            f"New Exam: {current.name}",
            description="A description of this exam (optional)"
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.description = resp.content
        while True:
            embed=create_embed(
                f"New Exam: {current.name}",
                description="When is this exam at? Format: mm/dd/yy hh:mm AM/PM\nExample: `03/09/21 3:45 PM`."
            )
            message = await context.send(embed=embed)
            resp = await self.client.wait_for("message", check=check(context.author, context.channel))
            resp = resp.content.split("/")
            if len(resp) == 3:
                try:
                    endsplit = resp[2].split(" ")
                    resp[2] = endsplit[0]
                    t = endsplit[1].split(":")
                    if len(t) == 2 and all(len(x) == 2 for x in resp):
                        for x in range(len(resp)):
                            resp[x] = int(resp[x])
                        for x in range(len(t)):
                            t[x] = int(t[x])
                        if endsplit[2].lower() == "pm":
                            t[0] += 12
                        current.date = datetime.datetime(resp[2], resp[0], resp[1], t[0], t[1])
                        current.at = current.date
                        break
                except Exception as e:
                    print(e)
                    await context.send("Invalid Date!")
            else:
                await context.send("Invalid Date!")
        embed = create_embed(
            f"Exam: {current.name}",
            description="Is this correct? ✅ or ❌",
            fields=[
                ["Description", current.description],
                ["Subject", current.subject, True],
                ["At", current.date.strftime("%A %B %d, %Y (%m/%d/%y) at %I:%M %p"), True]
            ]
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["✅", "❌"])
        if react.emoji == "✅":
            await context.send("Successfully added exam!")
        elif react.emoji == "❌":
            await context.send("Cancelled adding exam!")
        else:
            raise Exception("You've broken Python!")

    #New User
    @commands.command(name='profile', brief="Set up your profile", description="Set up your profile to start planning out your School Life")
    async def newuser(self, context):
        embed=create_embed(
            "Time to add a new user",
            description="To add a new user, we will need the person's name and school ID number. Please enter your full name."
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current = User()
        goodName = False
        while(goodName==False):
            if ' ' in resp.content:
                current.firstName = resp.content.split(' ')[0]
                current.lastName = resp.content.split(' ')[1]
                current.fullName = current.firstName + current.lastName
                goodName = True
            elif ',' in resp.content:
                current.firstName = resp.content.split(',')[0]
                current.lastName = resp.content.split(',')[1]
                current.fullName = current.firstName + current.lastName
                goodName = True
            else:
                await context.send("Invalid name! Please enter your full name")
                goodName = False
        embed=create_embed(
            f"Continue your Profile, {current.firstName}",
            description="Please enter your school student ID! This is so that you can be recognized by your teachers or students."
        )
        message = await context.send(embed=embed)
        resp = await self.client.wait_for("message", check=check(context.author, context.channel))
        current.schoolID = resp.content
        embed = create_embed(
            f"Confirm your information, {current.firstName}",
            description="Is this correct? ✅ or ❌",
            fields=[
                ["First Name:", current.firstName, True],
                ["Last Name", current.lastName, True],
                ["School ID", current.schoolID]
            ]
        )
        message = await context.send(embed=embed)
        react = await get_reacts(context, self.client, message, ["✅", "❌"])
        if react.emoji == "✅":
            await context.send("Successfully set your profile!")
        elif react.emoji == "❌":
            await context.send("Cancelled your profile!")
        else:
            raise Exception("You've broken Python!")

def setup(client):
    client.add_cog(Setup(client))


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from matplotlib.font_manager import *
import matplotlib
myfont = FontProperties(fname='/usr/share/fonts/truetype/arphic/ukai.ttc')
matplotlib.rcParams['axes.unicode_minus'] = False
matplotlib.rcParams['axes.unicode_minus'] = False

show_number=3
pf = pd.read_csv("ls_results.csv")
data_a = pf.values.T.tolist()
data_b = pf.values.T.tolist()
data_c = pf.values.T.tolist()
ticks = pf.columns.tolist()

def set_box_color(bp, color):
    plt.setp(bp['boxes'], color=color)
    plt.setp(bp['whiskers'], color=color)
    plt.setp(bp['caps'], color=color)
    plt.setp(bp['medians'], color=color)

plt.figure()

bpl = plt.boxplot(data_a, positions=np.array(range(len(data_a)))*show_number-0.2*show_number, sym='', widths=0.6)
bpr = plt.boxplot(data_b, positions=np.array(range(len(data_b)))*show_number, sym='', widths=0.6)
bpc = plt.boxplot(data_c, positions=np.array(range(len(data_c)))*show_number+0.2*show_number, sym='', widths=0.6)
set_box_color(bpl, 'r') # colors are from http://colorbrewer2.org/
set_box_color(bpr, 'b')
set_box_color(bpc, 'g')
# draw temporary red and blue lines and use them to create a legend
plt.plot([], c='r', label=u'ls算法')
plt.plot([], c='b', label=u'wls算法')
plt.plot([], c='g', label=u'cnn算法')
plt.legend(prop =myfont)

plt.xticks(range(0, len(ticks) * show_number, show_number), ticks)
plt.xlim(-show_number, len(ticks)*show_number)
plt.yticks([0.05 * temp for temp in range(10)])
plt.tight_layout()
plt.xlabel(u'高斯噪声的方差值', fontproperties=myfont)
plt.ylabel(u'MAPE值(x100%)', fontproperties=myfont)
plt.grid(True)
plt.savefig("results/boxcompare.png",dpi=300)
plt.show()


# -*- coding: utf-8 -*-

"""
Fusion d'un fichier au format pdf avec un ensemble de fichiers de même
format contenus dans un dossier passé en paramètre. Le contenu du premier
fichier est inséré une page sur deux dans le document final.

Il est nécessaire de fournir :
    - le nom du fichier de référence
    - le chemin du dossier contenant les fichiers pdf
    - le chemin du fichier à insérer.

Le résultat de la fusion_pdf est créé dans ce même dossier.
"""

from argparse import ArgumentParser
from PyPDF2 import PdfFileReader, PdfFileMerger, PdfFileWriter
import os
import sys

__author__ = "David Latreyte"
__date__ = "11/09/2015"

def split_pdf(path_to_pdf):
    """
    Création, à partir d'un fichier pdf contenant plusieurs pages, de fichiers
    pdf contenus dans un dossier de même nom que le fichier pdf.
    Chaque fichier créé correspond à une page du fichier original.

    Argument
    --------
    path_to_pdf: string
        Chemin (absolu ou relatif) du fichier à traiter.

    Retour
    ------
    chemin_vers_dossier: string
        Chemin (absolu ou relatif) vers le dossier contenant les pdf créés.
    """

    print("Début de la dissociation du pdf")

    # Récupération du nom complet du fichier (nom + l'extension)
    pdf_complete_filename = os.path.split(path_to_pdf)[1]

    # Si le fichier n'est pas au format pdf on quitte sinon on récupère le nom
    try:
        (pdf_filename, file_extension) = pdf_complete_filename.split(".")
    except ValueError:
        print("Le fichier ne possède pas d'extension.")
    # Vérification qu'il s'agit bien d'un fichier pdf
    if file_extension != "pdf":
        print("Le fichier n'est pas au format pdf.")
        sys.exit(0)

    # Création d'un répertoire du même nom que le fichier
    print("Création du dossier temporaire", end="...")
    path_to_dir = os.path.join(".", pdf_filename)
    os.mkdir(path_to_dir)
    print("Fait.", end="\n")

    # Buffer de lecture du pdf à traiter
    print("Création des fichiers pdf intermédiaires.", end="...")
    pdf_buffer_in = PdfFileReader(open(path_to_pdf, "rb"))

    for page_number in range(pdf_buffer_in.numPages):
        # Préparation du fichier de sortie
        pdf_objet = PdfFileWriter()
        pdf_objet.addPage(pdf_buffer_in.getPage(page_number))

        # Nom du nouveau pdf
        if page_number < 10:
            new_filename = "page-0{}.pdf".format(page_number)
        else:
            new_filename = "page-{}.pdf".format(page_number)

        # Création du fichier final
        with open(new_filename, "wb") as pdf_buffer_out:
            pdf_objet.write(pdf_buffer_out)
            pdf_buffer_out.close()

        # On déplace le fichier dans le dossier final
        final_path_to_new_pdf = os.path.join(path_to_dir, new_filename)
        actual_path_to_new_pdf = os.path.join(".", new_filename)
        os.rename(actual_path_to_new_pdf, final_path_to_new_pdf)
    print("Fait.", end="\n")

    # Retour du chemin vers le dossier contenant les pdf
    return path_to_dir

def fusion_pdf(path_bip, path_to_dir):

    print("Début de la fusion des fichiers...")

    # Détermination du nom du fichier de sortie
    pdf_out_name = os.path.split(path_to_dir)[1] + ".pdf"

    # Création du buffer
    merger = PdfFileMerger()

    # Création de la liste des noms des fichiers pdf
    pdf_files = (file for file in os.listdir(path_to_dir))

    # Traitement
    for file in pdf_files:
        # Ajout du nom du fichier à insérer (pages impaires)
        merger.append(PdfFileReader(path_bip, "rb"))

        # Ajout du nom d'un pdf issu de la liste
        merger.append(PdfFileReader(os.path.join(path_to_dir, file), "rb"))

    # Fin du traitement
    print("Fichiers tous traités...")
    print("Écriture du fichier final...")

    # Écriture du fichier
    merger.write(os.path.join(path_to_dir, pdf_out_name))

if __name__ == "__main__":
    # Traitement des arguments
    parser = ArgumentParser()

    parser.add_argument("-b", "--bip",
                        dest="bip_file",
                        default="BIP.pdf",
                        help="Chemin du fichier contenant le BIP.")
    parser.add_argument("-d", "--divisions",
                        dest="div_in",
                        default="./divisions",
                        help="Chemin du dossier contenant les fichiers.")
    args = parser.parse_args()

    # Traitement pour chaque fichier contenu dans le dossier passé en argument
    for file in os.listdir(args.div_in):
        print("")
        print("Traitement du fichier {}".format(file))
        print("")
        # Création du chemin vers le fichier
        path_to_file = os.path.join(args.div_in, file)
        # Appel de la fonction qui effectue le pré-traitement
        directory = split_pdf(path_to_file)

        # Appel de la fonction qui effectue le traitement
        fusion_pdf(args.bip_file, directory)
        print("")
        print("Fini pour {}".format(file))
        print("")
        print("############################################################")


from __future__ import print_function, absolute_import
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms

import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.use('Agg')
import time

from config import *

def init_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            nn.init.kaiming_normal_(m.weight, mode='fan_out')
            if m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.BatchNorm2d):
            nn.init.constant_(m.weight, 1)
            nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.Linear):
            nn.init.normal_(m.weight, std=1e-3)
            if m.bias is not None:
                nn.init.constant_(m.bias, 0)

def get_mean_and_std(dataset):
    '''Compute the mean and std value of dataset.'''
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=2)
    mean = torch.zeros(3)
    std = torch.zeros(3)
    print('==> Computing mean and std..')
    for inputs, targets in dataloader:
        for i in range(3):
            mean[i] += inputs[:,i,:,:].mean()
            std[i] += inputs[:,i,:,:].std()
    mean.div_(len(dataset))
    std.div_(len(dataset))
    return mean, std

def count_parameters(net, all=True):
    # If all= Flase, we only return the trainable parameters; tested
    return sum(p.numel() for p in net.parameters() if p.requires_grad or all)

def calculate_acc(dataloader, net, device):
    with torch.no_grad():
        correct = 0
        total = 0
        for data in dataloader:
            images, labels = data
            images = images.to(device)
            labels = labels.to(device)

            outputs = net(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    return (correct / total) * 100

# INPUTS: output have shape of [batch_size, category_count]
#    and target in the shape of [batch_size] * there is only one true class for each sample
# topk is tuple of classes to be included in the precision
# topk have to a tuple so if you are giving one number, do not forget the comma
def accuracy(output, target, topk=(1,5)):
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)
        _, pred = torch.topk(input=output, k=maxk, dim=1, largest=True, sorted=True)
        pred = pred.t()
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul(100.0/batch_size))
        return res

def get_network(network, dataset, device):

    # ResNet18 and Related Work
    if network == 'cc_resnet18':
        if dataset == 'cifar100':
            from cifar.cc_resnet import CC_ResNet18
        else:
            from imagenet.cc_resnet import CC_ResNet18
        net = CC_ResNet18()
    elif network == 'dy_resnet18':
        if dataset == 'cifar100':
            from cifar.dy_resnet import Dy_ResNet18
        else:
            from imagenet.dy_resnet import Dy_ResNet18
        net = Dy_ResNet18()
    elif network == 'wn_resnet18':
        if dataset == 'cifar100':
            from cifar.wn_resnet import WN_ResNet18
        else:
            from imagenet.wn_resnet import WN_ResNet18
        net = WN_ResNet18()
    elif network == 'resnet18':
        if dataset == 'cifar100':
            from cifar.resnet import ResNet18
        else:
            from imagenet.resnet import ResNet18
        net = ResNet18()

    # Experiment
    elif network == 'gcwn_resnet18':
        if dataset == 'cifar100':
            from cifar.gcwn_resnet import GCWN_ResNet18
        else:
            from imagenet.gcwn_resnet import GCWN_ResNet18
        net = GCWN_ResNet18()
    elif network == 'gcwnnl_resnet18':
        if dataset == 'cifar100':
            from cifar.gcwn_resnet import GCWNNL_ResNet18
        else:
            from imagenet.gcwn_resnet import GCWNNL_ResNet18
        net = GCWNNL_ResNet18()
    elif network == 'nlcwn_resnet18':
        if dataset == 'cifar100':
            from cifar.nlcwn_resnet import NLCWN_ResNet18
        else:
            from imagenet.nlcwn_resnet import NLCWN_ResNet18
        net = NLCWN_ResNet18()
    elif network == 'dnlcwnp_resnet18':
        if dataset == 'cifar100':
            from cifar.dnlcwn_resnet import DNLCWN_ResNet18
        else:
            from imagenet.dnlcwn_resnet import DNLCWN_ResNet18
        net = DNLCWN_ResNet18(bn=False, gap_mode='prior')
    elif network == 'dnlcwnbnp_resnet18':
        if dataset == 'cifar100':
            from cifar.dnlcwn_resnet import DNLCWN_ResNet18
        else:
            from imagenet.dnlcwn_resnet import DNLCWN_ResNet18
        net = DNLCWN_ResNet18(bn=True, gap_mode='prior')
    elif network == 'dnlcwnl_resnet18':
        if dataset == 'cifar100':
            from cifar.dnlcwn_resnet import DNLCWN_ResNet18
        else:
            from imagenet.dnlcwn_resnet import DNLCWN_ResNet18
        net = DNLCWN_ResNet18(bn=False, gap_mode='later')
    elif network == 'dnlcwnbnl_resnet18':
        if dataset == 'cifar100':
            from cifar.dnlcwn_resnet import DNLCWN_ResNet18
        else:
            from imagenet.dnlcwn_resnet import DNLCWN_ResNet18
        net = DNLCWN_ResNet18(bn=True, gap_mode='later')

    # AlexNet and Related Work
    elif network == 'cc_alexnet':
        if dataset == 'cifar100':
            from cifar.cc_alexnet import CC_AlexNet
        else:
            from imagenet.cc_alexnet import CC_AlexNet
        net = CC_AlexNet()
    elif network == 'dy_alexnet':
        if dataset == 'cifar100':
            from cifar.dy_alexnet import Dy_AlexNet
        else:
            from imagenet.dy_alexnet import Dy_AlexNet
        net = Dy_AlexNet()
    elif network == 'wn_alexnet':
        if dataset == 'cifar100':
            from cifar.wn_alexnet import WN_AlexNet
        else:
            from imagenet.wn_alexnet import WN_AlexNet
        net = WN_AlexNet()
    elif network == 'alexnet':
        if dataset == 'cifar100':
            from cifar.alexnet import AlexNet
        else:
            from imagenet.alexnet import AlexNet
        net = AlexNet()

    # Experiment
    elif network == 'gcwn_alexnet':
        if dataset == 'cifar100':
            from cifar.gcwn_alexnet import GCWN_AlexNet
        else:
            from imagenet.gcwn_alexnet import GCWN_AlexNet
        net = GCWN_AlexNet()
    elif network == 'gcwnnl_alexnet':
        if dataset == 'cifar100':
            from cifar.gcwn_alexnet import GCWN_AlexNet
        else:
            from imagenet.gcwn_alexnet import GCWN_AlexNet
        net = GCWN_AlexNet(normalized=True)
    elif network == 'nlcwn_alexnet':
        if dataset == 'cifar100':
            from cifar.nlcwn_alexnet import NLCWN_AlexNet
        else:
            from imagenet.nlcwn_alexnet import NLCWN_AlexNet
        net = NLCWN_AlexNet()
    elif network == 'dnlcwnp_alexnet':
        if dataset == 'cifar100':
            from cifar.dnlcwn_alexnet import DNLCWN_AlexNet
        else:
            from imagenet.dnlcwn_alexnet import DNLCWN_AlexNet
        net = DNLCWN_AlexNet(bn=False, gap_mode='prior')
    elif network == 'dnlcwnbnp_alexnet':
        if dataset == 'cifar100':
            from cifar.dnlcwn_alexnet import DNLCWN_AlexNet
        else:
            from imagenet.dnlcwn_alexnet import DNLCWN_AlexNet
        net = DNLCWN_AlexNet(bn=True, gap_mode='prior')
    elif network == 'dnlcwnl_alexnet':
        if dataset == 'cifar100':
            from cifar.dnlcwn_alexnet import DNLCWN_AlexNet
        else:
            from imagenet.dnlcwn_alexnet import DNLCWN_AlexNet
        net = DNLCWN_AlexNet(bn=False, gap_mode='later')
    elif network == 'dnlcwnbnl_alexnet':
        if dataset == 'cifar100':
            from cifar.dnlcwn_alexnet import DNLCWN_AlexNet
        else:
            from imagenet.dnlcwn_alexnet import DNLCWN_AlexNet
        net = DNLCWN_AlexNet(bn=True, gap_mode='later')

    # MobileNetV2 and Related Work
    elif network == 'cc_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.cc_mobilenetv2 import CC_MobileNetV2
        else:
            from imagenet.cc_mobilenetv2 import CC_MobileNetV2
        net = CC_MobileNetV2()
    elif network == 'dy_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.dy_mobilenetv2 import Dy_MobileNetV2
        else:
            from imagenet.dy_mobilenetv2 import Dy_MobileNetV2
        net = Dy_MobileNetV2()
    elif network == 'wn_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.wn_mobilenetv2 import WN_MobileNetV2
        else:
            from imagenet.wn_mobilenetv2 import WN_MobileNetV2
        net = WN_MobileNetV2()
    elif network == 'mobilenetv2':
        if dataset == 'cifar100':
            from cifar.mobilenetv2 import MobileNetV2
        else:
            from imagenet.mobilenetv2 import MobileNetV2
        net = MobileNetV2()

    # Experiment
    elif network == 'gcwn_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.gcwn_mobilenetv2 import GCWN_MobileNetV2
        else:
            from imagenet.gcwn_mobilenetv2 import GCWN_MobileNetV2
        net = GCWN_MobileNetV2()
    elif network == 'gcwnnl_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.gcwn_mobilenetv2 import GCWN_MobileNetV2
        else:
            from imagenet.gcwn_mobilenetv2 import GCWN_MobileNetV2
        net = GCWN_MobileNetV2(normalized=True)
    elif network == 'nlcwn_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.nlcwn_mobilenetv2 import NLCWN_MobileNetV2
        else:
            from imagenet.nlcwn_mobilenetv2 import NLCWN_MobileNetV2
        net = NLCWN_MobileNetV2()
    elif network == 'dnlcwnp_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        else:
            from imagenet.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        net = DNLCWN_MobileNetV2(bn=False, gap_mode='prior')
    elif network == 'dnlcwnbnp_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        else:
            from imagenet.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        net = DNLCWN_MobileNetV2(bn=True, gap_mode='prior')
    elif network == 'dnlcwnl_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        else:
            from imagenet.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        net = DNLCWN_MobileNetV2(bn=False, gap_mode='later')
    elif network == 'dnlcwnbnl_mobilenetv2':
        if dataset == 'cifar100':
            from cifar.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        else:
            from imagenet.dnlcwn_mobilenetv2 import DNLCWN_MobileNetV2
        net = DNLCWN_MobileNetV2(bn=True, gap_mode='later')

    else:
        print('the network is not supported')
        sys.exit()

    net = net.to(device)

    return net

def get_dataloader(dataset, batch_size):
    if dataset == 'cifar100':
        train_transform = transforms.Compose(
            [transforms.RandomCrop(size=32, padding=4),
            transforms.RandomHorizontalFlip(p=0.5),
            transforms.ToTensor(),
            transforms.Normalize(CIFAR100_MEAN, CIFAR100_STD)
            ])

        test_transform = transforms.Compose(
            [transforms.ToTensor(),
            transforms.Normalize(CIFAR100_MEAN, CIFAR100_STD)
            ])

        trainset = torchvision.datasets.CIFAR100(root=DATA_ROOT, train=True, transform=train_transform, download=True)
        testset = torchvision.datasets.CIFAR100(root=DATA_ROOT, train=False, transform=test_transform, download=True)

    elif dataset == 'imagenet':
        train_transform = transforms.Compose(
            [transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD)
            ])

        test_transform = transforms.Compose(
            [transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize(IMAGENET_MEAN, IMAGENET_STD)
            ])

        trainset = torchvision.datasets.ImageNet(root=IMAGENET_DATA_DIR, split='train', transform=train_transform)
        testset = torchvision.datasets.ImageNet(root=IMAGENET_DATA_DIR, split='val', transform=test_transform)

    else:
        print('Dataset not supported yet...')
        sys.exit()

    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)
    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)

    return trainloader, testloader

def save_plot(train_losses, train_accuracy, val_losses, val_accuracy, args, time_stamp):
    x = np.array([x for x in range(1, args.epoch + 1)])
    y1 = np.array(train_losses)
    y2 = np.array(val_losses)

    y3 = np.array(train_accuracy)
    y4 = np.array(val_accuracy)

    fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1)

    ax1.plot(x, y1, label='train loss')
    ax1.plot(x, y2, label='val loss')
    ax1.legend()
    ax1.xaxis.set_visible(False)
    ax1.set_ylabel('losses')

    ax2.plot(x, y3, label='train acc')
    ax2.plot(x, y4, label='val acc')
    ax2.legend()
    ax2.set_xlabel('batches')
    ax2.set_ylabel('acc')

    plt.savefig('plots/{}-losses-{}-b{}-e{}-{}.png'.format(args.network, args.dataset, args.batch, args.epoch, time_stamp))