import re
import requests
from bs4 import BeautifulSoup
from celery import current_app
from celery.contrib import rdb
from decouple import config
from django.db import models
from django.utils import timezone

class Platform(models.Model):
    name = models.CharField(max_length=50)
    url = models.URLField(max_length=500, blank=True)

    @classmethod
    def create(cls, name, url):
        platform = cls(name=name, url=url)
        return platform

class RequestLog(models.Model):
    STATUS_NEW = 1
    STATUS_PENDING = 2
    STATUS_COM = 5
    STATUS_ERROR = 6
    STATUS_CAN = 98
    STATUS_DELETED = 99
    PROCESSING = [STATUS_NEW, STATUS_PENDING]
    SCRAPED = [STATUS_PENDING, STATUS_COM]
    TERMINATED = [STATUS_COM, STATUS_ERROR]

    platform = models.ForeignKey(Platform, on_delete=models.DO_NOTHING)
    url = models.URLField(max_length=500, blank=True)
    title = models.CharField(max_length=250, blank=True)
    pub_date = models.DateTimeField(blank=True)
    status = models.SmallIntegerField()

    @classmethod
    def create(cls, platform, title, url, pub_date, status):
        log = cls(platform=platform, title=title, url=url, pub_date=pub_date, status=status)
        return log

    def is_status(self, *status):
        is_status = any(current_status == self.status for current_status in status)
        return is_status

    def scrape_complete(self):
        page_requests = PageRequest.objects.filter(request=self)
        if page_requests:
            pages = [page_request.page for page_request in page_requests]
            scrape_complete = all(page.is_status(Page.COMPLETED) for page in pages)
            return scrape_complete
        return False

    def update_status(self):
        import time
        if self.is_status(RequestLog.STATUS_NEW):
            print("Checking pages - status: " + str(self.status))
            pages_scraped = False
            while not pages_scraped:
                pages_scraped = self.scrape_complete()
                if not pages_scraped:
                    print('Idle: ' + str(self.status))
                    time.sleep(60)
            print('PEND: ' + self.url)
            self.status = RequestLog.STATUS_PENDING
            self.save()
        elif self.is_status(RequestLog.STATUS_PENDING):
            print('placeholder')

    @staticmethod
    def create_helper_search(request_url, platform, force):
        platform_model = platform
        request_log_model, created = RequestLog.objects.get_or_create(url=request_url,
                                                                      platform=platform_model,
                                                                      defaults={'title': request_url,
                                                                                'pub_date': timezone.now(),
                                                                                'status': RequestLog.STATUS_NEW
                                                                                }
                                                                      )
        playlist_model, created = Playlist.objects.get_or_create(platform=platform,
                                                                 request=request_log_model,
                                                                 )
        if not request_log_model.title == playlist_model.name:
            playlist_model.name = request_log_model.title
            playlist_model.save(update_fields=['name'])
        from .tasks import PROCESS_REQUEST
        current_app.send_task(
            PROCESS_REQUEST,
            args=[request_log_model.pk, force]
        )
        return playlist_model

    def create_search_for_request(self, force):
        request_log_model = self
        request_url = str(request_log_model.url)
        platform = request_log_model.platform
        if force:
            request_log_model.status = RequestLog.STATUS_PENDING
        # if request_log_model.is_status(*RequestLog.PROCESSING):
        if request_log_model.is_status(RequestLog.STATUS_NEW):
            # if created or request_log_model.is_status(RequestLog.PROCESSING):
            print('NEW')
            _response = requests.get(request_url, headers=Page.HEADER_DEFAULT)
            request_html = BeautifulSoup(_response.content, 'html.parser')
            # title = request_html.find('title').get_text()
            title = PageMetaInfo.title_from_html(request_html)
            request_log_model.title = title
            request_log_model.save(update_fields=['title'])
            playlist = Playlist.objects.get(request=request_log_model)
            playlist.name = title
            playlist.save(update_fields=['name'])
            page_meta_list = list()
            if request_url.__contains__('Category') or request_url.__contains__('brand'):
                print('Factory Pages URL setting: ' + platform.name)
                page_meta_list = PageMetaInfo.parse_from_html(platform.name, request_html)
            else:
                print(platform.name + ' page: ' + request_url)
                page_title = PageMetaInfo.title_from_html(request_html)
                page_meta_list.append(PageMetaInfo.factory(title=page_title, url=request_url))
            # page_list = [PageRequest.factory(page_meta, request_log_model) for page_meta in page_meta_list]
            page_list = [Page.repository(page_meta) for page_meta in page_meta_list]
            for page in page_list:
                PageRequest.factory(page_model=page, request_log=request_log_model)

            print(page.url for page in page_list)
            # Page.objects.bulk_create(page_list)
            search_list = list()
            print("PAGES: " + str(len(page_list)))
            from .tasks import REQUEST_STATUS_UPDATE
            current_app.send_task(
                REQUEST_STATUS_UPDATE,
                args=[self.pk]
            )
            for page_model in page_list:
                page_track_list = Search.objects.filter(page=page_model.id)
                if page_track_list:
                    search_list = search_list + list(page_track_list)
                else:
                    mode = request_log_model.platform.name  # rewrite async
                    # name, track_list = Search.factory(mode, page_model)
                    from .tasks import SCRAPE_PAGE_URL
                    current_app.send_task(
                        SCRAPE_PAGE_URL,
                        args=[mode, page_model.url]
                    )
                    # search_list = search_list + track_list
            # request_log_model.status = RequestLog.STATUS_PENDING
            # request_log_model.save()
            # print('PEND: ' + request_url)

        #return request_log_model

class Page(models.Model):
    NEW = 1
    PENDING = 2
    SCRAPED = 5
    EMPTY = 6
    RESCRAPE = 7
    CANCELLED = 98
    DELETED = 99
    COMPLETED = [SCRAPED, EMPTY]

    HEADER_DEFAULT = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) '}
    # 'AppleWebKit/537.36 (KHTML, like Gecko) ' \
    #  'Chrome/75.0.3770.80 Safari/537.36'}
    HEADER_MIXESDB = HEADER_DEFAULT

    url = models.URLField(max_length=500)
    title = models.CharField(max_length=250, blank=True)
    created_datetime = models.DateTimeField(default=timezone.now)
    modified_datetime = models.DateTimeField(blank=True, default=timezone.now)
    status = models.SmallIntegerField()

    @classmethod
    def create(cls, url, title, created_datetime, modified_datetime):
        page = cls(url=url, title=title, created_datetime=created_datetime, modified_datetime=modified_datetime)
        return page

    @classmethod
    def repository(cls, page_meta):
        page, created = Page.objects.get_or_create(
            url=page_meta.url,
            title=page_meta.title,
            status=Page.NEW
        )
        return page

    def is_status(self, *status):
        # return self.status == status
        is_status = any(current_status == self.status for current_status in status)
        return is_status

    def bbc_extract_tracklist(self):
        url = self.url
        print("PAGE TITLE: " + str(self.title))
        bbc_sounds_page_html = requests.get(str(url))
        bbc_sounds_soup = BeautifulSoup(bbc_sounds_page_html.content, 'html.parser')
        tracklist_tag = 'sc-u-flex-grow sc-c-basic-tile__text'
        bbc_tracklist = bbc_sounds_soup.find_all('div', class_=tracklist_tag)
        name = lambda track: track.get('title')
        tracklist = [name(track) for track in bbc_tracklist]
        if len(tracklist) == 0:
            print("NO TRACKS FOUND: " + self.url)
            self.status = Page.EMPTY
        else:
            search_list = [Search.create(track, page=self) for track in tracklist]
            Search.objects.bulk_create(search_list)
            self.status = Page.SCRAPED
        # from .tasks import SEARCH_LIST_CREATE
        # current_app.send_task(
        #     SEARCH_LIST_CREATE,
        #     args=[self.pk, tracklist]
        # )
        from .tasks import SEARCH_FOR_TRACK
        current_app.send_task(
            SEARCH_FOR_TRACK,
            args=[self.pk]
        )
        self.save()
        return self.title, []

    def mixesdb_extract_tracklist(self):
        # TODO retrieve any cached tracks once page status field added to model:
        # TODO remove logic that uses the page model directly
        mixesdb_html_page = requests.get(self.url, headers=Page.HEADER_MIXESDB)
        mixesdb_soup = BeautifulSoup(mixesdb_html_page.content, 'html.parser')
        # Mixipy.update_page_title(page_model, mixesdb_soup)
        print("PAGE TITLE: " + str(self.title))
        tracklist_h2_tag = mixesdb_soup.find('h2', id='Tracklist')
        current_page_element = tracklist_h2_tag
        search_raw_list = []
        while current_page_element:
            ol_section_tag = current_page_element.find_next('ol')
            if ol_section_tag:
                li_track_tags = ol_section_tag.find_all('li')
                if li_track_tags:
                    search_raw_list = search_raw_list + li_track_tags
            current_page_element = ol_section_tag
        if len(search_raw_list) == 0:
            print("PAGE " + self.url + " using fallback method")
            current_div_list_tag = mixesdb_soup.find('div', class_='list')
            while current_div_list_tag:
                class_list = ['list-track', 'aff-api-done', 'list-track aff-done aff-api-done',
                              'list-track aff-api-undone-search aff-done aff-api-done']
                div_list_track_tag = current_div_list_tag.find_next('div', {'class': class_list})
                if div_list_track_tag:
                    if div_list_track_tag.find_previous('h2', id='comments'):
                        current_div_list_tag = False
                    else:
                        search_raw_list.append(div_list_track_tag)
                        current_div_list_tag = div_list_track_tag
                else:
                    current_div_list_tag = False
        else:
            print("PAGE " + self.url + " using primary method")
        if len(search_raw_list) == 0:
            print("NO TRACKS FOUND: " + self.url)
            self.status = Page.EMPTY
        else:
            # TODO convert to listcomp
            format = lambda s: re.sub(r'\[[^\]]*\]', '', s.get_text()).strip().lower()
            valid = lambda s: Page.valid_mixesdb_search(s)
            formatted_search_results = [format(search_html_tag) for search_html_tag in search_raw_list]
            tracklist = [search for search in formatted_search_results if valid(search)]
            if len(tracklist) > 0:
                self.status = Page.PENDING
                from .tasks import SEARCH_LIST_CREATE
                # current_app.send_task(
                #     SEARCH_LIST_CREATE,
                #     args=[self.pk, tracklist]
                # )
                search_list = [Search.create(keywords=track, page=self) for track in tracklist]
                Search.objects.bulk_create(search_list)
                self.status = Page.SCRAPED
            else:
                self.status = Page.EMPTY
        self.save(update_fields=['status'])
        from .tasks import SEARCH_FOR_TRACK
        current_app.send_task(
            SEARCH_FOR_TRACK,
            args=[self.pk]
        )
        return self.title, []

    @staticmethod
    def valid_mixesdb_search(search_str):
        filt = ['?', 'intro', 'unknown']
        if filt.__contains__(search_str):
            return False
        else:
            if search_str.__contains__('-'):
                return True
            else:
                return False

class Search(models.Model):
    keywords = models.CharField(max_length=200)
    page = models.ForeignKey(Page, on_delete=models.DO_NOTHING)

    @classmethod
    def create(cls, keywords, page):
        search = cls(keywords=keywords, page=page)
        return search

    @staticmethod
    def extract_tracklist(mode, page):
        print([mode, page.url])
        name, search_list = '', []
        if mode == 'BBC':
            name, search_list = page.bbc_extract_tracklist()
        elif mode == 'MIXESDB':
            name, search_list = page.mixesdb_extract_tracklist()
        return name, search_list

    @staticmethod
    def create_search_for_page(page, mode):
        page_track_list = Search.objects.filter(page=page.id)
        if page_track_list:
            print("EXISTING TRACKS")
            return list(page_track_list)
        else:
            name, track_list = Search.extract_tracklist(mode, page)
            page.save(update_fields=['status'])
            print("NEW TRACKS")
            return track_list

    def has_been_searched(self):
        pk = self.pk
        page_track_list = Search.objects.filter(keywords=self.keywords).exclude(pk=pk)
        if page_track_list:
            print("ATTEMPTING KEYWORD MATCH")
            for search in list(page_track_list):
                exist_tracks = Track.objects.filter(search=search)
                if exist_tracks:
                    return exist_tracks.first()
        return False

class Track(models.Model):
    uri = models.CharField(max_length=50)
    search = models.ForeignKey(Search, on_delete=models.DO_NOTHING)

    @classmethod
    def create(cls, uri, search):
        track = cls(uri=uri, search=search)
        return track

    @classmethod
    def factory(cls, uri, search_model):
        return Track.create(uri=uri, search=search_model)

    @staticmethod
    def get_bulk(search_list):
        track_list = []
        for search_item in search_list:
            existing_track = Track.objects.filter(search=search_item)
            if existing_track:
                track_list.append(existing_track.first())
        print('TRACKS FOUND: ' + str(len(track_list)) + '/' + str(len(search_list)))  # TODO MOVE
        return track_list

class PageRequest(models.Model):
    request = models.ForeignKey(RequestLog, on_delete=models.DO_NOTHING)
    page = models.ForeignKey(Page, on_delete=models.DO_NOTHING)

    @classmethod
    def create(cls, request, page):
        page_request = cls(request=request, page=page)
        return page_request

    @classmethod
    def factory(cls, page_model, request_log):
        page_request = PageRequest.objects.get_or_create(page=page_model,
                                                         request=request_log)
        return page_request

class Playlist(models.Model):
    request = models.ForeignKey(RequestLog, on_delete=models.DO_NOTHING)
    name = models.CharField(max_length=250)
    platform = models.ForeignKey(Platform, on_delete=models.DO_NOTHING)
    uri = models.CharField(max_length=50, blank=True)
    description = models.CharField(max_length=250, blank=True)

    @classmethod
    def create(cls, request, name, platform, uri=None, description=None):
        playlist = cls(uri, platform, request, name, description)
        return playlist

    @staticmethod
    def facade(request, playlist, force):
        # rdb.set_trace()
        request.create_search_for_request(force)
        print(request.url + " STATUS " + str(request.status))
        playlist.add_available_tracks(force=force)
        return playlist

    def add_available_tracks(self, force):
        playlist = self
        request = self.request
        if force and request.is_status(RequestLog.STATUS_COM):
            request.status = RequestLog.STATUS_PENDING
        if request.is_status(RequestLog.STATUS_COM):
            print("RETURNED COMPLETED EXISTING")
            return playlist
        elif request.is_status(RequestLog.STATUS_PENDING):
            print("REQUEST IS PENDING: " + str(request.status))
            # TODO ONLY 100 tracks can be added at a time - batch api calls so that we search and add in chunks of 100
            linked_pages = PageRequest.objects.filter(request_id=playlist.request)
            page_id_list = [page_request.page_id for page_request in linked_pages]
            page_models = [Page.objects.get(pk=page_id) for page_id in page_id_list]
            page_search = set([Search.objects.filter(page=page) for page in page_models if
                               page.is_status(Page.SCRAPED)])
            track_list = list(set().union(*page_search))
            track_batch = []
            initial_trackList = track_list[0:100]
            remaining_tracks = []
            if len(initial_trackList) >= 100:
                remaining_tracks = track_list[100:]
                print("MORE THAN 100 tracks!")
            else:
                print("TRACKS: " + str(len(initial_trackList)))
            tracklist = Track.get_bulk(initial_trackList)
            playlist_pool = SpotipyPlaylistPool.getInstance()
            # playlist_endpoint = Spotipy.playlist_modify()
            playlist_endpoint = playlist_pool.getResource()
            playlist_uri = playlist.uri
            if not playlist_uri:
                # TODO add a link the playlist as viewed in Mixipy to the Spotify playlist
                request.refresh_from_db()
                title = request.title
                play_list_obj = playlist_endpoint.create_spotify_playlist(title)
                playlist_uri = play_list_obj['uri']
                playlist.uri = playlist_uri
                playlist.save()
            if len(tracklist) > 0:
                ready_batch = [track.uri for track in tracklist]
                print('READY: ' + str(len(ready_batch)) + ' TRACK BATCH: ' + str(len(track_batch)))
                ready_batch = set(ready_batch)
                # TODO we can only have a few of these at a time, maybe need to move to singleton task
                playlist_endpoint.post_tracks_to_playlist(playlist_uri, ready_batch)
            playlist_pool.returnResource(playlist_endpoint)
            if len(remaining_tracks) > 0:
                async_tracklist = [search.pk for search in remaining_tracks]
                from .tasks import TRACKS_TO_PLAYLIST
                current_app.send_task(
                    TRACKS_TO_PLAYLIST,
                    args=(playlist_uri, async_tracklist)
                )
            request.status = RequestLog.STATUS_COM
            request.save()
            print("COM - Playlist: " + str(playlist.id))
            return playlist
        else:
            print('ERR - Request Status: ' + str(request.status))

    def tracks_to_playlist(self, search_list):
        track_list = [Search.objects.get(pk=search_pk) for search_pk in search_list]
        search_chunks = [track_list[x: x + 100]
                         for x
                         in range(0, len(track_list), 100)]
        search_chunks_iter = iter(search_chunks)
        complete = False
        pool = SpotipyPlaylistPool.getInstance()
        spotipy_playlist = pool.getResource()
        print(spotipy_playlist.scope)
        # spotipy_playlist = Spotipy.playlist_modify()
        track_batch = []
        uri_cache = set()
        while not complete:
            next_batch = track_batch
            search_chunk = None
            if len(next_batch) < 100:
                search_chunk = next(search_chunks_iter, None)
                if search_chunk:
                    # TODO we can do the search elsewhere
                    spotify_track_uri_list = Track.get_bulk(search_chunk)
                    next_batch = next_batch + spotify_track_uri_list
            if len(next_batch) >= 100:
                track_batch = next_batch[100:]
                next_batch = next_batch[0:100]
            elif len(next_batch) < 100:
                if search_chunk:
                    track_batch = next_batch
                    next_batch = None
                else:
                    complete = True
            if next_batch:
                batch = [track.uri for track in next_batch]
                ready_batch = set(batch).difference(uri_cache)
                # print(ready_batch)
                print('READY: ' + str(len(ready_batch)) + ' TRACK BATCH: ' + str(len(track_batch)))
                spotipy_playlist.post_tracks_to_playlist(self.uri, ready_batch)
                uri_cache = uri_cache.union(ready_batch)
        pool.returnResource(spotipy_playlist)

class PageMetaInfo:

    def __init__(self, title, url):
        self.title = title
        self.url = url

    def __str__(self):
        return self.title + ', ' + self.url

    @classmethod
    def factory(cls, title, url):
        return cls(title, url)

    @staticmethod
    def title_from_html(html):
        return html.find('title').get_text()

    @staticmethod
    def html_from_url(url):
        return BeautifulSoup(requests.get(url, headers=Page.HEADER_DEFAULT).content, 'html.parser')

    def html(self):
        return BeautifulSoup(requests.get(self.url, headers=Page.HEADER_DEFAULT).content, 'html.parser')

    @staticmethod
    def parse_from_html(mode, html):
        if mode == 'BBC':
            return PageMetaInfo.bbc_brand_extract_page_urls(html)
        elif mode == 'MIXESDB':
            return PageMetaInfo.category_extract_page_urls(html)
        else:
            return []

    @staticmethod
    def bbc_brand_extract_page_urls(bbc_sounds_html):
        print("brand_extract_page_urls")
        bbc_brand = bbc_sounds_html.find_all('article')
        bbc_url = 'https://www.bbc.co.uk'
        title_tag = "sc-c-metadata__secondary gel-long-primer gs-u-mt-"
        url_tag = "sc-c-playable-list-card__link sc-o-link sc-u-flex-grow"
        # desc_tag = "sc-c-metadata__synopsis gel-brevier gs-u-mt- gs-u-mb"
        page_title = lambda e: e.find('p', class_=title_tag).get_text()
        url = lambda e: bbc_url + e.find('a', class_=url_tag).get('href')
        factory = lambda f: PageMetaInfo.factory(page_title(f), url(f))
        page_urls = [factory(mix_link) for mix_link in bbc_brand]
        return page_urls

    @staticmethod
    def category_extract_page_urls(mixesdb_html):
        page_urls = []
        page_html = mixesdb_html
        mixesdb_url = 'https://www.mixesdb.com'
        while page_html:
            urls = PageMetaInfo.mixesdb_extract_page_url_helper(page_html)
            page_urls = page_urls + urls
            # Find next page to parse
            pagination_links = page_html.find('div', class_='listPagination')
            if pagination_links:
                next200_pagination_element = pagination_links.find('a', string='next 200')
                if next200_pagination_element:
                    next200_hyperlink = next200_pagination_element.get('href')
                    next200_url = mixesdb_url + next200_hyperlink
                    next200_request = requests.get(next200_url, headers=Page.HEADER_MIXESDB)
                    print(next200_url)
                    page_html = BeautifulSoup(next200_request.content, 'html.parser')
                else:
                    page_html = ''
            else:
                page_html = ''
        return page_urls

    @staticmethod
    def mixesdb_extract_page_url_helper(mixesdb_html):
        ul_cat_mixes_list_tag = mixesdb_html.find('ul', id='catMixesList')
        page_meta_list = []
        if ul_cat_mixes_list_tag:
            li_cat_mixes_list = ul_cat_mixes_list_tag.find_all('li')
            if li_cat_mixes_list:
                mixlink_list = [mix_link.find('a') for mix_link in li_cat_mixes_list]
                mix_url = 'https://www.mixesdb.com'
                factory = lambda m: PageMetaInfo.factory(title=m.get_text(), url=mix_url + m.get('href'))
                page_meta_list = [factory(mix) for mix in mixlink_list]
        else:
            print("NONE FOUND")
        return page_meta_list

class Spotipy:
    client_id = config('SPOTIFY_CLIENT_ID')
    client_secret = config('SPOTIFY_CLIENT_SECRET')
    redirect_uri = config('REDIRECT_URI')
    search_port = config('REDIRECT_URI_SEARCH')
    playlist_port = config('REDIRECT_URI_PLAYLIST')

    def __init__(self, scope, instance):
        self.scope = scope
        self.instance = instance

    @staticmethod
    def factory(scope):
        if scope == "user-library-read":
            return Spotipy.search_instance()
        elif scope == "playlist-modify-public":
            return Spotipy.playlist_modify()
        elif scope == "playlist-modify-no-cred":
            return Spotipy.playlist_modify_test()
        else:
            print('ERR: OPTION NEEDED')

    @staticmethod
    def create_instance(scope):
        from spotipy import Spotify, SpotifyOAuth
        from spotipy.oauth2 import SpotifyClientCredentials
        if scope == "user-library-read":
            client_credentials_manager = SpotifyClientCredentials(client_id=Spotipy.client_id,
                                                                  client_secret=Spotipy.client_secret)
            return Spotify(client_credentials_manager=client_credentials_manager)
        elif scope == "playlist-modify-public":
            return Spotify(auth_manager=SpotifyOAuth(client_id=Spotipy.client_id,
                                                     client_secret=Spotipy.client_secret,
                                                     redirect_uri=Spotipy.redirect_uri,
                                                     scope="playlist-modify-public"))
        elif scope == 'playlist-modify-second':
            return Spotify(auth_manager=SpotifyOAuth(client_id=Spotipy.client_id,
                                                     client_secret=Spotipy.client_secret,
                                                     redirect_uri=Spotipy.playlist_port,
                                                     scope="playlist-modify-public"))
        elif scope == "playlist-modify-no-cred":
            client_credentials_manager = SpotifyClientCredentials(client_id=Spotipy.client_id,
                                                                  client_secret=Spotipy.client_secret,
                                                                  )
            return Spotify(client_credentials_manager=client_credentials_manager)

    def reset(self):
        self.instance = Spotipy.create_instance(scope=self.scope)

    @classmethod
    def search_instance(cls):
        return cls("user-library-read", Spotipy.create_instance("user-library-read"))

    @classmethod
    def playlist_modify(cls):
        return cls("playlist-modify-public", Spotipy.create_instance("playlist-modify-public"))

    @classmethod
    def playlist_second(cls):
        return cls("playlist-modify-second", Spotipy.create_instance("playlist-modify-second"))

    @classmethod
    def playlist_modify_test(cls):
        return cls("playlist-modify-no-cred", Spotipy.create_instance("playlist-modify-no-cred"))

    # TODO investigate: creating the playlist is a user action, but maybe we can use cred-manage for adding?
    def create_spotify_playlist(self, title):
        user_id = self.instance.me()['id']
        instance = self.instance
        playlist_obj = instance.user_playlist_create(user_id, title)
        return playlist_obj

    def post_tracks_to_playlist(self, playlist_uri, tracks):
        self.instance.playlist_add_items(playlist_id=playlist_uri, items=tracks, position=None)

    # @staticmethod
    # def search_spotify_track(search, sp, limit=10):
    #     if search:
    #         kw = search.keywords.lower()
    #         results = sp.search(q=kw, limit=limit)
    #         # for idx, track in enumerate(results['tracks']['items']):
    #         #     if str(search.keywords.lower()).__contains__(track['artists'][0].get('name').lower()):
    #         #         uri = track['uri']
    #         #         return SpotifyTrack.factory(spotify_track=track, uri=uri)
    #         search_result = [track for idx, track in enumerate(results['tracks']['items'])]
    #         validate = lambda s: kw.__contains__(s['artists'][0].get('name').lower())
    #         factory = lambda t: SpotifyTrack.factory(spotify_track=t, uri=t['uri'])
    #         filtered_search = [factory(track) for track in search_result if validate(track)]
    #         return filtered_search
    #         # else:
    #         # TODO log near-matches
    #         #    print([search, track['artists'][0].get('name')])

    def create_tracks_for_page(self, search_list):
        db_track_list = []
        for search_item in search_list:
            spotify_track_matches = self.search_track(search_item, 1)
            spotify_track = next((track for track in spotify_track_matches), None)
            if spotify_track:
                uri = spotify_track.uri
                if uri:
                    track_model = Track.factory(uri=uri, search_model=search_item)
                    db_track_list.append(track_model)
        Track.objects.bulk_create(db_track_list)

    def search_track(self, search, limit=10):
        if search:
            kw = search.keywords.lower()
            results = self.instance.search(q=kw, limit=limit)
            search_result = [track for idx, track in enumerate(results['tracks']['items'])]
            validate = lambda s: kw.__contains__(s['artists'][0].get('name').lower())
            factory = lambda t: SpotifyTrack.factory(spotify_track=t, uri=t['uri'])
            filtered_search = [factory(track) for track in search_result if validate(track)]
            return filtered_search

class SpotipyPlaylistPool:
    from queue import Queue
    __instance = None
    __resources = Queue()
    __resources.put(Spotipy.playlist_modify())
    __resources.put(Spotipy.playlist_second())

    def __init__(self):
        if SpotipyPlaylistPool.__instance is not None:
            raise NotImplemented("singleton class")

    @staticmethod
    def getInstance():
        if SpotipyPlaylistPool.__instance is None:
            SpotipyPlaylistPool.__instance = SpotipyPlaylistPool()

        return SpotipyPlaylistPool.__instance

    def getResource(self):
        if not self.__resources.empty():
            print("Using existing resource")
            return self.__resources.get(True)
        else:
            print("Waiting for new resource indefinitely")

    def returnResource(self, resource):
        resource.reset()
        self.__resources.put(resource)

# class SpotipyQueue:
#     _q = None
#     o = None
#
#     def __init__(self, dQ, autoGet=False):
#         self._q = dQ
#
#         if autoGet:
#             self.o == self._q.get()
#
#     def __enter__(self):
#         if self.o == None:
#             self.o = self._q.get()
#             return self.o
#         else:
#             return self.o
#
#     def __exit__(self, type, value, traceback):
#         if self.o is None:
#             self._q.put(self.o)
#             self.o = None
#
#     def __del__(self):
#         if self.o is None:
#             self._q.put(self.o)
#             self.o = None

class SpotifyTrack:

    def __init__(self, spotify_track, uri):
        self.spotify_track = spotify_track
        self.uri = uri

    def __str__(self):
        return str({**{'uri': self.uri}, **self.spotify_track})

    @classmethod
    def factory(cls, spotify_track, uri):
        return cls(spotify_track, uri)

class SearchQuery(models.Model):
    key = models.CharField(max_length=200, primary_key=True)

{'time': 'time', 'PROCESS_REQUEST': 'tasks.PROCESS_REQUEST'}.STATUS_PENDING

{'time': 'time', 'PROCESS_REQUEST': 'tasks.PROCESS_REQUEST'}.STATUS_NEW

{'SEARCH_FOR_TRACK': 'tasks.SEARCH_FOR_TRACK', 'SEARCH_LIST_CREATE': 'tasks.SEARCH_LIST_CREATE'}.SCRAPED

{'SEARCH_FOR_TRACK': 'tasks.SEARCH_FOR_TRACK', 'SEARCH_LIST_CREATE': 'tasks.SEARCH_LIST_CREATE'}.EMPTY

{'time': 'time', 'PROCESS_REQUEST': 'tasks.PROCESS_REQUEST', 'REQUEST_STATUS_UPDATE': 'tasks.REQUEST_STATUS_UPDATE', 'SCRAPE_PAGE_URL': 'tasks.SCRAPE_PAGE_URL'}.STATUS_COM

{'time': 'time', 'PROCESS_REQUEST': 'tasks.PROCESS_REQUEST', 'REQUEST_STATUS_UPDATE': 'tasks.REQUEST_STATUS_UPDATE', 'SCRAPE_PAGE_URL': 'tasks.SCRAPE_PAGE_URL'}.STATUS_PENDING

{'SEARCH_FOR_TRACK': 'tasks.SEARCH_FOR_TRACK', 'SEARCH_LIST_CREATE': 'tasks.SEARCH_LIST_CREATE'}.objects.get

{'SEARCH_FOR_TRACK': 'tasks.SEARCH_FOR_TRACK', 'SEARCH_LIST_CREATE': 'tasks.SEARCH_LIST_CREATE'}.HEADER_DEFAULT

{'SEARCH_FOR_TRACK': 'tasks.SEARCH_FOR_TRACK', 'SEARCH_LIST_CREATE': 'tasks.SEARCH_LIST_CREATE'}.HEADER_MIXESDB

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.client_id

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.client_secret

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.redirect_uri

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.playlist_port

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.create_instance

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.playlist_modify

{'Spotify': 'spotipy.Spotify', 'SpotifyOAuth': 'spotipy.SpotifyOAuth', 'SpotifyClientCredentials': 'spotipy.oauth2.SpotifyClientCredentials'}.playlist_second


from django.test import Client, TestCase
from django.core.urlresolvers import reverse

from smyt.forms import FORM_MAP
from smyt.models import MODEL_MAP, get_field_type
from smyt.views import get_items

import json
from datetime import date

class SimpleTest(TestCase):
    fixtures = ['smyt_data.json']
    conv = lambda self, x: json.loads(x.decode('utf-8'))
    data_map = {
        'int': 12345,
        'char': 'Огурец солёный',
        'date': date.today().strftime('%Y-%m-%d')
    }

    def setUp(self):
        self.client = Client()

    def test_details(self):
        response = self.client.get(reverse('dashboard'))
        self.assertEqual(response.status_code, 200)

    def test_view_items(self):
        for k, v in MODEL_MAP.items():
            response = self.client.get(reverse('view-items', args=(k,)))
            json_data = self.conv(response.content)
            self.assertEqual(json_data['ok'], 1)
            self.assertEqual(json_data['model'], k)

    def test_add_item(self):
        for k, v in FORM_MAP.items():
            data = {}
            for f in v().visible_fields():
                data[f.name] = ''
            response = self.client.post(reverse('add-item', args=(k,)), data)
            json_data = self.conv(response.content)
            self.assertEqual(json_data['ok'], 1)
            self.assertEqual('errs' in json_data, True)

            for f in v().visible_fields():
                data[f.name] = self.data_map[get_field_type(k, f.name)]
            response = self.client.post(reverse('add-item', args=(k,)), data)
            json_data = self.conv(response.content)
            self.assertEqual(json_data['ok'], 1)
            for v in data.values():
                self.assertEqual(v in json_data['item'][1], True)

    def test_update_item(self):
        for k, v in FORM_MAP.items():
            obj = MODEL_MAP[k].objects.get(pk=1)
            for f in v().visible_fields():
                data = {
                    'fname': f.name,
                    f.name: ''
                }
                response = self.client.post(reverse('update-item', args=(k, obj.id)), data)
                json_data = self.conv(response.content)
                self.assertEqual(json_data['ok'], 1)
                self.assertEqual('errs' in json_data, True)

                data = {
                    'fname': f.name,
                    f.name: self.data_map[get_field_type(k, f.name)]
                }
                response = self.client.post(reverse('update-item', args=(k, obj.id)), data)
                json_data = self.conv(response.content)
                self.assertEqual(json_data['ok'], 1)
                self.assertEqual(json_data['value'], str(data[f.name]))


"""Contains the definition for the StorageHeartbeat class."""

import time
import cPickle as pickle
from socket import error as SocketError
from datetime import datetime
from threading import Thread, Event
from Queue import Empty

from shared import Message, configure_logging

class StorageHeartbeat(object):
    """The StorageHeartbeat runs on the parent process of the client instance.
       It acts as the master to the StorageConsumer and StorageMonitor
       instances.

       Communication of task status from all children is done
       via a multiprocessing.Queue.  Additionally, there is a multiprocessing.Pipe
       connected to each child process for sending HEARTBEAT and KILL requests.

       All status tasks received on the queue are pickled and sent to the server
       over a socket.  HEARTBEAT and KILL responses are aggregated, pickled,
       and sent to the server over the same socket.
    """

    #These constants could optionally be converted to configuration parameters

    # (seconds) HEARTBEAT requests to child process must be reponded to within
    # this amout of time.
    HEARTBEAT_RESPONSE_TIMEOUT = 3
    # (seconds) KILL requests to child process must be reponded to within
    # this amout of time.
    HEARTBEAT_KILL_TIMEOUT = 10

    def __init__(self, consumers, monitor, report_in, runtime,
                 poll_period, client_socket, log_level=None):
        """Initializes a StorageHeartbeat with:

            Args:
                consumers: An iterable of ProcessData objects for the
                    StorageConsumer processes.
                monitor: A ProcessData object for the StorageMonitor process.
                report_in: Queue containing all child processes status.
                runtime: The configured runtime of the client.
                poll_period: The interval (seconds) on which to send
                    HEARTBEAT requests to child processes and then forward
                    the results to the server.
                client_socket: A connected socket to the server.
                log_level: A string matching the logging level.
                    (e.g. DEBUG, INFO, WARNING)
        """
        self.consumers = consumers
        self.monitor = monitor
        self.report_in = report_in
        self.runtime = runtime
        self.poll_period = poll_period
        self.socket = client_socket
        self.log = configure_logging(log_level, 'Client') \
            if log_level is not None else None

    def _log_message_received(self, message):
        """A helper method to log a message received from a child process.

            Args:
                message: A Message object received from a child process.
        """
        self.log.info('Message received from {}_{}: {}'.format(message.name,
                                                               message.id,
                                                               repr(message)))

    def _log_message_sent(self, message, process):
        """A helper method to log a message sent to a child process.

            Args:
                message: A Message object sent to a child process.
                process: The Process to which the message was sent.
        """
        self.log.info('Message sent to {}_{}: {}'.format(process.name,
                                                         process.id,
                                                         repr(message)))

    def _send_message_to_server(self, message):
        """Pickle a message and send it to the server.

           Sets the kill Event should the socket no longer be open.

            Args:
                message: A Message object to be sent to the server.
        """
        # pickle the message into a string
        # Not secure but sufficient for our purposes
        ps_message = pickle.dumps(message, pickle.HIGHEST_PROTOCOL)

        # Add a simple prefix to our message indicating the length that the
        # server should expect.  This allows us to leave the socket open for
        # all messages on this connection.
        ps_message = ':::{}:::{}'.format(len(ps_message), ps_message)
        try:
            self.socket.sendall(ps_message)
        except SocketError:
            # Server socket seems to have gone away.  Abort!
            self.kill.set()

    def _poll_processes(self, message, timeout, response_type, wait_to_send=None):
        """This method sends a message to all children processes and collects
           the response messages from each.  Processes that fail to respond are
           aggregated as a list of tuple of (name, id). The process responses
           and failed response list are sent to the server.

           This method is used by _do_heartbeat and _kill_all to prevent code
           duplication.

            Args:
                message: A Message object to send to all children.
                timeout: Time (s) to wait for responses.
                response_type: The Message.type expected in the responses.
                wait_to_send: And event signal that we wait on before sending
                    the message to the server.
        """
        #Send the message to the monitor and all consumers
        self.monitor.pipe.send(message)
        self._log_message_sent(message, self.monitor.process)

        for consumer in self.consumers:
            consumer.pipe.send(message)
            self._log_message_sent(message, consumer.process)

        # Poll the monitor and consumers until we get all responses or until
        # we timeout.
        responses = []
        response_stop = time.time() + timeout

        while time.time() < response_stop and \
            len(responses) < len(self.consumers) + 1:

            if self.monitor.pipe.poll():
                response = self.monitor.pipe.recv()
                self._log_message_received(response)
                if response.type == response_type:
                    responses.append(response)

            for consumer in self.consumers:
                if consumer.pipe.poll():
                    response = consumer.pipe.recv()
                    self._log_message_received(response)
                    if response.type == response_type:
                        responses.append(response)

        missing_responses = set([])
        if len(responses) < len(self.consumers) + 1:
            # We must have timed out.  Check for missed responses
            responding_processes = set([(proc.name, proc.id) for proc in responses])
            processes = set([(consumer.process.name, consumer.process.id) for
                             consumer in self.consumers])
            processes.add((self.monitor.process.name, self.monitor.process.id))

            # Items in the processes set but not in the responding_processes set
            # (symmetric difference) are the missing responses
            missing_responses = processes - responding_processes

        if wait_to_send is not None:
            # Signal message queue to finish processing
            self.kill.set()
            # Wait until work is done
            wait_to_send.wait()

        message = Message(name='Heartbeat',
                          id=0,
                          date_time=datetime.now(),
                          type=response_type,
                          payload=(responses, missing_responses))
        self._send_message_to_server(message)
        self.log.info('Message sent to server: {}'.format(repr(message)))

    def _do_heartbeat(self):
        """Periodically send heartbeat requests to child processes and forward
           results to the server until:
            1. Runtime is elapsed
            2. Kill event gets set (socket was closed)
        """
        heartbeat_stop = time.time() + self.runtime

        # Stop running if the kill event is set or if we've been running
        # for specified runtime
        while time.time() < heartbeat_stop and not self.kill.is_set():
            poll_start_time = time.time()

            message = Message(name='Heartbeat',
                              id=0,
                              date_time=None,
                              type='HEARTBEAT',
                              payload=None)

            self._poll_processes(message=message,
                                 timeout=self.HEARTBEAT_RESPONSE_TIMEOUT,
                                 response_type='HEARTBEAT')

            # Subtract the elapsed time from the poll_period for
            # more accurate heartbeat intervals
            sleep_time = self.poll_period - (time.time() - poll_start_time)
            if sleep_time > 0:
                time.sleep(sleep_time)

    def _kill_all(self):
        """Send kill message to all child processes.

           This run after the client runtime has elapsed or the kill event
           has been set.
        """
        message = Message(name='Heartbeat',
                          id=0,
                          date_time=None,
                          type='KILL',
                          payload=None)

        # Wait to send STOP message to server until the message queue has been
        # fully processed.  This hopes to arrange this as the last message to
        # arrive at the server.
        self._poll_processes(message=message,
                             timeout=self.HEARTBEAT_KILL_TIMEOUT,
                             response_type='STOP',
                             wait_to_send=self.queue_empty)

    def _process_message_queue(self):
        """Process messages in the report_in queue.

           The self.kill Event signals this thread to process the
           queue until empty and then exit.

           This method runs as a separate background thread while
           _do_heartbeat and _kill_all run in the foreground.
        """
        while not self.kill.is_set():
            try:
                message = self.report_in.get(timeout=2)
            except Empty:
                continue

            self._log_message_received(message)
            self._send_message_to_server(message)

        # Kill event was set. Finish processing the remaining events in the queue
        while not self.report_in.empty():
            try:
                message = self.report_in.get(block=False)
            except Empty:
                break # We shouldn't ever get here

            self._log_message_received(message)
            self._send_message_to_server(message)

        # Signal that the queue has been fully processed
        self.queue_empty.set()

    def run(self):
        """This is the main entry point for StorageHeartbeat."""
        self.kill = Event() # This signals _process_message_queue to finish up

        # Event used to delay sending STOP message to server until all messages
        # in the queue have been handled
        self.queue_empty = Event()

        t = Thread(target=self._process_message_queue)
        t.start()

        # Let _do_heartbeat decide how long to run for
        self._do_heartbeat()
        # Then kill all child processes
        self._kill_all()

        # Finish processing remaining messages from child processes
        t.join()

        self.socket.close()


import argparse
import json
import logging
import sys
from datetime import datetime
from typing import Union
from urllib.parse import urlparse

import pandas as pd
import pyangbind.lib.pybindJSON as pybindJSON

from catalog_connector.clients.ngsi_ld import NGSILDAPI
from catalog_connector.clients.yang_catalog import YangCatalogAPI
from catalog_connector.models.ngsi_ld.catalog import Module, Submodule
from catalog_connector.models.yang import yang_catalog as binding

logger = logging.getLogger(__name__)

BATCH_SIZE = 20

def chunks(lst, n):
    """
    Yield successive n-sized chunks from lst.

    Reference: https://stackoverflow.com/questions/312443/
               how-do-you-split-a-list-into-evenly-sized-chunks
    """
    for i in range(0, len(lst), n):
        yield lst[i : i + n]

def deserialize_yang(data: dict) -> binding.yang_catalog:
    return pybindJSON.loads_ietf(data, binding, "yang_catalog")

def compute_module_properties(
    data: binding.yc_module_yang_catalog__catalog_modules_module,
) -> dict:
    properties_dict = {}
    if data.ietf.ietf_wg:
        properties_dict["ietfWg"] = {"value": data.ietf.ietf_wg}
    if data.schema:
        properties_dict["schema"] = {"value": data.schema}
    if data.generated_from:
        properties_dict["generatedFrom"] = {"value": data.generated_from}
    if data.maturity_level:
        properties_dict["maturityLevel"] = {"value": data.maturity_level}
    if data.document_name:
        properties_dict["documentName"] = {"value": data.document_name}
    if data.author_email:
        properties_dict["authorEmail"] = {"value": data.author_email}
    if data.reference:
        properties_dict["reference"] = {"value": data.reference}
    if data.module_classification:
        properties_dict["moduleClassification"] = {"value": data.module_classification}
    if data.compilation_status:
        properties_dict["compilationStatus"] = {"value": data.compilation_status}
    if data.compilation_result:
        properties_dict["compilationResult"] = {"value": data.compilation_result}
    if data.prefix:
        properties_dict["prefix"] = {"value": data.prefix}
    if data.yang_version:
        properties_dict["yangVersion"] = {"value": data.yang_version}
    if data.description:
        properties_dict["description"] = {"value": data.description}
    if data.contact:
        properties_dict["contact"] = {"value": data.contact}
    if data.belongs_to:
        properties_dict["belongsTo"] = {"value": data.belongs_to}
    if data.tree_type:
        properties_dict["treeType"] = {"value": data.tree_type}
    if data.yang_tree:
        properties_dict["yangTree"] = {"value": data.yang_tree}
    if data.expires:
        properties_dict["expires"] = {"value": data.expires}
    if data.expired:
        properties_dict["expired"] = {"value": data.expired}
    if data.semantic_version:
        properties_dict["semanticVersion"] = {"value": data.semantic_version}
    if data.derived_semantic_version:
        properties_dict["derivedSemanticVersion"] = {
            "value": data.derived_semantic_version
        }
    return properties_dict

def build_dep(
    df: pd.DataFrame,
    dep_name: str,
    dep_revision: str = None,
    dep_schema: str = None,
) -> dict:
    dependency_id = None
    # (A) revision value takes preference when identifying the module
    if dep_revision:
        filtered_modules = df[
            (df["name"] == dep_name) & (df["revision"] == str(dep_revision))
        ]
        # Catch ghost dependency
        if filtered_modules.empty:
            organization = "unknown"
            module_type = "module"
        else:
            dependency_module = filtered_modules.iloc[0]
            module_type = dependency_module["module-type"]
        # Generate dependency entity ID
        dependency_id = "urn:ngsi-ld:{0}:{1}:{2}".format(
            module_type.capitalize(), dep_name, dep_revision
        )
    # (B) use schema URL to identify the module
    if not dep_revision and dep_schema:
        filtered_modules = df[
            (df["name"] == dep_name) & (df["schema"] == str(dep_schema))
        ]
        revision = None
        # Catch ghost dependency
        if filtered_modules.empty:
            # Try to figure out revision date from filename
            revision = (
                urlparse(dep_schema).path.split("/")[-1].split("@")[-1].split(".")[0]
            )
            try:
                _ = datetime.strptime(revision, "%Y-%m-%d")
                filtered_modules = df[
                    (df["name"] == dep_name) & (df["revision"] == revision)
                ]
                # Catch ghost dependency
                if filtered_modules.empty:
                    module_type = "module"
                else:
                    dependency_module = filtered_modules.iloc[0]
                    organization = dependency_module["organization"]
                    module_type = dependency_module["module-type"]
            except ValueError:
                revision = "unknown"
                module_type = "module"
        # Module found in the database
        else:
            dependency_module = filtered_modules.iloc[0]
            revision = dependency_module["revision"]
            module_type = dependency_module["module-type"]
        # Generate dependency entity ID
        dependency_id = "urn:ngsi-ld:{0}:{1}:{2}".format(
            module_type.capitalize(), dep_name, revision
        )
    # (C) fallback mechanism: set to unknown
    if not dep_revision and not dep_schema:
        revision = "unknown"
        organization = "unknown"
        module_type = "module"
        # Generate dependency entity ID
        dependency_id = "urn:ngsi-ld:{0}:{1}:{2}".format(
            module_type.capitalize(), dep_name, revision
        )

    return {
        "object": dependency_id,
        "datasetId": dependency_id,
    }

def collect_deps(
    module_id: str,
    df: pd.DataFrame,
    yang_data: binding.yc_module_yang_catalog__catalog_modules_module,
) -> dict:
    # Compute deps
    deps = {}
    # Then this entity becomes a dependency of its dependents
    if yang_data.dependents.items():
        logger.info("Collecting dependents of {0}".format(module_id))
        deps["hasDependents"] = []
        for _, dependent in yang_data.dependents.iteritems():
            dependent_object = build_dep(
                df, dependent.name, dependent.revision, dependent.schema
            )
            deps["hasDependents"].append(dependent_object)

    # Then this entity becomes a dependent of its dependents
    if yang_data.dependencies.items():
        logger.info("Collecting dependencies of {0}".format(module_id))
        deps["hasDependencies"] = []
        for _, dependency in yang_data.dependencies.iteritems():
            dependency_object = build_dep(
                df, dependency.name, dependency.revision, dependency.schema
            )
            deps["hasDependencies"].append(dependency_object)

    return deps

def build_module_entity(
    df: pd.DataFrame, yang_data: binding.yc_module_yang_catalog__catalog_modules_module
) -> Union[Module, Submodule]:

    # Compute properties
    yang_module_id = "{0}:{1}".format(
        yang_data.name, yang_data.revision)
    properties = compute_module_properties(yang_data)
    if yang_data.module_type == "module":
        id = "urn:ngsi-ld:Module:{0}".format(yang_module_id)
        deps = collect_deps(id, df, yang_data)
        if deps:
            properties.update(deps)
        module = Module(
            id=id,
            name={"value": yang_data.name},
            revision={"value": yang_data.revision},
            organization={"value": yang_data.organization},
            namespace={"value": yang_data.namespace},
            **properties
        )
        return module
    # Submodule then
    else:
        id = "urn:ngsi-ld:Submodule:{0}".format(yang_module_id)
        deps = collect_deps(id, df, yang_data)
        if deps:
            properties.update(deps)
        submodule = Submodule(
            id=id,
            name={"value": yang_data.name},
            revision={"value": yang_data.revision},
            organization={"value": yang_data.organization},
            namespace={"value": yang_data.namespace},
            **properties
        )
        return submodule

def main(ngsi_ld_api: NGSILDAPI, local_catalog: bool):
    if local_catalog:
        f = open("data.json")
        catalog_data = json.load(f)
        f.close()
    else:
        # Init YANGCatalog API Client
        yangcatalog_api = YangCatalogAPI()
        logger.info("Collecting data from YANG Catalog...")
        catalog_data = yangcatalog_api.get_whole_catalog()
        logger.info("Loaded module data from YANG Catalog!")
    # Build pandas DataFrame from module list for fast queries
    df = pd.json_normalize(catalog_data["yang-catalog:catalog"]["modules"]["module"])
    # Build Python generator from module list for NGSI-LD transformation
    module_list_generator = chunks(
        catalog_data["yang-catalog:catalog"]["modules"]["module"], BATCH_SIZE
    )
    while True:
        try:
            module_list_batch = next(module_list_generator)
        except StopIteration:
            break
        yang_batch = {
            "yang-catalog:catalog": {"modules": {"module": module_list_batch}}
        }
        yc = deserialize_yang(yang_batch)
        batch_entities = []
        for _, yang_module in yc.catalog.modules.module.iteritems():
            module_entity = build_module_entity(df, yang_module)
            batch_entities.append(module_entity.dict(exclude_none=True, by_alias=True))
        # Send batch of entities
        ngsi_ld_api.batchEntityUpsert(batch_entities, "update")

    logger.info("Synchronization with YANG Catalog completed!")

if __name__ == "__main__":
    logging.basicConfig(
        stream=sys.stdout,
        level=logging.INFO,
        format="'%(asctime)s - %(name)s - %(levelname)s - %(message)s'",
    )
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--broker-uri",
        dest="broker_uri",
        default="http://localhost:9090",
        required=False,
        help="NGSI-LD Context Broker URI.",
    )
    parser.add_argument(
        "--context-catalog-uri",
        dest="context_catalog_uri",
        default="http://context-catalog:8080/context.jsonld",
        required=False,
        help="Context Catalog URI.",
    )
    parser.add_argument(
        "--local-catalog", dest="local_catalog", default=False, required=False
    )
    argv = sys.argv[1:]
    known_args, _ = parser.parse_known_args(argv)

    # Init NGSI-LD API Client
    ngsi_ld_api = NGSILDAPI(
        url=known_args.broker_uri, context=known_args.context_catalog_uri
    )
    main(ngsi_ld_api, known_args.local_catalog)


from data import IsItEnglishTrainDataset, Vocab, Tokenize, TokensToIDs, ToTensor, character_tokenize, RidAttributes
from data import Collate
from torchvision.transforms import Compose
from train import BinaryClassification
from model import CNNForCorruptionClassification
import pytorch_lightning as pl
from pytorch_lightning.callbacks import ModelCheckpoint
from torch.utils.data import DataLoader
import torch

TRAIN_PATH = 'data/train.json'
VALID_PATH = 'data/valid.json'

def train(bsz: int,
          lr: float
          ) -> None:
    vocab = Vocab.construct_from_json_dataset(TRAIN_PATH)
    transform = Compose([
        Tokenize(tokenize_fn = character_tokenize),
        TokensToIDs(vocab = vocab),
        ToTensor(),
        RidAttributes(['original_tokens', 'corrupted_tokens', 'original_ids', 'corrupted_ids'])
    ])
    train_ds = IsItEnglishTrainDataset.from_json(TRAIN_PATH, transform=transform)
    valid_ds = IsItEnglishTrainDataset.from_json(VALID_PATH, transform=transform)
    train_loader = DataLoader(train_ds,
                              batch_size=bsz,
                              shuffle=True,
                              collate_fn=Collate(vocab))
    valid_loader = DataLoader(valid_ds,
                              batch_size=bsz,
                              shuffle=True,
                              collate_fn=Collate(vocab))
    model = CNNForCorruptionClassification(vocab = vocab,
                                           embed_dim = 128,
                                           out_channels = 32,
                                           window_sizes = [4, 8, 16])
    pl_module = BinaryClassification(model=model,
                                     lr=lr)
    trainer = pl.Trainer(log_every_n_steps=1,
                         #gpus=1,
                         max_epochs=10,
                         auto_lr_find='lr',
                         callbacks=[
                             ModelCheckpoint(monitor='valid_acc',
                                                 mode='max')
                         ])
    trainer.fit(pl_module,
                 train_dataloaders=train_loader,
                 val_dataloaders=valid_loader)

if __name__ == '__main__':
    train(bsz=32, lr=1e-2)


import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Optional, Union

import tifffile

from imctools.data import Acquisition, Channel
from imctools.data.acquisitiondata import AcquisitionData

class OmeTiffParser:
    """Parser of MCD compatible .OME-TIFF files.

    Allows to get a single IMC acquisition from a single OME-TIFF file.
    """

    def __init__(self, filepath: Union[str, Path], slide_id: int = 0, channel_id_offset: int = 0):
        if isinstance(filepath, str):
            filepath = Path(filepath)
        self._filepath = filepath
        self._slide_id = slide_id
        self._channel_id_offset = channel_id_offset
        self._acquisition_data = self._parse_acquisition(filepath)

    @property
    def origin(self):
        return "ome.tiff"

    @property
    def filepath(self):
        return self._filepath

    def get_mcd_xml(self) -> Optional[str]:
        """Original (raw) metadata from MCD file in XML format."""
        return self._mcd_xml

    def get_acquisition_data(self):
        """Returns AcquisitionData object with binary image data"""
        return self._acquisition_data

    @staticmethod
    def extract_acquisition_id(filepath: Union[str, Path]):
        """Extract acquisition ID from source OME-TIFF filepath.

        Filename should end with a numeric symbol!

        Parameters
        ----------
        filepath
            Input OME-TIFF filepath
        """
        if isinstance(filepath, str):
            filepath = Path(filepath)
        return int(filepath.stem.rstrip("_ac.ome").split("_")[-1].lstrip("a"))

    def _parse_acquisition(self, filepath: Path):
        image_data, ome_xml = OmeTiffParser._read_file(filepath)
        image_name, channel_names, channel_labels, self._mcd_xml = OmeTiffParser._parse_ome_xml(ome_xml)

        max_x = image_data.shape[2]
        max_y = image_data.shape[1]

        # TODO: implement a proper signal type extraction
        signal_type = "Dual"

        # Extract acquisition id from OME-TIFF file name
        acquisition_id = OmeTiffParser.extract_acquisition_id(filepath)

        # Offset should be 0 as we already got rid of 'X', 'Y', 'Z' channels!
        acquisition = Acquisition(
            self._slide_id,
            acquisition_id,
            self.origin,
            str(filepath),
            max_x,
            max_y,
            signal_type=signal_type,
            description=image_name,
        )

        for i in range(len(channel_names)):
            channel = Channel(acquisition.id, self._channel_id_offset, i, channel_names[i], channel_labels[i])
            self._channel_id_offset += 1
            channel.acquisition = acquisition
            acquisition.channels[channel.id] = channel

        acquisition_data = AcquisitionData(acquisition, image_data)
        # Calculate channels intensity range
        for ch in acquisition.channels.values():
            img = acquisition_data.get_image_by_name(ch.name)
            ch.min_intensity = round(float(img.min()), 4)
            ch.max_intensity = round(float(img.max()), 4)

        return acquisition_data

    @staticmethod
    def _parse_ome_xml(xml: str):
        ome: ET.Element = ET.fromstring(xml)
        ns = "{" + ome.tag.split("}")[0].strip("{") + "}"

        img = ome.find(ns + "Image")
        if img is None:
            raise ValueError("OME-TIFF XML is corrupted")

        channels = img.findall(f"{ns}Pixels/{ns}Channel")
        chan_dict = {
            int(chan.attrib["ID"].split(":")[2]): (chan.attrib.get("Name", chan.attrib["Fluor"]), chan.attrib["Fluor"])
            for chan in channels
        }

        image_name = img.attrib["Name"]
        channel_names = [chan_dict[i][1] for i in range(len(channels))]
        channel_labels = [chan_dict[i][0] for i in range(len(channels))]

        original_metadata_value = ome.find(
            f"{ns}StructuredAnnotations/{ns}XMLAnnotation/{ns}Value/{ns}OriginalMetadata/{ns}Value"
        )
        mcd_xml = original_metadata_value.text if original_metadata_value else None

        return image_name, channel_names, channel_labels, mcd_xml

    @staticmethod
    def _read_file(filepath: Path):
        with tifffile.TiffFile(filepath) as tif:
            data = tif.asarray(out="memmap")
            try:
                ome_xml = tif.pages[0].tags["ImageDescription"].value
            except:
                ome_xml = tif.pages[0].tags["image_description"].value
            return data, ome_xml

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

if __name__ == "__main__":
    import timeit

    tic = timeit.default_timer()

    with OmeTiffParser(
        "/home/anton/Downloads/imc_folder_v2/20170905_Fluidigmworkshopfinal_SEAJa/20170905_Fluidigmworkshopfinal_SEAJa_s0_a0_ac.ome.tiff"
    ) as parser:
        ac = parser.get_acquisition_data()
        pass

    print(timeit.default_timer() - tic)


import numpy as np
import cv2
import sys
from time import time

import kcftracker

selectingObject = False
initTracking = False
onTracking = False

ix, iy, cx, cy = -1, -1, -1, -1
w, h = 0, 0

inteval = 1
duration = 0.01

# mouse callback function
def draw_boundingbox(event, x, y, flags, param):
	global selectingObject, initTracking, onTracking, ix, iy, cx,cy, w, h

	if event == cv2.EVENT_LBUTTONDOWN:
		selectingObject = True
		onTracking = False
		ix, iy = x, y
		cx, cy = x, y

	elif event == cv2.EVENT_MOUSEMOVE:
		cx, cy = x, y

	elif event == cv2.EVENT_LBUTTONUP:
		selectingObject = False
		if(abs(x-ix)>10 and abs(y-iy)>10):
			w, h = abs(x - ix), abs(y - iy)
			ix, iy = min(x, ix), min(y, iy)
			initTracking = True
		else:
			onTracking = False

	elif event == cv2.EVENT_RBUTTONDOWN:
		onTracking = False
		if(w>0):
			ix, iy = x-w/2, y-h/2
			initTracking = True

if __name__ == '__main__':
	flag = 0

	if(len(sys.argv) == 1):
		cap = cv2.VideoCapture(0)
		flag = 1
	elif(len(sys.argv) == 3):
		## setting external camera
		if (sys.argv[1].isdigit() and sys.argv[2] == 'cam'):
			cap = cv2.VideoCapture(sys.argv[1])
			flag = 1
		elif (sys.argv[2] == 'vid'):
			cap = cv2.VideoCapture(sys.argv[1])
			flag = 2
		elif (sys.argv[2] == 'seq'):
			cap = cv2.VideoCapture(sys.argv[1] + '%04d.jpg', cv2.CAP_IMAGES)
			flag = 2
			inteval = 30
	else:
		assert(0), "Please check help for description"

	tracker = kcftracker.KCFTracker(True, True, True)
	# hog, fixed_window, multiscale

	cv2.namedWindow('Tracking')
	cv2.setMouseCallback('Tracking',draw_boundingbox)

	# Reading the first frame in case of stored video
	# This ensures we can mark the BB with stability.
	ret, frame = cap.read()
	if not ret:
		print('Nothing was returned.')
		exit(0)
	firstFrame = frame.copy() # Copy to restore each frame after BB creation
	while (not initTracking and flag == 2):
		if(selectingObject):
			cv2.rectangle(frame, (ix,iy), (cx,cy), (0,255,255), 1)
		cv2.imshow('Tracking', frame)
		cv2.waitKey(inteval)
		frame = firstFrame.copy() # Duplicate into frame the non BB overlay frame

	# Reading the subsequent frames in case of both camera & stored video
	while(cap.isOpened()):
		ret, frame = cap.read()
		if not ret:
			break
		if flag == 1 and selectingObject:
		 	cv2.rectangle(frame,(ix,iy), (cx,cy), (0,255,255), 1)
		elif(initTracking):
			initbb = [ix, iy, w, h]
			cv2.rectangle(frame,(ix,iy), (ix+w,iy+h), (0,255,255), 2)
			tracker.init([ix,iy,w,h], frame)
			initTracking = False
			onTracking = True
		elif(onTracking):
			t0 = time()
			bb = tracker.update(frame)
			t1 = time()
			bb = map(int, bb)
			amplitude = np.sqrt((initbb[0] + initbb[2]/2 - bb[0] - bb[2]/2)**2 + (initbb[1] + initbb[3]/2 - bb[1] - bb[3]/2)**2)
			cv2.rectangle(frame,(bb[0],bb[1]), (bb[0]+bb[2],bb[1]+bb[3]), (0,255,255), 1)

			duration = 0.8*duration + 0.2*(t1-t0)
			#duration = t1-t0
			cv2.putText(frame, 'FPS: '+str(1/duration)[:4].strip('.'), (8,20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,0,255), 2)
			cv2.putText(frame, 'Amplitude: '+ str(amplitude), (8,40), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,0,255), 2)
		cv2.imshow('Tracking', frame)
		# Exit on key 'esc' or 'q'
		c = cv2.waitKey(inteval) & 0xFF
		if c==27 or c==ord('q'):
			break

	cap.release()
	cv2.destroyAllWindows()


import pytest

from CheckEnvs import check_env
from CheckEnvs import MissingEnvVar

from TransportQuery import TransportQuery

from DataModels import Route
from DataModels import Stop

from main import find_most_stops
from main import find_min_stops
from main import find_stop_intersection

# Speaking of environment variables, my tests don't specify defaults.
# I think best practice would be mocking those.

class TestCheckEnvs:
    """
    Test the Check Envs functionality
    """
    def test_check_env(self):
        # Do we return a value when present?
        assert check_env('BASE_REQUEST_URL') is not None

        # Do we raise an error when not present?
        with pytest.raises(MissingEnvVar):
            check_env('FOO') is not None

class TestTransportQuery:
    def test_get_routes_by_type(self):
        new_transport_query = TransportQuery()
        # Does the function to generate a query return the right string?
        assert new_transport_query.generate_route_by_type_url([0,1]) == "routes?filter[type]=0,1"

        routes = new_transport_query.get_routes_by_type([0,1])
        # Does the function that queries the routes return the right results? This requires API access!
        assert [str(route) for route in routes] == ['Red Line', 'Mattapan Trolley', 'Orange Line', 'Green Line B', 'Green Line C', 'Green Line D', 'Green Line E', 'Blue Line']

    def test_generate_stop_by_routes(self):
        route = Route("Red", "Red Line")

        new_transport_query = TransportQuery()
        # Does the function to generate a query return the right string?
        assert new_transport_query.generate_stop_by_route_url(route) == "stops?fields[stop]=name&filter[route]=Red"

        stops = new_transport_query.get_stops_by_routes([route])
        # Does the query to get stops return the right count as a proxy for working function?
        assert len(stops["Red"]) == 22

class TestMain:

    route_stops = {Route("Red", "Red Line"): [Stop("1", "stop1"), Stop("2", "stop2"), Stop("3", "stop3")],
                        Route("Blue", "Blue Line"): [Stop("2", "stop2"), Stop("4", "stop4")],
                        Route("Green", "Green Line"): [Stop("5", "stop5")]}

    def test_max_stops(self):
        # Do we find the correct max stops route?
        assert find_most_stops(self.route_stops) == ("Red Line", 3)

    def test_min_stops(self):
        # Do we find the correct min stops route?
        assert find_min_stops(self.route_stops) == ("Green Line", 1)

    def test_find_stop_intersection(self):
        # Do we find the intersections?
        intersections = find_stop_intersection(self.route_stops)
        assert intersections[0][0] == "stop2"
        assert intersections[0][1][0].id == "Red"
        assert intersections[0][1][1].id == "Blue"


from flask import Flask
from flask_restful import Api, Resource, reqparse

app = Flask(__name__)
api = Api(app)

users = [
    {
        "name": "Nicholas",
        "age": 42,
        "occupation": "Network Engineer"
    },
    {
        "name": "Elvin",
        "age": 32,
        "occupation": "Doctor"
    },
    {
        "name": "Jass",
        "age": 22,
        "occupation": "Web Developer"
    }
]

class User(Resource):
    def get(self, name):
        for user in users:
            if(name == user["name"]):
                return user, 200
        return "User not found", 404

    def post(self, name):
        parser = reqparse.RequestParser()
        parser.add_argument("age")
        parser.add_argument("occupation")
        args = parser.parse_args()

        for user in users:
            if(name == user["name"]):
                return "User with name {} already exists".format(name), 400

        user = {
            "name": name,
            "age": args["age"],
            "occupation": args["occupation"]
        }
        users.append(user)
        return user, 201

    def put(self, name):
        parser = reqparse.RequestParser()
        parser.add_argument("age")
        parser.add_argument("occupation")
        args = parser.parse_args()

        for user in users:
            if(name == user["name"]):
                user["age"] = args["age"]
                user["occupation"] = args["occupation"]
                return user, 200

        user = {
            "name": name,
            "age": args["age"],
            "occupation": args["occupation"]
        }
        users.append(user)
        return user, 201

    def delete(self, name):
        global users
        users = [user for user in users if user["name"] != name]
        return "{} is deleted.".format(name), 200

api.add_resource(User, "/user/<string:name>")

app.run(debug=True)

#2. Accessing APIs using Clients
#    a. Using client Insomnia
#    - Install "Insomnia" client application to send RestAPI calls in chrome. Access it using (chrome://apps/)
#        GET: 127.0.0.1:5000/user/Elvin
#        POST: 127.0.0.1:5000/user/Amit
#        DELETE: 127.0.0.1:5000/user/Amit
#    b. Using firefox web Browser:        http://127.0.0.1:5000/user/Elvin
#    c. Using Python Code    https://www.geeksforgeeks.org/get-post-requests-using-python/


import math
from matplotlib import pyplot as plt

#This function causes the signal to be decimated by the block size
def block_average(buffer, block_size):
    block_index = 0
    output_buffer = []

    while(block_index < len(buffer)):
        stretch_factor = 0
        sum = 0
        for i in range(block_size):
            if(block_index + block_size >= len(buffer)):
                stretch_factor += 1
                break
            else:
                sum += buffer[block_index + block_size]
                stretch_factor += 1

        #We need to stretch the signal so that it matches the time scale of the input
        #signal
        output_buffer += [sum/len(buffer)] * stretch_factor
        block_index += block_size

    print(output_buffer)
    return output_buffer

##############################################
# Control values for the test signal
##############################################
signal_freq = 60 #Hz
signal_freq_rad = (signal_freq) * 2 * math.pi #60Hz in radians/second
phase = math.pi #seconds
dc_offset = 0.0
amplitude = 2.0

#Used to record signal values
buffer = []
#Used to record sampling time indices
time_buffer = []

#The frequency at which the signal is sampled
sampling_freq = 240 #Hz
sampling_freq_rad = sampling_freq * 2 * math.pi
#Amount of time to capture for in seconds
capture_time = .1 #second

#Use the sampling frequence and amount of time beign captured for
#to determine how long the buffer needs to be
buffer_len = math.ceil(sampling_freq_rad * capture_time)
print(buffer_len)

sample_time = 0
for i in range(buffer_len):
    buffer.append( amplitude * math.sin(signal_freq_rad * sample_time + phase) + dc_offset )
    sample_time += 1/sampling_freq_rad
    time_buffer.append(sample_time)

# plt.scatter(time_buffer, buffer)
plt.plot(time_buffer, buffer)
plt.plot(time_buffer, block_average(buffer, 15))
plt.show()


'''
Create a mask image from the spatial coordinates.
'''
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

## _initialize_mask
def CreateMaskImage(adata, scale_factor=1):
    adata_copy = adata.copy()
    A = 'spatial' in adata_copy.uns.keys()
    B = 'array_col' in adata_copy.obs.keys()
    C = 'array_row' in adata_copy.obs.keys()
    if A and B and C:
        coordinates = adata_copy.obs[['array_row', 'array_col']]
    else:
        coordinates = pd.DataFrame(adata_copy.obsm['spatial'], index=adata_copy.obs.index,
                                   columns=['array_col', 'array_row'])

    if (coordinates.max()>1000).any():
        coordinates = (coordinates/scale_factor).astype(int)

    evencol = coordinates[coordinates['array_col'] % 2 == 0]['array_row']
    oodcol = coordinates[coordinates['array_col'] % 2 != 0]['array_row']
    aligned = len(set(evencol).intersection(oodcol)) != 0
    coordinates = coordinates - coordinates.min()

    if aligned:
        nrow = int(coordinates['array_row'].max()) + 1
        ncol = int(coordinates['array_col'].max()) + 1
        Mask = np.zeros((nrow, ncol))
        spotID = []
        for i in range(coordinates.shape[0]):
            Mask[coordinates['array_row'][i], coordinates['array_col'][i]] = 1
            spotID.append(coordinates['array_row'][i]*ncol+coordinates['array_col'][i])
    else:
        nrow = coordinates['array_row'].max() + 1
        ncol = int((coordinates['array_col'].max() + 1)/2) + 1
        Mask = np.zeros((nrow, ncol))
        spotID = []
        for i in range(coordinates.shape[0]):
            Mask[coordinates['array_row'][i], coordinates['array_col'][i] // 2] = 1
            spotID.append(coordinates['array_row'][i]*ncol+(coordinates['array_col'][i] // 2))

    adata_copy.uns['Mask'] = Mask
    coordinates['spotID'] = spotID
    coordinates = coordinates.loc[adata_copy.obs.index,:]
    adata_copy.obs[['array_col', 'array_row', 'spotID']] = coordinates
    plt.imshow(adata_copy.uns['Mask'], cmap="binary")
    plt.xticks([])
    plt.yticks([])
    plt.title('Mask Image',fontsize=20)
    plt.show()
    return adata_copy


import contextlib, logging, math, sys
from OpenGL import GL as gl
import glfw

log = logging.getLogger(__name__)

@contextlib.contextmanager
def create_main_window():
    if not glfw.init():
        log.error('failed to initialize GLFW')
        sys.exit(1)
    try:
        log.debug('requiring modern OpenGL without any legacy features')
        glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 3)
        glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 3)
        glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True)
        glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_CORE_PROFILE)

        log.debug('opening window')
        title = 'Tutorial 1: Window'
        window = glfw.create_window(500, 400, title, None, None)
        if not window:
            log.error('failed to open GLFW window.')
            sys.exit(2)
        glfw.make_context_current(window)

        log.debug('set background to dark blue')
        gl.glClearColor(0, 0, 0.4, 0)

        yield window

    finally:
        glfw.terminate()

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    with create_main_window() as window:
        while (
            glfw.get_key(window, glfw.KEY_ESCAPE) != glfw.PRESS and
            not glfw.window_should_close(window)
        ):
            gl.glClear(gl.GL_COLOR_BUFFER_BIT)
            glfw.swap_buffers(window)
            glfw.poll_events()


# 요세푸스 문제

from collections import deque

n, k = map(int, input().split())
y = deque([i for i in range(1, n+1)])
ans = []

while y:
    for _ in range(k-1):
        y.append(y.popleft())
    ans.append(str(y.popleft()))

print('<' + ', '.join(ans[:]) + '>')


# -*- coding: utf-8 -*-
"""
Created on Wed Feb  1 01:02:25 2023

@author: Yunus
"""
# source: https://towardsdatascience.com/an-end-to-end-project-on-time-series-analysis-and-forecasting-with-python-4835e6bf050b

import warnings
import itertools
import numpy as np
import matplotlib.pyplot as plt

warnings.filterwarnings("ignore")
plt.style.use('fivethirtyeight')

import pandas as pd
import statsmodels.api as sm
import matplotlib

matplotlib.rcParams['axes.labelsize'] = 14
matplotlib.rcParams['xtick.labelsize'] = 12
matplotlib.rcParams['ytick.labelsize'] = 12
matplotlib.rcParams['text.color'] = 'k'

# We are going to do time series analysis and forecasting for furniture sales.

df = pd.read_excel("Superstore.xls")
furniture = df.loc[df['Category'] == 'Furniture']

print (furniture['Order Date'].min())

print (furniture['Order Date'].max())

# Data preprocessing

cols = ['Row ID', 'Order ID', 'Ship Date', 'Ship Mode', 'Customer ID', 'Customer Name', 'Segment', 'Country', 'City', 'State', 'Postal Code', 'Region', 'Product ID', 'Category', 'Sub-Category', 'Product Name', 'Quantity', 'Discount', 'Profit']

# removing columns we do not need
furniture.drop(cols, axis=1, inplace=True)

# sort sales by date
furniture = furniture.sort_values('Order Date')

# check missing values
furniture.isnull().sum()

# group sales by date
furniture = furniture.groupby('Order Date')['Sales'].sum().reset_index()

# check the last version of the data
print (furniture.head())

# Indexing with time series data
furniture = furniture.set_index('Order Date')
furniture.index

# datetime data is tricky to work with
# So, we will use the averages daily sales for that month
y = furniture['Sales'].resample('MS').mean()

# check 2017 sales data
print (y['2017':])

# Visualizing Furniture Sales Time Series Data
y.plot(figsize=(15, 6))
plt.show()

"""
Some distinguishable patterns appear when we plot the data.
The time-series has seasonality pattern;
sales are always low at the beginning of the year and high at the end of the year.
There is always a strong upward trend within any single year with a couple of low months in the mid of the year.
"""

# We can also visualize our data using a method called time-series decomposition
# that allows us to decompose our time series into three distinct components:
# trend, seasonality, and noise.

from pylab import rcParams
rcParams['figure.figsize'] = 18, 8
decomposition = sm.tsa.seasonal_decompose(y, model='additive')
fig = decomposition.plot()
plt.show()

"""
The plot clearly shows that the sales of furniture is unstable, along with its obvious seasonality.
"""

# Time series forecasting with ARIMA
# ARIMA, which stands for Autoregressive Integrated Moving Average.

# Parameter Selection for the ARIMA Time Series Model
p = d = q = range(0, 2)
pdq = list(itertools.product(p, d, q))
seasonal_pdq = [(x[0], x[1], x[2], 12) for x in list(itertools.product(p, d, q))]
print('Examples of parameter combinations for Seasonal ARIMA...')
print('SARIMAX: {} x {}'.format(pdq[1], seasonal_pdq[1]))
print('SARIMAX: {} x {}'.format(pdq[1], seasonal_pdq[2]))
print('SARIMAX: {} x {}'.format(pdq[2], seasonal_pdq[3]))
print('SARIMAX: {} x {}'.format(pdq[2], seasonal_pdq[4]))

for param in pdq:
    for param_seasonal in seasonal_pdq:
        try:
            mod = sm.tsa.statespace.SARIMAX(y, order=param, seasonal_order=param_seasonal, enforce_stationarity=False, enforce_invertibility=False)
            results = mod.fit()
            print('ARIMA{}x{}12 - AIC:{}'.format(param, param_seasonal, results.aic))
        except:
            continue

# Fitting the ARIMA model
mod = sm.tsa.statespace.SARIMAX(y,
                                order=(1, 1, 1),
                                seasonal_order=(1, 1, 0, 12),
                                #enforce_stationarity=False,
                                enforce_invertibility=False)
results = mod.fit()
print(results.summary().tables[1])

# run model diagnostics to investigate any unusual behavior
results.plot_diagnostics(figsize=(16, 8))
plt.show()

# Validating forecasts
"""
To help us understand the accuracy of our forecasts,
we compare predicted sales to real sales of the time series,
and we set forecasts to start at 2017-07-01 to the end of the data.
"""

pred = results.get_prediction(start=pd.to_datetime('2017-01-01'), dynamic=False)
pred_ci = pred.conf_int()

ax = y['2014':].plot(label='observed')
pred.predicted_mean.plot(ax=ax, label='One-step ahead Forecast', alpha=.7, figsize=(14, 7))

ax.fill_between(pred_ci.index,
                pred_ci.iloc[:, 0],
                pred_ci.iloc[:, 1], color='k', alpha=.2)

ax.set_xlabel('Date')
ax.set_ylabel('Furniture Sales')
plt.legend()
plt.show()

"""
The line plot is showing the observed values compared to the rolling forecast predictions.
Overall, our forecasts align with the true values very well, showing an upward trend starts from the beginning of the year.
"""

y_forecasted = pred.predicted_mean
y_truth = y['2017-01-01':]

# Compute the mean square error
mse = ((y_forecasted - y_truth) ** 2).mean()
print('The Mean Squared Error of our forecasts is {}'.format(round(mse, 2)))
# The Mean Squared Error of our forecasts is 22993.58

print('The Root Mean Squared Error of our forecasts is {}'.format(round(np.sqrt(mse), 2)))
# The Root Mean Squared Error of our forecasts is 151.64

"""
In statistics, the mean squared error (MSE) of an estimator measures the average of the squares of the errors —
that is, the average squared difference between the estimated values and what is estimated.
The MSE is a measure of the quality of an estimator—it is always non-negative,
and the smaller the MSE, the closer we are to finding the line of best fit.

Root Mean Square Error (RMSE) tells us that our model was able to forecast the average daily furniture sales in the test set within 151.64 of the real sales.
Our furniture daily sales range from around 400 to over 1200. In my opinion, this is a pretty good model so far.
"""

# Producing and visualizing forecasts
pred_uc = results.get_forecast(steps=100)
pred_ci = pred_uc.conf_int()

ax = y.plot(label='observed', figsize=(14, 7))
pred_uc.predicted_mean.plot(ax=ax, label='Forecast')
ax.fill_between(pred_ci.index,
                pred_ci.iloc[:, 0],
                pred_ci.iloc[:, 1], color='k', alpha=.25)
ax.set_xlabel('Date')
ax.set_ylabel('Furniture Sales')

plt.legend()
plt.show()

"""
Our model clearly captured furniture sales seasonality.
As we forecast further out into the future, it is natural for us to become less confident in our values.
This is reflected by the confidence intervals generated by our model, which grow larger as we move further out into the future.
"""

# Time Series of Furniture vs. Office Supplies
# Data preprocessing
furniture = df.loc[df['Category'] == 'Furniture']
office = df.loc[df['Category'] == 'Office Supplies']

furniture.shape, office.shape

cols = ['Row ID', 'Order ID', 'Ship Date', 'Ship Mode', 'Customer ID', 'Customer Name', 'Segment', 'Country', 'City', 'State', 'Postal Code', 'Region', 'Product ID', 'Category', 'Sub-Category', 'Product Name', 'Quantity', 'Discount', 'Profit']
furniture.drop(cols, axis=1, inplace=True)
office.drop(cols, axis=1, inplace=True)

furniture = furniture.sort_values('Order Date')
office = office.sort_values('Order Date')

furniture = furniture.groupby('Order Date')['Sales'].sum().reset_index()
office = office.groupby('Order Date')['Sales'].sum().reset_index()

# have a quick peek
furniture.head()
office.head()

# Data Exploration
furniture = furniture.set_index('Order Date')
office = office.set_index('Order Date')

y_furniture = furniture['Sales'].resample('MS').mean()
y_office = office['Sales'].resample('MS').mean()

furniture = pd.DataFrame({'Order Date':y_furniture.index, 'Sales':y_furniture.values})
office = pd.DataFrame({'Order Date': y_office.index, 'Sales': y_office.values})

store = furniture.merge(office, how='inner', on='Order Date')
store.rename(columns={'Sales_x': 'furniture_sales', 'Sales_y': 'office_sales'}, inplace=True)
store.head()

plt.figure(figsize=(20, 8))
plt.plot(store['Order Date'], store['furniture_sales'], 'b-', label = 'furniture')
plt.plot(store['Order Date'], store['office_sales'], 'r-', label = 'office supplies')
plt.xlabel('Date'); plt.ylabel('Sales'); plt.title('Sales of Furniture and Office Supplies')
plt.legend();

"""
We observe that sales of furniture and office supplies shared a similar seasonal pattern.
Early of the year is the off season for both of the two categories.
It seems summer time is quiet for office supplies too.
In addition, average daily sales for furniture are higher than those of office supplies in most of the months.
It is understandable, as the value of furniture should be much higher than those of office supplies.
Occationaly, office supplies passed furnitue on average daily sales.
"""

# Let's find out when was the first time office supplies' sales surpassed those of furniture's.

first_date = store.loc[np.min(list(np.where(store['office_sales'] > store['furniture_sales'])[0])), 'Order Date']
print("Office supplies first time produced higher sales than furniture is {}.".format(first_date.date()))

# Time Series Modeling with Prophet
# conda install -c conda-forge fbprophet -y
# pip install --upgrade plotly
from fbprophet import Prophet

furniture = furniture.rename(columns={'Order Date': 'ds', 'Sales': 'y'})
furniture_model = Prophet(interval_width=0.95)
furniture_model.fit(furniture)

office = office.rename(columns={'Order Date': 'ds', 'Sales': 'y'})
office_model = Prophet(interval_width=0.95)
office_model.fit(office)

furniture_forecast = furniture_model.make_future_dataframe(periods=36, freq='MS')
furniture_forecast = furniture_model.predict(furniture_forecast)

office_forecast = office_model.make_future_dataframe(periods=36, freq='MS')
office_forecast = office_model.predict(office_forecast)

plt.figure(figsize=(18, 6))
furniture_model.plot(furniture_forecast, xlabel = 'Date', ylabel = 'Sales')
plt.title('Furniture Sales');

plt.figure(figsize=(18, 6))
office_model.plot(office_forecast, xlabel = 'Date', ylabel = 'Sales')
plt.title('Office Supplies Sales');

# Compare forecasts
furniture_names = ['furniture_%s' % column for column in furniture_forecast.columns]
office_names = ['office_%s' % column for column in office_forecast.columns]

merge_furniture_forecast = furniture_forecast.copy()
merge_office_forecast = office_forecast.copy()

merge_furniture_forecast.columns = furniture_names
merge_office_forecast.columns = office_names

forecast = pd.merge(merge_furniture_forecast, merge_office_forecast, how = 'inner', left_on = 'furniture_ds', right_on = 'office_ds')

forecast = forecast.rename(columns={'furniture_ds': 'Date'}).drop('office_ds', axis=1)
forecast.head()

# Trend and Forecast Visualization
plt.figure(figsize=(10, 7))
plt.plot(forecast['Date'], forecast['furniture_trend'], 'b-')
plt.plot(forecast['Date'], forecast['office_trend'], 'r-')
plt.legend(); plt.xlabel('Date'); plt.ylabel('Sales')
plt.title('Furniture vs. Office Supplies Sales Trend');

plt.figure(figsize=(10, 7))
plt.plot(forecast['Date'], forecast['furniture_yhat'], 'b-')
plt.plot(forecast['Date'], forecast['office_yhat'], 'r-')
plt.legend(); plt.xlabel('Date'); plt.ylabel('Sales')
plt.title('Furniture vs. Office Supplies Estimate');

# Trends and patterns
furniture_model.plot_components(furniture_forecast);
office_model.plot_components(office_forecast);

"""
Good to see that the sales for both furniture and office supplies have been linearly increasing over time although office supplies' growth seems slightly stronger.

The worst month for furniture is April, the worst month for office supplies is February. The best month for furniture is December, and the best month for office supplies is November.

There are many time-series analysis we can explore from now on, such as forecast with uncertainty bounds, change point and anomaly detection, forecast time-series with external data source. We have only scratched the surface here. Stay tuned for future works on time-series analysis.
"""


from JSONdb.DBHelper import DBHelper
import datetime,json,random

DBName = 'database'

STUDENT_TABLE = 'students'
PREDMET_TABLE = 'predmets'
BALL_TABLE = 'balls'
OTDEL_TABLE = 'otdels'
USER_TABLE = 'users'

NAME_FIELD = 'name'
FIRSTNAME_FIELD = 'firstname'
LASTNAME_FIELD = 'lastname'
PATR_FIELD = 'patr'
NUMBER_FIELD = 'number'
OTDEL_ID_FIELD = 'otdel_id'
KOLVO_CHASOV_FIELD = 'kolvo_chasov'
BALL_FIELD = 'ball'
PREDMET_ID_FIELD = 'predmet_id'
STUDENT_ID_FIELD = 'student_id'
DATE_FIELD = 'date'
ID_FIELD = 'id'
EMAIL_FIELD = 'email'
PASSWORD_FIELD = 'password'

def getDate():
    now = datetime.datetime.now()
    return '{0}.{1}.{2} {3}:{4}'.format(now.day,now.month,now.year,now.hour,now.minute)

def createTables(DBName):
    try:
        with open(DBName + ".json", "r", encoding="utf-8") as file:
            json.load(file)
            db = DBHelper(DBName)
    except FileNotFoundError:
        db = DBHelper(DBName)
        db.newTable(STUDENT_TABLE,[FIRSTNAME_FIELD,LASTNAME_FIELD,PATR_FIELD,NUMBER_FIELD,OTDEL_ID_FIELD,DATE_FIELD])
        db.newTable(PREDMET_TABLE,[NAME_FIELD,KOLVO_CHASOV_FIELD,DATE_FIELD])
        db.newTable(OTDEL_TABLE,[NAME_FIELD,DATE_FIELD])
        db.newTable(BALL_TABLE,[BALL_FIELD,PREDMET_ID_FIELD,STUDENT_ID_FIELD,DATE_FIELD])
        db.newTable(USER_TABLE,[EMAIL_FIELD,PASSWORD_FIELD,DATE_FIELD])
    return db

def getStatByPredmet():
    db = createTables(DBName)
    balls = db.select(db.SELECT_TYPE_STANDARD, BALL_TABLE, 0)
    predmets = db.select(db.SELECT_TYPE_STANDARD, PREDMET_TABLE, 0)

    predmet_list = []
    for predmet in predmets:
        count = 0
        summ = 0
        string = ' '
        for ball in balls:
            if predmet != db.IDS and predmet != db.COLUMN_LIST and ball != db.IDS and ball != db.COLUMN_LIST:
                if predmet == balls[ball][PREDMET_ID_FIELD]:
                    count+=1
                    summ+=int(balls[ball][BALL_FIELD])
        try:
            string = '{:.4}'.format(str(summ/count))

            predmet_list.append([string,predmets[predmet][NAME_FIELD]])
        except:pass
    return predmet_list

def getStatByStudent():
    db = createTables(DBName)
    balls = db.select(db.SELECT_TYPE_STANDARD, BALL_TABLE, 0)
    students = db.select(db.SELECT_TYPE_STANDARD, STUDENT_TABLE, 0)

    student_list = []
    for student in students:
        count = 0
        summ = 0
        string = ' '

        for ball in balls:
            if student != db.IDS and students != db.COLUMN_LIST and ball != db.IDS and ball != db.COLUMN_LIST:
                if student == balls[ball][STUDENT_ID_FIELD]:
                    count+=1
                    summ+=int(balls[ball][BALL_FIELD])
        try:
            string = '{:.4}'.format(str(summ/count))

            student_list.append([string,students[student][LASTNAME_FIELD] + ' ' +
                        students[student][FIRSTNAME_FIELD]+ ' ' +
                        students[student][PATR_FIELD]])
        except:
            pass
    return student_list

def autoadd():
    db = DBHelper(DBName)
    for j in range(10):
        num = j + 1
        db.addRow(OTDEL_TABLE, ['Отделение' + str(num), getDate()])
        db.addRow(PREDMET_TABLE, ['Предмет' + str(num), str(random.randint(24, 89)), getDate()])
        db.addRow(STUDENT_TABLE,
                  ['Имя' + str(num),
                   'Фамилия' + str(num),
                   'Отчество' + str(num),
                   str(random.randint(87270000000, 87970000000)),
                   str(random.randint(1, 10)),
                   getDate()])
    for i in range(300):
        db.addRow(BALL_TABLE,[str(random.randint(2, 5)),
                              str(random.randint(1, 10)),
                              str(random.randint(1, 10)),
                              getDate()])
def addInTable(*args):
    db = createTables(DBName)
    db.addRow(args)

def addUser(ColumnListVariables,email):
    db = createTables(DBName)
    #ColumnListVariables.append(getDate())
    id = db.base[USER_TABLE][db.IDS] + 1
    db.base[USER_TABLE][db.IDS] = id
    list = db.base[USER_TABLE][db.COLUMN_LIST]
    column_dict = {}
    for i in range(len(list)):
        column_dict.update(dict([(list[i], ColumnListVariables[i])]))
    column_dict.update([('id', id)])
    di = dict([(email, column_dict)])
    db.base[USER_TABLE].update(di)

    # Update base
    db.CreateOrUpdateBase(db.DBName)