import sys
import os
import re
import datetime
import time
import json
import subprocess
import argparse
import configparser
import tempfile

import requests
import discogssmells

# grab the current year. Make sure to set the clock of your machine
# to the correct date or use NTP!
currentyear = datetime.datetime.utcnow().year

# grab the latest release from the API. Results tend to get cached
# by the Discogs nginx instance for some reason.
def get_latest_release(headers):
    latest = 'https://api.discogs.com/database/search?type=release&sort=date_added'
    r = requests.get(latest, headers=headers)
    if r.status_code != 200:
        return

    # now parse the response
    responsejson = r.json()
    if not 'results' in responsejson:
        return

    return responsejson['results'][0]['id']

# convenience method to check if roles are valid
def checkrole(artist, release_id, credits):
    invalidroles = []
    if not '[' in artist['role']:
        roles = map(lambda x: x.strip(), artist['role'].split(','))
        for role in roles:
            if role == '':
                continue
            if not role in credits:
                invalidroles.append(role)
    else:
        # sometimes there is an additional description in the role in
        # between [ and ]
        # This method is definitely not catching everything.
        rolesplit = artist['role'].split('[')
        for rs in rolesplit:
            if ']' in rs:
                rs_tmp = rs
                while ']' in rs_tmp:
                    rs_tmp = rs_tmp.split(']', 1)[1]
                roles = map(lambda x: x.strip(), rs_tmp.split(','))
                for role in roles:
                    if role == '':
                        continue
                    # ugly hack because sometimes the extra data between [ and ]
                    # appears halfway the words in a role, sigh.
                    if role == 'By':
                        continue
                    if not role in credits:
                        invalidroles.append(role)
    return invalidroles

# process the contents of a release
def processrelease(release, config_settings, count, credits, ibuddy, favourites):
    releaseurl = 'https://www.discogs.com/release/%s'

    # only process entries that have a status of 'Accepted'
    if release['status'] == 'Rejected':
        return count
    elif release['status'] == 'Draft':
        return count
    elif release['status'] == 'Deleted':
        return count

    errormsgs = []

    # store some data that is used by multiple checks
    founddeposito = False
    year = None
    release_id = release['id']

    # check for favourite artist, if defined
    for artist in release['artists']:
        if artist['name'] in favourites:
            if ibuddy != None:
                ibuddy.executecommand('HEART:WINGSHIGH:RED:GO:SHORTSLEEP:NOHEART:WINGSLOW:GO:SHORTSLEEP:HEART:LEFT::WINGSHIGH::GO:SHORTSLEEP:NOHEART:RIGHT:GO:HEART:GO:BLUE:SHORTSLEEP:WINGSLOW:GO:SHORTSLEEP:RESET')
                ibuddy.reset()
            if config_settings['use_notify_send']:
                count += 1
                errormsgs.append('%8d -- Favourite Artist (%s): https://www.discogs.com/release/%s' % (count, artist['name'], str(release_id)))

    # check for misspellings of Czechoslovak and Czech releases
    # People use 0x115 instead of 0x11B, which look very similar but 0x115
    # is not valid in the Czech alphabet. Check for all data except
    # the YouTube playlist.
    # https://www.discogs.com/group/thread/757556
    # This is important for the following elements:
    # * tracklist (title, subtracks not supported yet)
    # * artist and extraartists (including extraartists in tracklist)
    # * notes
    # * BaOI identifiers (both value and description)
    if config_settings['check_spelling_cs']:
        if 'country' in release:
            if release['country'] == 'Czechoslovakia' or release['country'] == 'Czech Republic':
                for t in release['tracklist']:
                    if chr(0x115) in t['title']:
                        count += 1
                        errormsgs.append('%8d -- Czech character (0x115, tracklist: %s): https://www.discogs.com/release/%s' % (count, t['position'], str(release_id)))
                    if 'extraartists' in t:
                        for artist in t['extraartists']:
                            if chr(0x115) in artist['name']:
                                count += 1
                                errormsgs.append('%8d -- Czech character (0x115, artist name at: %s): https://www.discogs.com/release/%s' % (count, t['position'], str(release_id)))
                if 'artists' in release:
                    for artist in release['artists']:
                        if chr(0x115) in artist['name']:
                            count += 1
                            errormsgs.append('%8d -- Czech character (0x115, artist name: %s): https://www.discogs.com/release/%s' % (count, artist['name'], str(release_id)))
                if 'extraartists' in release:
                    for artist in release['extraartists']:
                        if chr(0x115) in artist['name']:
                            count += 1
                            errormsgs.append('%8d -- Czech character (0x115, artist name: %s): https://www.discogs.com/release/%s' % (count, artist['name'], str(release_id)))
                for i in release['identifiers']:
                    if chr(0x115) in i['value']:
                        count += 1
                        errormsgs.append('%8d -- Czech character (0x115, BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    if 'description' in i:
                        if chr(0x115) in i['description']:
                            count += 1
                            errormsgs.append('%8d -- Czech character (0x115, BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                if 'notes' in release:
                    if chr(0x115) in release['notes']:
                        count += 1
                        errormsgs.append('%8d -- Czech character (0x115, Notes): https://www.discogs.com/release/%s' % (count, str(release_id)))

    # check credit roles in three places:
    # 1. artists
    # 2. extraartists (release level)
    # 3. extraartists (track level)
    if 'check_credits' in config_settings:
        if config_settings['check_credits']:
            if 'artists' in release:
                for artist in release['artists']:
                    if 'role' in artist:
                        invalidroles = checkrole(artist, release_id, credits)
                        for role in invalidroles:
                            count += 1
                            errormsgs.append('%8d -- Role \'%s\' invalid: https://www.discogs.com/release/%s' % (count, role, str(release_id)))
            if 'extraartists' in release:
                for artist in release['extraartists']:
                    if 'role' in artist:
                        invalidroles = checkrole(artist, release_id, credits)
                        for role in invalidroles:
                            count += 1
                            errormsgs.append('%8d -- Role \'%s\' invalid: https://www.discogs.com/release/%s' % (count, role, str(release_id)))
            for t in release['tracklist']:
                if 'extraartists' in t:
                    for artist in t['extraartists']:
                        if 'role' in artist:
                            invalidroles = checkrole(artist, release_id, credits)
                            for role in invalidroles:
                                count += 1
                                errormsgs.append('%8d -- Role \'%s\' invalid: https://www.discogs.com/release/%s' % (count, role, str(release_id)))

    # check release month and year
    if 'released' in release:
        if config_settings['check_month']:
            if '-' in release['released']:
                monthres = re.search('-(\d+)-', release['released'])
                if monthres != None:
                    monthnr = int(monthres.groups()[0])
                    if monthnr == 0:
                        count += 1
                        errormsgs.append('%8d -- Month 00: https://www.discogs.com/release/%s' % (count, str(release_id)))
                    elif monthnr > 12:
                        count += 1
                        errormsgs.append('%8d -- Month impossible (%d): https://www.discogs.com/release/%s' % (count, monthnr, str(release_id)))
        try:
            year = int(release['released'].split('-', 1)[0])
            # TODO: check for implausible old years
        except ValueError:
            if config_settings['check_year']:
                count += 1
                errormsgs.append('%8d -- Year \'%s\' invalid: https://www.discogs.com/release/%s' % (count, release['released'], str(release_id)))

    # check the tracklist
    tracklistcorrect = True
    tracklistpositions = set()
    formattexts = set()
    if config_settings['check_tracklisting'] and len(release['formats']) == 1:
        formattext = release['formats'][0]['name']
        formattexts.add(formattext)
        formatqty = int(release['formats'][0]['qty'])
        for t in release['tracklist']:
            if tracklistcorrect:
                if formattext in ['Vinyl', 'Cassette', 'Shellac', '8-Track Cartridge']:
                    try:
                        int(t['position'])
                        count += 1
                        errormsgs.append('%8d -- Tracklisting (%s): https://www.discogs.com/release/%s' % (count, formattext, str(release_id)))
                        tracklistcorrect = False
                        break
                    except:
                        pass
                if formatqty == 1:
                    if t['position'].strip() != '' and t['position'].strip() != '-' and t['type_'] != 'heading' and t['position'] in tracklistpositions:
                        count += 1
                        errormsgs.append('%8d -- Tracklisting reuse (%s, %s): https://www.discogs.com/release/%s' % (count, formattext, t['position'], str(release_id)))
                    tracklistpositions.add(t['position'])

    # various checks for labels
    for l in release['labels']:
        # check for several identifiers being used as catalog numbers
        if 'catno' in l:
            if config_settings['check_label_code']:
                if l['catno'].lower().startswith('lc'):
                    falsepositive = False
                    # American releases on Epic (label 1005 in Discogs)
                    # sometimes start with LC
                    if l['id'] == 1005:
                        falsepositive = True
                    if not falsepositive:
                        if discogssmells.labelcodere.match(l['catno'].lower()) != None:
                            count += 1
                            errormsgs.append('%8d -- Possible Label Code (in Catalogue Number): https://www.discogs.com/release/%s' % (count, str(release_id)))
            if config_settings['check_deposito']:
                # now check for D.L.
                dlfound = False
                for d in discogssmells.depositores:
                    result = d.search(l['catno'])
                    if result != None:
                        for depositovalre in discogssmells.depositovalres:
                            if depositovalre.search(l['catno']) != None:
                                dlfound = True
                                break

                if dlfound:
                    count += 1
                    errormsgs.append('%8d -- Possible Depósito Legal (in Catalogue Number): https://www.discogs.com/release/%s' % (count, str(release_id)))
        if 'name' in l:
            if config_settings['check_label_name']:
                if l['name'] == 'London' and l['id'] == 26905:
                    count += 1
                    errormsgs.append('%8d -- Wrong label (London): https://www.discogs.com/release/%s' % (count, str(release_id)))
    '''
    if name == 'format':
        for (k,v) in attrs.items():
            if k == 'name':
                if v == 'CD':
                    self.iscd = True
                self.formattexts.add(v)
            elif k == 'qty':
                if self.formatmaxqty == 0:
                    self.formatmaxqty = max(self.formatmaxqty, int(v))
                else:
                    self.formatmaxqty += int(v)
    '''

    # various checks for the formats
    formattexts = set()
    for f in release['formats']:
        if 'descriptions' in f:
            if 'Styrene' in f['descriptions']:
                pass
        # store the names of the formats. This is useful later for SID code checks
        if 'name' in f:
            formattexts.add(f['name'])
        if 'text' in f:
            if f['text'] != '':
                if config_settings['check_spars_code']:
                    tmpspars = f['text'].lower().strip()
                    for s in ['.', ' ', '•', '·', '[', ']', '-', '|', '/']:
                        tmpspars = tmpspars.replace(s, '')
                    if tmpspars in discogssmells.validsparscodes:
                        count += 1
                        errormsgs.append('%8d -- Possible SPARS Code (in Format): https://www.discogs.com/release/%s' % (count, str(release_id)))
                if config_settings['check_label_code']:
                    if f['text'].lower().startswith('lc'):
                        if discogssmells.labelcodere.match(f['text'].lower()) != None:
                            count += 1
                            errormsgs.append('%8d -- Possible Label Code (in Format): https://www.discogs.com/release/%s' % (count, str(release_id)))

    # walk through the BaOI identifiers
    for identifier in release['identifiers']:
        v = identifier['value']
        if config_settings['check_creative_commons']:
            if 'creative commons' in v.lower():
                count += 1
                errormsgs.append('%8d -- Creative Commons reference: https://www.discogs.com/release/%s' % (count, str(release)))
            if 'description' in identifier:
                if 'creative commons' in identifier['description'].lower():
                    count += 1
                    errormsgs.append('%8d -- Creative Commons reference: https://www.discogs.com/release/%s' % (count, str(release)))
        if config_settings['check_spars_code']:
            if identifier['type'] == 'SPARS Code':
                if v.lower() != "none":
                    # Sony format codes
                    # https://www.discogs.com/forum/thread/339244
                    # https://www.discogs.com/forum/thread/358285
                    if v == 'CDC' or v == 'CDM':
                        count += 1
                        errormsgs.append('%8d -- Sony Format Code in SPARS: https://www.discogs.com/release/%s' % (count, str(release_id)))
                    else:
                        tmpspars = v.lower().strip()
                        for s in ['.', ' ', '•', '·', '[', ']', '-', '|', '/']:
                            tmpspars = tmpspars.replace(s, '')
                        if not tmpspars in discogssmells.validsparscodes:
                            count += 1
                            errormsgs.append('%8d -- SPARS Code (format): https://www.discogs.com/release/%s' % (count, str(release_id)))
            else:
                # first check the description free text field
                sparsfound = False
                if 'description' in identifier:
                    for spars in discogssmells.spars_ftf:
                        if spars in identifier['description'].lower():
                            sparsfound = True
                # then also check the value to see if there is a valid SPARS
                if v.lower() in discogssmells.validsparscodes:
                    sparsfound = True
                else:
                    if 'd' in v.lower():
                        tmpspars = v.strip()
                        for s in ['.', ' ', '•', '·', '[', ']', '-', '|', '/']:
                            tmpspars = tmpspars.replace(s, '')
                        if tmpspars in discogssmells.validsparscodes:
                            sparsfound = True
                # print error if some SPARS code reference was found
                if sparsfound:
                    count += 1
                    errormsgs.append('%8d -- SPARS Code (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
        if config_settings['check_label_code']:
            if identifier['type'] == 'Label Code':
                # check how many people use 'O' instead of '0'
                if v.lower().startswith('lc'):
                    if 'O' in identifier['value']:
                        errormsgs.append('%8d -- Spelling error in Label Code): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        sys.stdout.flush()
                if discogssmells.labelcodere.match(v.lower()) is None:
                    count += 1
                    errormsgs.append('%8d -- Label Code (value): https://www.discogs.com/release/%s' % (count, str(release_id)))
            else:
                if identifier['type'] == 'Rights Society':
                    if v.lower().startswith('lc'):
                        if discogssmells.labelcodere.match(v.lower()) != None:
                            count += 1
                            errormsgs.append('%8d -- Label Code (in Rights Society): https://www.discogs.com/release/%s' % (count, str(release_id)))
                elif identifier['type'] == 'Barcode':
                    if v.lower().startswith('lc'):
                        if discogssmells.labelcodere.match(v.lower()) != None:
                            count += 1
                            errormsgs.append('%8d -- Label Code (in Barcode): https://www.discogs.com/release/%s' % (count, str(release_id)))
                else:
                    if 'description' in identifier:
                        if identifier['description'].lower() in discogssmells.label_code_ftf:
                            count += 1
                            errormsgs.append('%8d -- Label Code: https://www.discogs.com/release/%s' % (count, str(release_id)))
        if config_settings['check_rights_society']:
            if identifier['type'] != 'Rights Society':
                foundrightssociety = False
                for r in discogssmells.rights_societies:
                    if v.replace('.', '') == r or v.replace(' ', '') == r:
                        count += 1
                        foundrightssociety = True
                        if identifier['type'] == 'Barcode':
                            errormsgs.append('%8d -- Rights Society (Barcode): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        else:
                            errormsgs.append('%8d -- Rights Society (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        break
                if not foundrightssociety and 'description' in identifier:
                    if identifier['description'].lower() in discogssmells.rights_societies_ftf:
                        count += 1
                        errormsgs.append('%8d -- Rights Society: https://www.discogs.com/release/%s' % (count, str(release_id)))

        # temporary hack, move to own configuration option
        asinstrict = False
        if config_settings['check_asin']:
            if identifier['type'] == 'ASIN':
                if not asinstrict:
                    tmpasin = v.strip().replace('-', '')
                else:
                    tmpasin = v
                if not len(tmpasin.split(':')[-1].strip()) == 10:
                    count += 1
                    errormsgs.append('%8d -- ASIN (wrong length): https://www.discogs.com/release/%s' % (count, str(release_id)))
            else:
                if 'description' in identifier:
                    if identifier['description'].lower().startswith('asin'):
                        count += 1
                        errormsgs.append('%8d -- ASIN (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
        if config_settings['check_isrc']:
            if identifier['type'] == 'ISRC':
                # Check the length of ISRC fields. According to the
                # specifications these should be 12 in length. Some ISRC
                # identifiers that have been recorded in the database
                # span a range of tracks. These will be reported as wrong ISRC
                # codes. It is unclear what needs to be done with those.

                # first get rid of cruft
                isrc_tmp = v.strip().upper()
                if isrc_tmp.startswith('ISRC'):
                    isrc_tmp = isrc_tmp.split('ISRC')[-1].strip()
                if isrc_tmp.startswith('CODE'):
                    isrc_tmp = isrc_tmp.split('CODE')[-1].strip()

                # replace a few characters
                isrc_tmp = isrc_tmp.replace('-', '')
                isrc_tmp = isrc_tmp.replace(' ', '')
                isrc_tmp = isrc_tmp.replace('.', '')
                isrc_tmp = isrc_tmp.replace(':', '')
                isrc_tmp = isrc_tmp.replace('–', '')
                if not len(isrc_tmp) == 12:
                    count += 1
                    errormsgs.append('%8d -- ISRC (wrong length): https://www.discogs.com/release/%s' % (count, str(release_id)))
            else:
                if 'description' in identifier:
                    if identifier['description'].lower().startswith('isrc'):
                        count += 1
                        errormsgs.append('%8d -- ISRC Code (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    elif identifier['description'].lower().startswith('issrc'):
                        count += 1
                        errormsgs.append('%8d -- ISRC Code (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    else:
                        for isrc in discogssmells.isrc_ftf:
                            if isrc in identifier['description'].lower():
                                count += 1
                                errormsgs.append('%8d -- ISRC Code (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
        if identifier['type'] == 'Barcode':
            pass

        # check depósito legal in BaOI
        if config_settings['check_deposito']:
            if 'country' in release:
                if release['country'] == 'Spain':
                    if identifier['type'] == 'Depósito Legal':
                        founddeposito = True
                        if v.strip().endswith('.'):
                            count += 1
                            errormsgs.append('%8d -- Depósito Legal (formatting): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        if year != None:
                            # now try to find the year
                            depositoyear = None
                            if v.strip().endswith('℗'):
                                count += 1
                                errormsgs.append('%8d -- Depósito Legal (formatting, has ℗): https://www.discogs.com/release/%s' % (count, str(release_id)))
                                # ugly hack, remove ℗ to make at least be able to do some sort of check
                                v = v.strip().rsplit('℗', 1)[0]
                            # several separators, including some Unicode ones
                            for sep in ['-', '–', '/', '.', ' ', '\'', '_']:
                                try:
                                    depositoyeartext = v.strip().rsplit(sep, 1)[-1]
                                    if sep == '.' and len(depositoyeartext) == 3:
                                        continue
                                    if '.' in depositoyeartext:
                                        depositoyeartext = depositoyeartext.replace('.', '')
                                    depositoyear = int(depositoyeartext)
                                    if depositoyear < 100:
                                        # correct the year. This won't work correctly after 2099.
                                        if depositoyear <= currentyear - 2000:
                                            depositoyear += 2000
                                        else:
                                            depositoyear += 1900
                                    break
                                except:
                                    pass

                            # TODO, also allow (year), example: https://www.discogs.com/release/265497
                            if depositoyear != None:
                                if depositoyear < 1900:
                                    count += 1
                                    errormsgs.append("%8d -- Depósito Legal (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                elif depositoyear > currentyear:
                                    count += 1
                                    errormsgs.append("%8d -- Depósito Legal (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                elif year < depositoyear:
                                    count += 1
                                    errormsgs.append("%8d -- Depósito Legal (release date earlier): https://www.discogs.com/release/%s" % (count, str(release_id)))
                            else:
                                count += 1
                                errormsgs.append("%8d -- Depósito Legal (year not found): https://www.discogs.com/release/%s" % (count, str(release_id)))
                    elif identifier['type'] == 'Barcode':
                        for depositovalre in discogssmells.depositovalres:
                            if depositovalre.match(v.lower()) != None:
                                founddeposito = True
                                count += 1
                                errormsgs.append('%8d -- Depósito Legal (in Barcode): https://www.discogs.com/release/%s' % (count, str(release_id)))
                                break
                    else:
                        if v.startswith("Depósito"):
                            founddeposito = True
                            count += 1
                            errormsgs.append('%8d -- Depósito Legal (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        elif v.startswith("D.L."):
                            founddeposito = True
                            count += 1
                            errormsgs.append('%8d -- Depósito Legal (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        else:
                            if 'description' in identifier:
                                found = False
                                for d in discogssmells.depositores:
                                    result = d.search(identifier['description'].lower())
                                    if result != None:
                                        found = True
                                        break

                                # sometimes the depósito value itself can be found in the free text field
                                if not found:
                                    for depositovalre in discogssmells.depositovalres:
                                        deposres = depositovalre.match(identifier['description'].lower())
                                        if deposres != None:
                                            found = True
                                            break

                                if found:
                                    founddeposito = True
                                    count += 1
                                    errormsgs.append('%8d -- Depósito Legal (BaOI): https://www.discogs.com/release/%s' % (count, str(release_id)))

        # temporary hack, move to own configuration option
        mould_sid_strict = False
        if config_settings['check_mould_sid']:
            if identifier['type'] == 'Mould SID Code':
                if v.strip() != 'none':
                    # cleanup first for not so heavy formatting booboos
                    mould_tmp = v.strip().lower().replace(' ', '')
                    mould_tmp = mould_tmp.replace('-', '')
                    # some people insist on using ƒ instead of f
                    mould_tmp = mould_tmp.replace('ƒ', 'f')
                    res = discogssmells.mouldsidre.match(mould_tmp)
                    if res is None:
                        count += 1
                        errormsgs.append('%8d -- Mould SID Code (value): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    else:
                        if mould_sid_strict:
                            mould_split = mould_tmp.split('ifpi', 1)[-1]
                            for ch in ['i', 'o', 's', 'q']:
                                if ch in mould_split[-2:]:
                                    count += 1
                                    errormsgs.append('%8d -- Mould SID Code (strict value): https://www.discogs.com/release/%s' % (count, str(release_id)))
                        # rough check to find SID codes for formats other than CD/CD-like
                        if len(formattexts) == 1:
                            for fmt in set(['Vinyl', 'Cassette', 'Shellac', 'File', 'VHS', 'DCC', 'Memory Stick', 'Edison Disc']):
                                if fmt in formattexts:
                                    count += 1
                                    errormsgs.append('%8d -- Mould SID Code (Wrong Format: %s): https://www.discogs.com/release/%s' % (count, fmt, str(release_id)))
                                    break
                        if year != None:
                            if year < 1993:
                                count += 1
                                errormsgs.append('%8d -- SID Code (wrong year): https://www.discogs.com/release/%s' % (count, str(release_id)))

            else:
                if 'description' in identifier:
                    description = identifier['description'].lower()
                    # squash repeated spaces
                    description = re.sub('\s+', ' ', description)
                    description = description.strip()
                    if description in ['source identification code', 'sid', 'sid code', 'sid-code']:
                        count += 1
                        errormsgs.append('%8d -- Unspecified SID Code: https://www.discogs.com/release/%s' % (count, str(release_id)))
                    elif description in discogssmells.mouldsids:
                        count += 1
                        errormsgs.append('%8d -- Mould SID Code: https://www.discogs.com/release/%s' % (count, str(release_id)))

        if config_settings['check_mastering_sid']:
            if identifier['type'] == 'Mastering SID Code':
                if v.strip() != 'none':
                    # cleanup first for not so heavy formatting booboos
                    master_tmp = v.strip().lower().replace(' ', '')
                    master_tmp = master_tmp.replace('-', '')
                    # some people insist on using ƒ instead of f
                    master_tmp = master_tmp.replace('ƒ', 'f')
                    res = discogssmells.masteringsidre.match(master_tmp)
                    if res is None:
                        count += 1
                        errormsgs.append('%8d -- Mastering SID Code (value): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    else:
                        # rough check to find SID codes for formats other than CD/CD-like
                        if len(formattexts) == 1:
                            for fmt in set(['Vinyl', 'Cassette', 'Shellac', 'File', 'VHS', 'DCC', 'Memory Stick', 'Edison Disc']):
                                if fmt in formattexts:
                                    count += 1
                                    errormsgs.append('%8d -- Mastering SID Code (Wrong Format: %s): https://www.discogs.com/release/%s' % (count, fmt, str(release_id)))
                        if year != None:
                            if year < 1993:
                                count += 1
                                errormsgs.append('%8d -- SID Code (wrong year): https://www.discogs.com/release/%s' % (count, str(release_id)))
            else:
                if 'description' in identifier:
                    description = identifier['description'].lower()
                    # squash repeated spaces
                    description = re.sub('\s+', ' ', description)
                    description = description.strip()
                    if description in ['source identification code', 'sid', 'sid code', 'sid-code']:
                        count += 1
                        errormsgs.append('%8d -- Unspecified SID Code: https://www.discogs.com/release/%s' % (count, str(release_id)))
                    elif description in discogssmells.masteringsids:
                        count += 1
                        errormsgs.append('%8d -- Mastering SID Code: https://www.discogs.com/release/%s' % (count, str(release_id)))
                    elif description in ['sid code matrix', 'sid code - matrix', 'sid code (matrix)', 'sid-code, matrix', 'sid-code matrix', 'sid code (matrix ring)', 'sid code, matrix ring', 'sid code: matrix ring']:
                        count += 1
                        errormsgs.append('%8d -- Possible Mastering SID Code: https://www.discogs.com/release/%s' % (count, str(release_id)))
        if config_settings['check_pkd']:
            if 'country' in release:
                if release['country'] == 'India':
                    if 'pkd' in v.lower() or "production date" in v.lower():
                        if year != None:
                            # try a few variants
                            pkdres = re.search("\d{1,2}/((?:19|20)?\d{2})", v)
                            if pkdres != None:
                                pkdyear = int(pkdres.groups()[0])
                                if pkdyear < 100:
                                    # correct the year. This won't work correctly after 2099.
                                    if pkdyear <= currentyear - 2000:
                                        pkdyear += 2000
                                    else:
                                        pkdyear += 1900
                                if pkdyear < 1900:
                                    count += 1
                                    errormsgs.append("%8d -- Indian PKD (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                elif pkdyear > currentyear:
                                    count += 1
                                    errormsgs.append("%8d -- Indian PKD (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                elif year < pkdyear:
                                    count += 1
                                    errormsgs.append("%8d -- Indian PKD (release date earlier): https://www.discogs.com/release/%s" % (count, str(release_id)))
                        else:
                            count += 1
                            errormsgs.append('%8d -- India PKD code (no year): https://www.discogs.com/release/%s' % (count, str(release_id)))
                    else:
                        # now check the description
                        if 'description' in identifier:
                            description = identifier['description'].lower()
                            if 'pkd' in description or "production date" in description:
                                if year != None:
                                    # try a few variants
                                    pkdres = re.search("\d{1,2}/((?:19|20)?\d{2})", attrvalue)
                                    if pkdres != None:
                                        pkdyear = int(pkdres.groups()[0])
                                        if pkdyear < 100:
                                            # correct the year. This won't work correctly after 2099.
                                            if pkdyear <= currentyear - 2000:
                                                pkdyear += 2000
                                            else:
                                                pkdyear += 1900
                                        if pkdyear < 1900:
                                            count += 1
                                            errormsgs.append("%8d -- Indian PKD (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                        elif pkdyear > currentyear:
                                            count += 1
                                            errormsgs.append("%8d -- Indian PKD (impossible year): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                        elif year < pkdyear:
                                            count += 1
                                            errormsgs.append("%8d -- Indian PKD (release date earlier): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                    else:
                                        count += 1
                                        errormsgs.append('%8d -- India PKD code (no year): https://www.discogs.com/release/%s' % (count, str(release_id)))
        # check Czechoslovak manufacturing dates
        if config_settings['check_manufacturing_date_cs']:
            # config hack, needs to be in its own configuration option
            strict_cs = False
            strict_cs = True
            if 'country' in release:
                if release['country'] == 'Czechoslovakia':
                    if 'description' in identifier:
                        description = identifier['description'].lower()
                        if 'date' in description:
                            if year != None:
                                manufacturing_date_res = re.search("(\d{2})\s+\d$", identifier['value'].rstrip())
                                if manufacturing_date_res != None:
                                    manufacturing_year = int(manufacturing_date_res.groups()[0])
                                    if manufacturing_year < 100:
                                        manufacturing_year += 1900
                                        if manufacturing_year > year:
                                            count += 1
                                            errormsgs.append("%8d -- Czechoslovak manufacturing date (release year wrong): https://www.discogs.com/release/%s" % (count, str(release_id)))
                                        # possibly this check makes sense, but not always
                                        elif manufacturing_year < year and strict_cs:
                                            count += 1
                                            errormsgs.append("%8d -- Czechoslovak manufacturing date (release year possibly wrong): https://www.discogs.com/release/%s" % (count, str(release_id)))

    # finally check the notes for some errors
    if 'notes' in release:
        if '카지노' in release['notes']:
            # Korean casino spam that pops up every once in a while
            errormsgs.append('Spam: https://www.discogs.com/release/%s' % str(release_id))
        if 'country' in release:
            if release['country'] == 'Spain':
                if config_settings['check_deposito'] and not founddeposito:
                    # sometimes "deposito legal" can be found in the "notes" section
                    content_lower = release['notes'].lower()
                    for d in discogssmells.depositores:
                        result = d.search(content_lower)
                        if result != None:
                            count += 1
                            found = True
                            errormsgs.append('%8d -- Depósito Legal (Notes): https://www.discogs.com/release/%s' % (count, str(release_id)))
                            break
        if config_settings['check_html']:
            # see https://support.discogs.com/en/support/solutions/articles/13000014661-how-can-i-format-text-
            if '<a href="http://www.discogs.com/release/' in release['notes'].lower():
                count += 1
                errormsgs.append('%8d -- old link (Notes): https://www.discogs.com/release/%s' % (count, str(release_id)))
        if config_settings['check_creative_commons']:
            ccfound = False
            for cc in discogssmells.creativecommons:
                if cc in release['notes']:
                    count += 1
                    errormsgs.append('%8d -- Creative Commons reference (%s): https://www.discogs.com/release/%s' % (count, cc, str(release)))
                    ccfound = True
                    break

                if not ccfound:
                    if 'creative commons' in reales['notes'].lower():
                        count += 1
                        errormsgs.append('%8d -- Creative Commons reference: https://www.discogs.com/release/%s' % (count, str(release)))
                        ccfound = True
                        break

    for e in errormsgs:
        print(e)
        if config_settings['use_notify_send']:
            p = subprocess.Popen(['notify-send', "-t", "3000", "Error", e], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stanout, stanerr) = p.communicate()
    sys.stdout.flush()
    return count

def main(argv):
    parser = argparse.ArgumentParser()

    # the following options are provided on the commandline
    parser.add_argument("-c", "--config", action="store", dest="cfg", help="path to configuration file", metavar="FILE")
    parser.add_argument("-s", "--startvalue", action="store", dest="startvalue", help="start value for releases", metavar="STARTVALUE")
    parser.add_argument("-l", "--latest", action="store", dest="latest_value", help="value for latest release", metavar="LATEST")
    args = parser.parse_args()

    # some checks for the configuration file
    if args.cfg is None:
        parser.error("Configuration file missing")

    if not os.path.exists(args.cfg):
        parser.error("Configuration file does not exist")

    config = configparser.ConfigParser()

    configfile = open(args.cfg, 'r')

    try:
        config.read_file(configfile)
    except Exception:
        print("Cannot read configuration file", file=sys.stderr)
        sys.exit(1)

    startvalue = None
    # check for a startvalue
    if args.startvalue != None:
        try:
            startvalue = int(args.startvalue)
        except:
            parser.error("start value is not a valid integer, exciting")

    latest_release = None
    # check for a startvalue
    if args.latest_value != None:
        try:
            latest_release = int(args.latest_value)
        except:
            parser.error("latest value is not a valid integer, exciting")

    # process the configuration file and store settings
    config_settings = {}

    for section in config.sections():
        if section == 'cleanup':
            # store settings for depósito legal checks
            try:
                if config.get(section, 'deposito') == 'yes':
                    config_settings['check_deposito'] = True
                else:
                    config_settings['check_deposito'] = False
            except Exception:
                config_settings['check_deposito'] = True

            # store settings for rights society checks
            try:
                if config.get(section, 'rights_society') == 'yes':
                    config_settings['check_rights_society'] = True
                else:
                    config_settings['check_rights_society'] = False
            except Exception:
                config_settings['check_rights_society'] = True

            # store settings for label code checks
            try:
                if config.get(section, 'label_code') == 'yes':
                    config_settings['check_label_code'] = True
                else:
                    config_settings['check_label_code'] = False
            except Exception:
                config_settings['check_label_code'] = True

            # store settings for label name checks
            try:
                if config.get(section, 'label_name') == 'yes':
                    config_settings['check_label_name'] = True
                else:
                    config_settings['check_label_name'] = False
            except Exception:
                config_settings['check_label_name'] = True

            # store settings for ISRC checks
            try:
                if config.get(section, 'isrc') == 'yes':
                    config_settings['check_isrc'] = True
                else:
                    config_settings['check_isrc'] = False
            except Exception:
                config_settings['check_isrc'] = True

            # store settings for ASIN checks
            try:
                if config.get(section, 'asin') == 'yes':
                    config_settings['check_asin'] = True
                else:
                    config_settings['check_asin'] = False
            except Exception:
                config_settings['check_asin'] = True

            # store settings for mastering SID checks
            try:
                if config.get(section, 'mastering_sid') == 'yes':
                    config_settings['check_mastering_sid'] = True
                else:
                    config_settings['check_mastering_sid'] = False
            except Exception:
                config_settings['check_mastering_sid'] = True

            # store settings for mould SID checks
            try:
                if config.get(section, 'mould_sid') == 'yes':
                    config_settings['check_mould_sid'] = True
                else:
                    config_settings['check_mould_sid'] = False
            except Exception:
                config_settings['check_mould_sid'] = True

            # store settings for SPARS Code checks
            try:
                if config.get(section, 'spars') == 'yes':
                    config_settings['check_spars_code'] = True
                else:
                    config_settings['check_spars_code'] = False
            except Exception:
                config_settings['check_spars_code'] = True

            # store settings for Indian PKD checks
            try:
                if config.get(section, 'pkd') == 'yes':
                    config_settings['check_pkd'] = True
                else:
                    config_settings['check_pkd'] = False
            except Exception:
                config_settings['check_pkd'] = True

            # check for Czechoslovak manufacturing dates
            try:
                if config.get(section, 'manufacturing_date_cs') == 'yes':
                    config_settings['check_manufacturing_date_cs'] = True
                else:
                    config_settings['check_manufacturing_date_cs'] = False
            except Exception:
                config_settings['check_manufacturing_date_cs'] = True

            # check for Czechoslovak and Czech spelling (0x115 used instead of 0x11B)
            try:
                if config.get(section, 'spelling_cs') == 'yes':
                    config_settings['check_spelling_cs'] = True
                else:
                    config_settings['check_spelling_cs'] = False
            except Exception:
                config_settings['check_spelling_cs'] = True

            # store settings for tracklisting checks, default True
            try:
                if config.get(section, 'tracklisting') == 'yes':
                    config_settings['check_tracklisting'] = True
                else:
                    config_settings['check_tracklisting'] = False
            except Exception:
                config_settings['check_tracklisting'] = True

            # store settings for credits list checks
            try:
                if config.get(section, 'credits') == 'yes':
                    creditsfile = config.get(section, 'creditsfile')
                    if os.path.exists(creditsfile):
                        config_settings['creditsfile'] = creditsfile
                        config_settings['check_credits'] = True
                else:
                    config_settings['check_credits'] = False
            except Exception:
                config_settings['check_credits'] = False

            # store settings for URLs in Notes checks
            try:
                if config.get(section, 'html') == 'yes':
                    config_settings['check_html'] = True
                else:
                    config_settings['check_html'] = False
            except Exception:
                config_settings['check_html'] = True

            # month is 00 check: default is False
            try:
                if config.get(section, 'month') == 'yes':
                    config_settings['check_month'] = True
                else:
                    config_settings['check_month'] = False
            except Exception:
                config_settings['check_month'] = False

            # year is wrong check: default is False
            try:
                if config.get(section, 'year') == 'yes':
                    config_settings['check_year'] = True
                else:
                    config_settings['check_year'] = False
            except Exception:
                config_settings['check_year'] = False

            # reporting all: default is False
            try:
                if config.get(section, 'reportall') == 'yes':
                    config_settings['reportall'] = True
                else:
                    config_settings['reportall'] = False
            except Exception:
                config_settings['reportall'] = False

            # debug: default is False
            try:
                if config.get(section, 'debug') == 'yes':
                    config_settings['debug'] = True
                else:
                    config_settings['debug'] = False
            except Exception:
                config_settings['debug'] = False

            # report creative commons references: default is False
            try:
                if config.get(section, 'creative_commons') == 'yes':
                    config_settings['check_creative_commons'] = True
                else:
                    config_settings['check_creative_commons'] = False
            except Exception:
                config_settings['check_creative_commons'] = False

        elif section == 'api':
            # data directory to store JSON files
            try:
                storedir = config.get(section, 'storedir')
                if not os.path.exists(os.path.normpath(storedir)):
                    config_settings['storedir'] = None
                else:
                    # test if the directory is writable
                    testfile = tempfile.mkstemp(dir=storedir)
                    os.fdopen(testfile[0]).close()
                    os.unlink(testfile[1])
                    config_settings['storedir'] = storedir
            except Exception:
                config_settings['storedir'] = None
                break
            try:
                token = config.get(section, 'token')
                config_settings['token'] = token
            except Exception:
                config_settings['token'] = None
            try:
                username = config.get(section, 'username')
                config_settings['username'] = username
            except Exception:
                config_settings['username'] = None

            # skipdownloaded: default is False
            config_settings['skipdownloaded'] = False
            try:
                if config.get(section, 'skipdownloaded') == 'yes':
                    config_settings['skipdownloaded'] = True
            except Exception:
                pass

            # skip404: default is True
            config_settings['skip404'] = True
            try:
                if config.get(section, 'skip404') == 'yes':
                    config_settings['skip404'] = True
                else:
                    config_settings['skip404'] = False
            except Exception:
                pass

            # record404: default is True
            config_settings['record404'] = True
            try:
                if config.get(section, 'record404') == 'yes':
                    config_settings['record404'] = True
                else:
                    config_settings['record404'] = False
            except Exception:
                pass

            # specify location of 404 file
            try:
                release404 = os.path.normpath(config.get(section, '404file'))
                config_settings['404file'] = release404
            except:
                pass

            # specify whether or not notify-send (Linux desktops
            # should be used or not. Not recommended.
            config_settings['use_notify_send'] = True
            try:
                if config.get(section, 'notify') == 'yes':
                    config_settings['use_notify_send'] = True
                else:
                    config_settings['use_notify_send'] = False
            except Exception:
                pass

    if config_settings['use_notify_send']:
        try:
            p = subprocess.Popen(['notify-send', "-t", "3000", "Test for notify-send"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stanout, stanerr) = p.communicate()
        except Exception:
            config_settings['use_notify_send'] = False

    configfile.close()
    if config_settings['storedir'] is None:
        print("Data store directory non-existent or not writable, exiting.", file=sys.stderr)
        sys.exit(1)
    if config_settings['token'] is None:
        print("Token not specified, exiting.", file=sys.stderr)
        sys.exit(1)
    if config_settings['username'] is None:
        print("Discogs user name not specified, exiting.", file=sys.stderr)
        sys.exit(1)

    # a list of accepted roles. This is an external file, generated with extractcredits.py
    # from the 'helper-scripts' directory.
    credits = set()
    if 'check_credits' in config_settings:
        if config_settings['check_credits']:
            creditsfile = open(config_settings['creditsfile'], 'r')
            credits = set(map(lambda x: x.strip(), creditsfile.readlines()))
            creditsfile.close()

    # a file with release numbers that give a 404 error
    # This needs more work
    if config_settings['skip404']:
        if '404file' in config_settings:
            if not os.path.isabs(config_settings['404file']):
                release404filename = os.path.join(config_settings['storedir'], config_settings['404file'])
                if not os.path.exists(release404filename):
                    release404file = open(release404filename, 'w')
                    release404file.close()
            else:
                release404filename = config_settings['404file']
        else:
            # simply create the file
            pass

    # use a (somewhat) exponential backoff in case too many requests have been made
    ratelimitbackoff = 5

    # set the User Agent and Authorization header for each user request
    useragentstring = "DiscogsCleanupForUser-%s/0.1" % config_settings['username']
    headers = {'user-agent': useragentstring,
               'Authorization': 'Discogs token=%s' % config_settings['token']
              }

    if latest_release is None:
        latest_release = get_latest_release(headers)
        if latest_release is None:
            print("Something went wrong, try again later", file=sys.stderr)
            sys.exit(1)

    # if no start value has been provided start with the latest from the
    # Discogs website.
    if startvalue is None:
        startvalue = latest_release

    # populate a set with all the 404s that were found.
    skip404s = set()
    count = 0
    if config_settings['skip404']:
        release404file = open(release404filename, 'r')
        for l in release404file:
            # needs to be made more robust
            skip404s.add(int(l.strip()))
        release404file.close()
        # now open again for writing, so new 404 errors can be
        # stored.
        release404file = open(release404filename, 'a')

    # This is just something very silly: if you have an iBuddy device and
    # have the corresponding Python module installed it will respond to
    # data it finds (currently only favourite artists).
    #
    # https://github.com/armijnhemel/py3buddy
    #
    # Not recommended.
    ibuddy_enabled = False
    try:
        import py3buddy
        ibuddy_enabled = True
    except:
        pass

    ibuddy = None
    if ibuddy_enabled:
        ibuddy_config = {}
        ibuddy = py3buddy.iBuddy(ibuddy_config)
        if ibuddy.dev is None:
            ibuddy = None
            ibuddy_enabled = False

    # example:
    #favourites = set(['Bob Dylan', 'Iron Maiden', 'The Beatles'])
    favourites = set()

    newsleep = 600

    # now start a big loop
    # https://www.discogs.com/developers/#page:authentication
    while True:
        for releasenr in range(startvalue, latest_release+1):
            if startvalue == latest_release:
                break
            targetfilename = os.path.join(storedir, "%d" % (releasenr//1000000), "%d.json" % releasenr)
            os.makedirs(os.path.join(storedir, "%d" % (releasenr//1000000)), exist_ok=True)
            if config_settings['skip404']:
                if releasenr in skip404s:
                    continue
            if config_settings['skipdownloaded']:
                if os.path.exists(targetfilename):
                    if os.stat(targetfilename).st_size != 0:
                        responsejsonfile = open(targetfilename, 'r')
                        responsejson = json.loads(responsejsonfile.read())
                        responsejsonfile.close()
                        count = processrelease(responsejson, config_settings, count, credits, ibuddy, favourites)
                        continue
            print("downloading: %d" % releasenr, file=sys.stderr)
            r = requests.get('https://api.discogs.com/releases/%d' % releasenr, headers=headers)

            # now first check the headers to see if it is OK to do more requests
            if r.status_code != 200:
                if r.status_code == 404:
                    print("%d" % releasenr, file=release404file)
                    release404file.flush()
                if r.status_code == 429:
                    if 'Retry-After' in r.headers:
                        try:
                            retryafter = int(r.headers['Retry-After'])
                            print("Rate limiting, sleeping for %d seconds" % retryafter, file=sys.stderr)
                            time.sleep(retryafter)
                            sys.stderr.flush()
                        except:
                            print("Rate limiting, sleeping for %d seconds" % 60, file=sys.stderr)
                            time.sleep(60)
                            sys.stderr.flush()
                    else:
                        print("Rate limiting, sleeping for %d seconds" % 60, file=sys.stderr)
                        time.sleep(60)
                        sys.stderr.flush()
                # TODO: the current release will not have been downloaded and processed
                continue

            # in case there is no 429 response check the headers
            if 'X-Discogs-Ratelimit-Remaining' in r.headers:
                ratelimit = int(r.headers['X-Discogs-Ratelimit-Remaining'])
            if ratelimit == 0:
                # no more requests are allowed, so sleep for some
                # time, max 60 seconds
                time.sleep(ratelimitbackoff)
                print("Rate limiting, sleeping for %d seconds" % ratelimitbackoff, file=sys.stderr)
                sys.stderr.flush()
                if ratelimitbackoff < 60:
                    ratelimitbackoff = min(60, ratelimitbackoff * 2)
            else:
                ratelimitbackoff = 5

            # now process the response. This should be JSON, so decode it,
            # and also write the JSON data to a separate file for offline
            # processing (if necessary).
            try:
                responsejson = r.json()
                jsonreleasefile = open(targetfilename, 'w')
                jsonreleasefile.write(r.text)
                jsonreleasefile.close()
            except:
                # response doesn't contain JSON, so something is wrong.
                # sleep a bit then continue
                time.sleep(2)
                continue
            # now process the JSON content
            count = processrelease(responsejson, config_settings, count, credits, ibuddy, favourites)
            # be gentle for Discogs and sleep
            time.sleep(0.2)
            sys.stderr.flush()

        # now set startvalue to latest_release
        startvalue = latest_release

        # and find the newest release again
        print("Grabbing new data", file=sys.stderr)
        latest_release = get_latest_release(headers)
        if latest_release is None:
            print("Something went wrong, try again later", file=sys.stderr)
            break
        if latest_release < startvalue:
            pass
        print("Latest = %d" % latest_release, file=sys.stderr)
        print("Sleeping for %d seconds" % newsleep, file=sys.stderr)
        sys.stderr.flush()

        # sleep for ten minutes to make sure some new things
        # have been added to Discogs
        time.sleep(newsleep)
    release404file.close()

if __name__ == "__main__":
    main(sys.argv)


from os import path, environ
from types import FunctionType
from typing import Dict, List
import json
import yaml
from pathlib import Path

_DEF_CFG = {
    "github_token": "",
    "user": "",
    "target": "repositories",
    "ignore": "",
    "depth": "",
    "max_size": ""
}

def parse() -> Dict:
    cfg = _DEF_CFG
    _parse_config_files([
        path.join("/etc/stargrab/config"),
        path.join(Path.home(), ".stargrab/config"),
        "./config",
    ], cfg)
    _parse_env("SG", cfg)
    return cfg

def _parse_env(prefix: str, target: Dict):
    for k in target.keys():
        v = environ.get(f"{prefix}_{k}".upper())
        if v:
            target[k] = v

def _parse_config_files(pathes: List[str], target: Dict):
    for p in pathes:
        _merge(_parse_yaml(p+".yml"), target)
        _merge(_parse_yaml(p+".yaml"), target)
        _merge(_parse_json(p+".json"), target)

def _parse_json(path: str) -> Dict:
    return _parse_file(json.load, path)

def _parse_yaml(path: str) -> Dict:
    return _parse_file(lambda p: yaml.load(p, Loader=yaml.FullLoader), path)

def _parse_file(parser: FunctionType, path: str) -> Dict:
    try:
        with open(path) as f:
            return parser(f)
    except FileNotFoundError:
        return {}
    except Exception as e:
        raise e

def _merge(source: Dict, target: Dict) -> Dict:
    for k in target.keys():
        if source.get(k):
            target[k] = source[k]


import asyncio

import pytest

import zigpy.device
import zigpy.types as t
import zigpy.zdo as zdo
import zigpy.zdo.types as zdo_types

from .async_mock import AsyncMock, MagicMock, patch, sentinel

def test_commands():
    for cmdid, cmdspec in zdo.types.CLUSTERS.items():
        assert 0 <= cmdid <= 0xFFFF
        assert isinstance(cmdspec, tuple)
        for paramname, paramtype in zip(cmdspec[0], cmdspec[1]):
            assert isinstance(paramname, str)
            assert hasattr(paramtype, "serialize")
            assert hasattr(paramtype, "deserialize")

@pytest.fixture
def zdo_f(app):
    ieee = t.EUI64(map(t.uint8_t, [0, 1, 2, 3, 4, 5, 6, 7]))
    dev = zigpy.device.Device(app, ieee, 65535)
    dev.request = AsyncMock()

    app.devices[dev.ieee] = dev

    return zdo.ZDO(dev)

def test_deserialize(zdo_f):
    hdr, args = zdo_f.deserialize(2, b"\x01\x02\x03xx")
    assert hdr.tsn == 1
    assert hdr.is_reply is False
    assert args == [0x0302]

def test_deserialize_unknown(zdo_f):
    with pytest.raises(ValueError):
        hdr, args = zdo_f.deserialize(0x0100, b"\x01")

async def test_request(zdo_f):
    await zdo_f.request(2, 65535)
    app = zdo_f._device._application
    assert zdo_f.device.request.call_count == 1
    assert app.get_sequence.call_count == 1

async def test_bind(zdo_f):
    cluster = MagicMock()
    cluster.endpoint.endpoint_id = 1
    cluster.cluster_id = 1026
    await zdo_f.bind(cluster)
    assert zdo_f.device.request.call_count == 1
    assert zdo_f.device.request.call_args[0][1] == 0x0021

async def test_unbind(zdo_f):
    cluster = MagicMock()
    cluster.endpoint.endpoint_id = 1
    cluster.cluster_id = 1026
    await zdo_f.unbind(cluster)
    assert zdo_f.device.request.call_count == 1
    assert zdo_f.device.request.call_args[0][1] == 0x0022

@pytest.mark.parametrize(
    "remove_children, rejoin, flags",
    (
        (False, False, 0),
        (False, True, 0x80),
        (True, False, 0x40),
        (True, True, 0xC0),
    ),
)
async def test_leave(zdo_f, remove_children, rejoin, flags):
    """Test ZDO leave request options."""
    with patch.object(zdo_f, "request", AsyncMock()) as req_mock:
        await zdo_f.leave(remove_children, rejoin)
    assert req_mock.await_count == 1
    assert req_mock.await_args[0][0] == 0x0034
    assert req_mock.await_args[0][1] == t.EUI64.convert("07:06:05:04:03:02:01:00")
    assert req_mock.await_args[0][2] == flags

async def test_permit(zdo_f):
    await zdo_f.permit()
    assert zdo_f.device.request.call_count == 1
    assert zdo_f.device.request.call_args[0][1] == 0x0036

async def test_broadcast(app):
    await zigpy.zdo.broadcast(app, 0x0036, 0, 0, 60, 0)

    assert app.send_packet.call_count == 1

    packet = app.send_packet.mock_calls[0].args[0]
    assert packet.dst.addr_mode == t.AddrMode.Broadcast
    assert packet.cluster_id == 0x0036

def _handle_match_desc(zdo_f, profile):
    zdo_f.reply = AsyncMock()
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd.Match_Desc_req
    zdo_f.handle_message(5, 0x0006, hdr, [None, profile, [], []])
    assert zdo_f.reply.call_count == 1

async def test_handle_match_desc_zha(zdo_f):
    _handle_match_desc(zdo_f, 260)
    await asyncio.wait(asyncio.all_tasks(), return_when=asyncio.FIRST_COMPLETED)
    assert zdo_f.reply.await_count == 1
    assert zdo_f.reply.call_args[0][3]

async def test_handle_match_desc_generic(zdo_f):
    _handle_match_desc(zdo_f, 0)
    await asyncio.wait(asyncio.all_tasks(), return_when=asyncio.FIRST_COMPLETED)
    assert zdo_f.reply.await_count == 1
    assert not zdo_f.reply.call_args[0][3]

async def test_handle_nwk_addr(zdo_f):
    ieee = zdo_f._device.application.state.node_info.ieee
    zdo_f.reply = MagicMock()
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd.NWK_addr_req
    zdo_f.handle_message(5, 0x0000, hdr, [ieee, 0x00])
    assert zdo_f.reply.call_count == 1

async def test_handle_ieee_addr(zdo_f):
    nwk = zdo_f._device.application.state.node_info.nwk
    zdo_f.reply = MagicMock()
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd.IEEE_addr_req
    zdo_f.handle_message(5, 0x0001, hdr, [nwk, 0x00])
    assert zdo_f.reply.call_count == 1

def test_handle_announce(zdo_f):
    dev = zdo_f._device
    listener = MagicMock()
    zdo_f.add_listener(listener)

    dev._application.devices.pop(dev.ieee)
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd.Device_annce
    zdo_f.handle_message(
        5, 0x0013, hdr, [dev.nwk, dev.ieee, 0], dst_addressing=sentinel.dst_addr
    )

    assert listener.device_announce.call_count == 1
    assert listener.device_announce.call_args[0][0] is dev

    assert listener.zdo_device_annce.call_count == 1
    assert listener.zdo_device_annce.call_args[0][0] is dev
    assert listener.zdo_device_annce.call_args[0][1] is sentinel.dst_addr
    assert listener.zdo_device_annce.call_args[0][2] is hdr
    assert listener.zdo_device_annce.call_args[0][3] == [dev.nwk, dev.ieee, 0]

def test_handle_permit_join(zdo_f):
    listener = MagicMock()
    zdo_f.add_listener(listener)
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd.Mgmt_Permit_Joining_req
    zdo_f.handle_message(5, 0x0036, hdr, [100, 1])
    assert listener.permit_duration.call_count == 1

def test_handle_unsupported(zdo_f):
    listener = MagicMock()
    zdo_f.add_listener(listener)
    hdr = MagicMock()
    hdr.command_id = zdo_types.ZDOCmd(0xFFFF)
    assert hdr.command_id not in list(zdo_types.ZDOCmd)
    zdo_f.request = MagicMock()
    zdo_f.reply = MagicMock()
    zdo_f.handle_message(5, 0xFFFF, hdr, [])

    assert listener.zdo_undefined_0xffff.call_count == 1
    assert zdo_f.request.call_count == 0
    assert zdo_f.reply.call_count == 0

def test_device_accessor(zdo_f):
    assert zdo_f.device.nwk == 65535

async def test_reply(zdo_f):
    zdo_f.device.request = AsyncMock()
    await zdo_f.reply(0x0005)
    assert zdo_f.device.request.call_count == 1

def test_get_attr_error(zdo_f):
    with pytest.raises(AttributeError):
        zdo_f.no_such_attribute()

async def test_reply_tsn_override(zdo_f, monkeypatch):
    clusters = MagicMock()
    clusters.__getitem__.return_value = (
        sentinel.param_names,
        sentinel.scheam,
    )
    monkeypatch.setattr(zdo_types, "CLUSTERS", clusters)
    mock_ser = MagicMock()
    mock_ser.return_value = b"\xaa\x55"
    monkeypatch.setattr(t, "serialize", mock_ser)
    await zdo_f.reply(sentinel.cmd, sentinel.arg1, sentinel.arg2)
    seq = zdo_f.device.request.call_args[0][4]
    data = zdo_f.device.request.call_args[0][5]
    assert seq == 123
    assert data[0] == 123
    assert data[1:3] == b"\xaa\x55"

    # override tsn
    tsn = 0x23
    await zdo_f.reply(sentinel.cmd, sentinel.arg1, sentinel.arg2, tsn=tsn)
    seq = zdo_f.device.request.call_args[0][4]
    data = zdo_f.device.request.call_args[0][5]
    assert seq == tsn
    assert data[0] == tsn
    assert data[1:3] == b"\xaa\x55"


#!/usr/bin/env python3

"""
Module which consists of functions for basic number operations.
1. Drawing a set  of random numbers.
2. Putting numbers to a coupon.
"""

import random
import pygal

def draw_numbers_imp():
    line = set(random.sample(range(1,50),6))
    return line

def user_numbers():
    """
    Function which lets a user pick a valid set of 6 numbers
    :return: a set of 6 integers
    """
    coupon = set()
    while len(coupon) < 6:
        try:
            to_add = int(input("Please pick a number between 1-49: "))
            if to_add < 1 or to_add > 49 or to_add in coupon:
                raise ValueError
        except ValueError:
            print("Not a valid number")
        else:
            coupon.add(to_add)
            print("Number added!")
            print(coupon)
    return coupon

def classify_results(have, drawn):
    prizes = {0: 0, 1: 0, 2: 0, 3: 24, 4: 258, 5: 9500, 6: 2000000}
    hit = len(have.intersection(drawn))
    prize = prizes[hit]
    return hit, prize

def main():
    n = 4500
    have = user_numbers()
    money_lost = n * 3
    money_won = 0
    histogram = {}

    # simmulation
    print("Starting simulation:\n")
    for i in range(n):
        if i % 500 == 0:
            print(i)
        drawn = draw_numbers_imp()

        # cummulative functions
        x = classify_results(have, drawn)
        histogram[x[0]] = histogram.get(x[0], 0) + 1
        money_won += x[1]

    # print the results\n
    print("\nSimulation complete for {:d} number of draws".format(n))
    print("*" * 40)
    print("\nMoney spent on lottery: {:d} PLN".format(money_lost))
    print("Money won in the loterry: {:d} PLN".format(money_won))
    print("*" * 40)
    for key, value in histogram.items():
        if key >= 3:
            print("You've correctly picked {:d} numbers {:d} times".format(key, value))

    # create a bar chart
    print("\nWould you like to create a chart? Type 'y' to confirm")
    choice = str(input("\nType 'y' to confirm "))
    if choice == "y":
        line_chart = pygal.Bar()
        line_chart.title = "Results of " + str(n) + " lotto draws"
        line_chart.x_labels = ['1', '2', '3', '4', '5', '6']
        line_chart.x_title = "Numbers matching the draw"
        line_chart.y_title = "Frequency of result"

        line_chart.add('', histogram.values())
        line_chart.render_to_png('chart.png')
    else:
        pass

if __name__ == main():
    main()


import re
from pathlib import Path
from typing import Any, Union

from skops.io import dump, load

def load_model(path: Union[str, Path]) -> tuple[Any, float, float, str]:
    trusted_types = ['numpy.float64']
    typed_path = path if isinstance(path, Path) else Path(path)
    m = re.match(
        'svm_win_len_(?P<win_len>1\.0|0.\d+)_overlap_(?P<overlap>1.0|0\.\d+)_win_type_(?P<win_type>.*?)\..*',
        typed_path.name)
    win_length = float(m.group('win_len'))
    overlap_percentage = float(m.group('overlap'))
    win_type = m.group('win_type')

    model = load(path, trusted = trusted_types)
    return model, win_length, overlap_percentage, win_type

def save_model(model: Any, path: Union[str, Path]) -> None:
    Path(path).parent.mkdir(exist_ok = True, parents = True)
    dump(model, path)


from django.db import migrations

class Migration(migrations.Migration):
    def add_data_to_leagues(apps, schema_editor):
        League = apps.get_model('main', 'League')
        CustomDesignStyle = apps.get_model('overlays', 'CustomDesignStyle')

        leagues = League.objects.all()
        for league in leagues:
            custom_design_style = CustomDesignStyle(league=league)
            custom_design_style.save()

    def add_data_to_users(apps, schema_editor):
        User = apps.get_model('auth', 'User')
        UserOverlay = apps.get_model('overlays', 'UserOverlay')

        users = User.objects.all()
        for user in users:
            user_overlay = UserOverlay(user=user)
            user_overlay.save()

    dependencies = [
        ('overlays', '0003_alter_customdesignstyle_options_and_more')
    ]

    operations = [
        migrations.RunPython(add_data_to_leagues),
        migrations.RunPython(add_data_to_users)
    ]


# Math 10 Final Project
# Author: Leo Cheung
# ID: 19421084

import numpy as np
import pandas as pd
import altair as alt
import streamlit as st
from sklearn.linear_model import LinearRegression

#change title of the page to FP-LC, and change icon to sunrise
#emoji code acquired from streamlit's github link at https://www.webfx.com/tools/emoji-cheat-sheet/

st.set_page_config(page_title = "Final Project - Leo Cheung", page_icon = ":sunrise:")
st.title("Math 10 - Analysis of CPI based on pricing of consumer goods")

st.markdown("Author: Leo Cheung, [GitHub link](https://github.com/ctZN4)")

df_show_hide = st.radio(
     "Do you wish to see the select snippets of data displayed throughout this page?",
     ('Show', 'Hide'))

df_toggle = (df_show_hide == "Show")

# P1
st.header("Data cleaning")

st.write("We import the CPI data acquired from the U.S. Bureau of Labor Statistics. ")

#Link to database: https://www.bls.gov/cpi/data.htm
#Downloaded from: https://download.bls.gov/pub/time.series/ap/
df = pd.read_table('ap.data.2.Gasoline.txt', sep='\s+')

#check and see that since the last column is all empty, it is dropped
if df["footnote_codes"].notna().sum() == 0:
    df = df.drop("footnote_codes", axis = 1)

#print df to display the data
if df_toggle:
    st.dataframe(df)
    see_df = "As we can see above, the dataframe"
else:
    see_df = "Our imported data"

st.write(see_df + " contains 3 main columns: the ID that describes the type of data, the year, month, and finally price of the product.")

st.write("The database also has two additional useful files: `ap.area` references the area code of the item, `ap.item` explains the types of products the item code.")

st.write("Thus, for the sake of clarity, we will splice the `series_id` into two additional columns for ease of access.")

df["area_code"] = df["series_id"].astype(str).str[3:7]
df["item_code"] = df["series_id"].astype(str).str[7:]

if df_toggle:
    st.write("Here's a snippet of the modified dataframe:")
    st.write(df.head(5))

df_avg = df[df["area_code"] == "0000"]

st.write("We will count the different types of gasoline this dataset contains:")

#gas_counts = df_avg["item_code"].value_counts()

st.bar_chart(df_avg["item_code"].value_counts())

#Copied from ap.item

#item_code  item_name
#74712	    Gasoline, leaded regular (cost per gallon/3.8 liters)
#74713	    Gasoline, leaded premium (cost per gallon/3.8 liters)
#74714	    Gasoline, unleaded regular, per gallon/3.785 liters
#74715	    Gasoline, unleaded midgrade, per gallon/3.785 liters
#74716	    Gasoline, unleaded premium, per gallon/3.785 liters
#74717	    Automotive diesel fuel, per gallon/3.785 liters
#7471A	    Gasoline, all types, per gallon/3.785 liters

st.write("Note that the type of gasoline with most data points is code `74714`, which refers to the price of unleaded regular gasoline per US gallon. We will isolate those values and sort them by year and month for further calculation.")

df_unl_reg = df_avg[df_avg["item_code"] == "74714"]
df2 = df_unl_reg.sort_values(by = ["year", "period"], axis = 0, ascending = True)

if df_toggle:
    st.write("Here is a preview of the modified dataset:")
    st.write(df2.head(5))

st.write("Checking the shape of the dataframe reveals that we have correctly chosen the type of unleaded gasoline, since the number of rows matches the number above for code `74714` (550).")
st.write(df2.shape)

st.write("We will now perform a series of operations that convert our `year` and `period` columns into one \"DT\" (i.e. datetime) column to represent the time.")
df2["DT"] = df2["year"].astype(str) + df2["period"]
df2["DT"] = df2["DT"].apply(lambda s: s[:4] + "-" + s[5:])
df2["DT"] = pd.to_datetime(df2["DT"])
if df_toggle:
    st.write("Checking the `DT` column of the dataframe shows that our conversion is successful.")
    st.write(df2["DT"].head(3).astype(str))

#P2
st.header("Regression")
st.write("In order to graph the relationship between of the gas price and time, we will use the days elapsed from the first day of year 1970 as a reference.")
df2["ref"] = pd.to_datetime("1970-01-01")
df2["diff"] = df2["DT"] - df2["ref"]
df2["diff"] = df2["diff"].apply(lambda t: int(str(t).split(" ")[0]))

if df_toggle:
    st.write("Below is a sample of the modified dataframe with gas price values and their corresponding date difference:")
    st.write(df2[["value", "diff"]].head(3))

chart_choice = st.selectbox(
    'Which regression graphs would you want to see?',
    ("Both regular and premium gasoline", "Regular gasoline only", "Premium gasoline only"))
view_chart = chart_choice[0]

reg = LinearRegression()
dfr = pd.DataFrame(df2["diff"].copy())
dfr = dfr.rename(columns={"diff":"x"})

#degree of polynomial
deg = 4
for i in range(deg):
    dfr[str("x"+str(i+1))] = dfr["x"] ** (i+1)

#remove column of "x" and keep the remaining "x_i"
dfr.drop(labels = "x", axis = 1)
def make_xi_list(d):
    return [f"x{i}" for i in range(1,d+1)]

reg.fit(dfr[make_xi_list(deg)], df2["value"])
theta0 = reg.intercept_
thetas = reg.coef_

brush = alt.selection_interval()

## Marking original data points
choiceX = "diff"
choiceY = "value"
gasChart = alt.Chart(df2).mark_circle().encode(
    x = alt.X(choiceX, scale=alt.Scale(zero=False)),
    y = alt.Y(choiceY, scale=alt.Scale(zero=False)),
    #NOTE: replaced simple colors with interactive chart, now selected points are colored and unselected are grey
    #color = alt.Color(choiceY, scale = alt.Scale(scheme = "greenblue")),
    color = alt.condition(brush, alt.Color("value:Q", scale = alt.Scale(scheme = "greenblue")), alt.value('grey')),
    opacity = alt.condition(brush, alt.value(1), alt.value(0.2)),
    tooltip = [choiceX, choiceY]
).properties(
    width = 800,
    height = 400
).add_selection(
    brush
)
gasChart.encoding.x.title = 'Time elapsed'
gasChart.encoding.y.title = 'Actual gas price'

## Graphing regression

x = df2["diff"].values.astype(float)

gas_reg = pd.DataFrame({
  'x': x,
  'y': theta0 + thetas[0] * x + thetas[1] * x ** 2 + thetas[2] * x **3 + thetas[3] * x **4
})

gas_reg_chart = alt.Chart(gas_reg).mark_line().encode(
    x = alt.X("x", scale=alt.Scale(zero=False)),
    y = alt.Y("y", scale=alt.Scale(zero=False)),
).properties(
    width = 800,
    height = 400
)
gas_reg_chart.encoding.x.title = 'Time elapsed'
gas_reg_chart.encoding.y.title = 'Predicted price'

if view_chart == "R" or view_chart == "B":
    st.write("The following Altair chart displays the data points of the gas prices and the regression line that represents the general trend.")
    st.altair_chart(gasChart + gas_reg_chart)

st.write("We want to find out the relationship between the regular and premium gasoline prices, and see if there is a correlation between the two. We will redo our previous operations on the unleaded regular gasoline dataset on the unleaded premium gasoline dataset.")

#choose premium, sort by time, then calculate time elapsed as "diff"
df3 = df_avg[df_avg["item_code"] == "74716"].copy()
df3["DT"] = df3["year"].astype(str) + df3["period"]
df3["DT"] = df3["DT"].apply(lambda s: s[:4] + "-" + s[5:])
df3["DT"] = pd.to_datetime(df3["DT"])
df3.sort_values(by = "DT", axis = 0, ascending = True)
df3["ref"] = pd.to_datetime("1970-01-01")
df3["diff"] = df3["DT"] - df3["ref"]
df3["diff"] = df3["diff"].apply(lambda t: int(str(t).split(" ")[0]))

#regression for premium gasoline

reg_p = LinearRegression()
dfr = pd.DataFrame(df3["diff"].copy())
dfr = dfr.rename(columns={"diff":"x"})

#degree of polynomial
deg = 4
for i in range(deg):
    dfr[str("x"+str(i+1))] = dfr["x"] ** (i+1)

dfr.drop(labels = "x", axis = 1)
def make_xi_list(d):
    return [f"x{i}" for i in range(1,d+1)]

reg_p.fit(dfr[make_xi_list(deg)], df3["value"])
theta0 = reg_p.intercept_
thetas = reg_p.coef_

## Marking premium gas data points
choiceX = "diff"
choiceY = "value"
gasChartP = alt.Chart(df3).mark_circle().encode(
    x = alt.X(choiceX, scale=alt.Scale(zero=False)),
    y = alt.Y(choiceY, scale=alt.Scale(zero=False)),
    #NOTE: replaced simple colors with interactive chart, now selected points are colored and unselected are grey
    #color = alt.Color(choiceY, scale = alt.Scale(scheme = "greenblue")),
    color = alt.condition(brush, alt.Color("value:Q", scale = alt.Scale(scheme = "turbo")), alt.value('grey')),
    opacity = alt.condition(brush, alt.value(1), alt.value(0.2)),
    tooltip = [choiceX, choiceY]
).properties(
    width = 800,
    height = 400
).add_selection(
    brush
)
gasChartP.encoding.x.title = 'Time elapsed'
gasChartP.encoding.y.title = 'Actual gas price'

## Graphing regression

x = df3["diff"].values.astype(float)

gas_reg = pd.DataFrame({
  'x': x,
  'y': theta0 + thetas[0] * x + thetas[1] * x ** 2 + thetas[2] * x **3 + thetas[3] * x **4
})

gas_p_chart = alt.Chart(gas_reg).mark_line().encode(
    x = alt.X("x", scale=alt.Scale(zero=False)),
    y = alt.Y("y", scale=alt.Scale(zero=False)),
).properties(
    width = 800,
    height = 400
)
gas_p_chart.encoding.x.title = 'Time elapsed'
gas_p_chart.encoding.y.title = 'Predicted price'

if view_chart == "P" or view_chart == "B":
    st.write("Below is the regression line for the premium gasoline data graphed on its data points.")
    st.altair_chart(gas_p_chart + gasChartP)

st.write("Note the similar shape of the two curves, it indicates that the two values are extremely closely related. So, we will attempt to calculate the difference between the two. We will first store the absolute difference in a new dataframe.")
df2s = df2[df2["diff"] >= df3["diff"].iloc[0]]
df_gas = pd.DataFrame(data = {"days": df3["diff"].values, "regular": df2s["value"].values, "premium": df3["value"].values})
df_gas["delta"] = df_gas["premium"] - df_gas["regular"]

if df_toggle:
    st.write(df_gas)

reg_diff = LinearRegression()
reg_diff.fit(np.array(df_gas["days"]).reshape(-1,1), df_gas["delta"])
theta0 = reg_diff.intercept_
thetas = reg_diff.coef_
st.write("Calculating a regression for the difference of gas prices, we get an intercept of " + str(theta0) + " and coefficient of " + str(thetas[0]))
st.write("Therefore, we can conclude that the difference between the two types of gas prices as time goes on is almost constant, and the increased difference can be accounted for with inflation.")

#P3
st.header("References")

st.write("The project is completed based on the materials covered in the course, and I did not acquire help from external sources other than the instructions found in the documentation of each library, including Pandas, Altair, Sci-kit Learn and Streamlit.")
st.markdown("As stated above, the [CPI database](https://www.bls.gov/cpi/data.htm) is acquired from the US Bureau of Labor Statistics.")


#!/usr/bin/env python3
import numpy as np
import random
import scipy as sp
import sklearn as skl
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

# BINOMIAL EXAMPLE

dice_5times_6sides_100=[]
for i in range(100):
    dice=[random.randint(1,6) for n in range(5)]
    dice_5times_6sides_100.append(sum(dice))

y_100,x_100=np.histogram(dice_5times_6sides_100,range=(5,30),bins=25)

dice_5times_6sides_10000=[]
for i in range(10000):
    dice=[random.randint(1,6) for n in range(5)]
    dice_5times_6sides_10000.append(sum(dice))

y_10k,x_10k=np.histogram(dice_5times_6sides_10000,range=(5,30),bins=25)

plt.figure(1,figsize=(24,12))

plt.subplot2grid((1,2),(0,0))
plt.title("5 Dice, 6 Sides, 100 rolls")
plt.xticks(range(5,31))
plt.bar(x_100[1:],y_100,color='k',width=0.8)
plt.xlabel("Sum")
plt.ylabel("Count")

plt.subplot2grid((1,2),(0,1))
plt.title("5 Dice, 6 Sides, 10000 rolls")
plt.xticks(range(5,31))
plt.bar(x_10k[1:],y_10k,color='k',width=0.8)
plt.xlabel("Sum")
plt.ylabel("Count")

plt.savefig("dice_5times_6sides.pdf",format='pdf',dpi=300)

# POISSON EXAMPLE

dice_product_100=[]
for i in range(100):
    dice=random.randint(1,6)*random.randint(1,6)*random.randint(1,20)
    dice_product_100.append(dice)

y_100,x_100=np.histogram(dice_product_100,range=(1,720),bins=720)

dice_product_100k=[]
for i in range(100000):
    dice=random.randint(1,6)*random.randint(1,6)*random.randint(1,20)
    dice_product_100k.append(dice)

y_100k,x_100k=np.histogram(dice_product_100k,range=(1,720),bins=720)

plt.figure(1,figsize=(24,12))

plt.subplot2grid((1,2),(0,0))
plt.title("D6*D6*D20, 100 rolls")
plt.xticks(range(1,720,100))
plt.bar(x_100[1:],y_100,color='k',width=0.8)
plt.xlabel("Sum")
plt.ylabel("Count")

plt.subplot2grid((1,2),(0,1))
plt.title("D6*D6*D20, 10000 rolls")
plt.xticks(range(1,720,100))
plt.bar(x_100k[1:],y_100k,color='k',width=0.8)
plt.xlabel("Sum")
plt.ylabel("Count")

plt.savefig("dice_product.pdf",format='pdf',dpi=300)


from django.shortcuts import render
from marvelapi.models import Marvel
import requests
import hashlib
import datetime
from rest_framework.decorators import api_view
from rest_framework.response import Response
from dotenv import load_dotenv
import os

#environement variables
load_dotenv()
PRIVATE_KEY = os.getenv('PRIVATE_KEY')
PUBLIC_KEY = os.getenv('PUBLIC_KEY')

# Creating hash for API
timestamp = datetime.datetime.now().strftime('%Y-%m-%d%H:%M:%S')
def hash_params():
    hash_md5 = hashlib.md5()
    hash_md5.update(f'{timestamp}{PRIVATE_KEY}{PUBLIC_KEY}'.encode('utf-8'))
    hashed_params = hash_md5.hexdigest()
    return hashed_params

params = {'ts': timestamp, 'apikey': PUBLIC_KEY, 'hash': hash_params()}

# Getting data from Marvel API
def get_characters():
    url = f'https://gateway.marvel.com:443/v1/public/characters'
    response = requests.get(url, params=params).json()
    character = response['data']['results']
    return character

# Seed data from API to sqlite database
@api_view(['POST'])
def seed_characters(request):
    for i in get_characters():
        character = Marvel.objects.create(character=i['name'])
        character.save()
    return Response({'message': 'Characters seeded successfully'})

# Get data from sqlite database
@api_view(['GET'])
def get_from_database(request):
    for i in Marvel.objects.all():
        return Response({'name': i.character})
    characters = Marvel.objects.all()
    return characters