import re
import os
import argparse
import sys
import pickle
import hashlib
import difflib

# fasm keywords
keywords = [
    "align", "equ", "org", "while", "load", "store", "times", "repeat",
    "display", "err", "assert", "if", "aaa", "aad", "aam", "aas", "adc",
    "add", "addpd", "addps", "addsd", "addss", "addsubpd", "addsubps", "adox",
    "aesdeclast", "aesenc", "aesenclast", "aesimc", "aeskeygenassist", "and",
    "andnpd", "andnps", "andpd", "andps", "arpl", "bextr", "blendpd",
    "blendvpd", "blendvps", "blsi", "blsmsk", "blsr", "bndcl", "bndcn",
    "bndldx", "bndmk", "bndmov", "bndstx", "bound", "bsf", "bsr", "bswap",
    "btc", "btr", "bts", "bzhi", "call", "cbw", "cdq", "cdqe", "clac", "clc",
    "cldemote", "clflush", "clflushopt", "cli", "clts", "clwb", "cmc", "cmova",
    "cmovb", "cmovbe", "cmovc", "cmove", "cmovg", "cmovge", "cmovl", "cmovle",
    "cmovnae", "cmovnb", "cmovnbe", "cmovnc", "cmovne", "cmovng", "cmovnge",
    "cmovnle", "cmovno", "cmovnp", "cmovns", "cmovnz", "cmovo", "cmovp",
    "cmovpo", "cmovs", "cmovz", "cmp", "cmppd", "cmpps", "cmps", "cmpsb",
    "cmpsd", "cmpsq", "cmpss", "cmpsw", "cmpxchg", "cmpxchg16b", "cmpxchg8b",
    "comiss", "cpuid", "cqo", "crc32", "cvtdq2pd", "cvtdq2ps", "cvtpd2dq",
    "cvtpd2ps", "cvtpi2pd", "cvtpi2ps", "cvtps2dq", "cvtps2pd", "cvtps2pi",
    "cvtsd2ss", "cvtsi2sd", "cvtsi2ss", "cvtss2sd", "cvtss2si", "cvttpd2dq",
    "cvttps2dq", "cvttps2pi", "cvttsd2si", "cvttss2si", "cwd", "cwde", "daa",
    "dec", "div", "divpd", "divps", "divsd", "divss", "dppd", "dpps", "emms",
    "extractps", "f2xm1", "fabs", "fadd", "faddp", "fbld", "fbstp", "fchs",
    "fcmova", "fcmovae", "fcmovb", "fcmovbe", "fcmovc", "fcmove", "fcmovg",
    "fcmovl", "fcmovle", "fcmovna", "fcmovnae", "fcmovnb", "fcmovnbe",
    "fcmovne", "fcmovng", "fcmovnge", "fcmovnl", "fcmovnle", "fcmovno",
    "fcmovns", "fcmovnz", "fcmovo", "fcmovp", "fcmovpe", "fcmovpo", "fcmovs",
    "fcom", "fcomi", "fcomip", "fcomp", "fcompp", "fcos", "fdecstp", "fdiv",
    "fdivr", "fdivrp", "ffree", "fiadd", "ficom", "ficomp", "fidiv", "fidivr",
    "fimul", "fincstp", "finit", "fist", "fistp", "fisttp", "fisub", "fisubr",
    "fld1", "fldcw", "fldenv", "fldl2e", "fldl2t", "fldlg2", "fldln2", "fldpi",
    "fmul", "fmulp", "fnclex", "fninit", "fnop", "fnsave", "fnstcw", "fnstenv",
    "fpatan", "fprem", "fprem1", "fptan", "frndint", "frstor", "fsave",
    "fsin", "fsincos", "fsqrt", "fst", "fstcw", "fstenv", "fstp", "fstsw",
    "fsubp", "fsubr", "fsubrp", "ftst", "fucom", "fucomi", "fucomip", "fucomp",
    "fwait", "fxam", "fxch", "fxrstor", "fxsave", "fxtract", "fyl2x",
    "gf2p8affineinvqb", "gf2p8affineqb", "gf2p8mulb", "haddpd", "haddps",
    "hsubpd", "hsubps", "idiv", "imul", "in", "inc", "ins", "insb", "insd",
    "insw", "int", "int1", "int3", "into", "invd", "invlpg", "invpcid", "iret",
    "jmp", "ja", "jae", "jb", "jbe", "jc", "jcxz", "jecxz", "je", "jg", "jge",
    "jle", "jna", "jnae", "jnb", "jnbe", "jnc", "jne", "jng", "jnge", "jnl",
    "jno", "jnp", "jns", "jnz", "jo", "jp", "jpe", "jpo", "js", "jz", "kaddb",
    "kaddq", "kaddw", "kandb", "kandd", "kandnb", "kandnd", "kandnq", "kandnw",
    "kandw", "kmovb", "kmovd", "kmovq", "kmovw", "knotb", "knotd", "knotq",
    "korb", "kord", "korq", "kortestb", "kortestd", "kortestq", "kortestw",
    "kshiftlb", "kshiftld", "kshiftlq", "kshiftlw", "kshiftrb", "kshiftrd",
    "kshiftrw", "ktestb", "ktestd", "ktestq", "ktestw", "kunpckbw", "kunpckdq",
    "kxnorb", "kxnord", "kxnorq", "kxnorw", "kxorb", "kxord", "kxorq", "kxorw",
    "lar", "lddqu", "ldmxcsr", "lds", "lea", "leave", "les", "lfence", "lfs",
    "lgs", "lidt", "lldt", "lmsw", "lock", "lods", "lodsb", "lodsd", "lodsq",
    "loop", "loopa", "loopae", "loopb", "loopbe", "loopc", "loope", "loopg",
    "loopl", "loople", "loopna", "loopnae", "loopnb", "loopnbe", "loopnc",
    "loopng", "loopnge", "loopnl", "loopnle", "loopno", "loopnp", "loopns",
    "loopo", "loopp", "looppe", "looppo", "loops", "loopz", "lsl", "lss",
    "lzcnt", "maskmovdqu", "maskmovq", "maxpd", "maxps", "maxsd", "maxss",
    "minpd", "minps", "minsd", "minss", "monitor", "mov", "movapd", "movaps",
    "movd", "movddup", "movdir64b", "movdiri", "movdq2q", "movdqa", "movdqu",
    "movhpd", "movhps", "movlhps", "movlpd", "movlps", "movmskpd", "movmskps",
    "movntdqa", "movnti", "movntpd", "movntps", "movntq", "movq", "movq",
    "movs", "movsb", "movsd", "movsd", "movshdup", "movsldup", "movsq",
    "movsw", "movsx", "movsxd", "movupd", "movups", "movzx", "mpsadbw", "mul",
    "mulps", "mulsd", "mulss", "mulx", "mwait", "neg", "nop", "not", "or",
    "orps", "out", "outs", "outsb", "outsd", "outsw", "pabsb", "pabsd",
    "pabsw", "packssdw", "packsswb", "packusdw", "packuswb", "paddb", "paddd",
    "paddsb", "paddsw", "paddusb", "paddusw", "paddw", "palignr", "pand",
    "pause", "pavgb", "pavgw", "pblendvb", "pblendw", "pclmulqdq", "pcmpeqb",
    "pcmpeqq", "pcmpeqw", "pcmpestri", "pcmpestrm", "pcmpgtb", "pcmpgtd",
    "pcmpgtw", "pcmpistri", "pcmpistrm", "pdep", "pext", "pextrb", "pextrd",
    "pextrw", "phaddd", "phaddsw", "phaddw", "phminposuw", "phsubd", "phsubsw",
    "pinsrb", "pinsrd", "pinsrq", "pinsrw", "pmaddubsw", "pmaddwd", "pmaxsb",
    "pmaxsq", "pmaxsw", "pmaxub", "pmaxud", "pmaxuq", "pmaxuw", "pminsb",
    "pminsq", "pminsw", "pminub", "pminud", "pminuq", "pminuw", "pmovmskb",
    "pmovzx", "pmuldq", "pmulhrsw", "pmulhuw", "pmulhw", "pmulld", "pmullq",
    "pmuludq", "pop", "popa", "popad", "popcnt", "popf", "popfd", "popfq",
    "prefetchw", "prefetchh", "psadbw", "pshufb", "pshufd", "pshufhw",
    "pshufw", "psignb", "psignd", "psignw", "pslld", "pslldq", "psllq",
    "psrad", "psraq", "psraw", "psrld", "psrldq", "psrlq", "psrlw", "psubb",
    "psubq", "psubsb", "psubsw", "psubusb", "psubusw", "psubw", "ptest",
    "punpckhbw", "punpckhdq", "punpckhqdq", "punpckhwd", "punpcklbw",
    "punpcklqdq", "punpcklwd", "push", "pushw", "pushd", "pusha", "pushad",
    "pushfd", "pushfq", "pxor", "rcl", "rcpps", "rcpss", "rcr", "rdfsbase",
    "rdmsr", "rdpid", "rdpkru", "rdpmc", "rdrand", "rdseed", "rdtsc", "rdtscp",
    "repe", "repne", "repnz", "repz", "ret", "rol", "ror", "rorx", "roundpd",
    "roundsd", "roundss", "rsm", "rsqrtps", "rsqrtss", "sahf", "sal", "sar",
    "sbb", "scas", "scasb", "scasd", "scasw", "seta", "setae", "setb", "setbe",
    "sete", "setg", "setge", "setl", "setle", "setna", "setnae", "setnb",
    "setnc", "setne", "setng", "setnge", "setnl", "setnle", "setno", "setnp",
    "setnz", "seto", "setp", "setpe", "setpo", "sets", "setz", "sfence",
    "sha1msg1", "sha1msg2", "sha1nexte", "sha1rnds4", "sha256msg1",
    "sha256rnds2", "shl", "shld", "shlx", "shr", "shrd", "shrx", "shufpd",
    "sidt", "sldt", "smsw", "sqrtpd", "sqrtps", "sqrtsd", "sqrtss", "stac",
    "std", "sti", "stmxcsr", "stos", "stosb", "stosd", "stosq", "stosw", "str",
    "subpd", "subps", "subsd", "subss", "swapgs", "syscall", "sysenter",
    "sysret", "test", "tpause", "tzcnt", "ucomisd", "ucomiss", "ud",
    "umwait", "unpckhpd", "unpckhps", "unpcklpd", "unpcklps", "valignd",
    "vblendmpd", "vblendmps", "vbroadcast", "vcompresspd", "vcompressps",
    "vcvtpd2udq", "vcvtpd2uqq", "vcvtph2ps", "vcvtps2ph", "vcvtps2qq",
    "vcvtps2uqq", "vcvtqq2pd", "vcvtqq2ps", "vcvtsd2usi", "vcvtss2usi",
    "vcvttpd2udq", "vcvttpd2uqq", "vcvttps2qq", "vcvttps2udq", "vcvttps2uqq",
    "vcvttss2usi", "vcvtudq2pd", "vcvtudq2ps", "vcvtuqq2pd", "vcvtuqq2ps",
    "vcvtusi2ss", "vdbpsadbw", "verr", "verw", "vexpandpd", "vexpandps",
    "vextractf32x4", "vextractf32x8", "vextractf64x2", "vextractf64x4",
    "vextracti32x4", "vextracti32x8", "vextracti64x2", "vextracti64x4",
    "vfixupimmps", "vfixupimmsd", "vfixupimmss", "vfmadd132pd", "vfmadd132ps",
    "vfmadd132ss", "vfmadd213pd", "vfmadd213ps", "vfmadd213sd", "vfmadd213ss",
    "vfmadd231ps", "vfmadd231sd", "vfmadd231ss", "vfmaddsub132pd",
    "vfmaddsub213pd", "vfmaddsub213ps", "vfmaddsub231pd", "vfmaddsub231ps",
    "vfmsub132ps", "vfmsub132sd", "vfmsub132ss", "vfmsub213pd", "vfmsub213ps",
    "vfmsub213ss", "vfmsub231pd", "vfmsub231ps", "vfmsub231sd", "vfmsub231ss",
    "vfmsubadd132ps", "vfmsubadd213pd", "vfmsubadd213ps", "vfmsubadd231pd",
    "vfnmadd132pd", "vfnmadd132ps", "vfnmadd132sd", "vfnmadd132ss",
    "vfnmadd213ps", "vfnmadd213sd", "vfnmadd213ss", "vfnmadd231pd",
    "vfnmadd231sd", "vfnmadd231ss", "vfnmsub132pd", "vfnmsub132ps",
    "vfnmsub132ss", "vfnmsub213pd", "vfnmsub213ps", "vfnmsub213sd",
    "vfnmsub231pd", "vfnmsub231ps", "vfnmsub231sd", "vfnmsub231ss",
    "vfpclassps", "vfpclasssd", "vfpclassss", "vgatherdpd", "vgatherdpd",
    "vgatherdps", "vgatherqpd", "vgatherqpd", "vgatherqps", "vgatherqps",
    "vgetexpps", "vgetexpsd", "vgetexpss", "vgetmantpd", "vgetmantps",
    "vgetmantss", "vinsertf128", "vinsertf32x4", "vinsertf32x8",
    "vinsertf64x4", "vinserti128", "vinserti32x4", "vinserti32x8",
    "vinserti64x4", "vmaskmov", "vmovdqa32", "vmovdqa64", "vmovdqu16",
    "vmovdqu64", "vmovdqu8", "vpblendd", "vpblendmb", "vpblendmd", "vpblendmq",
    "vpbroadcast", "vpbroadcastb", "vpbroadcastd", "vpbroadcastm",
    "vpbroadcastw", "vpcmpb", "vpcmpd", "vpcmpq", "vpcmpub", "vpcmpud",
    "vpcmpuw", "vpcmpw", "vpcompressd", "vpcompressq", "vpconflictd",
    "vperm2f128", "vperm2i128", "vpermb", "vpermd", "vpermi2b", "vpermi2d",
    "vpermi2ps", "vpermi2q", "vpermi2w", "vpermilpd", "vpermilps", "vpermpd",
    "vpermq", "vpermt2b", "vpermt2d", "vpermt2pd", "vpermt2ps", "vpermt2q",
    "vpermw", "vpexpandd", "vpexpandq", "vpgatherdd", "vpgatherdd",
    "vpgatherdq", "vpgatherqd", "vpgatherqd", "vpgatherqq", "vpgatherqq",
    "vplzcntq", "vpmadd52huq", "vpmadd52luq", "vpmaskmov", "vpmovb2m",
    "vpmovdb", "vpmovdw", "vpmovm2b", "vpmovm2d", "vpmovm2q", "vpmovm2w",
    "vpmovqb", "vpmovqd", "vpmovqw", "vpmovsdb", "vpmovsdw", "vpmovsqb",
    "vpmovsqw", "vpmovswb", "vpmovusdb", "vpmovusdw", "vpmovusqb", "vpmovusqd",
    "vpmovuswb", "vpmovw2m", "vpmovwb", "vpmultishiftqb", "vprold", "vprolq",
    "vprolvq", "vprord", "vprorq", "vprorvd", "vprorvq", "vpscatterdd",
    "vpscatterqd", "vpscatterqq", "vpsllvd", "vpsllvq", "vpsllvw", "vpsravd",
    "vpsravw", "vpsrlvd", "vpsrlvq", "vpsrlvw", "vpternlogd", "vpternlogq",
    "vptestmd", "vptestmq", "vptestmw", "vptestnmb", "vptestnmd", "vptestnmq",
    "vrangepd", "vrangeps", "vrangesd", "vrangess", "vrcp14pd", "vrcp14ps",
    "vrcp14ss", "vreducepd", "vreduceps", "vreducesd", "vreducess",
    "vrndscaleps", "vrndscalesd", "vrndscaless", "vrsqrt14pd", "vrsqrt14ps",
    "vrsqrt14ss", "vscalefpd", "vscalefps", "vscalefsd", "vscalefss",
    "vscatterdps", "vscatterqpd", "vscatterqps", "vshuff32x4", "vshuff64x2",
    "vshufi64x2", "vtestpd", "vtestps", "vzeroall", "vzeroupper", "wait",
    "wrfsbase", "wrgsbase", "wrmsr", "wrpkru", "xabort", "xacquire", "xadd",
    "xchg", "xend", "xgetbv", "xlat", "xlatb", "xor", "xorpd", "xorps",
    "xrstor", "xrstors", "xsave", "xsavec", "xsaveopt", "xsaves", "xsetbv",
]

fasm_types = [
    "db", "rb",
    "dw", "rw",
    "dd", "rd",
    "dp", "rp",
    "df", "rf",
    "dq", "rq",
    "dt", "rt",
    "du",
]

# Add kind flag to identifier in id2kind
def id_add_kind(identifier, kind):
    if identifier not in id2kind:
        id2kind[identifier] = ''
    id2kind[identifier] += kind

# Remove kind flag of identifier in id2kind
def id_remove_kind(identifier, kind):
    if identifier in id2kind:
        if kind in id2kind[identifier]:
            id2kind[identifier] = id2kind[identifier].replace(kind, '')

# Get kind of an identifier
def id_get_kind(identifier):
    if identifier in id2kind:
        return id2kind[identifier]
    else:
        return ''

class LegacyAsmReader:
    def __init__(self, file):
        self.file = file
        self.lines = open(file, "r", encoding="utf-8").readlines()
        self.line_idx = 0
        self.i = 0

    def currline(self):
        return self.lines[self.line_idx]

    def curr(self):
        try:
            return self.lines[self.line_idx][self.i]
        except:
            return ''

    def step(self):
        c = self.curr()
        self.i += 1
        # Wrap the line if '\\' followed by whitespaces and/or comment
        while self.curr() == '\\':
            i_of_backslash = self.i
            self.i += 1
            while self.curr().isspace():
                self.i += 1
            if self.curr() == ';' or self.curr() == '':
                self.line_idx += 1
                self.i = 0
            else:
                # There's something other than a comment after the backslash
                # So don't interpret the backslash as a line wrap
                self.i = i_of_backslash
                break
        return c

    def nextline(self):
        c = self.curr()
        while c != '':
            c = self.step()
        self.line_idx += 1
        self.i = 0

    def no_lines(self):
        if self.line_idx >= len(self.lines):
            return True
        return False

    def location(self):
        return f"{self.file}:{self.line_idx + 1}"

    def skip_spaces(self):
        while self.curr().isspace():
            self.step()

class AsmReaderRecognizingStrings(LegacyAsmReader):
    def __init__(self, file):
        super().__init__(file)
        self.in_string = None
        self.should_recognize_strings = True

    def step(self):
        c = super().step()
        if self.should_recognize_strings and (c == '"' or c == "'"):
            # If just now we was at the double or single quotation mark
            # and we aren't in a string yet then say
            # "we are in a string openned with this quotation mark now"
            if self.in_string is None:
                self.in_string = c
            # If just now we was at the double or single quotation mark
            # and we are in the string entered with the same quotation mark
            # then say "we aren't in a string anymore"
            elif self.in_string == c:
                self.in_string = None
        return c

class AsmReaderReadingComments(AsmReaderRecognizingStrings):
    def __init__(self, file):
        super().__init__(file)
        self.status = dict()
        self.status_reset()
        self.comment = ''

    def status_reset(self):
        # If the line has non-comment code
        self.status_has_code = False
        # If the line has a comment at the end
        self.status_has_comment = False
        # Let it recognize strings further, we are definitely out of a comment
        self.should_recognize_strings = True

    def status_set_has_comment(self):
        self.status_has_comment = True
        # Don't let it recognize strings cause we are in a comment now
        self.should_recognize_strings = False

    def status_set_has_code(self):
        self.status_has_code = True

    def update_status(self):
        # If we aren't in a comment and we aren't in a string -
        # say we are now in a comment if ';' met
        if (not self.status_has_comment and
            not self.in_string and
            self.curr() == ';'):
            self.status_set_has_comment()
        # Else if we are in a comment - collect the comment
        elif self.status_has_comment:
            self.comment += self.curr()
        # Else if there's some non-whitespace character out of a comment
        # then the line has code
        elif not self.status_has_comment and not self.curr().isspace():
            self.status_set_has_code()

    def step(self):
        # Get to the next character
        c = super().step()
        # Update status of the line according to the next character
        self.update_status()
        return c

    def nextline(self):
        prev_line = self.currline()
        super().nextline()
        # If the line we leave was not a comment-only line
        # then forget the collected comment
        # Otherwise the collected comment should be complemented by
        # comment from next line in step()
        if self.status_has_code:
            # But we should preserve comment for the next line
            # If previous line set align (cause many functions re documented
            # right before align set, not before their labels)
            if not prev_line.startswith("align "):
                self.comment = ''
        # Reset the line status (now it's the status of the new line)
        self.status_reset()
        # Set new status for this line according to the
        # first character in the line
        self.update_status()

class AsmReaderFetchingIdentifiers(AsmReaderReadingComments):
    def __init__(self, file):
        super().__init__(file)

    def fetch_identifier(self):
        self.skip_spaces()
        result = ''
        while is_id(self.curr()):
            result += self.step()
        return result

class AsmReader(AsmReaderFetchingIdentifiers):
    def __init__(self, file):
        super().__init__(file)

def append_file(full_path, contents):
    if debug_mode:
        if full_path not in output_files:
            output_files[full_path] = ""
        output_files[full_path] += contents
    else:
        f = open(full_path, "a")
        f.write(contents)
        f.close()

class AsmElement:
    def __init__(self, location, name, comment):
        global warnings

        # If the element was constructed during this execution then
        # the element is new
        self.new = True
        self.location = location
        self.file = self.location.split(':')[0].replace('\\', '/')
        self.line = self.location.split(':')[1]
        self.name = name
        self.comment = comment

        if self.comment == '':
            warnings += f'{self.location}: Undocumented element\n'

    def dump(self):
        print(f"\n{self.location}: {self.name}")
        print(f"{self.comment}")

    def emit(self, dest, doxycomment='', declaration=''):
        # Do not emit anything if the symbol is marked as hidden in its comment
        if '@dont_give_a_doxygen' in self.comment:
            return

        global warnings
        # Redefine default declaration
        if declaration == '':
            declaration = f'#define {self.name}'
        # Check doxycomment
        if not doxycomment.endswith('\n'):
            doxycomment += '\n'
        if doxycomment.split('@brief ')[1][0].islower():
            warnings += (f"{self.location}: Brief comment starting from " +
                         "lowercase\n")
        # Build contents to emit
        contents = ''
        contents += '/**\n'
        contents += doxycomment
        contents += (f"@par Source\n" +
                     f"{self.file}:{self.line}\n")
        contents += '*/\n'
        contents += declaration
        contents += '\n\n'
        # Get path to file to emit this
        full_path = dest + '/' + self.file
        # Remove the file on first access if it was
        # created by previous generation
        if full_path not in created_files:
            if os.path.isfile(full_path):
                os.remove(full_path)
            created_files.append(full_path)
        # Create directories need for the file
        os.makedirs(os.path.dirname(full_path), exist_ok=True)
        contents = ''.join([i if ord(i) < 128 else '?' for i in contents])

        append_file(full_path, contents)

class AsmVariable(AsmElement):
    def __init__(self, location, name, comment, type, init):
        super().__init__(location, name, comment)
        self.type = type
        self.init = init

    def dump(self):
        super().dump()
        print(f"(Variable)\n---")

    def emit(self, dest):
        # Build doxycomment specific for the variable
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        doxycomment += (f"@par Initial value\n" +
                        f"{self.init}\n")
        # Build the declaration
        name = self.name.replace(".", "_")
        var_type = self.type.replace(".", "_")
        declaration = f"{var_type} {name};"
        # Emit this
        super().emit(dest, doxycomment, declaration)

class AsmFunction(AsmElement):
    def __init__(self, location, name, comment, calling_convention,
                 args, used_regs):
        super().__init__(location, name, comment)
        self.calling_convention = calling_convention
        self.args = args
        self.used_regs = used_regs

    def dump(self):
        super().dump()
        print(f"(Function)\n---")

    def emit(self, dest):
        # Build doxycomment specific for the variable
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        # If there was no arguments, maybe that's just a label
        # then parse parameters from its comment
        if len(self.args) == 0 and '@param' in self.comment:
            i = 0
            while '@param' in self.comment[i:]:
                i = self.comment.index('@param', i)
                # Skip '@param'
                i += len('@param')
                # Skip spaces after '@param'
                while self.comment[i].isspace():
                    i += 1
                # Get the parameter name
                name = ''
                while is_id(self.comment[i]):
                    name += self.comment[i]
                    i += 1
                # Save the parameter
                self.args.append((name, 'arg_t'))
        # Build the arg list for declaration
        arg_list = '('
        if len(self.args) > 0:
            argc = 0
            for arg in self.args:
                if argc != 0:
                    arg_list += ", "
                arg_list += f"{arg[1]} {arg[0]}"
                argc += 1
        arg_list += ')'
        # Build the declaration
        name = self.name.replace(".", "_")
        declaration = f"void {name}{arg_list};"
        # Emit this
        super().emit(dest, doxycomment, declaration)

class AsmLabel(AsmElement):
    def __init__(self, location, name, comment):
        super().__init__(location, name, comment)

    def dump(self):
        super().dump()
        print(f"(Label)\n---")

    def emit(self, dest):
        # Build doxycomment specific for the variable
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        # Build the declaration
        name = self.name.replace(".", "_")
        declaration = f"label {name};"
        # Emit this
        super().emit(dest, doxycomment, declaration)

class AsmMacro(AsmElement):
    def __init__(self, location, name, comment, args):
        super().__init__(location, name, comment)
        self.args = args

    def dump(self):
        super().dump()
        print(f"(Macro)\n---")

    def emit(self, dest):
        # Construct arg list without '['s, ']'s and '*'s
        args = [arg for arg in self.args if arg not in "[]*"]
        # Construct C-like arg list
        arg_list = ""
        if len(args) > 0:
            arg_list += '('
            argc = 0
            for arg in args:
                if argc != 0:
                    arg_list += ", "
                arg_list += arg
                argc += 1
            arg_list += ')'
        # Build doxycomment
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        # Build declaration
        declaration = f"#define {self.name}{arg_list}"
        # Emit this
        super().emit(dest, doxycomment, declaration)

class AsmStruct(AsmElement):
    def __init__(self, location, name, comment, members):
        super().__init__(location, name, comment)
        self.members = members

    def dump(self):
        super().dump()
        print(f"(Struct)\n---")

    def emit(self, dest):
        # Build doxycomment
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        doxycomment += '\n'
        # Build declaration
        declaration = f"struct {self.name}" + " {\n"
        for member in self.members:
            if type(member) == AsmVariable:
                declaration += (f'\t{member.type} {member.name}; ' +
                                f'/**< {member.comment} */\n')
        declaration += '};'
        # Emit this
        super().emit(dest, doxycomment, declaration)

class AsmUnion(AsmElement):
    def __init__(self, location, name, comment, members):
        super().__init__(location, name, comment)
        self.members = members

    def dump(self):
        super().dump()
        print(f"(Union)\n---")

    def emit(self, dest):
        # Build doxycomment
        doxycomment = ''
        doxycomment += self.comment
        if '@brief' not in doxycomment:
            doxycomment = '@brief ' + doxycomment
        # Build declaration
        declaration = f"union {self.name}" + " {};"
        # Emit this
        super().emit(dest, doxycomment, declaration)

class VariableNameIsMacroName:
    def __init__(self, name):
        self.name = name

def is_id(c):
    return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v"

def is_starts_as_id(s):
    return not s[0].isdigit()

def parse_after_macro(r):
    location = r.location()

    # Skip spaces after the "macro" keyword
    r.skip_spaces()
    # Read macro name
    name = ""
    while is_id(r.curr()) or r.curr() == '#':
        name += r.step()
    # Skip spaces after macro name
    r.skip_spaces()
    # Find all arguments
    args = []
    arg = ''
    while r.curr() and r.curr() != ';' and r.curr() != '{':
        # Collect identifier
        if is_id(r.curr()):
            arg += r.step()
        # Save the collected identifier
        elif r.curr() == ',':
            args.append(arg)
            arg = ''
            r.step()
        # Just push the '['
        elif r.curr() == '[':
            args.append(r.step())
        # Just push the identifier and get ']' ready to be pushed on next comma
        elif r.curr() == ']':
            args.append(arg)
            arg = r.step()
        # Just push the identifier and get '*' ready to be pushed on next comma
        elif r.curr() == '*':
            args.append(arg)
            arg = r.step()
        # Just skip whitespaces
        elif r.curr().isspace():
            r.step()
        # Something unexpected
        else:
            raise Exception(f"Unexpected symbol '{r.curr()}' " +
                            f"at index #{r.i} in the macro declaration " +
                            f"at {location} " +
                            f"(line: {r.lines[r.line_idx]})\n''")
    # Append the last argument
    if arg != '':
        args.append(arg)
    # Skip t spaces after the argument list
    r.skip_spaces()
    # Get a comment if it is: read till the end of the line and
    # get the comment from the reader
    while r.curr() != '':
        r.step()
    comment = r.comment
    # Find end of the macro
    prev = ''
    while True:
        if r.curr() == '}' and prev != '\\':
            break
        elif r.curr() == '':
            prev = ''
            r.nextline()
            continue
        prev = r.step()
    # Build the output
    return AsmMacro(location, name, comment, args)

def parse_variable(r, first_word=None):
    global warnings
    location = r.location()

    # Skip spaces before variable name
    r.skip_spaces()
    # Get variable name
    name = ""
    # Read it if it was not supplied
    if first_word is None:
        while is_id(r.curr()):
            name += r.step()
    # Or use the supplied one instead
    else:
        name = first_word
    # Check the name
    # If it's 0 len, that means threr's something else than an
    # identifier at the beginning
    if len(name) == 0:
        return None
    # If it starts from digit or othervice illegally it's illegal
    if not is_starts_as_id(name):
        return None
    # Get kind of the identifier from id2kind table
    kind = id_get_kind(name)
    # If it's a keyword, that's not a variable declaration
    if ID_KIND_KEYWORD in kind:
        return None
    # If it's a macro name, that's not a variable declaration
    if ID_KIND_MACRO_NAME in kind:
        return VariableNameIsMacroName(name)
    # If it's a datatype or a structure name that's not a
    # variable declaration: that's just a data
    # don't document just a data for now
    if ID_KIND_STRUCT_NAME in kind or ID_KIND_FASM_TYPE in kind:
        return None
    # Skip spaces before type name
    r.skip_spaces()
    # Read type name
    var_type = ""
    while is_id(r.curr()):
        var_type += r.step()
    # Check the type name
    if len(var_type) == 0:
        # If there's no type identifier after the name
        # maybe the name is something meaningful for the next parser
        # return it
        return name
    # If it starts from digit or othervice illegally it's illegal
    if not is_starts_as_id(var_type):
        return None
    # Get kind of type identifier
    type_kind = id_get_kind(var_type)
    # If it's a keyword, that's not a variable declaration
    # return the two words of the lexical structure
    if ID_KIND_KEYWORD in type_kind:
        return (name, var_type)
    # Skip spaces before the value
    r.skip_spaces()
    # Read the value until the comment or end of the line
    value = ""
    while r.curr() != ';' and r.curr() != '' and r.curr() != '\n':
        value += r.step()
    # Skip spaces after the value
    r.skip_spaces()
    # Read till end of the line to get a comment from the reader
    while r.curr() != '':
        r.step()
    # Build the result
    return AsmVariable(location, name, r.comment, var_type, value)

def parse_after_struct(r, as_union=True):
    global warnings
    location = r.location()

    # Skip spaces after "struct" keyword
    r.skip_spaces()
    # Read struct name
    name = ""
    while is_id(r.curr()):
        name += r.step()
    # Read till end of the line and get the comment from the reader
    while r.curr() != '':
        r.step()
    comment = r.comment
    # Get to the next line to parse struct members
    r.nextline()
    # Parse struct members
    members = []
    while True:
        r.skip_spaces()
        var = parse_variable(r)
        if type(var) == AsmVariable:
            members.append(var)
        elif type(var) == str:
            if var == 'union':
                # Parse the union as a struct
                union = parse_after_struct(r, as_union=True)
                members.append(union)
                # Skip the ends of the union
                r.nextline()
            elif r.curr() == ':':
                warnings += f"{r.location()}: Skept the label in the struct\n"
            else:
                raise Exception(f"Garbage in struct member at {location} " +
                                f" (got '{var}' identifier)")
        elif type(var) == VariableNameIsMacroName:
            if var.name == 'ends':
                break
        r.nextline()
    # Return the result
    if as_union:
        return AsmStruct(location, name, comment, members)
    else:
        return AsmUnion(location, name, comment, members)

def parse_after_proc(r):
    # Get proc name
    name = r.fetch_identifier()
    # Next identifier after the proc name
    identifier = r.fetch_identifier()
    # Check if the id is 'stdcall' or 'c' (calling convention specifier)
    # and if so - save the convention and lookup the next identifier
    calling_convention = ''
    if identifier == 'stdcall' or identifier == 'c':
        calling_convention = identifier
        # If next is a comma, just skip it
        if r.curr() == ',':
            r.step()
        # Read the next identifier
        identifier = r.fetch_identifier()
    # Check if the id is 'uses' (used register list specifier)
    # and if so save the used register list
    used_regs = []
    if identifier == 'uses':
        # Read the registers
        while True:
            reg_name = r.fetch_identifier()
            if reg_name != '':
                used_regs.append(reg_name)
            else:
                break
        # If next is a comma, just skip it
        if r.curr() == ',':
            r.step()
        # Read the next identifier
        identifier = r.fetch_identifier()
    # Check if there are argument identifiers
    args = []
    while identifier != '':
        arg_name = identifier
        arg_type = 'arg_t'
        # Skip spaces after argument name
        r.skip_spaces()
        # If there's a ':' after the name - the next identifier is type
        if r.curr() == ':':
            r.step()
            arg_type = r.fetch_identifier()
        # If there's a comma - there's one more argument
        # else no arguments anymore
        if r.curr() == ',':
            r.step()
            identifier = r.fetch_identifier()
        else:
            identifier = ''
        args.append((arg_name, arg_type))
    # Get to the end of the line and get a comment from the reader
    while r.curr() != '':
        r.step()
    comment = r.comment
    # Build the element
    return AsmFunction(r.location(), name, comment, calling_convention,
                       args, used_regs)

def get_declarations(asm_file_contents, asm_file_name):
    r = AsmReader(asm_file_name)

    while not r.no_lines():
        # Skip leading spaces
        r.skip_spaces()
        # Skip the line if it's starting with a comment
        if r.curr() == ';':
            r.nextline()
            continue
        # Get first word
        first_word = ""
        while is_id(r.curr()):
            first_word += r.step()
        # Match macro declaration
        if first_word == "macro":
            macro = parse_after_macro(r)
            elements.append(macro)
            id_add_kind(macro.name, ID_KIND_MACRO_NAME)
        # Match structure declaration
        elif first_word == "struct":
            struct = parse_after_struct(r)
            elements.append(struct)
            id_add_kind(struct.name, ID_KIND_STRUCT_NAME)
        # Match function definition
        elif first_word == "proc":
            proc = parse_after_proc(r)
            elements.append(proc)
        elif first_word == 'format':
            # Skip the format directive
            pass
        elif first_word == 'include':
            # Skip the include directive
            pass
        elif first_word == 'if':
            # Skip the conditional directive
            pass
        elif first_word == 'repeat':
            # Skip the repeat directive
            pass
        elif first_word == 'purge':
            while True:
                # Skip spaces after the 'purge' keyword or after
                # the comma what separated the previous macro name
                r.skip_spaces()
                # Get the purged macro name
                name = ''
                while is_id(r.curr()):
                    name += r.step()
                # Remove the purged macro from the macro names list
                try:
                    id_remove_kind(name, ID_KIND_MACRO_NAME)
                except:
                    pass
                # Skip spaces after the name
                r.skip_spaces()
                # If it's comma (',') after then that's not the last purged
                # macro, continue purging
                if r.curr() == ',':
                    r.step()
                    continue
                # Here we purged all the macros should be purged
                break
        # Match label or a variable
        elif len(first_word) != 0:
            # Skip spaces after the identifier
            r.skip_spaces()
            # Match a variable
            var = parse_variable(r, first_word)
            if type(var) == AsmVariable:
                elements.append(var)
            # If it wasn't a variable but there was an identifier
            # Maybe that's a label and the identifier is the label name
            # The parse_variable returns the first found or supplied identifier
            # In this case it returns the first_word which is supplied
            # If it didn't match a type identifier after the word
            elif type(var) == str:
                name = var
                # Match label beginning (':' after name)
                if r.curr() == ':':
                    # Get to the end of the line and
                    # get the coment from the reader
                    while r.curr() != '':
                        r.step()
                    comment = r.comment
                    # Only handle non-local labels
                    if name[0] != '.' and name != "@@" and name != "$Revision":
                        # Treate the label as function if there's @return or
                        # @param in its comment. Othervice it's just a variable
                        # with type `label` in generated doxygen C
                        if '@return' in comment or '@param' in comment:
                            element = AsmFunction(r.location(), name, comment,
                                                  '', [], [])
                        else:
                            element = AsmLabel(r.location(), name, comment)
                        elements.append(element)
                elif r.curr() == '=':
                    # Save the identifier as a set constant
                    id_add_kind(first_word, ID_KIND_SET_CONSTANT)
            elif type(var) == tuple:
                (word_one, word_two) = var
                if word_two == 'equ':
                    # Save the identifier as an equated constant
                    id_add_kind(word_one, ID_KIND_EQUATED_CONSTANT)
        r.nextline()

def it_neds_to_be_parsed(source_file):
    # If there's no symbols file saved - parse it anyway
    # cause we need to create the symbols file and use it
    # if we gonna generate proper doxygen
    if not os.path.isfile('asmxygen.elements.pickle'):
        return True
    dest = doxygen_src_path + '/' + source_file
    # If there's no the doxygen file it should be compiled to
    # then yes, we should compile it to doxygen
    if not os.path.isfile(dest):
        return True
    source_change_time = os.path.getmtime(source_file)
    dest_change_file = os.path.getmtime(dest)
    # If the source is newer than the doxygen it was compiled to
    # then the source should be recompiled (existing doxygen is old)
    if source_change_time > dest_change_file:
        return True
    return False

def handle_file(handled_files, asm_file_name, subdir="."):
    global elements
    # Canonicalize the file path and get it relative to cwd
    cwd = os.path.abspath(os.path.dirname(sys.argv[0]))
    asm_file_name = os.path.realpath(asm_file_name)
    asm_file_name = asm_file_name[len(cwd) + 1:]
    # If it's lang.inc - skip it
    if asm_file_name == 'lang.inc':
        return
    # If the file was handled in this execution before - skip it
    if asm_file_name in handled_files:
        return
    # Say that the file was handled in this execution
    handled_files.append(asm_file_name)
    # Check if the file should be parsed
    # (if it was modified or wasn't parsed yet)
    should_get_declarations = True
    if not it_neds_to_be_parsed(asm_file_name):
        print(f"Skipping {asm_file_name} (already newest)")
        should_get_declarations = False
    else:
        print(f"Handling {asm_file_name}")
        # Remove elements parsed from this file before if any
        elements_to_remove = [
            x for x in elements if x.location.split(':')[0] == asm_file_name
        ]
        elements = [
            x for x in elements if x.location.split(':')[0] != asm_file_name
        ]
        # Forget types of identifiers of names of the removed elements
        for element in elements_to_remove:
            if type(element) == AsmStruct:
                id_remove_kind(element.name, ID_KIND_STRUCT_NAME)
            elif type(element) == AsmMacro:
                id_remove_kind(element.name, ID_KIND_MACRO_NAME)
    # Read the source
    asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read()
    # Find includes, fix their paths and handle em recoursively
    includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents,
                          flags=re.MULTILINE)
    for include in includes:
        include = include[1].replace('\\', '/')
        full_path = subdir + '/' + include
        # If the path isn't valid, maybe that's not relative path
        if not os.path.isfile(full_path):
            full_path = include
        new_subdir = full_path.rsplit('/', 1)[0]
        handle_file(handled_files, full_path, new_subdir)
    # Only collect declarations from the file if it wasn't parsed before
    if should_get_declarations and not clean_generated_stuff:
        get_declarations(asm_file_contents, asm_file_name)

if __name__ == "__main__":
    link_root = "http://websvn.kolibrios.org/filedetails.php"
    link_root += "?repname=Kolibri+OS&path=/kernel/trunk"

    # Dict where an identifier is assicoated with a string
    # The string contains characters specifying flags
    # Available flags:
    #  k - Keyword
    #  m - Macro name
    #  t - fasm data Type name (db, rq, etc.)
    #  s - Struct type name
    #  e - equated constant (name equ value)
    #  = - set constants (name = value)
    ID_KIND_KEYWORD = 'k'
    ID_KIND_MACRO_NAME = 'm'
    ID_KIND_FASM_TYPE = 't'
    ID_KIND_STRUCT_NAME = 's'
    ID_KIND_EQUATED_CONSTANT = 'e'
    ID_KIND_SET_CONSTANT = '='
    id2kind = {}

    for keyword in keywords:
        id_add_kind(keyword, ID_KIND_KEYWORD)

    for fasm_type in fasm_types:
        id_add_kind(fasm_type, ID_KIND_FASM_TYPE)

    # Warning list
    warnings = ""

    # Parameters
    # Path to doxygen folder to make doxygen files in: -o <path>
    doxygen_src_path = 'docs/doxygen'
    # Remove generated doxygen files: --clean
    clean_generated_stuff = False
    # Dump all defined symbols: --dump
    dump_symbols = False
    # Print symbol stats: --stats
    print_stats = False
    # Do not write warnings file: --nowarn
    enable_warnings = True

    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-o", help="Doxygen output folder")
    parser.add_argument("--clean",
                        help="Remove generated files",
                        action="store_true")
    parser.add_argument("--dump",
                        help="Dump all defined symbols",
                        action="store_true")
    parser.add_argument("--stats",
                        help="Print symbol stats",
                        action="store_true")
    parser.add_argument("--nowarn",
                        help="Do not write warnings file",
                        action="store_true")
    parser.add_argument("--noemit",
                        help="Do not emit doxygen files (for testing)",
                        action="store_true")
    parser.add_argument("--debug",
                        help="Show hashes of files (for testing)",
                        action="store_true")
    args = parser.parse_args()
    doxygen_src_path = args.o if args.o else 'docs/doxygen'
    clean_generated_stuff = args.clean
    dump_symbols = args.dump
    print_stats = args.stats
    enable_warnings = not args.nowarn
    noemit = args.noemit
    debug_mode = args.debug

    # Variables, functions, labels, macros, structure types
    elements = []
    created_files = []
    kernel_files = []
    output_files = {}  # If --debug then all the files are written here

    # Load remembered list of symbols
    if os.path.isfile('asmxygen.elements.pickle'):
        print('Reading existing dump of symbols')
        pickle_file = open('asmxygen.elements.pickle', 'rb')
        (elements, id2kind) = pickle.load(pickle_file)
        pickle_file.close()

    handle_file(kernel_files, "./kernel.asm")

    if dump_symbols:
        stdout = sys.stdout
        sys.stdout = open('asmxygen.dump.txt', 'w', encoding='utf-8')
        for asm_element in elements:
            asm_element.dump()
        sys.stdout = stdout

    if clean_generated_stuff:
        kernel_files_set = set(kernel_files)
        for file in kernel_files:
            doxygen_file = f"{doxygen_src_path}/{file}"
            if (os.path.isfile(doxygen_file)):
                print(f"Removing {file}... ", end='')
                os.remove(doxygen_file)
                print("Done.")
    elif not noemit:
        print(f"Writing doumented sources to {doxygen_src_path}")

        i = 0
        new_elements = [x for x in elements if x.new]
        for element in new_elements:
            counter = f"[{i + 1}/{len(new_elements)}]"
            print(f"{counter} Emitting {element.name} from {element.location}")
            element.emit(doxygen_src_path)
            i += 1

        print(f"Writing dump of symbols to asmxygen.elements.pickle")

        # Now when the new elements already was written, there's no new
        # elements anymore
        for element in elements:
            element.new = False
        pickle_file = open('asmxygen.elements.pickle', 'wb')
        pickle.dump((elements, id2kind), pickle_file)
        pickle_file.close()

    if print_stats:
        var_count = 0
        mac_count = 0
        lab_count = 0
        fun_count = 0
        uni_count = 0
        str_count = 0
        for element in elements:
            if type(element) == AsmVariable:
                var_count += 1
            elif type(element) == AsmMacro:
                mac_count += 1
            elif type(element) == AsmLabel:
                lab_count += 1
            elif type(element) == AsmFunction:
                fun_count += 1
            elif type(element) == AsmUnion:
                uni_count += 1
            elif type(element) == AsmStruct:
                str_count += 1
        print(f'Parsed variable count: {var_count}')
        print(f'Parsed macro count: {mac_count}')
        print(f'Parsed label count: {lab_count}')
        print(f'Parsed function count: {fun_count}')
        print(f'Parsed union type count: {uni_count}')
        print(f'Parsed structure type count: {str_count}')

    if enable_warnings:
        open('asmxygen.txt', "w", encoding="utf-8").write(warnings)

    if debug_mode:
        hash_per_file = ""
        for file in output_files:
            h = hashlib.sha1(bytes(output_files[file], "ascii")).hexdigest()
            hash_per_file += f"{file}: {h}\n"
        if not os.path.exists("asmxygen_hash_per_file.txt"):
            open("asmxygen_hash_per_file.txt", "w").write(hash_per_file)
            print("NEW")
        else:
            reference_hash_per_file = open("asmxygen_hash_per_file.txt").read()
            if reference_hash_per_file != hash_per_file:
                diffs = difflib.ndiff(reference_hash_per_file, hash_per_file)
                print(''.join(diffs))
            else:
                print("SUCCESS")


from collections import deque

n = int(input())

cards = deque()

for i in range(1, n + 1):
    cards.append(i)

cards.reverse()

while len(cards) != 1:
    cards.pop()
    cards.rotate(1)

print(cards[0])


import geopandas
import matplotlib.pyplot as plt
import geoplot

world = geopandas.read_file(
    geopandas.datasets.get_path('naturalearth_lowres')
)
powiaty = 'powiaty.gpkg'

gdf = geopandas.read_file(powiaty, layer='powiaty')
gdf['area'] = gdf['geometry'].area

print(gdf.columns)

fig, ax = plt.subplots(1, figsize=(14, 14))
gdf.plot(ax=ax, edgecolor='0.4', linewidth=0.5,
         column='area', cmap='Greens', legend=True, scheme='quantiles',k=5)
ax.axis('off')
plt.show()

africa = world.query('continent == "Africa"')
#print(af)
ax = geoplot.cartogram(
    africa, scale='pop_est', limits=(0.2, 1),
    edgecolor='None', figsize=(7, 8)
)
geoplot.polyplot(africa, edgecolor='gray', ax=ax)
plt.show()


import requests

from django.conf import settings

def update_story(story_id, story_url):
    mutation_update_query = """
        mutation($story: UpdateStoryInput!) {
            updateStory(input: $story) {
                    story {
                        id,
                        url
                    }
                }
            }
    """
    variables = """
        {{
            "story" : {{
                "id": "{story_id}",
                "clientMutationId": "{story_id}",
                "story": {{
                    "url": "{story_url}"
                }}
            }}
        }}
    """.format(story_id=story_id, story_url='https://www.dpexnetwork.org' + story_url)
    headers = {
        'Authorization': 'Bearer {}'.format(settings.COMMENT_APP_TOKEN)
    }
    response = requests.post('https://comments.genexist.com/api/graphql/',
                             json={
                                 'query': mutation_update_query,
                                 'variables': variables,
                             },
                             headers=headers
                             )
    errors = response.json().get('errors')
    if errors:
        raise Exception('Can not update story: {}'.format(errors))
    return True


#-*- coding:utf-8 -*-

from scrapy.spiders import Spider
from scrapy import Request
from utils.files import get_content_list
from newspaper import Article
from en_p.items import EnPItem
from en_p.utils.files import save_mess

API_URL = "https://www.sciencedaily.com/xml/summaries.php?"\
                "section={}&topic={}&&start={}&end={}"

class sciSpider(Spider):

    name = "science"

    def start_requests(self):
        for line in get_content_list("science.txt"):
            values = line.split("/")
            if len(values) != 3:
                continue
            cur_page = 1
            url = API_URL.format(values[0], values[1], (cur_page-1)*20, cur_page*20)

            meta_data = {"page":cur_page,
                         "tags":[values[0], values[1]]}

            yield Request(url, meta=meta_data)

    def parse(self, response):
        page_idx = response.meta["page"]
        tags = response.meta["tags"]

        links = response.xpath('//*[@class="latest-head"]/a/@href').extract()
        for link in links:
            yield Request("https://www.sciencedaily.com%s"%link,
                          callback=self.parse_article,
                          meta={"tags":tags})

        url = API_URL.format(tags[0], tags[1], (page_idx) * 20, (page_idx+1) * 20)
        meta_data = {"page": page_idx+1,
                     "tags": [tags[0], tags[1]]}

        yield Request(url, meta=meta_data)

    def parse_article(self, response):

        tags = response.meta["tags"]
        item = EnPItem()
        art_parser = Article(response.url, language='en', fetch_images=False)
        # a.download()
        art_parser.set_html(response.text)
        art_parser.parse()

        item["home"] = response.url
        item["title"] = art_parser.title

        item["content"] = art_parser.text
        item["authors"] = art_parser.authors
        try:
            item["publish_date"] = art_parser.publish_date.strftime('%Y-%m-%d %H:%M:%S')
        except:
            pass
        item["images"] = list(art_parser.images)
        item["keywords"] = art_parser.keywords
        item["meta_keywords"] = art_parser.meta_keywords
        item["tags"] = tags  # list(art_parser.tags)

        print(item)
        save_mess("%s.txt" % self.name, json.dumps(dict(item), ensure_ascii=False))


on3
""" Module for implementing MRU Caching
"""

from collections import deque

BaseCaching = __import__("base_caching").BaseCaching

class MRUCache(BaseCaching):
    """  MRUCache class that inherits from BaseCaching which is a caching
    system
    """

    def __init__(self):
        """ Initialization function
        """
        super().__init__()
        self.queue = deque()

    def put(self, key, item):
        """ Assigns item value to the  self.cache_data library
        """
        if key and item:
            if key in self.cache_data:
                self.queue.remove(key)
            elif self.is_full():
                self.evict()
            self.queue.append(key)
            self.cache_data[key] = item

    def get(self, key):
        """Return the value in self.cache_data which is linked to key.
        """
        if key in self.cache_data:
            self.queue.remove(key)
            self.queue.append(key)
            return self.cache_data.get(key)

    def is_full(self):
        """ Checks the lenght of items in self.cache_data is higher that
        BaseCaching.MAX_ITEMS
        """
        return len(self.cache_data) >= self.MAX_ITEMS

    def evict(self):
        """ print DISCARD: with the key discarded and following by a
        new line
        """
        popped = self.queue.pop()
        del self.cache_data[popped]
        print("DISCARD: " + str(popped))


# Standard python and AWS imports:
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from matplotlib.dates import DateFormatter
from matplotlib import gridspec

def plot_timeseries(
    timeseries_df,
    tag_name,
    start=None,
    end=None,
    plot_rolling_avg=False,
    labels_df=None,
    predictions=None,
    tag_split=None,
    custom_grid=True,
    fig_width=18,
    prediction_titles=None,
    shutdown_ranges_df=None,
    evaluation_color=None,
    evaluation_alpha=0.5
):
    """
    This function plots a time series signal with a line plot and can combine
    this with labelled and predicted anomaly ranges.

    PARAMS
    ======
        timeseries_df: pandas.DataFrame
            A dataframe containing the time series to plot

        tag_name: string
            The name of the tag that we can add in the label

        start: string or pandas.Datetime (default: None)
            Starting timestamp of the signal to plot. If not provided, will use
            the whole signal

        end: string or pandas.Datetime (default: None)
            End timestamp of the signal to plot. If not provided, will use the
            whole signal

        plot_rolling_avg: boolean (default: False)
            If set to true, will add a rolling average curve on top of the
            line plot for the time series.

        labels_df: pandas.DataFrame (default: None)
            If provided, this is a dataframe with all the labelled anomalies.
            This will be rendered as a filled-in plots below the time series
            itself.

        predictions: pandas.DataFrame or list of pandas.DataFrame
            If provided, this is a dataframe with all the predicted anomalies.
            This will be rendered as a filled-in plots below the time series
            itself.

        tag_split: string or pandas.Datetime
            If provided, the line plot will plot the first part of the time
            series with a colour and the second part in grey. This can be
            used to show the split between training and evaluation period for
            instance.

        custom_grid: boolean (default: True)
            Will show a custom grid with month name mentionned for each quarter
            and lighter lines for the other month to prevent clutter on the
            horizontal axis.

        fig_width: integer (default: 18)
            Figure width.

        prediction_titles: list of strings (default: None)
            If we want to plot multiple predictions, we can set the titles for
            each of the prediction plot.

    RETURNS
    =======
        fig: matplotlib.pyplot.figure
            A figure where the plots are drawn

        ax: matplotlib.pyplot.Axis
            An axis where the plots are drawn
    """
    if start is None:
        start = timeseries_df.index.min()
    elif type(start) == str:
        start = pd.to_datetime(start)

    if end is None:
        end = timeseries_df.index.max()
    elif type(end) == str:
        end = pd.to_datetime(end)

    if (tag_split is not None) & (type(tag_split) == str):
        tag_split = pd.to_datetime(tag_split)

    # Prepare the figure:
    fig_height = 4
    height_ratios = [8]
    nb_plots = 1

    if labels_df is not None:
        fig_height += 1
        height_ratios += [1.5]
        nb_plots += 1

    if predictions is not None:
        if type(predictions) == pd.core.frame.DataFrame:
            fig_height += 1
            height_ratios += [1.5]
            nb_plots += 1
        elif type(predictions) == list:
            fig_height += 1 * len(predictions)
            height_ratios = height_ratios + [1.5] * len(predictions)
            nb_plots += len(predictions)

    fig = plt.figure(figsize=(fig_width, fig_height))
    gs = gridspec.GridSpec(nb_plots, 1, height_ratios=height_ratios, hspace=0.5)
    ax = []
    for i in range(nb_plots):
        ax.append(fig.add_subplot(gs[i]))

    # Plot the time series signal:
    data = timeseries_df[start:end].copy()
    if tag_split is not None:
        ax[0].plot(data.loc[start:tag_split, 'Value'],
                   linewidth=0.5,
                   alpha=0.5,
                   label=f'{tag_name} - Training',
                   color='tab:grey')

        if evaluation_color is not None:
            ax[0].plot(data.loc[tag_split:end, 'Value'],
                       linewidth=0.5,
                       alpha=evaluation_alpha,
                       color=evaluation_color,
                       label=f'{tag_name} - Evaluation')

        else:
            ax[0].plot(data.loc[tag_split:end, 'Value'],
                       linewidth=0.5,
                       alpha=0.8,
                       label=f'{tag_name} - Evaluation')
    else:
        ax[0].plot(data['Value'], linewidth=0.5, alpha=0.8, label=tag_name)
    ax[0].set_xlim(start, end)

    # Plot a daily rolling average:
    if plot_rolling_avg == True:
        daily_rolling_average = data['Value'].rolling(window=60*24).mean()
        ax[0].plot(data.index,
                   daily_rolling_average,
                   alpha=0.5,
                   color='white',
                   linewidth=3)
        ax[0].plot(data.index,
                   daily_rolling_average,
                   label='Daily rolling leverage',
                   color='tab:red',
                   linewidth=1)

    # Configure custom grid:
    ax_id = 0
    if custom_grid:
        date_format = DateFormatter("%Y-%m")
        major_ticks = np.arange(start, end, 3, dtype='datetime64[M]')
        minor_ticks = np.arange(start, end, 1, dtype='datetime64[M]')
        ax[ax_id].xaxis.set_major_formatter(date_format)
        ax[ax_id].set_xticks(major_ticks)
        ax[ax_id].set_xticks(minor_ticks, minor=True)
        ax[ax_id].grid(which='minor', axis='x', alpha=0.8)
        ax[ax_id].grid(which='major', axis='x', alpha=1.0, linewidth=2)
        ax[ax_id].xaxis.set_tick_params(rotation=30)

    # Add the labels on a second plot:
    if labels_df is not None:
        ax_id += 1
        label_index = pd.date_range(
            start=data.index.min(),
            end=data.index.max(),
            freq='1min'
        )
        label_data = pd.DataFrame(index=label_index)
        label_data.loc[:, 'Label'] = 0.0

        for index, row in labels_df.iterrows():
            event_start = row['start']
            event_end = row['end']
            label_data.loc[event_start:event_end, 'Label'] = 1.0

        ax[ax_id].plot(label_data['Label'], color='tab:green', linewidth=0.5)
        ax[ax_id].set_xlim(start, end)
        ax[ax_id].fill_between(label_index,
                               y1=label_data['Label'],
                               y2=0,
                               alpha=0.1,
                               color='tab:green',
                               label='Real anomaly range (label)')
        ax[ax_id].axes.get_xaxis().set_ticks([])
        ax[ax_id].axes.get_yaxis().set_ticks([])
        ax[ax_id].set_xlabel('Anomaly ranges (labels)', fontsize=12)

    # Add the labels (anomaly range) on a
    # third plot located below the main ones:
    if predictions is not None:
        pred_index = pd.date_range(
            start=data.index.min(),
            end=data.index.max(),
            freq='1min')
        pred_data = pd.DataFrame(index=pred_index)

        if type(predictions) == pd.core.frame.DataFrame:
            ax_id += 1
            pred_data.loc[:, 'prediction'] = 0.0

            for index, row in predictions.iterrows():
                event_start = row['start']
                event_end = row['end']
                pred_data.loc[event_start:event_end, 'prediction'] = 1.0

            ax[ax_id].plot(pred_data['prediction'],
                           color='tab:red',
                           linewidth=0.5)
            ax[ax_id].set_xlim(start, end)
            ax[ax_id].fill_between(pred_index,
                                   y1=pred_data['prediction'],
                                   y2=0,
                                   alpha=0.1,
                                   color='tab:red')
            ax[ax_id].axes.get_xaxis().set_ticks([])
            ax[ax_id].axes.get_yaxis().set_ticks([])
            ax[ax_id].set_xlabel('Anomaly ranges (Prediction)', fontsize=12)

        elif type(predictions) == list:
            for prediction_index, p in enumerate(predictions):
                ax_id += 1
                pred_data.loc[:, 'prediction'] = 0.0

                for index, row in p.iterrows():
                    event_start = row['start']
                    event_end = row['end']
                    pred_data.loc[event_start:event_end, 'prediction'] = 1.0

                if shutdown_ranges_df is not None:
                    for index, row in shutdown_ranges_df.iterrows():
                        shutdown_start = row['start']
                        shutdown_end = row['end']
                        pred_data.loc[shutdown_start:shutdown_end, 'prediction'] = 0.0

                ax[ax_id].plot(pred_data['prediction'],
                               color='tab:red',
                               linewidth=0.5)
                ax[ax_id].set_xlim(start, end)
                ax[ax_id].fill_between(pred_index,
                                       y1=pred_data['prediction'],
                                       y2=0,
                                       alpha=0.1,
                                       color='tab:red')
                ax[ax_id].axes.get_xaxis().set_ticks([])
                ax[ax_id].axes.get_yaxis().set_ticks([])
                ax[ax_id].set_xlabel(
                    prediction_titles[prediction_index],
                    fontsize=12
                )

    # Show the plot with a legend:
    ax[0].legend(fontsize=10, loc='upper right', framealpha=0.4)

    return fig, ax


from django.shortcuts import render, redirect, get_object_or_404
from .models import TodoItem
from django.http import HttpResponse, HttpResponseRedirect
from django.urls import reverse
from django.contrib import messages
from django.contrib.auth.decorators import login_required

def home(request):
    list_tasks = TodoItem.objects.all()

    context = {
        'tasks':list_tasks,
    }

    return render(request, "index.html", context=context)

@login_required
def task_details(request, id):
    desp = TodoItem.objects.get(pk=id)

    context = {
        'desp':desp,
    }

    return render(request, "task_details.html", context=context)

@login_required
def remove_task(request,id):
    task = get_object_or_404(TodoItem, pk=id)
    task.delete()
    return redirect("home")

@login_required
def edit_task(request,id):
    task = TodoItem.objects.get(pk=id)

    context = {
        'task': task
    }
    return render(request,"edit_task.html", context=context)

@login_required
def update_task(request,id):

    task = TodoItem.objects.get(pk=id)

    if request.POST.get('is_completed', False) == "on":
        task.task_name = request.POST['task_name']
        task.complete_task()
        task.save()

    return redirect('home')

@login_required
def create_task(request):

       #https://stackoverflow.com/questions/57288652/save-html-form-data-in-database-django?rq=1

    if request.method == 'POST':

        task_name = request.POST['task_name']
        due_date = request.POST['due_date']

        TodoItem.objects.create(task_name=task_name, due_date=due_date, )

        return redirect('create_task')
    else:
        return render(request,'create_task.html')


import os
import shutil
import easyocr

# Create an EasyOCR reader object
reader = easyocr.Reader(['en'])

# Set paths for input and output folders
input_folder = 'x'
output_folder = 'y'

# Create output folder if it does not exist
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

# Loop through files in input folder
for file_name in os.listdir(input_folder):
    # Check if file is an image
    if file_name.endswith('.png') or file_name.endswith('.jpg') or file_name.endswith('.jpeg'):
        # Construct full path for input and output files
        input_path = os.path.join(input_folder, file_name)
        output_path = os.path.join(output_folder, file_name)
        # Read image using EasyOCR
        results = reader.readtext(input_path)
        # Define keywords to search for in the text
        keywords = ['Device name', 'Processor', 'Installed RAM', 'Device ID', 'System type']
        # Create a dictionary to store the extracted information
        info = {}
        # Loop through the results and extract relevant information
        for i, result in enumerate(results):
            for keyword in keywords:
                if keyword in result[1]:
                    # Extract value after the keyword and store in dictionary
                    value = results[i+1][1].strip()
                    info[keyword] = value
        # Print extracted information

        if info=={}:
            shutil.move(input_path, 'delete')
            continue

        with open('data.txt', 'a') as f:
            f.write(f'File: {file_name}\n'
            f'Device Name: {info.get("Device name", "")}\n'
            f'Processor: {info.get("Processor", "")}\n'
            f'Installed RAM: {info.get("Installed RAM", "")}\n'
            f'Device ID: {info.get("Device ID", "")}\n'
            f'System Type: {info.get("System type", "")}\n'
            f'//////////////////////////////////////////////////////////////////////\n\n')

        print(info)

        # Move input file to output folder

        shutil.move(input_path, output_path)


import time
import datetime

from marshmallow import fields, validate, Schema, pre_load, post_dump
from flask import request

from .mixin import Mixin, NestedSchema
from .user_schema import UserSchema

class ImageJsonSchema(Schema):
    mime = fields.Str()
    data = fields.Str()

class ArtSchema(Mixin):
    aid = fields.Str()
    description = fields.Str(required=False)
    rpath = fields.Str(required=False, dump_only=True)
    status = fields.Str(
        required=False,
        dump_only=True,
        validate=validate.OneOf(["processing", "completed", "failed"]),
    )
    preimage = fields.Str(required=False, dump_only=True)
    index = fields.Int(required=False)
    width = fields.Int(required=False, dump_only=True)
    height = fields.Int(required=False, dump_only=True)
    twidth = fields.Int(required=False, dump_only=True)
    theight = fields.Int(required=False, dump_only=True)
    # is_publishing = fields.Bool(required=False, dump_only=True)

    image = fields.Nested(ImageJsonSchema, load_only=True, required=True)

    @post_dump
    def set_full_path(self, data, **kwargs):

        if not data.get("rpath"):
            return data

        r_full_path = f"https://{request.headers['Host']}{data.get('rpath')}"
        t_full_path = f"https://{request.headers['Host']}{data.get('rpath').split('.')[0]}_thumbnail.{data.get('rpath').split('.')[1]}"

        data.update({"rpath": r_full_path})
        data.update({"tpath": t_full_path})

        if data.get("created_date"):
            data.update(
                {
                    "created_date": time.mktime(
                        datetime.datetime.strptime(
                            data.get("created_date"), "%Y-%m-%d %H:%M:%S.%f"
                        ).timetuple()
                    )
                }
            )

        if data.get("updated_date"):
            data.update(
                {
                    "updated_date": time.mktime(
                        datetime.datetime.strptime(
                            data.get("updated_date"), "%Y-%m-%d %H:%M:%S.%f"
                        ).timetuple()
                    )
                }
            )

        return data

class NestedArtSchema(NestedSchema):
    arts = fields.Nested(ArtSchema, attribute="items", exclude=[], many=True)


from brownie import accounts,Lottery,config,network
from .helper import get_account,get_contract,fundWithLink
import time
def deploy_lottery():
    account = get_account()
    Lottery.deploy(get_contract("eth_usd_price_feed").address,get_contract("vrf_coordinator"),get_contract("link_token"),config["networks"][network.show_active()]["fee"],config["networks"][network.show_active()]["key_hash"],{'from':account})
    lottery = Lottery[-1]
    print(f"Deployed to {lottery.address}")

def startLottery():
    account = get_account()
    lottery = Lottery[-1]
    tx=lottery.startLottery({'from':account})
    tx.wait(1)
    print("Started the lottery!")

def enterLottery():
    account = get_account()
    lottery = Lottery[-1]
    tx=lottery.enter({'from':account,"value": lottery.getEntranceFee()+1000000})
    tx.wait(1)
    print("Entered the lottery with account = "+str(account))

def endLottery():
    account = get_account()
    lottery = Lottery[-1]
    tx = fundWithLink(lottery.address)
    tx.wait(1)
    end_tx = lottery.endLottery({'from':account})
    end_tx.wait(1)
    time.sleep(300)
    print(f"Ended lottery with winner = {lottery.recentWinner()}")

def main():
    deploy_lottery()
    startLottery()
    enterLottery()
    endLottery()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from libs.data import (
    scan_result,
    scan,
    system_settings,
    target
)
from gosint.celery import app
from celery.result import AsyncResult
from apps.assets.models import SubDomain, IP, Port, IPC
from time import sleep
import IPy, re

def domaininfo_scan():
    """
    域名信息收集
    :return:
    """
    # DONE: 先处理target.domain中的domain信息
    if target.domain:
        temp_domain = []
        if system_settings.debug:
            print(len(target.domain))
        for _ in target.domain:
            _ = _.strip()
            try:
                if _ not in scan_result.domain:
                    temp_domain.append(SubDomain(subdomain=_, scan_task=scan.task, tools='手工导入'))
                    scan_result.domain.append(_)
            except Exception as e:
                print(e)
                print(_)
                continue
        SubDomain.objects.bulk_create(temp_domain, ignore_conflicts=True)

    scan_queue = []
    if system_settings.debug:
        print(len(scan_result.domain))
    print("[+] Running domaininfo_scan...")
    # 多domain同时发送客户端, 以10个列表的额度发送到客户端，客户端结果接收服务端增加多result处理 by _lin9e 2021.7.28
    for i in range(0, len(scan_result.domain), 50):
        domains = scan_result.domain[i:i+50]
        domaininfo_scan = app.send_task('domaininfo.run', args=(domains,), queue='domaininfo')
        scan_queue.append(AsyncResult(domaininfo_scan.id))
    while True:
        for task in scan_queue:
            if task.successful():
                if task.result['result']:
                    for domaininfo_result in task.result['result']:
                        try:
                            domain = domaininfo_result['domain']
                            domain_type = domaininfo_result['type']
                            domain_record = ','.join(domaininfo_result['record'])
                            domain_ips = domaininfo_result['ips']
                            sub = SubDomain.objects.get(subdomain=domain)
                            sub.domain_type = domain_type
                            sub.record = domain_record
                            sub.domain_ips = domain_ips
                            sub.save()
                            domain_ips_list = []
                            # 是不是CND，都添加到webport==80中，用于web发现、其他端口，则正常通过端口扫描到web发现的过程
                            scan_result.webport.append({'domain': domain, 'port': 80})
                            if len(domain_ips) < 2:  # 如果一个域名解析的ip地址大于1个，则判定为CDN地址
                                cdn = False
                            else:
                                cdn = True
                                # TODO: cdn结果的端口插入问题
                                # Port.objects.get_or_create(port='80', scan=scan.task, service='', ip=ip,
                                #                            ip_port=ip_port, tools=task.result['tool'])
                            for ip in domain_ips:
                                domain_ips_list.append(IP(ip=ip, scan=scan.task, subdomain=sub, is_cdn=cdn, tools='subdomain'))
                                # fix bug: 2021.1.14: 如果为CDN的话，则不去扫描端口，而直接将域名添加到webport，进行httpx扫描
                                # 内网IP则直接不扫端口，IPy.IP(ip).iptype() == 'PUBLIC' add by _lin9e 2021.8.5
                                # 内网ip扫描 - 2021.12.13
                                # if not cdn and IPy.IP(ip).iptype() == 'PUBLIC':
                                if not cdn:
                                    scan_result.ip.append(ip)
                            IP.objects.bulk_create(domain_ips_list)
                        except Exception as e:
                            print(domaininfo_result['domain'])
                            print(e)
                            continue
                scan_queue.remove(task)
            elif task.failed():
                scan_queue.remove(task)
        if not scan_queue:
            break
        sleep(0.5)    # fix bug: 2021.1.14: 修复单核cpu占用过高的问题

    # DONE: 处理target.ip中的ip信息，如果有网段，需要解析网段
    # TODO：需要先判断ip是否存活
    # DONE：判断内网ip，如果是内网ip，直接跳过端口扫描(解析到的IP)，直接添加的内网IP则可以扫描
    if target.ip:
        temp_ip = []
        print(target.ip)
        for _ in target.ip:
            try:
                results = IPy.IP(_)
                for ip in results:
                    temp_ip.append(IP(ip=str(ip), scan=scan.task))
                    scan_result.ip.append(str(ip))
                    scan_result.webport.append({'domain': ip, 'port': 80})
            except Exception as e:
                print(e)
                print(ip)
                continue
        IP.objects.bulk_create(temp_ip, ignore_conflicts=True)

    # 扫描IP去重 2021.7.27 by _lin9e
    try:
        scan_result.ip = list(set(scan_result.ip))
    except Exception as e:
        print(e)

    print("[+] Finish domaininfo_scan. Counts of no_cdn and no_priv ip is {}! Counts of all 80 web is {}".format(
        str(len(scan_result.ip)), str(len(scan_result.webport))))
    # if system_settings.debug:
    #     # debug模式下只存5个ip进行扫描
    #     scan_result.ip = scan_result.ip[:5]
    #     print(scan_result.ip)

    """
    IPC C段情报统计 add by _lin9e in 2021.9.9
    """
    print("[*] Running IP C line...")
    try:
        ipcs = []   # django添加数组
        ipc_list = []    # 整理C段
        for ip in scan_result.ip:
            ipc_list.append(re.findall(r'\d+?\.\d+?\.\d+?\.', ip)[0] + '0/24')
        count_ipc = dict()  # C段IP及对应主机数量字典
        for item in ipc_list:
            if item in count_ipc:
                count_ipc[item] += 1
            else:
                count_ipc[item] = 1
        for ip_c in count_ipc:
            try:
                ipcs.append(IPC(ipc=ip_c, ipc_count=count_ipc[ip_c], scan=scan.task))
                scan_result.ipc.append({'ipc': ip_c, 'count': count_ipc[ip_c]})
            except Exception as e:
                print(e)
        IPC.objects.bulk_create(ipcs)
        print("[+] Finish IP C line, count of C is {}".format(len(count_ipc)))
    except Exception as e:
        print(e)


#!/usr/bin/python3
import matplotlib
matplotlib.use('Agg')
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import os
import sys

Request = sys.argv[1]
Area = sys.argv[2]
ToDay = sys.argv[3]

Node_Scale = [100,200,300,400,500,600,700,800,900,1000]

plt.figure(figsize=(6,6))

DIR="result/" + str(ToDay) +"/"+str(Area)+"area/"+str(Request)+"requests/"

Result = []
for i in Node_Scale:
    fp = open(DIR + str(i) + "_" + str(Request) + ".txt")
    R_lines = fp.readlines()
    rows = len(R_lines)-1
    average = float(R_lines[rows].strip('\n'))
    average = int(average)
    Result.append(average)

plt.xlabel('Node Scale')
plt.ylabel('Throughput')
# Charging Scale => 1
lines = plt.plot(Node_Scale,Result,'bo')
plt.setp(lines, color='r', linewidth=2.0)
lines = plt.plot(Node_Scale,Result)
plt.setp(lines, color='r', linewidth=2.0)

# r_patch = mpatches.Patch(color='r', label='1 node')
# g_patch = mpatches.Patch(color='g', label='2 node')
# b_patch = mpatches.Patch(color='b', label='4 node')
# plt.legend(handles=[r_patch,g_patch,b_patch])

out_path = DIR+"/Result.png"
plt.savefig(out_path)
plt.close()


from flask import Flask, request
from twilio.twiml.messaging_response import MessagingResponse
from twilio.rest import Client
from p2000_bot import run_p2000_bot
from decouple import config

# Your Twilio Account SID and Auth Token
account_sid = config('TWILIO_ACCOUNT_SID')
auth_token = config('TWILIO_AUTH_TOKEN')

# Initialize Twilio client
client = Client(account_sid, auth_token)

# Create a Flask app
app = Flask(__name__)

@app.route('/webhook', methods=['POST'])
def webhook():
    # Get the incoming message data from Twilio
    message_body = request.values.get('Body', '')
    sender_number = request.values.get('From', '')

    # Process the incoming message
    # create regex that checks for 'p2000' in message_body

    if message_body == 'P2000' or message_body == 'p':
        run_p2000_bot()
        return 'Standard P2000 messages sent'
    if 'P2000' in message_body:
        # get all characters after 'P2000'
        city = message_body[6:]
        run_p2000_bot(city)
        return f'P2000 messages sent for {city}'
    else:
        response = MessagingResponse()
        response.message('Unknown command. Please send "P2000" to get the latest P2000 messages.')
        return str(response)

if __name__ == "__main__":
    app.run(debug=True, port=5005)


import argparse
import asyncio
import logging
import re
from collections import deque

from .aobject import *

__all__ = ["ServerEndpoint", "ClientEndpoint"]

def endpoint(spec):
    m = re.match(r"^(unix):(.*)$", spec)
    if m: return (m[1], m[2])

    m = re.match(r"^(tcp):(?:()|\*|\[([a-fA-F0-9:]+)\]|(\d+(?:\.\d){3})|([a-zA-Z.-]+))"
                 r":(\d+)$", spec)
    if m: return (m[1], m[3] or m[4] or m[5] if m[2] is None else "localhost", int(m[6]))

    raise argparse.ArgumentTypeError("invalid format")

class ServerEndpoint(aobject, asyncio.Protocol):
    @classmethod
    def add_argument(cls, parser, name, default=None):
        metavar = name.upper().replace("_", "-")
        help    = "listen at %s, either unix:PATH or tcp:HOST:PORT" % metavar
        if default is None:
            nargs = None
        else:
            nargs = "?"
            help += " (default: %(default)s)"

        parser.add_argument(
            name, metavar=metavar, type=endpoint, nargs=nargs, default=default,
            help=help)

    async def __init__(self, name, logger, sock_addr, queue_size=None):
        assert isinstance(sock_addr, tuple)

        self.name    = name
        self._logger = logger

        proto, *proto_args = sock_addr
        loop = asyncio.get_event_loop()
        if proto == "unix":
            self.server = await loop.create_unix_server(lambda: self, *proto_args, backlog=1)
            unix_path, = proto_args
            self._log(logging.INFO, "listening at unix:%s", unix_path)
        elif proto == "tcp":
            self.server = await loop.create_server(lambda: self, *proto_args, backlog=1)
            tcp_host, tcp_port = proto_args
            self._log(logging.INFO, "listening at tcp:%s:%d", tcp_host or "*", tcp_port)
        else:
            raise ValueError("unknown protocol %s" % proto)

        self._transport     = None
        self._new_transport = None

        self._send_epoch = 0
        self._recv_epoch = 1
        self._queue      = deque()
        self._queued     = 0
        self._queue_size = queue_size
        self._future     = None

        self._buffer = None
        self._pos    = 0

        self._read_paused = False

    def _log(self, level, message, *args):
        self._logger.log(level, self.name + ": " + message, *args)

    def connection_made(self, transport):
        self._send_epoch += 1

        peername = transport.get_extra_info("peername")
        if peername:
            self._log(logging.INFO, "new connection from [%s]:%d", *peername[0:2])
        else:
            self._log(logging.INFO, "new connection")

        if self._transport is None:
            self._transport = transport
        else:
            self._log(logging.INFO, "closing old connection")
            self._transport.close()
            self._new_transport = transport

    def connection_lost(self, exc):
        peername = self._transport.get_extra_info("peername")
        if peername:
            self._log(logging.INFO, "connection from [%s]:%d lost", *peername[0:2])
        else:
            self._log(logging.INFO, "connection lost")

        self._transport, self._new_transport = self._new_transport, None
        self._queue.append(exc)
        self._check_future()

    def data_received(self, data):
        self._log(logging.TRACE, "endpoint received %d bytes", len(data))
        self._queue.append(data)
        self._queued += len(data)
        self._check_pushback()
        self._check_future()

    def _check_pushback(self):
        if self._queue_size is None:
            return
        elif not self._read_paused and self._queued >= self._queue_size:
            self._log(logging.TRACE, "queue full, pausing reads")
            self._transport.pause_reading()
            self._read_paused = True
        elif self._read_paused and self._queued < self._queue_size:
            self._log(logging.TRACE, "queue not full, resuming reads")
            self._transport.resume_reading()
            self._read_paused = False

    def _check_future(self):
        if self._queue and self._future is not None:
            item = self._queue.popleft()
            if isinstance(item, Exception):
                self._future.set_exception(item)
            else:
                self._future.set_result(item)
            self._future = None

    async def _refill(self):
        self._future = future = asyncio.Future()
        self._check_future()
        self._buffer = await future
        if self._buffer is None:
            self._buffer = b""
            self._log(logging.TRACE, "recv end-of-stream")
            self._recv_epoch += 1
            raise asyncio.CancelledError

    async def recv(self, length=0):
        data = bytearray()
        while length == 0 or len(data) < length:
            if not self._buffer:
                self._log(logging.TRACE, "recv waits for %d bytes", length - len(data))
                await self._refill()

            if length == 0:
                length = len(self._buffer)

            chunk = self._buffer[:length - len(data)]
            self._buffer = self._buffer[len(chunk):]
            self._queued -= len(chunk)
            self._check_pushback()
            data += chunk

        self._log(logging.TRACE, "recv <%s>", data.hex())
        return data

    async def recv_until(self, separator):
        separator = bytes(separator)
        data = bytearray()
        while True:
            if not self._buffer:
                self._log(logging.TRACE, "recv waits for <%s>", separator.hex())
                await self._refill()

            try:
                index = self._buffer.index(separator)
                chunk = self._buffer[:index]
                self._buffer = self._buffer[index + 1:]
                self._queued -= len(chunk)
                self._check_pushback()
                data += chunk
                break

            except ValueError:
                data += self._buffer
                self._queued -= len(self._buffer)
                self._check_pushback()
                self._buffer = None

        self._log(logging.TRACE, "recv <%s%s>", data.hex(), separator.hex())
        return data

    async def recv_wait(self):
        if not self._buffer:
            self._log(logging.TRACE, "recv wait")
            await self._refill()

    async def send(self, data):
        data = bytes(data)
        if self._send_epoch == self._recv_epoch:
            self._log(logging.TRACE, "send <%s>", data.hex())
            self._transport.write(data)
            return True
        else:
            self._log(logging.TRACE, "send to previous connection discarded")
            return False

    async def close(self):
        if self._transport:
            self._transport.close()

class ClientEndpoint(aobject, asyncio.Protocol):
    @classmethod
    def add_argument(cls, parser, name, default=None):
        metavar = name.upper().replace("_", "-")
        help    = "connect to %s, either unix:PATH or tcp:HOST:PORT" % metavar
        if default is None:
            nargs = None
        else:
            nargs = "?"
            help += " (default: %(default)s)"

        parser.add_argument(
            name, metavar=metavar, type=endpoint, nargs=nargs, default=default,
            help=help)

    # FIXME: finish this


import csv
import requests

def check_url(url):

    custom_header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }

    try:
        response = requests.get(url, timeout = 5)
        if response.status_code == 404:
            return 'Failure'
        else:
            return 'Success'
    except requests.exceptions.RequestException as e:
        return 'Error: ' + str(e)

def check_urls_from_csv(file_name, url_column):
    with open(file_name, 'r') as file:
        reader = csv.DictReader(file)
        row_number = 0
        websites_down = 0
        for row in reader:
            row_number += 1
            url = row[url_column]
            status = check_url(url)
            if(status == 'Failure'):
                websites_down += 1
                print(f'Row: {row_number}, URL: {url}, Status: {status}')
            elif(row_number%10 == 0):
                print(f'Current row: {row_number}')

        print(f'Done! Found {websites_down} website/s down\r\n')

# Replace with the name of your CSV file (same directory than py file)
# And the name of the column that contains the URLs
check_urls_from_csv('urls.csv', 'URL')


from django.urls import path
from api.views import ProjectDetailView, TaskDetailView, ProjectUpdateView, TaskUpdateView, ProjectDeleteView, TaskDeleteView

urlpatterns = [
    path('project/<int:pk>', ProjectDetailView.as_view()),
    path("task/<int:pk>", TaskDetailView.as_view()),
    path('project/<int:pk>/update', ProjectUpdateView.as_view()),
    path('task/<int:pk>/update', TaskUpdateView.as_view()),
    path('project/<int:pk>/delete', ProjectDeleteView.as_view()),
    path('task/<int:pk>/delete', TaskDeleteView.as_view())
]


from django.urls import include, path
from rest_framework.routers import DefaultRouter
from users.views import ExtraUsersViewSet, InviteToContactViewSet

app_name = "users"

router = DefaultRouter()
router.register("invites", InviteToContactViewSet)
router.register("users", ExtraUsersViewSet)

urlpatterns = [
    path(r"auth/", include("djoser.urls")),
    path(r"auth/", include("djoser.urls.jwt")),
    path("", include(router.urls)),
]