'''
MMU Simulator in python
Assignment 3, Computer Architecture
Authors: Akilesh K, Arjun Menon V
Sept 2021
Main Module: - Create kernel memory, user memory and process objects
             - Call functions for pagewalk, page eviction (LRU policy) and writing evicted PageTable
               back to Disk (WriteBack policy)
'''
import numpy as np
import argparse as ap
from inc.opts import *
from inc.parser import inputParser
from memory import memory
from translator import translate
from process import proc

def args():
    '''
    Parse command-line arguments

    '''
    parser = ap.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, metavar='file', help="Input file with requests", required=True)
    args = parser.parse_args()
    return args

def main():
    '''
    Main simulation function

    '''
    activeProcesses = [None]*NPROC
    userMem = memory()
    '''
     - kernel memory allocation type 1: give MMU access to the entire kernel space
     - for 0 page table evictions for the given page access pattern, only 40 page frames are needed by the mmu
    '''
    kernelMem = memory(256, 1024, "kernel")
    # kernel memory allocation type 2: Reserving only 64K for Page Directories and Page Tables
    #kernelMem = memory(16, 1024, "kernel")
    inFile = args().input
    fileParser = inputParser(inFile)
    reqs = fileParser.parse()
    totalRequests = len(reqs)
    totalHits = 0
    totalMisses = 0
    totalPageReplacements = 0
    totalTableReplacements = 0
    totalReads = 0
    totalWrites = 0
    for req in reqs:
        if req.typ == 'r':
            totalReads += 1
        elif req.typ == 'w':
            totalWrites += 1
        # check if the request is from a new process
        if activeProcesses[req.pid] == None:
            newProc = proc(req.pid)
            activeProcesses[req.pid] = newProc
        offsets = translate(req.va)
        p = activeProcesses[req.pid]
        page, rv1, rv2 = p.pagewalk(kernelMem, userMem, offsets)

        if rv1[0] != 0:
            evictedPID, evictedOffset, evictedTable = rv1[1], rv1[2], rv1[3]

            for pr in activeProcesses:
                if pr != None:
                    if pr.pid == evictedPID:
                        pr.pTableCopy[evictedOffset] = evictedTable

        if rv2[0] != 0:
            evictedUserFrame = rv2[1]
            for pr in activeProcesses:
                if pr != None:
                    for table in pr.TableCopy:
                        for entry in table:
                            if entry == evictedUserFrame:
                                entry = None

    for p in activeProcesses:
        if p != None:
            totalHits += p.hits
            totalMisses += p.misses
    print('Total Requests: {}'.format(totalRequests))
    print('Hit Rate: {:.2f}%'.format(100*totalHits/totalRequests))
    print('Miss Rate: {:.2f}%'.format(100*totalMisses/totalRequests))
    print('Read Requests: {}'.format(totalReads))
    print('Write Requests: {}'.format(totalWrites))
    for p in activeProcesses:
        if p != None:
            print('PID {}: {} requests'.format(p.pid, p.requests))
            totalPageReplacements += p.pageEvictions
            totalTableReplacements += p.tableEvictions
    print('Dirty Page Evictions: {}'.format(totalPageReplacements))
    print('Dirty Table Evictions: {}'.format(totalTableReplacements))

    totalKernelPFUsed = max(kernelMem.LRUctr)+1 # count starts from 0
    totalUserPFUsed = max(userMem.LRUctr)+1
    numActiveProcesses = 0
    for p in activeProcesses:
        if p != None:
            numActiveProcesses += 1
    print('Final number of pageframes used: {}'.format(totalKernelPFUsed+totalUserPFUsed+numActiveProcesses))
    print('\tKernel pageframes: {}'.format(totalKernelPFUsed+numActiveProcesses))
    print('\tUser pageframes: {}'.format(totalUserPFUsed))

if __name__=="__main__":
    main()
    # print("Please check back later. In development")


from SpotifyETL import SpotifyETL, clean_entities
import datetime
import time
import logging
import sqlalchemy
import pyodbc
import config

date_stage = datetime.datetime.today()
current_date = date_stage.strftime('%m-%d-%Y')

#defining logger conditions
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

formatter = logging.Formatter('	%(asctime)s - %(name)s - %(message)s')

file_handler = logging.FileHandler(f'./logs/{current_date} app.log')
file_handler.setFormatter(formatter)

start_time = time.time()

#initiate ShopifyETL Class
s = SpotifyETL()

#call get_recent_songs method
print('Getting Recent Songs')
s.get_recent_songs()
recent_length = len(s.get_recent_songs())
print(f'Retrieved {recent_length} songs')

#call get artist ids method
print('Getting Recent Artists')
s.get_artist_ids()
artist_length = len(s.get_artist_ids())
print(f'Retrieved {artist_length} artists')

#call get album ids method
print('Getting Recent Albums')
s.get_album_ids()
album_length = len(s.get_album_ids())
print(f'Retrieve {album_length} albums')

#create_Spotify csvs for staging
print('Creating Track Entity')
track_entity = s.create_track_entity()
print('Track Entity Created')
print('Creating Artist Entity')
artist_entity = s.create_artist_entity()
print('Artist Entity Created')
print('Creating Album Entity')
album_entity = s.create_album_entity()
print('Album Entity Created')

print('Storing Entities as CSVs')
track_entity.to_csv(f'./output/tracks {current_date}.csv')
artist_entity.to_csv(f'./output/artists {current_date}.csv')
album_entity.to_csv(f'./output/albums {current_date}.csv')

#establishing SQL Server Engine to Azure DB
engine = config.engine

# write the DataFrame to a table in the sql database
print('Writing DataFrames to Azure DB Tables')
track_entity.to_sql("tracks", schema='dbo', if_exists='append', con=engine)
artist_entity.to_sql("artists", schema='dbo', if_exists='append', con=engine)
album_entity.to_sql("albums", schema='dbo', if_exists='append', con=engine)

print('Cleaning Album and Artist Tables')
clean_tracks_df = clean_entities('tracks.sql', 'track_id')
clean_artist_df = clean_entities('artists.sql', 'artist_id')
clean_album_df = clean_entities('albums.sql', 'album_id')
track_data_types = {'track_id': sqlalchemy.types.VARCHAR(50)}
artist_data_types = {'artist_id': sqlalchemy.types.VARCHAR(50)}
album_data_types = {'album_id': sqlalchemy.types.VARCHAR(50)}
clean_tracks_df.to_sql("tracks", schema='dbo', if_exists='replace', dtype=track_data_types, con=engine)
clean_artist_df.to_sql("artists", schema='dbo', if_exists='replace', dtype=artist_data_types, con=engine)
clean_album_df.to_sql("albums", schema='dbo', if_exists='replace', dtype=album_data_types, con=engine)
print('Azure DB Tables Populated')

end_time = time.time()

total_time = end_time - start_time

logger.debug(f'Process took {int(total_time/60)} minutes and {(total_time % 60):.2f} seconds')


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myCv.settings')

import django
django.setup()

from faker import Faker
fake = Faker()

from accounts.models import User
from groups.models import Group, GroupMember
from posts.models import Post
import random

uName = []
gName = []

def initializeList():
    for i in range(10):
        uName.append(fake.profile(fields=['username'])['username'])

    for i in range(10):
        gName.append(fake.sentence(nb_words=6))

def addUser():
    user = User.objects.get_or_create(username=random.choice(uName))[0]
    user.save()
    user.set_password(fake.word())
    user.save()
    return user

def addGroup():
    name = random.choice(gName)
    admin = addUser()
    desc = fake.paragraph(nb_sentences=3)
    group = Group.objects.get_or_create(name=name)[0]
    group.admin=admin
    group.desc=desc
    group.save()
    return group

def joinGroup():
    user = addUser()
    group = addGroup()
    gMember = GroupMember.objects.get_or_create(member=user, group=group)[0]
    gMember.save()

def addPost():
    message = fake.text(max_nb_chars=200, ext_word_list=None)
    publisher = addUser()
    group = addGroup()
    post = Post(publisher=publisher, group=group, message=message)
    post.save()

def populate(n):
    initializeList()
    for i in range(n):
        joinGroup()
        addPost()

if(__name__=='__main__'):
    print('\n Populating Fake DAta !!!')
    populate(15)
    print('\n Fake data is inserted...\n')


import os
import sys
import logging
import time
import pathlib
import subprocess
import numpy as np
import re
from genotype_variants.run_cmd import run_cmd
from genotype_variants.create_duplex_simplex_dataframe import (
    create_duplex_simplex_dataframe as cdsd,
)
from genotype_variants.create_all_maf_dataframe import create_all_maf_dataframe as camd

try:
    import click
except ImportError as e:
    print(
        "small_variants: click is not installed, please install click as it is one of the requirements."
    )
    exit(1)
try:
    import click_log
except ImportError as e:
    print(
        "small_variants: click-log is not installed, please install click_log as it is one of the requirements."
    )
    exit(1)
try:
    import pandas as pd
except ImportError as e:
    print(
        "small_variants: pandas is not installed, please install pandas as it is one of the requirements."
    )
    exit(1)
"""
small_variants
~~~~~~~~~~~~~~~
:Description: console script for genotyping small variants
"""
"""
Created on January 29, 2020
Description: console script for genotyping small variants
@author: Ronak H Shah
@coauthor: Maysun Hasan
"""
BASE_DIR = pathlib.Path("__file__").resolve().parent
# Making logging possible
logger = logging.getLogger("genotype_variants")
click_log.basic_config(logger)
click_log.ColorFormatter.colors["info"] = dict(fg="green")

@click.group()
def cli():
    """Sub-commands for genotyping small variants"""
    pass

@cli.command()
@click.option(
    "-i",
    "--input-maf",
    required=True,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format",
)
@click.option(
    "-r",
    "--reference-fasta",
    required=True,
    type=click.Path(exists=True),
    help="Full path to reference file in FASTA format",
)
@click.option(
    "-p",
    "--patient-id",
    required=True,
    type=click.STRING,
    help="Alphanumeric string indicating patient identifier",
)
@click.option(
    "-b",
    "--standard-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to standard bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-d",
    "--duplex-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to duplex bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-s",
    "--simplex-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to simplex bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-g",
    "--gbcms-path",
    required=True,
    type=click.Path(exists=True),
    help="Full path to GetBaseCountMultiSample executable with fragment support",
)
@click.option(
    "-fd",
    "--filter-duplicate",
    required=False,
    default=0,
    type=click.INT,
    help="Filter duplicate parameter for GetBaseCountMultiSample",
)
@click.option(
    "-fc",
    "--fragment-count",
    required=False,
    default=1,
    type=click.INT,
    help="Fragment Count parameter for GetBaseCountMultiSample",
)
@click.option(
    "-mapq",
    "--mapping-quality",
    required=False,
    default=20,
    type=click.INT,
    help="Mapping quality for GetBaseCountMultiSample",
)
@click.option(
    "-t",
    "--threads",
    required=False,
    default=1,
    type=click.INT,
    help="Number of threads to use for GetBaseCountMultiSample",
)
@click_log.simple_verbosity_option(logger)
def generate(
    input_maf,
    reference_fasta,
    gbcms_path,
    patient_id,
    standard_bam,
    duplex_bam,
    simplex_bam,
    filter_duplicate,
    fragment_count,
    mapping_quality,
    threads
):
    """Command that helps to generate genotyped MAF,
    the output file will be labelled with
    patient identifier as prefix"""
    logger_output = pathlib.Path.cwd().joinpath("genotype_variants.log")
    fh = logging.FileHandler(logger_output)
    formatter = logging.Formatter(
        fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p",
    )
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.info(
        "=========================================================================="
    )
    logger.info(
        ">>> Running genotype_variants for small variants to generate genotypes <<<"
    )
    logger.info(
        "=========================================================================="
    )
    t1_start = time.perf_counter()
    t2_start = time.process_time()
    if standard_bam or duplex_bam or simplex_bam:
        pass
    else:
        logger.error(
            "Required to specify at-least one input BAM file option. Please refer to the README for more information"
        )
        exit(1)

    logger.info("small_variants: Patient ID: %s", patient_id)
    logger.info("small_variants: Input MAF: %s", input_maf)
    logger.info("small_variants: Reference FASTA: %s", reference_fasta)
    if standard_bam:
        logger.info("small_variants: Standard BAM: %s", standard_bam)
    if duplex_bam:
        logger.info("small_variants: Duplex BAM: %s", duplex_bam)
    if simplex_bam:
        logger.info("small_variants: Simplex BAM: %s", simplex_bam)
    logger.info("small_variants: GetBaseCountMultiSample -> Path: %s", gbcms_path)
    logger.info(
        "small_variants: GetBaseCountMultiSample -> Filter Duplicate: %s",
        str(filter_duplicate),
    )
    logger.info(
        "small_variants: GetBaseCountMultiSample -> Fragment Count: %s",
        str(fragment_count),
    )
    logger.info(
        "small_variants: GetBaseCountMultiSample -> Mapping Quality: %s",
        str(mapping_quality),
    )
    logger.info("small_variants: GetBaseCountMultiSample -> Threads: %s", str(threads))

    # Run GetBaseMultisampleCount for each available bam file
    std_output_maf = None
    duplex_output_maf = None
    simplex_output_maf = None
    p1, p2, p3 = None, None, None
    if standard_bam:
        btype = "STANDARD"
        (cmd, std_output_maf) = generate_gbcms_cmd(
            input_maf,
            btype,
            reference_fasta,
            gbcms_path,
            patient_id,
            standard_bam,
            filter_duplicate,
            fragment_count,
            mapping_quality,
            threads,
        )
        p1 = run_cmd(cmd)
        logger.info(
            "small_variants: Done running gbcms on %s and data has been written to %s",
            standard_bam,
            std_output_maf,
        )

    if duplex_bam:
        btype = "DUPLEX"
        (cmd, duplex_output_maf) = generate_gbcms_cmd(
            input_maf,
            btype,
            reference_fasta,
            gbcms_path,
            patient_id,
            duplex_bam,
            filter_duplicate,
            fragment_count,
            mapping_quality,
            threads,
        )
        p2 = run_cmd(cmd)
        logger.info(
            "small_variants: Done running gbcms on %s and data has been written to %s",
            duplex_bam,
            duplex_output_maf,
        )

    if simplex_bam:
        btype = "SIMPLEX"
        (cmd, simplex_output_maf) = generate_gbcms_cmd(
            input_maf,
            btype,
            reference_fasta,
            gbcms_path,
            patient_id,
            simplex_bam,
            filter_duplicate,
            fragment_count,
            mapping_quality,
            threads,
        )
        p3 = run_cmd(cmd)
        logger.info(
            "small_variants: Done running gbcms on %s and data has been written to %s",
            simplex_bam,
            simplex_output_maf,
        )

    logger.info("small_variants: Completed processing based on the given instructions")

    t1_stop = time.perf_counter()
    t2_stop = time.process_time()
    logger.info("--------------------------------------------------")
    logger.info("Elapsed time: %.1f [min]" % ((t1_stop - t1_start) / 60))
    logger.info("CPU process time: %.1f [min]" % ((t2_stop - t2_start) / 60))
    logger.info("--------------------------------------------------")
    return (std_output_maf, simplex_output_maf, duplex_output_maf)

def generate_gbcms_cmd(
    input_maf,
    btype,
    reference_fasta,
    gbcms_path,
    patient_id,
    bam,
    filter_duplicate,
    fragment_count,
    mapping_quality,
    threads,
):
    sample_id = patient_id + "-" + btype
    output_maf = pathlib.Path.cwd().joinpath(sample_id + "_genotyped.maf")

    cmd = (
        str(gbcms_path)
        + " --bam "
        + sample_id
        + ":"
        + str(bam)
        + " --filter_duplicate "
        + str(filter_duplicate)
        + " --fragment_count "
        + str(fragment_count)
        + " --maf "
        + str(input_maf)
        + " --maq "
        + str(mapping_quality)
        + " --omaf"
        + " --output "
        + str(output_maf)
        + " --fasta "
        + str(reference_fasta)
        + " --thread "
        + str(threads)
    )

    return (cmd, output_maf)

@cli.command()
@click.option(
    "-i",
    "--input-maf",
    required=False,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format used for input to GBCMS for generating genotypes",
)
@click.option(
    "-std",
    "--input-standard-maf",
    required=False,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format generated by GBCMS for standard_bam",
)
@click.option(
    "-d",
    "--input-duplex-maf",
    required=False,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format generated by GBCMS for duplex_bam",
)
@click.option(
    "-s",
    "--input-simplex-maf",
    required=False,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format generated by GBCMS for simplex_bam",
)
@click.option(
    "-p",
    "--patient-id",
    required=True,
    type=click.STRING,
    help="Alphanumeric string indicating patient identifier",
)
@click_log.simple_verbosity_option(logger)
def merge(
    patient_id, input_maf, input_standard_maf, input_duplex_maf, input_simplex_maf
):
    """
    Given original input MAF used as an input for GBCMS along with
    GBCMS generated output MAF for standard_bam, duplex_bam or simplex bam,
    Merge them into a single output MAF format.
    If both duplex_bam and simplex_bam based MAF are provided
    the program will generate merged genotypes as well.
    The output file will be based on the give alphanumeric patient identifier as prefix.
    """
    logger_output = pathlib.Path.cwd().joinpath("genotype_variants.log")
    fh = logging.FileHandler(logger_output)
    formatter = logging.Formatter(
        fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p",
    )
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.info(
        "========================================================================"
    )
    logger.info(
        ">>> Running genotype_variants for small variants to merge MAF output <<<"
    )
    logger.info(
        "========================================================================"
    )
    t1_start = time.perf_counter()
    t2_start = time.process_time()

    number_of_variables = 0
    for status in (input_maf, input_standard_maf, input_duplex_maf, input_simplex_maf):
        if status:
            number_of_variables += 1
    if number_of_variables >= 2:
        pass
    else:
        logger.error(
            "genotype_variants:small_variants:merge:: At least two MAF input need to be provided for us to merge."
        )
        exit(1)
    o_maf, i_maf, d_maf, s_maf = None, None, None, None
    if input_maf:
        logger.info(
            "genotype_variants:small_variants:merge:: Original MAF -> %s", input_maf
        )
        o_maf = pd.read_csv(input_maf, sep="\t", header="infer")
    if input_standard_maf:
        logger.info(
            "genotype_variants:small_variants:merge:: STANDARD BAM MAF -> %s",
            input_standard_maf,
        )
        i_maf = pd.read_csv(input_standard_maf, sep="\t", header="infer")
    if input_duplex_maf:
        d_maf = pd.read_csv(input_duplex_maf, sep="\t", header="infer")
        logger.info(
            "genotype_variants:small_variants:merge:: DUPLEX BAM MAF -> %s",
            input_duplex_maf,
        )
    if input_simplex_maf:
        s_maf = pd.read_csv(input_simplex_maf, sep="\t", header="infer")
        logger.info(
            "genotype_variants:small_variants:merge:: SIMPLEX BAM MAF -> %s",
            input_simplex_maf,
        )

    # generate duplex simplex data frame
    ds_maf = None

    if d_maf is not None and s_maf is not None:
        ds_maf = cdsd(s_maf, d_maf)

    # generate data frame based on satisfying conditions
    file_name = None
    (df_o_s_ds, df_s_ds, df_s_ds) = None, None, None
    if o_maf is not None and i_maf is not None and ds_maf is not None:
        df_o_s_ds = camd(o_maf, i_maf, ds_maf)
        file_name = pathlib.Path.cwd().joinpath(
            patient_id + "-ORG-STD-SIMPLEX-DUPLEX" + "_genotyped.maf"
        )
        write_csv(file_name, df_o_s_ds)
    elif o_maf is not None and i_maf is not None:
        pass
    elif o_maf is not None and ds_maf is not None:
        df_o_ds = camd(o_maf, None, ds_maf)
        file_name = pathlib.Path.cwd().joinpath(
            patient_id + "-ORG-SIMPLEX-DUPLEX" + "_genotyped.maf"
        )
        write_csv(file_name, df_o_ds)
    elif i_maf is not None and ds_maf is not None:
        df_s_ds = camd(None, i_maf, ds_maf)
        file_name = pathlib.Path.cwd().joinpath(
            patient_id + "-STD-SIMPLEX-DUPLEX" + "_genotyped.maf"
        )
        write_csv(file_name, df_s_ds)
    elif i_maf is not None and d_maf is not None:
        pass
    elif i_maf is not None and s_maf is not None:
        pass
    elif o_maf is not None and d_maf is not None:
        pass
    elif o_maf is not None and s_maf is not None:
        pass
    else:
        file_name = pathlib.Path.cwd().joinpath(
            patient_id + "-SIMPLEX-DUPLEX" + "_genotyped.maf"
        )
        write_csv(file_name, ds_maf)
    t1_stop = time.perf_counter()
    t2_stop = time.process_time()
    logger.info("--------------------------------------------------")
    logger.info("Elapsed time: %.1f [min]" % ((t1_stop - t1_start) / 60))
    logger.info("CPU process time: %.1f [min]" % ((t2_stop - t2_start) / 60))
    logger.info("--------------------------------------------------")
    return file_name

def write_csv(file_name, data_frame):
    try:
        data_frame.to_csv(str(file_name), sep="\t", index=False)
        logger.info(
            "genotype_variants:small_variants:create_csv:: merged genotyped data has been written to %s",
            file_name,
        )
    except:
        e = sys.exc_info()[0]
        logger.error(
            "genotype_variants:small_variants:create_csv:: could not write to CSV file, due to error: %s",
            e,
        )
        exit(1)

@cli.command()
@click.option(
    "-i",
    "--input-maf",
    required=True,
    type=click.Path(exists=True),
    help="Full path to small variants input file in MAF format",
)
@click.option(
    "-r",
    "--reference-fasta",
    required=True,
    type=click.Path(exists=True),
    help="Full path to reference file in FASTA format",
)
@click.option(
    "-p",
    "--patient-id",
    required=True,
    type=click.STRING,
    help="Alphanumeric string indicating patient identifier",
)
@click.option(
    "-b",
    "--standard-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to standard bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-d",
    "--duplex-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to duplex bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-s",
    "--simplex-bam",
    required=False,
    type=click.Path(exists=True),
    help="Full path to simplex bam file, Note: This option assumes that the .bai file is present at same location as the bam file",
)
@click.option(
    "-g",
    "--gbcms-path",
    required=True,
    type=click.Path(exists=True),
    help="Full path to GetBaseCountMultiSample executable with fragment support",
)
@click.option(
    "-fd",
    "--filter-duplicate",
    required=False,
    default=0,
    type=click.INT,
    help="Filter duplicate parameter for GetBaseCountMultiSample",
)
@click.option(
    "-fc",
    "--fragment-count",
    required=False,
    default=1,
    type=click.INT,
    help="Fragment Count parameter for GetBaseCountMultiSample",
)
@click.option(
    "-mapq",
    "--mapping-quality",
    required=False,
    default=20,
    type=click.INT,
    help="Mapping quality for GetBaseCountMultiSample",
)
@click.option(
    "-t",
    "--threads",
    required=False,
    default=1,
    type=click.INT,
    help="Number of threads to use for GetBaseCountMultiSample",
)
@click_log.simple_verbosity_option(logger)
def all(
    input_maf,
    reference_fasta,
    gbcms_path,
    patient_id,
    standard_bam,
    duplex_bam,
    simplex_bam,
    filter_duplicate,
    fragment_count,
    mapping_quality,
    threads,
):
    """
    Command that helps to generate genotyped MAF and
    merge the genotyped MAF.
    the output file will be labelled with
    patient identifier as prefix
    """
    logger_output = pathlib.Path.cwd().joinpath("genotype_variants.log")
    fh = logging.FileHandler(logger_output)
    formatter = logging.Formatter(
        fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p",
    )
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    logger.info(
        "========================================================================================"
    )
    logger.info(
        ">>> Running genotype_variants for small variants to generate genotypes and merge MAF <<<"
    )
    logger.info(
        "========================================================================================="
    )
    t1_start = time.perf_counter()
    t2_start = time.process_time()
    (standard_maf, simplex_maf, duplex_maf) = generate.callback(
        input_maf,
        reference_fasta,
        gbcms_path,
        patient_id,
        standard_bam,
        duplex_bam,
        simplex_bam,
        filter_duplicate,
        fragment_count,
        mapping_quality,
        threads,
    )
    final_file = merge.callback(patient_id, input_maf, standard_maf, simplex_maf, duplex_maf)
    t1_stop = time.perf_counter()
    t2_stop = time.process_time()
    logger.info("--------------------------------------------------")
    logger.info("Elapsed time: %.1f [min]" % ((t1_stop - t1_start) / 60))
    logger.info("CPU process time: %.1f [min]" % ((t2_stop - t2_start) / 60))
    logger.info("--------------------------------------------------")
    return


# !/usr/bin/env python
# -*- coding:utf-8 -*-
# Author:pylarva
# bolg:www.lichengbing.com

from __future__ import division

import os
import re
import sys
import time
import json
import math
import socket
import hashlib

# 客户端程序运行的本地目录
base_dir = os.path.dirname(os.path.abspath(__file__))

def connect_server():
    """
    客户端连接服务端：IP + PORT
    :return:
    """
    while True:
        ip_port = input('请输入服务端地址： ')
        if not re.match('^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}:\d+$', ip_port):
            print('地址有误...')
            continue
        else:
            ip_port = (ip_port.split(':')[0], int(ip_port.split(':')[1]))
            return ip_port

def login(s):
    """
    FTP登陆 如果登陆不成功即不允许与服务端会话
    :param s:
    :return:
    """
    while True:
        send_data = input('>>: ').strip()
        if len(send_data) == 0: continue
        s.send(bytes(send_data, encoding='utf-8'))
        recv_data = s.recv(1024)
        print(recv_data.decode())

        if re.match('欢迎', recv_data.decode()):
            break

def progressbar(cur, total):
    """
    上传文件或者下载文件过程中的进度条显示
    :param cur:
    :param total:
    :return:
    """
    percent = '{:.2%}'.format(cur / total)
    sys.stdout.write('\r')
    sys.stdout.write('[%-50s] %s' % ('=' * int(math.floor(cur * 50 / total)), percent))
    sys.stdout.flush()
    if cur == total:
        sys.stdout.write('\n')

def file_md5(file_path):
    """
    文件的MD5校验
    :param file_path:
    :return:
    """
    f = open(file_path, 'rb')
    md5obj = hashlib.md5()
    md5obj.update(f.read())
    md5_hash = md5obj.hexdigest()
    f.close()
    return str(md5_hash).upper()

def task_put(s, cmd_list):
    """
    执行上传文件put方法
    :param s:
    :param cmd_list:
    :return:
    """
    abs_filepath = cmd_list[1]
    if os.path.isfile(abs_filepath):
        file_size = os.stat(abs_filepath).st_size
        file_name = abs_filepath.split('\\')[-1]
        print('file:{} size:{}'.format(file_name, file_size))

        # 给服务端发送一个包含文件名 文件大小等信息的json文件
        msg_data = {"action": "put",
                    "file_name": file_name,
                    "file_size": file_size,
                    "file_path": abs_filepath}

        s.sendall(bytes(json.dumps(msg_data), encoding='utf-8'))

        # 接收服务端的确认信息
        server_confirm_msg = s.recv(1024)
        confirm_data = json.loads(server_confirm_msg.decode())

        if confirm_data['status'] == 200:

            # 如果服务端已有文件 得到文件大小 进行断点续传
            read_size = confirm_data['read_size']
            if read_size != 0:
                send_size = read_size
                print('服务器上检测到相同文件 已经启动断点续传...')
                time.sleep(0.5)
            else:
                send_size = 0
            print('Start sending file \033[31;0m{}\033[0m!'.format(msg_data['file_name']))

            # 打开文件 进行指针偏移 开始发送数据
            f = open(msg_data['file_path'], 'rb+')
            f.seek(read_size, 0)
            for line in f:
                s.send(line)
                send_size += len(line)
                progressbar(send_size, file_size)
            f.close()
            print('上传文件 \033[31;0m{}\033[0m 成功!'.format(file_name))

            # 上传成功后进行MD5校验
            print('正在校验 MD5 值...')
            md5_hash = file_md5(abs_filepath)
            ret = s.recv(1024)
            if md5_hash == ret.decode():
                print('MD5值 \033[31;0m{}\033[0m 校验成功!'.format(md5_hash))
            else:
                print('MD5值 \033[31;0m{}\033[0m 校验失败!'.format(md5_hash))

    else:
        print('\033[31;0m{}\033[0m文件不存在...'.format(abs_filepath))

def task_pull(s, cmd_list):
    """
    下载文件pull方法
    :param s:
    :param cmd_list:
    :return:
    """
    file_name = cmd_list[-1]

    file = os.path.join(base_dir, file_name)

    # 判断本地路径下是否有相同文件 得到该文件大小 方便断点续传
    if os.path.exists(file):
        recv_size = os.stat(file).st_size
        print('本地检测到相同文件 已经启动断点续传...')
    else:
        recv_size = 0

    # 发送请求给服务端
    msg_data = {"action": "pull", "file_name": file_name}
    s.send(bytes(json.dumps(msg_data), encoding='utf-8'))

    # 服务端收到下载请求 等待确认开始传输
    data = s.recv(1024)
    data = json.loads(data.decode())
    file_name = data.get('file_name')
    file_size = data.get('file_size')
    server_response = {'status': 200, "read_size": recv_size}
    s.send(bytes(json.dumps(server_response), encoding='utf-8'))
    f = open(os.path.join(base_dir, file_name), 'ab+')

    # 收到总的文件大小 开始循环接收
    while recv_size < file_size:
        data = s.recv(4096)
        f.write(data)
        recv_size += len(data)
        progressbar(recv_size, file_size)
    print('下载文件 \033[31;0m{}\033[0m 成功!\n本地路径：\033[31;0m{}\033[0m'.format(file_name, file))
    f.close()

    # 接受完成MD5校验
    print('正在校验 MD5 值...')
    md5_hash = file_md5(file)
    ret = s.recv(1024)
    if md5_hash == ret.decode():
        print('MD5值 \033[31;0m{}\033[0m 校验成功!'.format(md5_hash))
    else:
        print('MD5值 \033[31;0m{}\033[0m 校验失败!'.format(md5_hash))

def task_types(s, cmd_list):
    """
    对客户端命令进行有效性判断
    :param s:
    :param cmd_list:
    :return:
    """
    if len(cmd_list) == 1:
        if cmd_list[0] == 'mkdir':
            print('ERROR: mkdir + 文件夹名...')
        if cmd_list[0] == 'rm':
            print('ERROR: rm + 文件夹名...')
        else:
            msg_data = {"action":  cmd_list[0],
                        "file_name": None}
            msg_send(s, msg_data)

    if len(cmd_list) == 2:
        task_type = cmd_list[0]
        if task_type == 'put':
            task_put(s, cmd_list)
        if task_type == 'pull':
            task_pull(s, cmd_list)
        if task_type in ['cd', 'ls', 'mkdir', 'rm']:
            msg_data = {"action": cmd_list[0], "file_name": cmd_list[-1]}
            msg_send(s, msg_data)

def msg_send(s, msg_data):
    """
    向服务端发送一条消息并接收一条消息并打印
    :param s:
    :param msg_data:
    :return:
    """
    s.send(bytes(json.dumps(msg_data), encoding='utf-8'))
    recv_data = s.recv(1024)
    print(recv_data.decode())

def main():
    """
    客户端主程序函数：线路连接 + 循环会话
    :return:
    """
    print('  \033[32;0mFTP客户端程序\033[0m  '.center(50, '-'))

    while True:
        # 连接服务端
        ip_port = connect_server()
        # ip_port = ('10.0.0.150', 8000)
        s = socket.socket()
        s.settimeout(2)
        ret = s.connect_ex(ip_port)
        if ret != 0:
            print('服务器：\033[31;0m{}\033[0m 端口：\033[31;0m{}\033[0m 连接失败...'.format(ip_port[0], ip_port[1]))
            print('请检查IP和端口并重试!\n')
            continue

        welcome_msg = s.recv(1024)
        print(welcome_msg.decode())
        login(s)

        # 开始循环会话
        while True:
            task_list = ['ls', 'put', 'pull', 'mkdir', 'rm', 'cd']
            send_data = input('>>: ').strip()
            if len(send_data) == 0: continue

            cmd_list = send_data.split()
            if cmd_list[0].upper() == 'HELP':
                print('''\033[32;0m
                --------- Help 帮助信息 ----------
                1. ls    浏览目录
                2. cd    切换目录
                3. rm    删除文件
                4. mkdir 新建文件夹
                5. put   上传文件
                6. pull  下载文件
                \033[0m''')
                continue
            if cmd_list[0] not in task_list:
                print('不支持的命令！[Help]查看帮助...')
                continue

            task_types(s, cmd_list)

if __name__ == '__main__':
    main()


import asyncio
import json
from django.http import JsonResponse
from rest_framework import generics
from Ptt_Crawlers.ptt_search import Base_Crawler

# Create your views here.

def search_result(request):
    json_data = json.loads(request.body.decode('utf-8'))
    q = json_data.get('q', '')
    if q:
        query_string = q.replace(' ', '+')
        crawler = Base_Crawler(query_str=query_string)
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        loop.run_until_complete(crawler.parse())
        loop.close()

        items = crawler.items
        return JsonResponse(items, safe=False,  json_dumps_params={'ensure_ascii': False})


##################################################
###             tweets-k-means.py              ###
##################################################
# python tweets-k-means.py -s InitialSeeds.txt -i Tweets.json -o tweets-k-means-output.txt

import argparse
import random
from lib import ParseInput
from lib import kmean
from pprint import pprint

parser = argparse.ArgumentParser(description='tweet-k-menas')
parser.add_argument('-n', '--numberOfClusters', type=int, default=25, help='number of cluster')
parser.add_argument('-s', '--initialSeedsFile', help='initial seeds file')
parser.add_argument('-i', '--TweetsDataFile', help='input tweets data file')
parser.add_argument('-o', '--outputFile', help='output file')

args = parser.parse_args()
num_clusters = args.numberOfClusters
if num_clusters > 25:
    raise ValueError('number of cluster should be smaller than 25')
seeds = args.initialSeedsFile
tweets_path = args.TweetsDataFile
output_path = args.outputFile

print(num_clusters, seeds, tweets_path, output_path, sep='\n')

########## read json file ###########
data = ParseInput.readJson(jsonFile=tweets_path)
centers = ParseInput.readCenters(centerFile=seeds)
centers = random.sample(centers, num_clusters)

### first round
clusters = kmean.assignGroup(centers=centers, points=data)
new_centers = kmean.findCenters(clusters=clusters, points=data)

while set(new_centers) != set(centers):
    centers = new_centers.copy()
    clusters = kmean.assignGroup(centers=new_centers, points=data)
    new_centers = kmean.findCenters(clusters=clusters, points=data)
clusters = kmean.assignGroup(centers=new_centers, points=data)
# for center in new_centers:
#     for point in clusters[center]:
#         print(' '.join(data[point]))
#     print('--------------------')

sse = kmean.calculate_SSE(clusters=clusters, points=data)
with open(output_path, 'w') as f:
    for idx, item in enumerate(clusters.values()):
        text = str(idx+1) + ' ' + ','.join(map(str, item))
        print(text, file=f)
    print('-----------------', file=f)
    print('SSE: {}'.format(sse), file=f)


from darknet import darknet
import cv2, numpy as np, sys
from ctypes import *

net = darknet.load_net(b"D:/python_module/darknet-master/build/darknet/x64/project/myyolov3.cfg", b"D:/python_module/darknet-master/build/darknet/x64/project/backup/myyolov3_final.weights", 0)
meta = darknet.load_meta(b"D:/python_module/darknet-master/build/darknet/x64/project/my.data")
cap = cv2.VideoCapture("D:/python_module/darknet-master/build/darknet/x64/project/26-4.mp4")
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
fps = cap.get(cv2.CAP_PROP_FPS)
out = cv2.VideoWriter('otter_out26-4.avi', fourcc, fps, (640,480))
print(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
print(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
i = 0
while(cap.isOpened()):
    i += 1
    ret, image = cap.read()
    image = cv2.resize(image, dsize=(640, 480), interpolation=cv2.INTER_AREA)
    print(i)
    if not ret:
        break
    frame = darknet.nparray_to_image(image)
    r = darknet.detect_image(net, meta, frame, thresh=.5, hier_thresh=.5, nms=.45, debug= False)
    print(r)
    boxes = []

    for k in range(len(r)):
        width = r[k][2][2]
        height = r[k][2][3]
        center_x = r[k][2][0]
        center_y = r[k][2][1]
        bottomLeft_x = center_x - (width / 2)
        bottomLeft_y = center_y - (height / 2)
        x, y, w, h = bottomLeft_x, bottomLeft_y, width, height
        mytexts = r[k][0]
        mythresh = r[k][1]
        boxes.append((x, y, w, h, mytexts, mythresh))
    print(1)
    for k in range(len(boxes)):
        x, y, w, h, texts, threshs = boxes[k]
        top = max(0, np.floor(x + 0.5).astype(int))
        left = max(0, np.floor(y + 0.5).astype(int))
        right = min(image.shape[1], np.floor(x + w + 0.5).astype(int))
        bottom = min(image.shape[0], np.floor(y + h + 0.5).astype(int))
        cv2.rectangle(image, (top, left), (right, bottom), (255, 0, 0), 2)
        cv2.putText(image, texts.decode('utf-8') + '('+ str(threshs*100)[:5] + '%)', (top, left-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0))
        # cv2.line(image, (top + int(w / 2), left), (top + int(w / 2), left + int(h)), (0,255,0), 3)
        # cv2.line(image, (top, left + int(h / 2)), (top + int(w), left + int(h / 2)), (0,255,0), 3)
        # cv2.circle(image, (top + int(w / 2), left + int(h / 2)), 2, tuple((0,0,255)), 5)

    cv2.imshow('frame', image)
    out.write(image)
    darknet.free_image(frame) ## darknet에서 쓰는 c언어 동적할당 해제해주는 함수(써주어야 메모리가 버틴다.)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
out.release()
cv2.destroyAllWindows()


import pytest
pytestmark = [pytest.mark.setone, pytest.mark.working]

@pytest.fixture
def biosample_1(biosample_cc_wo_diff, GM12878_biosource, award, lab):
    return{
        "schema_version": '1',
        "award": award['@id'],
        "lab": lab['@id'],
        "cell_culture_details": biosample_cc_wo_diff['@id'],
        "biosource": [GM12878_biosource['@id']]
    }

def test_biosample_1_2(
        app, biosample_1, biosample_cc_wo_diff):
    migrator = app.registry['upgrader']
    value = migrator.upgrade('biosample', biosample_1, current_version='1', target_version='2')
    assert value['schema_version'] == '2'
    assert value['cell_culture_details'][0] == biosample_cc_wo_diff['@id']


#%% 0. Housekeeping

# =============================================================================
# 0.1 Import packages
# =============================================================================

import pandas as pd
import numpy as np
from datetime import datetime

# =============================================================================
# 0.2 Import data
# =============================================================================

train = pd.read_csv('data/train.csv', low_memory=False)
store = pd.read_csv('data/store.csv', low_memory=False)

#%% 1. Merging store to train data

# =============================================================================
# 1.1 Since data has to has the same size as 'train', a container is created
# =============================================================================

# Creating dataframe
expanded_store = train.loc[:,['Date','Store']]

# =============================================================================
# 1.2 Merging variables which do not have to be changed
# =============================================================================

# Variables which can be merged right away
easy = store.loc[:,['Store','StoreType','Assortment','CompetitionDistance']]

# Variables which can be simply merged
expanded_store = pd.merge(expanded_store, easy, on=['Store'])

# =============================================================================
# 1.3 Creating a dummy variable since when competition is open for every store
# =============================================================================

# Dataset of the variables which have to be transformed
competition = store.loc[:,['Store',
                           'CompetitionOpenSinceMonth','CompetitionOpenSinceYear']]

# For easier looping
competition.set_index('Store', inplace=True)

# Finding stores which have competition openend
open_comp = competition.any(axis=1)

# Stores with competition
open_stores = competition.loc[open_comp].index.unique()

# Generating Date when opened
for store_id in open_stores:
    year = competition.loc[store_id,'CompetitionOpenSinceYear'].astype(int)
    month = competition.loc[store_id,'CompetitionOpenSinceMonth'].astype(int)
    competition.loc[store_id,'CompetitionDate'] = datetime(year,month,1)

# Creating a dummy variable for whether competition openend for each store
expanded_store.loc[:,'CompetitionOpened'] = 0

for store_id in open_stores:

    # Getting the date when competition openend
    date = competition.loc[store_id,'CompetitionDate']
    date_str= date.strftime("%Y-%m-%d")

    store_number = expanded_store.loc[:,'Store'] == store_id

    maximum_date = expanded_store.loc[store_number,'Date'].max()

    # Whether it falls in time frame
    competition_existing = (expanded_store.loc[:,'Date'].between(date_str,maximum_date)) &  (expanded_store.loc[:,'Store'] == store_id)

    # Indicating whether competition is around
    expanded_store.loc[competition_existing, 'CompetitionOpened'] = 1

#%% Join


#!/usr/bin/env python3
# -*- encoding: utf-8; py-indent-offset: 4 -*-

# (c) Andreas Doehler <andreas.doehler@bechtle.com/andreas.doehler@gmail.com>

# This is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# ails.  You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Example Output:
#
#
from cmk.base.plugins.agent_based.agent_based_api.v1.type_defs import (
    CheckResult,
    DiscoveryResult,
)

from cmk.base.plugins.agent_based.agent_based_api.v1 import (
    register,
    Result,
    State,
    Service,
)

from .utils.redfish import parse_redfish_multiple, redfish_health_state

register.agent_section(
    name="redfish_drives",
    parse_function=parse_redfish_multiple,
)

def discovery_redfish_drives(section) -> DiscoveryResult:
    for key in section.keys():
        item = section[key]["Name"]
        yield Service(item=item)

def check_redfish_drives(item: str, section) -> CheckResult:
    data = None
    for key in section.keys():
        if item == section[key]["Name"]:
            data = section.get(key, None)
    if data is None:
        return

    disc_msg = "Size: %0.0fGB, Speed %s Gbs" % (
        data.get("CapacityBytes", 0) / 1024 / 1024 / 1024,
        data.get("CapableSpeedGbs", 0),
    )

    if data.get("MediaType") == "SSD":
        if data.get("PredictedMediaLifeLeftPercent"):
            disc_msg = disc_msg + ", Media Life Left: %d%%" % (
                int(data.get("PredictedMediaLifeLeftPercent", 0))
            )
        else:
            disc_msg = disc_msg + ", no SSD Media information available"

    yield Result(state=State(0), summary=disc_msg)

    dev_state, dev_msg = redfish_health_state(data["Status"])
    yield Result(state=State(dev_state), notice=dev_msg)

register.check_plugin(
    name="redfish_drives",
    service_name="Drive %s",
    sections=["redfish_drives"],
    discovery_function=discovery_redfish_drives,
    check_function=check_redfish_drives,
)


#!/usr/local/bin/python
# -*- coding: utf-8 -*-

import EllipticCurves as ec
from random import randint
from utils import prime_relative, is_divisor, has_modinv
from fractions import gcd

def lenstra(n):
    """
    Implementación del algoritmo de Lenstra para encontrar los factores
    primos de un número n de la forma n = p*q. Se asume que la proposición
    anterior es cierta, es decir, que en efecto n = p*q, regresando ambos
    factores de n.
    """
    a, x, y = randint(1,n-1), randint(1,n-1), randint(1,n-1)
    b = (pow(y,2) - pow(x,3) - (a * x)) % n
    E = ec.Curve(a, b, n)
    while not prime_relative(E.determinant(), n):
    	a, x, y = randint(1,n-1), randint(1,n-1), randint(1,n-1)
    	b = (pow(y,2) - pow(x,3) - (a * x)) % n
    	E = ec.Curve(a, b, n)
    p = (x, y)
    while not E.is_on_curve(p):
    	p = (randint(1,n), randint(1,n))
    lamb = calculate_lambda(p, p)
    point_aux = p
    while has_modinv(lamb, n):
    	point_aux = ec.add_points(p, point_aux, E)
    	lamb = calculate_lambda(p, point_aux)
    p = gcd(lamb, n)
    q = n/p
    if p==1 or q==1:
    	return lenstra(n)
    return (p, q)

def calculate_lambda(p, q):
	if p==q:
		return 2*p[1]
	return p[0]-q[0]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import logging
import importlib
import sys
import time
import os
import signal
import errno

def initlogger(config_basic):
    leveldict = {
        'CRITICAL' : logging.CRITICAL,
        'ERROR'    : logging.ERROR,
        'WARNING'  : logging.WARNING,
        'INFO'     : logging.INFO,
        'DEBUG'    : logging.DEBUG,
        'NOTSET'   : logging.NOTSET}
    defaultlogformat = \
        '%(asctime)s - %(levelname)s - %(name)s : %(message)s'
    # considering it is already in a daemon process which stdin/out/err has
    # been redirected, the info out put will be meaning less
    # however, you can still redirect stderr to a file to record any error
    # logging.info('...setting logger in main_job')
    logfilename = config_basic['filename'] # must specify one
    loglevel    = leveldict[config_basic.get('level','WARNING')]
    logformat    = config_basic.get('format',defaultlogformat)
    # logging.info("filename: {}, level: {}".format(logfilename,loglevel))
    name = config_basic.get('name','default')
    name = 'main_job.' + name
    logger = logging.getLogger(name)
    logger.setLevel(loglevel)
    filehandler = logging.FileHandler(logfilename)
    formatter = logging.Formatter(logformat)
    filehandler.setFormatter(formatter)
    logger.addHandler(filehandler)
    #prevent logger from propagating to parent
    logger.propagate = False
    ##set file handler to root so there will be no extra message in stderr
    #logging.getLogger('root').addHandler(filehandler)
    #set logging level to warning in prevention of too much
    #logging.basicConfig(level = logging.WARNING)
    return logger

#def killwait(interval, maxcnt) -> bool:
#    """kill-wait loop until it die
#delay

# Main job will fork a child to do main work and the sleep.
# When parent is awake but child still hasn't exited,
# it will try to kill the child process.
# Each kill happens between each wait(sec)
# and wait for at most maxwait times.
# Child process still not exit now?
# Parent process exits 1 then.
maxwait = 20
eachwait = 0.25

class main_job:
    def main_clean(self):
        pass

    # the main job to do
    def main_do(self):
        config = self.config
        logger = self.logger
        start_delay   = config['basic']['start_delay']
        repeat_time   = config['basic']['repeat_time']
        restart_delay = config['basic']['restart_delay']
        nodeconfig    = config['spec']
        mod = importlib.import_module(\
            "mod_" + config['basic']['nodetype'])
        handler_run = getattr(mod, "run")
        handler_fix = getattr(mod, "fix")

        logger.info('ready to start')
        #   wait a moment
        time.sleep(start_delay)

        logger.info('now fork for work process')
        #   fork for work process
        cnt = 0
        while cnt < repeat_time or repeat_time < 0:
            cnt = cnt + 1
            # fork for work func
            try:
                pid = os.fork()
                assert pid >= 0
            except Exception as e:
                logger.exception('failed to fork for work func')
                raise e
            # the child process
            if pid == 0:
                exitcode = 0
                logger = logging.getLogger(logger.name + '.' + hex(cnt))
                logger.info('forked for work process')
                try:
                    # don't care return value, if error happened, just raise it
                    handler_run(nodeconfig, logger, cnt)
                except SystemExit as e:
                    # although invode sys.exit is not recommended, but still
                    # ignore Exception raised by sys.exit(0)
                    logger.info('invoked sys.exit or just exit is \
not recommeneded in fun run')
                    if e.code!=0:
                        logger.exception('work fun exit {}'.format(e.code))
                        exitcode = e.code
                    else:
                        pass # invoked exit(0), which mean ok
                except:#except other error
                    logger.exception('a no-exit raise in work fun')
                logger.info('exit work process')
                os._exit(exitcode)# ensure 1-2-4-8...will not happen

            # parent process
            #logger.info('#{}: sleep now'.format(cnt))
            time.sleep(restart_delay)

            pidwait, status = os.waitpid(pid, os.WNOHANG)
            if pidwait != pid:
                #child process still not finished???
                logger.warning('parent awake now, but child havn\'t exited')
                logger.info('try to kill it')
                for waitcnt in range(0, maxwait):
                    if pidwait != 0:
                        break
                    try:
                        os.kill(pid, signal.SIGTERM)
                    except Exception as e:
                        if type(e) == OSError and e.errno == errno.ESRCH:
                            break
                    time.sleep(eachwait)
                pidwait, status = os.waitpid(pid, os.WNOHANG)
            if pidwait == pid:
                exitcode = int(status >> 8)
                if exitcode != 0:
                    logger.warning('work process exited {}'.format(exitcode))
                else:
                    logger.info('work process exited 0')
            else:
                logging.error('failed to terminate work process')
                sys.exit(1)
        # end of while

        logger.info('exit daemon process now')
        # this is a child process, thus just os._exit it
        os._exit(0)

    def start(self):
        self.main_clean()
        self.main_do()
    def stop(self):
        self.main_clean()

    def __init__(self, config):
        self.config = config
        self.logger = initlogger(config['basic'])

    def __enter__(self):
        self.start()

    def __exit__(self,exc_type, exc_val, exc_tb):
        self.stop()

#import sys
#import os
#import os.path
#if __name__ == '__main__':
#    modpath = os.path.abspath(sys.argv[0])
#    modpath = os.path.dirname(modpath)
#    modpath = os.path.join(modpath, "./mods")
#    sys.path.append(modpath)
#    nodeconfig = {"nodetype": "test",
#                  "start_delay": 2,
#                  "restart_delay": 2,
#                  "repeat_time": 5,
#                  "logpath": "./__pycache__/",
#                  "logformat": "%(asctime)s - %(levelname)s - %(name)s : %(message)s",
#                  "pidfile": "/temp/project_sperm_test0.pid",
#                  "dpidfile": "/temp/project_sperm_test0d.pid",
#                  "stdout": "/dev/null",
#                  "stdin": "/dev/null",
#                  "stderr": "/dev/null",
#\
#                  "exitcode": 1, "fixexitcode": 0}
#    nodeconfig['nodename'] = 'test0'
#    daemon_start(nodeconfig)
#
#   "test0":
#   {
#       "nodetype":"test",
#       "start_delay":5,
#       "restart_delay":5,
#       "repeat_time":5,
#       "logpath":"/var/log/project_sperm/",
#       "logformat":"%(asctime)s - %(levelname)s - %(name)s : %(message)s",
#       "pidfile":"/temp/project_sperm_test0.pid",
#       "dpidfile":"/temp/project_sperm_test0d.pid",
#       "stdout":"/dev/null",
#       "stdin":"/dev/null",
#       "stderr":"/dev/null",
#
#       "exitcode":0
#   }
#
#
#
#
#            # fork for fix func
#            try:
#                pid = os.fork()
#            except:
#                logger.exception('failed to fork to fix problems')
#            if pid == 0:
#                # child
#                handler_fix(nodeconfig, logger, exitcode)
#                logger.warning('fix func return')
#                exit(0)
#            # parent
#            pid, status = os.wait()
#            exitcode = int(status >> 8)
#            logger.info('fix process exited with {}'.format(exitcode))
#            if exitcode != 0:
#                # failed to fix problem
#                logger.error('failed to fix problems')
#                break
#            else:
#                logger.info('successfully fixed problems')
#


from rest_framework import serializers
from stands.models import Stand
from events.models import Event
from events.serializers import EventListSerializer

class StandListSerializer(serializers.ModelSerializer):
    class Meta:
        model = Stand
        fields = ('person', 'emotion', 'age', 'sex', 'people', 'description', 'id', 'eventId')

class StandOwnerListSerializer(serializers.ModelSerializer):
    eventId = EventListSerializer()

    class Meta:
        model = Stand
        fields = '__all__'

class StandDetailSerializer(serializers.ModelSerializer):
    class Meta:
        model = Stand
        fields = '__all__'

class ImageSerializer(serializers.Serializer):
    image = serializers.CharField()
    stand = serializers.PrimaryKeyRelatedField(queryset=Stand.objects.all())


from sys import pycache_prefix
import tensorflow as tf
import pydub
import wave
import scipy as sp
import matplotlib.pyplot as plt

mmodel=tf.keras.models.load_model('music_emotion.h5')

mdic={0:"Happy", 1:"Sad", 2:"Neutral", 3:"Happy"}
def mpredict(filepath:str):
    if ".mp3" in filepath:
        pydub.AudioSegment.from_mp3(filepath).export(filepath.replace('32kps/', 'wav/').replace('.mp3', '.wav'), format="wav")
        print(filepath)
    if ".wav" in filepath:
        wo = wave.open(filepath, 'rb' )
        chunk = 65536
        data = sp.fromstring(wo.readframes(chunk), sp.int16)
        srate = wo.getframerate()
        nFFT = 1024
        window = sp.hamming(nFFT)

        fig, ax = plt.subplots(1)
        fig.subplots_adjust(left=0,right=1,bottom=0,top=1)
        ax.set_axis_off()
        Pxx, freq, bins, im = ax.specgram(data,
                                        NFFT=nFFT,
                                        Fs=srate,
                                        noverlap=512,
                                        window=window)
        plt.savefig(filepath.replace('wav', 'png'))
        plt.close(fig)


from task import Task
import tensorflow as tf
from keras.layers import Input, Conv2D, Dense, GRU, LSTM, Activation, Flatten, ZeroPadding2D
from keras.layers.normalization import BatchNormalization
from keras.layers.core import Dropout, Reshape
from keras.models import Model, Sequential, load_model
from keras.layers.pooling import GlobalAveragePooling2D, MaxPool2D
from keras.optimizers import Adam, SGD
from keras.callbacks import TensorBoard

import time
from monitor import Monitor
import matplotlib.pyplot as plt
import config as cfg
import argparse

def describe(model):
    description = 'Model layers / shapes / parameters:\n'
    total_params = 0

    for layer in model.layers:
        layer_params = layer.count_params()
        description += '- {}'.format(layer.name).ljust(20)
        description += '{}'.format(layer.input_shape).ljust(20)
        description += '{0:,}'.format(layer_params).rjust(12)
        description += '\n'
        total_params += layer_params

    description += 'Total:'.ljust(30)
    description += '{0:,}'.format(total_params).rjust(22)

    print(description)

def convolution_block(x, filters):
    x = ZeroPadding2D()(x)
    x = Conv2D(filters, (3, 3), kernel_initializer='he_uniform')(x)
    x = BatchNormalization(axis=1)(x)
    x = Activation('relu')(x)
    x = Dropout(0.5)(x)
    x = ZeroPadding2D()(x)
    x = Conv2D(filters, (3, 3), kernel_initializer='he_uniform')(x)
    x = BatchNormalization(axis=1)(x)
    x = Activation('relu')(x)
    x = Dropout(0.5)(x)
    return x

def train(args):
    # arg parameters
    batch_size = args.batch_size
    epochs = args.epochs
    patience = args.patience

    task = Task(64, args)
    task.load_dataset()
    task.prepare_data()

    # model = Sequential()

    input = Input(shape=(64, 334, 1))
    x = convolution_block(input, 32)
    # x = convolution_block(x, 64)
    # x = convolution_block(x, 128)
    # x = convolution_block(x, 128)
    x = GlobalAveragePooling2D()(x)
    x = Dense(60, activation='relu')(x)
    x = BatchNormalization(axis=1)(x)
    x = Activation('relu')(x)
    x = Dense(10, activation='softmax')(x)
    model = Model(input=[input], outputs=[x])

    # model.add(GlobalAveragePooling2D())

    # model.add(Dense(512, activation='relu', kernel_initializer='he_uniform', ))
    # model.add(Dropout(0.5))
    # model.add(Dense(10, activation='softmax', kernel_initializer='he_uniform'))
    #
    # model.add(Dropout(0.5))

    # reshape (1,500,96)
    # model.add(Reshape(target_shape=(286, 32)))
    #
    # # three layers of GRU
    # model.add(GRU(32, dropout=0.2, bias_initializer='ones', return_sequences=True, activation='tanh'))
    # model.add(GRU(32, dropout=0.2, kernel_initializer='he_uniform', bias_initializer='ones', activation='tanh'))
    #
    # # FC
    # model.add(Dense(16))
    # model.add(BatchNormalization(axis=1))
    # model.add(Dropout(0.2))
    # model.add(Dense(10, activation='softmax', kernel_initializer='he_uniform'))

    model.compile(loss='categorical_crossentropy', optimizer=Adam(),
                  metrics=['accuracy'])
    describe(model)

    steps_per_epoch = len(task.train[0]) // batch_size

    tensorBoard = TensorBoard()

    current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

    checkpoint_path = "/data/check_points/2018_task1_cp/" + str(current_time) + "_cp-{epoch:04d}.ckpt"
    # checkpoint_dir = os.path.dirname(checkpoint_path)

    cp_callback = tf.keras.callbacks.ModelCheckpoint(
        checkpoint_path, verbose=1, save_weights_only=True,
        # Save weights, every 5-epochs.
        period=10)
    estop = tf.keras.callbacks.EarlyStopping(patience=patience)

    #   Keras的生成器要求是死循环，也就是说永远都有next元素可以抛出。在适当位置添加while True
    history = model.fit_generator(generator=task.iterbatches(batch_size, task.train, True),
                                  steps_per_epoch=steps_per_epoch, epochs=epochs,
                                  callbacks=[Monitor(1, model, task, args)],
                                  validation_data=task.iterbatches(batch_size, task.validation, False),
                                  validation_steps=len(task.validation[0]) // batch_size,
                                  )

    # model.save(os.path.join(MODEL_PATH, 'crnn_model_{}.h5'.format(time.time())))

    # score = model.evaluate_generator(task.iterbatches(batch_size, task.test, False),
    #                                  steps=len(task.test) // batch_size)
    # print('acc: {}'.format(score[1]))
    # print('loss: {}'.format(score[0]))
    # plot(history)
    # finish_instance()

def finish_instance():
    import os
    os.system('sh /data/stop_instance.sh')

def plot(history):
    plt.figure()
    plt.subplot(2, 1, 1)
    plt.plot(history.history['acc'], color="blue", linewidth=1.0)
    plt.plot(history.history['val_acc'], color="blue", linewidth=1.0, linestyle="-")
    plt.title('model acc')
    plt.xlabel('epoch')
    plt.ylabel('acc')
    plt.legend(['train', 'test'], loc='upper left')
    plt.subplot(2, 1, 2)
    plt.plot(history.history['loss'], color="red", linewidth=1.0)
    plt.plot(history.history['val_loss'], color="red", linewidth=1.0, linestyle="-")
    plt.title('val_loss')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(['train', 'test'], loc='upper left')
    plt.show()
    plt.savefig('/data/dl-result-image/2018_task_1_result')

if __name__ == '__main__':
    arg = argparse.ArgumentParser(description='')
    arg.add_argument('--batch_size', type=int, default=32)
    arg.add_argument('--epochs', type=int, default=40)
    arg.add_argument('--patience', type=int, default=10)
    arg.add_argument('--model_path', type=str, default=cfg.MODEL_PATH)
    arg.add_argument('--feature_path', type=str, required=True)
    # arg.add_argument('--plot_image_path', type=str)

    arg = arg.parse_args()

    train(arg)


import os

filter=[".txt"] #设置过滤后的文件类型 当然可以设置多个类型

def all_path(dirname):

    result = []#所有的文件

    for maindir, subdir, file_name_list in os.walk(dirname):

        # print("1:",maindir) #当前主目录
        # print("2:",subdir) #当前主目录下的所有目录
        # print("3:",file_name_list)  #当前主目录下的所有文件

        for filename in file_name_list:
            apath = os.path.join(maindir, filename)#合并成一个完整路径
            ext = os.path.splitext(apath)[1]  # 获取文件后缀 [0]获取的是除了文件名以外的内容
            dir = os.path.split(maindir)[1]
            #起诉书 裁定书 判决书
            if ext in filter and dir == "判决书":
                result.append(apath)
    result.sort()
    return result

#files=all_path("政法")
from collections import defaultdict
def merge(files):
    d = defaultdict(list)
    for filename in files:
        d[os.path.split(filename)[0]].append([os.path.split(filename)[1]])

    for key in d:
        newfile=open('政法\\'+key.split('\\')[1]+'.txt','a+')
        for value in d[key]:
            txt = open(os.path.join(key, value[0]),encoding='UTF-8')
            newfile.write(txt.read())
        newfile.close()

#merge(files)

import numpy
import re
def qishushu():
    #案号 原告 被告 案由 请求 事实
    listCase = []
    for maindir, subdir, file_name_list in os.walk("起诉书"):
        for file in file_name_list:
            d = defaultdict(set)
            d["案号"]=os.path.splitext(file)[0]
            f = open(os.path.join(maindir, file))
            txt=f.read()
            f.close()
            txt.strip()
            txts=re.split(r'[：，\n ]',txt)
            for index in range(len(txts)):
                if txts[index]=="原告":
                    if txts[index+1]=="单位名称" and txts[index+2]!='':
                        d["原告"].add(txts[index+2])
                        i=3
                        yuangaoStr=''
                        while(i<3+4 and index+i<len(txts) and txts[index+i] != "被告"):
                            yuangaoStr=yuangaoStr+txts[index+i]+','
                            i=i+1
                        d[txts[index+2]].add(yuangaoStr)
                    elif txts[index+1]!='':
                        d["原告"].add(txts[index+1])
                        i=2
                        yuangaoStr=''
                        while(i<2+4 and index+i<len(txts) and txts[index+i] != "被告"):
                            yuangaoStr=yuangaoStr+txts[index+i]+','
                            i=i+1
                        d[txts[index+1]].add(yuangaoStr)

                elif txts[index]=="被告":
                    if txts[index+1]=="单位名称" and txts[index+2]!='':
                        d["被告"].add(txts[index+2])
                        i=3
                        beigaoStr=''
                        while(i<3+4 and  index+i<len(txts) and txts[index+i] != "案由"and txts[index+i] != "诉讼请求"):
                            beigaoStr=beigaoStr+txts[index+i]+','
                            i=i+1
                        d[txts[index+2]].add(beigaoStr)
                    elif txts[index+1]!='':
                        d["被告"].add(txts[index+1])
                        i=2
                        beigaoStr=''
                        while(i<2+4 and index+i<len(txts) and txts[index+i] != "案由"and txts[index+i] != "诉讼请求"):
                            beigaoStr=beigaoStr+txts[index+i]+','
                            i=i+1
                        d[txts[index+1]].add(beigaoStr)
                elif txts[index]=="案由":
                    d["案由"].add(txts[index+1])
                elif txts[index]=="诉讼请求":
                    i=1
                    requestStr=''
                    while(i<10 and index+i<len(txts) and txts[index+i] != "事实和理由" and txts[index+i] != "事实与理由"):
                        requestStr=requestStr+txts[index+i]
                        i=i+1
                    d["诉讼请求"].add(requestStr)

                elif txts[index]=="事实和理由" or  txts[index] == "事实与理由":
                    i=1
                    evidenceStr=''
                    while(i<20 and index+i<len(txts) and  txts[index+i] != "此敬" and txts[index+i] != "具状人"):
                        evidenceStr=evidenceStr+txts[index+i]
                        i=i+1
                    d["事实和理由"].add(evidenceStr)
            listCase.append(d)
    return listCase

def panjueshu():
    #案号  法院 法律(判决) 判决
    listCase = []
    for maindir, subdir, file_name_list in os.walk("判决书"):
        for file in file_name_list:
            d = defaultdict(set)
            d["案号"]=os.path.splitext(file)[0]
            f = open(os.path.join(maindir, file))
            txt=f.read()
            f.close()
            txt.strip()
            txts=re.split(r'[：。\n ]',txt)
            for index in range(len(txts)):
                if txts[index].endswith('法院') and txts[index+1].endswith('判决书'):
                    d["法院"]=txts[index]
            txts=txt.replace('\n','')
            txts=re.split(r'[：。\n ]',txts)
            for index in range(len(txts)):
                if '判决如下' in txts[index]:
                    lawlist=re.split(r'[、， ]',txts[index])
                    lawname=''
                    #《最高人民法院关于适用《中华人民共和国民事诉讼法》的解释》第九十条之规定
                    for li in range(len(lawlist)):
                        l=lawlist[li]
                        if lawlist[li].find("《")!=-1 and lawlist[li].find("》")==-1 and lawlist[li+1].find("》")!=-1:
                            l=lawlist[li]+'、'+lawlist[li+1]
                        if '第' in l and '条' in l:
                            s=l[l.find('第'):l.find('条')+1]
                            if l.find("《")!=-1:
                                lawname=l[l.find('《'):l.rfind('》')+1]
                            d["法律(判决)"].add(lawname+s)
                    panjuestr=''
                    i=1
                    while(index+i<len(txts) and txts[index+i].startswith('如不服本判决') == False):
                        panjuestr=panjuestr+txts[index+i]+'。'
                        i=i+1
                    d['判决']=panjuestr
            listCase.append(d)
    return listCase

def caidingshu():
    #案号  法律 裁定
    listCase = []
    for maindir, subdir, file_name_list in os.walk("裁定书"):
        for file in file_name_list:
            d = defaultdict(set)
            d["案号"]=os.path.splitext(file)[0]
            f = open(os.path.join(maindir, file))
            txt=f.read()
            f.close()
            txt.strip()
            txts=txt.replace('\n','')
            txts=re.split(r'[：。\n ]',txts)
            for index in range(len(txts)):
                if '裁定如下' in txts[index]:
                    lawlist=re.split(r'[、， ]',txts[index])
                    lawname=''
                    #《最高人民法院关于民事执行中变更、追加当事人若干问题的规定》
                    for li in range(len(lawlist)):
                        l=lawlist[li]
                        if lawlist[li].find("《")!=-1 and lawlist[li].find("》")==-1 and lawlist[li+1].find("》")!=-1:
                            l=lawlist[li]+'、'+lawlist[li+1]
                        if '第' in l and '条' in l:
                            s=l[l.find('第'):l.find('条')+1]
                            if l.find("《")!=-1:
                                lawname=l[l.find('《'):l.rfind('》')+1]
                            d["法律(裁定)"].add(lawname+s)

                    caidingstr=''
                    i=1
                    while(index+i<len(txts) and txts[index+i].startswith('如不服本裁定') == False):
                        caidingstr=caidingstr+txts[index+i]+'。'
                        i=i+1
                    d['裁定']=caidingstr
            listCase.append(d)
    return listCase

def getallcaselist():
    #案号 原告 被告 案由 请求 事实 0 1 2 3 4 5
    qisushulist=qishushu()
    #案号  法院 法律(判决) 判决 6 7 8
    panjueshulist=panjueshu()
    #案号  法律(裁定) 裁定 9 10
    caidingshulist=caidingshu()

    for q in qisushulist:
        for p in panjueshulist:
            if q['案号']==p['案号']:
                q.update(p)
        for c in caidingshulist:
            if q['案号']==c['案号']:
                q.update(c)
    return qisushulist

def nodes(nodeset,group,valuedic):
    nodeList=[]
    for id in nodeset:
        dic={}
        dic["group"]=group
        dic["id"]=id
        if group==1 or group==2:
            if id in valuedic:
                dic["info"]=','.join(valuedic[id]).replace('\n','')
        if group==6:
            if os.path.exists('法院\\'+id+'.txt') == True:
                txtfile=open('法院\\'+id+'.txt','r')
                txts=txtfile.read()
                txtfile.close()
                txts=txts.replace('\n','')
                dic["info"]=txts
            else:
                print(id)
        if group==7 or  group==9:
            lawname=id[id.find('《'):id.rfind("》")+1]
            tiaoshu=id[id.rfind("》")+1:]
            lawstr=''
            if os.path.exists('法律\\'+lawname+'.txt') == True:
                txtfile=open('法律\\'+lawname+'.txt','r',encoding='gb2312')
                txts=txtfile.read()
                txtfile.close()
                txts=re.split(r'[\s]',txts)
                for index in range(len(txts)):
                    if txts[index]==tiaoshu:
                        i=1
                        while(index+i<len(txts) and (txts[index+i].startswith('第') ==False or txts[index+i].endswith('条')==False)):
                            lawstr=lawstr+txts[index+i]
                            i=i+1
                        dic["info"]=lawstr
            else:
                print("没有此文件"+lawname+','+tiaoshu+','+id)
            if lawstr=='':
                print(lawname+','+tiaoshu+','+id)
        nodeList.append(dic)
    return nodeList

def getJson():
    dic = defaultdict(list)
    linkList=[]
    nodeList=[]
    anhaoset=set()
    beigaoset=set()
    yuangaoset=set()
    anyouset=set()
    qingqiuset=set()
    shishiset=set()
    fayuanset=set()
    falvpanjueset=set()
    panjueset=set()
    falvcaidingset=set()
    caidingset=set()
    caseList = getallcaselist()
    #print(caseList)
    yuangaodic={}
    beigaodic={}
    for l in caseList:
        if "案号" in l:
            anhaoset.add(l["案号"])
            if "原告" in l:
                yuangaoset=yuangaoset|l["原告"]
                for t in l["原告"]:
                    yuangaodic[t]=l[t]
                    d={}
                    d["source"]=l["案号"]
                    d["value"]="原告为"
                    d["target"]=t
                    linkList.append(d)
            if "被告" in l:
                beigaoset=beigaoset|l["被告"]
                for t in l["被告"]:
                    beigaodic[t]=l[t]
                    d={}
                    d["source"]=l["案号"]
                    d["value"]="被告为"
                    d["target"]=t
                    linkList.append(d)
            if "案由" in l:
                d={}
                d["source"]=l["案号"]
                d["value"]="案由为"
                d["target"]=','.join(l["案由"])
                anyouset=anyouset|l["案由"]
                linkList.append(d)
            if "诉讼请求" in l:
                d={}
                d["source"]=l["案号"]
                d["value"]="请求为"
                d["target"]=','.join(l["诉讼请求"]).replace('\n','')
                qingqiuset.add(','.join(l["诉讼请求"]).replace('\n',''))
                linkList.append(d)
            if "事实和理由" in l:
                d={}
                d["source"]=l["案号"]
                d["value"]="事实和理由为"
                d["target"]=','.join(l["事实和理由"]).replace('\n','')
                shishiset.add(','.join(l["事实和理由"]).replace('\n',''))
                linkList.append(d)
            if "法院" in l:
                fayuanset.add(l["法院"])
                d={}
                d["source"]=l["案号"]
                d["value"]="受理"
                d["target"]=l["法院"]
                linkList.append(d)
            if "判决" in l:
                panjueset.add(l["判决"])
                d={}
                d["source"]=l["案号"]
                d["value"]="判决为"
                d["target"]=l["判决"]
                linkList.append(d)
            if "裁定" in l:
                caidingset.add(l["裁定"])
                d={}
                d["source"]=l["案号"]
                d["value"]="裁定为"
                d["target"]=l["裁定"]
                linkList.append(d)
        if "事实和理由" in l and "案由" in l:
            d={}
            d["source"]=','.join(l["事实和理由"]).replace('\n','')
            shishiset.add(','.join(l["事实和理由"]).replace('\n',''))
            d["value"]="证明"
            d["target"]=','.join(l["案由"]).replace('\n','')
            anyouset.add(','.join(l["案由"]).replace('\n',''))
            linkList.append(d)

        if "判决" in l and "法院" in l:
            fayuanset.add(l["法院"])
            panjueset.add(l["判决"])
            d={}
            d["source"]=l['法院']
            d["value"]="给出"
            d["target"]=l['判决']
            linkList.append(d)
        if "裁定" in l and "法院" in l:
            fayuanset.add(l["法院"])
            caidingset.add(l["裁定"])
            d={}
            d["source"]=l['法院']
            d["value"]="给出"
            d["target"]=l['裁定']
            linkList.append(d)
        if "裁定" in l and "法律(裁定)" in l:
            caidingset.add(l["裁定"])
            falvcaidingset=falvcaidingset|l["法律(裁定)"]
            for t in l["法律(裁定)"]:
                d={}
                d["source"]=l["裁定"]
                d["value"]="依据"
                d["target"]=t
                linkList.append(d)
        if "判决" in l and "法律(判决)" in l:
            panjueset.add(l["判决"])
            falvpanjueset=falvpanjueset|l["法律(判决)"]
            for t in l["法律(判决)"]:
                d={}
                d["source"]=l["判决"]
                d["value"]="依据"
                d["target"]=t
                linkList.append(d)
    dic['links']=linkList
    #案号  法院 法律(判决) 判决 6 7 8
    #案号  法律(裁定) 裁定 9 10
    nodeList=nodeList+nodes(anhaoset,0,{})
    nodeList=nodeList+nodes(yuangaoset,1,yuangaodic)
    nodeList=nodeList+nodes(beigaoset,2,beigaodic)
    nodeList=nodeList+nodes(anyouset,3,{})
    nodeList=nodeList+nodes(qingqiuset,4,{})
    nodeList=nodeList+nodes(shishiset,5,{})
    nodeList=nodeList+nodes(fayuanset,6,{})
    nodeList=nodeList+nodes(falvpanjueset,7,{})
    nodeList=nodeList+nodes(panjueset,8,{})
    nodeList=nodeList+nodes(falvcaidingset,9,{})
    nodeList=nodeList+nodes(caidingset,10,{})

    dic['nodes']=nodeList
    return dic

'''
dicJson=getJson()
import json
with open('a.txt','w') as f:
    f.write( json.dumps( dicJson,ensure_ascii=False,indent=2 ) )

'''


import re
import sys

import cv2

from algo import tools
from settings import *

if __name__ == '__main__':
    try:
        folder = os.path.join(LOG_DIR, sys.argv[1])
        assert os.path.isdir(folder), 'invalid folder path given: %s'%(folder,)
        regex = sys.argv[2]
        target_file = sys.argv[3]
        framerate = int(sys.argv[4])
    except Exception as e:
        print(str(e))
        print('\nUSAGE: %s <log-dir relative input-dir> <img_regex> <target_file> <framerate>'%(sys.argv[0],))
        quit()

    test = re.compile(regex)
    img_files = []
    for file in os.listdir(folder):
        if test.match(file):
            img_files.append(file)

    img_files = sorted(img_files)
    assert len(img_files)>3, 'too few images found: %s'%(img_files,)

    img0 = cv2.imread(os.path.join(folder, img_files[0]), cv2.IMREAD_COLOR)
    h, w, c = img0.shape
    writer = cv2.VideoWriter(target_file, cv2.VideoWriter_fourcc(*'DIVX'), framerate, (w, h))
    try:
        for i, f in enumerate(img_files):
            tools.show_progress(len(img_files), i)
            img = cv2.imread(os.path.join(folder, f), cv2.IMREAD_COLOR)
            writer.write(img)
    finally:
        writer.release()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.apps import apps
from django.utils.encoding import python_2_unicode_compatible
from core.util.misc_config import fee_comment_missing
from django.db.models import Min,Sum

import datetime

#Models
@python_2_unicode_compatible
class BookingComProperty(models.Model):
	id=models.IntegerField(primary_key=True)
	name=models.CharField(max_length=80,blank=True)
	def __str__(self):
		return str(self.name)

@python_2_unicode_compatible
class LodgifyProperty(models.Model):
	id=models.IntegerField(primary_key=True)
	name=models.CharField(max_length=80,blank=True)
	def __str__(self):
		return str(self.name)

@python_2_unicode_compatible
class LodgifyRoom(models.Model):
	id=models.IntegerField(primary_key=True)
	name=models.CharField(max_length=120,blank=True)
	def __str__(self):
		return str(self.name)

@python_2_unicode_compatible
class Property(models.Model):
	name=models.CharField(max_length=50,unique=True)
	label=models.CharField(max_length=50,null=True,blank=True)
	street=models.CharField(max_length=90,blank=True)
	zipCode=models.CharField(max_length=10,blank=True)
	city=models.CharField(max_length=40,blank=True)
	addedAt=models.DateTimeField(auto_now_add=True)
	updatedAt=models.DateTimeField(auto_now=True)
	lodgifyProperty=models.ForeignKey(LodgifyProperty,on_delete=models.PROTECT,null=True,blank=True)
	bookingComProperty=models.ForeignKey(BookingComProperty,on_delete=models.PROTECT,null=True,blank=True)
	def __str__(self):
		return str(self.name)
	def save(self, *args, **kwargs):
		o=Property.objects.filter(name=self.name)
		if (self.pk!=None or not o.exists()):
			super(Property, self).save(*args, **kwargs)

#@python_2_unicode_compatible
class Room(models.Model):
	name=models.CharField(max_length=50)
	label=models.CharField(max_length=50,null=True,blank=True)
	property=models.ForeignKey(Property,on_delete=models.PROTECT)
	stdOccupation=models.IntegerField()
	maxOccupation=models.IntegerField()
	bookableFrom=models.DateField(null=True,blank=True)
	addedAt=models.DateTimeField(auto_now_add=True)
	updatedAt=models.DateTimeField(auto_now=True)
	lodgifyRoom=models.ForeignKey(LodgifyRoom,on_delete=models.PROTECT,null=True,blank=True)
	accountingId=models.IntegerField()

	class Meta:
		unique_together = ('name', 'property',)
	def __str__(self):
		return str(self.name)
	def save(self, *args, **kwargs):
		o=Room.objects.filter(property=self.property,name=self.name)
		if (self.pk!=None or not o.exists()):
			super(Room, self).save(*args, **kwargs)

@python_2_unicode_compatible
class Guest(models.Model):
	name=models.CharField(max_length=60,blank=True,null=True)
	email=models.CharField(max_length=100,blank=True,null=True)
	phone=models.CharField(max_length=40,blank=True,null=True)
	country=models.CharField(max_length=40,blank=True,null=True)
	zipCode=models.CharField(max_length=10,blank=True,null=True)
	address=models.CharField(max_length=100,blank=True,null=True)
	mainGuest=models.ForeignKey("self",on_delete=models.PROTECT,null=True,blank=True)
	class Meta:
		unique_together = ('email', 'name',)
	def __str__(self):
		return (self.name or self.email) or str(self.pk)
	def save(self, *args, **kwargs):
		if self.email:
			self.email=self.email.lower()
		if self.name:
			self.name=self.name.title()
		o=Guest.objects.filter(email=self.email,name=self.name)
		if (self.pk!=None or not o.exists()):
			super(Guest, self).save(*args, **kwargs)
		else:
			og=Guest.objects.get(email=self.email,name=self.name)
			if (og.phone=="" or og.phone==None):
				o.update(phone=self.phone)
			if (og.country=="" or og.country==None):
				o.update(country=self.country)
			if (og.zipCode=="" or og.zipCode==None):
				o.update(zipCode=self.zipCode)
			if (og.address=="" or og.address==None):
				o.update(address=self.address)

	@property
	def isMainGuest(self):
		return self.guest_set.exists()

	@property
	def lastOrder(self):
		if len(self.order_set.all())>0:
			own=self.order_set.order_by('-bookedAt')[0]
			if self.isMainGuest:
				out=own
				for s in self.guest_set.all():
					if len(s.order_set.all())>0:
						sec=s.order_set.order_by('-bookedAt')[0]
						if sec.bookedAt>out.bookedAt:
							out=sec
				return out
			else:
				return own
		else:
			return None

	@property
	def firstOrder(self):
		if len(self.order_set.all())>0:
			own=self.order_set.order_by('bookedAt')[0]
			if self.isMainGuest:
				out=own
				for s in self.guest_set.all():
					if len(s.order_set.all())>0:
						sec=s.order_set.order_by('bookedAt')[0]
						if sec.bookedAt<out.bookedAt:
							out=sec
				return out
			else:
				return own
		else:
			return None

	@property
	def totalOrders(self):
		own=len(self.order_set.filter(lodgifyOrder__status="Booked"))
		if self.isMainGuest:
			out=own
			for s in self.guest_set.all():
				out+=len(s.order_set.filter(lodgifyOrder__status="Booked"))
			return out
		else:
			return own

	@property
	def totalNights(self):
		own=sum([o.nights for o in self.order_set.filter(lodgifyOrder__status="Booked")])
		if self.isMainGuest:
			out=own
			for s in self.guest_set.all():
				out+=sum([o.nights for o in s.order_set.filter(lodgifyOrder__status="Booked")])
			return out
		else:
			return own

	@property
	def totalRevenue(self):
		own=self.order_set.filter(lodgifyOrder__status="Booked").aggregate(Sum('total'))['total__sum'] if self.order_set.filter(lodgifyOrder__status="Booked").aggregate(Sum('total'))['total__sum'] else 0
		if self.isMainGuest:
			out=own
			for s in self.guest_set.all():
				rev=s.order_set.filter(lodgifyOrder__status="Booked").aggregate(Sum('total'))['total__sum'] if s.order_set.filter(lodgifyOrder__status="Booked").aggregate(Sum('total'))['total__sum'] else 0
				out+=rev
			return out
		else:
			return own

# @python_2_unicode_compatible
	# class Status(models.Model):
	# 	name=models.CharField(max_length=80,unique=True)
	# 	addedAt=models.DateTimeField(auto_now_add=True)
	# 	updatedAt=models.DateTimeField(auto_now=True)
	# 	def __str__(self):
	# 		return self.name
	# 	def __unicode__(self):
	# 		return unicode(self.pk) or u''
	# 	def save(self, *args, **kwargs):
	# 		o=Status.objects.filter(name=self.name)
	# 		if (self.pk!=None or not o.exists()):
	# 			super(Status, self).save(*args, **kwargs)

@python_2_unicode_compatible
class BookingComOrder(models.Model):
	id=models.BigIntegerField(primary_key=True)
	status=models.CharField(max_length=40,blank=True,null=True)
	remarks=models.CharField(max_length=400,blank=True,null=True)
	guests=models.CharField(max_length=300,blank=True,null=True)
	def __str__(self):
		return str(self.id)

@python_2_unicode_compatible
class LodgifyOrder(models.Model):
	id=models.BigIntegerField(primary_key=True)
	status=models.CharField(max_length=40,blank=True)
	type=models.CharField(max_length=40,blank=True)
	isDeleted=models.BooleanField()
	addedAt=models.DateTimeField()
	updatedAt=models.DateTimeField()

	def __str__(self):
		return str(self.pk)

@python_2_unicode_compatible
class Order(models.Model):
	bookedAt=models.DateTimeField()
	addedAt=models.DateTimeField(auto_now_add=True)
	updatedAt=models.DateTimeField(auto_now=True)
	room=models.ForeignKey(Room,on_delete=models.PROTECT,null=True,blank=True)
	guest=models.ForeignKey(Guest,on_delete=models.PROTECT,null=True,blank=True)
	#status=models.ForeignKey(Status,on_delete=models.PROTECT,null=True,blank=True)
	arrival=models.DateField(null=True,blank=True)
	departure=models.DateField(null=True,blank=True)
	rent=models.FloatField(null=True,blank=True)
	cleaningFee=models.FloatField(null=True,blank=True)
	otherFees=models.FloatField(null=True,blank=True)
	cancellationFee=models.FloatField(null=True,blank=True)
	total=models.FloatField(null=True,blank=True)
	paid=models.FloatField(null=True,blank=True)
	vat=models.FloatField(null=True,blank=True)
	people=models.IntegerField(null=True,blank=True)
	source=models.CharField(max_length=200,blank=True,null=True)
	sourceText=models.CharField(max_length=300,blank=True,null=True)
	bookingComOrder=models.ForeignKey(BookingComOrder,on_delete=models.PROTECT,null=True,blank=True)
	lodgifyOrder=models.OneToOneField(LodgifyOrder,on_delete=models.PROTECT,null=True,blank=True)
	notes=models.CharField(max_length=200,blank=True,null=True)
	addBeds=models.IntegerField(default=0)
	addKidBeds=models.IntegerField(default=0)
	kidChairs=models.IntegerField(default=0)
	mStatus=models.IntegerField(default=0)

	@property
	def nights(self):
		return (self.departure-self.arrival).days

	@property
	def arrivalTime(self):
		if self.bookingComOrder:
			if self.bookingComOrder.remarks:
				rem=self.bookingComOrder.remarks
				a=rem.find('arrival: between')
				if (a>0):
					return datetime.time(int(rem[a+17:a+19]))
		return None

	@property
	def customerRemarks(self):
		rem=None
		if self.bookingComOrder:
			rem=self.bookingComOrder.remarks
			remove_str=["You have a booker that prefers communication by email",
			"You have a booker that would like free parking. (based on availability)",
			"Upper-storey room request: this booker requests upper-storey room(s) - based on availability",
			"Ground-level room request: this booker requests ground-level room(s) - based on availability",
			"I'd like free parking (based upon the property's availability)",
			"  "]
			if rem:
				for r in remove_str:
					rem=rem.replace(r,"")
				at=rem.find('Approximate time of arrival: ')
				if (at>-1):
					rem=rem[:at]+rem[at+54:]
		return rem

	def __str__(self):
		return str(self.lodgifyOrder)+" "+str(self.guest)
	def save(self, *args, **kwargs):
		# print "pk:{}".format(self.pk)

		o=Order.objects.filter(lodgifyOrder=self.lodgifyOrder)

		if Guest.objects.filter(order=self).exists():
			if self.guest.name:
				if self.source=="KTS" or " kts" in self.guest.name.lower() \
						or " (kts" in self.guest.name.lower():
					self.source="KTS"
			if self.guest.email:
				if "kirstin.may" in self.guest.email.lower() \
						or "kaierbaeder-auf-usedom" in self.guest.email.lower():
					self.source="KTS"
		if (self.pk!=None or not o.exists()):
			super(Order, self).save(*args, **kwargs)
		else:
			self.id=Order.objects.get(lodgifyOrder=self.lodgifyOrder).id
			self.addedAt=Order.objects.get(lodgifyOrder=self.lodgifyOrder).addedAt

			super(Order, self).save(*args, **kwargs)

#@python_2_unicode_compatible
class ClosedPeriod(models.Model):
	addedAt=models.DateTimeField()
	updatedAt=models.DateTimeField(auto_now=True)
	arrival=models.DateField(null=True,blank=True)
	departure=models.DateField(null=True,blank=True)
	room=models.ForeignKey(Room,on_delete=models.PROTECT,null=True,blank=True)
	lodgifyId=models.BigIntegerField(null=True,blank=True,unique=True)
	def __unicode__(self):
		return str(self.room)+" ("+str(self.arrival)+"-"+str(self.departure)+")"
	def save(self, *args, **kwargs):
		o=ClosedPeriod.objects.filter(lodgifyId=self.lodgifyId)
		if (self.pk!=None or not o.exists()):
			super(ClosedPeriod, self).save(*args, **kwargs)
		else:
			self.id=ClosedPeriod.objects.get(lodgifyId=self.lodgifyId).id
			super(ClosedPeriod, self).save(*args, **kwargs)
			# o.update(addedAt=self.addedAt,room=self.room,
			# 	arrival=self.arrival,departure=self.departure,
			# 	updatedAt=self.updatedAt)

@python_2_unicode_compatible
class FeeType(models.Model):
	name=models.CharField(max_length=100) #not unique because it fucks up testing
	displayname=models.CharField(max_length=100,blank=True)
	addedAt=models.DateTimeField(auto_now_add=True)
	updatedAt=models.DateTimeField(auto_now=True)
	def __str__(self):
		return str(self.name)
	def save(self, *args, **kwargs):
		o=FeeType.objects.filter(name=self.name)
		if (self.pk!=None or not o.exists()):
			super(FeeType, self).save(*args, **kwargs)

@python_2_unicode_compatible
class FeeRate(models.Model):
	type=models.ForeignKey(FeeType,on_delete=models.PROTECT)
	amount=models.FloatField()
	addedAt=models.DateTimeField(auto_now_add=True)
	validUntil=models.DateTimeField(null=True,blank=True)
	room=models.ForeignKey(Room,on_delete=models.PROTECT,null=True,blank=True)
	channel=models.CharField(max_length=50,null=True,blank=True)
	def __str__(self):
		buf1=" (until "+str(self.validUntil)+")" #if (self.validUntil!=None) else buf1=""
		buf2=" ("+str(self.channel)+")" #if (self.channel!=None) else buf2=""
		return str(self.type)+" "+str(self.room)+buf1+buf2

@python_2_unicode_compatible
class Fee(models.Model):
	type=models.ForeignKey(FeeType,on_delete=models.PROTECT)
	order=models.ForeignKey(Order,on_delete=models.CASCADE)
	comment=models.CharField(max_length=100,blank=True)
	amount=models.FloatField()
	addedAt=models.DateTimeField(auto_now_add=True)
	updatedAt=models.DateTimeField(auto_now=True)
	def __str__(self):
		try:
			return str(self.type)+" "+str(self.order)+" ("+str(self.comment)+")"
		except:
			return str(self.pk)
	class Meta:
		unique_together = ('type', 'order',)

	# THIS WAS COMMENTED OUT BEFORE I RESETTED TEST DB AND THAN
	#.   HAD ERRORS:
	def save(self, *args, **kwargs):
		o=Fee.objects.filter(type=self.type,order=self.order)
		if (self.pk!=None or not o.exists()):
			super(Fee, self).save(*args, **kwargs)
		elif (Fee.objects.get(type=self.type,order=self.order).comment=='imported from booking.com'
			 or Fee.objects.get(type=self.type,order=self.order).comment==fee_comment_missing):
			self.addedAt=Fee.objects.get(type=self.type,order=self.order).addedAt
			self.id=Fee.objects.get(type=self.type,order=self.order).id
			super(Fee, self).save(*args, **kwargs) #not update because needs to call post save signal
			o.update(amount=self.amount,comment=self.comment)

@python_2_unicode_compatible
class TaskLog(models.Model):
	runAt=models.DateTimeField(auto_now=True)
	app=models.CharField(max_length=40)
	name=models.CharField(max_length=200)
	def __str__(self):
		return str(self.runAt.date())+' '+str(self.name)

@python_2_unicode_compatible
class ChangeLog(models.Model):
	changedAt=models.DateTimeField(auto_now=True)
	foreignId=models.BigIntegerField()
	added=models.BooleanField()
	app=models.CharField(max_length=40)
	model=models.CharField(max_length=60)
	field=models.CharField(max_length=60,blank=True,null=True)
	previousValue=models.CharField(max_length=300,blank=True,null=True)
	def __str__(self):
		return str(self.model)+" "+str(self.foreignId)+" at "+str(self.changedAt)