from setuptools import setup
from glob import glob

package_name = 'fitrobot'

setup(
    name=package_name,
    version='0.0.1',
    packages=[package_name],
    data_files=[
        ("share/ament_index/resource_index/packages", [f"resource/{package_name}"]),
        (f"share/{package_name}", ['package.xml']),
        (f"share/{package_name}/launch", glob("launch/*.launch.py")),
        (f"share/{package_name}/common", glob("common/*.py")),
        (f"share/{package_name}/script", glob("script/*.py")),
        (f"share/{package_name}/data", glob("data/*.json")),
        (f"share/{package_name}/maps", glob("maps/*")),
        (f"share/{package_name}/masks", glob("masks/*")),
    ],
    install_requires=['setuptools'],
    zip_safe=True,
    maintainer='ubuntu',
    maintainer_email='jason.kj.ke@fit-foxconn.com',
    description='TODO: Package description',
    license='TODO: License declaration',
    tests_require=['pytest'],
    entry_points={
        'console_scripts': [
            'tf_converter_node = fitrobot.tf_converter_node:main',
            'server_node = fitrobot.server_node:main',
            'bridge_node = fitrobot.bridge_node:main',
            'master_service = fitrobot.master_service:main',
            'save_map_service = fitrobot.save_map_service:main',
            'list_map_service = fitrobot.list_map_service:main',
            'waypoint_follower = fitrobot.waypoint_follower:main',
            'list_station_service = fitrobot.list_station_service:main',
            'nav2_params_loader = fitrobot.nav2_params_loader:main',
        ],
    },
)


from services.note import NoteService
from models.note import OwnNoteSchema
from db import with_session

from flask_restful import Resource, marshal_with
from flask_jwt_extended import jwt_required
from flasgger import swag_from

specs_dict = {
    "tags": ["Note"],

    "operationId": "resolveNote",

    "summary": "Resolve note",

    "parameters": [
        {
            "in": "path",
            "name": "noteId",
            "schema": { "type": "integer" },
            "required": "true",
            "description": "Note to be resolved"
        },
    ],

    "responses": {
        "200": {
            "description": "Resolved note",
            "content": {
                "application/json": {
                    "schema": OwnNoteSchema.schemaSpecRef()
                }
            }
        }
    }
}

class NoteResolveResource(Resource):
    decorators = [jwt_required()]

    @swag_from(specs_dict)
    @with_session
    @marshal_with(OwnNoteSchema.marshaller())
    def put(self, session, noteId):
        notes = NoteService(session)
        return notes.resolve(noteId)


from pyspark import SparkConf, SparkContext
import json, sys, datetime
from xgboost import XGBRegressor
import pandas as pd
import numpy as np
from collections import defaultdict
import pickle
from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating
from operator import add

def extract_features():
    # business_id: (stars, review count)   188593   all
    bsn_feature = sc.textFile(business_data_path).map(lambda x: json.loads(x))\
                        .map(lambda x: (x['business_id'], (x['stars'], x['review_count']))).collectAsMap()
    # business_id: (checkin days, total checkin days)   157075   lack 532
    checkin_feature = sc.textFile(checkin_path).map(lambda x: json.loads(x))\
                        .map(lambda x: (x['business_id'], (len(x['time']), sum(x['time'].values()))))\
                        .collectAsMap()
    # business_id: (food+drink+menu, inside+outside) 32976        lack 128374
    def encode(record):
        food = 0
        env = 0
        for tag in record[1]:
            if tag in ['food','drink', 'menu']:
                food += 1
            if tag in ['inside','outside']:
                env += 1
        return (record[0], (food, env))
    photo_feature = sc.textFile(photo_path).map(lambda x: json.loads(x))\
                        .map(lambda x: ((x['business_id']), x['label'])).groupByKey().mapValues(list)\
                        .map(lambda x: encode(x)).collectAsMap()
    # (user_id, business_id): tip times     999945       lack 409824
    tip_feature = sc.textFile(tip_path).map(lambda x: json.loads(x))\
                        .map(lambda x: ((x['user_id'], x['business_id']), 1))\
                        .reduceByKey(add).collectAsMap()
    # user_id: (number of reviews, average star, yelp since, fans, votes sent by users,
    #           elite years, nearest elite  year, compliments received by users)
    # 1518169       all
    userRDD = sc.textFile(user_data_path).map(lambda x: json.loads(x))
    user_feature = userRDD.map(lambda x: (x['user_id'],\
                                            (x['review_count'], x['average_stars'],\
                                            2019 - int(x['yelping_since'][:4]), x['fans'], x['useful']+x['funny']+x['cool'],\
                                            0 if x["elite"] == 'None' else (len(x["elite"])+2) // 6,\
                                            0 if x["elite"] == 'None' else int(x["elite"][-4:]) - 2004,\
                                            x['compliment_hot']+x['compliment_more']+x['compliment_plain']\
                                            +x['compliment_cool']+x['compliment_funny']+x['compliment_writer'])))\
                            .map(lambda x: (x[0], (x[1][0],x[1][1],x[1][2],x[1][3],x[1][4],x[1][5],x[1][6],\
                                                   0 if x[1][0] == 0 else x[1][7] / x[1][0])))\
                            .collectAsMap()
    return bsn_feature, checkin_feature, photo_feature, tip_feature, user_feature

def train_xgb(X_train, y_train):
    model = XGBRegressor(n_estimators = 500, learning_rate = 0.1, max_depth = 5, eval_metric='rmse')\
                .fit(X_train, y_train)
    return model

if __name__ == '__main__':
    start_time = datetime.datetime.now()

    conf = SparkConf().setAppName("competition").setMaster("local[*]")
    sc = SparkContext(conf = conf)
    sc.setLogLevel("WARN")
    conf.set("spark.executor.memory", "32g")
    conf.set("spark.driver.memory", "32g")

    folder_file = sys.argv[1]
    train_file = folder_file + 'yelp_train.csv'
    test_file = sys.argv[2]
    output_file_path = sys.argv[3]
    business_data_path = folder_file + 'business.json'
    checkin_path = folder_file + 'checkin.json'
    tip_path = folder_file + 'tip.json'
    user_data_path = folder_file + 'user.json'
    photo_path = folder_file + 'photo.json'
    xgb_model_file = './xgb.pickle'

    trainRDD = sc.textFile(train_file).map(lambda x: x.split(','))
    firstline = trainRDD.first()  # user, bsn, stars
    train_set = trainRDD.filter(lambda x: x != firstline).map(lambda x: (x[0], x[1], float(x[2])))

    # 142044
    testRDD = sc.textFile(test_file).map(lambda x: x.split(','))
    first = testRDD.first()
    test_set = testRDD.filter(lambda x: x != first).map(lambda x: (x[0], x[1], float(x[2])))

    bsn_feature, checkin_feature, photo_feature, tip_feature, user_feature = extract_features()

    train_rdd = train_set.map(lambda x: ((x[0], x[1]),user_feature[x[0]],bsn_feature[x[1]],\
                                        checkin_feature[x[1]] if x[1] in checkin_feature else (0, 0),\
                                        photo_feature[x[1]] if x[1] in photo_feature else (0, 0),\
                                        tip_feature[(x[0], x[1])] if (x[0], x[1]) in tip_feature else 0,\
                                        x[2]))\
                    .map(lambda x: (x[0], x[1][0], x[1][1], x[1][2], x[1][3], x[1][4], x[1][5], x[1][6], x[1][7],\
                                x[2][0], x[2][1], x[3][0], x[3][1], x[4][0], x[4][1], x[5], x[6]))
    train_data = train_rdd.collect()
    train_df = pd.DataFrame(train_data,
                        columns =
                        ['user_bsn','u_reviews', 'u_star', 'u_years', 'u_fans', 'u_votes','u_elite','u_elite_1','u_compliment',\
                        'b_stars','b_reviews', 'b_checkin','b_checkin_total','b_photo_food', 'b_photo_env', 'tip','rating'])\
                .set_index('user_bsn')
    X_train = train_df.iloc[:,:-1]
    y_train = train_df.iloc[:,-1]

    test_rdd = test_set.map(lambda x: ((x[0], x[1]),user_feature[x[0]],bsn_feature[x[1]],\
                                        checkin_feature[x[1]] if x[1] in checkin_feature else (0, 0),\
                                        photo_feature[x[1]] if x[1] in photo_feature else (0, 0),\
                                        tip_feature[(x[0], x[1])] if (x[0], x[1]) in tip_feature else 0,\
                                        x[2]))\
                    .map(lambda x: (x[0], x[1][0], x[1][1], x[1][2], x[1][3], x[1][4], x[1][5], x[1][6], x[1][7],\
                                x[2][0], x[2][1], x[3][0], x[3][1], x[4][0], x[4][1], x[5], x[6]))
    test_data = test_rdd.collect()
    test_df = pd.DataFrame(test_data,
                        columns =
                        ['user_bsn','u_reviews', 'u_star', 'u_years', 'u_fans', 'u_votes','u_elite','u_elite_1','u_compliment',\
                        'b_stars','b_reviews', 'b_checkin','b_checkin_total','b_photo_food', 'b_photo_env', 'tip', 'rating'])\
                .set_index('user_bsn')
    X_test = test_df.iloc[:,:-1]
    y_test = test_df.iloc[:,-1]

    # train model
    xgb = train_xgb(X_train, y_train)
    # # load model
    # with open(xgb_model_file, 'rb') as file:
    #     xgb = pickle.load(file)

    user_bsn_id = test_df.index
    xgb_predict = xgb.predict(X_test)

    # write into file
    file = open(output_file_path, 'w')
    file.write('user_id, business_id, prediction')
    for i in range(len(xgb_predict)):
        file.write('\n')
        key = user_bsn_id[i]
        file.write(str(key[0]) + ',' + str(key[1]) + ',' + str(xgb_predict[i]))
    file.close()

    end_time = datetime.datetime.now()
    print((end_time - start_time).seconds)


import base64

from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.db.models import Q
from rest_framework import status
from rest_framework.authentication import TokenAuthentication
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response

from .serializers import *

# Create your views here.

#### Appointments

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getAppointments(request, professor_id):
    professor = User.objects.get(id=professor_id)
    appointments = Appointment.objects.filter(student=request.user).filter(professor=professor)
    sr = AppointmentSerializer(appointments, many=True)
    return Response(sr.data)

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def getAllUserAppointments(request):
    appointments = Appointment.objects.filter(student=request.user).order_by('-time')
    sr = AppointmentSerializer(appointments, many=True)
    return Response(sr.data)

@api_view(['GET'])
def getAppointment(request, id):
    appointment = Appointment.objects.get(id=id)
    sr = AppointmentSerializer(appointment, many=False)
    return Response(sr.data)

@api_view(['GET'])
def getAppointmentForUser(request):
    appointments = Appointment.objects.filter(student=request.user).order_by('-id')
    sr = AppointmentSerializer(appointments, many=True)
    return Response(sr.data)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def addAppointment(request):
    appointment = Appointment.objects.create(
        title=request.data['title'],
        description=request.data['description'],
        professor_id=request.data['professor_id'],
        time=0,
        student=request.user
    )
    """
    context = {
        'subject': appointment.title,
        'content': f
    #                روز بخیر  {appointment.professor.name} \n
     #               دانشجو {appointment.student.name} درخواست قرار ملاقات  دارد.
      #              لطفا در اپلیکیشن برسی کنید.

    }
    message = get_template('layouts/mail/email_layout.html').render(context)
    msg = EmailMessage(
        appointment.title,
        message,
        'notification@gethermes.ir',
        [appointment.professor.email],
    )
    msg.content_subtype = "html"  # Main content is now text/html
    msg.send()"""

    return Response(AppointmentSerializer(appointment).data, status=status.HTTP_200_OK)

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def deleteAppointment(request, id):
    appointment = Appointment.objects.get(id=id)
    appointment.delete()
    return Response(status=status.HTTP_200_OK)

#### professor profile

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def updateProfile(request):
    user = request.user
    user.stud_code = request.data['stud_code']
    user.id_code = request.data['id_code']
    if request.data['email'] != '' and len(request.data['email']) > 0:
        user.email = request.data['email']
    user.bio = request.data['bio'][:200]
    user.phone = request.data['phone']
    user.name = request.data['name']
    user.family = request.data['family']
    user.save()
    return Response(UserSerializerSelf(user, many=False).data, status=status.HTTP_200_OK)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def updatePassword(request):
    success = request.user.check_password(request.data['previous_pass'])
    if not success:
        return Response(status=status.HTTP_403_FORBIDDEN)

    request.user.set_password(request.data['new_pass'])
    return Response(status=status.HTTP_202_ACCEPTED)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def updateAvatar(request):
    cover_data = request.data['avatar']

    format, imgstr = cover_data.split(';base64,')
    ext = format.split('/')[-1]
    data = ContentFile(base64.b64decode(imgstr))

    file_name = f'{time.time()}' + ext
    request.user.avatar.save(file_name, data, save=True)  # image is User's model field

    return Response(UserSerializerSelf(request.user, many=False).data, status=status.HTTP_200_OK)

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def updateAvatarFile(request):
    print(request.FILES)

    myfile = request.data.get('image')

    fs = FileSystemStorage()
    filename = fs.save(myfile.name, myfile)
    uploaded_file_url = fs.url(filename)
    request.user.avatar = uploaded_file_url

    return Response(UserSerializerSelf(request.user, many=False).data, status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def getProfessorProfile(request, id):
    professor = User.objects.get(id=id)
    if professor.mode != User.MODES[2][0]:
        return Response(status=status.HTTP_409_CONFLICT)
    appointments = Appointment.objects.filter(student=request.user).filter(professor=professor).order_by('-id')
    appointments_sr = AppointmentSerializer(appointments, many=True)
    professor_sr = UserSerializerProfile(professor, many=False)
    return Response({
        'professor': professor_sr.data,
        'appointments': appointments_sr.data
    })

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def getProfessors(request):
    professors = User.objects.filter(mode=2)
    return Response(
        UserSerializerListItem(professors, many=True).data
    )

### Main page

@api_view(['GET'])
def mainPage(request):
    main_page = Setting.objects.get(key=settings.MAIN_PAGE_PROFS_DISPLAY_LOCATION)
    location = Location.objects.get(id=main_page.value)
    profs = User.objects.filter(mode=User.MODES[2][0])
    sr = UserSerializerListItem(profs, many=True)
    return Response(
        {
            'profs': sr.data
        }
    )

#### posts

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def getPostDetails(request, id):
    post = Post.objects.get(id=id)
    return Response(
        PostSerializerDetailed(post, many=False).data, status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def allPosts(request):
    post = Post.objects.order_by('-id')
    return Response(
        PostSerializerDetailed(post, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def allPostsFilterByGroup(request):
    groups = request.POST['groups'].split(',')
    expertise = request.POST['expertise'].split(',')
    professors = request.POST['professors'].split(',')
    print(request.POST)
    print(groups)
    print(expertise)
    if groups == ['']:
        groups = []

    if expertise == ['']:
        print(expertise)
        expertise = []

    if professors == ['']:
        professors = []

    print((professors))
    posts = Post.objects.filter(Q(user__group__in=groups) | Q(user__expertise__in=expertise) | Q(user__in=professors))
    print(len(posts))
    # posts = Post.objects.all()

    return Response(
        PostSerializerDetailed(posts, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def allProfessorsFiltered(request):
    print(request.data['expertise'])
    groups = list(map(lambda x: x['id'], request.data['groups']))
    expertise = list(map(lambda x: x['id'], request.data['expertise']))
    locations = list(map(lambda x: x['id'], request.data['locations']))
    print(request.POST)
    print(groups)
    print(locations)
    if groups == ['']:
        groups = []

    if expertise == ['']:
        print(expertise)
        expertise = []

    if locations == ['']:
        locations = []

    profs = User.objects.filter(Q(group__in=groups) | Q(expertise__in=expertise) | Q(location__in=locations)).filter(
        mode=2)
    posts = Post.objects.all()

    return Response(
        UserSerializerListItem(profs, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def allAnnouncesFiltered(request):
    skills = request.POST['skills'].split(',')
    expertise = request.POST['expertise'].split(',')
    professors = request.POST['professors'].split(',')
    print(request.POST)
    if skills == ['']:
        skills = []

    if expertise == ['']:
        expertise = []

    if professors == ['']:
        professors = []

    announces = Announce.objects.filter(
        Q(user__expertise__in=expertise) | Q(skills__in=skills) | Q(user__in=professors))
    # posts = Post.objects.all()

    return Response(
        AnnounceSerializerList(announces, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def allAnnounces(request):
    announce = Announce.objects.order_by('-id')
    return Response(
        AnnounceSerializerList(announce, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getAnnounceDetails(request, id):
    announce = Announce.objects.get(id=id)
    return Response(
        AnnounceSerializerList(announce, many=False).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def addAnnounceRequest(request):
    announce, created = AnnounceRequest.objects.get_or_create(
        student=request.user,
        announce_id=request.data['announce_id'],
        description=request.data['description']
    )
    if created:
        return Response(
            status=status.HTTP_201_CREATED
        )
    else:
        return Response(
            status=status.HTTP_200_OK
        )

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def getMyAnnounceRequests(request):
    reqs = AnnounceRequest.objects.filter(student=request.user)

    return Response(AnnounceRequestSerializer(reqs, many=True).data,
                    status=status.HTTP_200_OK
                    )

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def deleteAnnounceRequest(request, id):
    announce = AnnounceRequest.objects.get(id=id)
    announce.delete()
    return Response(
        status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getPostFilters(request):
    groups = ProfessorGroup.objects.all()
    expertise = Expertise.objects.all()
    professors = User.objects.filter(mode=2)
    return Response(
        {
            'groups':
                GroupSerializer(groups, many=True).data,
            'expertise': ExpertiseSerializer(expertise, many=True).data,
            'professors': UserSerializerListItem(professors, many=True).data
        }
    )

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def getProfessorFilters(request):
    groups = ProfessorGroup.objects.all()
    expertise = Expertise.objects.all()
    locations = Location.objects.all()
    return Response(
        {
            'groups':
                GroupSerializer(groups, many=True).data,
            'expertise': ExpertiseSerializer(expertise, many=True).data,
            'locations': LocationSerializer(locations, many=True).data
        }
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getAnnounceFilters(request):
    skills = Skill.objects.all()
    expertise = Expertise.objects.all()
    professors = User.objects.filter(mode=2)
    return Response(
        {
            'skills':
                SkillSerializer(skills, many=True).data,
            'expertise': ExpertiseSerializer(expertise, many=True).data,
            'professors': UserSerializerListItem(professors, many=True).data
        }
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def filterPostsByText(request):
    search_term = request.POST['search_term']
    posts = Post.objects.filter(Q(title__contains=search_term) | Q(body__contains=search_term))
    return Response(
        PostSerializerDetailed(posts, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def filterProfessorsByText(request):
    search_term = request.POST['search_term']
    profs = User.objects.filter(name__contains=search_term).filter(mode=2)
    return Response(
        UserSerializerListItem(profs, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
def fullTextSearchProfessor(request):
    search_term = request.data['search_term']
    profs = User.objects.filter(
        Q(name__contains=search_term) | Q(family__contains=search_term) | Q(expertise__name__contains=search_term) | Q(
            location__name__contains=search_term)).filter(mode=2)
    return Response(
        UserSerializerListItem(profs, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def filterAnnouncesByText(request):
    search_term = request.POST['search_term']
    announces = Announce.objects.filter(Q(title__contains=search_term) | Q(body__contains=search_term))
    return Response(
        AnnounceSerializerList(announces, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getCircles(request):
    circles = Circle.objects.all()
    return Response(
        CircleSerializer(circles, many=True).data, status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def addCircleToProfile(request):
    print(request.POST)
    user = User.objects.get(id=request.user.id)
    circle = Circle.objects.get(id=request.POST['circle_id'])
    if circle not in user.circles.all():
        circle.user_set.add(user)
    else:
        pass
    return Response(
        status=status.HTTP_200_OK
    )

@api_view(['POST'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def deleteMyCircle(request):
    print(request.POST)
    user = User.objects.get(id=request.user.id)
    circle = Circle.objects.get(id=request.POST['circle_id'])
    user.circles.remove(circle)
    return Response(
        status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def getMyCircles(request):
    user = User.objects.get(id=request.user.id)
    return Response(
        CircleSerializer(
            user.circles.all(), many=True
        ).data, status=status.HTTP_200_OK
    )

@api_view(['GET'])
@authentication_classes([TokenAuthentication])
@permission_classes([IsAuthenticated])
def userData(request):
    user = User.objects.get(id=request.user.id)
    sr = UserSerializerSelf(user)
    return Response(
        sr.data, status=status.HTTP_200_OK
    )


from typing import Callable

from helper import choose_puzzle_input, print_solution

year = 2023
day = 10

puzzle_input = choose_puzzle_input(
    y=year,
    d=day,
    # sample_input_path=f"aoc_{year}_{day:02d}_input_sample.txt",
)

start = None

possible_tiles = ["7", "L", ".", "S", "J", "F", "|", "-"]

visited_1 = []
visited_2 = []

def start_connections(maze: list[str], startpos: list[int]) -> list[list[int]]:
    up = startpos[0] - 1
    down = startpos[0] + 1
    left = startpos[1] - 1
    right = startpos[1] + 1
    x, y = startpos[1], startpos[0]

    tile_up, tile_down, tile_left, tile_right = None, None, None, None

    if up >= 0 and maze[up][x] in ["7", "F", "|"]:
        tile_up = {"coord": [up, x], "tile": maze[up][x]}
    if down < len(maze) and maze[down][x] in ["L", "J", "|"]:
        tile_down = {"coord": [down, x], "tile": maze[down][x]}
    if left >= 0 and maze[y][left] in ["L", "F", "-"]:
        tile_left = {"coord": [y, left], "tile": maze[y][left]}
    if right < len(maze[0]) and maze[y][right] in ["7", "J", "-"]:
        tile_right = {"coord": [y, right], "tile": maze[y][right]}

    next_coordinates = [
        [tile["coord"][0], tile["coord"][1]]
        for tile in [tile_up, tile_down, tile_left, tile_right]
        if tile
    ]

    return next_coordinates

def next_connection(
    maze: list[str], startpos: list[int], prevpos: list[list[int]]
) -> list[int]:
    up = startpos[0] - 1
    down = startpos[0] + 1
    left = startpos[1] - 1
    right = startpos[1] + 1
    x, y = startpos[1], startpos[0]

    tile_up, tile_down, tile_left, tile_right = None, None, None, None

    if (
        up >= 0
        and maze[up][x] in ["7", "F", "|"]
        and maze[y][x] in ["L", "J", "|"]
    ):
        tile_up = [up, x]
    if (
        down < len(maze)
        and maze[down][x] in ["L", "J", "|"]
        and maze[y][x] in ["7", "F", "|"]
    ):
        tile_down = [down, x]
    if (
        left >= 0
        and maze[y][left] in ["L", "F", "-"]
        and maze[y][x] in ["7", "J", "-"]
    ):
        tile_left = [y, left]
    if (
        right < len(maze[0])
        and maze[y][right] in ["7", "J", "-"]
        and maze[y][x] in ["L", "F", "-"]
    ):
        tile_right = [y, right]

    next_coordinates = [
        [tile[0], tile[1]]
        for tile in [tile_up, tile_down, tile_left, tile_right]
        if tile and [tile[0], tile[1]] not in prevpos
    ][0]

    return next_coordinates

for y, line in enumerate(puzzle_input):
    start_index = line.find("S")
    if start_index > -1:
        start = [y, start_index]
        break

start_conns = start_connections(maze=puzzle_input, startpos=start)
visited_1.append(start_conns[0])
visited_2.append(start_conns[1])

while visited_1[-1] != visited_2[-1]:
    visited_1.append(
        next_connection(
            maze=puzzle_input, startpos=visited_1[-1], prevpos=visited_1
        )
    )
    visited_2.append(
        next_connection(
            maze=puzzle_input, startpos=visited_2[-1], prevpos=visited_2
        )
    )

solution = len(visited_1) if len(visited_1) == len(visited_2) else None
print_solution(solution=solution, y=year, d=day, part=1)


# plugin_pose_DeepLabCut.py

from qtpy.QtCore import Qt, QPointF
from qtpy.QtGui import QColor, QPainter, QPen, QPolygonF
from qtpy.QtWidgets import QMessageBox, QWidget
from pose.pose import PoseBase
from utils import get_colormap
import csv
import numpy as np
import h5py
from os.path import splitext

"""
Bento plugin that provides support for MARS-style pose files
represented MatLab .mat files.

Implements a class derived from PoseBase, which is an abstract class.
"""

class PoseDLCBase(PoseBase):
    """
    Base class for DeepLabCut pose support

    Includes implementation common to generic and mouse-specific DLC files
    """

    def __init__(self):
        super().__init__()

    def getFileSearchPattern(self) -> str:
        return "*.h5 *.csv"

    def _validateFileH5(self, parent_widget, file_path: str) -> bool:
        h5 = h5py.File(file_path, 'r')
        default_group = h5[list(h5.keys())[0]]
        if 'table' not in default_group.keys():
            QMessageBox.warning(parent_widget, "Add Pose ...", "No pose data found in pose file")
            return False
        return True

    def _validateFileCSV(self, parent_widget, file_path: str) -> bool:
        result = True
        with open(file_path, 'r') as csvFile:
            reader = csv.reader(csvFile)
            header1 = next(reader)
            if (not isinstance(header1[1], str) or
                header1[1] != 'bodyparts' or
                (len(header1) - 2) % 3 != 0):   # "", "bodyparts", 3 x <bodypart_name>, ...
                result = False
            csvFile.seek(0)
        return result

    def validateFile(self, parent_widget: QWidget, file_path: str) -> bool:
        """
        Default implementation does no checking,
        but we can do better than that
        """
        _, ext = splitext(file_path)
        ext = ext.lower()
        if ext == '.h5':
            return self._validateFileH5(parent_widget, file_path)
        elif ext == '.csv':
            return self._validateFileCSV(parent_widget, file_path)
        else:
            QMessageBox.warning(parent_widget, "Extension not supported",
                f"The file extension {ext} is not supported.")

    def _loadPoses_h5(self, parent_widget: QWidget, path: str):
        raise NotImplementedError("Please implement this in your derived class")

    def _loadPoses_csv(self, parent_widget: QWidget, path: str):
        raise NotImplementedError("Please implement this in your derived class")

    def loadPoses(self, parent_widget: QWidget, path: str):
        _, self.file_extension = splitext(path)
        self.file_extension = self.file_extension.lower()
        if self.file_extension == '.h5':
            self._loadPoses_h5(parent_widget, path)
        elif self.file_extension == '.csv':
            self._loadPoses_csv(parent_widget, path)
        else:
            QMessageBox.warning(parent_widget, "Extension not supported",
                f"The file extension {self.file_extension} is not supported.")

class PoseDLC_generic(PoseDLCBase):

    def __init__(self):
        super().__init__()
        self.file_extension = None
        self.frame_points = []
        self.num_frames = 0
        # Construct colors based on the number of body parts and the color map
        self.colormap_data = get_colormap('turbo')
        self.pose_colors = None

    def generatePoseColors(self, n):
        if n < 1 or n > 256:
            raise ValueError("generatePoseColors: n must be between 1 and 256 inclusive")
        if n == 1:
            self.pose_colors = [QColor(self.colormap_data[127])]
        else:
            step = 255. / (n - 1)
            self.pose_colors = [QColor(self.colormap_data[round(ix * step)]) for ix in range(n)]

    def drawPoses(self, painter: QPainter, frame_ix: int):
        frame_ix = min(frame_ix, self.num_frames)
        painter.setPen(Qt.NoPen)
        if len(self.frame_points[frame_ix]) > len(self.pose_colors):
            raise ValueError("drawPoses: number of points t draw exceeds number of body parts")
        for ix, point in enumerate(self.frame_points[frame_ix]):
            painter.setBrush(self.pose_colors[ix])
            painter.drawEllipse(point, 5.0, 5.0)

    def getFileSearchDescription(self) -> str:
        return "DeepLabCut generic pose files"

    def getFileFormat(self) -> str:
        return "DeepLabCut_generic"

    def _loadPoses_h5(self, parent_widget, path: str):
        # TODO: need to change this from the mouse form to a more generic
        # version that just stores the points.
        h5 = h5py.File(path, 'r')
        default_group = h5[list(h5.keys())[0]]
        table = default_group['table']
        self.num_frames = len(table)
        isColorsGenerated = False
        for frame_ix in range(self.num_frames):
            # DLC stores each frame as (frame_idx, (pose_array))
            # we don't care about the frame_idx
            this_frame_data = table[frame_ix][1]
            this_frame_points = []
            vals_per_pt = 3
            num_points = int(len(this_frame_data) / vals_per_pt)
            if not isColorsGenerated:
                self.generatePoseColors(num_points)
            for point_ix in range(num_points):
                # each pose point has x, y and confidence
                # we don't care about confidence
                pt_x_ix = (vals_per_pt * point_ix) + 0
                pt_y_ix = (vals_per_pt * point_ix) + 1
                this_frame_points.append(QPointF(this_frame_data[pt_x_ix], this_frame_data[pt_y_ix]))
            self.frame_points.append(this_frame_points)

    def _loadPoses_csv(self, parent_widget, path: str):
        with open(path, 'r') as csvFile:
            vals_per_pt = 3 # x, y, confidence.  We don't care about confidence.
            reader = csv.reader(csvFile)
            header1 = next(reader)
            num_pts = int((len(header1) - 2) / vals_per_pt)
            self.generatePoseColors(num_pts)
            _ = next(reader)    # skip over second row of header
            while True:
                try:
                    row = next(reader)
                except StopIteration:
                    break
                this_frame_points = []
                for point_ix in range(num_pts):
                    pt_x_ix = (vals_per_pt * point_ix) + 2
                    pt_y_ix = (vals_per_pt * point_ix) + 3
                    this_frame_points.append(QPointF(float(row[pt_x_ix]), float(row[pt_y_ix])))
                self.frame_points.append(this_frame_points)
            self.num_frames = len(self.frame_points)

class PoseDLC_mouse(PoseDLCBase):

    def __init__(self):
        super().__init__()
        self.pose_polys = []
        self.num_frames = 0
        self.pose_colors = [Qt.blue, Qt.green]

    def drawPoses(self, painter: QPainter, frame_ix: int):
        frame_ix = min(frame_ix, self.num_frames)
        try:
            for mouse_ix in range(len(self.pose_polys[frame_ix])):
                painter.setPen(QPen(self.pose_colors[mouse_ix], 2.0))
                painter.drawPolyline(self.pose_polys[frame_ix][mouse_ix])
                painter.setBrush(Qt.red)
                painter.setPen(Qt.NoPen)
                painter.drawEllipse(self.pose_polys[frame_ix][mouse_ix].first(), 5.0, 5.0) # red dot on nose
        except IndexError as e:
            # number of video frames is greater than number of pose frames.  Oh well!
            pass

    def getFileSearchDescription(self) -> str:
        return "DeepLabCut mouse pose files"

    def getFileFormat(self) -> str:
        return "DeepLabCut_mouse"

    def _loadPoses_h5(self, parent_widget: QWidget, path: str):
        h5 = h5py.File(path, 'r')
        default_group = h5[list(h5.keys())[0]]
        table = default_group['table']
        self.pose_polys = []
        self.num_frames = len(table)
        for frame_ix in range(self.num_frames):
            # DLC stores each frame as (frame_idx, (pose_array))
            # we don't care about the frame_idx
            this_frame_data = table[frame_ix][1]
            frame_polys = []
            vals_per_pt = 3
            pts_per_mouse = 7
            vals_per_mouse = vals_per_pt * pts_per_mouse
            num_mice = int(len(this_frame_data) / vals_per_mouse)
            for mouse_ix in range(num_mice):
                # each pose point has x, y and confidence
                # we don't care about confidence
                poly = QPolygonF()
                pt_x_ix = (vals_per_mouse * mouse_ix) + 0
                pt_y_ix = (vals_per_mouse * mouse_ix) + 1
                nose = QPointF(this_frame_data[pt_x_ix], this_frame_data[pt_y_ix])
                poly.append(nose)
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (1 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (1 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (3 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (3 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (4 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (4 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (6 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (6 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (5 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (5 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (3 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (3 * vals_per_pt)]))
                poly.append(QPointF(
                    this_frame_data[pt_x_ix + (2 * vals_per_pt)],
                    this_frame_data[pt_y_ix + (2 * vals_per_pt)]))
                poly.append(nose)
                frame_polys.append(poly)
            self.pose_polys.append(frame_polys)

    def _loadPoses_csv(self, parent_widget: QWidget, path: str):
        with open(path, 'r') as csvFile:
            vals_per_pt = 3 # x, y, confidence.  We don't care about confidence.
            vals_per_mouse = 7
            reader = csv.reader(csvFile)
            header1 = next(reader)
            num_pts = int((len(header1) - 2) / vals_per_pt)
            if num_pts % vals_per_mouse != 0:
                QMessageBox.warning(parent_widget, "Improper number of points for mice",
                    f"Expected a multiple of 7 points, got {num_pts}")
                return
            _ = next(reader)    # skip over second row of header
            while True:
                try:
                    row = next(reader)
                except StopIteration:
                    break
                num_mice = int(num_pts / vals_per_mouse)
                frame_polys = []
                for mouse_ix in range(num_mice):
                    # each pose point has x, y and confidence
                    # we don't care about confidence
                    poly = QPolygonF()
                    pt_x_ix = (vals_per_mouse * mouse_ix) + 2
                    pt_y_ix = (vals_per_mouse * mouse_ix) + 3
                    nose = QPointF(row[pt_x_ix], row[pt_y_ix])
                    poly.append(nose)
                    poly.append(QPointF(
                        row[pt_x_ix + (1 * vals_per_pt)],
                        row[pt_y_ix + (1 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (3 * vals_per_pt)],
                        row[pt_y_ix + (3 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (4 * vals_per_pt)],
                        row[pt_y_ix + (4 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (6 * vals_per_pt)],
                        row[pt_y_ix + (6 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (5 * vals_per_pt)],
                        row[pt_y_ix + (5 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (3 * vals_per_pt)],
                        row[pt_y_ix + (3 * vals_per_pt)]))
                    poly.append(QPointF(
                        row[pt_x_ix + (2 * vals_per_pt)],
                        row[pt_y_ix + (2 * vals_per_pt)]))
                    poly.append(nose)
                    frame_polys.append(poly)
                self.pose_polys.append(frame_polys)
            self.num_frames = len(self.frame_points)

def register(registry):
    # construct and register the generic plugin
    pose_plugin = PoseDLC_generic()
    registry.register(pose_plugin.getFileFormat(), pose_plugin)

    # construct and register the MARS-style mouse-specific plugin
    pose_plugin = PoseDLC_mouse()
    registry.register(pose_plugin.getFileFormat(), pose_plugin)


import pandas as pd
import numpy as np
import scipy.stats as ss
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv("./data/HR.csv")
dp_indices = df.groupby(by="department").indices

sales_values = df["left"].iloc[dp_indices["sales"]].values
print(sales_values)

technical_values = df["left"].iloc[dp_indices["technical"]].values
print(technical_values)

print(ss.ttest_ind(sales_values, technical_values))

dp_keys = list(dp_indices.keys())
dp_t_mat = np.zeros((len(dp_keys), len(dp_keys)))
for i in range(len(dp_keys)):
    for j in range(len(dp_keys)):
        p_value = ss.ttest_ind(df["left"].iloc[dp_indices[dp_keys[i]]].values, \
                               df["left"].iloc[dp_indices[dp_keys[j]]].values)[1]
        if p_value < 0.05:
            dp_t_mat[i][j] = -1
        else:
            dp_t_mat[i][j] = p_value
sns.heatmap(dp_t_mat, xticklabels=dp_keys, yticklabels=dp_keys)
plt.show()

# 判断数据之间的关系

piv_tb = pd.pivot_table(df, values="left", index=["promotion_last_5years", "salary"], columns=["Work_accident"],
                        aggfunc=np.mean)

print(piv_tb)

sns.heatmap(piv_tb, vmax=1, vmin=0, cmap=sns.color_palette("Reds", n_colors=12))
plt.show()

# 分组分析


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   splice_main.py
@Author  :   Liu Kaiyu
'''

import os
from PyQt5 import QtWidgets,QtCore,QtGui
from splice import Ui_Splice
from three_dialogs import HelpMenu,PreSet
from generators import FFMPEGGenerator,OverlayCMDGenerator,SpliceCMDGenerator
from splice_thread import SpliceThread
from myfilebrowser_main import MyFileDialog

# This will be shown on UI
date_modified = '30/08/2019'

# Hard code some commonly used image extensions
still_img_extensions = ['tif','tiff','png','jpeg','gif','exr','tga','jpg','jif']

################################################################################################################################################
# Customized Erros
class MyError(Exception):
    def __init__(self,message):
        self.message = message

################################################################################################################################################
# Define a global method, this will be used in FileList class below
def split_folder_file(file_path):
    folder,file_name_with_extension = os.path.split(file_path)
    file_name,extension = os.path.splitext(file_name_with_extension)

    return folder,file_name,extension

################################################################################################################################################
################################################################################################################################################
################################################################################################################################################
class FileList(QtWidgets.QTableWidget):

    refresh_signal = QtCore.pyqtSignal()

    def __init__(self,parent):
        super(FileList,self).__init__(parent)
        self.setAcceptDrops(True)

        # Some Settings, copy from UI codes
        # Default has one row, and two columns ("file name" and "duration" )
        self.setGeometry(QtCore.QRect(10, 50, 431, 141))
        self.setColumnCount(2)
        self.setRowCount(1)
        self.resizeRowsToContents()
        self.setHorizontalHeaderLabels(['File Name','Duration'])
        self.setColumnWidth(0,300)
        self.verticalHeader().setDefaultSectionSize(15)
        self.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
        self.setGridStyle(QtCore.Qt.NoPen)
        self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)

    ##############################################################################################################
    def dragMoveEvent(self, event):
        if event.mimeData().hasUrls:
            event.setDropAction(QtCore.Qt.CopyAction)
            event.accept()
        else:
            event.ignore()

    ##############################################################################################################
    def dragEnterEvent(self, e):
        m = e.mimeData()
        # File path is considered as URL
        if m.hasUrls():
            e.accept()
        else:
            e.ignore()

    ##############################################################################################################
    def dropEvent(self, e):
        m = e.mimeData()
        if m.hasUrls():
            for file in m.urls():
                file_path = file.toLocalFile()

                file_name = split_folder_file(file_path)[1]
                extension = split_folder_file(file_path)[2]

                self.parent().file_list_dd[file_name + extension] = {'file_path':'%s'%file_path}
                self.parent().get_file_identity(file_path,file_name + extension)
                self.parent().check_valid_image_sequence(file_name + extension)

            self.refresh_signal.emit()
            print('refresh signal emitted')

################################################################################################################################################
################################################################################################################################################
################################################################################################################################################

class SpliceTool(QtWidgets.QMainWindow,Ui_Splice):
    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        self.setupUi(self)

        # Re-write drop button
        self.file_list = FileList(self)
        self.file_list.refresh_signal.connect(self.refresh_file_list)

        # Update date label
        global date_modified
        _translate = QtCore.QCoreApplication.translate
        self.update_label.setText(_translate("MainWindow", "Update: %s"%date_modified))
        self.setWindowIcon(QtGui.QIcon(r'.\resources\milo.png'))

        # Connections
        self.input_browse.clicked.connect(lambda: self.browse_path(inp = True))
        self.output_browse.clicked.connect(lambda: self.browse_path(out = True))
        self.clear_btn.clicked.connect(self.clear_file_list_box)
        self.up_btn.clicked.connect(lambda: self.move_file('up'))
        self.down_btn.clicked.connect(lambda: self.move_file('down'))
        self.close_btn.clicked.connect(lambda: self.close())
        self.delete_btn.clicked.connect(self.delete_item)
        self.splice_btn.clicked.connect(self.splice_btn_function)
        self.overlay_checkbox.stateChanged.connect(self.use_overlay)
        self.file_list.itemClicked.connect(self.check_is_still_img)
        self.ok_btn.clicked.connect(self.ok_btn_function)
        self.clear_all_btn.clicked.connect(self.clear_all)

        # Store files and cmds
        self.file_list_dd = {}
        self.cmd_list = []

        # Initialize tool button menu and setting
        menu = QtWidgets.QMenu(self)
        self.help_Act = QtWidgets.QAction('Help',self)
        self.pre_setting_Act = QtWidgets.QAction('Pre Sets',self)

        menu.addAction(self.help_Act)
        menu.addAction(self.pre_setting_Act)
        self.toolButton.setMenu(menu)

        # Connect dialogs to mainwindows
        self.help_menu_dialog = HelpMenu()
        self.help_Act.triggered.connect(lambda: self.help_menu_dialog.show())

        self.preset_dialog = PreSet()
        self.pre_setting_Act.triggered.connect(lambda: self.preset_dialog.show())

        self.PRESETS = self.preset_dialog.read_preset_book()
        self.refresh_preset_dropdown(self.PRESETS)
        self.preset_dialog.refresh_signal.connect(lambda:  self.refresh_preset_dropdown(self.preset_dialog.read_preset_book()))

        self.fmt_generator = FFMPEGGenerator()
        self.splice_cmd_generator = SpliceCMDGenerator()

        # Thread
        self.threadpool = QtCore.QThreadPool()

    ################################################################################################################################################
    def create_a_thread(self):
        self.splice_tool = SpliceThread()
        self.splice_tool.signals.finished.connect(self.finish)

    ################################################################################################################################################
    def browse_path(self,inp = False, out = False):
        path = ''

        supported_video_fmt = self.fmt_generator.get_muxing_supported_fmt()

        formats = ''
        for each_fmt in supported_video_fmt:
            formats += '%s(*.%s);;'%(each_fmt.strip(),each_fmt.strip())

        if inp :
            file_dialog = MyFileDialog()
            path_list = file_dialog.exec_()

            if path_list:
                for path in path_list:
                    print(path)
                    self.add_files_to_list(path)

        elif out:
                path = QtWidgets.QFileDialog.getSaveFileName(self,'Choose File','.','%s'%formats)[0]

                if path != '':
                    path = path.replace('/','\\')
                    self.output_dropdown.clear()
                    self.output_dropdown.addItem(path)

    ################################################################################################################################################
    # Add files to file list box, and store the file path to 'file_list_dd'
    def add_files_to_list(self,file_path):
        file_name = os.path.split(file_path)[1].strip()
        print(file_name)

        self.file_list_dd[file_name] = {'file_path':'%s'%file_path}
        self.get_file_identity(file_path,file_name)

        # Check valid image sequence
        self.check_valid_image_sequence(file_name)

        self.refresh_file_list()

    ################################################################################################################################################
    def check_valid_image_sequence(self,file_name):

        if self.file_list_dd[file_name]['identity'] == 'image_sequence':
            folder_path = self.file_list_dd[file_name]['file_path']
            name_set = set()
            for each_img in os.listdir(folder_path):
                img_without_extension = os.path.splitext(each_img)[0]

                i = 1
                temp = []
                while True:

                    if i == len(img_without_extension):
                        break

                    if img_without_extension[-i].isdigit():
                        i += 1
                        continue
                    else:
                        temp.insert(0,img_without_extension[:-i])
                        break

                image_sequence_name = ''.join(i for i in temp)
                name_set.add(image_sequence_name)

            if len(name_set) > 1:
                QtWidgets.QMessageBox.about(self,' Warning ','%s has more than one image sequence'%file_name )
                del self.file_list_dd[file_name]

    ################################################################################################################################################
    def get_file_name_from_list(self,row_content):
        if '|' in row_content:
            file_name = row_content.split('|')[1].strip()
        else:
            file_name = row_content.strip()

        return file_name

    ################################################################################################################################################
    def refresh_file_list(self):
        print('refresh signal received')

        self.file_list.setRowCount(len(self.file_list_dd.keys()))

        for index,each_file in enumerate(self.file_list_dd.keys()):
            file_name = QtWidgets.QTableWidgetItem(each_file)
            self.file_list.setItem(index,0,file_name)            #Format : (row,col,item)

            if self.file_list_dd[each_file]['identity'] == 'still_image':
                if 'duration' in self.file_list_dd[each_file]:
                    duration = QtWidgets.QTableWidgetItem(self.file_list_dd[each_file]['duration'])
                else:
                    duration = QtWidgets.QTableWidgetItem('!')
                self.file_list.setItem(index,1,duration)

            else:
                NA = QtWidgets.QTableWidgetItem('NA')
                self.file_list.setItem(index,1,NA)

    ################################################################################################################################################
    def clear_file_list_box(self):
        self.file_list.clearContents()
        self.file_list.setRowCount(1)
        self.duration_inp.clear()
        self.file_list_dd ={}
        self.cmd_list = []

    ################################################################################################################################################
    def clear_all(self):
        self.clear_file_list_box()
        self.output_dropdown.clear()
        self.progressBar.setValue(0)

    #############################################################################################################
    def use_overlay(self):
        if self.overlay_checkbox.isChecked():
            self.overlay_group.setEnabled(True)
        else:
            self.overlay_group.setEnabled(False)

    ################################################################################################################################################
    def refresh_preset_dropdown(self,PRESETS):
        self.preset_dropdown.clear()
        self.preset_dropdown.addItem('None')

        for each_set in sorted(PRESETS.keys()):
            self.preset_dropdown.addItem(each_set)
        print(self.PRESETS)

    ################################################################################################################################################
    def finish(self,signal):
        if signal == 'done':
            self.progressBar.setRange(0,1)
            self.progressBar.setValue(1)
            QtWidgets.QMessageBox.about(self,' ','Done !')
            self.prevent_misclicking(finish=True)

    #############################################################################################################
    def closeEvent(self,event):
        '''
        overwrite closeEvent, pop a comfirmation window before closing
        '''
        reply = QtWidgets.QMessageBox.question(self,'Warning','Are you sure to Exit',QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No)
        if reply == QtWidgets.QMessageBox.Yes:
            self.help_menu_dialog.close()
            self.preset_dialog.close()
            self.close()
        else:
            event.ignore()

    #############################################################################################################
    def move_file(self,direction):
        selected_row = self.file_list.currentRow()

        # If no row has been selected, return value is -1
        if selected_row == -1 :
            QtWidgets.QMessageBox.about(self,' ','Please select a file !')
        else:
            file_name = self.file_list.item(selected_row,0).text()
            duration = self.file_list.item(selected_row,1).text()

            if direction == 'up' and selected_row != 0:
                self.file_list.insertRow(selected_row-1)
                self.file_list.setItem(selected_row-1,0,QtWidgets.QTableWidgetItem(file_name))
                self.file_list.setItem(selected_row-1,1,QtWidgets.QTableWidgetItem(duration))
                self.file_list.removeRow(selected_row+1)
                self.file_list.setCurrentCell(selected_row-1,0)

            elif direction == 'down' and selected_row != len(self.file_list_dd.keys())-1:
                self.file_list.insertRow(selected_row+2)
                self.file_list.setItem(selected_row+2,0,QtWidgets.QTableWidgetItem(file_name))
                self.file_list.setItem(selected_row+2,1,QtWidgets.QTableWidgetItem(duration))
                self.file_list.removeRow(selected_row)
                self.file_list.setCurrentCell(selected_row+1,0)

    ################################################################################################################################################
    def delete_item(self):
        selected_row = self.file_list.currentRow()

        if selected_row == -1 :
            QtWidgets.QMessageBox.about(self,' ','Please select a item !')

        else:
            file_name = self.file_list.item(selected_row,0).text()

            reply = QtWidgets.QMessageBox.question(self,'Warning','Are you sure to delete [%s]?'%file_name,QtWidgets.QMessageBox.Yes|QtWidgets.QMessageBox.No)

            if reply == QtWidgets.QMessageBox.Yes:
                del self.file_list_dd[file_name]
                self.duration_inp.clear()

                self.file_list.removeRow(selected_row)
                print(self.file_list_dd)

    ################################################################################################################################################
    def get_file_identity(self,file_path,file_name):
        global still_img_extensions
        if os.path.isfile(file_path):
            if file_name.split('.')[-1].strip().lower() in still_img_extensions:
                identity = 'still_image'
            else:
                identity = 'video'
        elif os.path.isdir(file_path):
            identity = 'image_sequence'
        else:
            QtWidgets.QMessageBox.about(self,' ',' Unknown File ')

        print('%s is %s'%(file_path,identity))

        self.file_list_dd[file_name]['identity'] = identity
        print(self.file_list_dd)

    ################################################################################################################################################
    # Check if a file is a still image
    def check_is_still_img(self):
        global still_img_extensions

        self.duration_inp.clear()
        selected_row = self.file_list.currentRow()
        file_name = self.file_list.item(selected_row,0).text()

        if file_name.split('.')[-1].strip().lower() in still_img_extensions:

            self.duration_inp.setEnabled(True)
            self.ok_btn.setEnabled(True)

            if 'duration' in self.file_list_dd[file_name].keys():
                self.duration_inp.setText(self.file_list_dd[file_name]['duration'])

        else:

            self.duration_inp.setEnabled(False)
            self.ok_btn.setEnabled(False)

    ################################################################################################################################################
    def ok_btn_function(self):
        if not self.duration_inp.text().strip().isdigit():
            QtWidgets.QMessageBox.about(self,' ERROR ',' Please input a valid duration ')
        elif self.duration_inp.text().strip() == '0':
            QtWidgets.QMessageBox.about(self,' ERROR ',' Duration must be larger than zero ')
        else:
            selected_row = self.file_list.currentRow()
            if selected_row == -1:
                QtWidgets.QMessageBox.about(self,' ERROR ',' Please select a file ')
            else:
                file_name = self.file_list.item(selected_row,0).text()
                self.file_list_dd[file_name]['duration']= self.duration_inp.text().strip()
                for obj in self.file_list.findItems(file_name,QtCore.Qt.MatchExactly):
                    self.file_list.setItem(obj.row(),1,QtWidgets.QTableWidgetItem(self.duration_inp.text().strip()))

                self.duration_inp.clear()

            print(self.file_list_dd)

            #self.refresh_file_list()

    ################################################################################################################################################
    def get_resolution(self):
        w = self.w_inp.text().strip()
        h = self.h_inp.text().strip()
        resolution = w + 'x' + h

        return resolution

    ################################################################################################################################################
    def splice_btn_function(self):
        if self.check_errors():

            self.create_a_thread()

            # Create a sequence list
            num_of_files = self.file_list.rowCount()
            sequence_list = []
            for i in range(num_of_files):
                file_name = self.file_list.item(i,0).text()
                sequence_list.append(file_name)
            print('file list in GUI is : \n %s'%sequence_list)

            # Get selcected preset parameters
            selected_preset = self.preset_dropdown.currentText()
            parameter_list = self.PRESETS[selected_preset].split(',')

            # Pass values to back end thread

            if self.overlay_checkbox.isChecked():
                print('overlay is checked')
                self.splice_tool.overlay_flag = True

                project_name = self.project_inp.text().strip()
                author = self.author_inp.text().strip()
                task = self.task_inp.text().strip()
                version = self.version_inp.text().strip()

                self.splice_tool.overlay_info_list = [project_name,author,task,version]

            self.splice_tool.preset_parameter_list = parameter_list
            self.splice_tool.output_file = self.output_dropdown.currentText().strip()
            self.splice_tool.target_resolution = self.get_resolution()
            self.splice_tool.sequence_list = sequence_list
            self.splice_tool.file_list_dd = self.file_list_dd

            self.progressBar.setRange(0,0)
            self.threadpool.start(self.splice_tool)
            self.prevent_misclicking()

    ################################################################################################################################################
    # Disable some buttons when it is in splicing process
    def prevent_misclicking(self,finish=False):
        self.output_group.setEnabled(finish)
        self.input_browse.setEnabled(finish)
        self.preser_group.setEnabled(finish)
        self.file_list.setEnabled(finish)
        self.clear_btn.setEnabled(finish)
        self.delete_btn.setEnabled(finish)
        self.duration_inp.setEnabled(finish)
        self.ok_btn.setEnabled(finish)
        self.overlay_checkbox.setEnabled(finish)
        self.splice_btn.setEnabled(finish)
        self.overlay_group.setEnabled(finish)
        self.clear_all_btn.setEnabled(finish)
        self.use_overlay()

    ################################################################################################################################################
    def check_errors(self):

        try:
            num_of_files = len(self.file_list_dd.keys())
            print('%s files in file list'%num_of_files)
            if not num_of_files:
                raise MyError(' File list is empty ! ')

        except MyError as e:
            QtWidgets.QMessageBox.about(self,' Warning ',e.message)

        else:
            try:

                if not self.output_dropdown.currentText().strip():
                    raise MyError(' Please specify output file ')

                if self.preset_dropdown.currentText().strip() == 'None':
                    raise MyError(' Please select a Pre-Set ')

                if not self.w_inp.text().strip().isdigit():
                    raise MyError(' Please input a valid resolution Width')

                if not self.h_inp.text().strip().isdigit():
                    raise MyError(' Please input a valid resolution Height')

                for i in range(num_of_files):
                    file_name = self.file_list.item(i,0).text()
                    # Identity is at index 1 {file name:[path,identity]}
                    identity = self.file_list_dd[file_name]['identity']

                    # Specific duration must be assigned to still images
                    if identity == 'still_image':
                        # If still image is not assigned with a duration
                        if 'duration' not in self.file_list_dd[file_name].keys():
                            raise MyError(' Please assign a duration for File %s '%file_name)

                        # Check if duration is a digit
                        elif not self.file_list_dd[file_name]['duration'].isdigit():
                            raise MyError(' Still image %s has invalid duration '%file_name)

                    else:
                        continue

            except MyError as e:
                print(e.message)
                QtWidgets.QMessageBox.about(self,' Warning ',e.message)

            else:
                return True

if __name__ == '__main__':
    os.environ["QT_AUTO_SCREEN_SCALE_FACTOR"] = "1"
    app = QtWidgets.QApplication(os.sys.argv)
    app.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling)

    splice_tool = SpliceTool()
    splice_tool.show()
    app.exec_()


"""empty message

Revision ID: 9d2828aa95a0
Revises: 452e194acb07
Create Date: 2017-11-08 12:54:00.270559

"""

# revision identifiers, used by Alembic.
revision = '9d2828aa95a0'
down_revision = '452e194acb07'

from alembic import op
import sqlalchemy as sa

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('db_meta_data',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('version', sa.Integer(), nullable=True),
    sa.Column('description', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('db_meta_data')
    # ### end Alembic commands ###


from fastapi import APIRouter
import model
from typing import Union
from data import Data

api_root = APIRouter()

@api_root.get("/book", response_model=Union[model.Response200, model.Response404])
async def book_description(book_id: str):
    response = Data.book(book_id)
    if response is not None:
        book_info = model.BookInfo(
            book_id=book_id,
            book_name=response.xpath(model.BookXpath.book_name)[0],
            author_name=response.xpath(model.BookXpath.author_name)[0],
            update_time=response.xpath(model.BookXpath.update_time)[0],
            update_chapter=response.xpath(model.BookXpath.update_chapter)[0],
            description=response.xpath(model.BookXpath.description)[0].replace(" ", ""),
            book_state=response.xpath(model.BookXpath.book_state)[0],
            cover_url="https://www.qu-la.com" + response.xpath(model.BookXpath.cover_url)[0],
            catalogue=[
                {"chapter_url": i.attrib['href'], "chapter_name": i.text} for i in
                response.xpath(model.BookXpath.catalogue)
            ]
        )
        return model.Response200(data=book_info.dict())
    return model.Response404(message="get book info failed, book_id is {}".format(book_id))

@api_root.get("/search", response_model=Union[model.Response200, model.Response404])
async def search_api(keyword: str):
    response = Data.search(keyword)
    if response is not None:
        search_result = []
        for index, data in enumerate(response.xpath(model.BookXpath.search_result), start=1):
            search_result.append({
                'index': index,
                'book_name': data.text,
                'book_url': data.attrib['href'].replace('booktxt', '').replace('/', ''),
            })
        return model.Response200(data=model.Search(search_result=search_result).dict())
    return model.Response404(message="search failed, keyword is {}".format(keyword))

@api_root.get("/chapter", response_model=Union[model.Response200, model.Response404])
async def content_api(book_id: Union[str, int] = 0, chapter_id: Union[str, int] = 0):
    if book_id == 0 or chapter_id == 0:
        return model.Response404(message="missing book_id or chapter_id")
    response = Data.chapter(book_id, chapter_id)
    if response is not None:
        chapter_info = model.Chapter(
            chapter_title=response.xpath(model.BookXpath.chapter_title)[0].strip(),
            content='\n'.join(response.xpath(model.BookXpath.content)).replace('\r', '\n')
        )
        return model.Response200(data=chapter_info.dict())
    return model.Response404(message="get chapter info failed, chapter_id is {}".format(chapter_id))

@api_root.get("/rank", response_model=Union[model.Response200, model.Response404])
async def rank_api(page: int, rank_type: model.Rank):
    rank_result = []
    if rank_type == model.Rank.day:
        response = Data.rank(page, model.Rank.day)
    elif rank_type == model.Rank.week:
        response = Data.rank(page, model.Rank.week)
    elif rank_type == model.Rank.month:
        response = Data.rank(page, model.Rank.month)
    elif rank_type == model.Rank.total:
        response = Data.rank(page, model.Rank.total)
    else:
        return model.Response404(message="rank_type error")

    if not response:
        return model.Response404(message="rank failed, page is {}".format(page))
    for index, data in enumerate(response.xpath(model.BookXpath.rank_result), start=1):
        rank_result.append({
            'index': index,
            'type': rank_type,
            'page': page,
            'book_name': data.text,
            'book_id': data.attrib['href'].replace('booktxt', '').replace('/', ''),
        })
    return model.Response200(data=model.Search(search_result=rank_result).dict())

@api_root.get("/class", response_model=Union[model.Response200, model.Response404])
async def class_api(page: int, class_type: model.BookClass):
    class_result = []
    if class_type == model.BookClass.xh:
        response = Data.book_class(page, model.BookClass.xh)
    elif class_type == model.BookClass.wy:
        response = Data.book_class(page, model.BookClass.wy)
    elif class_type == model.BookClass.cy:
        response = Data.book_class(page, model.BookClass.cy)
    elif class_type == model.BookClass.kh:
        response = Data.book_class(page, model.BookClass.kh)
    elif class_type == model.BookClass.finish:
        response = Data.book_class(page, model.BookClass.finish)
    elif class_type == model.BookClass.qt:
        response = Data.book_class(page, model.BookClass.qt)
    elif class_type == model.BookClass.ds:
        response = Data.book_class(page, model.BookClass.ds)
    elif class_type == model.BookClass.xj:
        response = Data.book_class(page, model.BookClass.xj)
    else:
        return model.Response404(message="class_type error")
    if not response is not None:
        return model.Response404(message="class failed, page is {}".format(page))
    for index, data in enumerate(response.xpath(model.BookXpath.class_result), start=1):
        class_result.append({
            'index': index,
            'type': class_type,
            'page': page,
            'book_name': data.text,
            'book_id': data.attrib['href'].replace('booktxt', '').replace('/', ''),
        })
    return model.Response200(data=model.Search(search_result=class_result).dict())


from glob import glob
from os.path import join,splitext,basename
from natsort import natsorted
import random
import tifffile
import numpy as np
import matplotlib.pyplot as plt

def read_vaihingen(path_dataset):
  """Read images and labels paths of the
      Vaihingen dataset

  Args:
      path_dataset (string): Path to the folder
      with the Vaihingen dataset

  Returns:
      tuple: two list with the path of the
      images and labels
  """
  path_top = join(path_dataset,"top")
  path_gts = join(path_dataset,"gts")

  # List with all TIF images in each folder
  list_top = glob(join(path_top,"*.tif"))
  list_top = natsorted(list_top, key=lambda y: y.lower())

  list_gts = glob(join(path_gts,"*.tif"))
  list_gts = natsorted(list_gts, key=lambda y: y.lower())

  return list_top, list_gts

def train_val_test_vaihingen(list_top,
                             list_gts,
                             val_size=0.25,
                             seed=42):
  """Function to generate the train, validation and test
  sets of the Vaihingen dataset

  Args:
      list_top (list): list with the paths of the images
      list_gts (list): list with the paths of the labels
      val_size (float, optional): Proportion to be used
        for validation. Defaults to 0.25.
      seed (int, optional): Seed to shuffle train samples
        before split. Defaults to 42.

  Returns:
      splits : dictionary with the paths of the images and
        labels for train, validation, and test
  """
  # Based on the benckmark rules,
  # these are the areas for train and test
  train_ids = [1,3,5,7,11,13,15,17,21,23,26,28,30,32,34,37]
  test_ids = [2,4,6,8,10,12,14,16,20,22,24,27,29,31,33,35,38]

  # Initialization of train, val and test sets
  list_top_train, list_top_val, list_top_test = ([] for i in range(3))
  list_gts_train, list_gts_val, list_gts_test = ([] for i in range(3))

  for top,gts in zip(list_top,list_gts):
    # Get the area ID from the filename
    # Example: ISPRS_semantic_labeling_Vaihingen/top/top_mosaic_09cm_area1.tif
    area_id = int(splitext(basename(top))[0].split("area")[-1])
    # Train set
    if area_id in train_ids:
      list_top_train.append(top)
      list_gts_train.append(gts)
    # Test set
    elif area_id in test_ids:
      list_top_test.append(top)
      list_gts_test.append(gts)

  # shuffle train lists
  temp = list(zip(list_top_train,list_gts_train))
  random.seed(seed)
  random.shuffle(temp)
  list_top_train,list_gts_train = zip(*temp)

  # take val_size as validation set
  n_val = int(val_size*len(list_top_train))

  list_top_val = list_top_train[:n_val]
  list_top_train = list_top_train[n_val:]

  list_gts_val = list_gts_train[:n_val]
  list_gts_train = list_gts_train[n_val:]

  splits = dict()
  splits["top_train"] = list_top_train
  splits["gts_train"] = list_gts_train
  splits["top_val"] = list_top_val
  splits["gts_val"] = list_gts_val
  splits["top_test"] = list_top_test
  splits["gts_test"] = list_gts_test

  return splits

def print_info(list_top, list_gts):
  """Function to print information of the Vaihingen dataset
   about number of classes

  Args:
      list_top (list): list with the paths of the true orthophotos
      list_gts (list): list with the paths of the laels
  """
  for top,gts in zip(list_top,list_gts):
    gt_temp = tifffile.imread(gts)
    n_classes = len(np.unique(gt_temp.reshape(-1,3), axis=0))
    print("TOP:{} , GT:{} , N:{}".format(basename(top),
                                          basename(gts),
                                          n_classes))

def show_batch(img, label, colormap):
  """Function to show the content in a batch

  Args:
      img (np.array): array with image data from
        a batch
      label (np.array): array with label data from
        a batch
      colormap (dict): dictionary with the colormap
        and labels for visualization
  """
  for x_i,y_i in zip(img,label):
    # For visualization: [0,1] => [0,255]
    x_i = x_i*255
    # Apply argmax to convert from one hot to integers
    y_i = np.argmax(y_i, axis=-1)
    # Convert each class to a color given a colormap
    y_i_rgb = np.zeros((y_i.shape[0],
                        y_i.shape[1],
                        3),
                       dtype=np.uint8)
    for key,value in colormap.items():
      y_i_rgb[y_i==value] = key
    # Visualization of RGB image and its ground truth or reference
    plt.figure(figsize=(5,5))
    plt.subplot(1,2,1)
    plt.imshow(x_i.astype("uint8"))
    plt.title("RGB Image")
    plt.axis("off")

    plt.subplot(1,2,2)
    plt.imshow(y_i_rgb, cmap="jet")
    plt.title("Ground Truth")
    plt.axis("off")
    plt.show()

Machine Learning Intermedio/4. Semantic Segmentation/utils.py


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 13 11:39:37 2019

@author: 10904
"""

import numpy as np
import random
import matplotlib.pyplot as plt
#from GMM_Cluster import GMM_Cluster

class K_Means:
    def __init__(self):
        self.X = None
        self.Ave = None
        self.C = None
        self.Metric = None

    def train(self, data, k, metric='Euc', max_iter=1000):
        #input('K-Means Starting')
        if k > len(data):
            raise Exception('k均值聚类时，k的值超过了样本数量！'+'\n样本数: ' + str(len(data)) +
                            '\nk值: ' + str(k))
        #初始化模型
        #均值初始化为随机k个样本的值
        self.X = np.array(data)
        index = random.sample([i for i in range(self.X.shape[0])], k)
        self.Ave = np.array([self.X[i] for i in index])
        #indexes = [random.sample([i for i in range(self.X.shape[0])], 2*k) for i in range(k)]
        #self.Ave = np.array([np.sum(np.array([self.X[j] for j in indexes[i]]), axis=0)/2*k for i in range(k)
        #print(self.Ave.shape)
        loop = 0
        loop_ctrl = True
        while loop_ctrl and loop < max_iter:
            print(loop)
            loop_ctrl = True
            self.set_cluster(metric)
            for i,cluster in enumerate(self.C):
                total_x = np.zeros(self.X[0].shape)
                for c in cluster:
                    total_x += c
                if len(cluster) != 0:
                    new_ave = total_x/len(cluster)
                    #用于提前停止循环的条件，待优化
                    #loop_ctrl = loop_ctrl or np.sum(np.absolute(new_ave-self.Ave[i])) > 1e-20
                    self.Ave[i] = new_ave
            loop += 1
        self.set_cluster(metric)

    #将所有样本归到对应的簇中
    def set_cluster(self, metric):
        #print(self.Ave.shape)
        self.C = [[] for i in range(len(self.Ave))]
        for x in self.X:
            dist = []
            for i,ave in enumerate(self.Ave):
                dist.append(self.cal_dist(x, ave, metric))
            self.C[list.index(dist, min(dist))].append(x)

    def cal_dist(self, x1, x2, metric):
        if metric == 'Euc':
            return (np.sum((x1-x2)**2)**0.5)
        elif metric == 'Man':
            return np.sum(np.absolute(x1-x2))

if __name__ == '__main__':
    def get_data(x, y, r, num):
        data = []
        for i in range(num):
            theta = random.uniform(-2*np.pi, 2*np.pi)
            R = random.uniform(0,r)
            data.append([x+R*np.cos(theta), y+R*np.sin(theta)])
        return data

    model = K_Means()
    data = []
    data += get_data(-10,10,5,50)
    data += (get_data(0,-5,5,50))
    data += (get_data(10,10,5,50))
    #for i in range(num):
    #    x = random.uniform(-20,20)
    #    y = random.uniform(-20,20)
    #    data.append([x,y])

    model.train(data, k=3, metric='Man')
    clusters = []
    for c in model.C:
        clusters.append(c)
    ave = model.Ave
    #print(clusters)
    plt.xlim(-20,20)
    plt.ylim(-20,20)
    plt.plot([x[0] for x in clusters[0]], [x[1] for x in clusters[0]], 'o', color='red')
    plt.plot([x[0] for x in clusters[1]], [x[1] for x in clusters[1]], 'o', color='green')
    plt.plot([x[0] for x in clusters[2]], [x[1] for x in clusters[2]], 'o', color='blue')
    plt.plot([x[0] for x in ave], [x[1] for x in ave], 'x', color='black')
    print(ave)
    #plt.plot([x[0] for x in clusters[3]], [x[1] for x in clusters[3]], 'o', color='yellow')
    plt.show()


"""Script for running transfer attack experiment."""

from __future__ import annotations

import timm

if not timm.__version__.startswith("0.6"):
    from timm.models import xcit

    xcit.XCiT = xcit.Xcit  # pylint: disable=no-member

# pylint: disable=wrong-import-order,wrong-import-position
import argparse
import copy
import logging
import sys
import time
from pathlib import Path
from typing import Any

from main import train_test_main
from src.attack.util import get_saved_adv_name
from src.models.util import ENSEMBLE_MODELS
from src.utils.argparser import get_args_parser, setup_output_dir
from src.utils.metric import safe_pickle
from src.utils.print_result import Aggregator, HiddenPrints, print_result

def _new_config(
    orig_config: dict[str, Any], model: str
) -> tuple[dict[str, Any], Path]:
    new_config = copy.deepcopy(orig_config)
    # "model" format is DATASET_ARCH_TRAIN_SEED
    # ARCH can be
    #   - single model: resnet18
    #   - ensemble: trs_resnet18+resnet50
    #   - ensemble with freeze: trs_freeze_resnet18+resnet50

    ensemble_type = None
    mod_model = model
    for ens in ENSEMBLE_MODELS:
        if f"_{ens}_" in model:
            mod_model = mod_model.replace(f"_{ens}_", "_")
            ensemble_type = ens
            break

    if ensemble_type is not None and "_freeze_" in model:
        # Handle freeze option
        mod_model = mod_model.replace("_freeze_", "_")
        ensemble_type = None

    new_config["ensemble_type"] = ensemble_type
    new_config["arch"] = mod_model.split("_")[1].split("+")
    if ensemble_type is None:
        new_config["arch"] = [new_config["arch"][0]]
    new_config["output_dir"] = f"results/{model}"
    new_config["evaluate"] = True
    output_path = setup_output_dir(new_config)
    return new_config, output_path

def main() -> None:
    """Main function."""
    source_models = config["src_models"]
    target_models = config["tgt_models"]
    attacks = config["transfer_attacks"]
    if not source_models:
        raise ValueError("No source models specified via --src-models.")
    if not target_models:
        raise ValueError("No target models specified via --tgt-models.")
    if not attacks:
        raise ValueError("No attacks specified via --transfer-attacks.")
    config["src_models"] = None
    config["tgt_models"] = None
    config["transfer_attacks"] = None
    run_exp(config, source_models, target_models, attacks)

def run_exp(
    config,  # pylint: disable=redefined-outer-name
    source_models,
    target_models,
    attacks,
) -> None:
    """Run transfer experiments for multiple sources, targets, and attacks.

    WARNING: metrics["idx_correct"] returned from train_test_main() should have
    a fixed length (= num test samples), but there is no guarantee that the
    order of the samples remained the same across runs. We fix random seed in
    all the scripts so the ordering should be correct when run in the same
    environment (same package version and machine), but beyond that is unclear.
    """
    results = {}

    for _, tgt in enumerate(target_models):
        aggregator = Aggregator()
        results[tgt] = {}
        tgt_config, tgt_path = _new_config(config, tgt)

        for src in source_models:
            results[tgt][src] = {}
            src_config, src_path = _new_config(config, src)

            for attack in attacks:
                print(f"{src} -> {tgt} using {attack} attack.")

                if (
                    (attack == "naa" and "robustbench-diffenderfer21" in src)
                    or (attack == "naa" and "imagenet_timm-vgg-11" in src)
                ):
                    print(f"[WARNING] Skipping {attack} on {src}")
                    continue

                start_time = time.time()
                saved_adv_path = src_path / get_saved_adv_name(
                    src_config, attack=attack, partition="test"
                )
                # Check if attack has been generated
                if not saved_adv_path.is_dir():
                    raise NotADirectoryError(
                        f"Given path {saved_adv_path} does not exist or is not "
                        "a directory."
                    )
                if not any(saved_adv_path.iterdir()):
                    raise FileNotFoundError(
                        f"No file found in {saved_adv_path}."
                    )
                # Get full attack name
                adv_name = saved_adv_path.name.replace("saved_", "")

                print("  Evaluating...")
                tgt_config["temperature"] = 1.0  # Reset eval temperature to 1
                tgt_config["test_load_paths"] = [str(saved_adv_path)]
                # Need to set train_load_paths because dataloader requires it
                tgt_config["train_load_paths"] = [str(saved_adv_path)]
                with HiddenPrints(not config["debug"]):
                    metrics = train_test_main(copy.deepcopy(tgt_config))
                # TODO(enhance): Collect metrics from last dataset assuming that
                # it is the adversarial one. Perhaps we can clean this up a bit.
                metrics = metrics["test"][-1]
                elapsed_time = (time.time() - start_time) / 60
                print(
                    f'  accuracy: {metrics["acc0"]:.2f}, time: '
                    f"{elapsed_time:.1f} min",
                    flush=True,
                )

                # Collect successes for best attack across sources and atk algo
                idx_success = ~metrics["idx_correct"]
                aggregator.update((src, attack), idx_success)
                results[tgt][src][adv_name] = idx_success

                # Dump all results after every run
                safe_pickle("results/all_results.pkl", obj=results, load=False)

                # Dump target-specific results
                tgt_result_path = tgt_path / "results.pkl"
                if tgt_result_path.exists():
                    # If results already exist, append to it
                    tgt_results = safe_pickle(tgt_result_path, load=True)
                    if src not in tgt_results:
                        tgt_results[src] = {}
                    tgt_results[src][adv_name] = idx_success
                else:
                    tgt_results = {src: {adv_name: idx_success}}
                safe_pickle(tgt_result_path, obj=tgt_results, load=False)

        print(f"Target: {tgt}")
        errs = print_result(attacks, source_models, aggregator)
        avg_acc, best_acc, overall_best_acc = [100 - err for err in errs]
        # metrics = safe_pickle(f"results/{tgt}/metrics.pkl", load=True)
        # The first metric should be from training
        # metrics = metrics[0]["test"][-1]
        # assert metrics["attack"] in ("none", "no_attack"), metrics
        # clean_acc = metrics["acc0"]
        clean_acc = tgt_results["test"]["none"]  # Gather new result with this line
        print(
            f"Accuracy (clean, avg, best, per-sample best): {clean_acc:.2f}, "
            f"{avg_acc:.2f}, {best_acc:.2f}, {overall_best_acc:.2f}"
        )

    safe_pickle("results/all_results.pkl", obj=results, load=False)
    print("Finished.")

if __name__ == "__main__":
    main_parser = argparse.ArgumentParser(
        "Run multiple transfer attacks across set of source and target models.",
        parents=[get_args_parser()],
    )
    main_parser.add_argument(
        "--tgt-models",
        default=None,
        nargs="+",
        type=str,
        help="List of target models used to evaluate against transfer attacks.",
    )
    args = main_parser.parse_args()
    config: dict[str, Any] = vars(args)

    # Set logging config
    logging.basicConfig(
        stream=sys.stdout,
        format="[%(asctime)s - %(name)s - %(levelname)s]: %(message)s",
        level=logging.DEBUG if args.debug else logging.INFO,
        force=True,
    )
    logging.getLogger("PIL").setLevel(logging.WARNING)

    main()


from google.appengine.ext import db
from model import IngredientCategory
import jinja2
import os
from base_handler import BaseHandler
from user_management import isUserCook
from cache_ingredient import modifyIngredient
from cache_ingredient_category import addIngredientCategory,\
	getIngredientCategoryWithIngredients, getIngredientCategories,\
	deleteIngredientCategory

jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

class CategoryIngredientDeletePage(BaseHandler):
	URL = '/deleteIngredientFromCategory'
	def post(self):
		if not isUserCook(self):
			self.redirect("/")
			return
		category = db.get(self.request.get('ingredientCategoryKey'))
		ingredient = db.get(self.request.get('ingredientKey'))
		ingredient.category=None
		ingredient.put()
		modifyIngredient(ingredient)
		self.redirect('/ingredientCategory?ingredientCategoryKey=%s' % category.key())

class IngredientCategoryPage(BaseHandler):
	URL = '/ingredientCategory'
	def post(self):
		if not isUserCook(self):
			self.redirect("/")
			return
		ingredientCategory = IngredientCategory()
		ingredientCategory.name = self.request.get('ingredient_category_name')
		ingredientCategory.put()
		addIngredientCategory(ingredientCategory)
		self.redirect('/ingredientCategory')
	def get(self):
		if not isUserCook(self):
			self.redirect("/")
			return
		ingredientCategoryKey = self.request.get('ingredientCategoryKey')
		if ingredientCategoryKey!= None and ingredientCategoryKey != "":
		# List every ingredient in the category
			ingredientCategory = getIngredientCategoryWithIngredients(ingredientCategoryKey)
			template_values = {
				'ingredientCategory': ingredientCategory,
				'add_url':"/addIngredientToCategory",
				'delete_url':"/deleteIngredientFromCategory"
			}
			template = jinja_environment.get_template('templates/ingredient_category.html')
			self.printPage(ingredientCategory['name'], template.render(template_values), False, False)
		else:
		# All categories
			ingredientCategories = getIngredientCategories()
			template_values = {
				'ingredientCategories': ingredientCategories,
				'delete_url':"/deleteIngredientCategory"
			}
			template = jinja_environment.get_template('templates/ingredient_category_list.html')
			self.printPage("Alapanyag kategoriak", template.render(template_values), False, False)

class IngredientCategoryDeletePage(BaseHandler):
	URL = '/deleteIngredientCategory'
	def post(self):
		if not isUserCook(self):
			self.redirect("/")
			return
		ingredientCategoryKey = self.request.get('ingredientCategoryKey')
		ingredientCategory = db.get(ingredientCategoryKey)
		ingredientCategory.delete()
		deleteIngredientCategory(ingredientCategoryKey)
		self.redirect('/ingredientCategory')


import json
import os

import requests as re

import streamlit as st

st.title("Resume Parser App using WordsAI")

st.image("./images/logo.png")

st.write("""
## About
This Resume Parser app uses the WordsAI API to parse a resume
""")

# write a file uploader using st.file_uploader
wav_file = st.file_uploader("Upload a pdf file", type=["pdf"])
# save the file to a temporary directory
if st.button ("Get Parsed Resume"):
    if wav_file is not None:
        files = wav_file.read()
        # save the file
        filename = "./temp/temp.pdf"
        with open(filename, "wb+") as f:
            f.write(files)
        # make a post request to the fastapi server
        response = re.post("http://wordsai-api.herokuapp.com/resume_parser", files={"file": open("./temp/temp.pdf", "rb")})
        response_json = response.json()
        st.write(response_json)


import pandas as pd
import json
file_name = '/import/cogsci/ravi/datasets/Embeddia/STY_24sata_comments_hr_001.csv'

# file_name = '../../../data/unsup/24h/classify/cro_test.csv'

df = pd.read_csv(file_name)
df['length'] = df.content.str.len()

data = {}
data['max'] = int(df["length"].max())
data['min'] = int(df["length"].min())
data['mean'] = int(df["length"].mean())
data['mode'] = int(df["length"].mode())
data['median'] = int(df["length"].median())

gr_mean = len(df[df['length'] > data['mean']])
gr_mean2 = len(df[df['length'] > 2*data['mean']  ])
gr_mean1 = len(df[df['length'] > 1.5*data['mean']  ])

print(gr_mean)
print(type(gr_mean))
# print(gr_mean.values,type(gr_mean.values))

data['gr_mean'] = gr_mean
data['gr_mean1'] = gr_mean1
data['gr_mean2'] = gr_mean2

# print(df["length"].max(), df["length"].min(), df["length"].mean(),df["length"].mode(), df["length"].median())
# print(df['length'])

with open('dataLen.json','w') as fid:
    json.dump(data,fid)

print(data)
# print(df[df['length']==967].content.values)


"""
Dataset loader for CIFAR100
"""
from copy import deepcopy

import torch
import numpy as np

from torch.utils.data import WeightedRandomSampler
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

def wif(id):
    """
    Used to fix randomization bug for pytorch dataloader + numpy
    Code from https://github.com/pytorch/pytorch/issues/5059
    """
    process_seed = torch.initial_seed()
    # Back out the base_seed so we can use all the bits.
    base_seed = process_seed - id
    ss = np.random.SeedSequence([id, base_seed])
    # More than 128 bits (4 32-bit words) would be overkill.
    np.random.seed(ss.generate_state(4))

class MultiTaskDataHandler():
    """
    Template class for a Multi-task data handler
    """
    def __init__(self) -> None:
        self.trainset: Dataset
        self.testset: Dataset

    def get_data_loader(self,
                        batch_size: int,
                        workers: int,
                        train: bool = True) -> DataLoader:
        """
        Get the Dataloader for the entire dataset
        Args:
            - shuf       : Shuffle
            - wtd_loss   : Dataloader also has wts along with targets
            - wtd_sampler: Sample data from dataloader with weights
                         according to self.tr_wts
        """
        data = self.trainset if train else self.testset
        loader = DataLoader(
            data, batch_size=batch_size, shuffle=train,
            num_workers=workers, pin_memory=True,
            worker_init_fn=wif)

        return loader

    def get_task_data_loader(self,
                             task: int,
                             batch_size: int,
                             workers: int,
                             train: bool = False) -> DataLoader:
        """
        Get Dataloader for a specific task
        """
        if train:
            task_set = deepcopy(self.trainset)
        else:
            task_set = deepcopy(self.testset)

        task_ind = [task == i[0] for i in task_set.targets]

        task_set.data = task_set.data[task_ind]
        task_set.targets = np.array(task_set.targets)[task_ind, :]
        task_set.targets = [(lab[0], lab[1]) for lab in task_set.targets]

        loader = DataLoader(
            task_set, batch_size=batch_size,
            shuffle=False, num_workers=workers, pin_memory=True,
            worker_init_fn=wif)

        return loader


# !/usr/bin/python3
# -*- coding:utf-8 -*-

import os
import logging

import django
from django.conf import settings

def setup_logger():
    """"
    配置日志相关
    """
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ImageRecognition.settings')
    django.setup()

    log_format = logging.Formatter(fmt='[%(asctime)s][%(pathname)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
    logger = logging.getLogger('')
    logger.setLevel(logging.DEBUG)

    if not logger.handlers:
        log_level = settings.LOG_OUTPUT_LEVEL
        log_output = logging.StreamHandler()
        log_output.setLevel(log_level)
        log_output.setFormatter(log_format)
        logger.addHandler(log_output)

        log_level = settings.LOG_FILE_LEVEL
        log_file_path = settings.LOG_FILE
        log_dir = os.path.dirname(log_file_path)
        if not os.path.isdir(log_dir):
            os.makedirs(log_dir)
        file_log = logging.FileHandler(log_file_path)
        file_log.setLevel(log_level)
        file_log.setFormatter(log_format)
        logger.addHandler(file_log)


#this parses the xml file, extracts raw contents of each doc, and sends it for processing
import xml.sax
from process import doc_to_ind
from indexer import write_to_disk

titles_total = 0
index_count = 0
out_dirname = ""
output_file = ""

class WikiDocParser(xml.sax.ContentHandler):
	lod = [{}, {}, {}, {}, {}, {}]
	sow = set()
	titles = {}
	global titles_total
	global out_dirname
	global output_file

	def __init__(self):
		self.title = ""
		self.text = ""
		self.titleflag = 0
		self.textflag = 0
		self.lod = [{}, {}, {}, {}, {}, {}]
		self.sow.clear()
		self.titles.clear()
		self.totaldocs = 0

	def startElement(self, lbl, attrs):
		if lbl == "title":
			self.title= ""
			self.titleflag = 1
		elif lbl == "text":
			self.text = ""
			self.textflag = 1

	def endElement(self, lbl):
		if lbl == "title": #page title
			self.titleflag = 0

		elif lbl == "text": #page content
			self.textflag = 0

		elif lbl == "page": #end of curr page
			global titles_total
			global index_count
			self.totaldocs += 1
			titles_total += 1

			title_ind = titles_total
			doc_to_ind(self.title, self.text, title_ind, self.lod, self.sow)
			self.titles[title_ind] = self.title

			if self.totaldocs %10 == 0:
				op = str(self.totaldocs)
				op += "\r"
				print(op, end="")

			if self.totaldocs % 10000 == 0:
				write_to_disk(self.lod, self.sow, out_dirname, "indexfile_" + str(index_count) + ".txt")
				write_titles(self.titles, out_dirname)
#				self.lod.clear()
				self.lod = [{}, {}, {}, {}, {}, {}]
				self.sow.clear()
				self.titles.clear()
				index_count += 1

		elif lbl == "mediawiki":	#end of the input file
			write_to_disk(self.lod, self.sow, out_dirname, "indexfile_" + str(index_count) + ".txt")
			write_titles(self.titles, out_dirname)
			self.lod = [{}, {}, {}, {}, {}, {}]
			self.sow.clear()
			self.titles.clear()

	def characters(self, data):
		if self.titleflag == 1:
			self.title += data
		elif self.textflag == 1:
			self.text += data

def parse_doc(filename, passed_dirname, passed_output):
	global out_dirname
	global output_file
	global titles_total
	global index_count

	print("Titles so far", titles_total)

	out_dirname = passed_dirname
	output_file = passed_output

	parser = xml.sax.make_parser()
	parser.setFeature(xml.sax.handler.feature_namespaces, 0)
	parser.setContentHandler(WikiDocParser())
	parser.parse(open("%s"%(filename),"r"))

	index_count += 1

	return titles_total

def write_titles(titles, dirname):
	with open(dirname+"/titles.txt","a+") as f:
		for key in titles:
			f.write(str(key) + ":" + str(titles[key]) + "\n")