#!/usr/bin/env python3
# This is a script 1 anon from 4chan wrote, and which I modified. The script
# will download all Microsoft patents. There are somewhere around 45500
# URLs / PDFs, so I set the range from 0 to 100 000, because it'll just finish
# and when you notice it hasn't created new files in a while, then you know the
# script is done fetching all the patents. If a URL can't be fetched, then the
# script will retry connecting 10 times, with a 10 second interval. After that
# the script gives up on that particular URL, and moves on to the next.

import requests
from requests import get
import bs4
import re
import time
import os

def download(url, file_name):
    if os.path.isfile(file_name):
        print('skipping ' + file_name + '...')
        return(True)
    connected = False
    tries_2 = 0
    while not connected:
         try:
             response = get(url)
             with open(file_name, 'wb') as file:
                 file.write(response.content)
             print('downloaded ' + str(num) + '\r\n')
             connected = True
         except:
             tries_2 = tries_2 + 1
             if tries_2 == 11:
                 connected = True
             else:
                 print('retrying ' + str(tries_2) + ' ' + url + '...')
                 time.sleep(10)
             pass

for num in range(0, 100000):
    start_url = "http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&Sect2=HITOFF&u=%2Fnetahtml%2FPTO%2Fsearch-adv.htm&r=" + str(num) + "&f=G&l=50&d=PTXT&p=1&S1=microsoft.ASNM.&OS=AN/microsoft&RS=AN/microsoft"
    headers = {'User-Agent':'HTC Mozilla/5.0 (Linux; Android 7.0; HTC 10 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36.'}

    connected = False
    tries = 0
    while not connected:
        try:
            content = requests.get(start_url,headers=headers)
            soup = bs4.BeautifulSoup(content.text,features='lxml')
            for t in soup.find_all('a', href=lambda x: x and 'View+first+page' in x):
                step2 = t['href'] + '&pagenum=0'
            sec_content = requests.get(step2)
            soup2 = bs4.BeautifulSoup(sec_content.text,features='lxml')
            download('http:' + soup2.find('embed').get('src'), soup2.find('embed').get('name') + '.pdf')
            connected = True
        except:
            tries = tries + 1
            if tries == 11:
                connected = True
            else:
                print('retrying ' + str(tries) + ' ' + start_url + '...')
                time.sleep(10)
            pass


import socket,_thread,threading,datetime
now = datetime.datetime.now()

clients = []

def recv(clientsocket,addr):
    while True:
        now = datetime.datetime.now()
        msg = clientsocket.recv(1024)
        ah = msg.decode()
        epic = ah.split(",")
        if epic[0] == "Disconnect#000000":
            clients.remove(clientsocket)
            print(f'[{now.strftime("%Y-%m-%d %H:%M:%S")}][DISCONNECT] Disconnected  {addr}')
            with open('log.csv', mode='a') as log:
                log.writelines("\n" + str([{now.strftime("%Y-%m-%d %H:%M:%S")} ,"[DIS]", addr, "N/A", "N/A"]))
        elif epic[0] == "INFO#000000":
             ah = f"[SERVER]#000000,{epic[1]} has joined the chat!"
             with open('log.csv', mode='a') as log:
                log.writelines("\n" + str([{now.strftime("%Y-%m-%d %H:%M:%S")} ,"[CON]", addr, epic[1], "N/A"]))
        else:
            print(f'[{now.strftime("%Y-%m-%d %H:%M:%S")}][MESSAGE] Message received: {epic[0]} saying {epic[1]}')
        with open('log.csv', mode='a') as log:
            log.writelines("\n" + str([{now.strftime("%Y-%m-%d %H:%M:%S")} , "[MSG]" ,addr ,  epic[0], epic[1]]))
        for i in range(len(clients)):
            client = clients[i]
            try:
             client.sendall(ah.encode())
            except BrokenPipeError:
                clients.remove(clients[i])
                print(f'[{now.strftime("%Y-%m-%d %H:%M:%S")}][IDC] Illegal Disconnect (Check Log)')
                with open('log.csv', mode='a') as log:
                    log.writelines("\n" + str([{now.strftime("%Y-%m-%d %H:%M:%S")} ,"[IDC]", "N/A","N/A",client]))
                break

    clientsocket.close()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host_name = socket.gethostname()
IP = socket.gethostbyname(host_name)
ipcorrect = input(f"Is your private IP: {IP}? [Y/N]")
answered = False

while not answered:
    if ipcorrect.lower() == "y":
        IP = IP
        answered = True
    elif ipcorrect.lower() == "n":
        IP = input("What is your private IP")
        answered = True
    else:
        print("Please enter Y/N")

PORT = int(input("port? "))

print("           Thanks for choosing PyChat!")
print("              PyChat Server V0.1.1")
print("                Server started!")
print("------------------------------------------------")
print(f"Listening for connections on {IP}:{PORT}...")

s.bind((IP, PORT))

while True:
    s.listen(5)
    c, addr = s.accept()
    _thread.start_new_thread(recv, (c, addr))
    print(f'[{now.strftime("%Y-%m-%d %H:%M:%S")}][CONNECT] Accepted new connection from {addr}')

    clients.append(c)

s.close()


'''Train CIFAR10/100 with PyTorch using standard Contrastive Learning. This script tunes the L2 reg weight of the
final classifier.'''
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn

import math
import os
import argparse

from models import *
from configs import get_datasets
from evaluate import train_clf, encode_feature_averaging

parser = argparse.ArgumentParser(description='Final evaluation with feature averaging.')
parser.add_argument("--num-workers", type=int, default=2, help='Number of threads for data loaders')
parser.add_argument("--load-from", type=str, default='ckpt.pth', help='File to load from')
parser.add_argument("--num-passes", type=int, default=10, help='Number of passes to average')
parser.add_argument("--reg-lower", type=float, default=-7, help='Minimum log regularization parameter (base 10)')
parser.add_argument("--reg-upper", type=float, default=-3, help='Maximum log regularization parameter (base 10)')
parser.add_argument("--num-steps", type=int, default=10, help='Number of log-linearly spaced reg parameters to try')
args = parser.parse_args()

# Load checkpoint.
print('==> Loading settings from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
resume_from = os.path.join('./checkpoint', args.load_from)
checkpoint = torch.load(resume_from)
args.dataset = checkpoint['args']['dataset']
args.arch = checkpoint['args']['arch']

device = 'cuda' if torch.cuda.is_available() else 'cpu'

# Data
print('==> Preparing data..')
_, testset, clftrainset, num_classes, stem = get_datasets(args.dataset, test_as_train=True)

testloader = torch.utils.data.DataLoader(testset, batch_size=1000, shuffle=False, num_workers=args.num_workers,
                                         pin_memory=True)
clftrainloader = torch.utils.data.DataLoader(clftrainset, batch_size=1000, shuffle=False, num_workers=args.num_workers,
                                             pin_memory=True)

# Model
print('==> Building model..')
##############################################################
# Encoder
##############################################################
if args.arch == 'resnet18':
    net = ResNet18(stem=stem)
elif args.arch == 'resnet34':
    net = ResNet34(stem=stem)
elif args.arch == 'resnet50':
    net = ResNet50(stem=stem)
else:
    raise ValueError("Bad architecture specification")
net = net.to(device)

if device == 'cuda':
    repr_dim = net.representation_dim
    net = torch.nn.DataParallel(net)
    net.representation_dim = repr_dim
    cudnn.benchmark = True

print('==> Loading encoder from checkpoint..')
net.load_state_dict(checkpoint['net'])

best_acc = 0
X, y = encode_feature_averaging(clftrainloader, device, net, num_passes=args.num_passes)
X_test, y_test = encode_feature_averaging(testloader, device, net, num_passes=args.num_passes)
for reg_weight in torch.exp(math.log(10) * torch.linspace(args.reg_lower, args.reg_upper, args.num_steps,
                                                          dtype=torch.float, device=device)):
    clf = train_clf(X, y, net.representation_dim, num_classes, device, reg_weight=reg_weight)
    raw_scores = clf(X_test)
    _, predicted = raw_scores.max(1)
    correct = predicted.eq(y_test).sum().item()
    acc = 100 * correct / predicted.shape[0]
    print('Test accuracy', acc, '%')
    if acc > best_acc:
        best_acc = acc
print("Best test accuracy", best_acc, "%")


import asyncio
import datetime
import random
import websockets
import serial
import io
import serial.tools.list_ports
global ser

def ports():
    global ser
    try:
        ports = list(serial.tools.list_ports.comports())
        if ports:
            for p in ports:
#                     print(p.vid,p.pid)
                if p.vid == 4292 and p.pid == 60000 or p.vid == 1240 and p.pid==61336:
                    global vid, pid, device
                    vid = p.vid
                    pid = p.pid
                    device = p.device
                    ser = serial.Serial(device,9600, timeout=0.1)
#                         print(vid ,device,pid)
                    return True
                else:
                    return False
                    print("")
        else:
                print("no active comports found")
    except Exception as e:
        print(e)

while ports() == False:
    pass

async def serialRead():
    global ser
    try:
        ##print("hh")
        serValue = ser.readline().decode('utf-8')
        if serValue:
            return(serValue)
        else:
            return False
    except Exception as e :
        while ports() == False:
            pass
        print(e)

async def time(websocket, path):
    while True:
        listener_task = asyncio.ensure_future(websocket.recv())
        producer_task = asyncio.ensure_future(serialRead())
        done, pending = await asyncio.wait(
            [listener_task, producer_task],
            return_when=asyncio.FIRST_COMPLETED)

        if producer_task in done:
            x=producer_task.result()
            if x:
                print(x)
                await websocket.send(x)
                ##await asyncio.sleep(random.random() * 3)
        else:
            producer_task.cancel()

        if listener_task in done:
            message = listener_task.result()
#             await consumer(message)
            ## do some task on receive do serial write.
            print(message)
            ser.write(message.encode("ascii"))
        else:
            listener_task.cancel()

start_server = websockets.serve(time, '0.0.0.0', 5678)

asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()


#!/usr/bin/env python3
import argparse
from datetime import datetime, timedelta
from enum import Enum
import gevent
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
import glob
import json
import os
from progress.bar import IncrementalBar
import re
import requests
from requests_toolbelt import sessions
from subprocess import Popen, PIPE, STDOUT
import sys
from time import sleep
import urllib.parse

VERSION_STRING="2021.03.18.3"

OPENREC=r'^(?:https?:\/\/)?(?:www\.)?openrec\.tv\/(?P<type>[^\/]+?)\/(?P<id>[^\/]+)$'
VALID_LIVE_ID=r'^(?P<id>[a-z0-9]+?)$'
FULL_SIZE_IMG=r'\.w[0-9]{1,}\.ttl[0-9]{1,}\.(?P<ext>[a-z]{1,4})\?'
FULL_SIZE_REP=r'.\g<ext>?q=100&quality=100&'
CLEAN_FILENAME_KINDA=r'[^\w\-_\. \[\]\(\)]'
API="https://public.openrec.tv/external/api/v5/"

class LogLevel(Enum):
    BASIC=1
    VERBOSE=2

class DownloadBar(IncrementalBar):
    suffix="%(percent).1f%% ETA %(time_remaining)s"
    @property
    def time_remaining(self):
        ts = self.eta_td.total_seconds()
        m = ts % 60
        s = ts / 60
        return f"{int(m)}:{int(s):02}"

class StreamDownloader():
    def __init__(self, playlist_base):
        self.m3u8_session = sessions.BaseUrlSession(base_url=playlist_base)
        adapter = requests.adapters.HTTPAdapter(pool_connections=5, pool_maxsize=5, max_retries=10)
        self.m3u8_session.mount('http://', adapter)
        self.m3u8_session.mount('https://', adapter)
        self.success = True
        self.failed_list = []
        self.completed = {}

    def run(self, stream_filename, ts_list, download_bar):
        self.stream_filename = stream_filename
        self.stream_file = open(os.path.join(args.directory, f"{stream_filename}.ts.tmp"), "ab")
        self.ts_count = len(ts_list)
        self.download_bar = download_bar
        join_thread = gevent.spawn(self._append_file)
        self._download_segments(ts_list)
        join_thread.join()

    def _download_segments(self, ts_list):
        Pool(10).map(self._download_worker, ts_list)
        if not self.success:
            ts_list = self.failed_list
            self.failed_list = []
            self._download_segments(ts_list)

    def _download_worker(self, ts_tuple):
        ts_segment = ts_tuple[0]
        ts_index = ts_tuple[1]
        for retry in range(0, 5):
            try:
                ts_r = self.m3u8_session.get(ts_segment)
                if ts_r.ok:
                    segment_filename = f"{self.stream_filename}.seg{ts_index}"
                    with open(os.path.join(args.directory, segment_filename), 'wb') as segment_file:
                        segment_file.write(ts_r.content)
                    self.completed[ts_index] = segment_filename
                    return
            except:
                print_log("download worker", f"failed to download {ts_segment}, retrying ({retry}/5)...")
        self.success = False

    def _append_file(self):
        index = 0
        while index < self.ts_count:
            segment_filename = self.completed.get(index, '')
            if segment_filename:
                with open(os.path.join(args.directory, segment_filename), 'rb') as segment_file:
                    self.stream_file.write(segment_file.read())
                os.remove(os.path.join(args.directory, segment_filename))
                self.download_bar.next()
                index += 1
            else:
                sleep(0.05)

def dl_channel(s, channel_id):
    # check for argument that cannot be used
    if args.list_formats:
        print_log(f"channel:{channel_id}", "cannot use argument --list-formats with channel link")
        return

    # get base channel data for checking validity
    init_response = s.get(f'channels/{channel_id}')
    if not init_response.ok:
        print_log(f"channel:{channel_id}", "failed to get channel information")
        print_log(f"channel:{channel_id}", f"API response returned status code {init_response.status_code}", LogLevel.VERBOSE)
        return
    c_init_json = init_response.json()
    if "status" in c_init_json:
        print_log(f"channel:{channel_id}", "failed to get channel information")
        print_log(f"channel:{channel_id}", f"API body returned status code {c_init_json['status']}: {c_init_json['message']}", LogLevel.VERBOSE)
        return

    # string to use in output channel filenames
    channel_string = re.sub(CLEAN_FILENAME_KINDA, "_", f"{c_init_json['name']} [{c_init_json['id']}]").strip()

    # use search api to retrieve more complete json data
    search_query_param = urllib.parse.urlencode({"search_query": c_init_json["name"]})
    search_response = s.get(f'search-users?{search_query_param}')

    # use /channels json if /search-users json doesn't exist for some reason
    c_json = c_init_json
    if search_response.ok:
        found_full_json = False
        for user_result in search_response.json():
            if user_result["name"] == c_init_json["name"]:
                c_json = user_result
                found_full_json = True
                break
        if not found_full_json:
            print_log(f"channel:{channel_id}", f"failed to get complete channel information from API")
    else:
        print_log(f"channel:{channel_id}", f"failed to get complete channel information from API")
        print_log(f"channel:{channel_id}", f"API response returned status code {search_response.status_code}", LogLevel.VERBOSE)

    if args.write_info_json:
        info_filename = f"{channel_string}.info.json"
        print_log(f"info:{channel_id}", f"writing channel information to '{info_filename}'")
        with open(os.path.join(args.directory, f"{info_filename}.tmp"), "w") as channel_info:
            channel_info.write(json.dumps(c_json))
        os.rename(os.path.join(args.directory, f"{info_filename}.tmp"), os.path.join(args.directory, info_filename))

    if args.write_thumbnail:
        # icon (avatar)
        full_size_icon_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, c_json["icon_image_url"])
        icon_format = urllib.parse.parse_qs(full_size_icon_url)["format"][0]
        icon_filename = f"{channel_string}-icon.{icon_format}"
        print_log(f"icon:{channel_id}", f"writing channel icon to '{icon_filename}'")
        icon_response = requests.get(full_size_icon_url)
        if icon_response.ok:
            with open(os.path.join(args.directory, f"{icon_filename}.tmp"), "wb") as channel_icon:
                channel_icon.write(icon_response.content)
            os.rename(os.path.join(args.directory, f"{icon_filename}.tmp"), os.path.join(args.directory, icon_filename))
        else:
            print_log(f"icon:{channel_id}", "failed to retrieve channel icon")
            print_log(f"icon:{channel_id}", f"API response returned status code {icon_response.status_code}", LogLevel.VERBOSE)

        # cover (banner)
        full_size_cover_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, c_json["cover_image_url"])
        cover_format = urllib.parse.parse_qs(full_size_cover_url)["format"][0]
        cover_filename = f"{channel_string}-cover.{cover_format}"
        print_log(f"cover:{channel_id}", f"writing channel cover to '{cover_filename}'")
        cover_response = requests.get(full_size_icon_url)
        if cover_response.ok:
            with open(os.path.join(args.directory, f"{cover_filename}.tmp"), "wb") as channel_cover:
                channel_cover.write(cover_response.content)
            os.rename(os.path.join(args.directory, f"{cover_filename}.tmp"), os.path.join(args.directory, cover_filename))
        else:
            print_log(f"icon:{channel_id}", "failed to retrieve channel cover")
            print_log(f"icon:{channel_id}", f"API response returned status code {cover_response.status_code}", LogLevel.VERBOSE)

    movie_list_response = s.get(f"movies?channel_ids={channel_id}").json()
    for movie_index in range(len(movie_list_response)):
        print_log(f"channel:{channel_id}", f"downloading video {movie_index + 1} of {len(movie_list_response)}")
        dl_movie(s, movie_list_response[movie_index]["id"])

def dl_movie(s, movie_id):
    # check if we can skip
    if args.download_archive:
        if os.path.isfile(args.download_archive):
            with open(args.download_archive, "r") as archive_file:
                skip_ids = archive_file.read().splitlines()
            if movie_id in skip_ids:
                print_log(f"movie:{movie_id}", "already recorded in archive")
                return

    # get video data and check validity
    movie_response = s.get(f"movies/{movie_id}")
    if not movie_response.ok:
        print_log(f"info:{movie_id}", "failed to get movie information")
        print_log(f"info:{movie_id}", f"API response returned status code {movie_response.status_code}", LogLevel.VERBOSE)
        return
    m_json = movie_response.json()
    if "status" in m_json:
        print_log(f"info:{movie_id}", "failed to get movie information")
        print_log(f"info:{movie_id}", f"API body returned status code {m_json['status']}: {m_json['message']}")
        return

    # string to use in output video names
    movie_string = re.sub(CLEAN_FILENAME_KINDA, "_", f"{m_json['title']} [{m_json['id']}]").strip()

    # remove ad info because who cares about ads
    m_json.pop("ad")

    # api only gives public m3u8 by default. we dont care :^)
    # derive m3u8 link from json[media][url_public]
    # these links might not actually exist, so downloading relies on values from playlist.m3u8
    m_json["media"]["url"] = urllib.parse.urljoin(m_json["media"]["url_public"], "playlist.m3u8")
    m_json["media"]["url_audio"] = urllib.parse.urljoin(m_json["media"]["url_public"], "aac.m3u8")
    m_json["media"]["url_source"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_source/chunklist.m3u8")
    m_json["media"]["url_low_latency"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_low/chunklist.m3u8")
    m_json["media"]["url_ull"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_144p/chunklist.m3u8")

    formats_list = get_m3u8_info(m_json["media"]["url"])

    if args.list_formats:
        print_log(f"movie:{movie_id}", f"available formats:")
        print_formats(formats_list)
        return

    downloading_format = None
    for format_settings in formats_list:
        if args.format in {format_settings["media"]["NAME"], format_settings["media"]["GROUP-ID"]}:
            downloading_format = urllib.parse.urljoin(m_json["media"]["url_public"], format_settings["location"])

    if args.write_info_json:
        info_filename = f"{movie_string}.info.json"
        print_log(f"info:{movie_id}", f"writing video information to '{info_filename}'")
        with open(os.path.join(args.directory, f"{info_filename}.tmp"), "w") as movie_info:
            movie_info.write(json.dumps(m_json))
        os.rename(os.path.join(args.directory, f"{info_filename}.tmp"), os.path.join(args.directory, info_filename))

    if args.write_thumbnail:
        full_size_thumb_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, m_json["thumbnail_url"])
        thumbnail_format = urllib.parse.parse_qs(full_size_thumb_url)["format"][0]
        thumbnail_filename = f"{movie_string}.{thumbnail_format}"
        print_log(f"thumbnail:{movie_id}", f"writing thumbnail to '{thumbnail_filename}'")
        thumb_response = requests.get(full_size_thumb_url)
        if thumb_response.ok:
            with open(os.path.join(args.directory, f"{thumbnail_filename}.tmp"), "wb") as movie_thumbnail:
                movie_thumbnail.write(thumb_response.content)
            os.rename(os.path.join(args.directory, f"{thumbnail_filename}.tmp"), os.path.join(args.directory, thumbnail_filename))
        else:
            print_log(f"thumbnail:{movie_id}", "failed to retrieve video thumbnail")
            print_log(f"thumbnail:{movie_id}", f"API response returned status code {thumb_response.status_code}", LogLevel.VERBOSE)

    if args.write_live_chat:
        dl_live_chat(s, movie_id, movie_string, m_json["started_at"])

    if not args.skip_download:
        if downloading_format:
            dl_m3u8_video(movie_id, movie_string, downloading_format)
        else:
            print_log(f"movie:{movie_id}", f"could not find video format '{args.format}'. to view all available formats, use --list-formats")
    else:
        print_log(f"movie:{movie_id}", f"skipping download")

def dl_m3u8_video(movie_id, movie_filename, m3u8_link):
    movie_path = os.path.join(args.directory, f"{movie_filename}")
    # don't re-download
    if os.path.isfile(f"{movie_path}.ts") or os.path.isfile(f"{movie_path}.mp4"):
        print_log(f"movie:{movie_id}", f"already downloaded")
    # can't resume downloads (atm), so remove progress
    else:
        if os.path.isfile(f"{movie_path}.ts.tmp"):
            os.remove(f"{movie_path}.ts.tmp")
            for seg_file in os.listdir(args.directory):
                if re.fullmatch(r"^" + f"{re.escape(movie_filename)}" + r"\.seg[0-9]{1,}$", seg_file):
                    os.remove(seg_file)

        # get necessary variables ready
        playlist_base = urllib.parse.urljoin(m3u8_link, ".")
        m3u8_text = requests.get(m3u8_link).text
        ts_list = [n for n in m3u8_text.splitlines() if n and not n.startswith("#")]
        ordered_ts_list = list(zip(ts_list, [n for n in range(len(ts_list))]))

        # run the downloader
        print_log(f"movie:{movie_id}", f"writing video to '{movie_filename}.ts'")
        stream_downloader = StreamDownloader(playlist_base)
        download_bar = DownloadBar(f"[movie:{movie_id}]", max=len(ts_list))
        stream_downloader.run(movie_filename, ordered_ts_list, download_bar)

        # if success, check if converting
        if stream_downloader.success:
            download_bar.finish()
            os.rename(f"{movie_path}.ts.tmp", f"{movie_path}.ts")
            if not args.skip_convert:
                mpeg_convert(os.path.join(args.directory, f"{movie_filename}"))
        else:
            print_log(f"movie:{movie_id}", f"failed to download")
            return
    if args.download_archive:
        with open(args.download_archive, "a") as archive_file:
            archive_file.write(f"{movie_id}\n")

def mpeg_convert(file_path):
    print_log("mpeg-convert", f"converting video to '{movie_filename}.mp4'")
    ffmpeg_list = ["ffmpeg", "-i", f"{file_path}.ts", "-acodec", "copy", "-vcodec", "copy", f"{file_path}.mp4"]
    try:
        ffmpeg_process = Popen(ffmpeg_list, stdout=PIPE, stderr=PIPE)
        stdout, stderr = ffmpeg_process.communicate()
    except Exception:
        print_log("mpeg-convert", "failure in executing ffmpeg")
        return
    os.remove(f"{file_path}.ts")

# based on parts of https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py
def get_m3u8_info(playlist_link):
    m3u8_info = []
    m3u8_text = requests.get(playlist_link).text
    media_details = None
    format_details = None
    for line in m3u8_text.splitlines():
        if line.startswith("#EXT-X-MEDIA:"):
            media_details = parse_m3u8_attributes(line)
            #media_type, group_id, name = media.get("TYPE"), media.get("GROUP-ID"), media.get("NAME")
        elif line.startswith("#EXT-X-STREAM-INF:"):
            format_details = parse_m3u8_attributes(line)
        elif not line.startswith("#"):
            if line.endswith(".m3u8"):
                if not media_details:
                    print_log("get-m3u8-info", f"could not find media details for stream with format '{str(format_details)}'", LogLevel.VERBOSE)
                elif not format_details:
                    print_log("get-m3u8-info", f"could not find format details for stream with media '{str(media_details)}'", LogLevel.VERBOSE)
                else:
                    m3u8_info += [{"location": line, "media": media_details, "format": format_details}]
                    media_details = None
                    format_details = None
            else:
                print_log("get-m3u8-info", f"unexpected line in m3u8 file: '{line}'", LogLevel.VERBOSE)
    return m3u8_info

# https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py#L5495
def parse_m3u8_attributes(attrib):
    info = {}
    for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
        if val.startswith("\""):
            val = val[1:-1]
        info[key] = val
    return info

def print_formats(formats_list):
    print(f"{'NAME':<7} {'GROUP-ID':<8} {'RESOLUTION':<10} {'FPS':<4} {'TBR':<6} {'CODECS':<24}")
    print(f"{'-' * 7} {'-' * 8} {'-' * 10} {'-' * 4} {'-' * 6} {'-' * 24}")
    for format_settings in formats_list:
        print(f"{format_settings['media']['NAME']:<7} " +
        f"{format_settings['media']['GROUP-ID']:<8} " +
        f"{format_settings['format']['RESOLUTION']:<10} " +
        f"{format_settings['format']['FRAME-RATE']:<4} " +
        f"{str(int(float(format_settings['format']['BANDWIDTH']) / 1000))+'k':<6} " +
        f"{format_settings['format']['CODECS']:<24}")

def dl_live_chat(s, movie_id, movie_filename, started_at):
    live_chat_filename = os.path.join(args.directory, f"{movie_filename}.live_chat.json")
    if os.path.isfile(live_chat_filename):
        print_log(f"live-chat:{movie_id}", "already downloaded")
        return
    # no resuming downloads, so just remove the temp
    elif os.path.isfile(f"{live_chat_filename}.tmp"):
        os.remove(f"{live_chat_filename}.tmp")

    # read datetime object and remove utc offset for using in the chat API
    chat_dt = datetime.strptime(started_at, "%Y-%m-%dT%H:%M:%S%z")
    chat_dt = (chat_dt - chat_dt.utcoffset()).replace(tzinfo=None)
    chat_response = s.get(f"movies/{movie_id}/chats?from_created_at={chat_dt.isoformat()}.000Z&is_including_system_message=false")

    print_log(f"live-chat:{movie_id}", f"writing live chat to {movie_filename}.live_chat.json")
    with open(f"{live_chat_filename}.tmp", "w") as live_chat_file:
        # loop until blank response
        while chat_response.ok and len(chat_response.json()) > 0:
            last_post_time = datetime.strptime(chat_response.json()[-1]["posted_at"], "%Y-%m-%dT%H:%M:%S%z")
            last_post_time = (last_post_time - last_post_time.utcoffset()).replace(tzinfo=None)
            # stop if this is the same date as previously requested
            if last_post_time.isoformat() == chat_dt.isoformat():
                break
            for chat_line in chat_response.json():
                live_chat_file.write(f"{json.dumps(chat_line)}\n")
            # use timestamp of last chat post retrieved to fill next url
            chat_dt = last_post_time
            chat_response = s.get(f"movies/{movie_id}/chats?from_created_at={chat_dt.isoformat()}.000Z&is_including_system_message=false")
    if not chat_response.ok:
        print_log(f"live-chat:{movie_id}", f"unexpected ending with API response status code {chat_response.status_code}", LogLevel.VERBOSE)
    os.rename(f"{live_chat_filename}.tmp", live_chat_filename)

def print_log(component, message, level=LogLevel.BASIC):
    if level == LogLevel.VERBOSE and not args.verbose:
        return
    print(f"[{component}] {message}")

def get_arguments():
    parser.add_argument("--version", action="store_true", help="print version string and exit")
    parser.add_argument("-V", "--verbose", action="store_true", help="print debugging information")
    parser.add_argument("-d", "--directory", type=str, help="save directory (defaults to current)", default=os.getcwd())
    parser.add_argument("--download-archive", metavar="FILE", type=str, help="download only videos not listed in the archive file and record the IDs of downloaded videos")
    parser.add_argument("--write-info-json", action="store_true", help="write metadata to .info.json file")
    parser.add_argument("--write-thumbnail", action="store_true", help="write thumbnail to image file")
    parser.add_argument("--write-live-chat", action="store_true", help="write live chat comments to .live_chat.json file")
    parser.add_argument("-f", "--format", type=str, help="video format, specified by either NAME or GROUP-ID (defaults to Source)", default="Source")
    parser.add_argument("-F", "--list-formats", action="store_true", help="print available format details for a video and exit")
    parser.add_argument("--skip-download", action="store_true", help="do not download the video")
    parser.add_argument("--skip-convert", action="store_true", help="do not use ffmpeg to convert the MPEG-TS stream to MPEG-4")
    parser.add_argument("links", metavar="LINK", nargs="*", help="openrec channel or video link(s)/ids")
    return parser.parse_args()

def main():
    api_session = sessions.BaseUrlSession(base_url=API)
    if args.version:
        print(VERSION_STRING)
        return
    if len(args.links) == 0:
        parser.print_usage()
    elif not os.path.isdir(args.directory):
        os.makedirs(args.directory)
    for link in args.links:
        # is openrec link
        openrec_m = re.search(OPENREC, link)
        id_m = re.search(VALID_LIVE_ID, link)
        if openrec_m:
            t = openrec_m.group("type")
            if t == "user":
                dl_channel(api_session, openrec_m.group("id"))
            elif t == "live":
                dl_movie(api_session, openrec_m.group("id"))
            else:
                print_log("openrec", f"unknown link type \'{t}\'")
        elif id_m:
            dl_movie(api_session, id_m.group("id"))
        else:
            print_log("openrec", f"invalid link or id \'{link}\'")

parser = argparse.ArgumentParser()
args = get_arguments()

if __name__ == "__main__":
    main()


#!/usr/bin/python
#-*- coding: UTF-8 -*-
import json
import time
import uuid
import random
import datetime
from django.http import HttpResponse
from django.shortcuts import render, redirect
from django.utils.translation import ugettext as _
from operator import itemgetter, attrgetter

from cloudengine.logger import logger
from cloudengine.models  import *
from cloudengine.utility import *
from cloudengine.sshapi  import *
from cloudengine.ldapapi import *
from cloudengine.virtapi import *
from cloudengine.license import checkLicense

def course_show(request):
    logger.info("Entering course_show...")
    username = request.session.get("username", "")
    userrole = request.session.get("userrole", "")
    fullname = request.session.get("fullname", "")
    logger.info("username: %s, userrole: %s, fullname: %s" % (username, userrole, fullname))

    roledict = {'admin':'管理员', 'teacher':'教师', 'student':'学生'}
    rolename = roledict.get(userrole, '')

    ecourses = []
    user = User.objects.get(tusername=username)
    for course in ECourse.objects.all():
        for tumap in ETUMap.objects.filter(euser=user):
            task = ETask.objects.get(etuuid=tumap.etask.etuuid)
            if course.ecuuid == task.ecourse.ecuuid:
                ecourse = {}
                ecourse['cuuid'] = course.ecuuid
                ecourse['cname'] = course.ecname
                ecourse['cdescription'] = course.ecdescription
                ecourses.append(ecourse)
                break

    lcode, lcount, ldays  = checkLicense()
    if lcount < 2:
        lcount = 2
    if ldays != 0 and compute_used_days() > ldays:
        lcount = 2

    total = len(ecourses)
    if total > lcount:
       total = lcount
    ecourses = ecourses[0:total]

    ecourses.sort(lambda x,y : cmp(x['cname'], y['cname']))
    logger.info(ecourses)
    return render(request, 'virtlab/course_show.html', {'ecourses': json.dumps(ecourses), 'username': username, 'userrole': userrole, 'fullname': fullname, 'rolename': rolename})

def task_show(request):
    logger.info("Entering task_show..")
    ecuuid   = request.GET.get('cuuid', '')
    username = request.session.get("username", "")
    userrole = request.session.get("userrole", "")
    fullname = request.session.get("fullname", "")
    logger.info("cuuid: %s, username: %s, userrole: %s, fullname: %s" % (ecuuid, username, userrole, fullname))

    roledict = {'admin':'管理员', 'teacher':'教师', 'student':'学生'}
    rolename = roledict.get(userrole, '')

    etasks = []
    course = ECourse.objects.get(ecuuid=ecuuid)
    ecourse = {}
    ecourse['cuuid'] = course.ecuuid
    ecourse['cname'] = course.ecname
    ecourse['cdescription'] = course.ecdescription

    user = User.objects.get(tusername=username)
    for task in ETask.objects.filter(ecourse=course):
        if len(ETUMap.objects.filter(etask=task, euser=user)) == 0:
            continue
        etask = {}
        etask['tuuid'] = task.etuuid
        etask['tname'] = task.etname
        etask['tdescription'] = task.etdescription
        etask['tsn']   = task.sn
        etasks.append(etask)
    etasks.sort(lambda x,y : cmp(x['tsn'], y['tsn']))

    logger.info(etasks)
    return render(request, 'virtlab/task_show.html', {'ecourse':json.dumps(ecourse), 'etasks': json.dumps(etasks), 'username': username, 'userrole': userrole, 'fullname': fullname, 'rolename': rolename})


import io, tempfile, unittest
from bibliopixel.util import json
from bibliopixel.util.persistent_dict import PersistentDict

JSON_TEST = """
{
  "a": {"foo": "bar", "bang": 1}
}
"""

class PersistentDictTest(unittest.TestCase):
    def test_reader_writer_json(self):
        with tempfile.NamedTemporaryFile('w') as tf:
            tf.write(JSON_TEST)
            tf.seek(0)

            pd = PersistentDict(tf.name)
            self.assertEqual(pd, {'a': {'foo': 'bar', 'bang': 1}})
            pd.clear()
            self.assertEqual(pd, {})
            self.assertEqual(json.load(tf.name), pd)

            pd.update(bang={'hi': 'there'})
            self.assertEqual(pd, {'bang': {'hi': 'there'}})
            self.assertEqual(json.load(tf.name), pd)


# -*- coding:utf-8 -*-
# ========================================================== #
# File name: vgg_19.py
# Author: BIGBALLON
# Date created: 07/22/2017
# Python Version: 3.5.2
# Description: implement vgg19 network to train cifar10
# Result: test accuracy about 93.28% - 93.32%
# ========================================================== #

import time
import math
import scipy.misc
import numpy as np
import tensorflow as tf
from loadface import *
from vggnet import *
from pyheatmap.heatmap import HeatMap
from visualize_filters import *

# 3641 165 7283
iterations      = 3641
batch_size      = 128
crop_size       = 256
total_epoch     = 200
weight_decay    = 0.0003
dropout_rate    = 0.5
momentum_rate   = 0.9
log_save_path   = './vgg_logs'
model_save_path = './model/test'
shuffle_sign    = False

restore = False
ckpt = 35

train_switch    = False

total           = 487168
feature_data = []

# ========================================================== #
# ├─ _random_crop()
# ├─ _random_flip_leftright()
# ├─ data_augmentation()
# ├─ data_preprocessing()
# └─ learning_rate_schedule()
# ========================================================== #

def data_preprocessing(x_train,x_test):

    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')

    x_train[:,:,:,0] = (x_train[:,:,:,0] - np.mean(x_train[:,:,:,0])) / np.std(x_train[:,:,:,0])
    x_train[:,:,:,1] = (x_train[:,:,:,1] - np.mean(x_train[:,:,:,1])) / np.std(x_train[:,:,:,1])
    x_train[:,:,:,2] = (x_train[:,:,:,2] - np.mean(x_train[:,:,:,2])) / np.std(x_train[:,:,:,2])

    x_test[:,:,:,0] = (x_test[:,:,:,0] - np.mean(x_test[:,:,:,0])) / np.std(x_test[:,:,:,0])
    x_test[:,:,:,1] = (x_test[:,:,:,1] - np.mean(x_test[:,:,:,1])) / np.std(x_test[:,:,:,1])
    x_test[:,:,:,2] = (x_test[:,:,:,2] - np.mean(x_test[:,:,:,2])) / np.std(x_test[:,:,:,2])

    return x_train, x_test

def learning_rate_schedule(epoch_num):

    return 0.1 * (0.96 ** epoch_num)

def run_testing(sess,ep):
    acc = 0.0
    loss = 0.0
    pre_index = 0
    add = 165
    for it in range(165):
        # batch_x = test_x[pre_index:pre_index+add]
        # batch_y = test_y[pre_index:pre_index+add]
        # pre_index = pre_index + add
        batch_x, batch_y = sess.run([test_batch_image, test_batch_labels])
        # batch_x, batch_y = sess.run([train_batch_image, train_batch_labels])
        loss_, acc_  = sess.run([cross_entropy,accuracy],feed_dict={x:batch_x, y_:batch_y, keep_prob: 1.0, train_flag: False})
        loss += loss_ / 165.0
        acc += acc_ / 165.0
    summary = tf.Summary(value=[tf.Summary.Value(tag="test_loss", simple_value=loss),
                            tf.Summary.Value(tag="test_accuracy", simple_value=acc)])
    return acc, loss, summary

# ========================================================== #
# ├─ main()
# Training and Testing
# Save train/teset loss and acc for visualization
# Save Model in ./model
# ========================================================== #

def generate_heatmap(array, percentile=0.5, sign=''):
    per = np.percentile(array, 1-percentile)
    height, width = array.shape
    hdata = []
    for i in range(height):
        for j in range(width):
            if (array[i][j] > per):
                hdata.append([i,j])
    hm = HeatMap(hdata)
    hm.clickmap(save_as='hit'+str(sign)+'.png')
    hm.heatmap(save_as='heat'+str(sign)+'.png')

def generate_picture(array, sign=''):
    # scipy.misc 会自动标准化图像
    amin, amax = array.min(), array.max()
    array = (array-amin)/(amax-amin)*255
    array.astype(np.uint8)
    scipy.misc.imsave('picture'+str(sign)+'.png', array)

def generate_path_picture(array, path):
    # scipy.misc 会自动标准化图像
    amin, amax = array.min(), array.max()
    array = (array-amin)/(amax-amin)*255
    array.astype(np.uint8)
    scipy.misc.imsave(path, array)

def get_feature(sess, image, layer, filter):
    feature = sess.run(layer, feed_dict={x:np.expand_dims(image, axis = 0), keep_prob: 1.0, train_flag: False})[0]
    return feature[:,:,filter]

differ = []
def modify_midlayer(sess, image, layer, filter=0, block_size=1, stride=1, percentile=0.1):
    height, width, _ = image.shape
    feature = get_feature(sess, image, layer, filter)
    # print(feature.shape, feature)

    details = True

    i,ii,count = 0,0,0
    while (i + block_size - 1 < height):
        j,jj = 0,0
        while (j + block_size - 1 < width):

            temp = image.copy()
            temp[i:i+block_size,j:j+block_size,:] = 255

            tem_feature = get_feature(sess, temp, layer, filter)
            differ_graph = feature - tem_feature
            differ.append(differ_graph)

            count += 1
            if (details and count % 10 == 0):
                scipy.misc.imsave('differ/'+str(ii)+'_'+str(jj)+'.png', temp)
                generate_path_picture(differ_graph, 'differ/'+str(ii)+'_'+str(jj)+'diff'+'.png')

            print("%.2f %%" % float(i*100/height), end='\r')
            j += stride
            jj += 1
        i += stride
        ii += 1

    res = sum(differ)
    # print(type(res))
    generate_picture(res, 'sum_differ')
    # generate_heatmap(res, percentile, 1)

def modify_acc(sess, image, label, block_size=1, stride=1, percentile=0.1):
    height, width, _ = image.shape
    feature = sess.run(prob, feed_dict={x:np.expand_dims(image, axis = 0), keep_prob: 1.0, train_flag: False})[0]
    print('the example predict is ', np.equal(np.argmax(feature), label))
    # print(feature.shape, feature)
    #import pdb
    #pdb.set_trace()
    h = math.ceil((height-block_size)/stride)+1
    w = math.ceil((width-block_size)/stride)+1
    hdata = np.zeros((h, w))
    i,ii = 0,0
    while (i + block_size - 1 < height):
        j,jj = 0,0
        while (j + block_size - 1 < width):

            temp = image.copy()
            temp[i:i+block_size,j:j+block_size,:] = 255

            tem_feature = sess.run(prob, feed_dict={x:np.expand_dims(temp, axis = 0), keep_prob: 1.0, train_flag: False})[0]
            p = tem_feature[label]

            predict = np.argmax(tem_feature)
            # print(label, predict, p)
            # print(tem_feature)
            # input()
            hdata[ii][jj] = p

            print("%.2f %%" % float(i*100/height), end='\r')
            j += stride
            jj += 1
        i += stride
        ii += 1

    # import pdb
    # pdb.set_trace()
    generate_path_picture(hdata, 'acc/'+str(block_size)+'_'+str(stride)+'.png')
    generate_picture(hdata,'acc')
    # generate_heatmap(hdata, percentile)

def visual_filter(sess, layer, channel):
    feature = sess.run(layer)
    # print(len(feature), type(feature[0]), feature[0].shape)
    # input()
    height, width, chanel, filt = feature.shape
    # print(feature.shape)
    if chanel == 3:
        for i in range(filt):
            filter_ = feature[:,:,:,i]
            generate_path_picture(filter_, 'filter/'+layer.name+'__'+str(i)+'_'+'.png')
    else:
        for i in range(filt):
            for j in range(chanel):
                filter_ = feature[:,:,j,i]
                generate_path_picture(filter_, 'filter/'+layer.name+'__'+str(i)+'_'+str(j)+'.png')

if __name__ == '__main__':

    train_image, train_label = decode_from_tfrecords('/home/code/lichangzhen/data/data_shuffle/vggnet/train/dataset.tfrecords')
    train_batch_image, train_batch_label = get_batch(train_image, train_label, batch_size, crop_size, shuffle_sign)        #batch 生成测试
    train_batch_labels = tf.one_hot(train_batch_label, 10575)
    test_image, test_label   = decode_from_tfrecords('/home/code/lichangzhen/data/data_shuffle/vggnet/test/dataset.tfrecords')
    test_batch_image, test_batch_label   = get_batch(test_image, test_label, batch_size, crop_size, shuffle_sign)       #batch 生成测试
    test_batch_labels  = tf.one_hot(test_batch_label, 10575)

    # define placeholder x, y_ , keep_prob, learning_rate
    x  = tf.placeholder(tf.float32,[None, crop_size, crop_size, 3])
    y_ = tf.placeholder(tf.float32, [None, 10575])
    keep_prob = tf.placeholder(tf.float32)
    learning_rate = tf.placeholder(tf.float32)
    train_flag = tf.placeholder(tf.bool)

    output, filt1, filt2, last_feature = inference(x,train_flag,keep_prob)

    # output = alexnet_v2(inputs=x,
    #            num_outputs=10575,
    #            dropout_keep_prob=keep_prob,
    #            is_training=train_flag
    #            # reuse=False,
    #            # scope=None
    #            )

    prob = tf.nn.softmax(output)

    # loss function: cross_entropy
    # train_step: training operation
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=output))
    l2 = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()])
    train_step = tf.train.MomentumOptimizer(learning_rate, momentum_rate,use_nesterov=True).minimize(cross_entropy + l2 * weight_decay)

    correct_prediction = tf.equal(tf.argmax(output,1), tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))

    # initial an saver to save model
    saver = tf.train.Saver()

    config = tf.ConfigProto(allow_soft_placement=True,
                            log_device_placement=False)
    config.gpu_options.allow_growth = True

    with tf.Session(config=config) as sess:

        sess.run(tf.global_variables_initializer())

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        if train_switch:

            summary_writer = tf.summary.FileWriter(log_save_path,sess.graph)
            # epoch = 164
            # make sure [bath_size * iteration = data_set_number]
            ep = 0
            for _ in range(1,total_epoch+1):
                ep += 1
                if restore:
                    restore = False
                    saver.restore(sess, model_save_path + str(ckpt))
                    ep = ckpt
                    print('strat from epoch %d' % (ckpt+1) )
                    continue
                lr = learning_rate_schedule(ep)
                pre_index = 0
                train_acc = 0.0
                train_loss = 0.0
                start_time = time.time()

                print("\nepoch %d/%d:" %(ep,total_epoch))

                for it in range(1,iterations+1):
                    # batch_x = train_x[pre_index:pre_index+batch_size]
                    # batch_y = train_y[pre_index:pre_index+batch_size]

                    # batch_x = data_augmentation(batch_x)
                    batch_x, batch_y = sess.run([train_batch_image, train_batch_labels])

                    # print((sess.run(output5_2,feed_dict={x:batch_x, y_:batch_y, keep_prob: dropout_rate, learning_rate: lr, train_flag: True})).shape)
                    # input()
                    _, batch_loss = sess.run([train_step, cross_entropy],feed_dict={x:batch_x, y_:batch_y, keep_prob: dropout_rate, learning_rate: lr, train_flag: True})
                    batch_acc = accuracy.eval(feed_dict={x:batch_x, y_:batch_y, keep_prob: 1.0, train_flag: True})

                    train_loss += batch_loss
                    train_acc  += batch_acc
                    pre_index  += batch_size

                    if it == iterations:
                        train_loss /= iterations
                        train_acc /= iterations

                        loss_, acc_  = sess.run([cross_entropy,accuracy],feed_dict={x:batch_x, y_:batch_y, keep_prob: 1.0, train_flag: True})
                        train_summary = tf.Summary(value=[tf.Summary.Value(tag="train_loss", simple_value=train_loss),
                                              tf.Summary.Value(tag="train_accuracy", simple_value=train_acc)])

                        val_acc, val_loss, test_summary = run_testing(sess,ep)

                        summary_writer.add_summary(train_summary, ep)
                        summary_writer.add_summary(test_summary, ep)
                        summary_writer.flush()

                        print("iteration: %d/%d, cost_time: %ds, train_loss: %.4f, train_acc: %.4f, test_loss: %.4f, test_acc: %.4f" %(it, iterations, int(time.time()-start_time), train_loss, train_acc, val_loss, val_acc))
                    else:
                        print("iteration: %d/%d, train_loss: %.4f, train_acc: %.4f" %(it, iterations, train_loss / it, train_acc / it) , end='\r')

                if ep % 5 == 0:
                    save_path = saver.save(sess, model_save_path + str(ep))
                    print("Model saved in file: %s" % save_path)

        else:
            saver.restore(sess, model_save_path + str(ckpt))
            print('load model complete')

            start = time.time()
            print('Start Manipulation DATA:')

            index = -1
            # 1000 * 128 + 100 - 1

            for it in range(3641):
                batch_x, batch_y = sess.run([train_batch_image, train_batch_labels])
                feature = sess.run(last_feature,feed_dict={x:batch_x, keep_prob: 1.0, train_flag: False})

                for ii in range(batch_size):
                    index += 1

                    feature_data.append(feature[ii])

                    if (index % 89 == 0):
                        print("loading : %.3f" %(index/total) , end='\r')

            for it in range(165):
                batch_x, batch_y = sess.run([test_batch_image, test_batch_labels])
                feature = sess.run(last_feature,feed_dict={x:batch_x, keep_prob: 1.0, train_flag: False})

                for ii in range(batch_size):
                    index += 1

                    feature_data.append(feature[ii])

                    if (index % 89 == 0):
                        print("loading : %.3f" %(index/total) , end='\r')

            np.save('feature.npy', feature_data)
            start = time.time() - start
            print("Total Time = %f, operations is completed" % start)

            # for i in range(100):

            #     im = batch_x[i]
            #     label = np.argmax(batch_y[i])

            #     scipy.misc.imsave('current_picture.png', im)
            #     print('label ', label)
            #     input()

            # im = scipy.misc.imread('example.png')
            # label = 5132

            # start = time.time()

            # visualize_filters(sess=sess, layer=filt1)

            # # visual_filter(sess=sess, layer=filt1, channel=0)
            # # modify_midlayer(sess=sess, image=im, layer=output1, filter=10, block_size=15, stride=1, percentile=0.05)
            # # print('/****************************************************************************/')
            # # for i in range(1,50):
            # #     for j in range(1,10):
            # #         modify_acc(sess=sess, image=im, label=label, block_size=i, stride=j, percentile=0.01)

            # start = time.time() - start
            # print("Total Time = %f" % (start))

        coord.request_stop()
        coord.join(threads)


from django import forms
from django.utils.translation import gettext, gettext_lazy as _
from django.forms.widgets import TextInput, Select

from files.models import File

class MultiUploadForm(forms.ModelForm):

    class Meta:
        model = File
        fields = [
            'upload_file',
        ]

class FileUpdateForm(forms.ModelForm):
    real_filename = forms.CharField(
        label=_('New file name'),
        max_length=60,
        widget=forms.TextInput(attrs={'autofocus': True, 'class': 'form-control form-control', 'placeholder': _('New file name')}),
    )

    class Meta:
        model = File
        fields = [
            'real_filename',
        ]


# uncompyle6 version 3.7.4
# Python bytecode 3.4 (3310)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/ale/progr/github/wasp-general/wasp_general/task/scheduler/task_source.py
# Compiled at: 2017-10-10 07:17:45
# Size of source mod 2**32: 15841 bytes
from wasp_general.version import __author__, __version__, __credits__, __license__, __copyright__, __email__
from wasp_general.version import __status__
from datetime import datetime, timedelta, MAXYEAR, timezone
from calendar import monthrange, weekday
from wasp_general.verify import verify_type, verify_value
from wasp_general.datetime import utc_datetime
from wasp_general.thread import WCriticalResource
from wasp_general.task.scheduler.proto import WTaskSourceProto, WScheduleRecord, WScheduleTask, WSchedulerServiceProto

class WBasicTaskSource(WTaskSourceProto):

    @verify_type(scheduler=WSchedulerServiceProto)
    def __init__(self, scheduler_service):
        self._WBasicTaskSource__service = scheduler_service

    def scheduler_service(self):
        return self._WBasicTaskSource__service

class WCronSchedule:
    __calendar__ = {1: 31,  2: 29,  3: 31,  4: 30,  5: 31,  6: 30,  7: 31,  8: 31,  9: 30,  10: 31,  11: 30,  12: 31}

    @verify_type(start_datetime=(datetime, None), minute=(int, None), hour=(int, None), day_of_month=(int, None))
    @verify_type(day_of_week=(int, None), month=(int, None))
    @verify_value(minute=lambda x: x is None or 0 <= x <= 59)
    @verify_value(hour=lambda x: x is None or 0 <= x <= 23)
    @verify_value(day_of_month=lambda x: x is None or 1 <= x <= 31)
    @verify_value(day_of_week=lambda x: x is None or 1 <= x <= 7)
    @verify_value(month=lambda x: x is None or 1 <= x <= 12)
    def __init__(self, start_datetime=None, minute=None, hour=None, day_of_month=None, day_of_week=None, month=None):
        if month is not None:
            if day_of_month is not None and day_of_month > WCronSchedule.__calendar__[month]:
                raise ValueError('Invalid day and month combination')
        self._WCronSchedule__start_datetime = start_datetime if start_datetime is not None else self.now()
        self._WCronSchedule__next_start = None
        self._WCronSchedule__minute = minute
        self._WCronSchedule__hour = hour
        self._WCronSchedule__day_of_month = day_of_month
        self._WCronSchedule__day_of_week = day_of_week
        self._WCronSchedule__month = month
        self.update()

    def start_datetime(self):
        return self._WCronSchedule__start_datetime

    @verify_type(start_datetime=datetime)
    def _set_start_datetime(self, start_datetime):
        self._WCronSchedule__start_datetime = start_datetime
        self._WCronSchedule__next_start = None

    @verify_type('paranoid', year=int, month=int, day=int, hour=int, minute=int)
    @verify_value('paranoid', year=lambda x: x > 0)
    @verify_value('paranoid', month=lambda x: x is None or 1 <= x <= 12)
    @verify_value('paranoid', day=lambda x: x is None or 1 <= x <= 31)
    @verify_value('paranoid', hour=lambda x: x is None or 0 <= x <= 23)
    @verify_value('paranoid', minute=lambda x: x is None or 0 <= x <= 59)
    def _datetime(self, year, month, day, hour, minute):
        return datetime(year, month, day, hour, minute)

    def next_start(self):
        return self._WCronSchedule__next_start

    def minute(self):
        return self._WCronSchedule__minute

    def hour(self):
        return self._WCronSchedule__hour

    def day_of_month(self):
        return self._WCronSchedule__day_of_month

    def day_of_week(self):
        return self._WCronSchedule__day_of_week

    def month(self):
        return self._WCronSchedule__month

    def no_frequency(self):
        result = self.minute() is None and self.hour() is None and self.day_of_month() is None
        return result is True and self.day_of_week() is None and self.month() is None

    def final_datetime(self):
        return datetime(MAXYEAR, 12, 31, 23, 59)

    @classmethod
    def now(cls):
        return datetime.now()

    def update(self):
        if self.no_frequency() is True:
            return
        for year, month in self.month_iterator():
            for day in self.day_iterator(year, month):
                for hour, minute in self.time_iterator(year, month, day):
                    self._WCronSchedule__next_start = self._datetime(year, month, day, hour, minute)
                    return

    @verify_type(omit_skipped=bool)
    def complete(self, omit_skipped=True):
        if self.no_frequency() is True:
            return
        next_start = self.next_start()
        seconds_last_minute = next_start.second
        seconds_last_hour = next_start.minute * 60 + seconds_last_minute
        seconds_last_day = next_start.hour * 24 * 60 * 60 + seconds_last_hour
        time_shift = 0
        if self.month() is not None:
            year = next_start.year
            month = next_start.month
            if month < 12:
                month += 1
            else:
                year += 1
                month = 1
            time_shift = (self._datetime(year, month, 1, 0, 0) - next_start).total_seconds()
        else:
            if self.day_of_week() is not None:
                time_shift = 604800 - seconds_last_day
            else:
                if self.day_of_month() is not None:
                    time_shift = 86400 - seconds_last_day
                else:
                    if self.hour() is not None:
                        time_shift = 3600 - seconds_last_hour
                    elif self.minute() is not None:
                        time_shift = 60 - seconds_last_minute
            time_shift += 1
            new_start = next_start + timedelta(seconds=time_shift)
            if omit_skipped is True:
                now = self.now()
                if now > new_start:
                    new_start = now
        self._set_start_datetime(new_start)
        self.update()

    def month_iterator(self):
        if self.no_frequency():
            return
        start_datetime = self.start_datetime()
        start_year = start_datetime.year
        start_month = start_datetime.month
        final_datetime = self.final_datetime()
        max_year = final_datetime.year
        max_month = final_datetime.month
        month = self.month()
        year = start_year
        if month is not None:
            if month < start_month:
                year += 1
            if year > max_year or year == max_year and month > max_month:
                return
            while year < max_year or year == max_year and month < max_month:
                yield (
                 year, month)
                year += 1

        else:
            month = start_month
            while year < max_year or year == max_year and month < max_month:
                yield (
                 year, month)
                if month < 12:
                    month += 1
                else:
                    year += 1
                    month = 1

    def day_iterator(self, year, month):
        if self.no_frequency():
            return
        start_datetime = self.start_datetime()
        days_in_month = monthrange(year, month)[1]
        day_of_month = self.day_of_month()
        day_of_week = self.day_of_week()
        final_datetime = self.final_datetime()
        if year == final_datetime.year:
            if month == final_datetime.month:
                days_in_month = final_datetime.day
        start_day = start_datetime.day
        if year != start_datetime.year or month != start_datetime.month:
            start_day = 1
        if day_of_month is None and day_of_week is None:
            day = start_day
            while day <= days_in_month:
                yield day
                day += 1

        else:
            if day_of_month is not None and day_of_week is None:
                if start_day <= days_in_month:
                    yield day_of_month
            else:
                if day_of_month is not None and day_of_week is not None:
                    if start_day <= days_in_month:
                        if weekday(year, month, day_of_month) == day_of_week:
                            yield day_of_month
                else:
                    current_weekday = weekday(year, month, start_day)
                    delta = day_of_week - current_weekday
                    if delta < 0:
                        delta += 7
                    day = start_day + delta
                    while day <= days_in_month:
                        yield day
                        day += 7

    def time_iterator(self, year, month, day):
        start_datetime = self.start_datetime()
        max_hour = 23
        max_minute = 59
        hour = self.hour()
        minute = self.minute()
        final_datetime = self.final_datetime()
        if year == final_datetime.year:
            if month == final_datetime.month and day == self.final_datetime():
                max_hour = final_datetime.hour
                max_minute = final_datetime.minute
        start_hour = start_datetime.hour
        start_minute = start_datetime.minute
        if year != start_datetime.year or month != start_datetime.month or day != start_datetime.day:
            start_hour = 0
            start_minute = 0
        if hour is None and minute is None:
            hour = start_hour
            minute = start_minute
            while (start_hour <= hour < max_hour or hour) == max_hour:
                if start_minute <= minute <= max_minute:
                    yield (
                     hour, minute)
                    if minute < 59:
                        minute += 1
                else:
                    minute = 0
                    hour += 1

        else:
            if hour is not None and minute is None:
                minute = start_minute
                if start_hour <= hour <= max_hour:
                    while minute <= max_minute:
                        yield (
                         hour, minute)
                        minute += 1

            elif hour is not None and minute is not None:
                if not start_hour <= hour < max_hour:
                    pass
        if hour == max_hour:
            if start_minute <= minute <= max_minute:
                yield (
                 hour, minute)
        else:
            hour = start_hour
            if start_minute > minute:
                hour += 1
            while (start_hour <= hour < max_hour or hour) == max_hour:
                if start_minute <= minute <= max_minute:
                    yield (
                     hour, minute)
                    hour += 1

    @classmethod
    @verify_type(scheduled=str)
    def from_string(cls, scheduled):
        return cls.from_string_tokens(*filter(lambda x: len(x) > 0, scheduled.strip().split(' ')))

    @classmethod
    @verify_type(tokens=str)
    def from_string_tokens(cls, *tokens):
        if len(tokens) != 5:
            raise ValueError('Malformed cron-schedule')
        tokens = [int(x) if x != '*' else None for x in tokens]
        if len(tokens) != 5:
            raise ValueError('Malformed cron-schedule')
        return cls(cls.now(), minute=tokens[0], hour=tokens[1], day_of_month=tokens[2], day_of_week=tokens[3], month=tokens[4])

class WCronLocalTZSchedule(WCronSchedule):
    pass

class WCronUTCSchedule(WCronSchedule):

    @verify_type('paranoid', start_datetime=(datetime, None), minute=(int, None), hour=(int, None))
    @verify_type('paranoid', day_of_month=(int, None), day_of_week=(int, None), month=(int, None))
    @verify_value('paranoid', minute=lambda x: x is None or 0 <= x <= 59)
    @verify_value('paranoid', hour=lambda x: x is None or 0 <= x <= 23)
    @verify_value('paranoid', day_of_month=lambda x: x is None or 1 <= x <= 31)
    @verify_value('paranoid', day_of_week=lambda x: x is None or 1 <= x <= 7)
    @verify_value('paranoid', month=lambda x: x is None or 1 <= x <= 12)
    @verify_value(start_datetime=lambda x: x.tzinfo is not None and x.tzinfo == timezone.utc)
    def __init__(self, start_datetime=None, minute=None, hour=None, day_of_month=None, day_of_week=None, month=None):
        WCronSchedule.__init__(self, start_datetime, minute=minute, hour=hour, day_of_month=day_of_month, day_of_week=day_of_week, month=month)

    @verify_type(start_datetime=datetime)
    @verify_value(start_datetime=lambda x: x.tzinfo is not None and x.tzinfo == timezone.utc)
    def _set_start_datetime(self, start_datetime):
        WCronSchedule._set_start_datetime(self, start_datetime)

    @verify_type('paranoid', year=int, month=int, day=int, hour=int, minute=int)
    @verify_value('paranoid', year=lambda x: x > 0)
    @verify_value('paranoid', month=lambda x: x is None or 1 <= x <= 12)
    @verify_value('paranoid', day=lambda x: x is None or 1 <= x <= 31)
    @verify_value('paranoid', hour=lambda x: x is None or 0 <= x <= 23)
    @verify_value('paranoid', minute=lambda x: x is None or 0 <= x <= 59)
    def _datetime(self, year, month, day, hour, minute):
        return utc_datetime(datetime(year, month, day, hour, minute), local_value=False)

    @classmethod
    def now(cls):
        return utc_datetime()

class WCronScheduleRecord(WScheduleRecord):

    @verify_type('paranoid', task=WScheduleTask, task_group_id=(str, None))
    @verify_value('paranoid', on_drop=lambda x: x is None or callable(x))
    @verify_value('paranoid', on_wait=lambda x: x is None or callable(x))
    @verify_type(schedule=WCronSchedule, omit_skipped=bool)
    @verify_value(schedule=lambda x: isinstance(x, WCronLocalTZSchedule) or isinstance(x, WCronUTCSchedule))
    def __init__(self, cron_schedule, task, policy=None, task_group_id=None, on_drop=None, on_wait=None, omit_skipped=True):
        WScheduleRecord.__init__(self, task, policy=policy, task_group_id=task_group_id, on_drop=on_drop, on_wait=on_wait)
        self._WCronScheduleRecord__schedule = cron_schedule
        self._WCronScheduleRecord__omit_skipped = omit_skipped

    def cron_schedule(self):
        return self._WCronScheduleRecord__schedule

    def next_start(self):
        cron = self.cron_schedule()
        next_start = cron.next_start()
        if isinstance(cron, WCronLocalTZSchedule):
            return utc_datetime(dt=next_start)
        if isinstance(cron, WCronUTCSchedule):
            return next_start
        raise RuntimeError('Corrupted object!')

    def complete(self):
        self.cron_schedule().complete(omit_skipped=self._WCronScheduleRecord__omit_skipped)

class WCronTaskSource(WBasicTaskSource, WCriticalResource):

    @verify_type('paranoid', scheduler_service=WSchedulerServiceProto)
    def __init__(self, scheduler_service):
        WBasicTaskSource.__init__(self, scheduler_service=scheduler_service)
        WCriticalResource.__init__(self)
        self._WCronTaskSource__tasks = []
        self._WCronTaskSource__next_task = None

    @verify_type(record=WCronScheduleRecord)
    def add_record(self, record):
        self._WCronTaskSource__add_record(record)
        self.scheduler_service().update(task_source=self)

    @WCriticalResource.critical_section()
    @verify_type('paranoid', record=WCronScheduleRecord)
    def __add_record(self, record):
        self._WCronTaskSource__tasks.append(record)
        self._WCronTaskSource__update(record)

    @verify_type('paranoid', task=(WCronScheduleRecord, None))
    def __update(self, task=None):
        if task is not None:
            next_start = task.next_start()
            if self._WCronTaskSource__next_task is None or next_start < self._WCronTaskSource__next_task.next_start():
                self._WCronTaskSource__next_task = task
        else:
            if len(self._WCronTaskSource__tasks) > 0:
                next_task = self._WCronTaskSource__tasks[0]
                for task in self._WCronTaskSource__tasks[1:]:
                    if next_task.next_start() is None:
                        next_task = task
                    elif task.next_start() is not None and task.next_start() < next_task.next_start():
                        next_task = task
                        continue

                self._WCronTaskSource__next_task = next_task
            else:
                self._WCronTaskSource__next_task = None

    @WCriticalResource.critical_section()
    def tasks_planned(self):
        return len(self._WCronTaskSource__tasks)

    @WCriticalResource.critical_section()
    def has_records(self):
        if self._WCronTaskSource__next_task is not None:
            next_start = self._WCronTaskSource__next_task.next_start()
            if next_start is not None and next_start <= utc_datetime():
                result = [
                 self._WCronTaskSource__next_task]
                self._WCronTaskSource__next_task.complete()
                self._WCronTaskSource__update()
                return tuple(result)

    @WCriticalResource.critical_section()
    def next_start(self):
        if self._WCronTaskSource__next_task is not None:
            return self._WCronTaskSource__next_task.next_start()

class WInstantTaskSource(WBasicTaskSource, WCriticalResource):
    __lock_acquiring_timeout__ = 5

    @verify_type('paranoid', scheduler_service=WSchedulerServiceProto)
    def __init__(self, scheduler_service):
        WBasicTaskSource.__init__(self, scheduler_service=scheduler_service)
        WCriticalResource.__init__(self)
        self._WInstantTaskSource__records = []

    @verify_type('paranoid', record=WScheduleRecord)
    def add_record(self, record):
        self._WInstantTaskSource__add_record(record)
        self.scheduler_service().update(self)

    @verify_type(record=WScheduleRecord)
    @WCriticalResource.critical_section(timeout=__lock_acquiring_timeout__)
    def __add_record(self, record):
        self._WInstantTaskSource__records.append(record)

    @WCriticalResource.critical_section(timeout=__lock_acquiring_timeout__)
    def has_records(self):
        if len(self._WInstantTaskSource__records) > 0:
            result = self._WInstantTaskSource__records.copy()
            self._WInstantTaskSource__records = []
            return tuple(result)

    @WCriticalResource.critical_section(timeout=__lock_acquiring_timeout__)
    def next_start(self):
        if len(self._WInstantTaskSource__records) > 0:
            return utc_datetime()

    @WCriticalResource.critical_section(timeout=__lock_acquiring_timeout__)
    def tasks_planned(self):
        return len(self._WInstantTaskSource__records)


"""
Name: 'Eurocom Interchange File'
Blender: 2.90.1
Group: 'Export'
Tooltip: 'Blender EIF Exporter for EuroLand'
Authors: Swyter and Jmarti856
"""
import bpy
import os
import math
import datetime
import bmesh
from mathutils import *
from math import *
from pathlib import Path
from bpy_extras.io_utils import axis_conversion
from bpy import context
from pprint import pprint
from decimal import Decimal
from mathutils import Euler

def _write(context, filepath,
            EXPORT_GLOBAL_MATRIX,
        ):

    #===============================================================================================
    #  GLOBAL VARIABLES
    #===============================================================================================
    ProjectContextScene = bpy.context.scene

    # Axis Conversion
    InvertAxisRotationMatrix = Matrix(((1, 0, 0),(0, 0, 1),(0, 1, 0)))

    #===============================================================================================
    #  MAIN
    #===============================================================================================
    def WriteFile():
        # Stop edit mode
        if bpy.ops.object.mode_set.poll():
            bpy.ops.object.mode_set(mode='OBJECT')

        #Create new file
        with open(filepath, 'w') as out:
            #Start writing
            AmbientValue = bpy.context.scene.world.light_settings.ao_factor
            out.write('*EUROCOM_INTERCHANGE_FILE 100\n')
            out.write('*COMMENT Eurocom Interchange File Version 1.00 %s\n' % (datetime.datetime.utcnow()).strftime('%A %B %d %Y %H:%M'))
            out.write('\n')
            out.write('*OPTIONS {\n')
            out.write('\t*COORD_SYSTEM %s\n' % "LH")
            out.write('}\n')
            out.write('\n')
            out.write('*SCENE {\n')
            out.write('\t*FILENAME "%s"\n' % (bpy.data.filepath))
            out.write('\t*FIRSTFRAME %d\n' % (ProjectContextScene.frame_start))
            out.write('\t*LASTFRAME %d\n' % (ProjectContextScene.frame_end))
            out.write('\t*FRAMESPEED %d\n' % (ProjectContextScene.render.fps))
            out.write('\t*STATICFRAME %d\n' % (ProjectContextScene.frame_current))
            out.write('\t*AMBIENTSTATIC %.6f %.6f %.6f\n' %(AmbientValue, AmbientValue, AmbientValue))
            out.write('}\n')
            out.write('\n')

            #===============================================================================================
            #  MATERIAL LIST
            #===============================================================================================
            MaterialIndex = 0
            out.write('*MATERIALS {\n')
            for mat in bpy.data.materials:
                if hasattr(mat.node_tree, 'nodes'):
                    DiffuseColor = mat.diffuse_color
                    ImageNode = mat.node_tree.nodes.get('Image Texture', None)

                    #Material has texture
                    if (ImageNode is not None):
                        ImageName = ImageNode.image.name
                        out.write('\t*MATERIAL %d {\n' % (MaterialIndex))
                        out.write('\t\t*NAME "%s"\n' % (os.path.splitext(ImageName)[0]))
                        out.write('\t\t*COL_DIFFUSE %.6f %.6f %.6f\n' % (DiffuseColor[0], DiffuseColor[1], DiffuseColor[2]))

                        #Check if the texture exists
                        if (os.path.exists(bpy.path.abspath(ImageNode.image.filepath))):
                            out.write('\t\t*TWOSIDED\n')
                            out.write('\t\t*MAP_DIFFUSE "%s"\n' % (bpy.path.abspath(ImageNode.image.filepath)))
                        out.write('\t\t*MAP_DIFFUSE_AMOUNT 1.0\n')

                        #Check if use Alpha
                        if mat.blend_method.startswith('ALPHA'):
                            out.write('\t\t*MAP_HASALPHA\n')

                        MaterialIndex +=1
                        out.write('\t}\n')

                    #Material has no texture
                    else:
                        Color = DiffuseColor[0] + DiffuseColor[1] + DiffuseColor[2]
                        out.write('\t*MATERIAL %d {\n' % (MaterialIndex))
                        out.write('\t\t*NAME "%s"\n' % (mat.name))
                        out.write('\t\t*COL_DIFFUSE %.6f %.6f %.6f\n' % (DiffuseColor[0], DiffuseColor[1], DiffuseColor[2]))
                        out.write('\t}\n')
                        MaterialIndex +=1
            out.write('}\n')
            out.write('\n')

            #===============================================================================================
            #  MESH DATA
            #===============================================================================================
            for obj in ProjectContextScene.objects:
                if obj.type == 'MESH':
                    if hasattr(obj, 'data'):
                        #===========================================[Clone Object]====================================================
                        depsgraph = bpy.context.evaluated_depsgraph_get()
                        ob_for_convert = obj.evaluated_get(depsgraph)

                        try:
                            MeshObject = ob_for_convert.to_mesh()
                        except RuntimeError:
                            MeshObject = None

                        if MeshObject is None:
                            continue

                        #===========================================[Apply Matrix]====================================================
                        MeshObject.transform(EXPORT_GLOBAL_MATRIX)
                        MeshObject.flip_normals()

                        #===========================================[Get Object Data]====================================================
                        #Get vertex list without duplicates
                        VertexList = []
                        for ObjVertex in MeshObject.vertices:
                            VertexList.append(ObjVertex.co)

                        #Get UVs list without duplicates
                        UVList = []
                        if hasattr(MeshObject.uv_layers.active, 'data'):
                            if len(MeshObject.uv_layers):
                                for layer in MeshObject.uv_layers:
                                    uv_layer = layer.data
                                    for pl_count, poly in enumerate(MeshObject.polygons):
                                        for li_count, loop_index in enumerate(poly.loop_indices):
                                            UVList.append(
                                                (uv_layer[loop_index].uv.x, 1.0 - uv_layer[loop_index].uv.y)
                                            )
                                UVList = list(dict.fromkeys(UVList))

                        #Get vertex color list without duplicates
                        VertexColList = []
                        if len(MeshObject.vertex_colors):
                            if hasattr(MeshObject.vertex_colors.active, 'data'):
                                for layer in MeshObject.vertex_colors:
                                    for vertex in layer.data:
                                        VertexColList.append(
                                            (vertex.color[0] * .5, vertex.color[1] * .5, vertex.color[2] * .5, vertex.color[3])
                                        )
                                VertexColList = list(dict.fromkeys(VertexColList))

                        FaceFormat = 'V'
                        #===========================================[Print Object Data]====================================================
                        out.write('*MESH {\n')
                        out.write('\t*NAME "%s"\n' % obj.name)
                        out.write('\t*VERTCOUNT %d\n' % len(VertexList))
                        out.write('\t*UVCOUNT %d\n' % len(UVList))
                        out.write('\t*VERTCOLCOUNT %d\n' % len(VertexColList))
                        out.write('\t*FACECOUNT %d\n' % len(MeshObject.polygons))
                        out.write('\t*TRIFACECOUNT %d\n' % sum(len(p.vertices) - 2 for p in MeshObject.polygons))

                        if len(MeshObject.uv_layers) > 0:
                            out.write('\t*FACELAYERSCOUNT %d\n' % len(MeshObject.uv_layers))
                        else:
                            out.write('\t*FACELAYERSCOUNT %d\n' % 1)

                        if len(MeshObject.uv_layers) > 1:
                            out.write('\t*FACESHADERCOUNT %d\n' % len(MeshObject.material_slots))

                        #Print Vertex List
                        out.write('\t*VERTEX_LIST {\n')
                        for VertexData in VertexList:
                            out.write('\t\t%.6f %.6f %.6f\n' % (VertexData.x, VertexData.y, VertexData.z))
                        out.write('\t}\n')

                        #Print UV data
                        out.write('\t*UV_LIST {\n')
                        if (len(UVList) > 0):
                            FaceFormat += 'T'
                            for uv in UVList:
                                out.write('\t\t%.6f %.6f\n' % (uv[0], uv[1]))
                        out.write('\t}\n')

                        #Print Vertex Color List
                        out.write('\t*VERTCOL_LIST {\n')
                        if(len(VertexColList) > 0):
                            FaceFormat += 'C'
                            for col in VertexColList:
                                out.write('\t\t%.6f %.6f %.6f %.6f\n' % (col[0], col[1], col[2], col[3]))
                        out.write('\t}\n')

                        if len(obj.material_slots) > 0:
                            FaceFormat += 'M'
                            FaceFormat += 'F'

                        #Print Shader faces
                        if (len(MeshObject.uv_layers) > 1):
                            ShaderIndex = 0
                            MaterialIndex = 0
                            out.write('\t*FACESHADERS {\n')
                            for mat in MeshObject.material_slots:
                                out.write('\t\t*SHADER %d {\n' % ShaderIndex)

                                if mat.material.blend_method == 'OPAQUE':
                                    out.write('\t\t\t%d %s\n' % (bpy.data.materials.find(mat.name),"Non"))
                                else:
                                    out.write('\t\t\t%d %s\n' % (bpy.data.materials.find(mat.name),"Alp"))
                                out.write('\t\t}\n')

                                #update Counter
                                MaterialIndex +=1
                            out.write('\t}\n')

                        #Get FaceFormat
                        out.write('\t*FACEFORMAT %s\n' % FaceFormat)

                        #Print Face List, the difficult part ;P
                        out.write('\t*FACE_LIST {\n')
                        for MeshPolys in MeshObject.polygons:
                            #Write vertices ---V
                            out.write('\t\t%d ' % (len(MeshPolys.vertices)))
                            for vert in MeshPolys.vertices:
                                out.write('%d ' % vert)

                            #Write UVs ---T
                            if ('T' in FaceFormat):
                                for vert_idx, loop_idx in enumerate(MeshPolys.loop_indices):
                                    for layer in MeshObject.uv_layers:
                                        uv_coords = layer.data[loop_idx].uv
                                        out.write('%d ' % UVList.index((uv_coords.x, 1.0 - float(uv_coords.y))))

                            #Write Colors ---C
                            if ('C' in FaceFormat):
                                for color_idx, loop_idx in enumerate(MeshPolys.loop_indices):
                                    # swy: this is wrong; it should be the same layer count as any other face format block, can't be a different size. me.vertex_colors != me.uv_layers
                                    for layerIndex in MeshObject.vertex_colors:
                                        vertex = layerIndex.data[loop_idx]
                                        out.write('%d ' % VertexColList.index((vertex.color[0] * .5, vertex.color[1] * .5, vertex.color[2] * .5, vertex.color[3])))
                            # swy: we're missing exporting (optional) face normals here

                            if ('M' in FaceFormat):
                                for layer in MeshObject.uv_layers:
                                    out.write('%d ' % MeshPolys.material_index)

                            # swy: we're missing exporting an optional shader index here

                            #Write Flags ---F
                            if ('F' in FaceFormat):
                                flags = 0

                                if obj.material_slots[MeshPolys.material_index].material.use_backface_culling:
                                    flags |= 1 << 16
                                out.write('%d ' % flags)

                            out.write('\n')
                        out.write('\t}\n')
                    out.write('}\n')
                    out.write('\n')

            for obj in ProjectContextScene.objects:
                if obj.type == 'MESH':
                    if hasattr(obj, 'data'):
                        #===============================================================================================
                        #  GEOM NODE
                        #===============================================================================================
                        out.write('*GEOMNODE {\n')
                        out.write('\t*NAME "%s"\n' % (obj.name))
                        out.write('\t*MESH "%s"\n' % (obj.name))
                        out.write('\t*WORLD_TM {\n')
                        out.write('\t\t*TMROW0 %.6f %.6f %.6f %.6f\n' % (1,0,0,0))
                        out.write('\t\t*TMROW1 %.6f %.6f %.6f %.6f\n' % (0,1,0,0))
                        out.write('\t\t*TMROW2 %.6f %.6f %.6f %.6f\n' % (0,0,1,0))
                        out.write('\t\t*TMROW3 %.6f %.6f %.6f %.6f\n' % (0,0,0,1))
                        out.write('\t\t*POS %.6f %.6f %.6f\n' % (0,0,0))
                        out.write('\t\t*ROT %.6f %.6f %.6f\n' % (0,0,0))
                        out.write('\t\t*SCL %.6f %.6f %.6f\n' % (1,1,1))
                        out.write('\t}\n')
                        out.write('\t*USER_FLAGS_COUNT %u\n' % 1)
                        out.write('\t*USER_FLAGS {\n')
                        out.write('\t\t*SET 0 0x00000000\n')
                        out.write('\t}\n')
                        out.write('}\n')

            for obj in ProjectContextScene.objects:
                if obj.type == 'MESH':
                    if hasattr(obj, 'data'):
                        #===============================================================================================
                        #  PLACE NODE
                        #===============================================================================================
                        out.write('*PLACENODE {\n')
                        out.write('\t*NAME "%s"\n' % (obj.name))
                        out.write('\t*MESH "%s"\n' % (obj.name))
                        out.write('\t*WORLD_TM {\n')

                        #Jump to the first frame
                        ProjectContextScene.frame_set(ProjectContextScene.frame_start)

                        ConvertedMatrix = obj.rotation_euler.to_matrix()
                        rot_mtx = InvertAxisRotationMatrix @ ConvertedMatrix
                        RotationMatrix = rot_mtx.transposed()
                        loc_conv = InvertAxisRotationMatrix @ obj.location

                        #Print Roation matrix
                        out.write('\t\t*TMROW0 %.6f %.6f %.6f %.6f\n' % (RotationMatrix[0].x, (RotationMatrix[0].y * -1), RotationMatrix[0].z, 0))
                        out.write('\t\t*TMROW1 %.6f %.6f %.6f %.6f\n' % (RotationMatrix[1].x, RotationMatrix[1].y, RotationMatrix[1].z, 0))
                        out.write('\t\t*TMROW2 %.6f %.6f %.6f %.6f\n' % ((RotationMatrix[2].x * -1), (RotationMatrix[2].y * -1), (RotationMatrix[2].z) * -1, 0))
                        out.write('\t\t*TMROW3 %.6f %.6f %.6f %.6f\n' % (loc_conv.x, loc_conv.y, loc_conv.z, 1))

                        # swy: these aren't actually used or read by this version of the importer
                        out.write('\t\t*POS %.6f %.6f %.6f\n' % (loc_conv.x, loc_conv.y, loc_conv.z))
                        out.write('\t\t*ROT %.6f %.6f %.6f\n' % (radians(obj.rotation_euler.x), radians(obj.rotation_euler.y), radians(obj.rotation_euler.z)))
                        out.write('\t\t*SCL %.6f %.6f %.6f\n' % (obj.scale.x, obj.scale.y, obj.scale.z))
                        out.write('\t}\n')
                        out.write('}\n')
            #Close File
            out.flush()
            out.close()
    WriteFile()

def save(context,
            filepath,
            *,
            global_matrix=None,
        ):

    _write(context, filepath,
            EXPORT_GLOBAL_MATRIX=global_matrix,
        )

    return {'FINISHED'}
if __name__ == '__main__':
    save({}, str(Path.home()) + '/Desktop/EurocomEIF.eif')


import io
from PIL import Image
import pyglet
from PIL.PngImagePlugin import PngImageFile

def replace_content_with_transparency(img: PngImageFile, x, y, width, height):
    pixels = img.load()
    for i in range(x, width):
        for j in range(y, height):
            pixels[i, j] = (0, 0, 0, 0)

desert_png = Image.open('assets/desert.png')
replace_content_with_transparency(desert_png, 32, 32, 123, 123)

fake_file = io.BytesIO()
desert_png.save(fake_file, format='PNG')

desert_img = pyglet.image.load('img.png', file=fake_file)
desert = pyglet.sprite.Sprite(desert_img, x=50, y=50)

winter_img = pyglet.image.load('assets/winter.jpg')
winter = pyglet.sprite.Sprite(winter_img, x=0, y=0)

window = pyglet.window.Window()

@window.event
def on_draw():
    winter.draw()
    desert.draw()

pyglet.app.run()


#!/usr/bin/env python
import MySQLdb

db = MySQLdb.connect("localhost", "greenhouse", "green", "greenhouse_db")
curs=db.cursor()

def write_log(message):
    try:
        sql="INSERT INTO log_message (message) VALUES ('" + message + "');"
        curs.execute (sql)
        db.commit()
        print("Done")
    except:
        print("Error. Rolling back.")
        db.rollback()
    return

def write_error(message):
    try:
        sql="INSERT INTO error_message (message) VALUES ('" + message + "');"
        curs.execute (sql)
        db.commit()
        print("Done")
    except:
        print("Error. Rolling back.")
        db.rollback()
    return


# -*- coding: utf-8 -*-
from moive.items import MoiveItem
# from userAgents import pcUserAgent
import time

import scrapy

class MeijuSpider(scrapy.Spider):
    name = 'meiju'
    allowed_domains = ['lab.scrapyd.cn']
    start_urls = ['http://lab.scrapyd.cn']
    custom_settings = {
        'LOG_LEVEL': 'DEBUG',
        'LOG_FILE': './log/log_{}.txt'.format(time.time()),
        "DEFAULT_REQUEST_HEADERS": {
            'User-Agent': 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
        }
    }
    # tag = '励志'

    def start_requests(self):
        url = 'http://lab.scrapyd.cn/'
        tag = getattr(self, 'tag', None)  # 获取tag值，也就是爬取时传过来的参数
        if tag is not None:  # 判断是否存在tag，若存在，重新构造url
            url = url + 'tag/' + tag  # 构造url若tag=爱情，url= "http://lab.scrapyd.cn/tag/爱情"
        yield scrapy.Request(url, self.parse)  # 发送请求爬取参数内容
        print('1', end=' ')

    def parse(self, response):
        mingyan = response.css('div.quote')
        for v in mingyan:
            item = MoiveItem()
            text = v.css('.text::text').extract_first()
            tags = v.css('.tags .tag::text').extract()
            tags = ','.join(tags)
            # filename = 'my_meiju.txt'
            # with open(filename, "a+", encoding='utf-8') as f:
            #     f.write(text + '\n')
            #     f.write('标签：' + tags)
            #     f.write('\n-------\n')

            item['text'] = text
            item['tags'] = tags
            yield item
        next_page = response.css('li.next a::attr(href)').extract_first()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)
            print('2', end=' ')

    # def parse(self, response):
    #     mingyan = response.css('div.quote')
    #     for v in mingyan:
    #         item = MoiveItem()
    #         item['text'] = v.css('.text::text').extract_first()
    #         item['tags'] = v.css('.tags .tag::text').extract()
    #         item['tags'] = ','.join(item['tags'])
    #         item['next_page'] = response.css('li.next a::attr(href)').extract_first()
    #         if item['next_page'] is not None:
    #             item['next_page'] = response.urljoin(item['next_page'] )
    #         yield item


import os
import sys
from hashlib import sha256

from PyQt5 import QtCore, QtGui, QtWidgets

from src.ui import Ui_MainWindow
from src.utils import (addBackupInfo, backup, calcHash, checkBackupDir,
                   deleteBackup, loadBackupsInfo, loadFile, makeFileInfo,
                   restoreBackup, saveAs, saveBackupsInfo)

class configManager(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        Ui_MainWindow.__init__(self)
        self.setupUi(self)
        self.btn_backup.clicked.connect(self.addBackup)
        self.btn_delete.clicked.connect(self.deleteBackup)
        self.btn_restore.clicked.connect(self.restoreBackup)
        self.btn_saveAs.clicked.connect(self.saveAs)

        self.backups = loadBackupsInfo() # 读取保存的备份信息
        self.lastBackupPath = sys.path[0] # 备份时默认的所在路径
        self.lastSavePath = sys.path[0]
        self.refreshTable()  # 刷新表格
        checkBackupDir() # 检测备份文件夹

    def closeEvent(self, event):
        saveBackupsInfo(self.backups)
        event.accept()

    def setItemInTable(self, row, col, text):
        item = QtWidgets.QTableWidgetItem()
        item.setText(text)
        self.mainTable.setItem(row, col, item)

    def refreshTable(self):
        rowCount = 1
        for hash_ in self.backups:
            for file in self.backups[hash_]:
                self.mainTable.setRowCount(rowCount)
                self.setItemInTable(rowCount - 1, 0, file["fileName"])
                self.setItemInTable(rowCount - 1, 1, file["dirPath"])
                self.setItemInTable(rowCount - 1, 2, hash_)
                rowCount += 1

    def getTextInTable(self, row, col):
        index = self.mainTable.model().index(row, col)
        return self.mainTable.model().data(index)

    def addBackup(self):
        path = QtWidgets.QFileDialog.getOpenFileName(self, "Select your file", self.lastBackupPath, "All Types (*.*)")
        path = path[0]
        if path: # 检查是否选择了一个路径
            self.lastBackupPath = os.path.split(path)[0]
            path = os.path.normpath(path) # 格式化路径
            hash_ = calcHash(loadFile(path))
            if not hash_ in self.backups:
                backup(path) # 如果没有备份过，备份一次
            fileInfo = makeFileInfo(path)
            if addBackupInfo(self.backups, fileInfo, hash_): # 添加成功，对新增的一行的操作
                currRowCount = self.mainTable.rowCount() + 1
                self.mainTable.setRowCount(currRowCount)
                self.setItemInTable(currRowCount - 1, 0, fileInfo["fileName"])
                self.setItemInTable(currRowCount - 1, 1, fileInfo["dirPath"])
                self.setItemInTable(currRowCount - 1, 2, hash_)

    def deleteBackup(self):
        if len(self.mainTable.selectedIndexes()) == 0: # 至少选中一个弹出确认窗口
            return
        ret = QtWidgets.QMessageBox.question(None, "确定", "确定删除么", QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
        if ret == QtWidgets.QMessageBox.Yes:
            selected = self.mainTable.selectedIndexes()
            rowNeedToRemove = list()
            for i in range(0, len(selected), 3): # 一次选中三列， 所以以三为步长遍历选中的行
                rowNeedToRemove.append(selected[i].row())
                hash_ = self.getTextInTable(selected[i].row(), 2)
                path = os.path.join(self.getTextInTable(selected[i].row(), 1), self.getTextInTable(selected[i].row(), 0)) # 拼成完整的路径
                self.backups[hash_].remove(makeFileInfo(path)) # 删掉这个记录
                if len(self.backups[hash_]) == 0: # 如果此文件不被任何记录关联，删除此文件
                    self.backups.pop(hash_)
                    deleteBackup(hash_)
            rowNeedToRemove.sort(reverse=True) #从大到小排序，以防止产生混乱
            for i in rowNeedToRemove:
                self.mainTable.removeRow(i)
        else:
            pass

    def restoreBackup(self):
        if len(self.mainTable.selectedIndexes()) == 0: # 至少选中一个弹出确认窗口
            return
        ret = QtWidgets.QMessageBox.question(None, "确定", "确定还原么，这将覆盖原来的文件", QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No, QtWidgets.QMessageBox.No)
        if ret == QtWidgets.QMessageBox.Yes:
            selected = self.mainTable.selectedIndexes()
            for i in range(0, len(selected), 3): # 一次选中三列， 所以以三为步长遍历选中的行
                hash_ = self.getTextInTable(selected[i].row(), 2)
                path = os.path.join(self.getTextInTable(selected[i].row(), 1), self.getTextInTable(selected[i].row(), 0)) # 拼成完整的路径
                restoreBackup(hash_, makeFileInfo(path)) # 还原文件
        else:
            pass

    def saveAs(self):
        path = QtWidgets.QFileDialog.getSaveFileName(self, "Select your save place", self.lastSavePath, "All File (*)")
        path = path[0]
        if path:
            path = os.path.normpath(path) # 格式化路径
            self.lastSavePath = os.path.split(path)[0]
            selectedRow = self.mainTable.currentRow() # 最后一个选择的列
            hash_ = self.getTextInTable(selectedRow, 2)
            saveAs(hash_, path)

app = QtWidgets.QApplication(sys.argv)
mygui = configManager()
mygui.show()
sys.exit(app.exec_())


'''
Digital map information type, contains a group of map data nodes

@author: stevenh47
'''
from google.appengine.ext import db

from Model.map.map_data_node import MapDataNode

class MapData(db.Model):
  '''Digital map information.
  '''
  map_id = db.StringProperty()
  nodes = db.StringListProperty(default=[])

  def __init__(self, map_id, **kwds):
    '''Constructor.
    map_id: Map instance id
    '''
    super(MapData, self).__init__(**kwds)
    self.map_id = map_id

  def add_node(self):
    """Add a new node.
    Return:
    A new MpaDataNode instance
    """
    new_node = MapDataNode(None, None, None, None, None, False, parent=self)
    self.nodes.append(new_node.key())
    return new_node

  def del_node(self, key='', node=None):
    """Delete a node.
    Args:
    key: key
    node: node
    Return:
    True for success, False for fail.
    """
    if node:
      key = node.key()
    if key == '' or key not in self.nodes:
      return False
    self.nodes.pop(self.nodes.index(key))
    if node:
      target = node
    else:
      target = MapDataNode.get(key)
    target.del_all_connect()
    target.delete()
    return True


from django.conf.urls import url
from django.urls import path

from .views import startpage, UserRegisterView, logout_view, UserLoginView, user_help_view,\
    contact_view, contact_success_view, contact_error_view, howitworks_startpage_view, contact_startpage_view,\
    contact_startpage_success_view, contact_startpage_error_view, faq_startpage_view

urlpatterns = [
    # landing page/startpage
    path('', startpage, name="startpage_view"),
    path('how-it-works/', howitworks_startpage_view, name="howitworks_startpage_view"),
    path('faq/', faq_startpage_view, name="faq_startpage_view"),
    path('contact/', contact_startpage_view, name="contact_startpage_view"),

    path('contact-success/', contact_startpage_success_view, name="contact_startpage_success_view"),
    path('contact-error/', contact_startpage_error_view, name="contact_startpage_error_view"),

    path('register-account/', UserRegisterView.as_view(), name="register_account"),
    path('login/', UserLoginView.as_view(), name="login_view"),
    path('logout/', logout_view, name="logout_view"),

    # inside app
    path('help/', user_help_view, name="help_view"),
    path('contact-app/', contact_view, name="contact_view"),

    path('contact-app-success/', contact_success_view, name="contact_success_view"),
    path('contact-app-error/', contact_error_view, name="contact_error_view"),

]