import datetime
import json
import os
import numpy as np
import torch
from PIL import Image
from matplotlib import pyplot as plt
from scipy.sparse import coo_matrix
from scipy.stats import multivariate_normal
from torch import nn
from torchvision.datasets import VisionDataset
from torchvision.transforms import ToTensor

from multiview_detector.utils.projection import *

class frameDataset(VisionDataset):
    def __init__(self, base, train=True, transform=ToTensor(), target_transform=ToTensor(),
                 reID=False, grid_reduce=4, img_reduce=4, train_ratio=0.9, force_download=True):
        # Totensor() Convert a PIL Image or numpy.ndarray to tensor. This transform does not support torchscript.
        super().__init__(base.root, transform=transform, target_transform=target_transform)

        map_sigma, map_kernel_size = 20 / grid_reduce, 20
        img_sigma, img_kernel_size = 20 / img_reduce, 20
        self.reID, self.grid_reduce, self.img_reduce = reID, grid_reduce, img_reduce

        self.base = base
        self.root, self.num_cam, self.num_frame = base.root, base.num_cam, base.num_frame
        self.img_shape, self.worldgrid_shape = base.img_shape, base.worldgrid_shape  # H,W; N_row,N_col
        self.reducedgrid_shape = list(map(lambda x: int(x / self.grid_reduce), self.worldgrid_shape))

        if train:
            frame_range = range(0, int(self.num_frame * train_ratio))  # 0.9
        else:
            frame_range = range(int(self.num_frame * train_ratio), self.num_frame)

        self.img_fpaths = self.base.get_image_fpaths(frame_range)
        self.map_gt = {}
        self.imgs_head_foot_gt = {}
        self.download(frame_range)

        self.gt_fpath = os.path.join(self.root, 'gt.txt')
        if not os.path.exists(self.gt_fpath) or force_download:
            self.prepare_gt()

        x, y = np.meshgrid(np.arange(-map_kernel_size, map_kernel_size + 1),
                           np.arange(-map_kernel_size, map_kernel_size + 1))

        pos = np.stack([x, y], axis=2)
        map_kernel = multivariate_normal.pdf(pos, [0, 0], np.identity(2) * map_sigma)
        map_kernel = map_kernel / map_kernel.max()
        kernel_size = map_kernel.shape[0]
        self.map_kernel = torch.zeros([1, 1, kernel_size, kernel_size], requires_grad=False)
        self.map_kernel[0, 0] = torch.from_numpy(map_kernel)

        x, y = np.meshgrid(np.arange(-img_kernel_size, img_kernel_size + 1),
                           np.arange(-img_kernel_size, img_kernel_size + 1))
        pos = np.stack([x, y], axis=2)
        img_kernel = multivariate_normal.pdf(pos, [0, 0], np.identity(2) * img_sigma)
        img_kernel = img_kernel / img_kernel.max()
        kernel_size = img_kernel.shape[0]
        self.img_kernel = torch.zeros([2, 2, kernel_size, kernel_size], requires_grad=False)
        self.img_kernel[0, 0] = torch.from_numpy(img_kernel)
        self.img_kernel[1, 1] = torch.from_numpy(img_kernel)
        pass

    def prepare_gt(self):
        og_gt = []
        for fname in sorted(os.listdir(os.path.join(self.root, 'annotations_positions'))):
            frame = int(fname.split('.')[0])
            with open(os.path.join(self.root, 'annotations_positions', fname)) as json_file:
                all_pedestrians = json.load(json_file)
            for single_pedestrian in all_pedestrians:
                def is_in_cam(cam):
                    return not (single_pedestrian['views'][cam]['xmin'] == -1 and
                                single_pedestrian['views'][cam]['xmax'] == -1 and
                                single_pedestrian['views'][cam]['ymin'] == -1 and
                                single_pedestrian['views'][cam]['ymax'] == -1)

                in_cam_range = sum(is_in_cam(cam) for cam in range(self.num_cam))
                if not in_cam_range:
                    continue
                grid_x, grid_y = self.base.get_worldgrid_from_pos(single_pedestrian['positionID'])
                og_gt.append(np.array([frame, grid_x, grid_y]))

        og_gt = np.stack(og_gt, axis=0)
        os.makedirs(os.path.dirname(self.gt_fpath), exist_ok=True)
        np.savetxt(self.gt_fpath, og_gt, '%d')

    def download(self, frame_range):
        # 每张图片中的每个人在每个相机下
        # 第一个for循环生成map_gt列表，包含1800帧图像，一个元素代表对应图像中每个人的位置
        # 第二个for循环生成imgs_head_foot列表，一个元素代表一帧图像中所有人在所有相机视角下head和foot位置
        for fname in sorted(os.listdir(os.path.join(self.root, 'annotations_positions'))):
            frame = int(fname.split('.')[0])
            if frame in frame_range:
                with open(os.path.join(self.root, 'annotations_positions', fname)) as json_file:
                    all_pedestrians = json.load(json_file)
                i_s, j_s, v_s = [], [], []  # world grid coord
                head_row_cam_s, head_col_cam_s = [[] for _ in range(self.num_cam)], \
                                                 [[] for _ in range(self.num_cam)]
                foot_row_cam_s, foot_col_cam_s, v_cam_s = [[] for _ in range(self.num_cam)], \
                                                          [[] for _ in range(self.num_cam)], \
                                                          [[] for _ in range(self.num_cam)]

                for single_pedestrian in all_pedestrians:
                    x, y = self.base.get_worldgrid_from_pos(single_pedestrian['positionID'])
                    if self.base.indexing == 'xy':
                        i_s.append(int(y / self.grid_reduce))
                        j_s.append(int(x / self.grid_reduce))
                    else:
                        i_s.append(int(x / self.grid_reduce))
                        j_s.append(int(y / self.grid_reduce))
                    v_s.append(single_pedestrian['personID'] + 1 if self.reID else 1)
                    for cam in range(self.num_cam):
                        x = max(min(int((single_pedestrian['views'][cam]['xmin'] +
                                         single_pedestrian['views'][cam]['xmax']) / 2), self.img_shape[1] - 1), 0)
                        y_head = max(single_pedestrian['views'][cam]['ymin'], 0)
                        y_foot = min(single_pedestrian['views'][cam]['ymax'], self.img_shape[0] - 1)
                        if x > 0 and y > 0:
                            head_row_cam_s[cam].append(y_head)
                            head_col_cam_s[cam].append(x)
                            foot_row_cam_s[cam].append(y_foot)
                            foot_col_cam_s[cam].append(x)
                            v_cam_s[cam].append(single_pedestrian['personID'] + 1 if self.reID else 1)
                occupancy_map = coo_matrix((v_s, (i_s, j_s)), shape=self.reducedgrid_shape)  # 120, 360
                self.map_gt[frame] = occupancy_map
                self.imgs_head_foot_gt[frame] = {}
                for cam in range(self.num_cam):
                    img_gt_head = coo_matrix((v_cam_s[cam], (head_row_cam_s[cam], head_col_cam_s[cam])),
                                             shape=self.img_shape)
                    img_gt_foot = coo_matrix((v_cam_s[cam], (foot_row_cam_s[cam], foot_col_cam_s[cam])),
                                             shape=self.img_shape)
                    self.imgs_head_foot_gt[frame][cam] = [img_gt_head, img_gt_foot]

    def __getitem__(self, index):  ####
        frame = list(self.map_gt.keys())[index]
        imgs = []
        for cam in range(self.num_cam):
            fpath = self.img_fpaths[cam][frame]
            img = Image.open(fpath).convert('RGB')
            if self.transform is not None:
                img = self.transform(img)
            imgs.append(img)
        imgs = torch.stack(imgs)
        map_gt = self.map_gt[frame].toarray()
        if self.reID:
            map_gt = (map_gt > 0).int()
        if self.target_transform is not None:
            map_gt = self.target_transform(map_gt)
        imgs_gt = []
        for cam in range(self.num_cam):
            img_gt_head = self.imgs_head_foot_gt[frame][cam][0].toarray()
            img_gt_foot = self.imgs_head_foot_gt[frame][cam][1].toarray()
            img_gt = np.stack([img_gt_head, img_gt_foot], axis=2)
            if self.reID:
                img_gt = (img_gt > 0).int()
            if self.target_transform is not None:
                img_gt = self.target_transform(img_gt)
            imgs_gt.append(img_gt.float())
        imgs_gt = torch.stack(imgs_gt)
        return imgs, map_gt.float(), imgs_gt, frame

    def __len__(self):
        return len(self.map_gt.keys())

def test():
    from datasets.W_M.Wildtrack import Wildtrack
    # from datasets.MultiviewX import MultiviewX
    from multiview_detector.utils.projection import get_worldcoord_from_imagecoord
    dataset = frameDataset(Wildtrack(os.path.expanduser('~/Data/Wildtrack')))
    import torch.nn.functional as F
    imgs, map_gt, imgs_gt, _ = dataset.__getitem__(0)

    print('map_gt shape', map_gt.shape)
    print('img_gt shape', imgs_gt[0].shape)
    print('sum of imgs-gt[0]', imgs_gt[0].sum().item())
    print('img max', imgs_gt[0].max())
    print('kernel shape==', dataset.map_kernel.shape)

    map_gt = F.conv2d(map_gt.detach().unsqueeze(0), dataset.map_kernel.float(),
                      padding=int((dataset.map_kernel.shape[-1] - 1) / 2))
    img_gt0 = F.adaptive_max_pool2d(imgs_gt[0][None], (270, 480))
    img_gt0 = F.conv2d(img_gt0, dataset.img_kernel.float(),
                       padding=int((dataset.img_kernel.shape[-1] - 1) / 2))
    print('img_gt0 shape', img_gt0.shape)
    # plt.imshow(map_gt.squeeze())
    # plt.show()
    plt.imshow(img_gt0[0][0].squeeze())
    plt.show()
    plt.imshow(img_gt0[0][1].squeeze())
    plt.show()
    print(datetime.datetime.now())

if __name__ == '__main__':
    test()


import itertools
row, max=[],[]

for i in range(9):
    a=list(map(int,input().split()))
    row.append(a)
    sort_a=sorted(a, reverse=True)   #행 숫자큰순서대로
    max.append(sort_a[0])  #각 행 최대값들
all=list(itertools.chain(*row))
max.sort(reverse=True)
ans=all.index(max[0])     #max[0]: 최댓값
r=ans//9+1
c=ans%9+1
print(max[0])
print(r,c)


import sys
from collections import deque

N, M = map(int, sys.stdin.readline().split())
data = list(map(int, sys.stdin.readline().split()))

queue = deque()
for i in range(1, N+1):
    queue.append(i)

count = 0
while len(data) > 0:
    idx = queue.index(data[0])
    distance = min(idx, len(queue) - idx)
    if distance == idx:
        for _ in range(idx):
            queue.append(queue.popleft())
    else:
        for _ in range(len(queue)-idx):
            queue.appendleft(queue.pop())
    count += distance
    data.pop(0)
    queue.popleft()

print(count)


import pygame
pygame.init()

win = pygame.display.set_mode((500, 500))
pygame.display.set_caption("POPSMOKE SIMULATOR")

player = pygame.image.load('graphics/popSmoke.jpg').convert_alpha()

x = 250
y = 250
velocity = 10

#GAME LOOP
run = True
while run:

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            run = False

    userInput = pygame.key.get_pressed()

    if userInput[pygame.K_LEFT]:
        x -= velocity
        print('left')
    if userInput[pygame.K_RIGHT]:
        x += velocity
        print('right')
    if userInput[pygame.K_UP]:
        y -= velocity
        print('up')
    if userInput[pygame.K_DOWN]:
        y += velocity
        print('down')

    win.blit(player, (250, 250))

    pygame.display.update()


#!/usr/bin/env python

import glob
import json

paths = glob.glob('assets/img/*.jpg')
images = []

for p in paths:
    images.append({
        'name': '',
        'path': p})

with open('cards.json', 'w') as f:
    json.dump(images, f, sort_keys=True, indent=2)


from django.db import models
from django.urls import reverse
from django.utils.text import slugify

from api.common.models.mixins import ViewsModelMixin
from api.v1.products.managers import ProductManager, ProductTypeManager

class Product(ViewsModelMixin, models.Model):
    name = models.CharField(max_length=128, unique=True)
    price = models.FloatField()
    card_description = models.CharField(max_length=128)
    description = models.TextField()
    image = models.ImageField(upload_to="products/products")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    store = models.ForeignKey(to="stores.Store", on_delete=models.CASCADE)
    product_type = models.ForeignKey(
        to="products.ProductType", on_delete=models.CASCADE
    )
    views = models.PositiveIntegerField(default=0)
    slug = models.SlugField(unique=True)

    objects = ProductManager()

    def __str__(self):
        return f"{self.name} | {self.store}"

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    def get_absolute_url(self) -> str:
        return reverse("api:v1:products:products-detail", args=(self.slug,))

class ProductType(ViewsModelMixin, models.Model):
    name = models.CharField(max_length=64, unique=True)
    description = models.CharField(max_length=128)
    image = models.ImageField(upload_to="products/product_types")
    views = models.PositiveIntegerField(default=0)
    slug = models.SlugField(unique=True)

    objects = ProductTypeManager()

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        if not self.slug:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    def get_absolute_url(self) -> str:
        return reverse("api:v1:products:product-types-detail", args=(self.slug,))


import torch.nn as nn
import torch
import torch.nn.functional as F
from torch import Tensor
from typing import Tuple, Dict
import math
from torchsummary import summary
from torch.nn.modules import ModuleList, Identity
from collections import OrderedDict

def PatchConvSASPPSE_MViT():
    model = PatchConvS_ASPP_SE_MViT(blocks=[1, 1, 1, 2], ratios=2, channels=64, dims=[64, 128, 256, 512])
    return model

class patchconv(nn.Module):
    def __init__(self, patch_size=(4, 4), in_dims=3, out_dims=16):
        super(patchconv, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_channels=in_dims, out_channels=out_dims, kernel_size=patch_size,
                                            stride=patch_size, padding=(0, 0), bias=False),
                                  nn.BatchNorm2d(out_dims),
                                  nn.LeakyReLU(negative_slope=0.2, inplace=True))

    def forward(self, x):
        x = self.conv(x)
        return x

class Resblock(nn.Module):
    def __init__(self, in_dims):
        super(Resblock, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_channels=in_dims, out_channels=in_dims, kernel_size=(3, 3),
                                            stride=(1, 1), padding=(1, 1), bias=False),
                                  nn.BatchNorm2d(in_dims),
                                  nn.LeakyReLU(negative_slope=0.2, inplace=True),
                                  nn.Conv2d(in_channels=in_dims, out_channels=in_dims, kernel_size=(3, 3),
                                            stride=(1, 1), padding=(1, 1), bias=False),
                                  nn.BatchNorm2d(in_dims),
                                  nn.LeakyReLU(negative_slope=0.2, inplace=True))

    def forward(self, x):
        residual = x
        x = self.conv(x)
        out = x + residual
        return out

class BasicRes(nn.Module):
    def __init__(self, num_blocks, dims, ratios, downsample=True):
        super(BasicRes, self).__init__()
        self.blocks = ModuleList()
        self.downsample = patchconv(patch_size=(2, 2), in_dims=dims, out_dims=dims*ratios) if downsample else Identity()
        for i in range(num_blocks):
            block = Resblock(in_dims=dims)
            self.blocks.append(block)

    def forward(self, x):
        for blk in self.blocks:
            x = blk(x)
        x = self.downsample(x)
        return x

class Upsample(nn.Module):
    def __init__(self, in_dims, out_dims):
        super(Upsample, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_channels=in_dims, out_channels=out_dims, kernel_size=(5, 5),
                                            stride=(1, 1), padding=(2, 2), bias=False),
                                  nn.BatchNorm2d(out_dims),
                                  nn.LeakyReLU(negative_slope=0.2, inplace=True))

    def forward(self, x):
        x = self.conv(x)
        return x

class Dropout(nn.Dropout):
    def __init__(self, p: float = 0.5, inplace: bool = False):
        """
        During training, randomly zeroes some of the elements of the input tensor with probability `p` using samples \
        from a Bernoulli distribution.

        :param p: probability of an element to be zeroed. Default: 0.5
        :param inplace: If set to ``True``, will do this operation in-place. Default: ``False``
        """
        super(Dropout, self).__init__(p=p, inplace=inplace)

    def profile_module(self, input: Tensor) -> (Tensor, float, float):
        input = self.forward(input)
        return input, 0.0, 0.0

class SEblock(nn.Module):
    def __init__(self, in_dim):
        super(SEblock, self).__init__()
        squeeze = int(in_dim // 16)
        self.fc1 = nn.Conv2d(in_dim, squeeze, (1, 1))
        self.fc2 = nn.Conv2d(squeeze, in_dim, (1, 1))

    def forward(self, x):
        scale = F.adaptive_avg_pool2d(x, output_size=(1, 1))
        scale = self.fc1(scale)
        scale = F.leaky_relu_(scale)
        scale = self.fc2(scale)
        scale = torch.sigmoid(scale)
        return scale * x

class ASPPConv(nn.Sequential):
    def __init__(self, in_channels: int, out_channels: int, dilation: int) -> None:
        super(ASPPConv, self).__init__(
            nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.LeakyReLU(negative_slope=0.2, inplace=True)
        )

class ASPPPooling(nn.Sequential):
    def __init__(self, in_channels: int, out_channels: int) -> None:
        super(ASPPPooling, self).__init__(
            nn.AdaptiveAvgPool2d(1),
            nn.Conv2d(in_channels, out_channels, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.LeakyReLU(negative_slope=0.2, inplace=True)
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        size = x.shape[-2:]
        for mod in self:
            x = mod(x)
        return F.interpolate(x, size=size, mode='bilinear', align_corners=False)

class ASPP(nn.Module):
    def __init__(self, in_channels: int, atrous_rates, out_channels: int = 256) -> None:
        super(ASPP, self).__init__()
        modules = [
            nn.Sequential(nn.Conv2d(in_channels, out_channels, 1, bias=False),
                          nn.BatchNorm2d(out_channels),
                          nn.LeakyReLU(negative_slope=0.2, inplace=True))
        ]

        rates = tuple(atrous_rates)
        for rate in rates:
            modules.append(ASPPConv(in_channels, out_channels, rate))

        modules.append(ASPPPooling(in_channels, out_channels))

        self.convs = nn.ModuleList(modules)

        self.project = nn.Sequential(
            nn.Conv2d(len(self.convs) * out_channels, out_channels, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.LeakyReLU(negative_slope=0.2, inplace=True),
            nn.Dropout(0.5)
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        _res = []
        for conv in self.convs:
            _res.append(conv(x))
        res = torch.cat(_res, dim=1)
        return self.project(res)

class LinearLayer(nn.Module):
    def __init__(self,
                 in_features: int,
                 out_features: int,
                 bias = True,
                 *args, **kwargs) -> None:
        """
            Applies a linear transformation to the input data

            :param in_features: size of each input sample
            :param out_features:  size of each output sample
            :param bias: Add bias (learnable) or not
        """
        super(LinearLayer, self).__init__()
        self.weight = nn.Parameter(torch.Tensor(out_features, in_features))
        self.bias = None
        if bias:
            self.bias = nn.Parameter(torch.Tensor(out_features))
        self.in_features = in_features
        self.out_features = out_features
        self.reset_params()

    def reset_params(self):
        if self.weight is not None:
            torch.nn.init.xavier_uniform_(self.weight)
        if self.bias is not None:
            torch.nn.init.constant_(self.bias, 0)

    def forward(self, x: Tensor) -> Tensor:
        if self.bias is not None and x.dim() == 2:
            x = torch.addmm(self.bias, x, self.weight.t())
        else:
            x = x.matmul(self.weight.t())
            if self.bias is not None:
                x += self.bias
        return x

class MultiHeadAttention(nn.Module):
    def __init__(self, embed_dim, num_heads, attn_dropout, bias=True):
        super(MultiHeadAttention, self).__init__()
        self.qkv_proj = LinearLayer(in_features=embed_dim, out_features=3 * embed_dim, bias=bias)

        self.attn_dropout = Dropout(p=attn_dropout)
        self.out_proj = LinearLayer(in_features=embed_dim, out_features=embed_dim, bias=bias)

        self.head_dim = embed_dim // num_heads
        self.scaling = self.head_dim ** -0.5
        self.softmax = nn.Softmax(dim=-1)
        self.num_heads = num_heads
        self.embed_dim = embed_dim

    def forward_other(self, x: Tensor) -> Tensor:
        # [B x N x C]
        b_sz, n_patches, in_channels = x.shape

        # [B x N x C] --> [B x N x 3 x h x C]
        qkv = (
            self.qkv_proj(x)
                .reshape(b_sz, n_patches, 3, self.num_heads, -1)
        )
        # [B x N x 3 x h x C] --> [B x h x 3 x N x C]
        qkv = qkv.transpose(1, 3)

        # [B x h x 3 x N x C] --> [B x h x N x C] x 3
        query, key, value = qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2]

        query = query * self.scaling

        # [B x h x N x C] --> [B x h x c x N]
        key = key.transpose(2, 3)

        # QK^T
        # [B x h x N x c] x [B x h x c x N] --> [B x h x N x N]
        attn = torch.matmul(query, key)
        attn = self.softmax(attn)
        attn = self.attn_dropout(attn)

        # weighted sum
        # [B x h x N x N] x [B x h x N x c] --> [B x h x N x c]
        out = torch.matmul(attn, value)

        # [B x h x N x c] --> [B x N x h x c] --> [B x N x C=ch]
        out = out.transpose(1, 2).reshape(b_sz, n_patches, -1)
        out = self.out_proj(out)

        return out

    def forward(self, x: Tensor) -> Tensor:
        return self.forward_other(x)

class TransformerBlock(nn.Module):
    def __init__(self, embed_dim, ffn_latent_dim, num_heads, attn_dropout, ffn_dropout, dropout):
        super(TransformerBlock, self).__init__()
        self.LN1 = nn.Sequential(nn.LayerNorm(embed_dim))
        self.MSAttn = MultiHeadAttention(embed_dim, num_heads=num_heads, attn_dropout=attn_dropout, bias=True)
        self.Drop1 = Dropout(p=dropout)
        self.LN2 = nn.Sequential(nn.LayerNorm(embed_dim))
        self.linear1 = LinearLayer(embed_dim, ffn_latent_dim, bias=True)
        self.act = nn.Sequential(nn.ReLU(inplace=False))
        self.Drop2 = Dropout(p=ffn_dropout)
        self.linear2 = LinearLayer(ffn_latent_dim, embed_dim,  bias=True)
        self.Drop3 = Dropout(p=dropout)

    def forward(self, x):
        res1 = x
        x = self.LN1(x)
        x = self.MSAttn(x)
        x = self.Drop1(x)
        res2 = res1 + x
        x = self.LN2(res2)
        x = self.linear1(x)
        x = self.act(x)
        x = self.Drop2(x)
        x = self.linear2(x)
        x = self.Drop3(x)
        x = res2 + x
        return x

class standard_down_conv(nn.Module):
    def __init__(self, in_dims, out_dims):
        super(standard_down_conv, self).__init__()
        self.conv = nn.Sequential(nn.Conv2d(in_channels=in_dims, out_channels=out_dims, kernel_size=(3, 3),
                                            stride=(2, 2), padding=(1, 1), bias=False),
                                  nn.BatchNorm2d(out_dims),
                                  nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.resconv = Resblock(out_dims)

    def forward(self, x):
        x = self.conv(x)
        x = self.resconv(x)
        return x

class MobileViTBlock(nn.Module):
    def __init__(self, in_channels, n_transformer_blocks, ffn_latent_dim, patch_w, patch_h,
                 num_heads=4, attn_dropout=0.0, ffn_dropout=0.0, dropout=0.1, fusion=True):
        super(MobileViTBlock, self).__init__()
        # self.conv5X5in = nn.Sequential(nn.Conv2d(in_channels, in_channels, kernel_size=(5, 5),
        #                                          stride=(1, 1), padding=(2, 2), bias=False),
        #                                nn.BatchNorm2d(in_channels),
        #                                nn.LeakyReLU(negative_slope=0.2, inplace=True))
        # self.conv1X1in = nn.Sequential(nn.Conv2d(in_channels, embed_dim, kernel_size=(1, 1),
        #                                          stride=(1, 1), padding=(0, 0), bias=False),
        #                                nn.BatchNorm2d(embed_dim),
        #                                nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.global_rep = nn.ModuleList()
        for i in range(n_transformer_blocks):
            global_rep = TransformerBlock(in_channels, ffn_latent_dim, num_heads, attn_dropout, ffn_dropout, dropout)
            self.global_rep.append(global_rep)
        # self.conv1X1out = nn.Sequential(nn.Conv2d(embed_dim, in_channels, kernel_size=(1, 1),
        #                                           stride=(1, 1), padding=(0, 0), bias=False),
        #                                 nn.BatchNorm2d(in_channels),
        #                                 nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.conv5X5out = nn.Sequential(nn.Conv2d(in_channels * 2, in_channels, kernel_size=(5, 5),
                                                  stride=(1, 1), padding=(2, 2), bias=False),
                                        nn.BatchNorm2d(in_channels),
                                        nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.fusion = fusion
        self.patch_w = patch_w
        self.patch_h = patch_h
        self.patch_area = patch_h * patch_w

    def unfolding(self, feature_map: Tensor) -> Tuple[Tensor, Dict]:
        patch_w, patch_h = self.patch_w, self.patch_h
        patch_area = int(patch_w * patch_h)
        batch_size, in_channels, orig_h, orig_w = feature_map.shape

        new_h = int(math.ceil(orig_h / self.patch_h) * self.patch_h)
        new_w = int(math.ceil(orig_w / self.patch_w) * self.patch_w)

        interpolate = False
        if new_w != orig_w or new_h != orig_h:
            # Note: Padding can be done, but then it needs to be handled in attention function.
            feature_map = F.interpolate(feature_map, size=(new_h, new_w), mode="bilinear", align_corners=False)
            interpolate = True

        # number of patches along width and height
        num_patch_w = new_w // patch_w # n_w
        num_patch_h = new_h // patch_h # n_h
        num_patches = num_patch_h * num_patch_w # N

        # [B, C, H, W] --> [B * C * n_h, p_h, n_w, p_w]
        reshaped_fm = feature_map.reshape(batch_size * in_channels * num_patch_h, patch_h, num_patch_w, patch_w)
        # [B * C * n_h, p_h, n_w, p_w] --> [B * C * n_h, n_w, p_h, p_w]
        transposed_fm = reshaped_fm.transpose(1, 2)
        # [B * C * n_h, n_w, p_h, p_w] --> [B, C, N, P] where P = p_h * p_w and N = n_h * n_w
        reshaped_fm = transposed_fm.reshape(batch_size, in_channels, num_patches, patch_area)
        # [B, C, N, P] --> [B, P, N, C]
        transposed_fm = reshaped_fm.transpose(1, 3)
        # [B, P, N, C] --> [BP, N, C]
        patches = transposed_fm.reshape(batch_size * patch_area, num_patches, -1)

        info_dict = {
            "orig_size": (orig_h, orig_w),
            "batch_size": batch_size,
            "interpolate": interpolate,
            "total_patches": num_patches,
            "num_patches_w": num_patch_w,
            "num_patches_h": num_patch_h
        }

        return patches, info_dict

    def folding(self, patches: Tensor, info_dict: Dict) -> Tensor:
        n_dim = patches.dim()
        assert n_dim == 3, "Tensor should be of shape BPxNxC. Got: {}".format(patches.shape)
        # [BP, N, C] --> [B, P, N, C]
        patches = patches.contiguous().view(info_dict["batch_size"], self.patch_area, info_dict["total_patches"], -1)

        batch_size, pixels, num_patches, channels = patches.size()
        num_patch_h = info_dict["num_patches_h"]
        num_patch_w = info_dict["num_patches_w"]

        # [B, P, N, C] --> [B, C, N, P]
        patches = patches.transpose(1, 3)

        # [B, C, N, P] --> [B*C*n_h, n_w, p_h, p_w]
        feature_map = patches.reshape(batch_size * channels * num_patch_h, num_patch_w, self.patch_h, self.patch_w)
        # [B*C*n_h, n_w, p_h, p_w] --> [B*C*n_h, p_h, n_w, p_w]
        feature_map = feature_map.transpose(1, 2)
        # [B*C*n_h, p_h, n_w, p_w] --> [B, C, H, W]
        feature_map = feature_map.reshape(batch_size, channels, num_patch_h * self.patch_h, num_patch_w * self.patch_w)
        if info_dict["interpolate"]:
            feature_map = F.interpolate(feature_map, size=info_dict["orig_size"], mode="bilinear", align_corners=False)
        return feature_map

    def forward(self, x: Tensor) -> Tensor:
        res = x
        # x = self.conv5X5in(x)
        # fm = self.conv1X1in(x)
        # convert feature map to patches
        patches, info_dict = self.unfolding(x)
        # learn global representations
        for i, gr in enumerate(self.global_rep):
            patches = gr(patches)
        # [B x Patch x Patches x C] --> [B x C x Patches x Patch]
        fm = self.folding(patches=patches, info_dict=info_dict)

        # fm = self.conv1X1out(fm)
        if self.fusion:
            fm = self.conv5X5out(torch.cat((res, fm), dim=1))
        return fm

class PatchConvS_ASPP_SE_MViT(nn.Module):
    def __init__(self, blocks, dims, ratios, channels):
        super(PatchConvS_ASPP_SE_MViT, self).__init__()
        self.patch_conv = patchconv(patch_size=(2, 2), in_dims=3, out_dims=channels)
        self.stages = nn.ModuleList()
        for i in range(0, len(blocks)):
            stage = BasicRes(num_blocks=blocks[i], dims=dims[i], ratios=ratios, downsample=(i + 1 < len(blocks)))
            self.stages.append(stage)
        self.ASPP = ASPP(dims[3], atrous_rates=[6, 12, 18], out_channels=dims[3])
        self.SE = SEblock(dims[3])
        self.up1 = Upsample(dims[2], dims[1])
        self.up2 = Upsample(dims[1], dims[0])
        self.up3 = Upsample(dims[0], dims[0])
        self.pwconv0 = nn.Sequential(nn.Conv2d(dims[3], dims[2], kernel_size=(1, 1), stride=(1, 1), padding=(0, 0), bias=False),
                                     nn.BatchNorm2d(dims[2]),
                                     nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.MobileViT_Block = MobileViTBlock(dims[2], n_transformer_blocks=1, ffn_latent_dim=dims[2]*3,
                                              patch_w=2, patch_h=2)
        self.pwconv = nn.Sequential(nn.Conv2d(dims[0], 1, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0), bias=False),
                                    nn.BatchNorm2d(1),
                                    nn.LeakyReLU(negative_slope=0.2, inplace=True))

    def forward(self, x):
        concat_block = []
        x = self.patch_conv(x)
        concat_block.append(x)
        for i, stage in enumerate(self.stages):
            x = stage(x)
            if i < 3:
                concat_block.append(x)
        x = self.ASPP(x)
        x = self.SE(x)
        x = self.pwconv0(x)
        x = self.MobileViT_Block(x)
        x = F.interpolate(x, scale_factor=2, mode='bilinear', align_corners=True)
        x = x + concat_block[2]
        x = self.up1(x)
        x = F.interpolate(x, scale_factor=2, mode='bilinear', align_corners=True)
        x = x + concat_block[1]
        x = self.up2(x)
        x = F.interpolate(x, scale_factor=2, mode='bilinear', align_corners=True)
        x = x + concat_block[0]
        x = self.up3(x)
        x = F.interpolate(x, scale_factor=2, mode='bilinear', align_corners=True)
        x = self.pwconv(x)
        return x

class PatchConvS_ASPP_SE_MViT_Seg(nn.Module):
    def __init__(self, blocks, dims, ratios, channels, num_classes):
        super(PatchConvS_ASPP_SE_MViT_Seg, self).__init__()
        self.patch_conv = patchconv(patch_size=(2, 2), in_dims=3, out_dims=channels)
        self.stages = nn.ModuleList()
        for i in range(0, len(blocks)):
            stage = BasicRes(num_blocks=blocks[i], dims=dims[i], ratios=ratios, downsample=(i + 1 < len(blocks)))
            self.stages.append(stage)
        self.ASPP = ASPP(dims[3], atrous_rates=[6, 12, 18], out_channels=dims[3])
        self.SE = SEblock(dims[3])
        self.pwconv0 = nn.Sequential(nn.Conv2d(dims[3], dims[2], kernel_size=(1, 1), stride=(1, 1), padding=(0, 0), bias=False),
                                     nn.BatchNorm2d(dims[2]),
                                     nn.LeakyReLU(negative_slope=0.2, inplace=True))
        self.MobileViT_Block = MobileViTBlock(dims[2], n_transformer_blocks=1, ffn_latent_dim=dims[2]*3,
                                              patch_w=2, patch_h=2)
        self.classifier = nn.Sequential(nn.Conv2d(dims[2], dims[2], kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False),
                                        nn.BatchNorm2d(dims[2]),
                                        nn.LeakyReLU(negative_slope=0.2, inplace=True),
                                        nn.Conv2d(dims[2], num_classes, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0)))
        # self.conv1 = nn.Sequential(nn.Conv2d(num_classes, num_classes, kernel_size=(1, 1), stride=(1, 1), padding=(0, 0), bias=False),
        #                            nn.BatchNorm2d(num_classes),
        #                            nn.LeakyReLU(negative_slope=0.2, inplace=True))

    def forward(self, x):
        result = OrderedDict()
        x = self.patch_conv(x)
        for i, stage in enumerate(self.stages):
            x = stage(x)
        x = self.ASPP(x)
        x = self.SE(x)
        x = self.pwconv0(x)
        x = self.MobileViT_Block(x)
        x = self.classifier(x)
        x = F.interpolate(x, size=(224, 224), mode='bilinear', align_corners=False)
        # x = self.conv1(x)
        result["out"] = x
        return result

if __name__ == '__main__':
    net = PatchConvS_ASPP_SE_MViT(blocks=[1, 1, 1, 2], ratios=2, channels=64, dims=[64, 128, 256, 512]).cpu()
    summary(net, (3, 224, 224), device='cpu')


"""This command will create and run survey jobs for each experiment
in the experiment_list. experiment list should be a file containing
one experiment accession code per line.
"""

import time
import uuid

from django.core.management.base import BaseCommand

import boto3
import botocore

from data_refinery_common.logging import get_and_configure_logger
from data_refinery_common.models import SurveyJob, SurveyJobKeyValue
from data_refinery_common.utils import parse_s3_url

logger = get_and_configure_logger(__name__)

SURVEYOR_JOB_NAME = "SURVEYOR_256"

def set_source_type_for_accession(survey_job, accession: str) -> None:
    """Type a surveyor based on accession structure"""
    if "GSE" in accession[:3]:
        survey_job.source_type = "GEO"
        survey_job.save()
        return
    elif "E-" in accession[:2]:
        survey_job.source_type = "ARRAY_EXPRESS"
        survey_job.save()
        return
    elif " " in accession:

        survey_job.source_type = "TRANSCRIPTOME_INDEX"
        survey_job.save()

        args = accession.split(",")
        # Allow organism to be unspecified so we survey the entire division.
        organism_name = args[0] if len(args[0]) > 0 else None
        if len(args) > 1:
            ensembl_division = args[1].strip()
        else:
            ensembl_division = "Ensembl"

        key_value_pair = SurveyJobKeyValue(
            survey_job=survey_job, key="ensembl_division", value=ensembl_division
        )
        key_value_pair.save()
        if organism_name:
            key_value_pair = SurveyJobKeyValue(
                survey_job=survey_job, key="organism_name", value=organism_name
            )
            key_value_pair.save()

        return
    else:
        survey_job.source_type = "SRA"
        survey_job.save()
        return

def queue_surveyor_for_accession(accession: str) -> None:
    """Dispatches a surveyor job for the accession code."""
    # Start at 1GB of RAM for surveyor jobs.
    survey_job = SurveyJob(ram_amount=1024)
    set_source_type_for_accession(survey_job, accession)

    key_value_pair = SurveyJobKeyValue(
        survey_job=survey_job, key="experiment_accession_code", value=accession
    )
    key_value_pair.save()

    # We don't actually send the job here, we just create it.
    # The foreman will pick it up and dispatch it when the time is appropriate.
    return survey_job

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            "--file",
            type=str,
            help=(
                """A file listing accession codes. s3:// URLs are also accepted."

Note: One entry per line, GSE* entries survey GEO, E-GEO-* entries survey ArrayExpress.
"""
            ),
        )

        parser.add_argument(
            "--offset", type=int, help=("Skip a number of lines at the beginning"), default=0
        )

    def handle(self, *args, **options):
        if options["file"] is None:
            logger.error("You must specify a file.")
            return "1"

        if "s3://" in options["file"]:
            bucket, key = parse_s3_url(options["file"])
            s3 = boto3.resource("s3")
            try:
                filepath = "/tmp/input_" + str(uuid.uuid4()) + ".txt"
                s3.Bucket(bucket).download_file(key, filepath)
            except botocore.exceptions.ClientError as e:
                if e.response["Error"]["Code"] == "404":
                    logger.error("The remote file does not exist.")
                    raise
                else:
                    raise
        else:
            filepath = options["file"]

        with open(filepath) as accession_file:
            for i, accession in enumerate(accession_file):
                if i < options["offset"]:
                    continue
                accession = accession.strip()
                try:
                    logger.info(f"Queuing surveyor job for {accession}.")
                    queue_surveyor_for_accession(accession)

                    # Sleep for 30 seconds so all surveyor jobs don't
                    # start at the exact same time and overload the
                    # database or ENA.
                    time.sleep(30)
                except Exception as e:
                    logger.exception(e)


from anki import hooks
from anki.utils import ids2str

from aqt import browser
from aqt.qt import *
from aqt.utils import shortcut, showInfo

def fastRepositionOnSortChanged(self, idx, ord):
    isDueSort = self.model.activeCols[idx] == 'cardDue'
    self.form.mvtotopAction.setEnabled(isDueSort)
    self.form.mvuponeAction.setEnabled(isDueSort)
    self.form.mvdownoneAction.setEnabled(isDueSort)

def setupFastRepositionActions(browser):
    """Add actions to the browser menu to move the cards up and down
    """
    # Set the actions active only if the cards are sorted by due date. This is necessary because the reposition
    # is done considering the current ordering in the browser
    mvtotopAction = QAction(_("Move to top"), browser)
    mvtotopAction.setShortcut(shortcut(_("Alt+0")))
    mvtotopAction.triggered.connect(browser.moveCardToTop)

    mvuponeAction = QAction(_("Move one up"), browser)
    mvuponeAction.setShortcut(shortcut(_("Alt+Up")))
    mvuponeAction.triggered.connect(browser.moveCardUp)

    mvdownoneAction = QAction(_("Move one down"), browser)
    mvdownoneAction.setShortcut(shortcut(_("Alt+Down")))
    mvdownoneAction.triggered.connect(browser.moveCardDown)

    browser.form.mvtotopAction = mvtotopAction
    browser.form.mvuponeAction = mvuponeAction
    browser.form.mvdownoneAction = mvdownoneAction

    browser.form.menu_Cards.addSeparator()
    browser.form.menu_Cards.addAction(mvtotopAction)
    browser.form.menu_Cards.addAction(mvuponeAction)
    browser.form.menu_Cards.addAction(mvdownoneAction)

    isDueSort = browser.col.conf['sortType'] == 'cardDue'
    browser.form.mvtotopAction.setEnabled(isDueSort)
    browser.form.mvuponeAction.setEnabled(isDueSort)
    browser.form.mvdownoneAction.setEnabled(isDueSort)

def moveCard(self, pos):
    revs = self.col.conf['sortBackwards']
    srows = self.form.tableView.selectionModel().selectedRows()

    #Get only new cards and exit if none are selected
    cids = self.selectedCards()
    cids2 = self.col.db.list(
            "select id from cards where type = 0 and id in " + ids2str(cids))
    if not cids2:
        return showInfo(_("Only new cards can be repositioned."))

    #Get the list of indexes of the selcted rows
    srowsidxes = []
    for crow in srows:
        srowsidxes.append(crow.row())

    #Check if the first (last) selected row is the first (last) on the table
    #and return in that case because it cannot moved up (down)
    if pos == -1:
        srowidx = min(srowsidxes)
        if srowidx == 0:
            return
    elif pos == 1:
        srowidx = max(srowsidxes)
        if srowidx == len(self.model.cards)-1:
            return

    #Get the index of the card on which the new due is calculated
    startidx = srowidx+pos

    #Check that the card on which the new due is calculated is a new card, otherwise the selected
    #card is at the boundary with the review cards and should not be moved
    cf = [self.model.cards[startidx]]
    cf2 = self.col.db.list(
            "select id from cards where type = 0 and id in " + ids2str(cf))
    if not cf2:
        return

    #When we move down (up) and the cards are in ascending (descending) order, the new due date must be greater by one
    #respect the due date of the next (previous) card, otherwise the due date of the selected card will be equal of that of
    #the next (previous) card but its position will be still before the next (previous) card
    inc = (revs==0 and pos>0) or (revs==1 and pos<0)

    start=self.col.getCard(self.model.cards[startidx]).due+inc

    #Perform repositioning. Copied from browser.Browser repositon method. Should be updated is changed upstream
    self.model.beginReset()
    self.mw.checkpoint(_("Reposition"))
    self.col.sched.sortCards(cids, start=start, step=1, shuffle=0, shift=1) # Preserve this line like this
    self.search()
    self.mw.requireReset()
    self.model.endReset()
    #Update the due position of the next card added.
    #This guarantees that the new cards are added a the end.
    self.col.conf['nextPos'] = self.col.db.scalar(
            "select max(due)+1 from cards where type = 0") or 0

def moveCardUp(self):
    self.moveCard(-1)

def moveCardDown(self):
    self.moveCard(1)

def moveCardToTop(self):
    #Get only new cards and exit if none are selected
    cids = self.selectedCards()
    cids2 = self.col.db.list(
            "select id from cards where type = 0 and id in " + ids2str(cids))
    if not cids2:
        return showInfo(_("Only new cards can be repositioned."))

    #Perform repositioning. Copied from browser.Browser repositon method. Should be updated is changed upstream
    self.model.beginReset()
    self.mw.checkpoint(_("Reposition"))
    self.col.sched.sortCards(cids, start=0, step=1, shuffle=0, shift=1) # Preserve this line like this
    self.search()
    self.mw.requireReset()
    self.model.endReset()
    #Update the due position of the next card added.
    #This guarantees that the new cards are added a the end.
    self.col.conf['nextPos'] = self.col.db.scalar(
            "select max(due)+1 from cards where type = 0") or 0

browser.Browser.moveCard = moveCard
browser.Browser.moveCardUp = moveCardUp
browser.Browser.moveCardDown = moveCardDown
browser.Browser.moveCardToTop = moveCardToTop

browser.Browser.onSortChanged = hooks.wrap(
    browser.Browser.onSortChanged, fastRepositionOnSortChanged)

hooks.addHook("browser.setupMenus", setupFastRepositionActions)


"""Handle the spec prescribing how to traverse and create gear(s)
"""

import click
import re
import sys
import os
import os.path as op
import shutil
import tempfile

from glob import glob
from os.path import join as opj
from inspect import ismodule
from itertools import chain

from .gear import (
    run_gear_native, run_gear_docker, create_gear, docker_push_gear,
    fw_upload_gear, copy_to_exchange
)
from . import get_logger
from .utils import import_module_from_file
from .consts import (
    GEAR_FLYWHEEL_DIR,
    GEAR_OUTPUT_DIR,
)

lgr = get_logger('spec')

from .spec_tests import _prepare, _check

def load_spec(path):
    mod_spec = import_module_from_file(op.join(path, 'spec.py'))
    spec = mod_spec.spec
    assert '%path' not in spec
    spec['%path'] = path
    return spec

def get_updated(old, new):
    """Return updated structure/value

    If new value is a scalar, it overrides the previous value.
    If it is a list, and old one is a list - returned value is a concatenation.
    If it is a dict,
    In either of the above cases if old value is None, new one is provided
    """
    if not old:
        return new
    if isinstance(new, list):
        assert isinstance(old, list)
        return old + new
    elif isinstance(new, dict):
        assert isinstance(old, dict)
        # we need to "meld" them while also updating values
        out = old.__class__()
        # we will try to preserve the order just in case original was
        # ordered dict or alike
        for k in chain(old, new):
            if k in out:
                continue  # we already saw it
            if k in old:
                old_ = old[k]
                if k in new:
                    v = get_updated(old_, new[k])
                else:
                    v = old_
            else:
                v = new[k]
            out[k] = v
        return out
    else:
        lgr.debug(
            "Forgetting old value %r since new one %r is neither dict or list",
            old, new
        )
        return new

def get_object_from_path(path, attr=None):
    """Get the object given a path

    If some analysis was done already and path was split into possible
    lead path and remaining attr, that was is provided in attr
    """
    if not path:
        raise ValueError(
            "Cannot figure out anything from an empty path. Remaining attr=%s"
            % attr)

    if path in sys.modules:
        # shortcut
        mod = sys.modules[path]
    else:
        # else we should try to import as is
        path_split = path.rsplit('.', 1)
        topmod = None if len(path_split) == 1 else path_split[0]
        try:
            mod = __import__(path, fromlist=[topmod])
        except ImportError:
            # Might have been not a module
            attr_full = path_split[-1] if not attr else '.'.join([path_split[-1], attr])
            return get_object_from_path(topmod, attr_full)

    assert mod
    # so we imported something
    if not attr:  # just a shortcut
        return mod

    # if there is attr we should get to it
    ret = mod  # start from the module
    for a in attr.split('.'):
        ret = getattr(ret, a)
    return ret

class SkipProcessing(Exception):
    pass

def get_gear_dir(path):
    """Custom tune up for the gear path to eliminate subpaths which are of no
    interest and convert into directories
    """
    # TODO: figure out a better/more reliable way
    comps = filter(lambda p: p not in {"interfaces"},
                   path.split('.'))
    return op.join(comps)

def _process(
        outputdir,  # if none provided -- nothing would be saved
        spec=None,  # TODO: make configurable
        regex=None,
        run_tests=False,
        run_tests_regex=None,
        run_testsdir=None,
        gear_actions=None,
        toppath=None,
        params={
            'recurse': False
        }
):
    """

    Parameters
    ----------
    spec: dict
    outputdir: str
    regex: str, optional
      Regular expression as to which paths to process
    include: dict, optional
    manifest: dict, optional
    params: dict, optional
    gear_actions: tuple of str
      What actions to perform to the gear, known ones: ...

    Returns
    -------
    None
    """
    gear_actions = gear_actions or ()
    lgr.log(5, toppath)
    # first process all % entries
    params_update = params.__class__()
    for param in spec or []:
        if not param.startswith('%'):
            continue
        param_ = param[1:]
        assert param_ in ('include', 'manifest', 'params', 'recurse', 'path')
        params_update[param_] = spec[param]

    # Get updated parameters
    new_params = get_updated(params, params_update)

    obj = None
    # TODO: move all the tests harnessing outside, since it should be global
    # and not straight in here
    if toppath:
        # if points to a class we need to process.  If to a module, then depends
        # on recurse
        try:
            obj = _process_gear(toppath, gear_actions, new_params,
                                outputdir, regex, run_tests, run_tests_regex,
                                run_testsdir)
        except SkipProcessing as exc:
            lgr.debug("SKIP(%s) %s", str(exc)[:100].replace('\n', ' '), toppath)

    paths_to_recurse = {
        path: path_spec
        for path, path_spec in spec.items()
        if not path.startswith('%')
    }

    if new_params['recurse'] and obj and ismodule(obj):
        # we were instructed to recurse so we will consider each path
        # which leads to either an object or another sub-module
        for attr in dir(obj):
            if attr.startswith('_'):
                continue
            subobj = getattr(obj, attr)
            subpath = '%s.%s' % (obj, attr)
            if attr not in paths_to_recurse and \
                (not ismodule(subobj) or
                    (ismodule(subobj) and
                     subobj.__name__.startswith(obj.__name__ + '.'))):
                    lgr.debug("Adding %s", subpath)
                    paths_to_recurse[attr] = {}   # no custom spec

    # after that we can traverse recursively for anything which is not %param
    for path, pathspec in paths_to_recurse.items():
        new_path = '.'.join([toppath, path]) if toppath else path

        _process(
                outputdir,
                spec=pathspec,
                regex=regex,
                run_tests=run_tests,
                run_tests_regex=run_tests_regex,
                gear_actions=gear_actions,
                toppath=new_path,
                params=new_params
        )

def _process_gear(toppath, gear_actions, params, outputdir, regex,
                  run_tests, run_tests_regex, run_testsdir):
    lgr.log(5, "Considering %s:", toppath)
    obj = get_object_from_path(toppath)
    if regex and not re.search(regex, toppath):
        raise SkipProcessing("regex")
    if 'include' in params and not params['include'](obj):
        raise SkipProcessing("%%include")
    if not obj.input_spec:
        raise SkipProcessing("no input spec")
    if not obj.output_spec:
        raise SkipProcessing("no output spec")
    if not outputdir:
        raise SkipProcessing("output_dir")
    # relative within hierarchy
    geardir = opj(*get_gear_dir(toppath))
    # full output path
    gearpath = opj(outputdir, geardir)
    gear_report = docker_image = None
    if 'skip-build' not in gear_actions:
        try:
            gear_report = create_gear(
                obj,
                gearpath,
                # Additional fields for the
                manifest_fields=params.get('manifest', {}),
                build_docker=gear_actions != "spec",  # For now
                dummy='dummy' in gear_actions,
                **params.get('params', {})
            )
            docker_image = gear_report["docker_image"]
            lgr.info("%s gear generated", toppath)
        except SyntaxError:
            # some grave error -- blow
            raise
        except NotImplementedError as exc:
            lgr.warning("Skipping: %s", exc)
            raise SkipProcessing("Not implemented yet")
        # except Exception as e:
        #     lgr.warning("ERROR happened: %s" % str(e))
        #     raise SkipProcessing("ERROR happened: %s" % str(e))
    elif op.exists(gearpath):
        lgr.info("Processing %s:", toppath)
    else:
        raise SkipProcessing("Gear is not built and there is no gear directory")
    if run_tests != "skip":
        # TODO Move away and generalize
        testspath = op.join(gearpath, 'tests')
        tests = glob(op.join(testspath, '*.yaml'))
        if not tests:
            lgr.warning(" TESTS: no tests were found")
        else:
            lgr.info(" TESTS: found %d tests", len(tests))
        for itest, test in enumerate(sorted(tests)):
            testname = op.splitext(op.basename(test))[0]
            testmsg = "  test #%d: %s" % (itest + 1, testname)
            if run_tests_regex:
                if not re.match(run_tests_regex, testname):
                    lgr.info(testmsg + " skipped")
                    continue
            lgr.debug("  running" + testmsg)
            # TODO: Redo all the below to just use one of the runners
            # such as pytest internally
            try:
                if run_testsdir is not None:
                    testdir = tempfile.mkdtemp(prefix='gf_test-%d_' % itest)
                else:
                    # create one under outputdir replicating testspath hierarchy
                    testdir = op.join(
                        params['path'], 'tests-run', geardir, testname)
                    if op.exists(testdir):
                        shutil.rmtree(testdir)

                _prepare(test, testdir)

                if run_tests == 'native':
                    run_gear_native(gearpath, testdir)
                elif run_tests == 'gear':
                    if not gear_report:
                        raise ValueError("-g option must not be 'skip-build'")
                    run_gear_docker(docker_image, testdir)
                    # change ownership back from root on output directory
                    # Redone via uid:gid mapping into Docker container
                    # and making all needed components readable with changes
                    # to Dockerfile
                    # run_gear_docker(docker_image, testdir,
                    #                 ["chown", "-R", os.getuid(),
                    #                  "%s/%s" % (GEAR_FLYWHEEL_DIR,
                    # GEAR_OUTPUT_DIR)])
                else:
                    raise ValueError(run_tests)

                _check(test, testdir)
                #  verify correspondence of # of files with target outputs
                #  run the tests specified in tests.yaml if any, if none -
                #  assume that they all must be identical
                lgr.info(testmsg + " passed")
            except Exception as exc:
                # for now pdb on generic --pdb if there was some setup
                lgr.error(testmsg + " FAILED: %s", exc)
                from .utils import _sys_excepthook
                if sys.excepthook != _sys_excepthook:
                    # TODO: we could call out to sys.excepthook with
                    #   (type, value, tb) details instead of reraising
                    #  that should allow to proceed after pdb session
                    raise
            finally:
                # TODO shutil.rmtree(testdir)
                import os
                # os.system("ls -lRa %s/*" % testdir)
                pass
        pass
    if 'docker-push' in gear_actions:
        if not gear_report:
            raise ValueError("-g option must not be 'skip-build'")
        docker_push_gear(docker_image)
    if 'fw-upload' in gear_actions:
        fw_upload_gear(gearpath)
    if 'exchange' in gear_actions:
        for exchange in glob(opj(outputdir, '..', 'exchanges', '*')):
            copy_to_exchange(gearpath, exchange)
    return obj

# CLI

from .cli_base import cli

@cli.group("spec")
def grp():
    """Commands to manipulate the spec"""
    pass

@grp.command()
@click.option('--regex', help='Regular expression to process only the '
                              'matching paths')
@click.option('--run-tests',
              type=click.Choice(['skip', 'native', 'gear']), default='gear',
              help='Run tests if present.  "native" runs on the host and '
                   '"gear" '
                   'via the dockerized gear')
@click.option('--run-tests-regex', help='Regular expression to run only the '
                                        'matching tests')
@click.option('--gear-actions', '-g', type=click.Choice(
              ['spec', 'skip-build', 'dummy', 'build', 'docker-push',
               'fw-upload', 'exchange']),
              multiple=True,
              help="Actions to perform on gear (dummy for testing UI or just spec "
                   "or full) or just skip (and possibly just do the tests etc)")
# @click.option('--docker', type=click.Choice(['skip', 'dummy', 'build']),
#               help='Either actually build a docker image. "dummy" would generate'
#                    ' a minimalistic image useful for quick upload to test '
#                    'web UI')
@click.option('--run-testsdir', help='Directory, under which run the tests. If none provided, will be tests-runs/ under outputdir')
@click.option('-o', '--outputdir', help='Output directory, to not place under gears/ alongside the spec')
@click.argument('inputdir') # , help='Directory with the spec.py and tests/, ...')
def process(
        inputdir,
        outputdir=None,  # if none provided -- nothing would be saved
        run_testsdir=None,
        **kwargs
):
    """Load and process the spec

    The "spec" ATM is a directory with

    \b
    - spec.py
       file which prescribes how to traverse class hierarchy, and provides
       custom fields for gears
    - inputs/
       directory containing datasets used in tests
    - tests/
       directory mimicing produced gears/ directory hierarchy but providing
       per gear testname.yaml with specification of the test, and testname/
       directory with target outputs
    - gears/
       directory gets generated/used as the default `outputdir/`

    Recommended to keep inputs/ and tests output directories content under
    git-annex to minimize storage requirement etc
    """
    spec = load_spec(inputdir)
    if outputdir is None:
        outputdir = op.join(inputdir, 'gears')
    return _process(outputdir, spec=spec, run_testsdir=run_testsdir, **kwargs)