# coding=utf-8
"""Unit tests for activitypub.py."""
from base64 import b64encode
import copy
from datetime import datetime, timedelta
from hashlib import sha256
import logging
from unittest import skip
from unittest.mock import patch

from flask import g
from google.cloud import ndb
from granary import as2, microformats2
from httpsig import HeaderSigner
from oauth_dropins.webutil.testutil import requests_response
from oauth_dropins.webutil.util import json_dumps, json_loads
import requests
from urllib3.exceptions import ReadTimeoutError
from werkzeug.exceptions import BadGateway

# import first so that Fake is defined before URL routes are registered
from .testutil import Fake, TestCase

import activitypub
from activitypub import ActivityPub, postprocess_as2
import common
from models import Follower, Object
import protocol
from web import Web

# have to import module, not attrs, to avoid circular import
from . import test_web

ACTOR = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://mas.to/users/swentel',
    'type': 'Person',
    'inbox': 'http://mas.to/inbox',
    'name': 'Mrs. ☕ Foo',
    'icon': {'type': 'Image', 'url': 'https://user.com/me.jpg'},
    'image': {'type': 'Image', 'url': 'https://user.com/me.jpg'},
}
ACTOR_BASE = {
    '@context': [
        'https://www.w3.org/ns/activitystreams',
        'https://w3id.org/security/v1',
    ],
    'type': 'Person',
    'id': 'http://localhost/user.com',
    'url': 'http://localhost/r/https://user.com/',
    'preferredUsername': 'user.com',
    'summary': '',
    'inbox': 'http://localhost/user.com/inbox',
    'outbox': 'http://localhost/user.com/outbox',
    'following': 'http://localhost/user.com/following',
    'followers': 'http://localhost/user.com/followers',
    'endpoints': {
        'sharedInbox': 'http://localhost/ap/sharedInbox',
    },
    'publicKey': {
        'id': 'http://localhost/user.com#key',
        'owner': 'http://localhost/user.com',
        'publicKeyPem': 'populated in setUp()',
    },
}
ACTOR_BASE_FULL = {
    **ACTOR_BASE,
    'name': 'Ms. ☕ Baz',
    'attachment': [{
        'name': 'Web site',
        'type': 'PropertyValue',
        'value': 'https://user.com/',
    }],
}
REPLY_OBJECT = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Note',
    'content': 'A ☕ reply',
    'id': 'http://mas.to/reply/id',
    'url': 'http://mas.to/reply',
    'inReplyTo': 'https://user.com/post',
    'to': [as2.PUBLIC_AUDIENCE],
}
REPLY_OBJECT_WRAPPED = copy.deepcopy(REPLY_OBJECT)
REPLY_OBJECT_WRAPPED['inReplyTo'] = 'http://localhost/r/https://user.com/post'
REPLY = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Create',
    'id': 'http://mas.to/reply/as2',
    'object': REPLY_OBJECT,
}
NOTE_OBJECT = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Note',
    'content': '☕ just a normal post',
    'id': 'http://mas.to/note/id',
    'url': 'http://mas.to/note',
    'to': [as2.PUBLIC_AUDIENCE],
    'cc': [
        'https://mas.to/author/followers',
        'https://masto.foo/@other',
        'http://localhost/target',  # redirect-wrapped
    ],
}
NOTE = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Create',
    'id': 'http://mas.to/note/as2',
    'actor': 'https://masto.foo/@author',
    'object': NOTE_OBJECT,
}
MENTION_OBJECT = copy.deepcopy(NOTE_OBJECT)
MENTION_OBJECT.update({
    'id': 'http://mas.to/mention/id',
    'url': 'http://mas.to/mention',
    'tag': [{
        'type': 'Mention',
        'href': 'https://masto.foo/@other',
        'name': '@other@masto.foo',
    }, {
        'type': 'Mention',
        'href': 'http://localhost/tar.get',  # redirect-wrapped
        'name': '@tar.get@tar.get',
    }],
})
MENTION = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Create',
    'id': 'http://mas.to/mention/as2',
    'object': MENTION_OBJECT,
}
# based on example Mastodon like:
# https://github.com/snarfed/bridgy-fed/issues/4#issuecomment-334212362
# (reposts are very similar)
LIKE = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'http://mas.to/like#ok',
    'type': 'Like',
    'object': 'https://user.com/post',
    'actor': 'https://mas.to/actor',
}
LIKE_WRAPPED = copy.deepcopy(LIKE)
LIKE_WRAPPED['object'] = 'http://localhost/r/https://user.com/post'
LIKE_ACTOR = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://mas.to/actor',
    'type': 'Person',
    'name': 'Ms. Actor',
    'preferredUsername': 'msactor',
    'icon': {'type': 'Image', 'url': 'https://user.com/pic.jpg'},
    'image': [
        {'type': 'Image', 'url': 'https://user.com/thumb.jpg'},
        {'type': 'Image', 'url': 'https://user.com/pic.jpg'},
    ],
}
LIKE_WITH_ACTOR = {
    **LIKE,
    'actor': LIKE_ACTOR,
}

# repost, should be delivered to followers if object is a fediverse post,
# translated to webmention if object is an indieweb post
REPOST = {
  '@context': 'https://www.w3.org/ns/activitystreams',
  'id': 'https://mas.to/users/alice/statuses/654/activity',
  'type': 'Announce',
  'actor': ACTOR['id'],
  'object': NOTE_OBJECT['id'],
  'published': '2023-02-08T17:44:16Z',
  'to': ['https://www.w3.org/ns/activitystreams#Public'],
}
REPOST_FULL = {
    **REPOST,
    'actor': ACTOR,
    'object': NOTE_OBJECT,
}

FOLLOW = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://mas.to/6d1a',
    'type': 'Follow',
    'actor': ACTOR['id'],
    'object': 'https://user.com/',
}
FOLLOW_WRAPPED = copy.deepcopy(FOLLOW)
FOLLOW_WRAPPED['object'] = 'http://localhost/user.com'
FOLLOW_WITH_ACTOR = copy.deepcopy(FOLLOW)
FOLLOW_WITH_ACTOR['actor'] = ACTOR
FOLLOW_WRAPPED_WITH_ACTOR = copy.deepcopy(FOLLOW_WRAPPED)
FOLLOW_WRAPPED_WITH_ACTOR['actor'] = ACTOR
FOLLOW_WITH_OBJECT = copy.deepcopy(FOLLOW)
FOLLOW_WITH_OBJECT['object'] = ACTOR

ACCEPT_FOLLOW = copy.deepcopy(FOLLOW_WITH_ACTOR)
del ACCEPT_FOLLOW['@context']
del ACCEPT_FOLLOW['actor']['@context']
ACCEPT_FOLLOW['actor']['image'] = {'type': 'Image', 'url': 'https://user.com/me.jpg'}
ACCEPT_FOLLOW['object'] = 'http://localhost/user.com'
ACCEPT = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'type': 'Accept',
    'id': 'http://localhost/web/user.com/followers#accept-https://mas.to/6d1a',
    'actor': 'http://localhost/user.com',
    'object': {
        **ACCEPT_FOLLOW,
        'url': 'https://mas.to/users/swentel#followed-https://user.com/',
        'to': ['https://www.w3.org/ns/activitystreams#Public'],
    },
   'to': ['https://www.w3.org/ns/activitystreams#Public'],
}

UNDO_FOLLOW_WRAPPED = {
  '@context': 'https://www.w3.org/ns/activitystreams',
  'id': 'https://mas.to/6d1b',
  'type': 'Undo',
  'actor': 'https://mas.to/users/swentel',
  'object': FOLLOW_WRAPPED,
}

DELETE = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://mas.to/users/swentel#delete',
    'type': 'Delete',
    'actor': 'https://mas.to/users/swentel',
    'object': 'https://mas.to/users/swentel',
}

UPDATE_PERSON = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://a/person#update',
    'type': 'Update',
    'actor': 'https://mas.to/users/swentel',
    'object': {
        'type': 'Person',
        'id': 'https://a/person',
    },
}
UPDATE_NOTE = {
    '@context': 'https://www.w3.org/ns/activitystreams',
    'id': 'https://a/note#update',
    'type': 'Update',
    'actor': 'https://mas.to/users/swentel',
    'object': {
        'type': 'Note',
        'id': 'https://a/note',
    },
}
WEBMENTION_DISCOVERY = requests_response(
    '<link rel="webmention" href="/webmention">')

HTML = requests_response('', headers={
    'Content-Type': common.CONTENT_TYPE_HTML,
})
HTML_WITH_AS2 = requests_response("""\
<meta>
<link href='http://as2' rel='alternate' type='application/activity+json'>
</meta>
""", headers={
    'Content-Type': common.CONTENT_TYPE_HTML,
})
AS2_OBJ = {'foo': ['bar']}
AS2 = requests_response(AS2_OBJ, headers={
    'Content-Type': as2.CONTENT_TYPE,
})
NOT_ACCEPTABLE = requests_response(status=406)

@patch('requests.post')
@patch('requests.get')
@patch('requests.head')
class ActivityPubTest(TestCase):

    def setUp(self):
        super().setUp()
        self.request_context.push()

        self.user = self.make_user('user.com', has_hcard=True, has_redirects=True,
                                   obj_as2={**ACTOR, 'id': 'https://user.com/'})
        self.swentel_key = ndb.Key(ActivityPub, 'https://mas.to/users/swentel')
        self.masto_actor_key = ndb.Key(ActivityPub, 'https://mas.to/actor')

        ACTOR_BASE['publicKey']['publicKeyPem'] = self.user.public_pem().decode()

        self.key_id_obj = Object(id='http://my/key/id', as2={
            **ACTOR,
            'publicKey': {
                'id': 'http://my/key/id#unused',
                'owner': 'http://own/er',
                'publicKeyPem': self.user.public_pem().decode(),
            },
        })
        self.key_id_obj.put()

    def assert_object(self, id, **props):
        props.setdefault('delivered_protocol', 'web')
        return super().assert_object(id, **props)

    def sign(self, path, body):
        """Constructs HTTP Signature, returns headers."""
        digest = b64encode(sha256(body.encode()).digest()).decode()
        headers = {
            'Date': 'Sun, 02 Jan 2022 03:04:05 GMT',
            'Host': 'localhost',
            'Content-Type': as2.CONTENT_TYPE,
            'Digest': f'SHA-256={digest}',
        }
        hs = HeaderSigner('http://my/key/id#unused', self.user.private_pem().decode(),
                          algorithm='rsa-sha256', sign_header='signature',
                          headers=('Date', 'Host', 'Digest', '(request-target)'))
        return hs.sign(headers, method='POST', path=path)

    def post(self, path, json=None):
        """Wrapper around self.client.post that adds signature."""
        body = json_dumps(json)
        return self.client.post(path, data=body, headers=self.sign(path, body))

    def test_actor_fake(self, *_):
        self.make_user('user.com', cls=Fake, obj_as2={
            'type': 'Person',
            'id': 'https://user.com/',
        })

        got = self.client.get('/ap/fake/user.com')
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))
        type = got.headers['Content-Type']
        self.assertTrue(type.startswith(as2.CONTENT_TYPE), type)
        self.assertEqual({
            '@context': ['https://w3id.org/security/v1'],
            'type': 'Person',
            'id': 'http://bf/fake/user.com/ap',
            'preferredUsername': 'user.com',
            'url': 'http://localhost/r/user.com',
            'summary': '',
            'inbox': 'http://bf/fake/user.com/ap/inbox',
            'outbox': 'http://bf/fake/user.com/ap/outbox',
            'following': 'http://bf/fake/user.com/ap/following',
            'followers': 'http://bf/fake/user.com/ap/followers',
            'endpoints': {'sharedInbox': 'http://localhost/ap/sharedInbox'},
            'publicKey': {
                'id': 'http://localhost/user.com#key',
                'owner': 'http://localhost/user.com',
                'publicKeyPem': self.user.public_pem().decode(),
            },
        }, got.json)

    def test_actor_web(self, *_):
        """Web users are special cased to drop the /web/ prefix."""
        got = self.client.get('/user.com')
        self.assertEqual(200, got.status_code)
        type = got.headers['Content-Type']
        self.assertTrue(type.startswith(as2.CONTENT_TYPE), type)
        self.assertEqual({
            **ACTOR_BASE,
            'name': 'Mrs. ☕ Foo',
            'icon': {'type': 'Image', 'url': 'https://user.com/me.jpg'},
            'image': {'type': 'Image', 'url': 'https://user.com/me.jpg'},
        }, got.json)

    def test_actor_blocked_tld(self, _, __, ___):
        got = self.client.get('/foo.json')
        self.assertEqual(404, got.status_code)

    def test_actor_new_user_fetch(self, _, mock_get, __):
        self.user.obj_key.delete()
        self.user.key.delete()
        protocol.objects_cache.clear()

        mock_get.return_value = requests_response(test_web.ACTOR_HTML)

        got = self.client.get('/user.com')
        self.assertEqual(200, got.status_code)
        self.assert_equals(ACTOR_BASE_FULL, got.json, ignore=['publicKeyPem'])

    def test_actor_new_user_fetch_no_mf2(self, _, mock_get, __):
        self.user.obj_key.delete()
        self.user.key.delete()
        protocol.objects_cache.clear()

        mock_get.return_value = requests_response('')

        got = self.client.get('/user.com')
        self.assertEqual(200, got.status_code)
        self.assert_equals(ACTOR_BASE, got.json, ignore=['publicKeyPem'])

    def test_actor_new_user_fetch_fails(self, _, mock_get, __):
        mock_get.side_effect = ReadTimeoutError(None, None, None)
        got = self.client.get('/nope.com')
        self.assertEqual(504, got.status_code)

    def test_individual_inbox_no_user(self, mock_head, mock_get, mock_post):
        self.user.key.delete()

        mock_get.side_effect = [self.as2_resp(LIKE_ACTOR)]

        reply = {
            **REPLY,
            'actor': LIKE_ACTOR,
        }
        self._test_inbox_reply(reply, mock_head, mock_get, mock_post)

        self.assert_user(ActivityPub, 'https://mas.to/actor',
                         obj_as2=LIKE_ACTOR, direct=True)

    def test_inbox_activity_without_id(self, *_):
        note = copy.deepcopy(NOTE)
        del note['id']
        resp = self.post('/ap/sharedInbox', json=note)
        self.assertEqual(400, resp.status_code)

    def test_inbox_reply_object(self, mock_head, mock_get, mock_post):
        self._test_inbox_reply(REPLY_OBJECT, mock_head, mock_get, mock_post)

        self.assert_object('http://mas.to/reply/id',
                           source_protocol='activitypub',
                           our_as1=as2.to_as1(REPLY_OBJECT),
                           type='comment')
        # auto-generated post activity
        self.assert_object(
            'http://mas.to/reply/id#bridgy-fed-create',
            source_protocol='activitypub',
            our_as1={
                **as2.to_as1(REPLY),
                'id': 'http://mas.to/reply/id#bridgy-fed-create',
                'published': '2022-01-02T03:04:05+00:00',
            },
            status='complete',
            delivered=['https://user.com/post'],
            type='post',
            notify=[self.user.key],
        )

    def test_inbox_reply_object_wrapped(self, mock_head, mock_get, mock_post):
        self._test_inbox_reply(REPLY_OBJECT_WRAPPED, mock_head, mock_get, mock_post)

        self.assert_object('http://mas.to/reply/id',
                           source_protocol='activitypub',
                           our_as1=as2.to_as1(REPLY_OBJECT),
                           type='comment')
        # auto-generated post activity
        self.assert_object(
            'http://mas.to/reply/id#bridgy-fed-create',
            source_protocol='activitypub',
            our_as1={
                **as2.to_as1(REPLY),
                'id': 'http://mas.to/reply/id#bridgy-fed-create',
                'published': '2022-01-02T03:04:05+00:00',
            },
            status='complete',
            delivered=['https://user.com/post'],
            type='post',
            notify=[self.user.key],
        )

    def test_inbox_reply_create_activity(self, mock_head, mock_get, mock_post):
        self._test_inbox_reply(REPLY, mock_head, mock_get, mock_post)

        self.assert_object('http://mas.to/reply/id',
                           source_protocol='activitypub',
                           our_as1=as2.to_as1({
                               **REPLY_OBJECT,
                               'author': None,
                           }),
                           type='comment')
        # sent activity
        self.assert_object(
            'http://mas.to/reply/as2',
            source_protocol='activitypub',
            as2=REPLY,
            status='complete',
            delivered=['https://user.com/post'],
            type='post',
            notify=[self.user.key],
        )

    def _test_inbox_reply(self, reply, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/post')
        mock_get.side_effect = (
            (list(mock_get.side_effect) if mock_get.side_effect else [])
            + [
                requests_response(test_web.NOTE_HTML),
                requests_response(test_web.NOTE_HTML),
                WEBMENTION_DISCOVERY,
            ])
        mock_post.return_value = requests_response()

        got = self.post('/ap/web/user.com/inbox', json=reply)
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))
        self.assert_req(mock_get, 'https://user.com/post')

        convert_id = reply['id'].replace('://', ':/')
        if reply['type'] != 'Create':
            convert_id += '%23bridgy-fed-create'

        self.assert_req(
            mock_post,
            'https://user.com/webmention',
            headers={'Accept': '*/*'},
            allow_redirects=False,
            data={
                'source': f'https://ap.brid.gy/convert/web/{convert_id}',
                'target': 'https://user.com/post',
            },
        )

    def test_inbox_reply_to_self_domain(self, *mocks):
        self._test_inbox_ignore_reply_to('http://localhost/mas.to', *mocks)

    def test_inbox_reply_to_in_blocklist(self, *mocks):
        self._test_inbox_ignore_reply_to('https://twitter.com/foo', *mocks)

    def _test_inbox_ignore_reply_to(self, reply_to, mock_head, mock_get, mock_post):
        reply = copy.deepcopy(REPLY_OBJECT)
        reply['inReplyTo'] = reply_to

        mock_head.return_value = requests_response(url='http://mas.to/')
        mock_get.side_effect = [
            # actor fetch
            self.as2_resp(ACTOR),
            # protocol inference
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
        ]

        got = self.post('/user.com/inbox', json=reply)
        self.assertEqual(204, got.status_code, got.get_data(as_text=True))
        mock_post.assert_not_called()

    def test_individual_inbox_create_obj(self, *mocks):
        self._test_inbox_create_obj('/user.com/inbox', *mocks)

    def test_shared_inbox_create_obj(self, *mocks):
        self._test_inbox_create_obj('/inbox', *mocks)

    def _test_inbox_create_obj(self, path, mock_head, mock_get, mock_post):
        swentel = self.make_user('https://mas.to/users/swentel', cls=ActivityPub)
        Follower.get_or_create(to=swentel, from_=self.user)
        bar = self.make_user('fake:bar', cls=Fake, obj_id='fake:bar')
        Follower.get_or_create(to=self.make_user('https://other.actor',
                                                 cls=ActivityPub),
                               from_=bar)
        baz = self.make_user('fake:baz', cls=Fake, obj_id='fake:baz')
        Follower.get_or_create(to=swentel, from_=baz)
        baj = self.make_user('fake:baj', cls=Fake, obj_id='fake:baj')
        Follower.get_or_create(to=swentel, from_=baj, status='inactive')

        mock_head.return_value = requests_response(url='http://target')
        mock_get.return_value = self.as2_resp(ACTOR)  # source actor
        mock_post.return_value = requests_response()

        got = self.post(path, json=NOTE)
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))

        expected_obj = {
            **as2.to_as1(NOTE_OBJECT),
            'author': {'id': 'https://masto.foo/@author'},
        }
        self.assert_object(NOTE_OBJECT['id'],
                           source_protocol='activitypub',
                           our_as1=expected_obj,
                           type='note',
                           feed=[self.user.key, baz.key])

        expected_create = as2.to_as1(common.redirect_unwrap(NOTE))
        expected_create.update({
            'actor': as2.to_as1(ACTOR),
            'object': expected_obj,
        })
        self.assert_object('http://mas.to/note/as2',
                           source_protocol='activitypub',
                           our_as1=expected_create,
                           users=[ndb.Key(ActivityPub, 'https://masto.foo/@author')],
                           type='post',
                           object_ids=[NOTE_OBJECT['id']],
                           status='complete',
                           delivered=['shared:target'],
                           delivered_protocol='fake')

    def test_repost_of_indieweb(self, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/orig')
        mock_get.return_value = WEBMENTION_DISCOVERY
        mock_post.return_value = requests_response()  # webmention

        orig_url = 'https://user.com/orig'
        note = {
            **NOTE_OBJECT,
            'id': 'https://user.com/orig',
        }
        del note['url']
        Object(id=orig_url, mf2=microformats2.object_to_json(as2.to_as1(note)),
               source_protocol='web').put()

        repost = copy.deepcopy(REPOST_FULL)
        repost['object'] = f'http://localhost/r/{orig_url}'
        got = self.post('/user.com/inbox', json=repost)
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))

        convert_id = REPOST['id'].replace('://', ':/')
        self.assert_req(
            mock_post,
            'https://user.com/webmention',
            headers={'Accept': '*/*'},
            allow_redirects=False,
            data={
                'source': f'https://ap.brid.gy/convert/web/{convert_id}',
                'target': orig_url,
            },
        )

        self.assert_object(REPOST_FULL['id'],
                           source_protocol='activitypub',
                           status='complete',
                           as2={
                               **REPOST,
                               'actor': ACTOR,
                               'object': orig_url,
                           },
                           users=[self.swentel_key],
                           delivered=['https://user.com/orig'],
                           type='share',
                           object_ids=['https://user.com/orig'])

    def test_shared_inbox_repost_of_fediverse(self, mock_head, mock_get, mock_post):
        Follower.get_or_create(to=ActivityPub.get_or_create(ACTOR['id']),
                               from_=self.user)
        baz = self.make_user('fake:baz', cls=Fake, obj_id='fake:baz')
        Follower.get_or_create(to=ActivityPub.get_or_create(ACTOR['id']), from_=baz)
        baj = self.make_user('fake:baj', cls=Fake, obj_id='fake:baj')
        Follower.get_or_create(to=ActivityPub.get_or_create(ACTOR['id']),
                               from_=baj, status='inactive')

        mock_head.return_value = requests_response(url='http://target')
        mock_get.side_effect = [
            self.as2_resp(ACTOR),  # source actor
            self.as2_resp(NOTE_OBJECT),  # object of repost
            # protocol inference
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
            HTML,  # no webmention endpoint
        ]

        got = self.post('/ap/sharedInbox', json=REPOST)
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))

        mock_post.assert_not_called()  # no webmention

        self.assert_object(REPOST['id'],
                           source_protocol='activitypub',
                           status='complete',
                           our_as1=as2.to_as1({**REPOST, 'actor': ACTOR}),
                           users=[self.swentel_key],
                           feed=[self.user.key, baz.key],
                           delivered=['shared:target'],
                           delivered_protocol='fake',
                           type='share',
                           object_ids=[REPOST['object']])

    def test_inbox_no_user(self, mock_head, mock_get, mock_post):
        mock_get.side_effect = [
            # source actor
            self.as2_resp(LIKE_WITH_ACTOR['actor']),
            # protocol inference
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
            # target post webmention discovery
            HTML,
        ]

        got = self.post('/ap/sharedInbox', json={
            **LIKE,
            'object': 'http://nope.com/post',
        })
        self.assertEqual(204, got.status_code)

        self.assert_object('http://mas.to/like#ok',
                           # no nope.com Web user key since it didn't exist
                           source_protocol='activitypub',
                           status='ignored',
                           our_as1=as2.to_as1({
                               **LIKE_WITH_ACTOR,
                               'object': 'http://nope.com/post',
                           }),
                           type='like',
                           notify=[self.user.key],
                           users=[self.masto_actor_key],
                           object_ids=['http://nope.com/post'])

    def test_inbox_not_public(self, mock_head, mock_get, mock_post):
        Follower.get_or_create(to=ActivityPub.get_or_create(ACTOR['id']),
                               from_=self.user)

        mock_head.return_value = requests_response(url='http://target')
        mock_get.return_value = self.as2_resp(ACTOR)  # source actor

        not_public = copy.deepcopy(NOTE)
        del not_public['object']['to']

        got = self.post('/user.com/inbox', json=not_public)
        self.assertEqual(200, got.status_code, got.get_data(as_text=True))

        self.assertIsNone(Object.get_by_id(not_public['id']))
        self.assertIsNone(Object.get_by_id(not_public['object']['id']))

    def test_inbox_like(self, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/post')
        mock_get.side_effect = [
            # source actor
            self.as2_resp(LIKE_WITH_ACTOR['actor']),
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        got = self.post('/user.com/inbox', json=LIKE)
        self.assertEqual(200, got.status_code)

        self.assertIn(self.as2_req('https://mas.to/actor'), mock_get.mock_calls)
        self.assertIn(self.req('https://user.com/post'), mock_get.mock_calls)

        args, kwargs = mock_post.call_args
        self.assertEqual(('https://user.com/webmention',), args)
        self.assertEqual({
            'source': 'https://ap.brid.gy/convert/web/http:/mas.to/like%23ok',
            'target': 'https://user.com/post',
        }, kwargs['data'])

        self.assert_object('http://mas.to/like#ok',
                           notify=[self.user.key],
                           users=[self.masto_actor_key],
                           source_protocol='activitypub',
                           status='complete',
                           our_as1=as2.to_as1(LIKE_WITH_ACTOR),
                           delivered=['https://user.com/post'],
                           type='like',
                           object_ids=[LIKE['object']])

    def test_inbox_like_indirect_user_creates_User(self, mock_get, *_):
        self.user.direct = False
        self.user.put()

        mock_get.return_value = self.as2_resp(LIKE_ACTOR)

        self.test_inbox_like()
        self.assert_user(ActivityPub, 'https://mas.to/actor',
                         obj_as2=LIKE_ACTOR, direct=True)

    def test_inbox_follow_accept_with_id(self, *mocks):
        self._test_inbox_follow_accept(FOLLOW_WRAPPED, ACCEPT, 200, *mocks)

        follow = {
            **FOLLOW_WITH_ACTOR,
            'url': 'https://mas.to/users/swentel#followed-https://user.com/',
        }
        self.assert_object('https://mas.to/6d1a',
                           users=[self.swentel_key],
                           notify=[self.user.key],
                           source_protocol='activitypub',
                           status='complete',
                           our_as1=as2.to_as1(follow),
                           delivered=['https://user.com/'],
                           type='follow',
                           object_ids=[FOLLOW['object']])

    def test_inbox_follow_accept_with_object(self, *mocks):
        follow = {
            **FOLLOW,
            'object': {
                'id': FOLLOW['object'],
                'url': FOLLOW['object'],
            },
        }
        self._test_inbox_follow_accept(follow, ACCEPT, 200, *mocks)

        follow.update({
            'actor': ACTOR,
            'url': 'https://mas.to/users/swentel#followed-https://user.com/',
        })
        self.assert_object('https://mas.to/6d1a',
                           users=[self.swentel_key],
                           notify=[self.user.key],
                           source_protocol='activitypub',
                           status='complete',
                           our_as1=as2.to_as1(follow),
                           delivered=['https://user.com/'],
                           type='follow',
                           object_ids=[FOLLOW['object']])

    def test_inbox_follow_accept_shared_inbox(self, *mocks):
        self._test_inbox_follow_accept(FOLLOW_WRAPPED, ACCEPT, 200, *mocks,
                                       inbox_path='/ap/sharedInbox')

        url = 'https://mas.to/users/swentel#followed-https://user.com/'
        self.assert_object('https://mas.to/6d1a',
                           users=[self.swentel_key],
                           notify=[self.user.key],
                           source_protocol='activitypub',
                           status='complete',
                           our_as1=as2.to_as1({**FOLLOW_WITH_ACTOR, 'url': url}),
                           delivered=['https://user.com/'],
                           type='follow',
                           object_ids=[FOLLOW['object']])

    def test_inbox_follow_accept_webmention_fails(self, mock_head, mock_get,
                                                  mock_post):
        mock_post.side_effect = [
            requests_response(),         # AP Accept
            requests.ConnectionError(),  # webmention
        ]
        self._test_inbox_follow_accept(FOLLOW_WRAPPED, ACCEPT, 502,
                                       mock_head, mock_get, mock_post)

        url = 'https://mas.to/users/swentel#followed-https://user.com/'
        self.assert_object('https://mas.to/6d1a',
                           users=[self.swentel_key],
                           notify=[self.user.key],
                           source_protocol='activitypub',
                           status='failed',
                           our_as1=as2.to_as1({**FOLLOW_WITH_ACTOR, 'url': url}),
                           delivered=[],
                           failed=['https://user.com/'],
                           type='follow',
                           object_ids=[FOLLOW['object']])

    def _test_inbox_follow_accept(self, follow_as2, accept_as2, expected_status,
                                  mock_head, mock_get, mock_post,
                                  inbox_path='/user.com/inbox'):
        # this should makes us make the follower ActivityPub as direct=True
        self.user.direct = False
        self.user.put()

        mock_head.return_value = requests_response(url='https://user.com/')
        mock_get.side_effect = [
            # source actor
            self.as2_resp(ACTOR),
            WEBMENTION_DISCOVERY,
        ]
        if not mock_post.return_value and not mock_post.side_effect:
            mock_post.return_value = requests_response()

        got = self.post(inbox_path, json=follow_as2)
        self.assertEqual(expected_status, got.status_code)

        mock_get.assert_has_calls((
            self.as2_req(FOLLOW['actor']),
        ))

        # check AP Accept
        self.assertEqual(2, len(mock_post.call_args_list))
        args, kwargs = mock_post.call_args_list[0]
        self.assertEqual(('http://mas.to/inbox',), args)
        self.assertEqual(accept_as2, json_loads(kwargs['data']))

        # check webmention
        args, kwargs = mock_post.call_args_list[1]
        self.assertEqual(('https://user.com/webmention',), args)
        self.assertEqual({
            'source': 'https://ap.brid.gy/convert/web/https:/mas.to/6d1a',
            'target': 'https://user.com/',
        }, kwargs['data'])

        # check that we stored Follower and ActivityPub user for the follower
        self.assert_entities_equal(
            Follower(to=self.user.key,
                     from_=ActivityPub(id=ACTOR['id']).key,
                     status='active',
                     follow=Object(id=FOLLOW['id']).key),
            Follower.query().fetch(),
            ignore=['created', 'updated'])

        self.assert_user(ActivityPub, 'https://mas.to/users/swentel',
                         obj_as2=ACTOR, direct=True)
        self.assert_user(Web, 'user.com', direct=False,
                         has_hcard=True, has_redirects=True)

    def test_inbox_follow_use_instead_strip_www(self, mock_head, mock_get, mock_post):
        self.make_user('www.user.com', use_instead=self.user.key)

        mock_head.return_value = requests_response(url='https://www.user.com/')
        mock_get.side_effect = [
            # source actor
            self.as2_resp(ACTOR),
            # target post webmention discovery
            requests_response(''),
        ]
        mock_post.return_value = requests_response()

        got = self.post('/user.com/inbox', json=FOLLOW_WRAPPED)
        self.assertEqual(204, got.status_code)

        follower = Follower.query().get()
        self.assert_entities_equal(
            Follower(to=self.user.key,
                     from_=ActivityPub(id=ACTOR['id']).key,
                     status='active',
                     follow=Object(id=FOLLOW['id']).key),
            follower,
            ignore=['created', 'updated'])

        # double check that Follower doesn't have www
        self.assertEqual('user.com', follower.to.id())

        # double check that follow Object doesn't have www
        self.assertEqual('active', follower.status)
        self.assertEqual('https://mas.to/users/swentel#followed-https://user.com/',
                         follower.follow.get().as2['url'])

    def test_inbox_undo_follow(self, mock_head, mock_get, mock_post):
        follower = Follower(to=self.user.key,
                            from_=ActivityPub.get_or_create(ACTOR['id']).key,
                            status='active')
        follower.put()

        mock_get.side_effect = [
            self.as2_resp(ACTOR),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        got = self.post('/user.com/inbox', json=UNDO_FOLLOW_WRAPPED)
        self.assertEqual(200, got.status_code)

        # check that the Follower is now inactive
        self.assertEqual('inactive', follower.key.get().status)

    def test_inbox_follow_inactive(self, mock_head, mock_get, mock_post):
        follower = Follower.get_or_create(to=self.user,
                                          from_=ActivityPub.get_or_create(ACTOR['id']),
                                          status='inactive')

        mock_head.return_value = requests_response(url='https://user.com/')
        mock_get.side_effect = [
            # source actor
            self.as2_resp(FOLLOW_WITH_ACTOR['actor']),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        got = self.post('/user.com/inbox', json=FOLLOW_WRAPPED)
        self.assertEqual(200, got.status_code)

        # check that the Follower is now active
        self.assertEqual('active', follower.key.get().status)

    def test_inbox_undo_follow_doesnt_exist(self, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/')
        mock_get.side_effect = [
            self.as2_resp(ACTOR),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        got = self.post('/user.com/inbox', json=UNDO_FOLLOW_WRAPPED)
        self.assertEqual(200, got.status_code)

    def test_inbox_undo_follow_inactive(self, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/')
        mock_get.side_effect = [
            self.as2_resp(ACTOR),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        follower = Follower.get_or_create(to=self.user,
                                          from_=ActivityPub.get_or_create(ACTOR['id']),
                                          status='inactive')

        got = self.post('/user.com/inbox', json=UNDO_FOLLOW_WRAPPED)
        self.assertEqual(200, got.status_code)
        self.assertEqual('inactive', follower.key.get().status)

    def test_inbox_undo_follow_composite_object(self, mock_head, mock_get, mock_post):
        mock_head.return_value = requests_response(url='https://user.com/')
        mock_get.side_effect = [
            self.as2_resp(ACTOR),
            WEBMENTION_DISCOVERY,
        ]
        mock_post.return_value = requests_response()

        follower = Follower.get_or_create(to=self.user,
                                          from_=ActivityPub.get_or_create(ACTOR['id']),
                                          status='inactive')

        undo_follow = copy.deepcopy(UNDO_FOLLOW_WRAPPED)
        undo_follow['object']['object'] = {'id': undo_follow['object']['object']}
        got = self.post('/user.com/inbox', json=undo_follow)
        self.assertEqual(200, got.status_code)
        self.assertEqual('inactive', follower.key.get().status)

    def test_inbox_unsupported_type(self, *_):
        got = self.post('/user.com/inbox', json={
            '@context': ['https://www.w3.org/ns/activitystreams'],
            'id': 'https://xoxo.zone/users/aaronpk#follows/40',
            'type': 'Block',
            'actor': 'https://xoxo.zone/users/aaronpk',
            'object': 'http://snarfed.org/',
        })
        self.assertEqual(501, got.status_code)

    def test_inbox_bad_object_url(self, mock_head, mock_get, mock_post):
        # https://console.cloud.google.com/errors/detail/CMKn7tqbq-GIRA;time=P30D?project=bridgy-federated
        mock_get.return_value = self.as2_resp(ACTOR)  # source actor

        id = 'https://mas.to/users/tmichellemoore#likes/56486252'
        bad_url = 'http://localhost/r/Testing \u2013 Brid.gy \u2013 Post to Mastodon 3'
        bad = {
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': id,
            'type': 'Like',
            'actor': ACTOR['id'],
            'object': bad_url,
        }
        got = self.post('/user.com/inbox', json=bad)

        # bad object, should ignore activity
        self.assertEqual(204, got.status_code)
        mock_post.assert_not_called()

        self.assert_object(id,
                           our_as1={
                               **as2.to_as1(bad),
                               'actor': as2.to_as1(ACTOR),
                           },
                           users=[self.swentel_key],
                           source_protocol='activitypub',
                           status='ignored',
                           )
        self.assertIsNone(Object.get_by_id(bad_url))

    @patch('activitypub.logger.info', side_effect=logging.info)
    @patch('common.logger.info', side_effect=logging.info)
    @patch('oauth_dropins.webutil.appengine_info.DEBUG', False)
    def test_inbox_verify_http_signature(self, mock_common_log, mock_activitypub_log,
                                         _, mock_get, ___):
        # actor with a public key
        self.key_id_obj.key.delete()
        protocol.objects_cache.clear()
        actor_as2 = {
            **ACTOR,
            'publicKey': {
                'id': 'http://my/key/id#unused',
                'owner': 'http://own/er',
                'publicKeyPem': self.user.public_pem().decode(),
            },
        }
        mock_get.return_value = self.as2_resp(actor_as2)

        # valid signature
        body = json_dumps(NOTE)
        headers = self.sign('/ap/sharedInbox', json_dumps(NOTE))
        resp = self.client.post('/ap/sharedInbox', data=body, headers=headers)
        self.assertEqual(204, resp.status_code, resp.get_data(as_text=True))
        mock_get.assert_has_calls((
            self.as2_req('http://my/key/id'),
        ))
        mock_activitypub_log.assert_any_call('HTTP Signature verified!')

        # valid signature, Object has no key
        self.key_id_obj.as2 = ACTOR
        self.key_id_obj.put()
        resp = self.client.post('/ap/sharedInbox', data=body, headers=headers)
        self.assertEqual(401, resp.status_code, resp.get_data(as_text=True))

        # valid signature, Object has our_as1 instead of as2
        self.key_id_obj.clear()
        self.key_id_obj.our_as1 = as2.to_as1(actor_as2)
        self.key_id_obj.put()
        resp = self.client.post('/ap/sharedInbox', data=body, headers=headers)
        self.assertEqual(204, resp.status_code, resp.get_data(as_text=True))
        mock_activitypub_log.assert_any_call('HTTP Signature verified!')

        # invalid signature, missing keyId
        protocol.seen_ids.clear()
        obj_key = ndb.Key(Object, NOTE['id'])
        obj_key.delete()

        resp = self.client.post('/ap/sharedInbox', data=body, headers={
            **headers,
            'signature': headers['signature'].replace(
                'keyId="http://my/key/id#unused",', ''),
        })
        self.assertEqual(401, resp.status_code)
        self.assertEqual({'error': 'HTTP Signature missing keyId'}, resp.json)
        mock_common_log.assert_any_call('Returning 401: HTTP Signature missing keyId', exc_info=None)

        # invalid signature, content changed
        protocol.seen_ids.clear()
        obj_key = ndb.Key(Object, NOTE['id'])
        obj_key.delete()

        resp = self.client.post('/ap/sharedInbox', json={**NOTE, 'content': 'z'}, headers=headers)
        self.assertEqual(401, resp.status_code)
        self.assertEqual({'error': 'Invalid Digest header, required for HTTP Signature'},
                         resp.json)
        mock_common_log.assert_any_call('Returning 401: Invalid Digest header, required for HTTP Signature', exc_info=None)

        # invalid signature, header changed
        protocol.seen_ids.clear()
        obj_key.delete()

        resp = self.client.post('/ap/sharedInbox', data=body, headers={**headers, 'Date': 'X'})
        self.assertEqual(401, resp.status_code)
        self.assertEqual({'error': 'HTTP Signature verification failed'}, resp.json)
        mock_common_log.assert_any_call('Returning 401: HTTP Signature verification failed', exc_info=None)

        # no signature
        protocol.seen_ids.clear()
        obj_key.delete()
        resp = self.client.post('/ap/sharedInbox', json=NOTE)
        self.assertEqual(401, resp.status_code, resp.get_data(as_text=True))
        self.assertEqual({'error': 'No HTTP Signature'}, resp.json)
        mock_common_log.assert_any_call('Returning 401: No HTTP Signature', exc_info=None)

    def test_delete_actor(self, *mocks):
        follower = Follower.get_or_create(
            to=self.user, from_=ActivityPub.get_or_create(DELETE['actor']))
        followee = Follower.get_or_create(
            to=ActivityPub.get_or_create(DELETE['actor']),
            from_=Fake.get_or_create('snarfed.org'))
        # other unrelated follower
        other = Follower.get_or_create(
            to=self.user, from_=ActivityPub.get_or_create('https://mas.to/users/other'))
        self.assertEqual(3, Follower.query().count())

        got = self.post('/ap/sharedInbox', json=DELETE)
        self.assertEqual(204, got.status_code)
        self.assertEqual('inactive', follower.key.get().status)
        self.assertEqual('inactive', followee.key.get().status)
        self.assertEqual('active', other.key.get().status)

    def test_delete_actor_not_fetchable(self, _, mock_get, ___):
        self.key_id_obj.key.delete()
        protocol.objects_cache.clear()

        mock_get.return_value = requests_response(status=410)
        got = self.post('/ap/sharedInbox', json={**DELETE, 'object': 'http://my/key/id'})
        self.assertEqual(202, got.status_code)

    def test_delete_actor_empty_deleted_object(self, _, mock_get, ___):
        self.key_id_obj.as2 = None
        self.key_id_obj.deleted = True
        self.key_id_obj.put()
        protocol.objects_cache.clear()

        got = self.post('/ap/sharedInbox', json={**DELETE, 'object': 'http://my/key/id'})
        self.assertEqual(202, got.status_code)
        mock_get.assert_not_called()

    def test_delete_note(self, _, mock_get, ___):
        obj = Object(id='http://an/obj')
        obj.put()

        mock_get.side_effect = [
            self.as2_resp(ACTOR),
        ]

        delete = {
            **DELETE,
            'object': 'http://an/obj',
        }
        resp = self.post('/ap/sharedInbox', json=delete)
        self.assertEqual(204, resp.status_code)
        self.assertTrue(obj.key.get().deleted)
        self.assert_object(delete['id'],
                           our_as1={
                               **as2.to_as1(delete),
                               'actor': as2.to_as1(ACTOR),
                           },
                           type='delete',
                           source_protocol='activitypub',
                           status='ignored',
                           users=[ActivityPub(id='https://mas.to/users/swentel').key])

        obj.populate(deleted=True, as2=None)
        self.assert_entities_equal(obj,
                                   protocol.objects_cache['http://an/obj'],
                                   ignore=['expire', 'created', 'updated'])

    def test_update_note(self, *mocks):
        Object(id='https://a/note', as2={}).put()
        self._test_update(*mocks)

    def test_update_unknown(self, *mocks):
        self._test_update(*mocks)

    def _test_update(self, _, mock_get, ___):
        mock_get.side_effect = [
            self.as2_resp(ACTOR),
        ]

        resp = self.post('/ap/sharedInbox', json=UPDATE_NOTE)
        self.assertEqual(204, resp.status_code)

        note_as1 = as2.to_as1({
            **UPDATE_NOTE['object'],
            'author': {'id': 'https://mas.to/users/swentel'},
        })
        self.assert_object('https://a/note',
                           type='note',
                           our_as1=note_as1,
                           source_protocol='activitypub')

        update_as1 = {
            **as2.to_as1(UPDATE_NOTE),
            'object': note_as1,
            'actor': as2.to_as1(ACTOR),
        }
        self.assert_object(UPDATE_NOTE['id'],
                           source_protocol='activitypub',
                           type='update',
                           status='ignored',
                           our_as1=update_as1,
                           users=[self.swentel_key])

        self.assert_entities_equal(Object.get_by_id('https://a/note'),
                                   protocol.objects_cache['https://a/note'])

    def test_inbox_webmention_discovery_connection_fails(self, mock_head,
                                                         mock_get, mock_post):
        mock_get.side_effect = [
            # source actor
            self.as2_resp(LIKE_WITH_ACTOR['actor']),
            # protocol inference
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
            # target post webmention discovery
            ReadTimeoutError(None, None, None),
        ]

        got = self.post('/user.com/inbox', json=LIKE)
        self.assertEqual(502, got.status_code)

    def test_inbox_no_webmention_endpoint(self, mock_head, mock_get, mock_post):
        mock_get.side_effect = [
            # source actor
            self.as2_resp(LIKE_WITH_ACTOR['actor']),
            # protocol inference
            requests_response(test_web.NOTE_HTML),
            requests_response(test_web.NOTE_HTML),
            # target post webmention discovery
            HTML,
        ]

        got = self.post('/user.com/inbox', json=LIKE)
        self.assertEqual(204, got.status_code)

        self.assert_object('http://mas.to/like#ok',
                           notify=[self.user.key],
                           users=[self.masto_actor_key],
                           source_protocol='activitypub',
                           status='ignored',
                           our_as1=as2.to_as1(LIKE_WITH_ACTOR),
                           type='like',
                           object_ids=[LIKE['object']])

    def test_inbox_id_already_seen(self, *mocks):
        obj_key = Object(id=FOLLOW_WRAPPED['id'], as2={}).put()

        got = self.post('/user.com/inbox', json=FOLLOW_WRAPPED)
        self.assertEqual(204, got.status_code)
        self.assertEqual(0, Follower.query().count())

        # second time should use in memory cache
        obj_key.delete()
        got = self.post('/user.com/inbox', json=FOLLOW_WRAPPED)
        self.assertEqual(204, got.status_code)
        self.assertEqual(0, Follower.query().count())

    def test_followers_collection_unknown_user(self, *_):
        resp = self.client.get('/nope.com/followers')
        self.assertEqual(404, resp.status_code)

    def test_followers_collection_empty(self, *_):
        resp = self.client.get('/user.com/followers')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/user.com/followers',
            'type': 'Collection',
            'summary': "user.com's followers",
            'totalItems': 0,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/user.com/followers',
                'items': [],
            },
        }, resp.json)

    def store_followers(self):
        follow = Object(id=FOLLOW_WITH_ACTOR['id'], as2=FOLLOW_WITH_ACTOR).put()

        Follower.get_or_create(
            to=self.user,
            from_=self.make_user('http://bar', cls=ActivityPub, obj_as2=ACTOR),
            follow=follow)
        Follower.get_or_create(
            to=self.make_user('https://other.actor', cls=ActivityPub),
            from_=self.user)
        Follower.get_or_create(
            to=self.user,
            from_=self.make_user('http://baz', cls=ActivityPub, obj_as2=ACTOR),
            follow=follow)
        Follower.get_or_create(
            to=self.user,
            from_=self.make_user('http://baj', cls=Fake),
            status='inactive')

    def test_followers_collection_fake(self, *_):
        self.make_user('foo.com', cls=Fake)

        resp = self.client.get('/ap/fake/foo.com/followers')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/ap/fake/foo.com/followers',
            'type': 'Collection',
            'summary': "foo.com's followers",
            'totalItems': 0,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/ap/fake/foo.com/followers',
                'items': [],
            },
        }, resp.json)

    def test_followers_collection(self, *_):
        self.store_followers()

        resp = self.client.get('/user.com/followers')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/user.com/followers',
            'type': 'Collection',
            'summary': "user.com's followers",
            'totalItems': 2,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/user.com/followers',
                'items': [ACTOR, ACTOR],
            },
        }, resp.json)

    @patch('models.PAGE_SIZE', 1)
    def test_followers_collection_page(self, *_):
        self.store_followers()
        before = (datetime.utcnow() + timedelta(seconds=1)).isoformat()
        next = Follower.query(Follower.from_ == ActivityPub(id='http://baz').key,
                              Follower.to == self.user.key,
                              ).get().updated.isoformat()

        resp = self.client.get(f'/user.com/followers?before={before}')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': f'http://localhost/user.com/followers?before={before}',
            'type': 'CollectionPage',
            'partOf': 'http://localhost/user.com/followers',
            'next': f'http://localhost/user.com/followers?before={next}',
            'prev': f'http://localhost/user.com/followers?after={before}',
            'items': [ACTOR],
        }, resp.json)

    def test_following_collection_unknown_user(self, *_):
        resp = self.client.get('/nope.com/following')
        self.assertEqual(404, resp.status_code)

    def test_following_collection_empty(self, *_):
        resp = self.client.get('/user.com/following')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/user.com/following',
            'summary': "user.com's following",
            'type': 'Collection',
            'totalItems': 0,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/user.com/following',
                'items': [],
            },
        }, resp.json)

    def store_following(self):
        follow = Object(id=FOLLOW_WITH_ACTOR['id'], as2=FOLLOW_WITH_ACTOR).put()

        Follower.get_or_create(
            to=self.make_user('http://bar', cls=ActivityPub, obj_as2=ACTOR),
            from_=self.user,
            follow=follow)
        Follower.get_or_create(
            to=self.user,
            from_=self.make_user('https://other.actor', cls=ActivityPub))
        Follower.get_or_create(
            to=self.make_user('http://baz', cls=ActivityPub, obj_as2=ACTOR),
            from_=self.user, follow=follow)
        Follower.get_or_create(
            to=self.make_user('http://baj', cls=ActivityPub),
            from_=self.user,
            status='inactive')

    def test_following_collection(self, *_):
        self.store_following()

        resp = self.client.get('/user.com/following')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/user.com/following',
            'summary': "user.com's following",
            'type': 'Collection',
            'totalItems': 2,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/user.com/following',
                'items': [ACTOR, ACTOR],
            },
        }, resp.json)

    @patch('models.PAGE_SIZE', 1)
    def test_following_collection_page(self, *_):
        self.store_following()
        after = datetime(1900, 1, 1).isoformat()
        prev = Follower.query(Follower.to == ActivityPub(id='http://baz').key,
                              Follower.from_ == self.user.key,
                              ).get().updated.isoformat()

        resp = self.client.get(f'/user.com/following?after={after}')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': f'http://localhost/user.com/following?after={after}',
            'type': 'CollectionPage',
            'partOf': 'http://localhost/user.com/following',
            'prev': f'http://localhost/user.com/following?after={prev}',
            'next': f'http://localhost/user.com/following?before={after}',
            'items': [ACTOR],
        }, resp.json)

    def test_outbox_fake(self, *_):
        self.make_user('foo.com', cls=Fake)
        resp = self.client.get(f'/ap/fake/foo.com/outbox')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/ap/fake/foo.com/outbox',
            'summary': "foo.com's outbox",
            'type': 'OrderedCollection',
            'totalItems': 0,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/ap/fake/foo.com/outbox',
                'items': [],
            },
        }, resp.json)

    def test_outbox_web(self, *_):
        resp = self.client.get(f'/user.com/outbox')
        self.assertEqual(200, resp.status_code)
        self.assertEqual({
            '@context': 'https://www.w3.org/ns/activitystreams',
            'id': 'http://localhost/user.com/outbox',
            'summary': "user.com's outbox",
            'type': 'OrderedCollection',
            'totalItems': 0,
            'first': {
                'type': 'CollectionPage',
                'partOf': 'http://localhost/user.com/outbox',
                'items': [],
            },
        }, resp.json)

class ActivityPubUtilsTest(TestCase):
    def setUp(self):
        super().setUp()
        g.user = self.make_user('user.com', has_hcard=True, obj_as2=ACTOR)

    def test_put_validates_id(self, *_):
        for bad in (
            '',
            'not a url',
            'ftp://not.web/url',
            'https:///no/domain',
            'https://fed.brid.gy/foo',
            'https://ap.brid.gy/foo',
            'http://localhost/foo',
        ):
            with self.assertRaises(AssertionError):
                ActivityPub(id=bad).put()

    def test_owns_id(self):
        self.assertIsNone(ActivityPub.owns_id('http://foo'))
        self.assertIsNone(ActivityPub.owns_id('https://bar/baz'))
        self.assertFalse(ActivityPub.owns_id('at://did:plc:foo/bar/123'))
        self.assertFalse(ActivityPub.owns_id('e45fab982'))

        self.assertFalse(ActivityPub.owns_id('https://twitter.com/foo'))
        self.assertFalse(ActivityPub.owns_id('https://fed.brid.gy/foo'))

    def test_postprocess_as2_multiple_in_reply_tos(self):
        self.assert_equals({
            'id': 'http://localhost/r/xyz',
            'inReplyTo': 'foo',
            'to': [as2.PUBLIC_AUDIENCE],
        }, postprocess_as2({
            'id': 'xyz',
            'inReplyTo': ['foo', 'bar'],
        }))

    def test_postprocess_as2_multiple_url(self):
        self.assert_equals({
            'id': 'http://localhost/r/xyz',
            'url': ['http://localhost/r/foo', 'http://localhost/r/bar'],
            'to': [as2.PUBLIC_AUDIENCE],
        }, postprocess_as2({
            'id': 'xyz',
            'url': ['foo', 'bar'],
        }))

    def test_postprocess_as2_multiple_image(self):
        self.assert_equals({
            'id': 'http://localhost/r/xyz',
            'attachment': [{'url': 'http://r/foo'}, {'url': 'http://r/bar'}],
            'image': [{'url': 'http://r/foo'}, {'url': 'http://r/bar'}],
            'to': [as2.PUBLIC_AUDIENCE],
        }, postprocess_as2({
            'id': 'xyz',
            'image': [{'url': 'http://r/foo'}, {'url': 'http://r/bar'}],
        }))

    def test_postprocess_as2_actor_attributedTo(self):
        g.user = Fake(id='site')
        self.assert_equals({
            'actor': {
                'id': 'baj',
                'preferredUsername': 'site',
                'url': 'http://localhost/r/site',
                'inbox': 'http://bf/fake/site/ap/inbox',
                'outbox': 'http://bf/fake/site/ap/outbox',
            },
            'attributedTo': [{
                'id': 'bar',
                'preferredUsername': 'site',
                'url': 'http://localhost/r/site',
                'inbox': 'http://bf/fake/site/ap/inbox',
                'outbox': 'http://bf/fake/site/ap/outbox',
            }, {
                'id': 'baz',
                'preferredUsername': 'site',
                'url': 'http://localhost/r/site',
                'inbox': 'http://bf/fake/site/ap/inbox',
                'outbox': 'http://bf/fake/site/ap/outbox',
            }],
            'to': [as2.PUBLIC_AUDIENCE],
        }, postprocess_as2({
            'attributedTo': [{'id': 'bar'}, {'id': 'baz'}],
            'actor': {'id': 'baj'},
        }))

    def test_postprocess_as2_note(self):
        self.assert_equals({
            'id': 'http://localhost/r/xyz',
            'type': 'Note',
            'to': [as2.PUBLIC_AUDIENCE],
        }, postprocess_as2({
            'id': 'xyz',
            'type': 'Note',
        }))

    def test_postprocess_as2_hashtag(self):
        """https://github.com/snarfed/bridgy-fed/issues/45"""
        self.assert_equals({
            'tag': [
                {'type': 'Hashtag', 'name': '#bar', 'href': 'bar'},
                {'type': 'Hashtag', 'name': '#baz', 'href': 'http://localhost/hashtag/baz'},
                {'type': 'Mention', 'href': 'foo'},
            ],
            'to': ['https://www.w3.org/ns/activitystreams#Public'],
        }, postprocess_as2({
            'tag': [
                {'name': 'bar', 'href': 'bar'},
                {'type': 'Tag', 'name': '#baz'},
                # should leave alone
                {'type': 'Mention', 'href': 'foo'},
            ],
        }))

    def test_postprocess_as2_url_attachments(self):
        got = postprocess_as2(as2.from_as1({
            'objectType': 'person',
            'urls': [
                {
                    'value': 'https://user.com/about-me',
                    'displayName': 'Mrs. \u2615 Foo',
                }, {
                    'value': 'https://user.com/',
                    'displayName': 'should be ignored',
                }, {
                    'value': 'http://one',
                    'displayName': 'one text',
                }, {
                    'value': 'https://two',
                    'displayName': 'two title',
                },
            ]
        }))

        self.assert_equals([{
            'type': 'PropertyValue',
            'name': 'Mrs. ☕ Foo',
            'value': 'https://user.com/about-me',

        }, {
            'type': 'PropertyValue',
            'name': 'Web site',
            'value': 'https://user.com/',
        }, {
            'type': 'PropertyValue',
            'name': 'one text',
            'value': 'http://one',
        }, {
            'type': 'PropertyValue',
            'name': 'two title',
            'value': 'https://two',
        }], got['attachment'])

    def test_postprocess_as2_preserves_preferredUsername(self):
        # preferredUsername stays y.z despite user's username. since Mastodon
        # queries Webfinger for preferredUsername@fed.brid.gy
        # https://github.com/snarfed/bridgy-fed/issues/77#issuecomment-949955109
        self.assertEqual('user.com', postprocess_as2({
            'type': 'Person',
            'url': 'https://user.com/about-me',
            'preferredUsername': 'nick',
            'attachment': [{
                'type': 'PropertyValue',
                'name': 'nick',
                'value': 'https://user.com/about-me',
            }],
        })['preferredUsername'])

    # TODO: make these generic and use Fake
    @patch('requests.get')
    def test_load_http(self, mock_get):
        mock_get.return_value = AS2

        id = 'http://the/id'
        self.assertIsNone(Object.get_by_id(id))

        # first time fetches over HTTP
        got = ActivityPub.load(id)
        self.assert_equals(id, got.key.id())
        self.assert_equals(AS2_OBJ, got.as2)
        mock_get.assert_has_calls([self.as2_req(id)])

        # second time is in cache
        got.key.delete()
        mock_get.reset_mock()

        got = ActivityPub.load(id)
        self.assert_equals(id, got.key.id())
        self.assert_equals(AS2_OBJ, got.as2)
        mock_get.assert_not_called()

    @patch('requests.get')
    def test_load_datastore(self, mock_get):
        id = 'http://the/id'
        stored = Object(id=id, as2=AS2_OBJ)
        stored.put()
        protocol.objects_cache.clear()

        # first time loads from datastore
        got = ActivityPub.load(id)
        self.assert_entities_equal(stored, got)
        mock_get.assert_not_called()

        # second time is in cache
        stored.key.delete()
        got = ActivityPub.load(id)
        self.assert_entities_equal(stored, got)
        mock_get.assert_not_called()

    @patch('requests.get')
    def test_load_preserves_fragment(self, mock_get):
        stored = Object(id='http://the/id#frag', as2=AS2_OBJ)
        stored.put()
        protocol.objects_cache.clear()

        got = ActivityPub.load('http://the/id#frag')
        self.assert_entities_equal(stored, got)
        mock_get.assert_not_called()

    @patch('requests.get')
    def test_load_datastore_no_as2(self, mock_get):
        """If the stored Object has no as2, we should fall back to HTTP."""
        id = 'http://the/id'
        stored = Object(id=id, as2={}, status='in progress')
        stored.put()
        protocol.objects_cache.clear()

        mock_get.return_value = AS2
        got = ActivityPub.load(id)
        mock_get.assert_has_calls([self.as2_req(id)])

        self.assert_equals(id, got.key.id())
        self.assert_equals(AS2_OBJ, got.as2)
        mock_get.assert_has_calls([self.as2_req(id)])

        self.assert_object(id,
                           as2=AS2_OBJ,
                           as1={**AS2_OBJ, 'id': id},
                           source_protocol='activitypub',
                           # check that it reused our original Object
                           status='in progress')

    @patch('requests.get')
    def test_signed_get_redirects_manually_with_new_sig_headers(self, mock_get):
        mock_get.side_effect = [
            requests_response(status=302, redirected_url='http://second',
                              allow_redirects=False),
            requests_response(status=200, allow_redirects=False),
        ]
        activitypub.signed_get('https://first')

        first = mock_get.call_args_list[0][1]
        second = mock_get.call_args_list[1][1]
        self.assertNotEqual(first['headers'], second['headers'])

    @patch('requests.get')
    def test_signed_get_redirects_to_relative_url(self, mock_get):
        mock_get.side_effect = [
            # redirected URL is relative, we have to resolve it
            requests_response(status=302, redirected_url='/second',
                              allow_redirects=False),
            requests_response(status=200, allow_redirects=False),
        ]
        activitypub.signed_get('https://first')

        self.assertEqual(('https://first/second',), mock_get.call_args_list[1][0])

        first = mock_get.call_args_list[0][1]
        second = mock_get.call_args_list[1][1]

        # headers are equal because host is the same
        self.assertEqual(first['headers'], second['headers'])
        self.assertEqual(
            first['auth'].header_signer.sign(first['headers'], method='GET', path='/'),
            second['auth'].header_signer.sign(second['headers'], method='GET', path='/'))

    @patch('requests.post')
    def test_signed_post_ignores_redirect(self, mock_post):
        mock_post.side_effect = [
            requests_response(status=302, redirected_url='http://second',
                              allow_redirects=False),
        ]

        resp = activitypub.signed_post('https://first')
        mock_post.assert_called_once()
        self.assertEqual(302, resp.status_code)

    @patch('requests.get')
    def test_fetch_direct(self, mock_get):
        mock_get.return_value = AS2
        obj = Object(id='http://orig')
        ActivityPub.fetch(obj)
        self.assertEqual(AS2_OBJ, obj.as2)

        mock_get.assert_has_calls((
            self.as2_req('http://orig'),
        ))

    @patch('requests.get')
    def test_fetch_direct_ld_content_type(self, mock_get):
        mock_get.return_value = requests_response(AS2_OBJ, headers={
            'Content-Type': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
        })
        obj = Object(id='http://orig')
        ActivityPub.fetch(obj)
        self.assertEqual(AS2_OBJ, obj.as2)

        mock_get.assert_has_calls((
            self.as2_req('http://orig'),
        ))

    @patch('requests.get')
    def test_fetch_via_html(self, mock_get):
        mock_get.side_effect = [HTML_WITH_AS2, AS2]
        obj = Object(id='http://orig')
        ActivityPub.fetch(obj)
        self.assertEqual(AS2_OBJ, obj.as2)

        mock_get.assert_has_calls((
            self.as2_req('http://orig'),
            self.as2_req('http://as2', headers=as2.CONNEG_HEADERS),
        ))

    @patch('requests.get')
    def test_fetch_only_html(self, mock_get):
        mock_get.return_value = HTML

        obj = Object(id='http://orig')
        self.assertFalse(ActivityPub.fetch(obj))
        self.assertIsNone(obj.as1)

    @patch('requests.get')
    def test_fetch_not_acceptable(self, mock_get):
        mock_get.return_value = NOT_ACCEPTABLE

        obj = Object(id='http://orig')
        self.assertFalse(ActivityPub.fetch(obj))
        self.assertIsNone(obj.as1)

    @patch('requests.get')
    def test_fetch_ssl_error(self, mock_get):
        mock_get.side_effect = requests.exceptions.SSLError
        with self.assertRaises(BadGateway):
            ActivityPub.fetch(Object(id='http://orig'))

    @patch('requests.get')
    def test_fetch_no_content(self, mock_get):
        mock_get.return_value = self.as2_resp('')

        with self.assertRaises(BadGateway):
            ActivityPub.fetch(Object(id='http://the/id'))

        mock_get.assert_has_calls([self.as2_req('http://the/id')])

    @patch('requests.get')
    def test_fetch_not_json(self, mock_get):
        mock_get.return_value = self.as2_resp('XYZ not JSON')

        with self.assertRaises(BadGateway):
            ActivityPub.fetch(Object(id='http://the/id'))

        mock_get.assert_has_calls([self.as2_req('http://the/id')])

    def test_fetch_non_url(self):
        obj = Object(id='x y z')
        self.assertFalse(ActivityPub.fetch(obj))
        self.assertIsNone(obj.as1)

    @skip
    def test_serve(self):
        obj = Object(id='http://orig', as2=LIKE)
        self.assertEqual((LIKE_WRAPPED, {'Content-Type': 'application/activity+json'}),
                         ActivityPub.serve(obj))

    def test_postprocess_as2_idempotent(self):
        g.user = self.make_user('foo.com')

        for obj in (ACTOR, REPLY_OBJECT, REPLY_OBJECT_WRAPPED, REPLY,
                    NOTE_OBJECT, NOTE, MENTION_OBJECT, MENTION, LIKE,
                    LIKE_WRAPPED, REPOST, FOLLOW, FOLLOW_WRAPPED, ACCEPT,
                    UNDO_FOLLOW_WRAPPED, DELETE, UPDATE_NOTE,
                    # TODO: these currently fail
                    # LIKE_WITH_ACTOR, REPOST_FULL, FOLLOW_WITH_ACTOR,
                    # FOLLOW_WRAPPED_WITH_ACTOR, FOLLOW_WITH_OBJECT, UPDATE_PERSON,
                    ):
            with self.subTest(obj=obj):
                obj = copy.deepcopy(obj)
                self.assert_equals(postprocess_as2(obj),
                                   postprocess_as2(postprocess_as2(obj)),
                                   ignore=['to'])

    def test_ap_address(self):
        user = ActivityPub(obj=Object(id='a', as2={**ACTOR, 'preferredUsername': 'me'}))
        self.assertEqual('@me@mas.to', user.ap_address())
        self.assertEqual('@me@mas.to', user.readable_id)

        user.obj.as2 = ACTOR
        self.assertEqual('@swentel@mas.to', user.ap_address())
        self.assertEqual('@swentel@mas.to', user.readable_id)

        user = ActivityPub(id='https://mas.to/users/alice')
        self.assertEqual('@alice@mas.to', user.ap_address())
        self.assertEqual('@alice@mas.to', user.readable_id)

    def test_ap_actor(self):
        user = self.make_user('http://foo/actor', cls=ActivityPub)
        self.assertEqual('http://foo/actor', user.ap_actor())

    def test_web_url(self):
        user = self.make_user('http://foo/actor', cls=ActivityPub)
        self.assertEqual('http://foo/actor', user.web_url())

        user.obj = Object(id='a', as2=copy.deepcopy(ACTOR))  # no url
        self.assertEqual('http://foo/actor', user.web_url())

        user.obj.as2['url'] = ['http://my/url']
        self.assertEqual('http://my/url', user.web_url())

    def test_readable_id(self):
        user = self.make_user('http://foo', cls=ActivityPub)
        self.assertIsNone(user.readable_id)
        self.assertEqual('http://foo', user.readable_or_key_id())

        user.obj = Object(id='a', as2=ACTOR)
        self.assertEqual('@swentel@mas.to', user.readable_id)
        self.assertEqual('@swentel@mas.to', user.readable_or_key_id())

    @skip
    def test_target_for_not_activitypub(self):
        with self.assertRaises(AssertionError):
            ActivityPub.target_for(Object(source_protocol='web'))

    def test_target_for_actor(self):
        self.assertEqual(ACTOR['inbox'], ActivityPub.target_for(
            Object(source_protocol='ap', as2=ACTOR)))

        actor = copy.deepcopy(ACTOR)
        del actor['inbox']
        self.assertIsNone(ActivityPub.target_for(
            Object(source_protocol='ap', as2=actor)))

        actor['publicInbox'] = 'so-public'
        self.assertEqual('so-public', ActivityPub.target_for(
            Object(source_protocol='ap', as2=actor)))

        # sharedInbox
        self.assertEqual('so-public', ActivityPub.target_for(
            Object(source_protocol='ap', as2=actor), shared=True))
        actor['endpoints'] = {
            'sharedInbox': 'so-shared',
        }
        self.assertEqual('so-public', ActivityPub.target_for(
            Object(source_protocol='ap', as2=actor)))
        self.assertEqual('so-shared', ActivityPub.target_for(
            Object(source_protocol='ap', as2=actor), shared=True))

    def test_target_for_object(self):
        obj = Object(as2=NOTE_OBJECT, source_protocol='ap')
        self.assertIsNone(ActivityPub.target_for(obj))

        Object(id=ACTOR['id'], as2=ACTOR).put()
        obj.as2 = {
            **NOTE_OBJECT,
            'author': ACTOR['id'],
        }
        self.assertEqual('http://mas.to/inbox', ActivityPub.target_for(obj))

        del obj.as2['author']
        obj.as2['actor'] = copy.deepcopy(ACTOR)
        obj.as2['actor']['url'] = [obj.as2['actor'].pop('id')]
        self.assertEqual('http://mas.to/inbox', ActivityPub.target_for(obj))

    @patch('requests.get')
    def test_target_for_object_fetch(self, mock_get):
        mock_get.return_value = self.as2_resp(ACTOR)

        obj = Object(as2={
            **NOTE_OBJECT,
            'author': 'http://the/author',
        }, source_protocol='ap')
        self.assertEqual('http://mas.to/inbox', ActivityPub.target_for(obj))
        mock_get.assert_has_calls([self.as2_req('http://the/author')])

    @patch('requests.get')
    def test_target_for_author_is_object_id(self, mock_get):
        obj = self.store_object(id='http://the/author', our_as1={
            'author': 'http://the/author',
        })
        # test is that we short circuit out instead of infinite recursion
        self.assertIsNone(ActivityPub.target_for(obj))

    @patch('requests.post')
    def test_send_blocklisted(self, mock_post):
        self.assertFalse(ActivityPub.send(Object(as2=NOTE),
                                          'https://fed.brid.gy/ap/sharedInbox'))
        mock_post.assert_not_called()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""

import sys
import os
from os import path
import numpy as np
from PIL import Image
import datetime
import matplotlib.pyplot as plt
import time

sys.path.append(os.getcwd() + "/lib/wordcloud")
from wordcloud import WordCloud

text = "初鳩,初花,初針,初春,初日,初日影,初飛行,初披講,初日の出,初雲雀,初富士,初冬,初風呂,初箒,初蛍,初参り,初松風,初神籤,初彌撒,春小袖," \
       "春炬燵,春寒,春雨,春芝居,春近し,春隣,春の馬,春の海,春の蚊,春の風,春の川,春の草,春の雲,春の暮,春の蟬,春の空,春の田,春の月,春の虹," \
       "春の野,春の日,春の服,春,2,3,春の星,はるのほし,春,2,3,春の水,はるのみず,春,2,3,春の山,はるのやま,春,2,3,春の闇,はるのやみ,春," \
       "2,3,春の雪,はるのゆき,春,2,3,春の夜,はるのよ,春,2,春の宵,はるのよい,2,3,春日傘,はるひがさ,春,2,3,春深し,はるふかし,春,2,3,春待つ," \
       "はるまつ,冬,2,春祭,はるまつり,春,2,春めく,はるめく,春,2,春休み,はるやすみ,春,2,春夕焼,はるゆうやけ,春,2,バレンタインデー," \
       "ばれんたいんでー,春,晩夏,ばんか,夏,晩菊,ばんぎく,秋,万愚節,ばんぐせつ,春,半夏,はんげ,夏,半夏生,はんげしょう,夏,パンジー,ぱんじー," \
       "春,晩秋,ばんしゅう,秋,晩春,ばんしゅん,春,半仙戯,はんせんぎ,春,晩霜,ばんそう,春,斑猫,はんみょう,夏,晩涼,ばんりょう,夏,万緑," \
       "ばんりょく,夏,日脚伸ぶ,ひあしのぶ,ビーチパラソル,びーちぱらそる,夏,ひひな,ひいな,春,柊,ひいらぎ,冬,麦酒,びーる,夏".encode('utf-8')
d = path.dirname(__file__)

start_day = datetime.date.today()

logo_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/logo_mask.png")))
spring_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/spring-mask.png")))
newyear_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/newyear-mask.png")))
summer_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/summer-mask.png")))
autumn_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/autumn-mask.png")))
winter_mask = np.array(Image.open(
    path.join(d, os.getcwd() + "/mask/winter-mask.png")))

total_f = open("./text/total_wordcloud.txt")
total_text = total_f.read()

weekly_f = open("./text/weekly_wordcloud.txt")
weekly_text = weekly_f.read()

monthly_f = open("./text/monthly_wordcloud.txt")
monthly_text = monthly_f.read()

newyear_f = open("./text/newyear_wordcloud.txt")
newyear_text = newyear_f.read()

spring_f = open("./text/spring_wordcloud.txt")
spring_text = spring_f.read()

summer_f = open("./text/summer_wordcloud.txt")
summer_text = summer_f.read()

autumn_f = open("./text/autumn_wordcloud.txt")
autumn_text = autumn_f.read()

winter_f = open("./text/winter_wordcloud.txt")
winter_text = winter_f.read()

while True:
    today = datetime.date.today()
    wordcloud = WordCloud(background_color="lightcyan", mode="RGB", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=logo_mask, width=600, height=600, colormap="gist_rainbow").generate(total_text)
    wordcloud.to_file("./image/total_wordcloud.png")
    print("total_wordcloud was generated.")

    wordcloud = WordCloud(background_color="lightcyan", mode="RGB", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=logo_mask, width=600, height=600, colormap="gist_rainbow").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/weekly_wordcloud.png")
    print("weekly_wordcloud was generated.")

    wordcloud = WordCloud(background_color="lightcyan", mode="RGB", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=logo_mask, width=600, height=600, colormap="gist_rainbow").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/monthly_wordcloud.png")
    print("monthly_wordcloud was generated.")

    wordcloud = WordCloud(background_color="moccasin", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=newyear_mask, width=600, height=600, colormap="Reds").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/newyear_wordcloud.png")
    print("newyear_wordcloud was generated.")

    wordcloud = WordCloud(background_color="palegreen", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=spring_mask, width=600, height=600, colormap="spring").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/spring_wordcloud.png")
    print("spring_wordcloud was generated.")

    wordcloud = WordCloud(background_color="paleturquoise", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=summer_mask, width=600, height=600, colormap="summer").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/summer_wordcloud.png")
    print("summer_wordcloud was generated.")

    wordcloud = WordCloud(background_color="darkslategray", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=autumn_mask, width=600, height=600, colormap="autumn").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/autumn_wordcloud.png")
    print("autumn_wordcloud was generated.")

    wordcloud = WordCloud(background_color="midnightblue", font_path="./ヒラギノ角ゴシック W5.ttc",
                          mask=winter_mask, width=600, height=600, colormap="PuBuGn").generate(text.decode('utf-8'))
    wordcloud.to_file("./image/winter_wordcloud.png")
    print("winter_wordcloud was generated.")

    time.sleep(86400)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Filters results from taxonomic classifiers and extracts taxonomic IDs for filtered hits"""
"""To do: Add argparse and logging"""
import os
import sys
import subprocess
import pandas as pd

def main():
    # Input
    classifier = sys.argv[1]
    params = sys.argv[2]
    hits_report = sys.argv[3]
    ntc = None
    if len(sys.argv) > 4:
        ntc = sys.argv[4]
    reference_summary_refseq = "ftp://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_refseq.txt"
    reference_summary_genbank = "ftp://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt"
    #reference_summary_refseq = "rsync://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_refseq.txt"
    #reference_summary_genbank = "rsync://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt"

    # Get thresholds for filtering
    min_reads = ""
    min_reads_w_ntc = ""
    max_reads_ntc = ""
    min_ratio = ""
    min_reads_ratio = ""
    try:
        params = params.strip().split(",")
        if len(params) == 1:
            min_reads = int(params[0])
        elif len(params) == 4:
            min_reads_w_ntc = int(params[0])
            max_reads_ntc = int(params[1])
            min_ratio = float(params[2])
            min_reads_ratio = int(params[3])
        else:
            print("No filter parameters specified")
            sys.exit(1)
    except Exception as e:
        print("Failed parsing filtering options\n" + str(e))
        sys.exit(1)

    print("min_reads", min_reads)
    print("min_reads_w_ntc", min_reads_w_ntc)
    print("max_reads_ntc", max_reads_ntc)
    print("min_ratio", min_ratio)
    print("min_reads_ratio", min_reads_ratio)

    # Output
    summary_filtered = "{}_{}_filtered_taxonomic_hits.tsv".format(classifier,hits_report)
    summary_skipped = "{}_{}_skipped_taxonomic_hits.tsv".format(classifier,hits_report)

    # Parse hits
    all_hits = get_all_hits(hits_report, classifier)
    if ntc:
        ntc_hits = get_all_hits(ntc, classifier)
    # Filter hits
    filtered_hits = {}
    skipped_hits = {}
    for taxid, data in all_hits.items():
        reads = parse_reads(data)
        # Filter based on ntc sample
        if ntc:
            # Get number of reads for ntc
            if taxid in ntc_hits.keys():
                ntc_data = ntc_hits[taxid]
                ntc_reads = parse_reads(ntc_data)
            else:
                ntc_reads = 0
            # Filter based on difference in read counts between sample and ntc
            if (reads > min_reads_w_ntc) and (ntc_reads < max_reads_ntc):
                add_taxid(all_hits, taxid, filtered_hits, skipped_hits)
            elif ((ntc_reads / reads) < min_ratio) and (reads > min_reads_ratio):
                add_taxid(all_hits, taxid, filtered_hits, skipped_hits)
        # Filter based on number of reads
        else:
            if reads > min_reads:
                add_taxid(all_hits, taxid, filtered_hits, skipped_hits)
    sorted_hits = sorted(list(filtered_hits.items()), key=lambda k: k[1][0], reverse=True)
    print("Found {} species in the sample".format(len(filtered_hits)))
    for hit in filtered_hits:
      print (hit, filtered_hits[hit])

    # Download refseq assembly summary file from ncbi
    print("Downloading NCBI refseq reference summary file")
    download_ftp(reference_summary_refseq)

    # Get reference ids for filtered taxids
    taxid_accesion_map = {}
    # Search refseq for references
    missing_acc_refseq = parse_accession(reference_summary_refseq, filtered_hits.keys(), taxid_accesion_map)
    if len(missing_acc_refseq) > 0:
        # Download genbank assembly summary file from ncbi
        print("Downloading NCBI genbank reference summary file")
        download_ftp(reference_summary_genbank)
        # Search genbank for references that are not found in refseq
        missing_acc = parse_accession(reference_summary_genbank, missing_acc_refseq, taxid_accesion_map)
        if len(missing_acc) > 0:
            print("Failed to identify accession number of reference for taxons: {}".format(", ".join(missing_acc)))

    # Create summary files
    with open(summary_filtered, "w") as out:
        out.write("taxonomic_id\treads\tspecies\taccession_id\n")
        for taxid, info in sorted_hits:
            out.write("\t".join([taxid, info[0], info[1], taxid_accesion_map[str(taxid)]]) + "\n")
    with open(summary_skipped, "w") as out:
        out.write("taxonomic_id\tspecies\n")
        for taxid, name in skipped_hits.items():
            out.write(str(taxid) + "\t" + name + "\n")

def get_all_hits(results, classifier):
    """Parse all hits in classifier report for species level"""
    all_hits = {}
    with open(results, 'r') as f:
        res = f.readlines()[1:]
        for hit in res:
            hit = hit.strip().split("\t")
            if classifier == "kaiju":
                all_hits[hit[3]] = [hit[2], hit[4]]
            elif classifier in ["kraken2", "centrifuge"]:
                reads = hit[1]
                level = hit[3]
                name = hit[5].strip().rstrip()
                if level == "S":
                    taxid = hit[4]
                    all_hits[taxid] = [reads, name]
                elif level.startswith("S") or level == "-":
                    if name not in ["root","cellular organisms","Opisthokonta","Eumetazoa","Bilateria","Deuterostomia","Craniata","Vertebrata","Gnathostomata","Teleostomi","Euteleostomi","Sarcopterygii"," Eutheria","Dipnotetrapodomorpha","Tetrapoda","Amniota","Theria","Eutheria","Boreoeutheria","Euarchontoglires","Haplorrhini","Simiiformes","Catarrhini","Hominoidea","Homininae","Pseudomonas fluorescens group"]:
                         all_hits[taxid].append([reads, name, hit[4]])
            else:
                print("No classifier specified. Supported classifiers are kraken2, centrifuge and kaiju.")
                sys.exit(1)
    return all_hits

def parse_reads(data):
    reads = int(data[0])
    return reads

def add_taxid(all_hits, taxid, output, skipped):
    """Add taxonomic ID of species or one of the strains of the species to the output"""
    data = all_hits[taxid]
    # Check if substrain of taxon should be used
    if len(data) > 2:
        # Compare reads with substrains for species
        reads = parse_reads(data)
        for sub in data[2:]:
            if int(sub[0])/reads > 0.75:
                taxid = sub[2]
                data = sub
    name = data[1]
    # Filter out unwanted species
    if "phage" in name.lower() or taxid == "9606" or "group" in name.lower() or "unclassified" in name.lower():
        skipped[taxid] = name
        return
    #elif contaminant
    output[taxid] = [data[0], name]

def download_ftp(path):
    """Download file"""
    try:
        command = "wget {}".format(path)
        proc = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
        output, error = proc.communicate()
        print("output", output)
        print("error", error)
    except Exception as e:
        print("Could not download file: {}\n".format(path) + e)
        sys.exit(1)

def parse_accession(referencefile, taxids, taxid_accesion_map):
    """Parses NCBI summary file and chooses one reference accession number for each species in input list, returns list of missing taxons"""
    missing_acc = []
    with open(os.path.basename(referencefile), "r") as f:
        references = pd.read_csv(f, sep='\t', skiprows=1, index_col=0, dtype=str)
        for taxid in taxids:
            refs = references.copy()
            refs = refs.loc[refs["taxid"] == taxid]
            if len(refs) == 1:
                acc = refs.index.tolist()[0]
            elif "reference genome" in list(refs["refseq_category"]):
                acc = refs.index[refs["refseq_category"] == "reference genome"].tolist()[0]
            elif "representative genome" in list(refs["refseq_category"]):
                acc = refs.index[refs["refseq_category"] == "representative genome"].tolist()[0]
            elif "Complete Genome" in list(refs["assembly_level"]):
                acc = refs.index[refs["assembly_level"] == "Complete Genome"].tolist()[0]
            elif len(refs) > 0:
                acc = refs.index.tolist()[0]
            else:
                acc = None
                missing_acc.append(taxid)
            # Add identified references
            taxid_accesion_map[taxid] = acc
    return missing_acc

main()


from __future__ import annotations

import asyncio
import dataclasses
import logging
import uuid
from typing import TYPE_CHECKING, Self

from PySide6.QtCore import QObject, Signal

from randovania.bitpacking.json_dataclass import JsonDataclass
from randovania.interface_common.players_configuration import INVALID_UUID
from randovania.lib import json_lib, migration_lib

if TYPE_CHECKING:
    from collections.abc import Iterable
    from pathlib import Path

_MIGRATIONS = [
    # lambda data: data,
]
CURRENT_VERSION = migration_lib.get_version(_MIGRATIONS)

def migrate_to_current(data: dict):
    return migration_lib.apply_migrations(data, _MIGRATIONS, copy_before_migrating=True)

@dataclasses.dataclass(frozen=True)
class WorldServerData(JsonDataclass):
    world_name: str
    session_id: int
    session_name: str

def _combine_tuples(existing: tuple[int, ...], new_indices: Iterable[int]) -> tuple[int, ...]:
    new = set(existing)
    for it in new_indices:
        new.add(it)
    return tuple(sorted(new))

@dataclasses.dataclass(frozen=True)
class WorldData(JsonDataclass):
    collected_locations: tuple[int, ...] = ()
    uploaded_locations: tuple[int, ...] = ()
    latest_message_displayed: int = 0
    server_data: WorldServerData | None = None

    def extend_collected_location(self, new_indices: Iterable[int]) -> Self:
        return dataclasses.replace(
            self,
            collected_locations=_combine_tuples(self.collected_locations, new_indices),
        )

    def extend_uploaded_locations(self, new_indices: Iterable[int]) -> Self:
        return dataclasses.replace(
            self,
            uploaded_locations=_combine_tuples(self.uploaded_locations, new_indices),
        )

class WorldDatabase(QObject):
    _all_data: dict[uuid.UUID, WorldData]
    _persist_path: Path

    WorldDataUpdate = Signal()

    def __init__(self, persist_path: Path):
        super().__init__()
        self.logger = logging.getLogger(__name__)

        persist_path.mkdir(parents=True, exist_ok=True)
        self._persist_path = persist_path
        self.logger.info("Using %s as database path", persist_path)

        self._all_data = {}
        self._lock = asyncio.Lock()

    async def _read_data(self, uid: uuid.UUID) -> WorldData:
        raw_data = await json_lib.read_path_async(self._persist_path.joinpath(f"{uid}.json"))
        return WorldData.from_json(migrate_to_current(raw_data)["data"])

    async def _write_data(self, uid: uuid.UUID, data: WorldData):
        json_lib.write_path(
            self._persist_path.joinpath(f"{uid}.json"),
            {
                "schema_version": CURRENT_VERSION,
                "data": data.as_json,
            },
        )

    async def load_existing_data(self):
        for f in self._persist_path.glob("*.json"):
            try:
                uid = uuid.UUID(f.stem)
            except ValueError:
                self.logger.warning("File name is not a UUID: %s", f)
                continue

            if uid != INVALID_UUID:
                self._all_data[uid] = await self._read_data(uid)

    def get_data_for(self, uid: uuid.UUID) -> WorldData:
        if uid == INVALID_UUID:
            raise ValueError("UID not allowed for Multiworld")

        if uid not in self._all_data:
            self._all_data[uid] = WorldData()

        return self._all_data[uid]

    async def set_data_for(self, uid: uuid.UUID, data: WorldData):
        await self.set_many_data({uid: data})

    async def set_many_data(self, new_data: dict[uuid.UUID, WorldData]):
        async with self._lock:
            for uid, data in new_data.items():
                if data != self._all_data.get(uid):
                    self._all_data[uid] = data
                    await self._write_data(uid, data)
            self.WorldDataUpdate.emit()

    def get_locations_to_upload(self, uid: uuid.UUID) -> tuple[int, ...]:
        data = self.get_data_for(uid)
        return tuple(i for i in sorted(data.collected_locations) if i not in data.uploaded_locations)

    def all_known_data(self) -> Iterable[uuid.UUID]:
        yield from self._all_data.keys()


def get_config():
    import os

    from board_game.states.gomoku_state import GomokuState
    from .gomoku_dqn import GomokuDqnModel

    board_shape = (9, 9)
    target = 5
    print('board_shape:', board_shape)
    print('target:', target)

    state = GomokuState(board_shape = board_shape, target = target)
    model_path = 'gomoku_dqn_model_{0}_{1}_{2}'.format(board_shape[0], board_shape[1], target)
    model_path = os.path.join(os.path.dirname(__file__), model_path)
    model = GomokuDqnModel(board_shape = state.board_shape, action_dim = state.get_action_dim(), model_path = model_path)

    config = {
        'model_path' : model_path,
        'replay_memory_size' : 64 * 1024,
        'discount' : 0.95,
        'batch_size' : 16,
        'epoch_num' : 4,
        'learning_rate' : 0.001,
        'episode_num' : 2000000,
    }

    return state, model, config

def main():
    from . import dqn_train

    dqn_train.main('gomoku', get_config)


import inflect

def main():

    p = inflect.engine()
    names = []

    while True:
        try:
            name = input("Name: ")
            names.append(name)

        except(EOFError, KeyboardInterrupt):

            names = p.join(names)
            print("Adieu, adieu, to " + names)
            quit()

main()


import cv2

#Configurable Parameters
inputValue = int(input("Enter the scale value to resize the image: (0 - 100): "))
if inputValue >= 0 and inputValue <= 100:

    source = "wx.jpg"
    destination = 'newImage.png'
    scale_percent = inputValue

    src = cv2.imread(source, cv2.IMREAD_UNCHANGED)
    #cv2.imshow("title", src)

    #Percentage by which the image is resize

    #Calculate the 50 percent of original dimensions
    new_width = int(src.shape[1] * scale_percent / 100)
    new_height = int(src.shape[0] * scale_percent / 100)

    dsize = (new_width, new_height)

    output = cv2.resize(src, dsize)

    cv2.imwrite(destination, output)
    #cv2.waitKey(0)
else:
    print("Enter correct values")


from rest_framework.response import Response
from rest_framework.decorators import api_view
from .serializers import PersonSerializer
from .models import Person
from rest_framework import status
from rest_framework.permissions import IsAdminUser
from rest_framework.decorators import api_view, permission_classes

@api_view(['GET','POST'])
def home(request):
    if request.method == 'POST':
        name = request.data['name']
        return Response({'name':f'my name is {name}'})

    else:

        return Response({'name':'sina'})

@api_view()
def persons(request):
    person = Person.objects.all()
    ser_data = PersonSerializer(person,many=True)
    return Response(ser_data.data,status=status.HTTP_200_OK)

@api_view()
@permission_classes([IsAdminUser])
def person(request,name):
    try:
        person = Person.objects.get(name=name)
    except Person.DoesNotExist:
        return Response({'error':'this user does not exist'},status=status.HTTP_404_NOT_FOUND)

    ser_data = PersonSerializer(person)
    return Response(ser_data.data,status=status.HTTP_200_OK)

#create_user
@api_view(['POST'])
def person_create(request):
    info=PersonSerializer(data=request.data)
    if info.is_valid():
        info.save()

        return Response({'message':'ok'},status=status.HTTP_201_CREATED)
    else:
        return Response(info.errors,status=status.HTTP_400_BAD_REQUEST)