Merge from master

Author: Taaku18

CHANGELOG.md

@@ -45,13 +45,20 @@ A `?delete` command, which is an alternative to manually deleting a message. Thi
 When updating to this version, all prior permission settings with guild-based permissions will be invalidated. You will need to convert to the above system.
 `OWNERS` will also get removed, you will need to set owners through `?permissions add level owner 212931293123129` or any way listed above.
 
-# v2.17.1
+
+# v2.17.2
 
 ### Changed
 
-- Stricter fallback genesis embed search. This update shouldn't affect anyone.
+- Logs search command will search through log keys as well now. 
+- For example, `?logs search e7499e82f8ff`.
 
-### Info
+# v2.17.1
+### What's new?
+
+Stricter fallback genesis embed search.
+
+### Changed
 How modmail checks if a channel is a thread: 
 
 1. First the bot checks if the channel topic is in the format `User ID: xxxx`, this means it is a thread.
@@ -79,10 +86,21 @@ An issue where a scheduled close would not execute over a long period of time if
 
 # v2.16.0
 
-### What's new?
+### Changed
+
+All support for Modmail API (api.modmail.tk) has terminated. 
+If you're still using api.modmail.tk, you will need to migrate to the self-hosted database
+option ASAP. Your bot will not work unless you switch to the self-hosted option. Refer to the 
+installation tutorial for information regarding self-hosted Modmail.
 
 If a member leaves/joins (again) while they are a recipient of a thread, a message will be sent to notify you that this has occured.
 
+# v2.15.1
+
+### Fixed
+
+Emergency patch of a SyntaxError.
+
 # v2.15.0
 
 ### What's new?
@@ -270,7 +288,7 @@ Huge thanks to Sasiko for reporting these issues.
 
 # v2.12.0
 
-### Important 
+### Important
 **In the future, the Modmail API (https://modmail.tk) will be deprecated. This is due to the fact that we are providing a free service without getting anything in return, and thus we do not have the resources to scale to accommodate for more users. 
 We recommend using your own database for logs. In the future you will soon get a `backup` command so you can download all your pre-existing data and migrate to your own database.** 
 

README.md

@@ -13,10 +13,6 @@
     <img src="https://img.shields.io/discord/515071617815019520.svg?style=for-the-badge&colorB=7289DA" alt="Support">
   </a>
 
-  <a href="https://github.com/kyb3r/modmail/">
-    <img src="https://api.modmail.tk/badges/instances.svg" alt="Bot instances">
-  </a>
-  
   <a href="https://patreon.com/kyber">
     <img src="https://img.shields.io/badge/patreon-donate-orange.svg?style=for-the-badge" alt="Python 3.7">
   </a>
@@ -56,8 +52,7 @@ Have you sent something with the `?reply` command by accident? Don't fret, you c
 
 ## Thread Logs
 
-Thread conversations are automatically logged with a generated viewable website of the complete thread. Logs are rendered with styled HTML and presented in an aesthetically pleasing way—it blends seamlessly with the mobile version of Discord. An example of a logged conversation: https://logs.modmail.tk/02032d65a6f3
-
+Thread conversations are automatically logged with a generated viewable website of the complete thread. Logs are rendered with styled HTML and presented in an aesthetically pleasing way—it blends seamlessly with the mobile version of Discord. An example of a logged conversation: https://modmail-logs.herokuapp.com/logs/02032d65a6f3.
 
 # Contributing
 

bot.py

@@ -31,6 +31,7 @@
 import sys
 
 from datetime import datetime
+from pkg_resources import parse_version
 from types import SimpleNamespace
 
 import discord
@@ -45,7 +46,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 
 from core.changelog import Changelog
-from core.clients import ModmailApiClient, SelfHostedClient, PluginDatabaseClient
+from core.clients import SelfHostedClient, PluginDatabaseClient
 from core.config import ConfigManager
 from core.utils import info, error
 from core.models import Bot, PermissionLevel
@@ -103,14 +104,10 @@ def __init__(self):
 
         self._configure_logging()
 
-        if self.self_hosted:
-            self._db = AsyncIOMotorClient(self.config.mongo_uri).modmail_bot
-            self._api = SelfHostedClient(self)
-        else:
-            self._api = ModmailApiClient(self)
+        self._db = AsyncIOMotorClient(self.config.mongo_uri).modmail_bot
+        self._api = SelfHostedClient(self)
         self.plugin_db = PluginDatabaseClient(self)
 
-        self.data_task = self.loop.create_task(self.data_loop())
         self.autoupdate_task = self.loop.create_task(self.autoupdate_loop())
         self._load_extensions()
         self.owner = None
@@ -143,10 +140,6 @@ def version(self):
     def db(self):
         return self._db
 
-    @property
-    def self_hosted(self):
-        return bool(self.config.get('mongo_uri', ''))
-
     @property
     def api(self):
         return self._api
@@ -204,12 +197,6 @@ def run(self, *args, **kwargs):
         except Exception:
             logger.critical(error('Fatal exception'), exc_info=True)
         finally:
-            try:
-                self.data_task.cancel()
-                self.loop.run_until_complete(self.data_task)
-            except asyncio.CancelledError:
-                logger.debug('data_task has been cancelled')
-
             try:
                 self.autoupdate_task.cancel()
                 self.loop.run_until_complete(self.autoupdate_task)
@@ -339,15 +326,8 @@ def main_color(self):
 
     async def on_connect(self):
         logger.info(LINE)
-        if not self.self_hosted:
-            logger.info(info('MODE: Using the Modmail API'))
-            logger.info(LINE)
-            await self.validate_api_token()
-            logger.info(LINE)
-        else:
-            logger.info(info('Mode: Self-hosting logs.'))
-            await self.validate_database_connection()
-            logger.info(LINE)
+        await self.validate_database_connection()
+        logger.info(LINE)
         logger.info(info('Connected to gateway.'))
 
         await self.config.refresh()
@@ -358,12 +338,12 @@ async def on_connect(self):
     async def setup_indexes(self):
         """Setup text indexes so we can use the $search operator"""
         coll = self.db.logs
-        index_name = 'messages.content_text_messages.author.name_text'
+        index_name = 'messages.content_text_messages.author.name_text_key_text'
 
         index_info = await coll.index_information()
 
         # Backwards compatibility
-        old_index = 'messages.content_text'
+        old_index = 'messages.content_text_messages.author.name_text'
         if old_index in index_info:
             logger.info(info(f'Dropping old index: {old_index}'))
             await coll.drop_index(old_index)
@@ -373,7 +353,8 @@ async def setup_indexes(self):
             logger.info(info('Name: ' + index_name))
             await coll.create_index([
                 ('messages.content', 'text'),
-                ('messages.author.name', 'text')
+                ('messages.author.name', 'text'),
+                ('key', 'text')
             ])
 
     async def on_ready(self):
@@ -823,32 +804,6 @@ def overwrites(ctx):
                 )
         return overwrites
 
-    async def validate_api_token(self):
-        try:
-            self.config.modmail_api_token
-        except KeyError:
-            logger.critical(error(f'MODMAIL_API_TOKEN not found.'))
-            logger.critical(error('Set a config variable called '
-                                  'MODMAIL_API_TOKEN with a token from '
-                                  'https://dashboard.modmail.tk.'))
-            logger.critical(error('If you want to self-host logs, '
-                                  'input a MONGO_URI config variable.'))
-            logger.critical(error('A Modmail API token is not needed '
-                                  'if you are self-hosting logs.'))
-
-            return await self.logout()
-        else:
-            valid = await self.api.validate_token()
-            if not valid:
-                logger.critical(error('Invalid MODMAIL_API_TOKEN - get one '
-                                      'from https://dashboard.modmail.tk'))
-                return await self.logout()
-
-        user = await self.api.get_user_info()
-        username = user['user']['username']
-        logger.info(info('Validated token.'))
-        logger.info(info('GitHub user: ' + username))
-
     async def validate_database_connection(self):
         try:
             await self.db.command('buildinfo')
@@ -860,32 +815,6 @@ async def validate_database_connection(self):
         else:
             logger.info(info('Successfully connected to the database.'))
 
-    async def data_loop(self):
-        await self.wait_until_ready()
-        self.owner = (await self.application_info()).owner
-
-        while not self.is_closed():
-            data = {
-                "owner_name": str(self.owner),
-                "owner_id": self.owner.id,
-                "bot_id": self.user.id,
-                "bot_name": str(self.user),
-                "avatar_url": self.user.avatar_url,
-                "guild_id": self.guild_id,
-                "guild_name": self.guild.name,
-                "member_count": len(self.guild.members),
-                "uptime": (datetime.utcnow() -
-                           self.start_time).total_seconds(),
-                "latency": f'{self.ws.latency * 1000:.4f}',
-                "version": self.version,
-                # TODO: change to `self_hosted`
-                "selfhosted": self.self_hosted,
-                "last_updated": str(datetime.utcnow())
-            }
-
-            await self.api.post_metadata(data)
-            await asyncio.sleep(3600)
-
     async def autoupdate_loop(self):
         await self.wait_until_ready()
 
@@ -894,16 +823,19 @@ async def autoupdate_loop(self):
             logger.info(LINE)
             return
 
-        if self.self_hosted and not self.config.get('github_access_token'):
+        if not self.config.get('github_access_token'):
             logger.warning(info('GitHub access token not found.'))
             logger.warning(info('Autoupdates disabled.'))
             logger.info(LINE)
             return
 
+        logger.info(info('Autoupdate loop started.'))
+
         while not self.is_closed():
-            metadata = await self.api.get_metadata()
+            changelog = await Changelog.from_url(self)
+            latest = changelog.latest_version
 
-            if metadata['latest_version'] != self.version:
+            if parse_version(self.version) < parse_version(latest.version):
                 data = await self.api.update_repository()
 
                 embed = discord.Embed(color=discord.Color.green())
@@ -913,11 +845,10 @@ async def autoupdate_loop(self):
                 embed.set_author(name=user['username'] + ' - Updating Bot',
                                  icon_url=user['avatar_url'],
                                  url=user['url'])
+
                 embed.set_footer(text=f'Updating Modmail v{self.version} '
-                                      f"-> v{metadata['latest_version']}")
+                                      f'-> v{latest.version}')
 
-                changelog = await Changelog.from_url(self)
-                latest = changelog.latest_version
                 embed.description = latest.description
                 for name, value in latest.fields.items():
                     embed.add_field(name=name, value=value)

cogs/modmail.py

@@ -380,11 +380,7 @@ def format_log_embeds(self, logs, avatar_url):
 
             created_at = parser.parse(entry['created_at'])
 
-            log_url = (
-                f"https://logs.modmail.tk/{key}"
-                if not self.bot.self_hosted else
-                self.bot.config.log_url.strip('/') + f'/logs/{key}'
-            )
+            log_url = self.bot.config.log_url.strip('/') + f'/logs/{key}'
 
             username = entry['recipient']['name'] + '#'
             username += entry['recipient']['discriminator']
@@ -451,14 +447,6 @@ async def logs(self, ctx, *, member: User = None):
     @checks.has_permissions(PermissionLevel.SUPPORTER)
     async def closed_by(self, ctx, *, user: User = None):
         """Returns all logs closed by a user."""
-        if not self.bot.self_hosted:
-            embed = discord.Embed(
-                color=discord.Color.red(),
-                description='This command only works if '
-                            'you are self-hosting your logs.'
-                )
-            return await ctx.send(embed=embed)
-
         user = user or ctx.author
 
         query = {
@@ -493,14 +481,6 @@ async def search(self, ctx, limit: Optional[int] = None, *, query):
 
         await ctx.trigger_typing()
 
-        if not self.bot.self_hosted:
-            embed = discord.Embed(
-                color=discord.Color.red(),
-                description='This command only works if you '
-                            'are self-hosting your logs.'
-                )
-            return await ctx.send(embed=embed)
-
         query = {
             'guild_id': str(self.bot.guild_id),
             'open': False,