send content to channels
This commit is contained in:
parent
4f4d827d3c
commit
3c1aebc8c7
11
src/api.py
11
src/api.py
@ -108,7 +108,7 @@ class API:
|
||||
url=url
|
||||
)
|
||||
|
||||
async def get_subscriptions(self, **filters):
|
||||
async def get_subscriptions(self, **filters) -> tuple[list[dict], int]:
|
||||
"""
|
||||
Get multiple subscriptions.
|
||||
"""
|
||||
@ -117,6 +117,15 @@ class API:
|
||||
|
||||
return await self._get_many(self.API_ENDPOINT + "subscription/", filters)
|
||||
|
||||
async def get_subscription_channels(self, **filters) -> tuple[list[dict], int]:
|
||||
"""
|
||||
Get many subscription channels.
|
||||
"""
|
||||
|
||||
log.debug("getting multiple channels")
|
||||
|
||||
return await self._get_many(self.API_ENDPOINT + "subchannel/", filters)
|
||||
|
||||
# async def create_subscription(self, name: str, rss_url: str, image_url: str, server_id: str, targets: list) -> dict:
|
||||
# """
|
||||
# Create a new Subscription.
|
||||
|
||||
@ -11,7 +11,7 @@ import validators
|
||||
from feedparser import FeedParserDict, parse
|
||||
from discord.ext import commands
|
||||
from discord import Interaction, TextChannel
|
||||
from discord.app_commands import Choice, Group, autocomplete, rename
|
||||
from discord.app_commands import Choice, Group, autocomplete, rename, command
|
||||
|
||||
from api import API
|
||||
from feed import Subscription, SubscriptionChannel, TrackedContent
|
||||
@ -92,240 +92,240 @@ class FeedCog(commands.Cog):
|
||||
|
||||
log.info("%s cog is ready", self.__class__.__name__)
|
||||
|
||||
async def autocomplete_subscriptions(self, inter: Interaction, name: str) -> list[Choice]:
|
||||
""""""
|
||||
# async def autocomplete_subscriptions(self, inter: Interaction, name: str) -> list[Choice]:
|
||||
# """"""
|
||||
|
||||
log.debug("autocompleting subscriptions '%s'", name)
|
||||
# log.debug("autocompleting subscriptions '%s'", name)
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
results, _ = await api.get_subscriptions(server=inter.guild_id, search=name)
|
||||
# try:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# results, _ = await api.get_subscriptions(server=inter.guild_id, search=name)
|
||||
|
||||
except Exception as exc:
|
||||
log.error(exc)
|
||||
return []
|
||||
# except Exception as exc:
|
||||
# log.error(exc)
|
||||
# return []
|
||||
|
||||
subscriptions = Subscription.from_list(results)
|
||||
# subscriptions = Subscription.from_list(results)
|
||||
|
||||
return [
|
||||
Choice(name=sub.name, value=sub.uuid)
|
||||
for sub in subscriptions
|
||||
]
|
||||
# return [
|
||||
# Choice(name=sub.name, value=sub.uuid)
|
||||
# for sub in subscriptions
|
||||
# ]
|
||||
|
||||
async def autocomplete_subscription_channels(self, inter: Interaction, uuid: str):
|
||||
""""""
|
||||
# async def autocomplete_subscription_channels(self, inter: Interaction, uuid: str):
|
||||
# """"""
|
||||
|
||||
log.debug("autocompleting subscription channels")
|
||||
# log.debug("autocompleting subscription channels")
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
results, _ = await api.get_subscription_channels()
|
||||
# try:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# results, _ = await api.get_subscription_channels()
|
||||
|
||||
except Exception as exc:
|
||||
log.error(exc)
|
||||
return []
|
||||
# except Exception as exc:
|
||||
# log.error(exc)
|
||||
# return []
|
||||
|
||||
subscription_channels = SubscriptionChannel.from_list(results)
|
||||
# subscription_channels = SubscriptionChannel.from_list(results)
|
||||
|
||||
async def name(link):
|
||||
result = self.bot.get_channel(link.id) or await self.bot.fetch_channel(link.id)
|
||||
return f"{link.subscription.name} -> #{result.name}"
|
||||
# async def name(link):
|
||||
# result = self.bot.get_channel(link.id) or await self.bot.fetch_channel(link.id)
|
||||
# return f"{link.subscription.name} -> #{result.name}"
|
||||
|
||||
return [
|
||||
Choice(name=await name(link), value=link.uuid)
|
||||
for link in subscription_channels
|
||||
]
|
||||
# return [
|
||||
# Choice(name=await name(link), value=link.uuid)
|
||||
# for link in subscription_channels
|
||||
# ]
|
||||
|
||||
subscription_group = Group(
|
||||
name="subscriptions",
|
||||
description="subscription commands",
|
||||
guild_only=True
|
||||
)
|
||||
# subscription_group = Group(
|
||||
# name="subscriptions",
|
||||
# description="subscription commands",
|
||||
# guild_only=True
|
||||
# )
|
||||
|
||||
@subscription_group.command(name="link")
|
||||
@autocomplete(sub_uuid=autocomplete_subscriptions)
|
||||
@rename(sub_uuid="subscription")
|
||||
async def link_subscription_channel(self, inter: Interaction, sub_uuid: str, channel: TextChannel):
|
||||
"""
|
||||
Link Subscription to discord.TextChannel.
|
||||
"""
|
||||
# @subscription_group.command(name="link")
|
||||
# @autocomplete(sub_uuid=autocomplete_subscriptions)
|
||||
# @rename(sub_uuid="subscription")
|
||||
# async def link_subscription_channel(self, inter: Interaction, sub_uuid: str, channel: TextChannel):
|
||||
# """
|
||||
# Link Subscription to discord.TextChannel.
|
||||
# """
|
||||
|
||||
await inter.response.defer()
|
||||
# await inter.response.defer()
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
data = await api.create_subscription_channel(str(channel.id), sub_uuid)
|
||||
# try:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# data = await api.create_subscription_channel(str(channel.id), sub_uuid)
|
||||
|
||||
except aiohttp.ClientResponseError as exc:
|
||||
return await (
|
||||
Followup(
|
||||
f"Error · {exc.message}",
|
||||
"Ensure you haven't: \n"
|
||||
"- Already linked this subscription to this channel\n"
|
||||
"- Already linked this subscription to the maximum of 4 channels"
|
||||
)
|
||||
.footer(f"HTTP {exc.code}")
|
||||
.error()
|
||||
.send(inter)
|
||||
)
|
||||
# except aiohttp.ClientResponseError as exc:
|
||||
# return await (
|
||||
# Followup(
|
||||
# f"Error · {exc.message}",
|
||||
# "Ensure you haven't: \n"
|
||||
# "- Already linked this subscription to this channel\n"
|
||||
# "- Already linked this subscription to the maximum of 4 channels"
|
||||
# )
|
||||
# .footer(f"HTTP {exc.code}")
|
||||
# .error()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
subscription = Subscription.from_dict(data.pop("subscription"))
|
||||
data["subscription"] = (
|
||||
f"{subscription.name}\n"
|
||||
f"[RSS]({subscription.rss_url}) · "
|
||||
f"[API Subscription]({API.SUBSCRIPTION_ENDPOINT}{subscription.uuid}) · "
|
||||
f"[API Link]({API.CHANNEL_ENDPOINT}{data['uuid']})"
|
||||
)
|
||||
# subscription = Subscription.from_dict(data.pop("subscription"))
|
||||
# data["subscription"] = (
|
||||
# f"{subscription.name}\n"
|
||||
# f"[RSS]({subscription.rss_url}) · "
|
||||
# f"[API Subscription]({API.SUBSCRIPTION_ENDPOINT}{subscription.uuid}) · "
|
||||
# f"[API Link]({API.CHANNEL_ENDPOINT}{data['uuid']})"
|
||||
# )
|
||||
|
||||
channel_id = int(data.pop("id"))
|
||||
channel = self.bot.get_channel(channel_id) or await self.bot.fetch_channel(channel_id)
|
||||
data["channel"] = channel.mention
|
||||
# channel_id = int(data.pop("id"))
|
||||
# channel = self.bot.get_channel(channel_id) or await self.bot.fetch_channel(channel_id)
|
||||
# data["channel"] = channel.mention
|
||||
|
||||
data.pop("creation_datetime")
|
||||
data.pop("uuid")
|
||||
# data.pop("creation_datetime")
|
||||
# data.pop("uuid")
|
||||
|
||||
await (
|
||||
Followup("Linked!")
|
||||
.fields(**data)
|
||||
.added()
|
||||
.send(inter)
|
||||
)
|
||||
# await (
|
||||
# Followup("Linked!")
|
||||
# .fields(**data)
|
||||
# .added()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
@subscription_group.command(name="unlink")
|
||||
@autocomplete(uuid=autocomplete_subscription_channels)
|
||||
@rename(uuid="link")
|
||||
async def unlink_subscription_channel(self, inter: Interaction, uuid: str):
|
||||
"""
|
||||
Unlink subscription from discord.TextChannel.
|
||||
"""
|
||||
# @subscription_group.command(name="unlink")
|
||||
# @autocomplete(uuid=autocomplete_subscription_channels)
|
||||
# @rename(uuid="link")
|
||||
# async def unlink_subscription_channel(self, inter: Interaction, uuid: str):
|
||||
# """
|
||||
# Unlink subscription from discord.TextChannel.
|
||||
# """
|
||||
|
||||
await inter.response.defer()
|
||||
# await inter.response.defer()
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
# data = await api.get_subscription(uuid=uuid)
|
||||
await api.delete_subscription_channel(uuid=uuid)
|
||||
# sub_channel = await SubscriptionChannel.from_dict(data)
|
||||
# try:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# # data = await api.get_subscription(uuid=uuid)
|
||||
# await api.delete_subscription_channel(uuid=uuid)
|
||||
# # sub_channel = await SubscriptionChannel.from_dict(data)
|
||||
|
||||
except Exception as exc:
|
||||
return await (
|
||||
Followup(exc.__class__.__name__, str(exc))
|
||||
.error()
|
||||
.send(inter)
|
||||
)
|
||||
# except Exception as exc:
|
||||
# return await (
|
||||
# Followup(exc.__class__.__name__, str(exc))
|
||||
# .error()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
await (
|
||||
Followup("Subscription unlinked!", uuid)
|
||||
.added()
|
||||
.send(inter)
|
||||
)
|
||||
# await (
|
||||
# Followup("Subscription unlinked!", uuid)
|
||||
# .added()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
@subscription_group.command(name="list-links")
|
||||
async def list_subscription(self, inter: Interaction):
|
||||
"""List Subscriptions Channels in this server."""
|
||||
# @subscription_group.command(name="list-links")
|
||||
# async def list_subscription(self, inter: Interaction):
|
||||
# """List Subscriptions Channels in this server."""
|
||||
|
||||
await inter.response.defer()
|
||||
# await inter.response.defer()
|
||||
|
||||
async def formatdata(index: int, item: dict) -> tuple[str, str]:
|
||||
item = SubscriptionChannel.from_dict(item)
|
||||
next_emoji = self.bot.get_emoji(1204542366602502265)
|
||||
key = f"{index}. {item.subscription.name} {next_emoji} {item.mention}"
|
||||
return key, item.hyperlinks_string
|
||||
# async def formatdata(index: int, item: dict) -> tuple[str, str]:
|
||||
# item = SubscriptionChannel.from_dict(item)
|
||||
# next_emoji = self.bot.get_emoji(1204542366602502265)
|
||||
# key = f"{index}. {item.subscription.name} {next_emoji} {item.mention}"
|
||||
# return key, item.hyperlinks_string
|
||||
|
||||
async def getdata(page: int, pagesize: int) -> dict:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
return await api.get_subscription_channels(
|
||||
subscription__server=inter.guild.id, page=page, page_size=pagesize
|
||||
)
|
||||
# async def getdata(page: int, pagesize: int) -> dict:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# return await api.get_subscription_channels(
|
||||
# subscription__server=inter.guild.id, page=page, page_size=pagesize
|
||||
# )
|
||||
|
||||
embed = Followup(f"Links in {inter.guild.name}").info()._embed
|
||||
pagination = PaginationView(
|
||||
self.bot,
|
||||
inter=inter,
|
||||
embed=embed,
|
||||
getdata=getdata,
|
||||
formatdata=formatdata,
|
||||
pagesize=10,
|
||||
initpage=1
|
||||
)
|
||||
await pagination.send()
|
||||
# embed = Followup(f"Links in {inter.guild.name}").info()._embed
|
||||
# pagination = PaginationView(
|
||||
# self.bot,
|
||||
# inter=inter,
|
||||
# embed=embed,
|
||||
# getdata=getdata,
|
||||
# formatdata=formatdata,
|
||||
# pagesize=10,
|
||||
# initpage=1
|
||||
# )
|
||||
# await pagination.send()
|
||||
|
||||
@subscription_group.command(name="add")
|
||||
async def new_subscription(self, inter: Interaction, name: str, rss_url: str):
|
||||
"""Subscribe this server to a new RSS Feed."""
|
||||
# @subscription_group.command(name="add")
|
||||
# async def new_subscription(self, inter: Interaction, name: str, rss_url: str):
|
||||
# """Subscribe this server to a new RSS Feed."""
|
||||
|
||||
await inter.response.defer()
|
||||
# await inter.response.defer()
|
||||
|
||||
try:
|
||||
parsed_rssfeed = await self.bot.functions.validate_feed(name, rss_url)
|
||||
image_url = parsed_rssfeed.get("feed", {}).get("image", {}).get("href")
|
||||
# try:
|
||||
# parsed_rssfeed = await self.bot.functions.validate_feed(name, rss_url)
|
||||
# image_url = parsed_rssfeed.get("feed", {}).get("image", {}).get("href")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
data = await api.create_subscription(name, rss_url, image_url, str(inter.guild_id), [-1])
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# data = await api.create_subscription(name, rss_url, image_url, str(inter.guild_id), [-1])
|
||||
|
||||
except aiohttp.ClientResponseError as exc:
|
||||
return await (
|
||||
Followup(
|
||||
f"Error · {exc.message}",
|
||||
"Ensure you haven't: \n"
|
||||
"- Reused an identical name of an existing Subscription\n"
|
||||
"- Already created the maximum of 25 Subscriptions"
|
||||
)
|
||||
.footer(f"HTTP {exc.code}")
|
||||
.error()
|
||||
.send(inter)
|
||||
)
|
||||
# except aiohttp.ClientResponseError as exc:
|
||||
# return await (
|
||||
# Followup(
|
||||
# f"Error · {exc.message}",
|
||||
# "Ensure you haven't: \n"
|
||||
# "- Reused an identical name of an existing Subscription\n"
|
||||
# "- Already created the maximum of 25 Subscriptions"
|
||||
# )
|
||||
# .footer(f"HTTP {exc.code}")
|
||||
# .error()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
# Omit data we dont want the user to see
|
||||
data.pop("uuid")
|
||||
data.pop("image")
|
||||
data.pop("server")
|
||||
data.pop("creation_datetime")
|
||||
# # Omit data we dont want the user to see
|
||||
# data.pop("uuid")
|
||||
# data.pop("image")
|
||||
# data.pop("server")
|
||||
# data.pop("creation_datetime")
|
||||
|
||||
# Update keys to be more human readable
|
||||
data["url"] = data.pop("rss_url")
|
||||
# # Update keys to be more human readable
|
||||
# data["url"] = data.pop("rss_url")
|
||||
|
||||
await (
|
||||
Followup("Subscription Added!")
|
||||
.fields(**data)
|
||||
.image(image_url)
|
||||
.added()
|
||||
.send(inter)
|
||||
)
|
||||
# await (
|
||||
# Followup("Subscription Added!")
|
||||
# .fields(**data)
|
||||
# .image(image_url)
|
||||
# .added()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
@subscription_group.command(name="remove")
|
||||
@autocomplete(uuid=autocomplete_subscriptions)
|
||||
@rename(uuid="choice")
|
||||
async def remove_subscriptions(self, inter: Interaction, uuid: str):
|
||||
"""Unsubscribe this server from an existing RSS Feed."""
|
||||
# @subscription_group.command(name="remove")
|
||||
# @autocomplete(uuid=autocomplete_subscriptions)
|
||||
# @rename(uuid="choice")
|
||||
# async def remove_subscriptions(self, inter: Interaction, uuid: str):
|
||||
# """Unsubscribe this server from an existing RSS Feed."""
|
||||
|
||||
await inter.response.defer()
|
||||
# await inter.response.defer()
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
await api.delete_subscription(uuid)
|
||||
# try:
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
# api = API(self.bot.api_token, session)
|
||||
# await api.delete_subscription(uuid)
|
||||
|
||||
except Exception as exc:
|
||||
return await (
|
||||
Followup(exc.__class__.__name__, str(exc))
|
||||
.error()
|
||||
.send(inter)
|
||||
)
|
||||
# except Exception as exc:
|
||||
# return await (
|
||||
# Followup(exc.__class__.__name__, str(exc))
|
||||
# .error()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
await (
|
||||
Followup("Subscription Removed!", uuid)
|
||||
.trash()
|
||||
.send(inter)
|
||||
)
|
||||
# await (
|
||||
# Followup("Subscription Removed!", uuid)
|
||||
# .trash()
|
||||
# .send(inter)
|
||||
# )
|
||||
|
||||
@subscription_group.command(name="list")
|
||||
@command(name="subscriptions")
|
||||
async def list_subscription(self, inter: Interaction):
|
||||
"""List Subscriptions from this server."""
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ Extension for the `TaskCog`.
|
||||
Loading this file via `commands.Bot.load_extension` will add `TaskCog` to the bot.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import datetime
|
||||
from os import getenv
|
||||
@ -16,7 +17,7 @@ from discord.errors import Forbidden
|
||||
from sqlalchemy import insert, select, and_
|
||||
from feedparser import parse
|
||||
|
||||
from feed import Source, Article, RSSFeed, Subscription, SubscriptionChannel
|
||||
from feed import Source, Article, RSSFeed, Subscription, SubscriptionChannel, SubChannel
|
||||
from db import (
|
||||
DatabaseManager,
|
||||
FeedChannelModel,
|
||||
@ -75,133 +76,166 @@ class TaskCog(commands.Cog):
|
||||
async def rss_task(self):
|
||||
"""Automated task responsible for processing rss feeds."""
|
||||
|
||||
log.debug("sub task disabled")
|
||||
return
|
||||
|
||||
log.info("Running subscription task")
|
||||
time = process_time()
|
||||
|
||||
# async with DatabaseManager() as database:
|
||||
# query = select(FeedChannelModel, RssSourceModel).join(RssSourceModel)
|
||||
# result = await database.session.execute(query)
|
||||
# feeds = result.scalars().all()
|
||||
|
||||
# for feed in feeds:
|
||||
# await self.process_feed(feed, database)
|
||||
|
||||
guild_ids = [guild.id for guild in self.bot.guilds]
|
||||
page = 1
|
||||
page_size = 10
|
||||
|
||||
async def get_subs(api, page=1):
|
||||
return await api.get_subscriptions(server__in=guild_ids, page_size=page_size)
|
||||
data = []
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
api = API(self.bot.api_token, session)
|
||||
data, total_subs = await get_subs(api)
|
||||
await self.batch_process_subs(data)
|
||||
processed_subs = len(data)
|
||||
|
||||
while processed_subs < total_subs:
|
||||
log.debug("we are missing '%s' items, fetching next page '%s'", total_subs - processed_subs, page + 1)
|
||||
page = 0
|
||||
|
||||
while True:
|
||||
page += 1
|
||||
data, _ = await get_subs(api, page)
|
||||
await self.batch_process_subs(next_page_data)
|
||||
processed_subs += len(data)
|
||||
page_data = await self.get_subscriptions(api, guild_ids, page)
|
||||
|
||||
else:
|
||||
log.debug("we have all '%s' items, ending while loop", total_subs)
|
||||
if not page_data:
|
||||
break
|
||||
|
||||
log.info("Finished subscription task, time elapsed: %s", process_time() - time)
|
||||
data.extend(page_data)
|
||||
log.debug("extending data by '%s' items", len(page_data))
|
||||
|
||||
async def batch_process_subs(self, data: list):
|
||||
log.debug("finished api data collection, browsed %s pages for %s subscriptions", page, len(data))
|
||||
|
||||
log.debug("batch process subs, count '%s'", len(data))
|
||||
subscriptions = Subscription.from_list(data)
|
||||
subscriptions = Subscription.from_list(data)
|
||||
for sub in subscriptions:
|
||||
await self.process_subscription(api, session, sub)
|
||||
|
||||
for sub in subscriptions:
|
||||
log.info(sub.name)
|
||||
log.info("Finished subscription task, time elapsed: %s", process_time() - time)
|
||||
|
||||
async def get_subscriptions(self, api, guild_ids: list[int], page: int):
|
||||
|
||||
async def process_feed(self, feed: FeedChannelModel, database: DatabaseManager):
|
||||
"""Process the passed feed. Will also call process for each article found in the feed.
|
||||
log.debug("attempting to get subscriptions for page: %s", page)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
feed : FeedChannelModel
|
||||
Database model for the feed.
|
||||
database : DatabaseManager
|
||||
Database connection handler, must be open.
|
||||
"""
|
||||
|
||||
log.debug("Processing feed: %s", feed.id)
|
||||
|
||||
channel = self.bot.get_channel(feed.discord_channel_id)
|
||||
|
||||
# TODO: integrate the `validate_feed` code into here, also do on list command and show errors.
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
||||
unparsed_content = await get_unparsed_feed(feed.rss_source.rss_url)
|
||||
parsed_feed = parse(unparsed_content)
|
||||
source = Source.from_parsed(parsed_feed)
|
||||
articles = source.get_latest_articles(5)
|
||||
|
||||
if not articles:
|
||||
log.info("No articles to process for %s in ", feed.rss_source.nick, feed.discord_server_id)
|
||||
return
|
||||
|
||||
for article in articles:
|
||||
await self.process_article(feed.id, article, channel, database, session)
|
||||
|
||||
async def process_article(
|
||||
self, feed_id: int, article: Article, channel: TextChannel, database: DatabaseManager,
|
||||
session: aiohttp.ClientSession
|
||||
):
|
||||
"""Process the passed article. Will send the embed to a channel if all is valid.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
feed_id : int
|
||||
The feed model ID, used to log the sent article.
|
||||
article : Article
|
||||
Database model for the article.
|
||||
channel : TextChannel
|
||||
Where the article will be sent to.
|
||||
database : DatabaseManager
|
||||
Database connection handler, must be open.
|
||||
"""
|
||||
|
||||
log.debug("Processing article: %s", article.url)
|
||||
|
||||
query = select(SentArticleModel).where(and_(
|
||||
SentArticleModel.article_url == article.url,
|
||||
SentArticleModel.discord_channel_id == channel.id,
|
||||
))
|
||||
result = await database.session.execute(query)
|
||||
|
||||
if result.scalars().all():
|
||||
log.debug("Article already processed: %s", article.url)
|
||||
return
|
||||
|
||||
embed = await article.to_embed(session)
|
||||
try:
|
||||
await channel.send(embed=embed)
|
||||
except Forbidden as error: # TODO: find some way of informing the user about this error.
|
||||
log.error("Can't send article to channel: %s · %s · %s", channel.name, channel.id, error)
|
||||
return (await api.get_subscriptions(server__in=guild_ids, page=page))[0]
|
||||
except aiohttp.ClientResponseError as error:
|
||||
if error.status == 404:
|
||||
log.debug(error)
|
||||
return []
|
||||
|
||||
log.error(error)
|
||||
|
||||
async def process_subscription(self, api, session, sub: Subscription):
|
||||
log.debug("processing subscription '%s' '%s' for '%s'", sub.id, sub.name, sub.guild_id)
|
||||
|
||||
if not sub.active:
|
||||
log.debug("skipping sub because it's active flag is 'False'")
|
||||
return
|
||||
|
||||
query = insert(SentArticleModel).values(
|
||||
article_url = article.url,
|
||||
discord_channel_id = channel.id,
|
||||
discord_server_id = channel.guild.id,
|
||||
discord_message_id = -1,
|
||||
feed_channel_id = feed_id
|
||||
)
|
||||
await database.session.execute(query)
|
||||
channels = [self.bot.get_channel(subchannel.channel_id) for subchannel in await sub.get_channels(api)]
|
||||
if not channels:
|
||||
log.warning("No channels to send this to")
|
||||
return
|
||||
|
||||
log.debug("new Article processed: %s", article.url)
|
||||
unparsed_content = await get_unparsed_feed(sub.url, session)
|
||||
parsed_content = parse(unparsed_content)
|
||||
source = Source.from_parsed(parsed_content)
|
||||
articles = source.get_latest_articles(3)
|
||||
|
||||
if not articles:
|
||||
log.debug("No articles found")
|
||||
|
||||
for article in articles:
|
||||
await self.process_article(session, channels, article)
|
||||
|
||||
async def process_article(self, session, channels: list[SubChannel], article: Article):
|
||||
embed = await article.to_embed(session)
|
||||
|
||||
log.debug("attempting to send embed to %s channel(s)", len(channels))
|
||||
|
||||
for channel in channels:
|
||||
await channel.send(embed=embed)
|
||||
|
||||
|
||||
# async def batch_process_subs(self, data: list):
|
||||
|
||||
# log.debug("batch process subs, count '%s'", len(data))
|
||||
# subscriptions = Subscription.from_list(data)
|
||||
|
||||
# for sub in subscriptions:
|
||||
# log.info(sub.name)
|
||||
|
||||
|
||||
# async def process_feed(self, feed: FeedChannelModel, database: DatabaseManager):
|
||||
# """Process the passed feed. Will also call process for each article found in the feed.
|
||||
|
||||
# Parameters
|
||||
# ----------
|
||||
# feed : FeedChannelModel
|
||||
# Database model for the feed.
|
||||
# database : DatabaseManager
|
||||
# Database connection handler, must be open.
|
||||
# """
|
||||
|
||||
# log.debug("Processing feed: %s", feed.id)
|
||||
|
||||
# channel = self.bot.get_channel(feed.discord_channel_id)
|
||||
|
||||
# # TODO: integrate the `validate_feed` code into here, also do on list command and show errors.
|
||||
|
||||
# async with aiohttp.ClientSession() as session:
|
||||
|
||||
# unparsed_content = await get_unparsed_feed(feed.rss_source.rss_url)
|
||||
# parsed_feed = parse(unparsed_content)
|
||||
# source = Source.from_parsed(parsed_feed)
|
||||
# articles = source.get_latest_articles(5)
|
||||
|
||||
# if not articles:
|
||||
# log.info("No articles to process for %s in ", feed.rss_source.nick, feed.discord_server_id)
|
||||
# return
|
||||
|
||||
# for article in articles:
|
||||
# await self.process_article(feed.id, article, channel, database, session)
|
||||
|
||||
# async def process_article(
|
||||
# self, feed_id: int, article: Article, channel: TextChannel, database: DatabaseManager,
|
||||
# session: aiohttp.ClientSession
|
||||
# ):
|
||||
# """Process the passed article. Will send the embed to a channel if all is valid.
|
||||
|
||||
# Parameters
|
||||
# ----------
|
||||
# feed_id : int
|
||||
# The feed model ID, used to log the sent article.
|
||||
# article : Article
|
||||
# Database model for the article.
|
||||
# channel : TextChannel
|
||||
# Where the article will be sent to.
|
||||
# database : DatabaseManager
|
||||
# Database connection handler, must be open.
|
||||
# """
|
||||
|
||||
# log.debug("Processing article: %s", article.url)
|
||||
|
||||
# query = select(SentArticleModel).where(and_(
|
||||
# SentArticleModel.article_url == article.url,
|
||||
# SentArticleModel.discord_channel_id == channel.id,
|
||||
# ))
|
||||
# result = await database.session.execute(query)
|
||||
|
||||
# if result.scalars().all():
|
||||
# log.debug("Article already processed: %s", article.url)
|
||||
# return
|
||||
|
||||
# embed = await article.to_embed(session)
|
||||
# try:
|
||||
# await channel.send(embed=embed)
|
||||
# except Forbidden as error: # TODO: find some way of informing the user about this error.
|
||||
# log.error("Can't send article to channel: %s · %s · %s", channel.name, channel.id, error)
|
||||
# return
|
||||
|
||||
# query = insert(SentArticleModel).values(
|
||||
# article_url = article.url,
|
||||
# discord_channel_id = channel.id,
|
||||
# discord_server_id = channel.guild.id,
|
||||
# discord_message_id = -1,
|
||||
# feed_channel_id = feed_id
|
||||
# )
|
||||
# await database.session.execute(query)
|
||||
|
||||
# log.debug("new Article processed: %s", article.url)
|
||||
|
||||
|
||||
async def setup(bot):
|
||||
|
||||
46
src/feed.py
46
src/feed.py
@ -52,7 +52,7 @@ class Article:
|
||||
The Article created from the feed entry.
|
||||
"""
|
||||
|
||||
log.debug("Creating Article from entry: %s", dumps(entry))
|
||||
# log.debug("Creating Article from entry: %s", dumps(entry))
|
||||
|
||||
published_parsed = entry.get("published_parsed")
|
||||
published = datetime(*entry.published_parsed[0:-2]) if published_parsed else None
|
||||
@ -80,7 +80,7 @@ class Article:
|
||||
The thumbnail URL, or None if not found.
|
||||
"""
|
||||
|
||||
log.debug("Fetching thumbnail for article: %s", self)
|
||||
# log.debug("Fetching thumbnail for article: %s", self)
|
||||
|
||||
try:
|
||||
async with session.get(self.url, timeout=15) as response:
|
||||
@ -111,7 +111,7 @@ class Article:
|
||||
A Discord Embed object representing the article.
|
||||
"""
|
||||
|
||||
log.debug(f"Creating embed from article: {self}")
|
||||
# log.debug(f"Creating embed from article: {self}")
|
||||
|
||||
# Replace HTML with Markdown, and shorten text.
|
||||
title = shorten(markdownify(self.title, strip=["img", "a"]), 256)
|
||||
@ -164,7 +164,7 @@ class Source:
|
||||
The Source object
|
||||
"""
|
||||
|
||||
log.debug("Creating Source from feed: %s", dumps(feed))
|
||||
# log.debug("Creating Source from feed: %s", dumps(feed))
|
||||
|
||||
return cls(
|
||||
name=feed.get("channel", {}).get("title"),
|
||||
@ -194,7 +194,7 @@ class Source:
|
||||
A list of Article objects.
|
||||
"""
|
||||
|
||||
log.debug("Fetching latest articles from %s, max=%s", self, max)
|
||||
# log.debug("Fetching latest articles from %s, max=%s", self, max)
|
||||
|
||||
return [
|
||||
Article.from_entry(self, entry)
|
||||
@ -270,24 +270,28 @@ class Subscription(DjangoDataModel):
|
||||
|
||||
return item
|
||||
|
||||
# uuid: str
|
||||
# name: str
|
||||
# rss_url: str
|
||||
# image_url: str
|
||||
# creation_datetime: datetime
|
||||
# server: int
|
||||
# targets: list[int]
|
||||
# extra_notes: str
|
||||
# active: bool
|
||||
async def get_channels(self, api):
|
||||
channel_data, _ = await api.get_subscription_channels(subscription=self.id)
|
||||
return SubChannel.from_list(channel_data)
|
||||
|
||||
# @staticmethod
|
||||
# def parser(item: dict) -> dict:
|
||||
|
||||
# item["image_url"] = item.pop("image")
|
||||
# item["creation_datetime"] = datetime.strptime(item["creation_datetime"], DATETIME_FORMAT)
|
||||
# item["server"] = int(item["server"])
|
||||
# item["targets"] = item["targets"].split(";")
|
||||
# return item
|
||||
@dataclass(slots=True)
|
||||
class SubChannel(DjangoDataModel):
|
||||
|
||||
id: int
|
||||
channel_id: int
|
||||
subscription: int
|
||||
|
||||
@staticmethod
|
||||
def parser(item: dict) -> dict:
|
||||
item["channel_id"] = int(item["channel_id"])
|
||||
item["subscription"] = int(item["subscription"])
|
||||
|
||||
return item
|
||||
|
||||
@property
|
||||
def mention(self) -> str:
|
||||
return f"<#{self.channel_id}>"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user