corbz/PYRSS-Bot
corbz/PYRSS-Bot
1
0
Fork 0
Code Issues 14 Pull Requests Actions Releases 2 Activity

working on tasks

Browse Source
This commit is contained in:
Corban-Lee Jones 2024-02-17 19:09:52 +00:00
parent 4e3dc174cb
commit a0921c1859
1 changed files with 25 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/extensions/tasks.py
View File

@ -75,7 +75,7 @@ class TaskCog(commands.Cog):
async def rss_task(self):
"""Automated task responsible for processing rss feeds."""
log.info("Running rss task")
log.info("Running subscription task")
time = process_time()
# async with DatabaseManager() as database:
@ -87,27 +87,39 @@ class TaskCog(commands.Cog):
# await self.process_feed(feed, database)
guild_ids = [guild.id for guild in self.bot.guilds]
page = 1
page_size = 10
async def get_subs(api, page=1):
return await api.get_subscriptions(server__in=guild_ids, page_size=page_size)
async with aiohttp.ClientSession() as session:
api = API(self.bot.api_token, session)
sub_data = api.get_subscriptions(server__in=guild_ids)
subscriptions = Subscriptions.from_list(sub_data)
data, total_subs = await get_subs(api)
await self.batch_process_subs(data)
processed_subs = len(data)
for sub in subscriptions:
channel_data = api.get_subscription_channels(subscription__uuid=sub.uuid)
channels = SubscriptionChannel.from_list(channel_data)
while processed_subs < total_subs:
log.debug("we are missing '%s' items, fetching next page '%s'", total_subs - processed_subs, page + 1)
page += 1
data, _ = await get_subs(api, page)
await self.batch_process_subs(next_page_data)
processed_subs += len(data)
# async with aiohttp.ClientSession() as session:
# api = API(self.bot.api_token, session)
# data, count = await api.get_rssfeed_list(discord_server_id__in=guild_ids)
# rssfeeds = RSSFeed.from_list(data)
# for item in rssfeeds:
# log.info(item.name)
else:
log.debug("we have all '%s' items, ending while loop", total_subs)
log.info("Finished subscription task, time elapsed: %s", process_time() - time)
async def batch_process_subs(self, data: list):
log.debug("batch process subs, count '%s'", len(data))
subscriptions = Subscription.from_list(data)
for sub in subscriptions:
log.info(sub.name)
log.info("Finished rss task, time elapsed: %s", process_time() - time)
async def process_feed(self, feed: FeedChannelModel, database: DatabaseManager):
"""Process the passed feed. Will also call process for each article found in the feed.