From a0921c18596db1618fa5814f4b4bb125dacefdd1 Mon Sep 17 00:00:00 2001
From: Corban-Lee Jones <corbz.jones@gmail.com>
Date: Sat, 17 Feb 2024 19:09:52 +0000
Subject: [PATCH] working on tasks

---
 src/extensions/tasks.py | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/src/extensions/tasks.py b/src/extensions/tasks.py
index 883d152..a0ca4e9 100644
--- a/src/extensions/tasks.py
+++ b/src/extensions/tasks.py
@@ -75,7 +75,7 @@ class TaskCog(commands.Cog):
     async def rss_task(self):
         """Automated task responsible for processing rss feeds."""
 
-        log.info("Running rss task")
+        log.info("Running subscription task")
         time = process_time()
 
         # async with DatabaseManager() as database:
@@ -87,27 +87,39 @@ class TaskCog(commands.Cog):
         #         await self.process_feed(feed, database)
 
         guild_ids = [guild.id for guild in self.bot.guilds]
+        page = 1
+        page_size = 10
+
+        async def get_subs(api, page=1):
+            return await api.get_subscriptions(server__in=guild_ids, page_size=page_size)
 
         async with aiohttp.ClientSession() as session:
             api = API(self.bot.api_token, session)
-            sub_data = api.get_subscriptions(server__in=guild_ids)
-            subscriptions = Subscriptions.from_list(sub_data)
+            data, total_subs = await get_subs(api)
+            await self.batch_process_subs(data)
+            processed_subs = len(data)
 
-            for sub in subscriptions:
-                channel_data = api.get_subscription_channels(subscription__uuid=sub.uuid)
-                channels = SubscriptionChannel.from_list(channel_data)
+            while processed_subs < total_subs:
+                log.debug("we are missing '%s' items, fetching next page '%s'", total_subs - processed_subs, page + 1)
 
+                page += 1
+                data, _ = await get_subs(api, page)
+                await self.batch_process_subs(next_page_data)
+                processed_subs += len(data)
 
-        # async with aiohttp.ClientSession() as session:
-        #     api = API(self.bot.api_token, session)
-        #     data, count = await api.get_rssfeed_list(discord_server_id__in=guild_ids)
-        #     rssfeeds = RSSFeed.from_list(data)
-        #     for item in rssfeeds:
-        #         log.info(item.name)
+            else:
+                log.debug("we have all '%s' items, ending while loop", total_subs)
 
+        log.info("Finished subscription task, time elapsed: %s", process_time() - time)
 
+    async def batch_process_subs(self, data: list):
+
+        log.debug("batch process subs, count '%s'", len(data))
+        subscriptions = Subscription.from_list(data)
+
+        for sub in subscriptions:
+            log.info(sub.name)
 
-        log.info("Finished rss task, time elapsed: %s", process_time() - time)
 
     async def process_feed(self, feed: FeedChannelModel, database: DatabaseManager):
         """Process the passed feed. Will also call process for each article found in the feed.