152 lines
5.1 KiB
TypeScript
152 lines
5.1 KiB
TypeScript
import { Client, EmbedBuilder, Guild, HexColorString, Channel as DiscordChannel, TextChannel } from "discord.js";
|
|
import RssParser from "rss-parser";
|
|
import { parse as HtmlParser } from "node-html-parser";
|
|
import { Feed, Filter, MessageStyle, Channel, MatchingAlgorithms } from "../../generated/prisma";
|
|
import * as filters from "@bot/filter";
|
|
import prisma from "@server/prisma";
|
|
import { getLogger } from "@server/../log";
|
|
|
|
const logger = getLogger(__filename);
|
|
|
|
export const triggerTask = async (client: Client) => {
|
|
for (const [_, guild] of client.guilds.cache) {
|
|
await processGuild(guild, client);
|
|
}
|
|
};
|
|
|
|
interface ExpandedFeed extends Feed {
|
|
channels: Channel[],
|
|
filters: Filter[],
|
|
message_style: MessageStyle
|
|
}
|
|
|
|
const processGuild = async (guild: Guild, client: Client) => {
|
|
const feeds = await prisma.feed.findMany({
|
|
where: { guild_id: guild.id, active: true },
|
|
include: { channels: true, filters: true, message_style: true }
|
|
}) as ExpandedFeed[];
|
|
|
|
for (const feed of feeds) {
|
|
await processFeed(feed, client);
|
|
}
|
|
};
|
|
|
|
const getParsedUrl = async (url: string) => {
|
|
try {
|
|
return new RssParser().parseURL(url)
|
|
}
|
|
catch (error) {
|
|
console.error(error);
|
|
return undefined
|
|
}
|
|
};
|
|
|
|
const processFeed = async (feed: ExpandedFeed, client: Client) => {
|
|
const parsed = await getParsedUrl(feed.url);
|
|
if (!parsed) return;
|
|
|
|
logger.debug(`Processing feed: ${feed.name}`);
|
|
|
|
for (const channelId of feed.channels.map(channel => channel.channel_id)) {
|
|
const channel = client.channels.cache.get(channelId);
|
|
if (channel) await processItems(parsed.items, feed, channel);
|
|
}
|
|
};
|
|
|
|
const processItems = async (items: RssParser.Item[], feed: ExpandedFeed, channel: DiscordChannel) => {
|
|
logger.debug(`Processing ${items.length} items`);
|
|
|
|
for (let i = items.length; i--;) {
|
|
if (new Date(items[i].pubDate!) < feed.published_threshold) {
|
|
logger.debug(`skipping outdated item: ${items[i].title}`)
|
|
items.splice(i, 1);
|
|
continue;
|
|
}
|
|
|
|
if (!(await Promise.all(feed.filters.map(f => passesFilter(f, items[i])))).every(Boolean)) {
|
|
items.splice(i, 1);
|
|
}
|
|
}
|
|
|
|
logger.debug(`Processing ${items.length} items (post-filter)`)
|
|
|
|
const batchSize = 4;
|
|
const totalBatches = Math.floor((items.length + batchSize - 1) / batchSize);
|
|
|
|
logger.debug(`batchSize: ${batchSize}, totalBatches: ${totalBatches}`)
|
|
|
|
for (let batchNumber = 0; batchNumber * batchSize < items.length; batchNumber++) {
|
|
logger.debug(`Processing items batch [${batchNumber+1}/${totalBatches}]`);
|
|
|
|
const i = batchNumber * batchSize;
|
|
const batch = items.slice(i, i + batchSize);
|
|
|
|
const embeds = await createEmbedFromItems(batch, feed, batchNumber, totalBatches);
|
|
|
|
await (channel as TextChannel).send({ embeds: embeds });
|
|
}
|
|
};
|
|
|
|
const createEmbedFromItems = async (items: RssParser.Item[], feed: ExpandedFeed, batchNumber: number, totalBatches: number) => {
|
|
if (!items.length) {
|
|
throw new Error("Items empty, expected at least 1 item.");
|
|
}
|
|
|
|
const mainEmbed = new EmbedBuilder();
|
|
const embeds = [mainEmbed]
|
|
|
|
mainEmbed.setTitle(totalBatches > 1 ? `${feed.name} [${batchNumber+1}/${totalBatches}]` : feed.name);
|
|
mainEmbed.setColor(feed.message_style.colour as HexColorString);
|
|
mainEmbed.setURL(process.env.PUBLIC_URL ?? null);
|
|
|
|
if (items.length == 1) {
|
|
mainEmbed.setImage(await getItemImageUrl(items[0].link ?? "") ?? null);
|
|
}
|
|
|
|
for (const item of items) {
|
|
const contentSnippet = item.contentSnippet + `\n[View Article](${item.link})`;
|
|
mainEmbed.addFields({
|
|
name: item.title ?? "- no title found -",
|
|
value: contentSnippet ?? "- no desc found -",
|
|
inline: false
|
|
})
|
|
|
|
if (embeds.length <= 5) {
|
|
const imageEmbed = new EmbedBuilder({ title: "dummy", url: process.env.PUBLIC_URL });
|
|
imageEmbed.setImage(await getItemImageUrl(item.link ?? "") ?? null);
|
|
embeds.push(imageEmbed);
|
|
}
|
|
}
|
|
|
|
return embeds
|
|
};
|
|
|
|
const getItemImageUrl = async (url: string) => {
|
|
const response = await fetch(url);
|
|
const html = HtmlParser.parse(await response.text());
|
|
|
|
const imageElement = html.querySelector("meta[property='og:image']");
|
|
if (!imageElement) return "";
|
|
|
|
return imageElement.getAttribute("content");
|
|
};
|
|
|
|
const passesFilter = async (filter: Filter, item: RssParser.Item) => {
|
|
if (!filter.value.trim()) return !filter.is_whitelist;
|
|
|
|
let matchFound = false;
|
|
|
|
if (filter.matching_algorithm === MatchingAlgorithms.ALL) {
|
|
matchFound = filters.all(filter, `${item.title} ${item.content}`);
|
|
} else {
|
|
matchFound = (
|
|
filters.mapAlgorithmToFunction(filter, item.title ?? "")
|
|
|| filters.mapAlgorithmToFunction(filter, item.content ?? "")
|
|
);
|
|
}
|
|
|
|
logger.debug(`Filter result: matchFound=${matchFound}, is_whitelist=${filter.is_whitelist}, willSend=${filter.is_whitelist ? matchFound : !matchFound}`);
|
|
|
|
return filter.is_whitelist ? matchFound : !matchFound;
|
|
};
|