improved filters

apps/api/serializers.py

@@ -125,7 +125,7 @@ class FilterSerializer(DynamicModelSerializer):
 
     class Meta:
         model = Filter
-        fields = ("id", "name", "keywords", "regex", "whitelist", "guild_id")
+        fields = ("id", "name", "matching_algorithm", "match", "is_insensitive", "is_whitelist", "guild_id")
 
 
 class ArticleMutatorSerializer(DynamicModelSerializer):

apps/api/views.py

@@ -110,7 +110,7 @@ class Filter_ListView(generics.ListCreateAPIView):
     serializer_class = FilterSerializer
 
     filter_backends = [filters.SearchFilter, rest_filters.DjangoFilterBackend, filters.OrderingFilter]
-    filterset_fields = ["id", "name", "keywords", "regex", "whitelist", "guild_id"]
+    filterset_fields = ["id", "name", "matching_algorithm", "match", "is_insensitive", "is_whitelist", "guild_id"]
     search_fields = ["name", "keywords", "regex"]
 
     def get_queryset(self):

apps/home/admin.py

@@ -22,7 +22,7 @@ class SubChannelAdmin(admin.ModelAdmin):
 @admin.register(Filter)
 class FilterAdmin(admin.ModelAdmin):
     list_display = [
-        "id", "name", "keywords", "regex", "whitelist", "guild_id"
+        "id", "name", "guild_id"
     ]
 
 

apps/home/migrations/0015_alter_filter_options_remove_filter_keywords_and_more.py

@@ -0,0 +1,67 @@
+# Generated by Django 5.0.4 on 2024-07-10 13:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('home', '0014_rename_published_theshold_subscription_published_threshold'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='filter',
+            options={'ordering': ('name',)},
+        ),
+        migrations.RemoveField(
+            model_name='filter',
+            name='keywords',
+        ),
+        migrations.RemoveField(
+            model_name='filter',
+            name='regex',
+        ),
+        migrations.RemoveField(
+            model_name='filter',
+            name='whitelist',
+        ),
+        migrations.RemoveField(
+            model_name='subscription',
+            name='article_fetch_limit',
+        ),
+        migrations.RemoveField(
+            model_name='subscription',
+            name='reset_article_fetch_limit',
+        ),
+        migrations.AddField(
+            model_name='filter',
+            name='is_insensitive',
+            field=models.BooleanField(default=True, verbose_name='is insensitive'),
+        ),
+        migrations.AddField(
+            model_name='filter',
+            name='is_whitelist',
+            field=models.BooleanField(default=False, verbose_name='is whitelist'),
+        ),
+        migrations.AddField(
+            model_name='filter',
+            name='match',
+            field=models.CharField(blank=True, max_length=256, verbose_name='match'),
+        ),
+        migrations.AddField(
+            model_name='filter',
+            name='matching_algorithm',
+            field=models.PositiveIntegerField(choices=[(0, 'None'), (1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
+        ),
+        migrations.AlterField(
+            model_name='filter',
+            name='guild_id',
+            field=models.CharField(max_length=128, verbose_name='guild id'),
+        ),
+        migrations.AlterField(
+            model_name='filter',
+            name='name',
+            field=models.CharField(max_length=128, verbose_name='name'),
+        ),
+    ]

apps/home/models.py

@@ -119,44 +119,49 @@ class SubChannel(models.Model):
         return self.channel_id
 
 
+# using a brilliant matching model design from paperless-ngx src
 class Filter(models.Model):
+    MATCH_NONE = 0
+    MATCH_ANY = 1
+    MATCH_ALL = 2
+    MATCH_LITERAL = 3
+    MATCH_REGEX = 4
+    MATCH_FUZZY = 5
+    MATCH_AUTO = 6
+
+    MATCHING_ALGORITHMS = (
+        # (MATCH_NONE, _("None")),
+        (MATCH_ANY, _("Any word")),
+        (MATCH_ALL, _("All words")),
+        (MATCH_LITERAL, _("Exact match")),
+        (MATCH_REGEX, _("Regular expression")),
+        (MATCH_FUZZY, _("Fuzzy word")),
+        # (MATCH_AUTO, _("Automatic")),
+    )
+
     id = models.AutoField(primary_key=True)
 
-    name = models.CharField(
+    name = models.CharField(_("name"), max_length=128)
-        max_length=32,
+
-        null=False,
+    match = models.CharField(_("match"), max_length=256, blank=True)
-        blank=False
+
+    matching_algorithm = models.PositiveIntegerField(
+        _("matching algorithm"),
+        choices=MATCHING_ALGORITHMS,
+        default=MATCH_ANY,
     )
 
-    keywords = models.CharField(
+    is_insensitive = models.BooleanField(_("is insensitive"), default=True)
-        max_length=128,
-        null=True,
-        blank=True
-    )
 
-    regex = models.CharField(
+    is_whitelist = models.BooleanField(_("is whitelist"), default=False)
-        max_length=128,
-        null=True,
-        blank=True
-    )
-
-    whitelist = models.BooleanField(default=False)
 
     # Have to use charfield instead of positiveBigIntegerField due to an Sqlite
     # issue that rounds down the value
     # https://github.com/sequelize/sequelize/issues/9335
-    guild_id = models.CharField(
+    guild_id = models.CharField(_("guild id"), max_length=128)
-        max_length=128
-    )
 
     class Meta:
-        """
+        ordering = ("name",)
-        Metadata for the Filter Model.
-        """
-
-        verbose_name = "filter"
-        verbose_name_plural = "filters"
-        get_latest_by = "id"
         constraints = [
             models.UniqueConstraint(
                 fields=["name", "guild_id"],
@@ -164,8 +169,8 @@ class Filter(models.Model):
             )
         ]
 
-    def __str__(self) -> str:
+    def __str__(self):
-        return self.name
+        return f"{self.guild_id} - {self.name}"
 
 
 class Subscription(models.Model):
@@ -217,13 +222,6 @@ class Subscription(models.Model):
         blank=True,
     )
 
-    article_fetch_limit = models.PositiveSmallIntegerField(
-        validators = [MaxValueValidator(1), MinValueValidator(10)],
-        default=10
-    )
-
-    reset_article_fetch_limit = models.BooleanField(default=False)
-
     embed_colour = models.CharField(
         max_length=6,
         default="3498db",

apps/static/js/home/filters.js

@@ -39,49 +39,43 @@ function initFiltersTable() {
                 }
             },
             {
-                title: "Match",
+                title: "Matching Algorithm",
-                // data: "keywords",
+                data: "matching_algorithm",
-                className: "text-start text-truncate",
+                render: function(data) {
-                render: function(data, type, row) {
+                    switch (data) {
-                    return row.regex ? row.regex : row.keywords;
+                        case 1: return "Any Word";
+                        case 2: return "All Words";
+                        case 3: return "Exact Match";
+                        case 4: return "Regular Expression";
+                        case 5: return "Fuzzy Match";
+                        default:
+                            console.error(`unknown matching algorithm '${data}'`);
+                            return data;
+                    }
                 }
             },
             {
-                title: "Type",
+                title: "Match",
-                // data: "regex",
+                data: "match"
-                render: function(data, type, row) {
+            },
-                    return row.regex ? "Regex" : "Keywords";
+            {
+                title: "Case Sensitivity",
+                data: "is_insensitive",
+                render: function(data) {
+                    return data ? "Insensitive" : "Sensitive"
                 }
             },
             {
                 title: "Control List",
-                data: "whitelist",
+                data: "is_whitelist",
                 render: function(data) {
-                    return data ? "whitelist" : "blacklist";
+                    return data ? "Whitelist" : "Blacklist";
                 }
             }
         ]
     });
 }
 
-$("#filterAdvancedMode").on("change", function() {
-    const advancedMode = $(this).prop("checked");
-    setFilterAdvancedMode(advancedMode);
-});
-
-function setFilterAdvancedMode(advancedMode) {
-    if (advancedMode) {
-        $("#filterForm .simple-filtering").hide();
-        $("#filterForm .advanced-filtering").show();
-        $("#filterForm .advanced-filtering").val("");
-    }
-    else {
-        $("#filterForm .advanced-filtering").hide();
-        $("#filterForm .simple-filtering").show();
-        $("#filterForm .simple-filtering").val("");
-    }
-}
-
 $("#addFilterBtn").on("click", async function() {
     await showEditFilterModal(-1);
 })
@@ -90,11 +84,10 @@ async function showEditFilterModal(filterId) {
 
     if (filterId === -1) {
         $("#filterFormModal input, #filterFormModal textarea").val("");
+        $("#filterFormModal input:checkbox").prop("checked", false);
+
         $("#filterFormModal .form-create").show();
         $("#filterFormModal .form-edit").hide();
-        $("#filterAdvancedMode").prop("checked", false);
-        $("#filterAdvancedMode").trigger("change");
-        $("#filterWhitelist").prop("checked", false);
     }
     else {
         const filter = filtersTable.row(function(idx, data, node) {
@@ -102,14 +95,10 @@ async function showEditFilterModal(filterId) {
         }).data();
 
         $("#filterName").val(filter.name);
-        $("#filterKeywords").val(filter.keywords);
+        $("#filterAlgorithm").val(filter.matching_algorithm);
-        $("#filterRegex").val(filter.regex);
+        $("#filterMatch").val(filter.match);
-
+        $("#filterWhitelist").prop("checked", filter.is_whitelist);
-        $("#filterAdvancedMode").prop("checked", filter.regex !== "");
+        $("#filterInsensitive").prop("checked", filter.is_insensitive);
-        $("#filterAdvancedMode").trigger("change");
-
-        $("#filterWhitelist").prop("checked", filter.whitelist);
-        $("#filterWhitelist").trigger("change");
 
         $("#filterFormModal .form-create").hide();
         $("#filterFormModal .form-edit").show();
@@ -122,22 +111,15 @@ async function showEditFilterModal(filterId) {
 $("#filterForm").on("submit", async function(event) {
     event.preventDefault();
 
-    var advancedFiltering = $("#filterAdvancedMode").prop("checked");
+    var id = $("#filterId").val();
-        id = $("#filterId").val();
         name = $("#filterName").val();
-        keywords = advancedFiltering ? "" : $("#filterKeywords").val();
+        algorithm = $("#filterAlgorithm option:selected").val();
-        regex = advancedFiltering ? $("#filterRegex").val() : "";
+        match = $("#filterMatch").val();
-        whitelistMode = $("#filterWhitelist").prop("checked");
+        isWhitelist = $("#filterWhitelist").prop("checked");
+        isInsensitive = $("#filterInsensitive").prop("checked");
         guildId = getCurrentlyActiveServer().guild_id;
 
-    if (advancedFiltering) {
+    var filterPrimaryKey = await saveFilter(id, name, algorithm, match, isWhitelist, isInsensitive, guildId);
-        keywords = "";
-    }
-    else {
-        regex = ""
-    }
-
-    var filterPrimaryKey = await saveFilter(id, name, keywords, regex, whitelistMode, guildId);
 
     if (filterPrimaryKey) {
         showToast("success", "Filter Saved", "Filter ID " + filterPrimaryKey);
@@ -148,12 +130,13 @@ $("#filterForm").on("submit", async function(event) {
     $("#filterFormModal").modal("hide");
 });
 
-async function saveFilter(id, name, keywords, regex, whitelistMode, guildId) {
+async function saveFilter(id, name, algorithm, match, isWhitelist, isInsensitive, guildId) {
     var formData = new FormData();
     formData.append("name", name);
-    formData.append("keywords", keywords);
+    formData.append("matching_algorithm", algorithm);
-    formData.append("regex", regex);
+    formData.append("match", match);
-    formData.append("whitelist", whitelistMode)
+    formData.append("is_whitelist", isWhitelist);
+    formData.append("is_insensitive", isInsensitive);
     formData.append("guild_id", guildId);
 
     var response;
@@ -206,6 +189,44 @@ async function loadFilters(guildId, page=1, pageSize=null) {
     }
 }
 
+$(document).ready(function() {
+    loadMatchingAlgorithms();
+});
+
+function loadMatchingAlgorithms() {
+    // Disable input while options are loading
+    $("#filterAlgorithm").prop("disabled", true);
+
+    // Delete existing options
+    $("#filterAlgorithm option").each(function() {
+        if ($(this).val())
+            $(this).remove();
+    });
+
+    // Clear select2 input
+    $("#filterAlgorithm").val("").change();
+
+    algorithms = [
+        {id: "", name: "None"},
+        {id: 1, name: "Any Words"},
+        {id: 2, name: "All Words"},
+        {id: 3, name: "Exact Match"},
+        {id: 4, name: "Regular Expression"},
+        {id: 5, name: "Fuzzy Match"},
+    ]
+
+    algorithms.forEach(algorithm => {
+        $("#filterAlgorithm").append($("<option>", {
+            text: algorithm.name,
+            value: algorithm.id
+        }));
+    });
+
+
+    // Re-enable the input
+    $("#filterAlgorithm").prop("disabled", false);
+};
+
 $(document).on("selectedServerChange", async function() {
     const activeServer = getCurrentlyActiveServer();
     await loadFilters(activeServer.guild_id);

apps/templates/home/includes/filtermodal.html

@@ -12,29 +12,39 @@
                 </div>
                 <div class="modal-body p-4">
                     <input type="hidden" id="filterId" name="filterId">
-                    <div class="mb-4">
+                    <div class="row">
-                        <label for="filterName" class="form-label">Name</label>
+                        <div class="col-lg-6 pe-lg-4">
-                        <input type="text" id="filterName" name="filterName" class="form-control rounded-1" placeholder="Remove Common Words">
+                            <div class="mb-4">
-                    </div>
+                                <label for="filterName" class="form-label">Name</label>
-                    <div class="mb-4">
+                                <input type="text" id="filterName" name="filterName" class="form-control rounded-1" tabindex="0">
-                        <div class="form-switch form-check-inline pe-4">
+                            </div>
-                            <input type="checkbox" id="filterAdvancedMode" name="filterAdvancedMode" class="form-check-input" />
-                            <label for="filterAdvancedMode" class="form-check-label d-inline">Advanced Filtering</label>
                         </div>
-                        <div class="form-switch form-check-inline">
+                        <div class="col-lg-6 ps-lg-4">
-                            <input type="checkbox" id="filterWhitelist" name="filterWhitelist" class="form-check-input" />
+                            <div class="mb-4">
-                            <label for="filterWhitelist" class="form-check-label d-inline">Whitelist Mode</label>
+                                <label for="filterAlgorithm" class="form-label">Matching Algorithm</label>
+                                <select name="filterAlgorithm" id="filterAlgorithm" class="select-2" data-dropdownparent="#filterFormModal" tabindex="1"></select>
+                            </div>
+                        </div>
+                        <div class="col-12">
+                            <div class="mb-4">
+                                <label for="filterMatch" class="form-label">Matching Pattern</label>
+                                <input type="text" id="filterMatch" name="filterMatch" class="form-control rounded-1" placeholder="" tabindex="2">
+                            </div>
+                        </div>
+                        <div class="col-lg-6 pe-lg-4">
+                            <div class="form-switch ps-0">
+                                <label for="filterWhitelist" class="form-check-label mb-2">Is Whitelist?</label>
+                                <br>
+                                <input type="checkbox" id="filterWhitelist" name="filterWhitelist" class="form-check-input ms-0 mt-0" tabindex="3">
+                            </div>
+                        </div>
+                        <div class="col-lg-6 ps-lg-4">
+                            <div class="form-switch ps-0">
+                                <label for="filterInsensitive" class="form-check-label mb-2">Case Insensitive?</label>
+                                <br>
+                                <input type="checkbox" id="filterInsensitive" name="filterInsensitive" class="form-check-input ms-0 mt-0" tabindex="4">
+                            </div>
                         </div>
-                    </div>
-                    <div class="simple-filtering">
-                        <label for="filterKeywords" class="form-label">Keywords</label>
-                        <textarea id="filterKeywords" name="filterKeywords" class="form-control rounded-1" placeholder="one,common,word,or,another"></textarea>
-                        <div class="form-text">Commma separated words to block.</div>
-                    </div>
-                    <div class="advanced-filtering">
-                        <label for="filterRegex" class="form-label">Regex</label>
-                        <input type="text" id="filterRegex" name="filterRegex" class="form-control rounded-1" placeholder="(?:^|(?<= ))(one|common|word|or|another)(?:(?= )|$)">
-                        <div class="form-text">Block content matching the provided regex.</div>
                     </div>
                 </div>
                 <div class="modal-footer">