improved filters
This commit is contained in:
parent
dbf943591a
commit
88f3fff950
@ -125,7 +125,7 @@ class FilterSerializer(DynamicModelSerializer):
|
||||
|
||||
class Meta:
|
||||
model = Filter
|
||||
fields = ("id", "name", "keywords", "regex", "whitelist", "guild_id")
|
||||
fields = ("id", "name", "matching_algorithm", "match", "is_insensitive", "is_whitelist", "guild_id")
|
||||
|
||||
|
||||
class ArticleMutatorSerializer(DynamicModelSerializer):
|
||||
|
||||
@ -110,7 +110,7 @@ class Filter_ListView(generics.ListCreateAPIView):
|
||||
serializer_class = FilterSerializer
|
||||
|
||||
filter_backends = [filters.SearchFilter, rest_filters.DjangoFilterBackend, filters.OrderingFilter]
|
||||
filterset_fields = ["id", "name", "keywords", "regex", "whitelist", "guild_id"]
|
||||
filterset_fields = ["id", "name", "matching_algorithm", "match", "is_insensitive", "is_whitelist", "guild_id"]
|
||||
search_fields = ["name", "keywords", "regex"]
|
||||
|
||||
def get_queryset(self):
|
||||
|
||||
@ -22,7 +22,7 @@ class SubChannelAdmin(admin.ModelAdmin):
|
||||
@admin.register(Filter)
|
||||
class FilterAdmin(admin.ModelAdmin):
|
||||
list_display = [
|
||||
"id", "name", "keywords", "regex", "whitelist", "guild_id"
|
||||
"id", "name", "guild_id"
|
||||
]
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,67 @@
|
||||
# Generated by Django 5.0.4 on 2024-07-10 13:45
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('home', '0014_rename_published_theshold_subscription_published_threshold'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='filter',
|
||||
options={'ordering': ('name',)},
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='filter',
|
||||
name='keywords',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='filter',
|
||||
name='regex',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='filter',
|
||||
name='whitelist',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='subscription',
|
||||
name='article_fetch_limit',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='subscription',
|
||||
name='reset_article_fetch_limit',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='filter',
|
||||
name='is_insensitive',
|
||||
field=models.BooleanField(default=True, verbose_name='is insensitive'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='filter',
|
||||
name='is_whitelist',
|
||||
field=models.BooleanField(default=False, verbose_name='is whitelist'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='filter',
|
||||
name='match',
|
||||
field=models.CharField(blank=True, max_length=256, verbose_name='match'),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='filter',
|
||||
name='matching_algorithm',
|
||||
field=models.PositiveIntegerField(choices=[(0, 'None'), (1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='filter',
|
||||
name='guild_id',
|
||||
field=models.CharField(max_length=128, verbose_name='guild id'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='filter',
|
||||
name='name',
|
||||
field=models.CharField(max_length=128, verbose_name='name'),
|
||||
),
|
||||
]
|
||||
@ -119,44 +119,49 @@ class SubChannel(models.Model):
|
||||
return self.channel_id
|
||||
|
||||
|
||||
# using a brilliant matching model design from paperless-ngx src
|
||||
class Filter(models.Model):
|
||||
MATCH_NONE = 0
|
||||
MATCH_ANY = 1
|
||||
MATCH_ALL = 2
|
||||
MATCH_LITERAL = 3
|
||||
MATCH_REGEX = 4
|
||||
MATCH_FUZZY = 5
|
||||
MATCH_AUTO = 6
|
||||
|
||||
MATCHING_ALGORITHMS = (
|
||||
# (MATCH_NONE, _("None")),
|
||||
(MATCH_ANY, _("Any word")),
|
||||
(MATCH_ALL, _("All words")),
|
||||
(MATCH_LITERAL, _("Exact match")),
|
||||
(MATCH_REGEX, _("Regular expression")),
|
||||
(MATCH_FUZZY, _("Fuzzy word")),
|
||||
# (MATCH_AUTO, _("Automatic")),
|
||||
)
|
||||
|
||||
id = models.AutoField(primary_key=True)
|
||||
|
||||
name = models.CharField(
|
||||
max_length=32,
|
||||
null=False,
|
||||
blank=False
|
||||
name = models.CharField(_("name"), max_length=128)
|
||||
|
||||
match = models.CharField(_("match"), max_length=256, blank=True)
|
||||
|
||||
matching_algorithm = models.PositiveIntegerField(
|
||||
_("matching algorithm"),
|
||||
choices=MATCHING_ALGORITHMS,
|
||||
default=MATCH_ANY,
|
||||
)
|
||||
|
||||
keywords = models.CharField(
|
||||
max_length=128,
|
||||
null=True,
|
||||
blank=True
|
||||
)
|
||||
is_insensitive = models.BooleanField(_("is insensitive"), default=True)
|
||||
|
||||
regex = models.CharField(
|
||||
max_length=128,
|
||||
null=True,
|
||||
blank=True
|
||||
)
|
||||
|
||||
whitelist = models.BooleanField(default=False)
|
||||
is_whitelist = models.BooleanField(_("is whitelist"), default=False)
|
||||
|
||||
# Have to use charfield instead of positiveBigIntegerField due to an Sqlite
|
||||
# issue that rounds down the value
|
||||
# https://github.com/sequelize/sequelize/issues/9335
|
||||
guild_id = models.CharField(
|
||||
max_length=128
|
||||
)
|
||||
guild_id = models.CharField(_("guild id"), max_length=128)
|
||||
|
||||
class Meta:
|
||||
"""
|
||||
Metadata for the Filter Model.
|
||||
"""
|
||||
|
||||
verbose_name = "filter"
|
||||
verbose_name_plural = "filters"
|
||||
get_latest_by = "id"
|
||||
ordering = ("name",)
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["name", "guild_id"],
|
||||
@ -164,8 +169,8 @@ class Filter(models.Model):
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
def __str__(self):
|
||||
return f"{self.guild_id} - {self.name}"
|
||||
|
||||
|
||||
class Subscription(models.Model):
|
||||
@ -217,13 +222,6 @@ class Subscription(models.Model):
|
||||
blank=True,
|
||||
)
|
||||
|
||||
article_fetch_limit = models.PositiveSmallIntegerField(
|
||||
validators = [MaxValueValidator(1), MinValueValidator(10)],
|
||||
default=10
|
||||
)
|
||||
|
||||
reset_article_fetch_limit = models.BooleanField(default=False)
|
||||
|
||||
embed_colour = models.CharField(
|
||||
max_length=6,
|
||||
default="3498db",
|
||||
|
||||
@ -39,49 +39,43 @@ function initFiltersTable() {
|
||||
}
|
||||
},
|
||||
{
|
||||
title: "Match",
|
||||
// data: "keywords",
|
||||
className: "text-start text-truncate",
|
||||
render: function(data, type, row) {
|
||||
return row.regex ? row.regex : row.keywords;
|
||||
title: "Matching Algorithm",
|
||||
data: "matching_algorithm",
|
||||
render: function(data) {
|
||||
switch (data) {
|
||||
case 1: return "Any Word";
|
||||
case 2: return "All Words";
|
||||
case 3: return "Exact Match";
|
||||
case 4: return "Regular Expression";
|
||||
case 5: return "Fuzzy Match";
|
||||
default:
|
||||
console.error(`unknown matching algorithm '${data}'`);
|
||||
return data;
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
title: "Type",
|
||||
// data: "regex",
|
||||
render: function(data, type, row) {
|
||||
return row.regex ? "Regex" : "Keywords";
|
||||
title: "Match",
|
||||
data: "match"
|
||||
},
|
||||
{
|
||||
title: "Case Sensitivity",
|
||||
data: "is_insensitive",
|
||||
render: function(data) {
|
||||
return data ? "Insensitive" : "Sensitive"
|
||||
}
|
||||
},
|
||||
{
|
||||
title: "Control List",
|
||||
data: "whitelist",
|
||||
data: "is_whitelist",
|
||||
render: function(data) {
|
||||
return data ? "whitelist" : "blacklist";
|
||||
return data ? "Whitelist" : "Blacklist";
|
||||
}
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
$("#filterAdvancedMode").on("change", function() {
|
||||
const advancedMode = $(this).prop("checked");
|
||||
setFilterAdvancedMode(advancedMode);
|
||||
});
|
||||
|
||||
function setFilterAdvancedMode(advancedMode) {
|
||||
if (advancedMode) {
|
||||
$("#filterForm .simple-filtering").hide();
|
||||
$("#filterForm .advanced-filtering").show();
|
||||
$("#filterForm .advanced-filtering").val("");
|
||||
}
|
||||
else {
|
||||
$("#filterForm .advanced-filtering").hide();
|
||||
$("#filterForm .simple-filtering").show();
|
||||
$("#filterForm .simple-filtering").val("");
|
||||
}
|
||||
}
|
||||
|
||||
$("#addFilterBtn").on("click", async function() {
|
||||
await showEditFilterModal(-1);
|
||||
})
|
||||
@ -90,11 +84,10 @@ async function showEditFilterModal(filterId) {
|
||||
|
||||
if (filterId === -1) {
|
||||
$("#filterFormModal input, #filterFormModal textarea").val("");
|
||||
$("#filterFormModal input:checkbox").prop("checked", false);
|
||||
|
||||
$("#filterFormModal .form-create").show();
|
||||
$("#filterFormModal .form-edit").hide();
|
||||
$("#filterAdvancedMode").prop("checked", false);
|
||||
$("#filterAdvancedMode").trigger("change");
|
||||
$("#filterWhitelist").prop("checked", false);
|
||||
}
|
||||
else {
|
||||
const filter = filtersTable.row(function(idx, data, node) {
|
||||
@ -102,14 +95,10 @@ async function showEditFilterModal(filterId) {
|
||||
}).data();
|
||||
|
||||
$("#filterName").val(filter.name);
|
||||
$("#filterKeywords").val(filter.keywords);
|
||||
$("#filterRegex").val(filter.regex);
|
||||
|
||||
$("#filterAdvancedMode").prop("checked", filter.regex !== "");
|
||||
$("#filterAdvancedMode").trigger("change");
|
||||
|
||||
$("#filterWhitelist").prop("checked", filter.whitelist);
|
||||
$("#filterWhitelist").trigger("change");
|
||||
$("#filterAlgorithm").val(filter.matching_algorithm);
|
||||
$("#filterMatch").val(filter.match);
|
||||
$("#filterWhitelist").prop("checked", filter.is_whitelist);
|
||||
$("#filterInsensitive").prop("checked", filter.is_insensitive);
|
||||
|
||||
$("#filterFormModal .form-create").hide();
|
||||
$("#filterFormModal .form-edit").show();
|
||||
@ -122,22 +111,15 @@ async function showEditFilterModal(filterId) {
|
||||
$("#filterForm").on("submit", async function(event) {
|
||||
event.preventDefault();
|
||||
|
||||
var advancedFiltering = $("#filterAdvancedMode").prop("checked");
|
||||
id = $("#filterId").val();
|
||||
var id = $("#filterId").val();
|
||||
name = $("#filterName").val();
|
||||
keywords = advancedFiltering ? "" : $("#filterKeywords").val();
|
||||
regex = advancedFiltering ? $("#filterRegex").val() : "";
|
||||
whitelistMode = $("#filterWhitelist").prop("checked");
|
||||
algorithm = $("#filterAlgorithm option:selected").val();
|
||||
match = $("#filterMatch").val();
|
||||
isWhitelist = $("#filterWhitelist").prop("checked");
|
||||
isInsensitive = $("#filterInsensitive").prop("checked");
|
||||
guildId = getCurrentlyActiveServer().guild_id;
|
||||
|
||||
if (advancedFiltering) {
|
||||
keywords = "";
|
||||
}
|
||||
else {
|
||||
regex = ""
|
||||
}
|
||||
|
||||
var filterPrimaryKey = await saveFilter(id, name, keywords, regex, whitelistMode, guildId);
|
||||
var filterPrimaryKey = await saveFilter(id, name, algorithm, match, isWhitelist, isInsensitive, guildId);
|
||||
|
||||
if (filterPrimaryKey) {
|
||||
showToast("success", "Filter Saved", "Filter ID " + filterPrimaryKey);
|
||||
@ -148,12 +130,13 @@ $("#filterForm").on("submit", async function(event) {
|
||||
$("#filterFormModal").modal("hide");
|
||||
});
|
||||
|
||||
async function saveFilter(id, name, keywords, regex, whitelistMode, guildId) {
|
||||
async function saveFilter(id, name, algorithm, match, isWhitelist, isInsensitive, guildId) {
|
||||
var formData = new FormData();
|
||||
formData.append("name", name);
|
||||
formData.append("keywords", keywords);
|
||||
formData.append("regex", regex);
|
||||
formData.append("whitelist", whitelistMode)
|
||||
formData.append("matching_algorithm", algorithm);
|
||||
formData.append("match", match);
|
||||
formData.append("is_whitelist", isWhitelist);
|
||||
formData.append("is_insensitive", isInsensitive);
|
||||
formData.append("guild_id", guildId);
|
||||
|
||||
var response;
|
||||
@ -206,6 +189,44 @@ async function loadFilters(guildId, page=1, pageSize=null) {
|
||||
}
|
||||
}
|
||||
|
||||
$(document).ready(function() {
|
||||
loadMatchingAlgorithms();
|
||||
});
|
||||
|
||||
function loadMatchingAlgorithms() {
|
||||
// Disable input while options are loading
|
||||
$("#filterAlgorithm").prop("disabled", true);
|
||||
|
||||
// Delete existing options
|
||||
$("#filterAlgorithm option").each(function() {
|
||||
if ($(this).val())
|
||||
$(this).remove();
|
||||
});
|
||||
|
||||
// Clear select2 input
|
||||
$("#filterAlgorithm").val("").change();
|
||||
|
||||
algorithms = [
|
||||
{id: "", name: "None"},
|
||||
{id: 1, name: "Any Words"},
|
||||
{id: 2, name: "All Words"},
|
||||
{id: 3, name: "Exact Match"},
|
||||
{id: 4, name: "Regular Expression"},
|
||||
{id: 5, name: "Fuzzy Match"},
|
||||
]
|
||||
|
||||
algorithms.forEach(algorithm => {
|
||||
$("#filterAlgorithm").append($("<option>", {
|
||||
text: algorithm.name,
|
||||
value: algorithm.id
|
||||
}));
|
||||
});
|
||||
|
||||
|
||||
// Re-enable the input
|
||||
$("#filterAlgorithm").prop("disabled", false);
|
||||
};
|
||||
|
||||
$(document).on("selectedServerChange", async function() {
|
||||
const activeServer = getCurrentlyActiveServer();
|
||||
await loadFilters(activeServer.guild_id);
|
||||
|
||||
@ -12,29 +12,39 @@
|
||||
</div>
|
||||
<div class="modal-body p-4">
|
||||
<input type="hidden" id="filterId" name="filterId">
|
||||
<div class="mb-4">
|
||||
<label for="filterName" class="form-label">Name</label>
|
||||
<input type="text" id="filterName" name="filterName" class="form-control rounded-1" placeholder="Remove Common Words">
|
||||
</div>
|
||||
<div class="mb-4">
|
||||
<div class="form-switch form-check-inline pe-4">
|
||||
<input type="checkbox" id="filterAdvancedMode" name="filterAdvancedMode" class="form-check-input" />
|
||||
<label for="filterAdvancedMode" class="form-check-label d-inline">Advanced Filtering</label>
|
||||
<div class="row">
|
||||
<div class="col-lg-6 pe-lg-4">
|
||||
<div class="mb-4">
|
||||
<label for="filterName" class="form-label">Name</label>
|
||||
<input type="text" id="filterName" name="filterName" class="form-control rounded-1" tabindex="0">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-switch form-check-inline">
|
||||
<input type="checkbox" id="filterWhitelist" name="filterWhitelist" class="form-check-input" />
|
||||
<label for="filterWhitelist" class="form-check-label d-inline">Whitelist Mode</label>
|
||||
<div class="col-lg-6 ps-lg-4">
|
||||
<div class="mb-4">
|
||||
<label for="filterAlgorithm" class="form-label">Matching Algorithm</label>
|
||||
<select name="filterAlgorithm" id="filterAlgorithm" class="select-2" data-dropdownparent="#filterFormModal" tabindex="1"></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="mb-4">
|
||||
<label for="filterMatch" class="form-label">Matching Pattern</label>
|
||||
<input type="text" id="filterMatch" name="filterMatch" class="form-control rounded-1" placeholder="" tabindex="2">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6 pe-lg-4">
|
||||
<div class="form-switch ps-0">
|
||||
<label for="filterWhitelist" class="form-check-label mb-2">Is Whitelist?</label>
|
||||
<br>
|
||||
<input type="checkbox" id="filterWhitelist" name="filterWhitelist" class="form-check-input ms-0 mt-0" tabindex="3">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6 ps-lg-4">
|
||||
<div class="form-switch ps-0">
|
||||
<label for="filterInsensitive" class="form-check-label mb-2">Case Insensitive?</label>
|
||||
<br>
|
||||
<input type="checkbox" id="filterInsensitive" name="filterInsensitive" class="form-check-input ms-0 mt-0" tabindex="4">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="simple-filtering">
|
||||
<label for="filterKeywords" class="form-label">Keywords</label>
|
||||
<textarea id="filterKeywords" name="filterKeywords" class="form-control rounded-1" placeholder="one,common,word,or,another"></textarea>
|
||||
<div class="form-text">Commma separated words to block.</div>
|
||||
</div>
|
||||
<div class="advanced-filtering">
|
||||
<label for="filterRegex" class="form-label">Regex</label>
|
||||
<input type="text" id="filterRegex" name="filterRegex" class="form-control rounded-1" placeholder="(?:^|(?<= ))(one|common|word|or|another)(?:(?= )|$)">
|
||||
<div class="form-text">Block content matching the provided regex.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user