Add optional FILTER_FEED environment variable
parent
e0fc853b23
commit
6aa9cfb20b
13
README.md
13
README.md
|
@ -103,6 +103,19 @@ If you're debugging the feed compatibility code, then setting the `DEBUG_FEEDITE
|
||||||
|
|
||||||
If you're working on the CSS, note that Eleventy will *not* auto-rebuild & reload the browser. For that, you need to re-save a file it *does* watch, such as `index.html`, `.eleventy.js`, etc (even if you haven't made any changes).
|
If you're working on the CSS, note that Eleventy will *not* auto-rebuild & reload the browser. For that, you need to re-save a file it *does* watch, such as `index.html`, `.eleventy.js`, etc (even if you haven't made any changes).
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
The following environment variables can be used to influence the behaviour of the aggregator.
|
||||||
|
|
||||||
|
Environment Variable | Purpose
|
||||||
|
------------------------|------------------------------
|
||||||
|
`FILTER_FEED` | Set to any value to enable the filtering of posts based on whether they contain any bad words. Uses [futility](https://www.npmjs.com/package/futility), but with a custom algorithm on top to avoid the "[Scunthorpe Problem](https://en.wikipedia.org/wiki/Scunthorpe_problem)".
|
||||||
|
|
||||||
|
Example of setting an environment variable when running a build:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
FILTER_FEED=true npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
### Design Decisions
|
### Design Decisions
|
||||||
A number of decisions were made in the design process of this website. These are documented with the reasoning behind them here.
|
A number of decisions were made in the design process of this website. These are documented with the reasoning behind them here.
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
"dateformat": "^4.5.1",
|
"dateformat": "^4.5.1",
|
||||||
"feedme": "^2.0.2",
|
"feedme": "^2.0.2",
|
||||||
"file-type": "^16.5.3",
|
"file-type": "^16.5.3",
|
||||||
|
"futility": "^1.1.2",
|
||||||
"html-entities": "^2.3.2",
|
"html-entities": "^2.3.2",
|
||||||
"p-reflect": "^2.1.0",
|
"p-reflect": "^2.1.0",
|
||||||
"phin": "^3.6.0",
|
"phin": "^3.6.0",
|
||||||
|
@ -1217,6 +1218,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
|
||||||
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
|
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
|
||||||
},
|
},
|
||||||
|
"node_modules/futility": {
|
||||||
|
"version": "1.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/futility/-/futility-1.1.2.tgz",
|
||||||
|
"integrity": "sha1-w55ZYFAH8jnL7Vpu/RZRuFiHy/s="
|
||||||
|
},
|
||||||
"node_modules/get-caller-file": {
|
"node_modules/get-caller-file": {
|
||||||
"version": "2.0.5",
|
"version": "2.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||||
|
@ -4598,6 +4604,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
|
||||||
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
|
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
|
||||||
},
|
},
|
||||||
|
"futility": {
|
||||||
|
"version": "1.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/futility/-/futility-1.1.2.tgz",
|
||||||
|
"integrity": "sha1-w55ZYFAH8jnL7Vpu/RZRuFiHy/s="
|
||||||
|
},
|
||||||
"get-caller-file": {
|
"get-caller-file": {
|
||||||
"version": "2.0.5",
|
"version": "2.0.5",
|
||||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||||
|
|
|
@ -25,6 +25,7 @@
|
||||||
"dateformat": "^4.5.1",
|
"dateformat": "^4.5.1",
|
||||||
"feedme": "^2.0.2",
|
"feedme": "^2.0.2",
|
||||||
"file-type": "^16.5.3",
|
"file-type": "^16.5.3",
|
||||||
|
"futility": "^1.1.2",
|
||||||
"html-entities": "^2.3.2",
|
"html-entities": "^2.3.2",
|
||||||
"p-reflect": "^2.1.0",
|
"p-reflect": "^2.1.0",
|
||||||
"phin": "^3.6.0",
|
"phin": "^3.6.0",
|
||||||
|
|
|
@ -6,6 +6,7 @@ const dateformat = require("dateformat");
|
||||||
const striptags = require("striptags");
|
const striptags = require("striptags");
|
||||||
|
|
||||||
const fetch_feed = require("./lib/fetch_feed.js");
|
const fetch_feed = require("./lib/fetch_feed.js");
|
||||||
|
const check_text = require("./lib/check_text.js");
|
||||||
|
|
||||||
// The length of auto-generated descriptions if one isn't provided.
|
// The length of auto-generated descriptions if one isn't provided.
|
||||||
const DESCRIPTION_LENGTH = 200;
|
const DESCRIPTION_LENGTH = 200;
|
||||||
|
@ -72,6 +73,15 @@ module.exports = async function() {
|
||||||
})));
|
})));
|
||||||
global.feed_items.sort((a, b) => b.pubdate_obj - a.pubdate_obj);
|
global.feed_items.sort((a, b) => b.pubdate_obj - a.pubdate_obj);
|
||||||
// console.log(feed_items.map(el => el.title));
|
// console.log(feed_items.map(el => el.title));
|
||||||
|
|
||||||
|
if(process.env.FILTER_FEED) {
|
||||||
|
const feed_items_count = global.feed_items.length;
|
||||||
|
global.feed_items = global.feed_items.filter((item) => !check_text(item.title)
|
||||||
|
&& !check_text(item.description)
|
||||||
|
&& !check_text(item.content));
|
||||||
|
|
||||||
|
console.log(`>>> Feed filtering enabled, removed ${feed_items_count - global.feed_items.length} posts`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
const Futility = require("futility").default;
|
||||||
|
const striptags = require("striptags");
|
||||||
|
|
||||||
|
const checker = new Futility();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks the words in the given string for bad (swear) words.
|
||||||
|
* Checks on a word level - therefore avoiding the "scunthorpe problem".
|
||||||
|
* @param {string} str The string to check.
|
||||||
|
* @return {boolean} Truee if the given string contains any bad words, or false if it does not.
|
||||||
|
*/
|
||||||
|
function check_text(str) {
|
||||||
|
const words = striptags(str).split(/\s+/);
|
||||||
|
|
||||||
|
for(let word of words) {
|
||||||
|
if(word.trim().search(/^\*+$/) > -1) continue;
|
||||||
|
let replaced = checker.censor(word, "*");
|
||||||
|
if(replaced.trim().search(/^\*+$/) > -1)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = check_text;
|
Loading…
Reference in New Issue