From d02259dca1854eaf3d1f9c93e00c65b18eb02858 Mon Sep 17 00:00:00 2001 From: Sefinek Date: Wed, 13 Nov 2024 09:08:14 +0100 Subject: [PATCH] 1.2.1 --- .env.default | 8 ++++---- README.md | 5 ++--- index.js | 32 ++++++++++++++++++++------------ package-lock.json | 4 ++-- package.json | 8 ++++---- scripts/config.js | 4 ++-- scripts/generateComment.js | 12 +++++++++--- scripts/whitelist.js | 4 ++-- services/payload.js | 2 +- services/sefinekAPI.js | 17 +++++++++++------ 10 files changed, 57 insertions(+), 39 deletions(-) diff --git a/.env.default b/.env.default index bc6f6ce..2883e87 100644 --- a/.env.default +++ b/.env.default @@ -15,7 +15,7 @@ CYCLE_INTERVAL=120 # The minimum time (in hours) that must pass after reporting an IP address before it can be reported again. # The required time is >= 15 minutes, according to AbuseIPDB API limits. -REPORTED_IP_COOLDOWN_MS=7 +REPORTED_IP_COOLDOWN=7 # The maximum URI length that can be reported to AbuseIPDB. # If Cloudflare returns a longer URI, the API request will fail. @@ -29,12 +29,12 @@ SUCCESS_COOLDOWN=80 IP_REFRESH_INTERVAL=80 ############################### SEFINEK API ############################### -# Secret key for api.sefinek.net -SEFINEK_API_SECRET= - # Report IP addresses to api.sefinek.net to support the development of the repository at https://github.com/sefinek/Malicious-IP-Addresses. SEFINEK_API_SECRET is required if true. REPORT_TO_SEFINEK_API=false +# Secret key for api.sefinek.net +SEFINEK_API_SECRET= + # How often should the log (reported_ips.csv) be analyzed and sent to the Sefinek API? In hours. SEFINEK_API_INTERVAL=1 diff --git a/README.md b/README.md index 89335c1..7e26766 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,8 @@ Also, take a look at [sefinek/UFW-AbuseIPDB-Reporter](https://github.com/sefinek > If you like this repository or find it useful, I would greatly appreciate it if you could give it a star ā­. Thanks a lot! ## šŸ› ļø Prerequisites -- [Node.js](https://nodejs.org) -- [npm](https://www.npmjs.com) (Node Package Manager) -- [PM2](https://www.npmjs.com/package/pm2) (optional) +- [Node.js + npm](https://nodejs.org) +- [PM2](https://www.npmjs.com/package/pm2) (recommended) ## šŸ“ƒ Information diff --git a/index.js b/index.js index 00b3123..d4a5fb6 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ require('dotenv').config(); const { axios, moduleVersion } = require('./services/axios.js'); -const { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN_MS, MAX_URL_LENGTH, SUCCESS_COOLDOWN, SEFINEK_API_INTERVAL, REPORT_TO_SEFINEK_API } = require('./scripts/config.js'); +const { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN, MAX_URL_LENGTH, SUCCESS_COOLDOWN, SEFINEK_API_INTERVAL, REPORT_TO_SEFINEK_API } = require('./scripts/config.js'); const PAYLOAD = require('./services/payload.js'); const generateComment = require('./scripts/generateComment.js'); const SefinekAPI = require('./services/sefinekAPI.js'); @@ -20,9 +20,15 @@ const fetchBlockedIPs = async () => { if (events) { const filtered = events.filter(x => x.ip !== clientIp.getAddress() && - !whitelist.subdomains.some(subdomain => x.clientRequestHTTPHost?.includes(subdomain)) && // Subdomains - !whitelist.userAgents.some(ua => x.userAgent?.includes(ua)) && // User-agents - !whitelist.endpoints.some(endpoint => x.clientRequestPath?.includes(endpoint)) // Endpoints + ( + x.source === 'securitylevel' || + x.source === 'badscore' || + ( + !whitelist.domains.some(subdomain => x.clientRequestHTTPHost?.includes(subdomain)) && + !whitelist.userAgents.some(ua => x.userAgent?.includes(ua)) && + !whitelist.endpoints.some(endpoint => x.clientRequestPath?.includes(endpoint)) + ) + ) ); log('log', `Fetched ${events.length} (filtered ${filtered.length}) events from Cloudflare`); @@ -46,7 +52,7 @@ const isIPReportedRecently = (rayId, ip, reportedIPs) => { return latest; }, null); - if (lastReport && (Date.now() - lastReport.timestamp) < REPORTED_IP_COOLDOWN_MS) { + if (lastReport && (Date.now() - lastReport.timestamp) < REPORTED_IP_COOLDOWN) { return { recentlyReported: true, timeDifference: Date.now() - lastReport.timestamp, reason: lastReport.status === 'TOO_MANY_REQUESTS' ? 'RATE-LIMITED' : 'REPORTED' }; } @@ -118,7 +124,7 @@ const reportIP = async (event, uri, country, hostname, endpoint, cycleErrorCount // AbuseIPDB let cycleId = 1; while (true) { - log('log', `================ New Reporting Cycle v${moduleVersion}; ID: ${cycleId} ================`); + log('log', `===================== Reporting Cycle No. ${cycleId} =====================`); const blockedIPEvents = await fetchBlockedIPs(); if (!blockedIPEvents) { @@ -145,13 +151,15 @@ const reportIP = async (event, uri, country, hostname, endpoint, cycleErrorCount if (whitelist.endpoints.includes(event.clientRequestPath)) return log('log', `Skipping ${event.clientRequestPath}...`); const reportedIPs = readReportedIPs(); - const { recentlyReported, timeDifference, reason } = isIPReportedRecently(event.rayName, ip, reportedIPs); + const { recentlyReported } = isIPReportedRecently(event.rayName, ip, reportedIPs); if (recentlyReported) { - const hoursAgo = Math.floor(timeDifference / (1000 * 60 * 60)); - const minutesAgo = Math.floor((timeDifference % (1000 * 60 * 60)) / (1000 * 60)); - const secondsAgo = Math.floor((timeDifference % (1000 * 60)) / 1000); + // if (process.env.NODE_ENV === 'development') { + // const hoursAgo = Math.floor(timeDifference / (1000 * 60 * 60)); + // const minutesAgo = Math.floor((timeDifference % (1000 * 60 * 60)) / (1000 * 60)); + // const secondsAgo = Math.floor((timeDifference % (1000 * 60)) / 1000); + // log('log', `${ip} was ${reason} ${hoursAgo}h ${minutesAgo}m ${secondsAgo}s ago. Skipping...`); + // } - if (process.env.NODE_ENV === 'development') log('log', `${ip} was ${reason} ${hoursAgo}h ${minutesAgo}m ${secondsAgo}s ago. Skipping...`); cycleSkippedCount++; continue; } @@ -177,7 +185,7 @@ const reportIP = async (event, uri, country, hostname, endpoint, cycleErrorCount log('log', `- Reported IPs: ${cycleReportedCount}`); log('log', `- Total IPs processed: ${cycleProcessedCount}`); log('log', `- Skipped IPs: ${cycleSkippedCount}`); - log('log', `- Skipped due to image requests: ${cycleImageSkippedCount}`); + log('log', `- Ignored image requests: ${cycleImageSkippedCount}`); log('log', `- Rate-limits: ${cycleErrorCounts.blocked}`); log('log', `- Other errors: ${cycleErrorCounts.otherErrors}`); log('log', '===================== End of Reporting Cycle ====================='); diff --git a/package-lock.json b/package-lock.json index 2aeab8c..ad983ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "waf-to-abuseipdb", - "version": "1.2.0", + "version": "1.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "waf-to-abuseipdb", - "version": "1.2.0", + "version": "1.2.1", "license": "MIT", "dependencies": { "axios": "^1.7.7", diff --git a/package.json b/package.json index f11b31d..eb857d2 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,15 @@ { "name": "waf-to-abuseipdb", - "version": "1.2.0", + "version": "1.2.1", "description": "A Node.js project that automates the reporting of incidents detected by Cloudflare WAF to the AbuseIPDB database.", "keywords": [ "abuseipdb", "cloudflare", - "cloudflare-waf", + "cloudflare waf", "waf", "reporting", - "abuseipdb-reporting", - "node-abuseipdb" + "abuseipdb reporting", + "node abuseipdb" ], "homepage": "https://github.com/sefinek/Cloudflare-WAF-To-AbuseIPDB", "bugs": { diff --git a/scripts/config.js b/scripts/config.js index ef9bc46..6d706b0 100644 --- a/scripts/config.js +++ b/scripts/config.js @@ -1,7 +1,7 @@ const CYCLE_INTERVAL = process.env.NODE_ENV === 'production' ? parseInt(process.env.CYCLE_INTERVAL || '120') * 60 * 1000 : 8 * 1000; -const REPORTED_IP_COOLDOWN_MS = parseInt(process.env.REPORTED_IP_COOLDOWN_MS || '6') * 60 * 60 * 1000; +const REPORTED_IP_COOLDOWN = parseInt(process.env.REPORTED_IP_COOLDOWN || '6') * 60 * 60 * 1000; const MAX_URL_LENGTH = parseInt(process.env.MAX_URL_LENGTH || '920'); @@ -16,7 +16,7 @@ const SEFINEK_API_INTERVAL = process.env.NODE_ENV === 'production' ? module.exports = { CYCLE_INTERVAL, - REPORTED_IP_COOLDOWN_MS, + REPORTED_IP_COOLDOWN, MAX_URL_LENGTH, SUCCESS_COOLDOWN, IP_REFRESH_INTERVAL, diff --git a/scripts/generateComment.js b/scripts/generateComment.js index 044fe32..62d684b 100644 --- a/scripts/generateComment.js +++ b/scripts/generateComment.js @@ -3,12 +3,12 @@ module.exports = ({ action, clientAsn, clientASNDescription, clientRequestHTTPPr { label: 'Action taken', value: action?.toUpperCase() }, { label: 'ASN', value: `${clientAsn} (${clientASNDescription})` }, { label: 'Protocol', value: `${clientRequestHTTPProtocol} (${clientRequestHTTPMethodName} method)` }, - { label: 'Domain', value: clientRequestHTTPHost }, + { label: 'Zone', value: clientRequestHTTPHost }, { label: 'Endpoint', value: clientRequestPath }, { label: 'Query', value: clientRequestQuery }, { label: 'Timestamp', value: datetime }, { label: 'Ray ID', value: rayName }, - { label: 'Rule ID', value: ruleId }, + // { label: 'Rule ID', value: ruleId }, { label: 'UA', value: userAgent }, ]; @@ -21,4 +21,10 @@ ${reportLines.join('\n')} Report generated by Cloudflare-WAF-To-AbuseIPDB: https://github.com/sefinek/Cloudflare-WAF-To-AbuseIPDB`; -}; \ No newline at end of file +}; + +/* + * Hello! šŸ‘‹ I'm really glad you're here. + * Please do not remove the repository URL in the comment above. + * Iā€™d really appreciate it (: + */ \ No newline at end of file diff --git a/scripts/whitelist.js b/scripts/whitelist.js index 028a6f5..54a7238 100644 --- a/scripts/whitelist.js +++ b/scripts/whitelist.js @@ -1,5 +1,5 @@ -const subdomains = ['api.', 'cdn.', 'bucket.']; +const domains = ['api.', 'cdn.', 'blocklist.sefinek.net', 'bucket.sefinek.net']; const userAgents = ['Chrome/129', 'Chrome/130', 'Chrome/131', 'Chrome/132', 'StellaLauncher', 'PrepareStella']; const endpoints = ['/api/', '//video', '//js', '//images', '//imgs', 'favicon.ico', 'sitemap.xml', 'robots.txt']; -module.exports = { subdomains, userAgents, endpoints }; \ No newline at end of file +module.exports = { domains, userAgents, endpoints }; \ No newline at end of file diff --git a/services/payload.js b/services/payload.js index 8d6fc03..7c6ee1d 100644 --- a/services/payload.js +++ b/services/payload.js @@ -3,7 +3,7 @@ const query = `query ListFirewallEvents($zoneTag: string, $filter: FirewallEvent zones(filter: { zoneTag: $zoneTag }) { firewallEventsAdaptive( filter: $filter, - limit: 1500, + limit: 1000, orderBy: [datetime_DESC] ) { action diff --git a/services/sefinekAPI.js b/services/sefinekAPI.js index b3692e4..7c9dbf9 100644 --- a/services/sefinekAPI.js +++ b/services/sefinekAPI.js @@ -10,12 +10,17 @@ module.exports = async () => { const reportedIPs = (readReportedIPs() || []).filter(x => x.status === 'REPORTED' && x.ip !== clientIp.getAddress() && - x.hostname !== 'blocklist.sefinek.net' && // Domain - !whitelist.subdomains.some(subdomain => x.clientRequestHTTPHost?.includes(subdomain)) && // Subdomains - !whitelist.userAgents.some(ua => x.userAgent?.includes(ua)) && // User-agents - !whitelist.endpoints.some(endpoint => x.clientRequestPath?.includes(endpoint)) && // Endpoints - !(/crawler|spider|bot/gi).test(x.userAgent) && - !x.sefinekAPI + !x.sefinekAPI && + ( + x.source === 'securitylevel' || + x.source === 'badscore' || + ( + !(/crawler|spider|bot/gi).test(x.userAgent) && + !whitelist.domains.some(subdomain => x.clientRequestHTTPHost?.includes(subdomain)) && + !whitelist.userAgents.some(ua => x.userAgent?.includes(ua)) && + !whitelist.endpoints.some(endpoint => x.clientRequestPath?.includes(endpoint)) + ) + ) ); if (!reportedIPs.length) return;