JoshS/Cloudflare-WAF-To-AbuseIPDB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update

Browse source
This commit is contained in:
Sefinek 2024-08-17 18:12:35 +02:00
parent 8f9c3a6230
commit d5d3280a4a
3 changed files with 74 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
index.js
View file

@ -1,17 +1,17 @@
require('dotenv').config(); require('dotenv').config();
const axios = require('axios'); const axios = require('axios');
const PAYLOAD = require('./scripts/payload.js'); const PAYLOAD = require('./scripts/payload.js');
const generateComment = require('./scripts/generateComment.js'); const generateComment = require('./scripts/generateComment.js');
const isImageRequest = require('./scripts/isImageRequest.js'); const isImageRequest = require('./scripts/isImageRequest.js');
const headers = require('./scripts/headers.js'); const headers = require('./scripts/headers.js');
const { logToCSV, readReportedIPs } = require('./scripts/csv.js'); const { logToCSV, readReportedIPs, wasImageRequestLogged } = require('./scripts/csv.js');
const formatDelay = require('./scripts/formatDelay.js'); const formatDelay = require('./scripts/formatDelay.js');
const log = require('./scripts/log.js'); const log = require('./scripts/log.js');
const TIME_WINDOW_MS = 20 * 60 * 1000;
const COOLDOWN_MS = 2000; const COOLDOWN_MS = 2000;
const BLOCK_TIME_MS = 5 * 60 * 60 * 1000; // 5h const BLOCK_TIME_MS = 5 * 60 * 60 * 1000; // 5h
const exceptedRuleIds = new Set(['fa01280809254f82978e827892db4e46']); const REPORTED_IP_COOLDOWN_MS = 7 * 60 * 60 * 1000; // 7h
const fetchBlockedIPs = async () => { const fetchBlockedIPs = async () => {
try { try {
@ -21,32 +21,25 @@ const fetchBlockedIPs = async () => {
log('info', `Fetched ${events.length} events from Cloudflare`); log('info', `Fetched ${events.length} events from Cloudflare`);
return events; return events;
} else { } else {
throw new Error(`Failed to retrieve data from Cloudflare (status ${response.status}). Missing permissions? Check your token. The required permission is Zone.Analytics.Read.`); throw new Error(`Failed to retrieve data from Cloudflare. Status: ${response.status}`);
} }
} catch (err) { } catch (err) {
if (err.response) { log('error', err.response ? `${err.response.status} HTTP ERROR (Cloudflare)\n${JSON.stringify(err.response.data, null, 2)}` : `Unknown error with Cloudflare: ${err.message}`);
log('error', `${err.response.status} HTTP ERROR (Cloudflare)\n${JSON.stringify(err.response.data, null, 2)}`);
} else if (err.request) {
log('error', 'No response received from Cloudflare');
} else {
log('error', `Unknown error with Cloudflare. ${err.message}`);
}
return null; return null;
} }
}; };
const shouldReportDomain = (domain, reportedIPs) => { const isIPBlockedRecently = (ip, reportedIPs) => {
const lastReport = reportedIPs.find(entry => entry.domain === domain); const lastBlock = reportedIPs.find(entry => entry.ip === ip && entry.action.includes('429'));
return !lastReport || (Date.now() - lastReport.timestamp.getTime()) > TIME_WINDOW_MS; return lastBlock && (Date.now() - new Date(lastBlock.timestamp).getTime()) < BLOCK_TIME_MS;
}; };
const isIPBlockedRecently = (ip, blockedIPs) => { const isIPReportedRecently = (ip, reportedIPs) => {
const lastBlockTime = blockedIPs.get(ip); const lastReport = reportedIPs.find(entry => entry.ip === ip && entry.action === 'Reported');
return lastBlockTime && (Date.now() - lastBlockTime) < BLOCK_TIME_MS; return lastReport && (Date.now() - new Date(lastReport.timestamp).getTime()) < REPORTED_IP_COOLDOWN_MS;
}; };
const reportIP = async (event, url, country, blockedIPs, cycleErrorCounts) => { const reportIP = async (event, url, country, cycleErrorCounts) => {
try { try {
await axios.post('https://api.abuseipdb.com/api/v2/report', { await axios.post('https://api.abuseipdb.com/api/v2/report', {
ip: event.clientIP, ip: event.clientIP,
@ -60,22 +53,17 @@ const reportIP = async (event, url, country, blockedIPs, cycleErrorCounts) => {
} catch (err) { } catch (err) {
if (err.response) { if (err.response) {
if (err.response.status === 429) { if (err.response.status === 429) {
blockedIPs.set(event.clientIP, Date.now()); logToCSV(new Date(), event.rayName, event.clientIP, url, 'Blocked - 429 Too Many Requests', country);
logToCSV(new Date(), event.rayName, event.clientIP, url, 'Blocked - 429 Too Many Requests', event.clientCountryName);
log('warn', `Rate limited (429) while reporting: ${event.clientIP}; URL: ${url}; (Will retry after 5 hours)`); log('warn', `Rate limited (429) while reporting: ${event.clientIP}; URL: ${url}; (Will retry after 5 hours)`);
cycleErrorCounts.blocked++; cycleErrorCounts.blocked++;
} else { } else {
log('error', `Error ${err.response.status} while reporting: ${event.clientIP}; URL: ${url}; Message: ${err.response.data.message}`); log('error', `Error ${err.response.status} while reporting: ${event.clientIP}; URL: ${url}; Message: ${err.response.data.message}`);
cycleErrorCounts.otherErrors++; cycleErrorCounts.otherErrors++;
} }
} else if (err.request) { } else {
log('error', `No response from AbuseIPDB while reporting: ${event.clientIP}; URL: ${url}`); log('error', `No response from AbuseIPDB while reporting: ${event.clientIP}; URL: ${url}`);
cycleErrorCounts.noResponse++; cycleErrorCounts.noResponse++;
} else {
log('error', `Unknown error: ${err.message} while reporting: ${event.clientIP}; URL: ${url}`);
cycleErrorCounts.otherErrors++;
} }
return false; return false;
} }
}; };
@ -83,50 +71,48 @@ const reportIP = async (event, url, country, blockedIPs, cycleErrorCounts) => {
(async () => { (async () => {
log('info', 'Starting IP reporting process...'); log('info', 'Starting IP reporting process...');
const reportedIPs = readReportedIPs();
const skippedRayIds = new Set(reportedIPs.filter(ip => ip.action.startsWith('Skipped')).map(ip => ip.rayid));
const blockedIPs = new Map(reportedIPs.filter(ip => ip.action.includes('429')).map(ip => [ip.ip, ip.timestamp.getTime()]));
while (true) { while (true) {
log('info', '===================== New Reporting Cycle ====================='); log('info', '===================== New Reporting Cycle =====================');
const reportedIPs = readReportedIPs();
let cycleImageSkippedCount = 0, cycleProcessedCount = 0, cycleReportedCount = 0, cycleSkippedCount = 0; let cycleImageSkippedCount = 0, cycleProcessedCount = 0, cycleReportedCount = 0, cycleSkippedCount = 0;
const cycleErrorCounts = { blocked: 0, noResponse: 0, otherErrors: 0 }; const cycleErrorCounts = { blocked: 0, noResponse: 0, otherErrors: 0 };
let imageRequestLogged = false;
const blockedIPEvents = await fetchBlockedIPs(); const blockedIPEvents = await fetchBlockedIPs();
if (blockedIPEvents) { if (!blockedIPEvents) {
for (const event of blockedIPEvents) { log('warn', 'No events fetched, skipping cycle...');
cycleProcessedCount++;
if (skippedRayIds.has(event.rayName) || isIPBlockedRecently(event.clientIP, blockedIPs)) {
cycleSkippedCount++;
continue; continue;
} }
for (const event of blockedIPEvents) {
cycleProcessedCount++;
const ip = event.clientIP;
const url = `${event.clientRequestHTTPHost}${event.clientRequestPath}`; const url = `${event.clientRequestHTTPHost}${event.clientRequestPath}`;
const country = event.clientCountryName; const country = event.clientCountryName;
if (isImageRequest(event.clientRequestPath)) { if (isIPReportedRecently(ip, reportedIPs) || isIPBlockedRecently(ip, reportedIPs)) {
skippedRayIds.add(event.rayName); cycleSkippedCount++;
logToCSV(new Date(), event.rayName, event.clientIP, url, 'Skipped - Image Request', country);
log('info', `Skipping: ${event.clientIP}; URL: ${url}; (Image request detected)`);
cycleImageSkippedCount++;
continue; continue;
} }
if (!exceptedRuleIds.has(event.ruleId) && shouldReportDomain(event.clientRequestHTTPHost, reportedIPs)) { if (isImageRequest(event.clientRequestPath)) {
const wasReported = await reportIP(event, url, country, blockedIPs, cycleErrorCounts); cycleImageSkippedCount++;
if (!wasImageRequestLogged(ip, reportedIPs)) {
logToCSV(new Date(), event.rayName, ip, url, 'Skipped - Image Request', country);
if (!imageRequestLogged) {
log('info', 'Skipping image requests in this cycle.');
imageRequestLogged = true;
}
}
continue;
}
const wasReported = await reportIP(event, url, country, cycleErrorCounts);
if (wasReported) { if (wasReported) {
cycleReportedCount++; cycleReportedCount++;
await new Promise(resolve => setTimeout(resolve, COOLDOWN_MS)); await new Promise(resolve => setTimeout(resolve, COOLDOWN_MS));
} }
} else {
skippedRayIds.add(event.rayName);
logToCSV(new Date(), event.rayName, event.clientIP, url, 'Skipped - Already Reported', country);
log('info', `Skipping: ${event.clientIP} (domain ${event.clientRequestHTTPHost}); URL: ${url}; (Already reported recently)`);
cycleSkippedCount++;
}
}
} }
log('info', 'Cycle Summary:'); log('info', 'Cycle Summary:');

30
scripts/csv.js
View file

@ -1,11 +1,23 @@
const fs = require('node:fs'); const fs = require('node:fs');
const path = require('node:path'); const path = require('node:path');
const log = require('./log.js');
const CSV_FILE_PATH = path.join(__dirname, '..', 'reported_ips.csv'); const CSV_FILE_PATH = path.join(__dirname, '..', 'reported_ips.csv');
const MAX_CSV_SIZE_BYTES = 4 * 1024 * 1024; // 4 MB
const CSV_HEADER = 'Timestamp,RayID,IP,Endpoint,Action,Country\n';
if (!fs.existsSync(CSV_FILE_PATH)) fs.writeFileSync(CSV_FILE_PATH, 'Timestamp,RayID,IP,Endpoint,Action,Country\n'); if (!fs.existsSync(CSV_FILE_PATH)) fs.writeFileSync(CSV_FILE_PATH, CSV_HEADER);
const checkCSVSize = () => {
const stats = fs.statSync(CSV_FILE_PATH);
if (stats.size > MAX_CSV_SIZE_BYTES) {
fs.writeFileSync(CSV_FILE_PATH, CSV_HEADER);
log('info', `CSV file size exceeded ${MAX_CSV_SIZE_BYTES / (1024 * 1024)} MB. File has been reset.`);
}
};
const logToCSV = (timestamp, rayid, ip, endpoint, action, country) => { const logToCSV = (timestamp, rayid, ip, endpoint, action, country) => {
checkCSVSize();
const logLine = `${timestamp.toISOString()},${rayid},${ip},${endpoint},${action},${country}\n`; const logLine = `${timestamp.toISOString()},${rayid},${ip},${endpoint},${action},${country}\n`;
fs.appendFileSync(CSV_FILE_PATH, logLine); fs.appendFileSync(CSV_FILE_PATH, logLine);
}; };
@ -14,10 +26,16 @@ const readReportedIPs = () => {
if (!fs.existsSync(CSV_FILE_PATH)) return []; if (!fs.existsSync(CSV_FILE_PATH)) return [];
const content = fs.readFileSync(CSV_FILE_PATH, 'utf8'); const content = fs.readFileSync(CSV_FILE_PATH, 'utf8');
return content.split('\n').slice(1).map(line => { return content
const [timestamp, rayid, ip, domain, action, country] = line.split(','); .split('\n')
return { timestamp: new Date(timestamp), rayid, ip, domain, action, country }; .slice(1)
}).filter(entry => entry.timestamp && entry.rayid && entry.ip); .filter(line => line.trim() !== '')
.map(line => {
const [timestamp, rayid, ip, endpoint, action, country] = line.split(',');
return { timestamp: new Date(timestamp), rayid, ip, endpoint, action, country };
});
}; };
module.exports = { logToCSV, readReportedIPs }; const wasImageRequestLogged = (ip, reportedIPs) => reportedIPs.some(entry => entry.ip === ip && entry.action === 'Skipped - Image Request');
module.exports = { logToCSV, readReportedIPs, wasImageRequestLogged };

12
scripts/log.js
View file

@ -1,10 +1,10 @@
module.exports = (level, message) => { const levels = {
const logLevels = {
info: '[INFO]', info: '[INFO]',
warn: '[WARN]', warn: '[WARN]',
error: '[FAIL]' error: '[FAIL]'
}; };
const timestamp = process.env.NODE_ENV === 'development' ? `${new Date().toISOString()}: ` : ''; module.exports = (level, message) => {
console[level](`${logLevels[level]} ${timestamp}${message}`); const timestamp = process.env.NODE_ENV === 'development' ? `${new Date().toISOString()}: ` : '';
console[level](`${levels[level]} ${timestamp}${message}`);
}; };