Update (not finished yet)
This commit is contained in:
Sefinek
parent
6f47f85547
commit
f37cacbfd1
4 changed files with 108 additions and 26 deletions
11
config.js
11
config.js
|
|
@ -17,4 +17,13 @@ const SUCCESS_COOLDOWN_MS = 2 * 1000; // 2s
|
|||
// This ensures that WAF violations originating from your IP address are not reported to AbuseIPDB.
|
||||
const IP_REFRESH_INTERVAL = 9 * 60 * 1000; // 9m
|
||||
|
||||
module.exports = { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN_MS, MAX_URL_LENGTH, SUCCESS_COOLDOWN_MS, IP_REFRESH_INTERVAL };
|
||||
// Zgłaszaj także adresy IP do api.sefinek.net aby pomóc w rozbudowie repozytorium https://github.com/sefinek24/malicious-ip-addresses
|
||||
const REPORT_TO_SEFINEK_API = true;
|
||||
|
||||
// Token do api.sefinek.net. Poproś mnie o ten klucz jeśli chcesz przyczynić się do rozbudowy listy sefinek24/malicious-ip-addresses.
|
||||
const SEFINEK_API_SECRET = 'keyboardcat';
|
||||
|
||||
// Co ile mają być analizowane logi (reported_ips.csv) i wysyłane do Sefinek API?
|
||||
const SEFINEK_API_INTERVAL = process.env.NODE_ENV === 'production' ? 60 * 60 * 1000 : 2 * 1000;
|
||||
|
||||
module.exports = { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN_MS, MAX_URL_LENGTH, SUCCESS_COOLDOWN_MS, IP_REFRESH_INTERVAL, REPORT_TO_SEFINEK_API, SEFINEK_API_SECRET, SEFINEK_API_INTERVAL };
|
||||
46
index.js
46
index.js
|
|
@ -1,9 +1,10 @@
|
|||
require('dotenv').config();
|
||||
|
||||
const { axios, moduleVersion } = require('./services/axios.js');
|
||||
const { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN_MS, MAX_URL_LENGTH, SUCCESS_COOLDOWN_MS } = require('./config.js');
|
||||
const { CYCLE_INTERVAL, REPORTED_IP_COOLDOWN_MS, MAX_URL_LENGTH, SUCCESS_COOLDOWN_MS, IP_REFRESH_INTERVAL, REPORT_TO_SEFINEK_API, SEFINEK_API_SECRET, SEFINEK_API_INTERVAL } = require('./config.js');
|
||||
const PAYLOAD = require('./scripts/payload.js');
|
||||
const generateComment = require('./scripts/generateComment.js');
|
||||
const SefinekAPI = require('./scripts/sefinekAPI.js');
|
||||
const isImageRequest = require('./scripts/isImageRequest.js');
|
||||
const headers = require('./scripts/headers.js');
|
||||
const { logToCSV, readReportedIPs, wasImageRequestLogged } = require('./scripts/csv.js');
|
||||
|
|
@ -40,22 +41,24 @@ const isIPReportedRecently = (ip, reportedIPs) => {
|
|||
return { recentlyReported: false };
|
||||
};
|
||||
|
||||
const reportIP = async (event, url, country, cycleErrorCounts) => {
|
||||
if (!url) {
|
||||
logToCSV(event.rayName, event.clientIP, url, 'Failed - Missing URL', country);
|
||||
log('warn', `Missing URL ${event.clientIP}; URI: ${url};`);
|
||||
const reportIP = async (event, hostname, endpoint, userAgent, country, cycleErrorCounts) => {
|
||||
const uri = `${hostname}${endpoint}`;
|
||||
|
||||
if (!uri) {
|
||||
logToCSV(event.rayName, event.clientIP, hostname, endpoint, event.userAgent, 'Failed - Missing URL', country);
|
||||
log('warn', `Missing URL ${event.clientIP}; URI: ${uri};`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (event.clientIP === clientIp.address) {
|
||||
logToCSV(event.rayName, event.clientIP, url, 'Your IP address', country);
|
||||
log('log', `Your IP address (${event.clientIP}) was unexpectedly received from Cloudflare. URI: ${url}; Ignoring...`);
|
||||
logToCSV(event.rayName, event.clientIP, hostname, endpoint, event.userAgent, 'Your IP address', country);
|
||||
log('log', `Your IP address (${event.clientIP}) was unexpectedly received from Cloudflare. URI: ${uri}; Ignoring...`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (url.length > MAX_URL_LENGTH) {
|
||||
logToCSV(event.rayName, event.clientIP, url, 'Failed - URL too long', country);
|
||||
log('log', `URL too long ${event.clientIP}; URI: ${url};`);
|
||||
if (uri.length > MAX_URL_LENGTH) {
|
||||
logToCSV(event.rayName, event.clientIP, hostname, endpoint, event.userAgent, 'Failed - URL too long', country);
|
||||
log('log', `URL too long ${event.clientIP}; URI: ${uri};`);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -66,17 +69,17 @@ const reportIP = async (event, url, country, cycleErrorCounts) => {
|
|||
comment: generateComment(event)
|
||||
}, { headers: headers.ABUSEIPDB });
|
||||
|
||||
logToCSV(event.rayName, event.clientIP, url, 'Reported', country);
|
||||
log('info', `Reported ${event.clientIP}; URI: ${url}`);
|
||||
logToCSV(event.rayName, event.clientIP, hostname, endpoint, event.userAgent, 'Reported', country);
|
||||
log('info', `Reported ${event.clientIP}; URI: ${uri}`);
|
||||
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (err.response?.status === 429) {
|
||||
logToCSV(event.rayName, event.clientIP, url, 'Failed - 429 Too Many Requests', country);
|
||||
log('info', `Rate limited (429) while reporting ${event.clientIP}; URI: ${url};`);
|
||||
logToCSV(event.rayName, event.clientIP, hostname, endpoint, event.userAgent, 'Failed - 429 Too Many Requests', country);
|
||||
log('info', `Rate limited (429) while reporting ${event.clientIP}; URI: ${uri};`);
|
||||
cycleErrorCounts.blocked++;
|
||||
} else {
|
||||
log('error', `Error ${err.response?.status} while reporting ${event.clientIP}; URI: ${url}; (${err.response?.data})`);
|
||||
log('error', `Error ${err.response?.status} while reporting ${event.clientIP}; URI: ${uri}; (${err.response?.data})`);
|
||||
cycleErrorCounts.otherErrors++;
|
||||
}
|
||||
|
||||
|
|
@ -96,6 +99,12 @@ const reportIP = async (event, url, country, cycleErrorCounts) => {
|
|||
log('info', 'Loading data, please wait...');
|
||||
await clientIp.fetchIPAddress();
|
||||
|
||||
// Sefinek API
|
||||
setInterval(async () => {
|
||||
await SefinekAPI();
|
||||
}, SEFINEK_API_INTERVAL);
|
||||
|
||||
// AbuseIPDB
|
||||
let cycleId = 1;
|
||||
while (true) {
|
||||
log('info', `===================== New Reporting Cycle (v${moduleVersion}) =====================`);
|
||||
|
|
@ -117,7 +126,8 @@ const reportIP = async (event, url, country, cycleErrorCounts) => {
|
|||
for (const event of blockedIPEvents) {
|
||||
cycleProcessedCount++;
|
||||
const ip = event.clientIP;
|
||||
const url = `${event.clientRequestHTTPHost}${event.clientRequestPath}`;
|
||||
const hostname = event.clientRequestHTTPHost;
|
||||
const endpoint = event.clientRequestPath;
|
||||
const country = event.clientCountryName;
|
||||
|
||||
const { recentlyReported, timeDifference } = isIPReportedRecently(ip, reportedIPs);
|
||||
|
|
@ -133,7 +143,7 @@ const reportIP = async (event, url, country, cycleErrorCounts) => {
|
|||
if (isImageRequest(event.clientRequestPath)) {
|
||||
cycleImageSkippedCount++;
|
||||
if (!wasImageRequestLogged(ip, reportedIPs)) {
|
||||
logToCSV(event.rayName, ip, url, 'Skipped - Image Request', country);
|
||||
logToCSV(event.rayName, ip, hostname, endpoint, null, 'Skipped - Image Request', country);
|
||||
|
||||
if (imageRequestLogged) continue;
|
||||
log('info', 'Skipping image requests in this cycle...');
|
||||
|
|
@ -143,7 +153,7 @@ const reportIP = async (event, url, country, cycleErrorCounts) => {
|
|||
continue;
|
||||
}
|
||||
|
||||
const wasReported = await reportIP(event, url, country, cycleErrorCounts);
|
||||
const wasReported = await reportIP(event, hostname, endpoint, event.userAgent, country, cycleErrorCounts);
|
||||
if (wasReported) {
|
||||
cycleReportedCount++;
|
||||
await new Promise(resolve => setTimeout(resolve, SUCCESS_COOLDOWN_MS));
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ const log = require('./log.js');
|
|||
|
||||
const CSV_FILE_PATH = path.join(__dirname, '..', 'reported_ips.csv');
|
||||
const MAX_CSV_SIZE_BYTES = 4 * 1024 * 1024; // 4 MB
|
||||
const CSV_HEADER = 'Timestamp,RayID,IP,Endpoint,Action,Country\n';
|
||||
const CSV_HEADER = 'Timestamp,RayID,IP,Hostname,Endpoint,User-Agent,Action,Country,SefinekAPI\n';
|
||||
|
||||
if (!fs.existsSync(CSV_FILE_PATH)) fs.writeFileSync(CSV_FILE_PATH, CSV_HEADER);
|
||||
|
||||
|
|
@ -16,10 +16,17 @@ const checkCSVSize = () => {
|
|||
}
|
||||
};
|
||||
|
||||
const logToCSV = (rayId, ip, endpoint, action, country) => {
|
||||
const escapeCSVValue = value => {
|
||||
if (typeof value === 'string' && value.includes(',')) {
|
||||
return `"${value.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const logToCSV = (rayId, ip, hostname, endpoint, useragent, action, country, sefinekAPI) => {
|
||||
checkCSVSize();
|
||||
const logLine = `${new Date().toISOString()},${rayId},${ip},${endpoint},${action},${country}\n`;
|
||||
fs.appendFileSync(CSV_FILE_PATH, logLine);
|
||||
const logLine = `${new Date().toISOString()},${rayId},${ip},${hostname},${escapeCSVValue(endpoint)},${escapeCSVValue(useragent || '')},${action},${country},${sefinekAPI || false}`;
|
||||
fs.appendFileSync(CSV_FILE_PATH, logLine + '\n');
|
||||
};
|
||||
|
||||
const readReportedIPs = () => {
|
||||
|
|
@ -31,11 +38,33 @@ const readReportedIPs = () => {
|
|||
.slice(1)
|
||||
.filter(line => line.trim() !== '')
|
||||
.map(line => {
|
||||
const [timestamp, rayid, ip, endpoint, action, country] = line.split(',');
|
||||
return { timestamp: new Date(timestamp), rayid, ip, endpoint, action, country };
|
||||
const [timestamp, rayId, ip, hostname, endpoint, useragent, action, country, sefinekAPI] = line.split(',');
|
||||
return { timestamp: new Date(timestamp), rayId, ip, hostname, endpoint, useragent, action, country, sefinekAPI };
|
||||
});
|
||||
};
|
||||
|
||||
const updateSefinekAPIInCSV = (rayId, reportedToSefinekAPI) => {
|
||||
if (!fs.existsSync(CSV_FILE_PATH)) {
|
||||
log('error', 'CSV file does not exist');
|
||||
return;
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(CSV_FILE_PATH, 'utf8');
|
||||
const lines = content.split('\n');
|
||||
|
||||
const updatedLines = lines.map(line => {
|
||||
if (line.includes(rayId)) {
|
||||
const [timestamp, rayIdExisting, ip, hostname, endpoint, useragent, action, country] = line.split(',');
|
||||
if (rayIdExisting === rayId) {
|
||||
return `${timestamp},${rayId},${ip},${hostname},${escapeCSVValue(endpoint)},${escapeCSVValue(useragent)},${action},${country},${reportedToSefinekAPI}`;
|
||||
}
|
||||
}
|
||||
return line;
|
||||
});
|
||||
|
||||
fs.writeFileSync(CSV_FILE_PATH, updatedLines.join('\n'));
|
||||
};
|
||||
|
||||
const wasImageRequestLogged = (ip, reportedIPs) => reportedIPs.some(entry => entry.ip === ip && entry.action === 'Skipped - Image Request');
|
||||
|
||||
module.exports = { logToCSV, readReportedIPs, wasImageRequestLogged };
|
||||
module.exports = { logToCSV, readReportedIPs, updateSefinekAPIInCSV, wasImageRequestLogged };
|
||||
34
scripts/sefinekAPI.js
Normal file
34
scripts/sefinekAPI.js
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
const { axios } = require('../services/axios.js');
|
||||
const { readReportedIPs, updateSefinekAPIInCSV } = require('./csv.js');
|
||||
const log = require('./log.js');
|
||||
|
||||
const SEFINEK_API_URL = `${process.env.NODE_ENV === 'production' ? 'https://api.sefinek.net' : 'http://127.0.0.1:4010'}/api/v2/cloudflare-waf-abuseipdb/post`;
|
||||
|
||||
module.exports = async () => {
|
||||
const reportedIPs = readReportedIPs();
|
||||
if (reportedIPs.length === 0) {
|
||||
log('info', 'No reported IPs to send to Sefinek API.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await axios.post(SEFINEK_API_URL, {
|
||||
reportedIPs: reportedIPs.map(ip => ({
|
||||
rayId: ip.rayId,
|
||||
ip: ip.ip,
|
||||
endpoint: ip.endpoint,
|
||||
action: ip.action,
|
||||
country: ip.country
|
||||
}))
|
||||
});
|
||||
|
||||
log('info', `Logs (${res.data.count}) sent to Sefinek API. Status: ${res.status}`);
|
||||
|
||||
reportedIPs.forEach(ip => {
|
||||
updateSefinekAPIInCSV(ip.rayId, true);
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
log('error', `Failed to send logs to Sefinek API. Error: ${err.message}`);
|
||||
}
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue