diff --git a/.env.example b/.env.example index d0e0eed3..cfff13bb 100644 --- a/.env.example +++ b/.env.example @@ -78,7 +78,8 @@ NASA_FIRMS_API_KEY= # ------ Railway Relay (scripts/ais-relay.cjs) ------ -# The relay server handles AIS vessel tracking and OpenSky aircraft data. +# The relay server handles AIS vessel tracking + OpenSky aircraft data + RSS proxy. +# It can also run the Telegram OSINT poller (stateful MTProto) when configured. # Deploy on Railway with: node scripts/ais-relay.cjs # AISStream API key for live vessel positions @@ -91,6 +92,17 @@ OPENSKY_CLIENT_ID= OPENSKY_CLIENT_SECRET= +# ------ Telegram OSINT (Railway relay) ------ +# Telegram MTProto keys (free): https://my.telegram.org/apps +TELEGRAM_API_ID= +TELEGRAM_API_HASH= + +# GramJS StringSession generated locally (see: scripts/telegram/session-auth.mjs) +TELEGRAM_SESSION= + +# Which curated list bucket to ingest: full | tech | finance +TELEGRAM_CHANNEL_SET=full + # ------ Railway Relay Connection (Vercel → Railway) ------ # Server-side URL (https://) — used by Vercel edge functions to reach the relay diff --git a/api/telegram-feed.js b/api/telegram-feed.js new file mode 100644 index 00000000..b9ece75e --- /dev/null +++ b/api/telegram-feed.js @@ -0,0 +1,80 @@ +// Telegram feed proxy (web) +// Fetches Telegram Early Signals from the Railway relay (stateful MTProto lives there). + +import { getCorsHeaders, isDisallowedOrigin } from './_cors.js'; + +export const config = { runtime: 'edge' }; + +async function fetchWithTimeout(url, options, timeoutMs = 25000) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { ...options, signal: controller.signal }); + } finally { + clearTimeout(timeout); + } +} + +export default async function handler(req) { + const cors = getCorsHeaders(req, 'GET, OPTIONS'); + + if (req.method === 'OPTIONS') { + return new Response(null, { status: 204, headers: cors }); + } + + if (isDisallowedOrigin(req)) { + return new Response(JSON.stringify({ error: 'Origin not allowed' }), { status: 403, headers: cors }); + } + + let relay = process.env.WS_RELAY_URL; + if (!relay) { + return new Response(JSON.stringify({ error: 'WS_RELAY_URL not configured' }), { + status: 503, + headers: { 'Content-Type': 'application/json', ...cors }, + }); + } + + // Guard: WS_RELAY_URL should be HTTP(S) for server-side fetches. + // If someone accidentally sets a ws:// or wss:// URL, normalize it. + if (relay.startsWith('wss://')) relay = relay.replace('wss://', 'https://'); + if (relay.startsWith('ws://')) relay = relay.replace('ws://', 'http://'); + + const url = new URL(req.url); + const limit = Math.max(1, Math.min(200, parseInt(url.searchParams.get('limit') || '50', 10) || 50)); + const topic = (url.searchParams.get('topic') || '').trim(); + const channel = (url.searchParams.get('channel') || '').trim(); + + const relayUrl = new URL('/telegram/feed', relay); + relayUrl.searchParams.set('limit', String(limit)); + if (topic) relayUrl.searchParams.set('topic', topic); + if (channel) relayUrl.searchParams.set('channel', channel); + + try { + const res = await fetchWithTimeout(relayUrl.toString(), { + headers: { 'Accept': 'application/json' }, + }, 25000); + + const text = await res.text(); + return new Response(text, { + status: res.status, + headers: { + 'Content-Type': res.headers.get('content-type') || 'application/json', + // Short cache. Telegram is near-real-time. + 'Cache-Control': 'public, max-age=10', + ...cors, + }, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + const isAbort = err && (err.name === 'AbortError' || /aborted/i.test(msg)); + return new Response(JSON.stringify({ + error: isAbort ? 'Telegram relay request aborted (timeout)' : 'Telegram relay fetch failed', + detail: msg, + relayUrl: relayUrl.toString(), + hint: 'Check that WS_RELAY_URL is https://..., Railway relay is up, and /health responds. This endpoint times out after 25s.' + }), { + status: isAbort ? 504 : 502, + headers: { 'Content-Type': 'application/json', ...cors }, + }); + } +} diff --git a/data/telegram-channels.json b/data/telegram-channels.json new file mode 100644 index 00000000..76c4dc09 --- /dev/null +++ b/data/telegram-channels.json @@ -0,0 +1,254 @@ +{ + "version": 1, + "updatedAt": "2026-02-23T18:37:10Z", + "note": "Product-managed curated list. Not user-configurable.", + "channels": { + "full": [ + { + "handle": "VahidOnline", + "label": "Vahid Online", + "topic": "politics", + "tier": 1, + "enabled": true, + "region": "iran", + "maxMessages": 20 + }, + { + "handle": "abualiexpress", + "label": "Abu Ali Express", + "topic": "middleeast", + "tier": 2, + "enabled": true, + "region": "middleeast", + "maxMessages": 25 + }, + { + "handle": "air_alert_ua", + "label": "Повітряна Тривога", + "topic": "alerts", + "tier": 2, + "enabled": true, + "region": "ukraine", + "maxMessages": 20 + }, + { + "handle": "AuroraIntel", + "label": "Aurora Intel", + "topic": "conflict", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 20 + }, + { + "handle": "BNONews", + "label": "BNO News", + "topic": "breaking", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 25 + }, + { + "handle": "ClashReport", + "label": "Clash Report", + "topic": "conflict", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 30 + }, + { + "handle": "DeepStateUA", + "label": "DeepState", + "topic": "conflict", + "tier": 2, + "enabled": true, + "region": "ukraine", + "maxMessages": 20 + }, + { + "handle": "DefenderDome", + "label": "The Defender Dome", + "topic": "conflict", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 25 + }, + { + "handle": "englishabuali", + "label": "Abu Ali Express EN", + "topic": "middleeast", + "tier": 2, + "enabled": true, + "region": "middleeast", + "maxMessages": 25 + }, + { + "handle": "IranIntl", + "label": "Iran International", + "topic": "politics", + "tier": 2, + "enabled": true, + "region": "iran", + "maxMessages": 20 + }, + { + "handle": "kpszsu", + "label": "Air Force of the Armed Forces of Ukraine", + "topic": "alerts", + "tier": 2, + "enabled": true, + "region": "ukraine", + "maxMessages": 20 + }, + { + "handle": "LiveUAMap", + "label": "LiveUAMap", + "topic": "breaking", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 25 + }, + { + "handle": "OSINTdefender", + "label": "OSINTdefender", + "topic": "conflict", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 25 + }, + { + "handle": "OsintUpdates", + "label": "Osint Updates", + "topic": "breaking", + "tier": 2, + "enabled": true, + "region": "global", + "maxMessages": 20 + }, + { + "handle": "bellingcat", + "label": "Bellingcat", + "topic": "osint", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 10 + }, + { + "handle": "CyberDetective", + "label": "CyberDetective", + "topic": "cyber", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 15 + }, + { + "handle": "GeopoliticalCenter", + "label": "GeopoliticalCenter", + "topic": "geopolitics", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 20 + }, + { + "handle": "Middle_East_Spectator", + "label": "Middle East Spectator", + "topic": "middleeast", + "tier": 3, + "enabled": true, + "region": "middleeast", + "maxMessages": 20 + }, + { + "handle": "MiddleEastNow_Breaking", + "label": "Middle East Now Breaking", + "topic": "middleeast", + "tier": 3, + "enabled": true, + "region": "middleeast", + "maxMessages": 15 + }, + { + "handle": "nexta_live", + "label": "NEXTA Live", + "topic": "breaking", + "tier": 3, + "enabled": true, + "region": "europe", + "maxMessages": 15 + }, + { + "handle": "nexta_tv", + "label": "NEXTA", + "topic": "politics", + "tier": 3, + "enabled": true, + "region": "europe", + "maxMessages": 15 + }, + { + "handle": "OSINTIndustries", + "label": "OSINT Industries", + "topic": "osint", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 15 + }, + { + "handle": "Osintlatestnews", + "label": "OSIntOps News", + "topic": "osint", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 20 + }, + { + "handle": "osintlive", + "label": "OSINT Live", + "topic": "osint", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 15 + }, + { + "handle": "OsintTv", + "label": "OsintTV", + "topic": "geopolitics", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 15 + }, + { + "handle": "spectatorindex", + "label": "The Spectator Index", + "topic": "breaking", + "tier": 3, + "enabled": true, + "region": "global", + "maxMessages": 15 + }, + { + "handle": "war_monitor", + "label": "monitor", + "topic": "alerts", + "tier": 3, + "enabled": true, + "region": "ukraine", + "maxMessages": 20 + } + ], + "tech": [], + "finance": [] + } +} diff --git a/scripts/ais-relay.cjs b/scripts/ais-relay.cjs index daba7a8a..e6028ba8 100644 --- a/scripts/ais-relay.cjs +++ b/scripts/ais-relay.cjs @@ -11,6 +11,8 @@ const http = require('http'); const zlib = require('zlib'); +const path = require('path'); +const { readFileSync, writeFileSync, existsSync } = require('fs'); const { WebSocketServer, WebSocket } = require('ws'); const AISSTREAM_URL = 'wss://stream.aisstream.io/v0/stream'; @@ -48,6 +50,168 @@ function sendCompressed(req, res, statusCode, headers, body) { } } +// ───────────────────────────────────────────────────────────── +// Telegram OSINT ingestion (public channels) → Early Signals +// Web-first: runs on this Railway relay process, serves /telegram/feed +// Requires env: +// - TELEGRAM_API_ID +// - TELEGRAM_API_HASH +// - TELEGRAM_SESSION (StringSession) +// ───────────────────────────────────────────────────────────── +const TELEGRAM_ENABLED = Boolean(process.env.TELEGRAM_API_ID && process.env.TELEGRAM_API_HASH && process.env.TELEGRAM_SESSION); +const TELEGRAM_POLL_INTERVAL_MS = Math.max(15_000, Number(process.env.TELEGRAM_POLL_INTERVAL_MS || 60_000)); +const TELEGRAM_MAX_FEED_ITEMS = Math.max(50, Number(process.env.TELEGRAM_MAX_FEED_ITEMS || 200)); +const TELEGRAM_MAX_TEXT_CHARS = Math.max(200, Number(process.env.TELEGRAM_MAX_TEXT_CHARS || 800)); + +const telegramState = { + client: null, + channels: [], + cursorByHandle: Object.create(null), + items: [], + lastPollAt: 0, + lastError: null, + startedAt: Date.now(), +}; + +function loadTelegramChannels() { + // Product-managed curated list lives in repo root under data/ (shared by web + desktop). + // Relay is executed from scripts/, so resolve ../data. + const p = path.join(__dirname, '..', 'data', 'telegram-channels.json'); + const set = String(process.env.TELEGRAM_CHANNEL_SET || 'full').toLowerCase(); + try { + const raw = JSON.parse(readFileSync(p, 'utf8')); + const bucket = raw?.channels?.[set]; + const channels = Array.isArray(bucket) ? bucket : []; + + telegramState.channels = channels + .filter(c => c && typeof c.handle === 'string' && c.handle.length > 1) + .map(c => ({ + handle: String(c.handle).replace(/^@/, ''), + label: c.label ? String(c.label) : undefined, + topic: c.topic ? String(c.topic) : undefined, + region: c.region ? String(c.region) : undefined, + tier: c.tier != null ? Number(c.tier) : undefined, + enabled: c.enabled !== false, + maxMessages: c.maxMessages != null ? Number(c.maxMessages) : undefined, + })) + .filter(c => c.enabled); + + return telegramState.channels; + } catch (e) { + telegramState.channels = []; + telegramState.lastError = `failed to load telegram-channels.json: ${e?.message || String(e)}`; + return []; + } +} + +function normalizeTelegramMessage(msg, channel) { + const textRaw = String(msg?.message || ''); + const text = textRaw.slice(0, TELEGRAM_MAX_TEXT_CHARS); + const ts = msg?.date ? new Date(msg.date * 1000).toISOString() : new Date().toISOString(); + return { + id: `${channel.handle}:${msg.id}`, + source: 'telegram', + channel: channel.handle, + channelTitle: channel.label || channel.handle, + url: `https://t.me/${channel.handle}/${msg.id}`, + ts, + text, + topic: channel.topic || 'other', + tags: [channel.region].filter(Boolean), + earlySignal: true, + }; +} + +async function initTelegramClientIfNeeded() { + if (!TELEGRAM_ENABLED) return false; + if (telegramState.client) return true; + + const apiId = parseInt(String(process.env.TELEGRAM_API_ID || ''), 10); + const apiHash = String(process.env.TELEGRAM_API_HASH || ''); + const sessionStr = String(process.env.TELEGRAM_SESSION || ''); + + if (!apiId || !apiHash || !sessionStr) return false; + + try { + const { TelegramClient } = await import('telegram'); + const { StringSession } = await import('telegram/sessions'); + + const client = new TelegramClient(new StringSession(sessionStr), apiId, apiHash, { + connectionRetries: 3, + }); + + await client.connect(); + telegramState.client = client; + telegramState.lastError = null; + console.log('[Relay] Telegram client connected'); + return true; + } catch (e) { + telegramState.lastError = `telegram init failed: ${e?.message || String(e)}`; + console.warn('[Relay] Telegram init failed:', telegramState.lastError); + return false; + } +} + +async function pollTelegramOnce() { + const ok = await initTelegramClientIfNeeded(); + if (!ok) return; + + const channels = telegramState.channels.length ? telegramState.channels : loadTelegramChannels(); + if (!channels.length) return; + + const client = telegramState.client; + const newItems = []; + + for (const channel of channels) { + const handle = channel.handle; + const minId = telegramState.cursorByHandle[handle] || 0; + + try { + const entity = await client.getEntity(handle); + const msgs = await client.getMessages(entity, { + limit: Math.max(1, Math.min(50, channel.maxMessages || 25)), + minId, + }); + + for (const msg of msgs) { + if (!msg || !msg.id || !msg.message) continue; + const item = normalizeTelegramMessage(msg, channel); + newItems.push(item); + if (!telegramState.cursorByHandle[handle] || msg.id > telegramState.cursorByHandle[handle]) { + telegramState.cursorByHandle[handle] = msg.id; + } + } + + // Gentle rate limiting between channels + await new Promise(r => setTimeout(r, Math.max(300, Number(process.env.TELEGRAM_RATE_LIMIT_MS || 800)))); + } catch (e) { + const em = e?.message || String(e); + telegramState.lastError = `poll ${handle} failed: ${em}`; + console.warn('[Relay] Telegram poll error:', telegramState.lastError); + } + } + + if (newItems.length) { + telegramState.items = [...newItems, ...telegramState.items] + .filter((item, idx, arr) => arr.findIndex(x => x.id === item.id) === idx) + .sort((a, b) => (b.ts || '').localeCompare(a.ts || '')) + .slice(0, TELEGRAM_MAX_FEED_ITEMS); + } + + telegramState.lastPollAt = Date.now(); +} + +function startTelegramPollLoop() { + if (!TELEGRAM_ENABLED) return; + loadTelegramChannels(); + // Don’t block server startup. + pollTelegramOnce().catch(() => {}); + setInterval(() => { + pollTelegramOnce().catch(() => {}); + }, TELEGRAM_POLL_INTERVAL_MS).unref?.(); + console.log('[Relay] Telegram poll loop started'); +} + // AIS aggregate state for snapshot API (server-side fanout) const GRID_SIZE = 2; const DENSITY_WINDOW = 30 * 60 * 1000; // 30 minutes @@ -1102,6 +1266,13 @@ const server = http.createServer(async (req, res) => { connected: upstreamSocket?.readyState === WebSocket.OPEN, vessels: vessels.size, densityZones: Array.from(densityGrid.values()).filter(c => c.vessels.size >= 2).length, + telegram: { + enabled: TELEGRAM_ENABLED, + channels: telegramState.channels?.length || 0, + items: telegramState.items?.length || 0, + lastPollAt: telegramState.lastPollAt ? new Date(telegramState.lastPollAt).toISOString() : null, + lastError: telegramState.lastError, + }, cache: { opensky: openskyResponseCache.size, rss: rssResponseCache.size, @@ -1187,6 +1358,37 @@ const server = http.createServer(async (req, res) => { res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.end(JSON.stringify(diag, null, 2)); + } else if (req.url.startsWith('/telegram')) { + // Telegram Early Signals feed (public channels) + try { + const url = new URL(req.url, `http://localhost:${PORT}`); + const limit = Math.max(1, Math.min(200, Number(url.searchParams.get('limit') || 50))); + const topic = (url.searchParams.get('topic') || '').trim().toLowerCase(); + const channel = (url.searchParams.get('channel') || '').trim().toLowerCase(); + + const items = Array.isArray(telegramState.items) ? telegramState.items : []; + const filtered = items.filter((it) => { + if (topic && String(it.topic || '').toLowerCase() !== topic) return false; + if (channel && String(it.channel || '').toLowerCase() !== channel) return false; + return true; + }).slice(0, limit); + + sendCompressed(req, res, 200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'public, max-age=10', + }, JSON.stringify({ + source: 'telegram', + earlySignal: true, + enabled: TELEGRAM_ENABLED, + count: filtered.length, + updatedAt: telegramState.lastPollAt ? new Date(telegramState.lastPollAt).toISOString() : null, + lastError: telegramState.lastError, + items: filtered, + })); + } catch (e) { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: e?.message || String(e) })); + } } else if (req.url.startsWith('/rss')) { // Proxy RSS feeds that block Vercel IPs try { @@ -1442,6 +1644,7 @@ const wss = new WebSocketServer({ server }); server.listen(PORT, () => { console.log(`[Relay] WebSocket relay on port ${PORT}`); + startTelegramPollLoop(); }); wss.on('connection', (ws, req) => { diff --git a/scripts/package.json b/scripts/package.json index a4e5e8ef..f877b57d 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -1,13 +1,15 @@ { - "name": "ais-relay", - "version": "1.0.0", - "description": "WebSocket relay for aisstream.io vessel tracking", + "name": "worldmonitor-railway-relay", + "version": "1.1.0", + "description": "Railway relay: AIS/OpenSky + RSS proxy + Telegram OSINT poller", "main": "ais-relay.cjs", "scripts": { - "start": "node ais-relay.cjs" + "start": "node ais-relay.cjs", + "telegram:session": "node telegram/session-auth.mjs" }, "dependencies": { - "ws": "^8.18.0" + "ws": "^8.18.0", + "telegram": "^2.22.2" }, "engines": { "node": ">=18" diff --git a/scripts/telegram/session-auth.mjs b/scripts/telegram/session-auth.mjs new file mode 100644 index 00000000..2c2462c8 --- /dev/null +++ b/scripts/telegram/session-auth.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +/** + * Generate a TELEGRAM_SESSION (GramJS StringSession) for the Railway Telegram OSINT poller. + * + * Usage (local only): + * cd scripts + * npm install + * TELEGRAM_API_ID=... TELEGRAM_API_HASH=... node telegram/session-auth.mjs + * + * Output: + * Prints TELEGRAM_SESSION=... to stdout. + */ + +import { TelegramClient } from 'telegram'; +import { StringSession } from 'telegram/sessions'; +import readline from 'node:readline/promises'; +import { stdin as input, stdout as output } from 'node:process'; + +const apiId = parseInt(String(process.env.TELEGRAM_API_ID || ''), 10); +const apiHash = String(process.env.TELEGRAM_API_HASH || ''); + +if (!apiId || !apiHash) { + console.error('Missing TELEGRAM_API_ID or TELEGRAM_API_HASH. Get them from https://my.telegram.org/apps'); + process.exit(1); +} + +const rl = readline.createInterface({ input, output }); + +try { + const phoneNumber = (await rl.question('Phone number (with country code, e.g. +971...): ')).trim(); + const password = (await rl.question('2FA password (press enter if none): ')).trim(); + + const client = new TelegramClient(new StringSession(''), apiId, apiHash, { connectionRetries: 3 }); + + await client.start({ + phoneNumber: async () => phoneNumber, + password: async () => password || undefined, + phoneCode: async () => (await rl.question('Verification code from Telegram: ')).trim(), + onError: (err) => console.error(err), + }); + + const session = client.session.save(); + console.log('\n✅ Generated session. Add this as a Railway secret:'); + console.log(`TELEGRAM_SESSION=${session}`); + + await client.disconnect(); +} finally { + rl.close(); +}