const express = require('express');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
const dns = require('dns').promises;
const { AsyncLocalStorage } = require('async_hooks');
const cors = require('cors');
const { createWorker } = require('tesseract.js');
const { ProxyAgent } = require('undici');
const apiRequestContext = new AsyncLocalStorage();

// Load local environment variables without exposing server secrets to the browser.
const localEnvPath = path.join(__dirname, '.env');
if (fs.existsSync(localEnvPath)) {
  // A checked-in/local project .env is the authoritative database selection
  // for this local server. Inherited terminal variables may otherwise point a
  // restarted child process at a different Supabase project.
  const localDatabaseVariables = new Set(['SUPABASE_URL', 'SUPABASE_SERVICE_ROLE_KEY', 'SUPABASE_SECRET_KEY', 'SESSION_SECRET', 'REQUIRE_SUPABASE']);
  for (const line of fs.readFileSync(localEnvPath, 'utf8').split(/\r?\n/)) {
    const match = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/.exec(line);
    if (!match || (process.env[match[1]] !== undefined && !localDatabaseVariables.has(match[1]))) continue;
    process.env[match[1]] = match[2].replace(/^(['"])(.*)\1$/, '$2');
  }
}

const app = express();
const PORT = process.env.PORT || 3000;
const SUPABASE_URL = String(process.env.SUPABASE_URL || '').replace(/\/rest\/v1\/?$/i, '').replace(/\/$/, '');
const SUPABASE_SERVICE_ROLE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_SECRET_KEY || '';
const USE_SUPABASE = Boolean(SUPABASE_URL && SUPABASE_SERVICE_ROLE_KEY);
const REQUIRE_SUPABASE = process.env.NODE_ENV === 'production' || process.env.REQUIRE_SUPABASE === 'true';
const SESSION_SECRET = process.env.SESSION_SECRET || SUPABASE_SERVICE_ROLE_KEY || crypto.randomBytes(32).toString('hex');
const ENV_GOOGLE_OAUTH_CLIENT_ID = process.env.GOOGLE_OAUTH_CLIENT_ID || '';
const ENV_GOOGLE_OAUTH_CLIENT_SECRET = process.env.GOOGLE_OAUTH_CLIENT_SECRET || '';
const ENV_GOOGLE_OAUTH_REDIRECT_URI = process.env.GOOGLE_OAUTH_REDIRECT_URI || '';
const ENV_GOOGLE_TOKEN_ENCRYPTION_KEY = process.env.GOOGLE_TOKEN_ENCRYPTION_KEY || '';
const INTERNAL_API_KEY = crypto.createHmac('sha256', SESSION_SECRET).update('quocdev-internal-api').digest('hex');
const INTERNAL_BASE_URL = String(process.env.INTERNAL_BASE_URL || (process.env.VERCEL_URL ? `https://${process.env.VERCEL_URL}` : `http://127.0.0.1:${PORT}`)).replace(/\/$/, '');
let supabaseWriteQueue = Promise.resolve();
async function supabaseStateRequest(key, value, timeoutMs = 4500) {
  const headers = { apikey: SUPABASE_SERVICE_ROLE_KEY, Authorization: `Bearer ${SUPABASE_SERVICE_ROLE_KEY}`, 'Content-Type': 'application/json' };
  if (value !== undefined) headers.Prefer = 'resolution=merge-duplicates';
  let lastError;
  for (let attempt = 1; attempt <= 3; attempt++) {
    try {
      const response = await fetch(`${SUPABASE_URL}/rest/v1/app_state${value === undefined ? `?key=eq.${encodeURIComponent(key)}&select=value` : '?on_conflict=key'}`, {
        method: value === undefined ? 'GET' : 'POST',
        headers,
        body: value === undefined ? undefined : JSON.stringify({ key, value, updated_at: new Date().toISOString() }),
        signal: AbortSignal.timeout(Math.max(8000, timeoutMs))
      });
      if (!response.ok) throw new Error(`Supabase ${response.status}: ${await response.text()}`);
      if (value === undefined) return (await response.json())[0]?.value;
      return;
    } catch (error) {
      lastError = error;
      if (attempt < 3) await new Promise(resolve => setTimeout(resolve, attempt * 750));
    }
  }
  const cause = lastError?.cause?.code || lastError?.cause?.message || '';
  throw new Error(`${lastError?.message || 'Supabase request failed'}${cause ? ` (${cause})` : ''}`);
}
function persistSupabaseState(key, value) {
  if (!USE_SUPABASE) return Promise.resolve();
  const snapshot = JSON.parse(JSON.stringify(value));
  const operation = supabaseWriteQueue.then(() => supabaseStateRequest(key, snapshot));
  supabaseWriteQueue = operation.catch(error => console.error(`Supabase save ${key} failed:`, error.message));
  return operation;
}
const localPresence = new Map();
// Two missed 30-second heartbeats are tolerated before a user is offline.
const PRESENCE_ONLINE_MS = 70 * 1000;

async function listPresence() {
  if (!USE_SUPABASE) return Array.from(localPresence.values());
  const headers = { apikey: SUPABASE_SERVICE_ROLE_KEY, Authorization: `Bearer ${SUPABASE_SERVICE_ROLE_KEY}` };
  const response = await fetch(`${SUPABASE_URL}/rest/v1/app_state?key=like.presence%3A%25&select=value`, {
    headers,
    signal: AbortSignal.timeout(4500)
  });
  if (!response.ok) throw new Error(`Supabase ${response.status}: ${await response.text()}`);
  return (await response.json()).map(row => row.value).filter(Boolean);
}

function publicPresenceRows(rows) {
  const now = Date.now();
  const latestByUser = new Map();
  rows.forEach(row => {
    if (!row?.userId) return;
    const current = latestByUser.get(row.userId);
    if (!current || new Date(row.lastSeenAt).getTime() > new Date(current.lastSeenAt).getTime()) latestByUser.set(row.userId, row);
  });
  return Array.from(latestByUser.values()).map(row => ({
    ...row,
    online: now - new Date(row.lastSeenAt).getTime() <= PRESENCE_ONLINE_MS
  }));
}
const AUTH_FILE = path.join(__dirname, 'data', 'auth-store.json');
let authStore = { users: [], settings: { proxyUrl: '' } };
try { authStore = JSON.parse(fs.readFileSync(AUTH_FILE, 'utf8')); } catch (error) {}
if (!Array.isArray(authStore.users)) authStore.users = [];
if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = { proxyUrl: '', curlCommand: '' };
if (typeof authStore.settings.curlCommand !== 'string') authStore.settings.curlCommand = '';
if (!['workfast', 'trafficlens', 'trafficlens_api', 'sitetraffic_api', 'workfast_sitetraffic'].includes(authStore.settings.trafficProvider)) authStore.settings.trafficProvider = 'workfast';
if (!authStore.settings.trafficLens || typeof authStore.settings.trafficLens !== 'object') {
  authStore.settings.trafficLens = { sessionCookie: '' };
}
if (!authStore.settings.trafficLensApi || typeof authStore.settings.trafficLensApi !== 'object') {
  authStore.settings.trafficLensApi = { apiKey: '' };
}
if (!authStore.settings.siteTrafficApi || typeof authStore.settings.siteTrafficApi !== 'object') {
  authStore.settings.siteTrafficApi = { apiKey: '' };
}
if (!authStore.settings.extensionBridge || typeof authStore.settings.extensionBridge !== 'object') {
  authStore.settings.extensionBridge = { token: '' };
}
if (!Array.isArray(authStore.extensionJobs)) authStore.extensionJobs = [];
if (!authStore.settings.nicheClassification || typeof authStore.settings.nicheClassification !== 'object') {
  authStore.settings.nicheClassification = { deepseekApiKey: '', firecrawlApiKey: '', categories: ['AI', 'Vật lý', 'Khác'], tavilyKeys: [], firecrawlKeys: [] };
}
if (!Array.isArray(authStore.settings.nicheClassification.categories) || !authStore.settings.nicheClassification.categories.length) {
  authStore.settings.nicheClassification.categories = ['AI', 'Vật lý', 'Khác'];
}
if (!Array.isArray(authStore.settings.nicheClassification.tavilyKeys)) authStore.settings.nicheClassification.tavilyKeys = [];
if (!authStore.settings.traceGoogleSheet || typeof authStore.settings.traceGoogleSheet !== 'object') {
  authStore.settings.traceGoogleSheet = { spreadsheetUrl: '', sheetName: 'Data', lastSync: null };
}
if (!Array.isArray(authStore.settings.trafficProfiles) || !authStore.settings.trafficProfiles.length) {
  authStore.settings.trafficProfiles = Array.from({ length: 10 }, (_, index) => ({
    id: String(index + 1), name: `Profile ${index + 1}`, enabled: true
  }));
}
authStore.settings.trafficProfiles = authStore.settings.trafficProfiles.map(item => ({
  id: String(item.id), name: `Profile ${item.id}`, enabled: item.enabled !== false
}));
if (!Array.isArray(authStore.loginHistory)) authStore.loginHistory = [];

function requestClientIp(req) {
  return String(req.get('x-vercel-forwarded-for') || req.get('x-forwarded-for') || req.get('x-real-ip') || req.socket.remoteAddress || '')
    .split(',')[0].trim() || null;
}

function recordLoginAttempt(req, user, username, success, reason = '') {
  if (!Array.isArray(authStore.loginHistory)) authStore.loginHistory = [];
  authStore.loginHistory.unshift({
    id: crypto.randomUUID(),
    timestamp: new Date().toISOString(),
    userId: user?.id || null,
    username: user?.username || username || 'unknown',
    displayName: user?.displayName || user?.username || username || 'Không xác định',
    success: Boolean(success),
    reason,
    ip: requestClientIp(req),
    country: req.get('x-vercel-ip-country') || null,
    region: decodeHeader(req.get('x-vercel-ip-country-region')),
    city: decodeHeader(req.get('x-vercel-ip-city')),
    userAgent: req.get('user-agent') || null
  });
  if (authStore.loginHistory.length > 500) authStore.loginHistory.length = 500;
  saveAuthStore().catch(error => console.error('Login history save failed:', error.message));
}
function saveAuthStore() {
  if (!USE_SUPABASE) {
    fs.mkdirSync(path.dirname(AUTH_FILE), { recursive: true });
    fs.writeFileSync(AUTH_FILE, JSON.stringify(authStore, null, 2));
  }
  return persistSupabaseState('auth', authStore);
}
function hashPassword(password, salt = crypto.randomBytes(16).toString('hex')) {
  return { salt, hash: crypto.scryptSync(String(password), salt, 64).toString('hex') };
}
function verifyPassword(password, user) {
  try {
    const actual = Buffer.from(crypto.scryptSync(String(password), user.passwordSalt, 64));
    const expected = Buffer.from(user.passwordHash, 'hex');
    return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
  } catch (error) { return false; }
}
function parseCookies(req) {
  return Object.fromEntries(String(req.headers.cookie || '').split(';').map(item => item.trim()).filter(Boolean).map(item => {
    const index = item.indexOf('=');
    return [decodeURIComponent(index < 0 ? item : item.slice(0, index)), decodeURIComponent(index < 0 ? '' : item.slice(index + 1))];
  }));
}
const ACCESS_TOKEN_MAX_AGE_SECONDS = Math.max(300, Number(process.env.ACCESS_TOKEN_MAX_AGE_SECONDS) || 30 * 60);
const REFRESH_TOKEN_MAX_AGE_SECONDS = Math.max(86400, Number(process.env.REFRESH_TOKEN_MAX_AGE_SECONDS) || 7 * 24 * 60 * 60);
function publicUser(user) {
  if (user && user.role !== 'admin' && user.canUseDomainStore === undefined) user.canUseDomainStore = Boolean(user.canUseTrace);
  return user ? {
    id: user.id,
    username: user.username,
    displayName: user.displayName || user.username,
    role: user.role,
    active: user.active !== false,
    canUseProxy: user.role === 'admin' || Boolean(user.canUseProxy),
    canUseAdvertiserCount: user.role === 'admin' || Boolean(user.canUseAdvertiserCount),
    canUseTraffic: user.role === 'admin' || Boolean(user.canUseTraffic),
    canUseTrace: user.role === 'admin' || Boolean(user.canUseTrace),
    canUseDomainStore: user.role === 'admin' || Boolean(user.canUseDomainStore ?? user.canUseTrace),
    canUseNiche: user.role === 'admin' || Boolean(user.canUseNiche),
    canUseDomainDuplicate: user.role === 'admin' || Boolean(user.canUseDomainDuplicate),
    canUseAffiliate: user.role === 'admin' || Boolean(user.canUseAffiliate),
    canUseAdminFirecrawl: user.role === 'admin' || Boolean(user.canUseAdminFirecrawl),
    personalFirecrawlConfigured: Boolean(decryptAdminSetting(user.personalFirecrawlApiKey)) || (Array.isArray(user.personalFirecrawlKeys) && user.personalFirecrawlKeys.some(item => item.active !== false && decryptAdminSetting(item.secret))),
    affiliateFirecrawlSource: ['user', 'admin', 'both'].includes(user.affiliateFirecrawlSource) ? user.affiliateFirecrawlSource : (user.role === 'admin' || user.canUseAdminFirecrawl ? 'admin' : 'user')
  } : null;
}
function verifySignedSessionToken(token, expectedKind = '') {
  const separator = token.lastIndexOf('.');
  if (separator < 1) return null;
  const encoded = token.slice(0, separator);
  const signature = token.slice(separator + 1);
  const expected = crypto.createHmac('sha256', SESSION_SECRET).update(encoded).digest('base64url');
  if (signature.length !== expected.length || !crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) return null;
  let payload;
  try { payload = JSON.parse(Buffer.from(encoded, 'base64url').toString('utf8')); } catch (error) { return null; }
  if (!payload?.userId || payload.expiresAt < Date.now()) return null;
  if (expectedKind && payload.kind !== expectedKind) return null;
  return payload;
}
function signSessionToken(user, kind, maxAgeSeconds) {
  const payload = {
    userId: user.id,
    kind,
    sessionVersion: Number(user.sessionVersion || 1),
    issuedAt: Date.now(),
    expiresAt: Date.now() + maxAgeSeconds * 1000
  };
  const encoded = Buffer.from(JSON.stringify(payload)).toString('base64url');
  const signature = crypto.createHmac('sha256', SESSION_SECRET).update(encoded).digest('base64url');
  return `${encoded}.${signature}`;
}
function userFromSessionPayload(payload) {
  if (!payload) return null;
  const user = authStore.users.find(item => item.id === payload.userId && item.active !== false);
  if (!user) return null;
  if (payload.kind && Number(payload.sessionVersion || 1) !== Number(user.sessionVersion || 1)) return null;
  return user || null;
}
function currentUser(req) {
  const cookies = parseCookies(req);
  const accessPayload = verifySignedSessionToken(cookies.qdev_access || '', 'access');
  if (accessPayload) return userFromSessionPayload(accessPayload);
  // Temporary migration support for the former 12-hour qdev_session cookie.
  return userFromSessionPayload(verifySignedSessionToken(cookies.qdev_session || ''));
}
function refreshSessionUser(req) {
  return userFromSessionPayload(verifySignedSessionToken(parseCookies(req).qdev_refresh || '', 'refresh'));
}
function sessionCookie(name, token, maxAgeSeconds) {
  const secure = process.env.NODE_ENV === 'production' ? '; Secure' : '';
  return `${name}=${token}; HttpOnly; SameSite=Strict; Path=/; Max-Age=${maxAgeSeconds}${secure}`;
}
function setAccessSession(res, user) {
  res.append('Set-Cookie', sessionCookie('qdev_access', signSessionToken(user, 'access', ACCESS_TOKEN_MAX_AGE_SECONDS), ACCESS_TOKEN_MAX_AGE_SECONDS));
}
function setSession(res, user) {
  setAccessSession(res, user);
  res.append('Set-Cookie', sessionCookie('qdev_refresh', signSessionToken(user, 'refresh', REFRESH_TOKEN_MAX_AGE_SECONDS), REFRESH_TOKEN_MAX_AGE_SECONDS));
  res.append('Set-Cookie', sessionCookie('qdev_session', '', 0));
}
function resolveSessionUser(req, res) {
  const cookies = parseCookies(req);
  const accessUser = currentUser(req);
  if (accessUser) {
    if (!cookies.qdev_access && cookies.qdev_session) setSession(res, accessUser);
    return accessUser;
  }
  const refreshedUser = refreshSessionUser(req);
  if (!refreshedUser) return null;
  setAccessSession(res, refreshedUser);
  res.setHeader('X-Session-Refreshed', '1');
  return refreshedUser;
}
function normalizeProxyUrl(value) {
  const raw = String(value || '').trim();
  if (!raw) return '';
  const candidate = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `http://${raw}`;
  const parsed = new URL(candidate);
  if (!['http:', 'https:'].includes(parsed.protocol) || !parsed.hostname || !parsed.username || !parsed.password) throw new Error('Proxy phải có dạng user:password@host:port hoặc http://user:password@host:port.');
  return parsed.toString().replace(/\/$/, '');
}
function parseAdminCurl(curlString) {
  const clean = String(curlString || '').replace(/\\\r?\n/g, ' ').replace(/\r?\n/g, ' ');
  const headers = {};
  const headerRegex = /(?:-H|--header)\s+(['"])(.*?)\1/gi;
  let match;
  while ((match = headerRegex.exec(clean)) !== null) {
    const colon = match[2].indexOf(':');
    if (colon > 0) headers[match[2].slice(0, colon).trim().toLowerCase()] = match[2].slice(colon + 1).trim();
  }
  const cookie = /(?:-b|--cookie)\s+(['"])(.*?)\1/i.exec(clean);
  if (cookie) headers.cookie = cookie[2];
  const data = /(?:--data-raw|--data|-d)\s+(['"])(.*?)\1/i.exec(clean);
  let fReq = null;
  if (data) {
    try {
      const raw = data[2].startsWith('f.req=') ? decodeURIComponent(data[2].slice(6)) : data[2];
      fReq = JSON.parse(raw);
    } catch (error) {}
  }
  return Object.keys(headers).length ? { headers, fReq } : null;
}
const MONITOR_FILE = path.join(__dirname, 'data', 'competitor-monitor.json');
let monitorStore = { schedules: [], notifications: [] };
try { monitorStore = JSON.parse(fs.readFileSync(MONITOR_FILE, 'utf8')); } catch (error) {}
if (!Array.isArray(monitorStore.schedules)) monitorStore.schedules = [];
if (!Array.isArray(monitorStore.notifications)) monitorStore.notifications = [];
if (!monitorStore.telegram || typeof monitorStore.telegram !== 'object') {
  const legacyTelegram = monitorStore.schedules.find(item => item.telegram?.botToken && item.telegram?.chatId)?.telegram;
  monitorStore.telegram = legacyTelegram ? { ...legacyTelegram } : { enabled: false, botToken: '', chatId: '' };
}
if (!monitorStore.telegramByUser || typeof monitorStore.telegramByUser !== 'object' || Array.isArray(monitorStore.telegramByUser)) {
  monitorStore.telegramByUser = {};
}
function getUserTelegram(userId) {
  if (!monitorStore.telegramByUser || typeof monitorStore.telegramByUser !== 'object' || Array.isArray(monitorStore.telegramByUser)) {
    monitorStore.telegramByUser = {};
  }
  const personal = monitorStore.telegramByUser?.[userId];
  if (personal && typeof personal === 'object') return personal;
  const user = authStore.users.find(item => item.id === userId);
  if (user?.role === 'admin' && monitorStore.telegram && typeof monitorStore.telegram === 'object') return monitorStore.telegram;
  return { enabled: false, botToken: '', chatId: '' };
}
function saveMonitorStore() {
  if (monitorStore.notifications.length > 300) monitorStore.notifications.length = 300;
  if (!USE_SUPABASE) {
    fs.mkdirSync(path.dirname(MONITOR_FILE), { recursive: true });
    fs.writeFileSync(MONITOR_FILE, JSON.stringify(monitorStore, null, 2));
  }
  return persistSupabaseState('monitor', monitorStore);
}
function monitorId(prefix) { return `${prefix}_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; }
function escapeTelegramHtml(value) {
  return String(value ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
}
async function sendTelegramNotification(schedule, advertiserId, domains) {
  const telegram = getUserTelegram(schedule.ownerId);
  if (!telegram.enabled || !domains.length) return;
  if (!/^\d+:[A-Za-z0-9_-]+$/.test(String(telegram.botToken || '')) || !String(telegram.chatId || '').trim()) {
    throw new Error('Bot Token hoặc Chat ID không hợp lệ.');
  }
  const detectedAt = new Date().toLocaleString('vi-VN', { timeZone: 'Asia/Bangkok' });
  const domainLines = domains.map((item, index) => {
    const link = item.adUrl ? `\n   <a href="${escapeTelegramHtml(item.adUrl)}">Mở mẫu quảng cáo</a>` : '';
    return `${index + 1}. <b>${escapeTelegramHtml(item.domain)}</b>${link}`;
  }).join('\n');
  const text = `🔔 <b>Phát hiện dự án mới</b>\n\n<b>Nhà quảng cáo:</b> <code>${escapeTelegramHtml(advertiserId)}</code>\n<b>Thời điểm:</b> ${escapeTelegramHtml(detectedAt)}\n<b>Domain mới:</b>\n${domainLines}`;
  const response = await fetch(`https://api.telegram.org/bot${telegram.botToken}/sendMessage`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ chat_id: telegram.chatId, text, parse_mode: 'HTML', disable_web_page_preview: true })
  });
  const result = await response.json().catch(() => ({}));
  if (!response.ok || !result.ok) throw new Error(result.description || `Telegram HTTP ${response.status}`);
}

async function sendTelegramTest(botToken, chatId) {
  if (!/^\d+:[A-Za-z0-9_-]+$/.test(String(botToken || '')) || !String(chatId || '').trim()) {
    throw new Error('Bot Token hoặc Chat ID không hợp lệ.');
  }
  const identityResponse = await fetch(`https://api.telegram.org/bot${botToken}/getMe`);
  const identity = await identityResponse.json().catch(() => ({}));
  if (!identityResponse.ok || !identity.ok) throw new Error(identity.description || 'Bot Token không hợp lệ.');
  const chatResponse = await fetch(`https://api.telegram.org/bot${botToken}/getChat?chat_id=${encodeURIComponent(chatId)}`);
  const chatResult = await chatResponse.json().catch(() => ({}));
  if (!chatResponse.ok || !chatResult.ok) {
    throw new Error(`Bot @${identity.result?.username || 'không rõ'} không truy cập được Chat ID ${chatId}: ${chatResult.description || `HTTP ${chatResponse.status}`}`);
  }
  const response = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      chat_id: chatId,
      text: '✅ <b>Kết nối thành công!</b>\nBot Telegram đã sẵn sàng nhận thông báo domain mới từ Ads Transparency.',
      parse_mode: 'HTML',
      disable_web_page_preview: true
    })
  });
  const result = await response.json().catch(() => ({}));
  if (!response.ok || !result.ok) throw new Error(result.description || `Telegram HTTP ${response.status}`);
  return { chat: result.result?.chat || {}, bot: identity.result || {} };
}
const DEFAULT_PROXY_URL = process.env.PROXY_URL
  || '';

// Enable CORS and JSON parsing
app.use(cors());
// Các phiên phân loại/nghiệp vụ hàng loạt có thể chứa hàng chục nghìn domain.
// Giới hạn rõ ràng 10 MB thay cho mặc định 100 KB của Express.
app.use(express.json({ limit: '10mb' }));
app.use('/api/auth', (req, res, next) => { res.setHeader('Cache-Control', 'no-store, private'); next(); });
app.get('/health', (req, res) => res.json({
  ok: true,
  uptimeSeconds: Math.round(process.uptime()),
  timestamp: new Date().toISOString(),
  storage: USE_SUPABASE ? 'supabase' : 'local',
  databaseProject: USE_SUPABASE ? (() => { try { return new URL(SUPABASE_URL).hostname.split('.')[0]; } catch (_) { return 'invalid-url'; } })() : null,
  traceDomainStorage: USE_SUPABASE ? (traceDomainTableAvailable === true ? 'supabase-table' : traceDomainTableAvailable === false ? 'supabase-app-state' : 'detecting') : 'local-file',
  authUserCount: authStore.users.length,
  supabaseLastRefreshAt: lastSupabaseRefreshAt ? new Date(lastSupabaseRefreshAt).toISOString() : null
}));
let stateHydrationPromise;
let supabaseRefreshPromise;
let lastSupabaseRefreshAt = 0;
app.use(async (req, res, next) => {
  if (req.path === '/login' || req.path === '/setup' || req.path === '/api/auth/logout') return next();
  const hydration = ensureSupabaseHydrated();
  // A cached local auth snapshot may keep the UI/login available while
  // Supabase is temporarily slow. Domain data itself never falls back to the
  // legacy store, so this cannot silently swap the visible database.
  if (!authStore.users.length) {
    try {
      await hydration;
    } catch (error) {
      return res.status(503).json({ error: 'Kho dữ liệu đang phản hồi chậm. Vui lòng thử lại sau vài giây.' });
    }
  } else {
    hydration.catch(error => console.warn('Using cached state after background Supabase hydration error:', error.message));
  }
  const mustRefresh = req.path === '/api/auth/login' || req.path.startsWith('/api/admin/');
  const authOnlyRefresh = req.path === '/api/auth/me' || mustRefresh;
  const refresh = refreshSupabaseSharedState(mustRefresh, authOnlyRefresh);
  if (mustRefresh) {
    try { await refresh; } catch (error) { console.warn('Using cached state after Supabase refresh error:', error.message); }
  } else refresh?.catch(error => console.warn('Background Supabase refresh failed:', error.message));
  next();
});

app.get('/login', (req, res) => res.sendFile(path.join(__dirname, 'public', 'login.html')));
app.get('/setup', (req, res) => res.sendFile(path.join(__dirname, 'public', 'login.html')));
app.get('/api/auth/status', (req, res) => res.json({ needsSetup: !authStore.users.some(user => user.role === 'admin'), authenticated: Boolean(resolveSessionUser(req, res)), storage: USE_SUPABASE ? 'supabase' : 'local', accessTokenMinutes: Math.round(ACCESS_TOKEN_MAX_AGE_SECONDS / 60), refreshTokenDays: Math.round(REFRESH_TOKEN_MAX_AGE_SECONDS / 86400) }));
app.post('/api/auth/setup', (req, res) => {
  if (authStore.users.some(user => user.role === 'admin')) return res.status(409).json({ error: 'Admin đã được thiết lập.' });
  const username = String(req.body?.username || '').trim().toLowerCase();
  const password = String(req.body?.password || '');
  const displayName = String(req.body?.displayName || 'QuocDev Admin').trim();
  if (!/^[a-z0-9._-]{3,40}$/.test(username) || password.length < 8) return res.status(400).json({ error: 'Username từ 3 ký tự; mật khẩu tối thiểu 8 ký tự.' });
  const secured = hashPassword(password);
  const user = { id: crypto.randomUUID(), username, displayName, role: 'admin', active: true, canUseProxy: true, sessionVersion: 1, passwordSalt: secured.salt, passwordHash: secured.hash, createdAt: new Date().toISOString() };
  authStore.users.push(user); recordLoginAttempt(req, user, username, true, 'Thiết lập Admin lần đầu'); setSession(res, user);
  res.status(201).json({ ok: true, user: publicUser(user) });
});
app.post('/api/auth/login', (req, res) => {
  const username = String(req.body?.username || '').trim().toLowerCase();
  const user = authStore.users.find(item => item.username === username);
  if (!user || user.active === false || !verifyPassword(req.body?.password || '', user)) {
    recordLoginAttempt(req, user, username, false, !user ? 'Tài khoản không tồn tại' : user.active === false ? 'Tài khoản bị khóa' : 'Sai mật khẩu');
    return res.status(401).json({ error: 'Tài khoản hoặc mật khẩu không đúng.' });
  }
  recordLoginAttempt(req, user, username, true, 'Đăng nhập thành công');
  setSession(res, user);
  res.json({ ok: true, user: publicUser(user) });
});
app.post('/api/auth/logout', (req, res) => {
  res.setHeader('Set-Cookie', [
    sessionCookie('qdev_access', '', 0),
    sessionCookie('qdev_refresh', '', 0),
    sessionCookie('qdev_session', '', 0)
  ]);
  res.json({ ok: true });
});

app.use((req, res, next) => {
  if (req.headers['x-internal-key'] === INTERNAL_API_KEY) {
    req.user = { id: 'system', username: 'system', role: 'admin', active: true, canUseProxy: true };
    return next();
  }
  if (req.path.startsWith('/api/extension/worker/')) {
    const suppliedToken = String(req.get('authorization') || '').replace(/^Bearer\s+/i, '').trim();
    const configuredToken = decryptAdminSetting(authStore.settings.extensionBridge?.token);
    if (suppliedToken && configuredToken && suppliedToken.length === configuredToken.length && crypto.timingSafeEqual(Buffer.from(suppliedToken), Buffer.from(configuredToken))) {
      req.user = { id: 'extension', username: 'chrome-extension', role: 'extension', active: true };
      req.extensionAuthenticated = true;
      return next();
    }
    return res.status(401).json({ error: 'Extension token kh\u00f4ng h\u1ee3p l\u1ec7.' });
  }
  if (req.path === '/api/google-sheets/oauth/callback') {
    const oauthState = verifyGoogleOAuthState(req.query.state);
    const oauthUser = oauthState && authStore.users.find(item => item.id === oauthState.userId && item.active !== false);
    if (!oauthUser) return res.redirect('/login?googleSheets=error');
    req.user = oauthUser;
    return next();
  }
  const user = resolveSessionUser(req, res);
  if (!user) {
    if (req.path.startsWith('/api/')) return res.status(401).json({ error: 'Bạn cần đăng nhập.' });
    return res.redirect('/login');
  }
  req.user = user;
  if (req.path === '/admin.html' && user.role !== 'admin') return res.status(403).send('Forbidden');
  next();
});

app.use((req, res, next) => apiRequestContext.run({
  ownerId: req.user?.id || null,
  username: req.user?.username || 'system',
  role: req.user?.role || 'system',
  requestIp: requestClientIp(req)
}, next));

app.use((req, res, next) => {
  const isTraceRequest = req.body?.traceMode === true || String(req.query?.traceMode || '') === '1';
  if (isTraceRequest && req.user.role !== 'admin' && !req.user.canUseTrace) {
    return res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Truy vết theo cấp độ.' });
  }
  next();
});

app.get('/api/auth/me', (req, res) => res.json({ user: publicUser(req.user), proxyConfigured: Boolean(authStore.settings.proxyUrl) }));
function publicUserExclusions(user) {
  const value = user?.exclusionSettings && typeof user.exclusionSettings === 'object' ? user.exclusionSettings : {};
  return {
    enabled: value.enabled === true,
    domains: [...new Set((Array.isArray(value.domains) ? value.domains : []).map(normalizeProjectDomain).filter(Boolean))].slice(0, 10000),
    advertisers: (Array.isArray(value.advertisers) ? value.advertisers : []).map(item => ({ id: String(item?.id || '').trim(), name: String(item?.name || '').trim() })).filter(item => item.id).slice(0, 10000)
  };
}
app.get('/api/user/exclusions', (req, res) => res.json(publicUserExclusions(req.user)));
app.put('/api/user/exclusions', async (req, res) => {
  const domains = [...new Set((Array.isArray(req.body?.domains) ? req.body.domains : []).map(normalizeProjectDomain).filter(Boolean))].slice(0, 10000);
  const seen = new Set();
  const advertisers = (Array.isArray(req.body?.advertisers) ? req.body.advertisers : []).map(item => ({ id: String(item?.id || '').trim(), name: String(item?.name || '').trim().slice(0, 300) })).filter(item => item.id && !seen.has(item.id) && seen.add(item.id)).slice(0, 10000);
  req.user.exclusionSettings = { enabled: req.body?.enabled === true, domains, advertisers, updatedAt: new Date().toISOString() };
  await saveAuthStore();
  res.json(publicUserExclusions(req.user));
});
app.post('/api/presence', async (req, res) => {
  const page = String(req.body?.page || '/').slice(0, 120);
  const section = String(req.body?.section || 'Ứng dụng').slice(0, 100);
  const record = {
    userId: req.user.id,
    username: req.user.username,
    displayName: req.user.displayName || req.user.username,
    role: req.user.role,
    page,
    section,
    ip: requestClientIp(req),
    lastSeenAt: new Date().toISOString()
  };
  localPresence.set(req.user.id, record);
  try {
    await persistSupabaseState(`presence:${req.user.id}`, record);
  } catch (error) {
    console.warn('Presence heartbeat save failed:', error.message);
  }
  res.json({ ok: true, onlineUntil: new Date(Date.now() + PRESENCE_ONLINE_MS).toISOString() });
});
app.get('/admin', (req, res) => req.user.role === 'admin' ? res.sendFile(path.join(__dirname, 'public', 'admin.html')) : res.status(403).send('Forbidden'));
function requireAdmin(req, res, next) { return req.user.role === 'admin' ? next() : res.status(403).json({ error: 'Chỉ Admin được sử dụng chức năng này.' }); }
function requireTrace(req, res, next) { return req.user.role === 'admin' || req.user.canUseTrace ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Truy vết theo cấp độ.' }); }
function requireDomainStore(req, res, next) { return req.user.role === 'admin' || Boolean(req.user.canUseDomainStore ?? req.user.canUseTrace) ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Kho Domain.' }); }
function requireNiche(req, res, next) { return req.user.role === 'admin' || req.user.canUseNiche ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Phân loại ngách.' }); }
function requireDomainDuplicate(req, res, next) { return req.user.role === 'admin' || req.user.canUseDomainDuplicate ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Check trùng Domain.' }); }
function requireAffiliate(req, res, next) { return req.user.role === 'admin' || req.user.canUseAffiliate ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Tìm nền tảng Affiliate.' }); }
function requireDomainDuplicateOrStore(req, res, next) { return req.user.role === 'admin' || req.user.canUseDomainDuplicate || req.user.canUseDomainStore ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Check trùng Domain hoặc Kho Domain.' }); }
function requireTraceOrDomainStore(req, res, next) { return req.body?.traceMode === true || String(req.query?.traceMode || '') === '1' ? requireTrace(req, res, next) : requireDomainStore(req, res, next); }
function requireTraceOrDomainStoreAccess(req, res, next) { return req.user.role === 'admin' || req.user.canUseTrace || req.user.canUseDomainStore ? next() : res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Truy vết hoặc Kho Domain.' }); }

let traceDomainsStore = [];
let traceHistoryStore = [];
let traceStoreLoaded = false;
let traceStoreLoadedAt = 0;
let traceStoreQueue = Promise.resolve();
let traceDomainTableAvailable = null;
let traceDomainMigrationPromise = null;

function supabaseTraceHeaders(prefer = '') {
  const headers = { apikey: SUPABASE_SERVICE_ROLE_KEY, Authorization: `Bearer ${SUPABASE_SERVICE_ROLE_KEY}`, 'Content-Type': 'application/json' };
  if (prefer) headers.Prefer = prefer;
  return headers;
}

async function ensureLegacyTraceDomainsMigrated() {
  if (!USE_SUPABASE || traceDomainTableAvailable === false) return;
  if (traceDomainMigrationPromise) return traceDomainMigrationPromise;
  traceDomainMigrationPromise = (async () => {
    await ensureTraceStores(false);
    if (!traceDomainsStore.length) return;
    const countResponse = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?select=domain`, { method: 'HEAD', headers: { ...supabaseTraceHeaders('count=exact'), Range: '0-0' }, signal: AbortSignal.timeout(10000) });
    if (countResponse.status === 404) { traceDomainTableAvailable = false; return; }
    if (!countResponse.ok) throw new Error(`Supabase trace_domains ${countResponse.status}`);
    traceDomainTableAvailable = true;
    for (let index = 0; index < traceDomainsStore.length; index += 1000) {
      const rows = traceDomainsStore.slice(index, index + 1000).map(domain => ({ domain, first_seen_at: new Date().toISOString(), scan_id: 'legacy-import', root_domain: null, added_by_username: 'Dữ liệu cũ' }));
      const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?on_conflict=domain`, { method: 'POST', headers: supabaseTraceHeaders('resolution=ignore-duplicates,return=minimal'), body: JSON.stringify(rows), signal: AbortSignal.timeout(20000) });
      if (!response.ok) throw new Error(`Không thể chuyển kho domain cũ: Supabase ${response.status}: ${await response.text()}`);
    }
    // Đã sao chép thành công sang bảng quan hệ: giải phóng JSONB cũ để các lần
    // tải sau không phải download/parse lại toàn bộ danh sách lớn.
    await persistSupabaseState('trace_domains', []);
    traceDomainsStore = [];
  })().catch(error => { traceDomainMigrationPromise = null; throw error; });
  return traceDomainMigrationPromise;
}

async function resolveTraceDomainsTable(candidates, metadata = {}) {
  if (!USE_SUPABASE || traceDomainTableAvailable === false) return null;
  await ensureLegacyTraceDomainsMigrated();
  if (traceDomainTableAvailable === false) return null;
  const now = new Date().toISOString();
  const rows = candidates.map(domain => ({ domain, first_seen_at: now, scan_id: metadata.scanId || null, root_domain: metadata.rootDomain || null, added_by_user_id: metadata.userId || null, added_by_username: metadata.username || null }));
  const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?on_conflict=domain&select=domain`, {
    method: 'POST', headers: supabaseTraceHeaders('resolution=ignore-duplicates,return=representation'), body: JSON.stringify(rows), signal: AbortSignal.timeout(15000)
  });
  if (response.status === 404) { traceDomainTableAvailable = false; return null; }
  if (!response.ok) throw new Error(`Supabase trace_domains ${response.status}: ${await response.text()}`);
  traceDomainTableAvailable = true;
  const added = (await response.json()).map(row => normalizeProjectDomain(row.domain)).filter(Boolean);
  const addedSet = new Set(added);
  return { added, existing: candidates.filter(domain => !addedSet.has(domain)) };
}

async function listTraceDomainsTable({ page, pageSize, query, addedBy, from, to, scanIds = [] }) {
  if (!USE_SUPABASE || traceDomainTableAvailable === false) return null;
  await ensureLegacyTraceDomainsMigrated();
  if (traceDomainTableAvailable === false) return null;
  const params = new URLSearchParams({ select: 'domain,first_seen_at,added_by_user_id,added_by_username,niche,niche_updated_at', order: 'first_seen_at.desc,domain.asc' });
  if (query) params.set('domain', `ilike.*${query.replace(/[,*()]/g, '')}*`);
  if (addedBy) params.set('added_by_username', `eq.${addedBy.replace(/[,()*]/g, '')}`);
  if (from && !Number.isNaN(from.getTime())) params.set('first_seen_at', `gte.${from.toISOString()}`);
  if (to && !Number.isNaN(to.getTime())) params.append('first_seen_at', `lte.${to.toISOString()}`);
  if (scanIds.length) params.set('scan_id', `in.(${scanIds.slice(0, 50).map(id => `"${String(id).replace(/["(),]/g, '')}"`).join(',')})`);
  const start = (page - 1) * pageSize;
  const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?${params}`, { headers: { ...supabaseTraceHeaders('count=exact'), Range: `${start}-${start + pageSize - 1}` }, signal: AbortSignal.timeout(15000) });
  if (response.status === 404) { traceDomainTableAvailable = false; return null; }
  if (!response.ok) throw new Error(`Supabase trace_domains ${response.status}: ${await response.text()}`);
  traceDomainTableAvailable = true;
  const filteredTotal = Number(String(response.headers.get('content-range') || '').split('/')[1]) || 0;
  const rows = await response.json();
  const canonicalRows = new Map();
  for (const row of rows) {
    const domain = projectDomainIdentity(row.domain);
    if (!domain || canonicalRows.has(domain)) continue;
    canonicalRows.set(domain, { domain, sourceDomain: normalizeProjectDomain(row.domain), firstSeenAt: row.first_seen_at, addedByUserId: row.added_by_user_id || '', addedByUsername: row.added_by_username || 'Không rõ', niche: row.niche || '', nicheUpdatedAt: row.niche_updated_at || null });
  }
  return { domains: [...canonicalRows.values()], filteredTotal: query ? canonicalRows.size : filteredTotal };
}

async function countTraceDomainsTable() {
  if (!USE_SUPABASE || traceDomainTableAvailable === false) return null;
  const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?select=domain`, { method: 'HEAD', headers: { ...supabaseTraceHeaders('count=exact'), Range: '0-0' }, signal: AbortSignal.timeout(10000) });
  if (response.status === 404) { traceDomainTableAvailable = false; return null; }
  if (!response.ok) throw new Error(`Supabase trace_domains ${response.status}`);
  traceDomainTableAvailable = true;
  return Number(String(response.headers.get('content-range') || '').split('/')[1]) || 0;
}

async function ensureTraceStores(force = false) {
  if (traceStoreLoaded && (!force || Date.now() - traceStoreLoadedAt < 5000)) return;
  if (USE_SUPABASE) {
    const [domains, history] = await Promise.all([
      supabaseStateRequest('trace_domains'),
      supabaseStateRequest('trace_history')
    ]);
    traceDomainsStore = Array.isArray(domains) ? domains.map(normalizeProjectDomain).filter(Boolean) : [];
    traceHistoryStore = Array.isArray(history) ? history : [];
  }
  traceDomainsStore = [...new Set(traceDomainsStore)];
  traceStoreLoaded = true;
  traceStoreLoadedAt = Date.now();
}

async function handleDomainResolve(req, res) {
  const candidates = uniqueProjectDomains(req.body?.domains);
  if (!candidates.length) return res.json({ added: [], existing: [], total: traceDomainsStore.length });
  const operation = traceStoreQueue.then(async () => {
    const tableResult = await resolveTraceDomainsTable(candidates, { scanId: String(req.body?.scanId || ''), rootDomain: normalizeProjectDomain(req.body?.rootDomain), userId: req.user?.id, username: req.user?.username });
    if (tableResult) return { ...tableResult, total: await countTraceDomainsTable() };
    await ensureTraceStores(true);
    const known = new Set(traceDomainsStore);
    const added = candidates.filter(domain => !known.has(domain));
    const existing = candidates.filter(domain => known.has(domain));
    if (added.length) {
      traceDomainsStore.push(...added);
      await persistSupabaseState('trace_domains', traceDomainsStore);
    }
    return { added, existing, total: traceDomainsStore.length };
  });
  traceStoreQueue = operation.catch(() => {});
  try { res.json(await operation); } catch (error) { res.status(503).json({ error: `Không thể cập nhật kho domain: ${error.message}` }); }
}
app.post('/api/trace/domains/resolve', requireTraceOrDomainStore, handleDomainResolve);
app.post('/api/advertiser/domains/resolve', requireDomainStore, handleDomainResolve);
app.post('/api/advertiser/domains/check', requireDomainStore, async (req, res) => {
  const inputDomains = [...new Set((Array.isArray(req.body?.domains) ? req.body.domains : []).map(normalizeProjectDomain).filter(Boolean))];
  const candidates = uniqueProjectDomains(inputDomains);
  try {
    if (!candidates.length) return res.json({ existing: [], missing: [] });
    if (USE_SUPABASE && traceDomainTableAvailable !== false) {
      await ensureLegacyTraceDomainsMigrated();
      if (traceDomainTableAvailable !== false) {
        const existing = [];
        for (let index = 0; index < candidates.length; index += 200) {
          const batch = candidates.slice(index, index + 200);
          const brands = batch.map(domain => domain.slice(0, -4));
          const params = new URLSearchParams({ select: 'domain', or: `(${brands.map(brand => `domain.like.${brand}.*`).join(',')})` });
          const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?${params}`, { headers: supabaseTraceHeaders(), signal: AbortSignal.timeout(15000) });
          if (response.status === 404) { traceDomainTableAvailable = false; break; }
          if (!response.ok) throw new Error(`Supabase trace_domains ${response.status}: ${await response.text()}`);
          existing.push(...(await response.json()).map(row => projectDomainIdentity(row.domain)).filter(Boolean));
        }
        if (traceDomainTableAvailable !== false) { const known = new Set(existing); return res.json({ existing: [...known], missing: candidates.filter(domain => !known.has(domain)) }); }
      }
    }
    await ensureTraceStores(true); const known = new Set(traceDomainsStore.map(projectDomainIdentity).filter(Boolean));
    res.json({ existing: candidates.filter(domain => known.has(domain)), missing: candidates.filter(domain => !known.has(domain)) });
  } catch (error) { res.status(503).json({ error: `Không thể đối chiếu Kho Domain: ${error.message}` }); }
});

app.post('/api/trace/history', requireTrace, async (req, res) => {
  const operation = traceStoreQueue.then(async () => {
    await ensureTraceStores(true);
    const record = {
      id: String(req.body?.id || crypto.randomUUID()),
      ownerId: req.user.id,
      username: req.user.username,
      rootDomain: normalizeProjectDomain(req.body?.rootDomain),
      startedAt: String(req.body?.startedAt || new Date().toISOString()),
      completedAt: new Date().toISOString(),
      maxLevel: Math.max(1, Number(req.body?.maxLevel) || 1),
      newDomainCount: [...new Set((req.body?.newDomains || []).map(normalizeProjectDomain).filter(Boolean))].length,
      // Domain chi tiết nằm trong public.trace_domains. Không nhân bản hàng chục
      // nghìn chuỗi vào từng bản ghi lịch sử nữa.
      newDomains: traceDomainTableAvailable === true ? [] : [...new Set((req.body?.newDomains || []).map(normalizeProjectDomain).filter(Boolean))],
      scannedDomains: Math.max(0, Number(req.body?.scannedDomains) || 0),
      advertisers: Math.max(0, Number(req.body?.advertisers) || 0),
      skippedOwners: Math.max(0, Number(req.body?.skippedOwners) || 0),
      errors: Math.max(0, Number(req.body?.errors) || 0),
      filters: { days: 7, platform: 'Google Search', format: 'Text' }
    };
    traceHistoryStore.unshift(record);
    traceHistoryStore = traceHistoryStore.slice(0, 500);
    await persistSupabaseState('trace_history', traceHistoryStore);
    return record;
  });
  traceStoreQueue = operation.catch(() => {});
  try { res.status(201).json({ ok: true, record: await operation }); } catch (error) { res.status(503).json({ error: `Không thể lưu lịch sử truy vết: ${error.message}` }); }
});

app.get('/api/trace/domains', requireDomainStore, async (req, res) => {
  try {
    await ensureTraceStores(true);
    let history = [...traceHistoryStore].sort((a, b) => new Date(b.completedAt) - new Date(a.completedAt));
    const lastScans = Math.max(0, Number(req.query.lastScans) || 0);
    const days = Math.max(0, Number(req.query.days) || 0);
    const from = req.query.from ? new Date(`${req.query.from}T00:00:00`) : null;
    const to = req.query.to ? new Date(`${req.query.to}T23:59:59.999`) : null;
    if (lastScans) history = history.slice(0, lastScans);
    if (days) { const cutoff = Date.now() - days * 86400000; history = history.filter(item => new Date(item.completedAt).getTime() >= cutoff); }
    if (from && !Number.isNaN(from.getTime())) history = history.filter(item => new Date(item.completedAt) >= from);
    if (to && !Number.isNaN(to.getTime())) history = history.filter(item => new Date(item.completedAt) <= to);
    const pageSize = Math.max(25, Math.min(500, Number(req.query.pageSize) || 100));
    const page = Math.max(1, Number(req.query.page) || 1);
    const hasFilter = Boolean(lastScans || days || from || to || req.query.addedBy);
    const query = String(req.query.q || '').trim().toLowerCase();
    const addedBy = String(req.query.addedBy || '').trim();
    const sheetSync = authStore.settings.traceGoogleSheet?.lastSync || null;
    const sheetRegistrationGroups = new Map();
    for (const item of (Array.isArray(sheetSync?.registrations) ? sheetSync.registrations : [])) {
      const domain = projectDomainIdentity(item.domain);
      if (!domain) continue;
      if (!sheetRegistrationGroups.has(domain)) sheetRegistrationGroups.set(domain, new Set());
      const names = Array.isArray(item.registrants) ? item.registrants : [item.registrant];
      names.map(String).map(name => name.trim()).filter(Boolean).forEach(name => sheetRegistrationGroups.get(domain).add(name));
    }
    const sheetRegistrations = new Map([...sheetRegistrationGroups].map(([domain, names]) => [domain, [...names].join(', ')]));
    const queryIdentity = projectDomainIdentity(query);
    const sheetMatches = query && !addedBy && !lastScans && !days && !from && !to ? [...sheetRegistrations]
      .filter(([domain]) => domain.includes(queryIdentity || query))
      .map(([domain, registrant]) => ({ domain, firstSeenAt: sheetSync.syncedAt || null, addedByUserId: '', addedByUsername: 'Google Sheet', source: 'google-sheet', registrant })) : [];
    const attachRegistration = item => ({ ...item, registrant: sheetRegistrations.get(projectDomainIdentity(item.domain)) || '' });
    {
      const tablePage = await listTraceDomainsTable({ page, pageSize, query, addedBy, from: days ? new Date(Date.now() - days * 86400000) : from, to, scanIds: lastScans ? history.map(item => item.id).filter(Boolean) : [] });
      if (tablePage) {
        const total = await countTraceDomainsTable();
        if (sheetMatches.length) {
          const stored = new Set(tablePage.domains.map(item => projectDomainIdentity(item.domain)));
          const sheetOnly = sheetMatches.filter(item => !stored.has(item.domain));
          const domains = (page === 1 ? [...tablePage.domains, ...sheetOnly].slice(0, pageSize) : tablePage.domains).map(attachRegistration);
          const filteredTotal = tablePage.filteredTotal + sheetOnly.length;
          return res.json({ total, filteredTotal, domains, sheetMatches: sheetMatches.length, adders: ['Google Sheet', ...authStore.users.map(user => user.username).filter(Boolean)], history, page, pageSize, pageCount: Math.max(1, Math.ceil(filteredTotal / pageSize)) });
        }
        return res.json({ total, filteredTotal: tablePage.filteredTotal, domains: tablePage.domains.map(attachRegistration), adders: ['Dữ liệu cũ', ...authStore.users.map(user => user.username).filter(Boolean)], history, page, pageSize, pageCount: Math.max(1, Math.ceil(tablePage.filteredTotal / pageSize)) });
      }
    }
    let domains = hasFilter ? [...new Set(history.flatMap(item => item.newDomains || []))] : [...traceDomainsStore];
    if (query) domains = domains.filter(domain => domain.includes(query));
    if (addedBy && addedBy !== 'Dữ liệu cũ') domains = [];
    const storedDomains = new Set(domains.map(projectDomainIdentity));
    if (sheetMatches.length) domains.push(...sheetMatches.filter(item => !storedDomains.has(item.domain)).map(item => item.domain));
    const filteredTotal = domains.length;
    domains = domains.slice((page - 1) * pageSize, page * pageSize);
    res.setHeader('Cache-Control', 'no-store, private');
    res.json({ total: traceDomainsStore.length, filteredTotal, domains: domains.map(domain => attachRegistration({ domain, firstSeenAt: null, addedByUserId: '', addedByUsername: 'Dữ liệu cũ' })), adders: ['Dữ liệu cũ', ...authStore.users.map(user => user.username).filter(Boolean)], history, page, pageSize, pageCount: Math.max(1, Math.ceil(filteredTotal / pageSize)) });
  } catch (error) { res.status(503).json({ error: `Không thể tải kho domain: ${error.message}` }); }
});
app.get('/api/admin/config', requireAdmin, async (req, res) => {
  const proxy = String(authStore.settings.proxyUrl || '');
  const googleOAuth = googleOAuthRuntimeConfig();
  const maskedSetting = value => value ? `${'•'.repeat(10)}${String(value).slice(-4)}` : 'Chưa cấu hình';
  let presence = publicPresenceRows(Array.from(localPresence.values()));
  try { presence = publicPresenceRows(await listPresence()); } catch (error) { console.warn('Presence list failed:', error.message); }
  const trafficLensSession = decryptAdminSetting(authStore.settings.trafficLens?.sessionCookie);
  const trafficLensApiKey = decryptAdminSetting(authStore.settings.trafficLensApi?.apiKey);
  res.json({ users: authStore.users.map(publicUser), presence, onlineCount: presence.filter(item => item.online).length, proxyConfigured: Boolean(proxy), proxyUrl: proxy, proxyHint: proxy ? proxy.replace(/\/\/([^:@]+):([^@]+)@/, '//$1:••••••@') : '', curlConfigured: Boolean(parseAdminCurl(authStore.settings.curlCommand)), trafficProfiles: authStore.settings.trafficProfiles, trafficProvider: authStore.settings.trafficProvider, trafficLens: { sessionConfigured: Boolean(trafficLensSession), sessionMasked: trafficLensSession ? `${'•'.repeat(12)}${trafficLensSession.slice(-6)}` : 'Chưa cấu hình', callsPerIp: 1 }, trafficLensApi: { apiKeyConfigured: Boolean(trafficLensApiKey), apiKeyMasked: trafficLensApiKey ? `${trafficLensApiKey.slice(0, 8)}${'•'.repeat(10)}${trafficLensApiKey.slice(-4)}` : 'Chưa cấu hình', requestsPerMinute: 60 }, traceGoogleSheet: authStore.settings.traceGoogleSheet || {}, googleOAuth: { clientId: googleOAuth.clientId, redirectUri: googleOAuth.redirectUri, clientSecretConfigured: Boolean(googleOAuth.clientSecret), tokenEncryptionKeyConfigured: googleOAuth.tokenEncryptionKeyConfigured, clientIdMasked: googleOAuth.clientId ? `${googleOAuth.clientId.slice(0, 8)}••••${googleOAuth.clientId.slice(-12)}` : 'Chưa cấu hình', clientSecretMasked: maskedSetting(googleOAuth.clientSecret), tokenEncryptionKeyMasked: googleOAuth.tokenEncryptionKeyConfigured ? maskedSetting(googleOAuth.rawTokenEncryptionKey) : 'Chưa cấu hình' } });
});

app.put('/api/admin/trace-google-sheet', requireAdmin, async (req, res) => {
  const current = authStore.settings.traceGoogleSheet || {};
  const spreadsheetUrl = String(req.body?.spreadsheetUrl ?? current.spreadsheetUrl ?? '').trim();
  const sheetName = String(req.body?.sheetName ?? current.sheetName ?? 'Data').trim().slice(0, 100) || 'Data';
  if (spreadsheetUrl && !parseGoogleSpreadsheetId(spreadsheetUrl)) return res.status(400).json({ error: 'Link Google Sheet không hợp lệ.' });
  const incomingSync = req.body?.lastSync && typeof req.body.lastSync === 'object' ? req.body.lastSync : null;
  const currentBackup = current.lastGoodSync || (Number(current.lastSync?.uniqueDomains || current.lastSync?.domains?.length || 0) > 0 ? current.lastSync : null);
  let lastGoodSync = currentBackup;
  let lastSync = current.lastSync || currentBackup || null;
  if (incomingSync) {
    const incomingCount = Number(incomingSync.uniqueDomains || incomingSync.domains?.length || 0);
    if (incomingCount > 0) { lastGoodSync = { ...incomingSync, usingBackup: false }; lastSync = lastGoodSync; }
    else if (currentBackup) lastSync = { ...currentBackup, usingBackup: true, backupSyncedAt: currentBackup.syncedAt || null, failedSyncAt: new Date().toISOString(), liveUniqueDomains: 0 };
  }
  if (incomingSync && Number(incomingSync.uniqueDomains || incomingSync.domains?.length || 0) === 0 && currentBackup && lastSync) {
    lastSync.message = `Google Sheet tr\u1ea3 v\u1ec1 0 domain; \u0111ang d\u00f9ng backup g\u1ea7n nh\u1ea5t ${currentBackup.domains?.length || currentBackup.uniqueDomains || 0} domain.`;
  }
  authStore.settings.traceGoogleSheet = { spreadsheetUrl, sheetName, lastSync, lastGoodSync, lastCheckedAt: incomingSync ? new Date().toISOString() : current.lastCheckedAt || null };
  await saveAuthStore();
  res.json({ ok: true, traceGoogleSheet: authStore.settings.traceGoogleSheet });
});

app.get('/api/trace/google-sheet/status', requireTraceOrDomainStoreAccess, async (req, res) => {
  let syncWarning = '';
  if (String(req.query?.refresh || '') === '1') {
    try { await refreshTraceGoogleSheetRegistrations(req.user); }
    catch (error) { syncWarning = error.message || 'Không thể tự đồng bộ Google Sheet.'; }
  }
  const config = authStore.settings.traceGoogleSheet || {};
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({ configured: Boolean(config.spreadsheetUrl), sheetName: config.sheetName || 'Data', lastSync: config.lastSync || null, syncWarning });
});

// Trạng thái đăng ký dự án là dữ liệu dùng chung của Tra cứu nhà quảng cáo,
// không phụ thuộc quyền xem hoặc chỉnh sửa Kho Domain.
let traceGoogleSheetRefreshPromise = null;
async function refreshTraceGoogleSheetRegistrations(requestUser, force = false) {
  const config = authStore.settings.traceGoogleSheet || {};
  const spreadsheetId = parseGoogleSpreadsheetId(config.spreadsheetUrl);
  if (!spreadsheetId) throw new Error('Chưa cấu hình link Google Sheet của Kho Domain.');
  const lastSyncAt = new Date(config.lastCheckedAt || config.lastSync?.syncedAt || 0).getTime();
  if (!force && config.lastSync?.domainParserVersion === 3 && Number.isFinite(lastSyncAt) && Date.now() - lastSyncAt < 60000) return config.lastSync;
  if (traceGoogleSheetRefreshPromise) return traceGoogleSheetRefreshPromise;
  traceGoogleSheetRefreshPromise = (async () => {
    const connectedUser = requestUser?.googleSheetsOAuth?.refreshToken ? requestUser : authStore.users.find(user => user.role === 'admin' && user.googleSheetsOAuth?.refreshToken) || authStore.users.find(user => user.googleSheetsOAuth?.refreshToken);
    if (!connectedUser) throw new Error('Chưa có tài khoản Google được kết nối để tự đồng bộ.');
    const token = await googleAccessTokenForUser(connectedUser);
    const googleHeaders = { authorization: `Bearer ${token.access_token}` };
    const sheetName = String(config.sheetName || 'Data').trim().slice(0, 100);
    const metadataResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}?fields=properties.title,sheets.properties.title`, { headers: googleHeaders, signal: AbortSignal.timeout(20000) });
    const metadata = await metadataResponse.json().catch(() => ({}));
    if (!metadataResponse.ok) throw new Error(metadata.error?.message || `Google Sheets HTTP ${metadataResponse.status}`);
    const availableSheets = (metadata.sheets || []).map(item => String(item.properties?.title || '')).filter(Boolean);
    const resolvedSheetName = availableSheets.find(title => title === sheetName) || availableSheets.find(title => title.toLocaleLowerCase('vi') === sheetName.toLocaleLowerCase('vi'));
    if (!resolvedSheetName) throw new Error(`Không tìm thấy tab “${sheetName}” trong Google Sheet.`);
    const range = `'${resolvedSheetName.replace(/'/g, "''")}'!A:ZZ`;
    const valuesResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?majorDimension=ROWS`, { headers: googleHeaders, signal: AbortSignal.timeout(30000) });
    const valuesPayload = await valuesResponse.json().catch(() => ({}));
    if (!valuesResponse.ok) throw new Error(valuesPayload.error?.message || `Google Sheets HTTP ${valuesResponse.status}`);
    const rows = Array.isArray(valuesPayload.values) ? valuesPayload.values : [];
    const headers = (rows[0] || []).map(value => String(value || '').trim().toLowerCase());
    let domainColumn = headers.findIndex(value => /^(domain|domain dự án|link dự án|link domain|tên miền|website|url)$/i.test(value));
    if (domainColumn < 0) domainColumn = 0;
    let registrantColumn = headers.findIndex(value => /^(nhân sự|người đăng ký|người phụ trách|người sở hữu|owner|assignee|registered by|registrant)$/i.test(value));
    if (registrantColumn < 0 && domainColumn !== 1 && (rows[0] || []).length > 1) registrantColumn = 1;
    const registrationMap = new Map(); const firstDomainRow = new Map(); const duplicateDetails = []; const invalidDetails = []; let validRows = 0; let invalidRows = 0;
    rows.slice(1).forEach((row, index) => {
      const rowNumber = index + 2;
      const rawDomain = String(row[domainColumn] || '').trim();
      const domain = projectDomainIdentity(rawDomain);
      if (!domain) { invalidRows++; invalidDetails.push({ rowNumber, value: rawDomain || '(trống)' }); return; }
      validRows++;
      if (firstDomainRow.has(domain)) duplicateDetails.push({ domain, rowNumber, firstRowNumber: firstDomainRow.get(domain), value: rawDomain });
      else firstDomainRow.set(domain, rowNumber);
      if (!registrationMap.has(domain)) registrationMap.set(domain, new Set());
      const registrant = registrantColumn >= 0 ? String(row[registrantColumn] || '').trim() : '';
      if (registrant) registrationMap.get(domain).add(registrant);
    });
    const registrations = [...registrationMap].map(([domain, names]) => ({ domain, registrants: [...names], registrant: [...names].join(', ') }));
    const domains = registrations.map(item => item.domain);
    const duplicateRows = duplicateDetails.length;
    const message = `Tự động đồng bộ tab “${resolvedSheetName}”: ${domains.length.toLocaleString('vi-VN')} domain duy nhất · ${validRows.toLocaleString('vi-VN')} dòng hợp lệ · ${duplicateRows.toLocaleString('vi-VN')} dòng trùng · ${invalidRows.toLocaleString('vi-VN')} dòng lỗi.`;
    const checkedAt = new Date().toISOString();
    if (!domains.length) {
      const backup = config.lastGoodSync || (Number(config.lastSync?.uniqueDomains || config.lastSync?.domains?.length || 0) > 0 ? config.lastSync : null);
      if (backup) {
        const fallback = { ...backup, usingBackup: true, backupSyncedAt: backup.syncedAt || null, failedSyncAt: checkedAt, liveUniqueDomains: 0 };
        fallback.message = `Google Sheet tr\u1ea3 v\u1ec1 0 domain; \u0111ang d\u00f9ng backup g\u1ea7n nh\u1ea5t ${Number(backup.domains?.length || backup.uniqueDomains || 0).toLocaleString('vi-VN')} domain t\u1eeb ${backup.syncedAt ? new Date(backup.syncedAt).toLocaleString('vi-VN') : 'l\u1ea7n \u0111\u1ed3ng b\u1ed9 tr\u01b0\u1edbc'}.`;
        authStore.settings.traceGoogleSheet = { ...config, sheetName: resolvedSheetName, lastSync: fallback, lastGoodSync: backup, lastCheckedAt: checkedAt };
        await saveAuthStore();
        const backupError = new Error('Google Sheet tr\u1ea3 v\u1ec1 0 domain, c\u00f3 th\u1ec3 \u0111ang l\u1ed7i; h\u1ec7 th\u1ed1ng \u0111ang d\u00f9ng b\u1ea3n backup g\u1ea7n nh\u1ea5t.');
        backupError.usingBackup = true;
        throw backupError;
      }
      throw new Error('Google Sheet tr\u1ea3 v\u1ec1 0 domain v\u00e0 ch\u01b0a c\u00f3 b\u1ea3n backup h\u1ee3p l\u1ec7; kh\u00f4ng ghi \u0111\u00e8 d\u1eef li\u1ec7u hi\u1ec7n t\u1ea1i.');
    }
    const lastSync = { message, syncedAt: checkedAt, autoSynced: true, usingBackup: false, domainParserVersion: 3, syncedBy: connectedUser.googleSheetsOAuth?.email || connectedUser.username, domains, registrations, totalRows: Math.max(0, rows.length - 1), validRows, duplicateRows, invalidRows, duplicateDetails, invalidDetails, uniqueDomains: domains.length, domainHeader: String((rows[0] || [])[domainColumn] || ''), sheetName: resolvedSheetName };
    authStore.settings.traceGoogleSheet = { ...config, sheetName: resolvedSheetName, lastSync, lastGoodSync: lastSync, lastCheckedAt: checkedAt };
    await saveAuthStore();
    return lastSync;
  })().finally(() => { traceGoogleSheetRefreshPromise = null; });
  return traceGoogleSheetRefreshPromise;
}

async function runTraceGoogleSheetAutoRefresh() {
  if (!authStore.settings.traceGoogleSheet?.spreadsheetUrl) return;
  try { await refreshTraceGoogleSheetRegistrations(null); }
  catch (error) { console.warn('Google Sheet auto sync skipped:', error.message); }
}
setTimeout(runTraceGoogleSheetAutoRefresh, 30000).unref?.();
setInterval(runTraceGoogleSheetAutoRefresh, 5 * 60 * 1000).unref?.();

app.post('/api/project-registrations/check', requireDomainDuplicateOrStore, async (req, res) => {
  const candidates = new Set(uniqueProjectDomains(req.body?.domains).slice(0, 10000));
  let syncWarning = '';
  try { await refreshTraceGoogleSheetRegistrations(req.user); }
  catch (error) { syncWarning = error.message || 'Không thể tự đồng bộ Google Sheet.'; }
  const lastSync = authStore.settings.traceGoogleSheet?.lastSync || null;
  const grouped = new Map();
  for (const item of (Array.isArray(lastSync?.registrations) ? lastSync.registrations : [])) {
    const domain = projectDomainIdentity(item.domain);
    if (!domain || !candidates.has(domain)) continue;
    if (!grouped.has(domain)) grouped.set(domain, { domain, sourceDomains: new Set(), registrants: new Set() });
    const group = grouped.get(domain);
    const sourceDomain = normalizeProjectDomain(item.domain);
    if (sourceDomain) group.sourceDomains.add(sourceDomain);
    const names = Array.isArray(item.registrants) ? item.registrants : [item.registrant];
    names.map(String).map(name => name.trim()).filter(Boolean).forEach(name => group.registrants.add(name));
  }
  const registrations = [...grouped.values()].map(item => ({ domain: item.domain, sourceDomains: [...item.sourceDomains], registrants: [...item.registrants], registrant: [...item.registrants].join(', ') }));
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({ synced: Boolean(lastSync), syncedAt: lastSync?.syncedAt || null, autoSynced: Boolean(lastSync?.autoSynced), syncWarning, registrations });
});

app.get('/api/admin/site-traffic-api', requireAdmin, (req, res) => {
  const apiKey = decryptAdminSetting(authStore.settings.siteTrafficApi?.apiKey);
  res.json({ apiKeyConfigured: Boolean(apiKey), apiKeyMasked: apiKey ? `${apiKey.slice(0, 6)}${'\u2022'.repeat(10)}${apiKey.slice(-4)}` : '' });
});

app.get('/api/admin/extension-bridge', requireAdmin, (req, res) => {
  const token = decryptAdminSetting(authStore.settings.extensionBridge?.token);
  res.json({ tokenConfigured: Boolean(token), tokenMasked: token ? `${token.slice(0, 6)}${'\u2022'.repeat(12)}${token.slice(-4)}` : '' });
});

app.put('/api/admin/extension-bridge', requireAdmin, async (req, res) => {
  const suppliedToken = String(req.body?.token || '').trim();
  if (suppliedToken && suppliedToken.length < 16) return res.status(400).json({ error: 'Extension token ph\u1ea3i c\u00f3 \u00edt nh\u1ea5t 16 k\u00fd t\u1ef1.' });
  const token = suppliedToken || crypto.randomBytes(32).toString('hex');
  authStore.settings.extensionBridge = { token: encryptAdminSetting(token), updatedAt: new Date().toISOString() };
  await saveAuthStore();
  res.json({ ok: true, token, tokenConfigured: true, tokenMasked: `${token.slice(0, 6)}${'\u2022'.repeat(12)}${token.slice(-4)}` });
});

app.put('/api/admin/site-traffic-api', requireAdmin, async (req, res) => {
  const suppliedApiKey = String(req.body?.apiKey || '').trim();
  const currentEncryptedApiKey = authStore.settings.siteTrafficApi?.apiKey || '';
  if (!suppliedApiKey && !decryptAdminSetting(currentEncryptedApiKey)) {
    return res.status(400).json({ error: 'C\u1ea7n nh\u1eadp API key c\u1ee7a SiteTraffic API.' });
  }
  authStore.settings.siteTrafficApi = { apiKey: suppliedApiKey ? encryptAdminSetting(suppliedApiKey) : currentEncryptedApiKey };
  await saveAuthStore();
  const savedApiKey = decryptAdminSetting(authStore.settings.siteTrafficApi.apiKey);
  res.json({ ok: true, apiKeyConfigured: Boolean(savedApiKey), apiKeyMasked: `${savedApiKey.slice(0, 6)}${'\u2022'.repeat(10)}${savedApiKey.slice(-4)}` });
});

app.put('/api/admin/traffic-provider', requireAdmin, async (req, res, next) => {
  const requestedProvider = String(req.body?.provider || '').trim().toLowerCase();
  if (!['sitetraffic_api', 'workfast_sitetraffic'].includes(requestedProvider)) return next();
  const suppliedApiKey = String(req.body?.siteTrafficApiKey || '').trim();
  const currentEncryptedApiKey = authStore.settings.siteTrafficApi?.apiKey || '';
  if (!suppliedApiKey && !decryptAdminSetting(currentEncryptedApiKey)) {
    return res.status(400).json({ error: 'C\u1ea7n nh\u1eadp API key c\u1ee7a SiteTraffic API.' });
  }
  authStore.settings.trafficProvider = requestedProvider;
  if (suppliedApiKey) authStore.settings.siteTrafficApi = { apiKey: encryptAdminSetting(suppliedApiKey) };
  await saveAuthStore();
  res.json({ ok: true, provider: requestedProvider, apiKeyConfigured: true });
});

app.put('/api/admin/traffic-provider', requireAdmin, async (req, res) => {
  const provider = String(req.body?.provider || '').trim().toLowerCase();
  if (!['workfast', 'trafficlens', 'trafficlens_api'].includes(provider)) return res.status(400).json({ error: 'Nguồn Traffic không hợp lệ.' });
  const suppliedSession = String(req.body?.sessionCookie || '').trim();
  const suppliedApiKey = String(req.body?.apiKey || '').trim();
  const currentEncryptedSession = authStore.settings.trafficLens?.sessionCookie || '';
  const currentEncryptedApiKey = authStore.settings.trafficLensApi?.apiKey || '';
  if (provider === 'trafficlens' && !authStore.settings.proxyUrl) {
    return res.status(400).json({ error: 'TrafficLens cần Proxy chung để đổi IP ở mỗi lượt gọi.' });
  }
  if (provider === 'trafficlens_api' && !suppliedApiKey && !decryptAdminSetting(currentEncryptedApiKey)) {
    return res.status(400).json({ error: 'Cần nhập X-API-Key của TrafficLens API.' });
  }
  authStore.settings.trafficProvider = provider;
  authStore.settings.trafficLens = {
    sessionCookie: suppliedSession ? encryptAdminSetting(suppliedSession.replace(/^session=/i, '')) : currentEncryptedSession
  };
  authStore.settings.trafficLensApi = {
    apiKey: suppliedApiKey ? encryptAdminSetting(suppliedApiKey) : currentEncryptedApiKey
  };
  resetTrafficLensProxySlot();
  await saveAuthStore();
  res.json({ ok: true, provider, sessionConfigured: Boolean(decryptAdminSetting(authStore.settings.trafficLens.sessionCookie)), apiKeyConfigured: Boolean(decryptAdminSetting(authStore.settings.trafficLensApi.apiKey)) });
});

function extensionJobPublic(job) {
  return { id: job.id, action: job.action, status: job.status, total: job.domains.length, completed: job.results.length, successCount: job.results.filter(item => item.success !== false).length, errorCount: job.results.filter(item => item.success === false).length, domains: job.domains, results: job.results, createdAt: job.createdAt, claimedAt: job.claimedAt || null, startedAt: job.startedAt || null, updatedAt: job.updatedAt, completedAt: job.completedAt || null, message: job.message || '' };
}

function findExtensionJob(id) { return authStore.extensionJobs.find(item => item.id === id); }
function canAccessExtensionJob(req, job) { return req.user?.role === 'admin' || job?.ownerId === req.user?.id; }
function trimExtensionJobs() { authStore.extensionJobs = authStore.extensionJobs.sort((a, b) => new Date(b.updatedAt) - new Date(a.updatedAt)).slice(0, 30); }

app.post('/api/extension/jobs', async (req, res) => {
  if (req.user.role !== 'admin' && !req.user.canUseTraffic) return res.status(403).json({ error: 'T\u00e0i kho\u1ea3n ch\u01b0a \u0111\u01b0\u1ee3c c\u1ea5p quy\u1ec1n Qu\u00e9t Traffic.' });
  const domains = uniqueProjectDomains(req.body?.domains).slice(0, 10000);
  if (!domains.length) return res.status(400).json({ error: 'Danh s\u00e1ch domain kh\u00f4ng h\u1ee3p l\u1ec7.' });
  const now = new Date().toISOString();
  const job = { id: `ext_${Date.now()}_${crypto.randomBytes(5).toString('hex')}`, action: 'scan_traffic', ownerId: req.user.id, ownerUsername: req.user.username, status: 'pending', domains, results: [], createdAt: now, updatedAt: now, claimedAt: null, startedAt: null, completedAt: null, message: '' };
  authStore.extensionJobs.unshift(job); trimExtensionJobs(); await saveAuthStore();
  res.status(201).json({ ok: true, job: extensionJobPublic(job) });
});

app.get('/api/extension/jobs/:id', (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job || !canAccessExtensionJob(req, job)) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({ job: extensionJobPublic(job) });
});

app.post('/api/extension/jobs/:id/cancel', async (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job || !canAccessExtensionJob(req, job)) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  job.status = 'cancelled'; job.updatedAt = new Date().toISOString(); job.completedAt = job.updatedAt; job.message = 'Ng\u01b0\u1eddi d\u00f9ng \u0111\u00e3 h\u1ee7y.'; await saveAuthStore();
  res.json({ ok: true, job: extensionJobPublic(job) });
});

app.get('/api/extension/worker/jobs/next', async (req, res) => {
  const staleBefore = Date.now() - 2 * 60 * 1000;
  authStore.extensionJobs.forEach(job => { if (['claimed', 'running'].includes(job.status) && new Date(job.updatedAt).getTime() < staleBefore) { job.status = 'pending'; job.message = 'Tr\u1ea3 l\u1ea1i h\u00e0ng \u0111\u1ee3i do extension m\u1ea5t k\u1ebft n\u1ed1i.'; } });
  const job = authStore.extensionJobs.find(item => item.status === 'pending');
  if (!job) return res.status(204).end();
  job.status = 'claimed'; job.claimedAt = new Date().toISOString(); job.updatedAt = job.claimedAt; job.message = 'Extension \u0111\u00e3 nh\u1eadn job.'; await saveAuthStore();
  res.json({ job: extensionJobPublic(job), callback: { results: `/api/extension/worker/jobs/${job.id}/results`, heartbeat: `/api/extension/worker/jobs/${job.id}/heartbeat`, complete: `/api/extension/worker/jobs/${job.id}/complete`, fail: `/api/extension/worker/jobs/${job.id}/fail` } });
});

app.post('/api/extension/worker/jobs/:id/heartbeat', async (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  if (job.status === 'cancelled') return res.status(409).json({ error: 'Job \u0111\u00e3 b\u1ecb h\u1ee7y.', cancelled: true });
  if (job.status === 'claimed') { job.status = 'running'; job.startedAt = new Date().toISOString(); }
  job.updatedAt = new Date().toISOString(); job.message = String(req.body?.message || job.message || '').slice(0, 500); await saveAuthStore();
  res.json({ ok: true, status: job.status });
});

app.post('/api/extension/worker/jobs/:id/results', async (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  if (job.status === 'cancelled') return res.status(409).json({ error: 'Job \u0111\u00e3 b\u1ecb h\u1ee7y.', cancelled: true });
  const incoming = (Array.isArray(req.body?.items) ? req.body.items : [req.body]).slice(0, 100);
  const allowedDomains = new Set(job.domains.map(projectDomainIdentity));
  const byDomain = new Map(job.results.map(item => [projectDomainIdentity(item.domain), item]));
  let accepted = 0;
  for (const item of incoming) {
    const domain = projectDomainIdentity(item?.domain || item?.itemId);
    if (!domain || !allowedDomains.has(domain)) continue;
    const safeData = item?.data && typeof item.data === 'object' && JSON.stringify(item.data).length <= 250000 ? item.data : null;
    byDomain.set(domain, { itemId: String(item?.itemId || domain).slice(0, 250), domain, success: item?.success !== false, data: safeData, error: String(item?.error || '').slice(0, 2000), receivedAt: new Date().toISOString() }); accepted++;
  }
  job.results = [...byDomain.values()]; job.status = 'running'; job.startedAt ||= new Date().toISOString(); job.updatedAt = new Date().toISOString(); job.message = `\u0110\u00e3 nh\u1eadn ${job.results.length}/${job.domains.length} k\u1ebft qu\u1ea3.`; await saveAuthStore();
  res.json({ ok: true, accepted, completed: job.results.length, total: job.domains.length });
});

app.post('/api/extension/worker/jobs/:id/complete', async (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  job.status = 'completed'; job.completedAt = new Date().toISOString(); job.updatedAt = job.completedAt; job.message = String(req.body?.message || 'Extension \u0111\u00e3 ho\u00e0n t\u1ea5t job.').slice(0, 500); await saveAuthStore();
  res.json({ ok: true, job: extensionJobPublic(job) });
});

app.post('/api/extension/worker/jobs/:id/fail', async (req, res) => {
  const job = findExtensionJob(req.params.id);
  if (!job) return res.status(404).json({ error: 'Kh\u00f4ng t\u00ecm th\u1ea5y extension job.' });
  job.status = req.body?.retryable ? 'pending' : 'failed'; job.updatedAt = new Date().toISOString(); job.message = String(req.body?.error || 'Extension job th\u1ea5t b\u1ea1i.').slice(0, 2000); await saveAuthStore();
  res.json({ ok: true, job: extensionJobPublic(job) });
});

app.get('/api/admin/presence', requireAdmin, async (req, res) => {
  let presence = publicPresenceRows(Array.from(localPresence.values()));
  try { presence = publicPresenceRows(await listPresence()); } catch (error) { console.warn('Presence refresh failed:', error.message); }
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({ presence, onlineCount: presence.filter(item => item.online).length, serverTime: new Date().toISOString() });
});

app.put('/api/admin/google-oauth', requireAdmin, async (req, res) => {
  const password = String(req.body?.password || '');
  const expected = Buffer.from('Abc@12123');
  const actual = Buffer.from(password);
  if (actual.length !== expected.length || !crypto.timingSafeEqual(actual, expected)) return res.status(403).json({ error: 'Mật khẩu xác nhận không đúng.' });
  const current = authStore.settings.googleOAuth || {};
  const clientId = String(req.body?.clientId || '').trim();
  const redirectUri = String(req.body?.redirectUri || '').trim();
  const clientSecret = String(req.body?.clientSecret || '');
  const tokenEncryptionKey = String(req.body?.tokenEncryptionKey || '');
  if (!clientId) return res.status(400).json({ error: 'GOOGLE_OAUTH_CLIENT_ID không được để trống.' });
  if (!clientId.endsWith('.apps.googleusercontent.com')) return res.status(400).json({ error: 'GOOGLE_OAUTH_CLIENT_ID không đúng định dạng.' });
  if (!redirectUri) return res.status(400).json({ error: 'GOOGLE_OAUTH_REDIRECT_URI không được để trống.' });
  try {
    const parsedRedirect = new URL(redirectUri);
    const localHttp = parsedRedirect.protocol === 'http:' && ['localhost', '127.0.0.1'].includes(parsedRedirect.hostname);
    if (parsedRedirect.protocol !== 'https:' && !localHttp) throw new Error();
  } catch (_) { return res.status(400).json({ error: 'GOOGLE_OAUTH_REDIRECT_URI phải là HTTPS (hoặc HTTP localhost khi chạy local).' }); }
  if (!clientSecret && !current.clientSecret && !ENV_GOOGLE_OAUTH_CLIENT_SECRET) return res.status(400).json({ error: 'GOOGLE_OAUTH_CLIENT_SECRET không được để trống.' });
  if (!tokenEncryptionKey && !current.tokenEncryptionKey && !ENV_GOOGLE_TOKEN_ENCRYPTION_KEY) return res.status(400).json({ error: 'GOOGLE_TOKEN_ENCRYPTION_KEY không được để trống.' });
  if (tokenEncryptionKey && tokenEncryptionKey.length < 3) return res.status(400).json({ error: 'GOOGLE_TOKEN_ENCRYPTION_KEY cần tối thiểu 3 ký tự.' });
  const keyChanged = Boolean(tokenEncryptionKey);
  authStore.settings.googleOAuth = {
    clientId,
    redirectUri,
    clientSecret: clientSecret ? encryptAdminSetting(clientSecret) : current.clientSecret,
    tokenEncryptionKey: tokenEncryptionKey ? encryptAdminSetting(tokenEncryptionKey) : current.tokenEncryptionKey
  };
  if (keyChanged) authStore.users.forEach(user => { delete user.googleSheetsOAuth; });
  await saveAuthStore();
  res.json({ ok: true, disconnectedUsers: keyChanged });
});

app.post('/api/admin/google-oauth/reveal', requireAdmin, (req, res) => {
  const password = String(req.body?.password || '');
  const expected = Buffer.from('Abc@12123');
  const actual = Buffer.from(password);
  if (actual.length !== expected.length || !crypto.timingSafeEqual(actual, expected)) return res.status(403).json({ error: 'Mật khẩu xác nhận không đúng.' });
  const config = googleOAuthRuntimeConfig();
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({
    clientId: config.clientId,
    clientSecret: config.clientSecret,
    redirectUri: config.redirectUri,
    tokenEncryptionKey: config.tokenEncryptionKeyConfigured ? config.rawTokenEncryptionKey : ''
  });
});

app.post('/api/admin/google-oauth/test', requireAdmin, async (req, res) => {
  const config = googleOAuthRuntimeConfig();
  const checks = [];
  try {
    if (!config.clientId || !config.clientId.endsWith('.apps.googleusercontent.com')) throw new Error('Client ID chưa đúng định dạng Google OAuth.');
    checks.push({ key: 'clientId', ok: true, message: 'Client ID đúng định dạng.' });
    if (!config.clientSecret) throw new Error('Client Secret chưa được cấu hình.');
    checks.push({ key: 'clientSecret', ok: true, message: 'Client Secret đã được lưu.' });
    const parsedRedirect = new URL(config.redirectUri);
    const localHttp = parsedRedirect.protocol === 'http:' && ['localhost', '127.0.0.1'].includes(parsedRedirect.hostname);
    if (parsedRedirect.protocol !== 'https:' && !localHttp) throw new Error('Redirect URI phải dùng HTTPS, trừ localhost.');
    checks.push({ key: 'redirectUri', ok: true, message: `Redirect URI hợp lệ: ${config.redirectUri}` });
    if (!config.tokenEncryptionKeyConfigured || String(config.rawTokenEncryptionKey || '').length < 3) throw new Error('Encryption Key chưa cấu hình hoặc ngắn hơn 3 ký tự.');
    const probe = `google-oauth-config-test:${crypto.randomUUID()}`;
    if (decryptGoogleToken(encryptGoogleToken(probe)) !== probe) throw new Error('Encryption Key không mã hóa/giải mã được token.');
    checks.push({ key: 'encryption', ok: true, message: 'Encryption Key mã hóa và giải mã AES-256-GCM thành công.' });

    const connectedUser = authStore.users.find(user => user.googleSheetsOAuth?.refreshToken);
    if (connectedUser) {
      await googleAccessTokenForUser(connectedUser);
      checks.push({ key: 'google', ok: true, message: `Google xác nhận Client ID, Client Secret, Encryption Key và refresh token của ${connectedUser.googleSheetsOAuth?.email || connectedUser.username} hoạt động.` });
    } else {
      const response = await fetch('https://oauth2.googleapis.com/token', { method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ client_id: config.clientId, client_secret: config.clientSecret, refresh_token: 'configuration-test-not-a-real-token', grant_type: 'refresh_token' }), signal: AbortSignal.timeout(20000) });
      const payload = await response.json().catch(() => ({}));
      if (payload.error === 'invalid_client' || response.status === 401) throw new Error('Google từ chối Client ID hoặc Client Secret.');
      checks.push({ key: 'google', ok: true, message: 'Google chấp nhận thông tin ứng dụng. Chưa có tài khoản kết nối để kiểm tra refresh token.' });
    }
    res.json({ ok: true, checks });
  } catch (error) { res.status(400).json({ ok: false, error: error.message, checks }); }
});

app.get('/api/admin/traffic-profiles', requireAdmin, async (req, res) => {
  const calls = await getApiHistory();
  const history = calls.filter(call => /Workfast Traffic API/i.test(String(call.direction || ''))).map(call => ({
    timestamp: call.timestamp,
    profile: call.trafficProfile || null,
    profileName: call.trafficProfile ? `Profile ${call.trafficProfile}` : 'Không xác định',
    domain: call.trafficDomain || null,
    attempt: call.trafficAttempt || 1,
    status: call.status,
    durationMs: call.durationMs
  })).slice(0, 200);
  res.json({ profiles: authStore.settings.trafficProfiles, history });
});

app.put('/api/admin/traffic-profiles', requireAdmin, async (req, res) => {
  const input = Array.isArray(req.body?.profiles) ? req.body.profiles : [];
  const seen = new Set();
  const profiles = input.map((item, index) => {
    const id = String(item?.id ?? '').trim();
    if (!id || !/^[a-zA-Z0-9._-]{1,40}$/.test(id)) throw new Error(`Profile dòng ${index + 1} không hợp lệ.`);
    if (seen.has(id)) throw new Error(`Profile ${id} bị trùng.`);
    seen.add(id);
    return { id, name: `Profile ${id}`, enabled: item?.enabled !== false };
  });
  if (!profiles.length || !profiles.some(item => item.enabled)) {
    return res.status(400).json({ error: 'Cần ít nhất một Profile đang bật.' });
  }
  if (profiles.length > 200) return res.status(400).json({ error: 'Tối đa 200 Profile.' });
  authStore.settings.trafficProfiles = profiles;
  cachedTrafficAuth = null;
  pendingAuthPromise = null;
  currentProfileIndex = 0;
  await saveAuthStore();
  res.json({ ok: true, profiles });
});

function startOfIsoWeek(dateValue) {
  const date = new Date(dateValue);
  const day = date.getUTCDay() || 7;
  date.setUTCDate(date.getUTCDate() - day + 1);
  return date.toISOString().slice(0, 10);
}

function buildSecurityAlerts(users, proxyCalls, logins) {
  const alerts = [];
  const now = Date.now();
  const userMap = new Map(users.map(user => [user.id, user]));
  for (const user of users) {
    const userLogins = logins.filter(item => item.userId === user.id);
    const recentDay = userLogins.filter(item => now - new Date(item.timestamp).getTime() <= 86400000 && item.success);
    const recentWeek = userLogins.filter(item => now - new Date(item.timestamp).getTime() <= 7 * 86400000 && item.success);
    const ips = new Set(recentDay.map(item => item.ip).filter(Boolean));
    const countries = new Set(recentWeek.map(item => item.country).filter(Boolean));
    if (ips.size >= 4) alerts.push({ severity: 'high', userId: user.id, username: user.username, title: 'Đăng nhập từ nhiều IP', message: `${ips.size} IP đăng nhập thành công trong 24 giờ. Nên xác minh lại với nhân viên.` });
    if (countries.size >= 3) alerts.push({ severity: 'high', userId: user.id, username: user.username, title: 'Vị trí đăng nhập thay đổi bất thường', message: `Tài khoản xuất hiện tại ${countries.size} quốc gia trong 7 ngày.` });
    const agents = new Set(recentDay.map(item => item.userAgent).filter(Boolean));
    if (agents.size >= 4) alerts.push({ severity: 'medium', userId: user.id, username: user.username, title: 'Nhiều thiết bị/trình duyệt', message: `${agents.size} dấu vân tay trình duyệt trong 24 giờ.` });
  }
  const failuresByIp = new Map();
  logins.filter(item => !item.success && now - new Date(item.timestamp).getTime() <= 30 * 60000).forEach(item => {
    const key = item.ip || 'unknown';
    failuresByIp.set(key, (failuresByIp.get(key) || 0) + 1);
  });
  for (const [ip, count] of failuresByIp) {
    if (count >= 5) alerts.push({ severity: 'high', userId: null, username: null, title: 'Nhiều lần đăng nhập thất bại', message: `${count} lần thất bại từ IP ${ip} trong 30 phút.` });
  }
  const unauthorizedProxyUsers = new Set();
  proxyCalls.forEach(call => {
    const user = userMap.get(call.ownerId);
    if (user && user.role !== 'admin' && !user.canUseProxy && !unauthorizedProxyUsers.has(user.id)) {
      unauthorizedProxyUsers.add(user.id);
      alerts.push({ severity: 'critical', userId: user.id, username: user.username, title: 'Dùng Proxy khi chưa được cấp quyền', message: `Phát hiện API Proxy lúc ${new Date(call.timestamp).toLocaleString('vi-VN')}.` });
    }
  });
  return alerts.slice(0, 50);
}

app.get('/api/admin/security-overview', requireAdmin, async (req, res) => {
  const calls = await getApiHistory();
  const users = authStore.users.map(publicUser);
  const userId = String(req.query.userId || 'all');
  const proxyCallsAll = calls.filter(call => call.category === 'proxy' || Boolean(call.proxyIpChecked));
  const proxyCalls = userId === 'all' ? proxyCallsAll : proxyCallsAll.filter(call => call.ownerId === userId);
  const allLogins = Array.isArray(authStore.loginHistory) ? authStore.loginHistory : [];
  const logins = userId === 'all' ? allLogins : allLogins.filter(item => item.userId === userId);
  const group = (rows, keyFn) => Array.from(rows.reduce((map, row) => {
    const key = keyFn(row.timestamp);
    map.set(key, (map.get(key) || 0) + 1);
    return map;
  }, new Map()), ([period, count]) => ({ period, count })).sort((a, b) => b.period.localeCompare(a.period));
  const perUser = users.map(user => {
    const userCalls = proxyCallsAll.filter(call => call.ownerId === user.id);
    const userLogins = allLogins.filter(item => item.userId === user.id);
    return {
      user,
      proxyTotal: userCalls.length,
      proxyToday: userCalls.filter(call => String(call.timestamp).slice(0, 10) === new Date().toISOString().slice(0, 10)).length,
      proxyThisWeek: userCalls.filter(call => startOfIsoWeek(call.timestamp) === startOfIsoWeek(new Date())).length,
      loginCount: userLogins.filter(item => item.success).length,
      lastLogin: userLogins.find(item => item.success) || null,
      uniqueLoginIps: new Set(userLogins.filter(item => item.success).map(item => item.ip).filter(Boolean)).size
    };
  });
  res.setHeader('Cache-Control', 'no-store, max-age=0');
  res.json({
    generatedAt: new Date().toISOString(),
    totals: { proxyCalls: proxyCalls.length, successfulLogins: logins.filter(item => item.success).length, failedLogins: logins.filter(item => !item.success).length, uniqueLoginIps: new Set(logins.map(item => item.ip).filter(Boolean)).size },
    daily: group(proxyCalls, timestamp => String(timestamp).slice(0, 10)).slice(0, 31),
    weekly: group(proxyCalls, startOfIsoWeek).slice(0, 12),
    perUser,
    proxyCalls: proxyCalls.slice(0, 300),
    loginHistory: logins.slice(0, 300),
    alerts: buildSecurityAlerts(users, proxyCallsAll, allLogins)
  });
});
app.put('/api/admin/proxy', requireAdmin, async (req, res) => {
  try {
    authStore.settings.proxyUrl = normalizeProxyUrl(req.body?.proxyUrl);
    resetTrafficLensProxySlot();
    await saveAuthStore();
    res.json({ ok: true, configured: Boolean(authStore.settings.proxyUrl) });
  } catch (error) { res.status(400).json({ error: error.message }); }
});
app.put('/api/admin/curl', requireAdmin, (req, res) => {
  const curlCommand = String(req.body?.curlCommand || '').trim();
  if (curlCommand && !parseAdminCurl(curlCommand)) return res.status(400).json({ error: 'Cấu hình không hợp lệ hoặc không chứa header xác thực.' });
  authStore.settings.curlCommand = curlCommand;
  saveAuthStore();
  res.json({ ok: true, configured: Boolean(curlCommand) });
});
function nicheSettings() {
  if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = {};
  if (!authStore.settings.nicheClassification || typeof authStore.settings.nicheClassification !== 'object') {
    authStore.settings.nicheClassification = { deepseekApiKey: '', firecrawlApiKey: '', categories: ['AI', 'Vật lý', 'Khác'], tavilyKeys: [] };
  }
  const settings = authStore.settings.nicheClassification;
  if (!Array.isArray(settings.categories) || !settings.categories.length) settings.categories = ['AI', 'Vật lý', 'Khác'];
  if (!Array.isArray(settings.tavilyKeys)) settings.tavilyKeys = [];
  if (!Array.isArray(settings.firecrawlKeys)) settings.firecrawlKeys = [];
  if (!['deepseek', '9router'].includes(settings.aiProvider)) settings.aiProvider = 'deepseek';
  if (!settings.routerBaseUrl) settings.routerBaseUrl = 'http://127.0.0.1:20128/v1';
  if (!settings.routerModel) settings.routerModel = '';
  if (!settings.systemPrompt) settings.systemPrompt = 'Phân loại website vào đúng một nhãn: {{categories}}. AI: sản phẩm số có AI là chức năng chính; Vật lý: hàng hóa hữu hình; Khác: dịch vụ, tài chính hoặc thiếu dữ liệu. Chỉ trả JSON: {"category":"nhãn"}.';
  if (!settings.userPrompt) settings.userPrompt = 'Domain: {{domain}}\nDữ liệu Tavily Extract:\n{{context}}';
  if (!['deepseek-v4-flash', 'deepseek-v4-pro'].includes(settings.model)) settings.model = 'deepseek-v4-flash';
  return settings;
}
function tavilyUsageMonth() { return new Date().toISOString().slice(0, 7); }
function normalizeTavilyMonthlyUsage(key) {
  const month = tavilyUsageMonth();
  if (key.usageMonth !== month) { key.usageMonth = month; key.usedCount = 0; key.planLimitReached = false; key.planLimitReachedAt = null; key.cooldownUntil = null; key.lastError = ''; }
  return key;
}
function publicTavilyKey(key) { normalizeTavilyMonthlyUsage(key); return { id: key.id, name: key.name, active: key.active !== false, status: key.active === false ? 'disabled' : key.planLimitReached ? 'monthly-limit' : key.cooldownUntil && new Date(key.cooldownUntil).getTime() > Date.now() ? 'cooldown' : 'ready', usedCount: Number(key.usedCount || 0), usageMonth: key.usageMonth, lastUsedAt: key.lastUsedAt || null, cooldownUntil: key.cooldownUntil || null, lastError: key.lastError || '', planLimitReached: Boolean(key.planLimitReached) }; }
function firecrawlUsageMonth() { return new Date().toISOString().slice(0, 7); }
function normalizeFirecrawlMonthlyUsage(key) {
  const month = firecrawlUsageMonth();
  if (key.usageMonth !== month) { key.usageMonth = month; key.usedCount = 0; key.planLimitReached = false; key.planLimitReachedAt = null; key.cooldownUntil = null; key.lastError = ''; }
  // Các phiên bản cũ đánh dấu hết quota bằng cách dò chuỗi lỗi quá rộng. Chỉ giữ cờ
  // khi Firecrawl đã xác nhận bằng HTTP 402 (Payment Required: Insufficient credits).
  if (key.planLimitReached && Number(key.quotaConfirmedStatus || 0) !== 402) {
    key.planLimitReached = false; key.planLimitReachedAt = null; key.cooldownUntil = null;
  }
  return key;
}
function publicFirecrawlKey(key) { normalizeFirecrawlMonthlyUsage(key); return { id: key.id, name: key.name, active: key.active !== false, status: key.active === false ? 'disabled' : key.planLimitReached ? 'monthly-limit' : key.cooldownUntil && new Date(key.cooldownUntil).getTime() > Date.now() ? 'cooldown' : 'ready', usedCount: Number(key.usedCount || 0), usageMonth: key.usageMonth, lastUsedAt: key.lastUsedAt || null, cooldownUntil: key.cooldownUntil || null, lastError: key.lastError || '', planLimitReached: Boolean(key.planLimitReached) }; }
const firecrawlKeyInFlight = new Map();
let firecrawlProxyRotation = 0;
function acquireFirecrawlKey() {
  const now = Date.now();
  const keys = (nicheSettings().firecrawlKeys || []).map(normalizeFirecrawlMonthlyUsage).filter(key => {
    if (key.cooldownUntil && key.lastUsedAt && new Date(key.cooldownUntil).getTime() - new Date(key.lastUsedAt).getTime() > 5 * 60000) key.cooldownUntil = new Date(new Date(key.lastUsedAt).getTime() + 60000).toISOString();
    return key.active !== false && key.secret && !key.planLimitReached && (!key.cooldownUntil || new Date(key.cooldownUntil).getTime() <= now);
  }).sort((a, b) => (Number(a.usedCount || 0) + Number(firecrawlKeyInFlight.get(a.id) || 0)) - (Number(b.usedCount || 0) + Number(firecrawlKeyInFlight.get(b.id) || 0)) || new Date(a.lastUsedAt || 0) - new Date(b.lastUsedAt || 0));
  if (keys.length) { const key = keys[0]; firecrawlKeyInFlight.set(key.id, Number(firecrawlKeyInFlight.get(key.id) || 0) + 1); return key; }
  const legacySecret = nicheSettings().firecrawlApiKey || process.env.FIRECRAWL_API_KEY;
  if (legacySecret && !nicheSettings().firecrawlKeys.length) return { id: 'legacy-firecrawl-key', name: 'Firecrawl Default Key', secret: legacySecret, active: true, isLegacy: true };
  return null;
}
function releaseFirecrawlKey(key) { if (!key || key.isLegacy) return; const count = Math.max(0, Number(firecrawlKeyInFlight.get(key.id) || 0) - 1); if (count) firecrawlKeyInFlight.set(key.id, count); else firecrawlKeyInFlight.delete(key.id); }
function firecrawlAvailabilitySnapshot() {
  const now = Date.now(); const keys = nicheSettings().firecrawlKeys.map(normalizeFirecrawlMonthlyUsage);
  const active = keys.filter(key => key.active !== false && key.secret); const cooldown = active.filter(key => key.cooldownUntil && new Date(key.cooldownUntil).getTime() > now); const monthlyLimit = active.filter(key => key.planLimitReached);
  return { total: keys.length, active: active.length, ready: active.filter(key => !key.planLimitReached && (!key.cooldownUntil || new Date(key.cooldownUntil).getTime() <= now)).length, cooldown: cooldown.length, monthlyLimit: monthlyLimit.length, disabled: keys.length - active.length, inFlight: [...firecrawlKeyInFlight.values()].reduce((sum, count) => sum + Number(count || 0), 0), nextAvailableAt: cooldown.map(key => key.cooldownUntil).sort()[0] || null, usageMonth: firecrawlUsageMonth() };
}
async function waitForFirecrawlKey(maxWaitMs = 90000) {
  const startedAt = Date.now();
  let refreshedExhaustedPool = false;
  while (true) {
    const key = acquireFirecrawlKey(); if (key) return key;
    const now = Date.now();
    const eligible = nicheSettings().firecrawlKeys.map(normalizeFirecrawlMonthlyUsage).filter(item => item.active !== false && item.secret && !item.planLimitReached);
    const configured = nicheSettings().firecrawlKeys.filter(item => item.active !== false && item.secret);
    if (!eligible.length && configured.length && configured.every(item => item.planLimitReached)) {
      if (USE_SUPABASE && !refreshedExhaustedPool) {
        refreshedExhaustedPool = true;
        await syncNicheSettingsFromSupabase();
        continue;
      }
      const error = new Error('Firecrawl đã hết lượt sử dụng trên tất cả API key. Phiên phân loại đã dừng.');
      error.status = 503; error.quotaExhausted = true; error.provider = 'Firecrawl';
      error.firecrawlResponse = { type: 'quota_exhausted', keyPool: firecrawlAvailabilitySnapshot() };
      throw error;
    }
    if (!eligible.length) {
      const legacySecret = nicheSettings().firecrawlApiKey || process.env.FIRECRAWL_API_KEY;
      if (legacySecret) return { id: 'legacy-firecrawl-key', name: 'Firecrawl Default Key', secret: legacySecret, active: true, isLegacy: true };
      const error = new Error('Không có Firecrawl API key khả dụng. Xem chi tiết trạng thái kho key trong Admin.'); error.status = 503; error.firecrawlResponse = { type: 'key_pool_unavailable', keyPool: firecrawlAvailabilitySnapshot() }; throw error;
    }
    const earliest = Math.min(...eligible.map(item => Math.max(now, new Date(item.cooldownUntil || now).getTime())));
    const waitMs = Math.max(250, earliest - now + 150);
    if (Date.now() - startedAt + waitMs > maxWaitMs) { const error = new Error(`Tất cả Firecrawl key đang tạm nghỉ do giới hạn tần suất. Hãy quét lại sau ${Math.max(1, Math.ceil(waitMs / 1000))} giây.`); error.status = 429; error.firecrawlResponse = { type: 'key_pool_cooldown', waitSeconds: Math.max(1, Math.ceil(waitMs / 1000)), keyPool: firecrawlAvailabilitySnapshot() }; throw error; }
    await new Promise(resolve => setTimeout(resolve, waitMs));
  }
}
function isFirecrawlPlanLimitError(error) {
  const status = Number(error?.status || 0);
  if (status) return status === 402;
  return /^payment required:\s*insufficient credits|^insufficient credits(?:\b|$)/i.test(String(error?.message || '').trim());
}
function parseNicheCategories(value) { const items = Array.isArray(value) ? value : String(value || '').split(/[\n,;|]+/); return [...new Set(items.map(item => String(item).trim()).filter(Boolean))].slice(0, 30); }
function renderNichePrompt(template, values) { return String(template || '').replace(/\{\{(categories|domain|context)\}\}/g, (_, key) => String(values[key] || '')); }
function normalizeOpenAiBaseUrl(value) {
  const parsed = new URL(String(value || 'http://127.0.0.1:20128/v1').trim());
  if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error('9Router Base URL phải dùng http hoặc https.');
  return parsed.toString().replace(/\/$/, '');
}
function openAiChatUrl(baseUrl) { return `${normalizeOpenAiBaseUrl(baseUrl).replace(/\/chat\/completions$/i, '')}/chat/completions`; }
function parseAiJsonContent(content) {
  const text = String(content || '').trim();
  try { return JSON.parse(text); } catch (_) {}
  const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text)?.[1];
  if (fenced) { try { return JSON.parse(fenced.trim()); } catch (_) {} }
  const object = /\{[\s\S]*\}/.exec(text)?.[0];
  if (object) { try { return JSON.parse(object); } catch (_) {} }
  return {};
}
const tavilyKeyInFlight = new Map();
let tavilyProxyRotation = 0;
function acquireTavilyKey() {
  const now = Date.now();
  const keys = nicheSettings().tavilyKeys.map(normalizeTavilyMonthlyUsage).filter(key => {
    if (key.cooldownUntil && key.lastUsedAt && new Date(key.cooldownUntil).getTime() - new Date(key.lastUsedAt).getTime() > 5 * 60000) key.cooldownUntil = new Date(new Date(key.lastUsedAt).getTime() + 60000).toISOString();
    return key.active !== false && key.secret && !key.planLimitReached && (!key.cooldownUntil || new Date(key.cooldownUntil).getTime() <= now);
  }).sort((a, b) => (Number(a.usedCount || 0) + Number(tavilyKeyInFlight.get(a.id) || 0)) - (Number(b.usedCount || 0) + Number(tavilyKeyInFlight.get(b.id) || 0)) || new Date(a.lastUsedAt || 0) - new Date(b.lastUsedAt || 0));
  const key = keys[0]; if (key) tavilyKeyInFlight.set(key.id, Number(tavilyKeyInFlight.get(key.id) || 0) + 1); return key;
}
function releaseTavilyKey(key) { if (!key) return; const count = Math.max(0, Number(tavilyKeyInFlight.get(key.id) || 0) - 1); if (count) tavilyKeyInFlight.set(key.id, count); else tavilyKeyInFlight.delete(key.id); }
function tavilyAvailabilitySnapshot() {
  const now = Date.now(); const keys = nicheSettings().tavilyKeys.map(normalizeTavilyMonthlyUsage);
  const active = keys.filter(key => key.active !== false && key.secret); const cooldown = active.filter(key => key.cooldownUntil && new Date(key.cooldownUntil).getTime() > now); const monthlyLimit = active.filter(key => key.planLimitReached);
  return { total: keys.length, active: active.length, ready: active.filter(key => !key.planLimitReached && (!key.cooldownUntil || new Date(key.cooldownUntil).getTime() <= now)).length, cooldown: cooldown.length, monthlyLimit: monthlyLimit.length, disabled: keys.length - active.length, inFlight: [...tavilyKeyInFlight.values()].reduce((sum, count) => sum + Number(count || 0), 0), nextAvailableAt: cooldown.map(key => key.cooldownUntil).sort()[0] || null, usageMonth: tavilyUsageMonth() };
}
function isTavilyPoolExhausted(keys) {
  const active = (Array.isArray(keys) ? keys : []).filter(item => item.active !== false && item.secret);
  return active.length > 0 && active.every(item => item.planLimitReached === true);
}
async function reconfirmTavilyPoolExhausted() {
  if (USE_SUPABASE) await syncNicheSettingsFromSupabase();
  const active = nicheSettings().tavilyKeys.map(normalizeTavilyMonthlyUsage).filter(item => item.active !== false && item.secret);
  if (!active.length || !isTavilyPoolExhausted(active)) return false;
  const checks = await Promise.all(active.map(key => verifyTavilyKeyQuota(key)));
  let changed = false;
  active.forEach((key, index) => {
    if (checks[index]?.confirmed === true || checks[index]?.status !== 200) return;
    key.planLimitReached = false;
    key.planLimitReachedAt = null;
    key.cooldownUntil = null;
    changed = true;
  });
  if (changed) await saveAuthStore();
  if (active.every((key, index) => checks[index]?.confirmed === true)) return true;
  if (checks.some(check => check?.status === 200 && check.confirmed === false)) return false;
  const error = new Error('Chưa thể xác minh lại lượt Tavily của kho key. Hãy thử tiếp tục phiên sau ít phút.');
  error.status = 503;
  throw error;
}
async function syncNicheSettingsFromSupabase() {
  if (!USE_SUPABASE) return nicheSettings();
  const remoteAuth = await supabaseStateRequest('auth', undefined, 12000);
  const remoteNiche = remoteAuth?.settings?.nicheClassification;
  if (!remoteNiche || !Array.isArray(remoteNiche.tavilyKeys)) throw new Error('Supabase chưa trả về cấu hình kho Tavily key hợp lệ.');
  if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = {};
  authStore.settings.nicheClassification = remoteNiche;
  return nicheSettings();
}
function tavilyErrorText(payload, fallback = '') {
  const candidates = [payload?.error?.message, payload?.error?.detail, payload?.error?.error, payload?.message, payload?.detail, payload?.error];
  for (const value of candidates) {
    if (typeof value === 'string' && value.trim()) return value.trim();
    if (Array.isArray(value) && value.length) return value.map(item => tavilyErrorText(item, '') || (typeof item === 'string' ? item : '')).filter(Boolean).join('; ');
    if (value && typeof value === 'object') { const nested = tavilyErrorText(value, ''); if (nested) return nested; try { const json = JSON.stringify(value); if (json && json !== '{}') return json; } catch (_) {} }
  }
  return fallback;
}
function nicheHttpError(status, payload, response) { const retryAfter = Number(response?.headers?.get('retry-after')) || 0; const detail = tavilyErrorText(payload, `Tavily HTTP ${status}${retryAfter ? ` · thử lại sau ${retryAfter} giây` : ''}`); const error = new Error(detail); error.status = status; error.retryAfter = retryAfter; error.tavilyResponse = { status, statusText: response?.statusText || '', retryAfter, requestId: payload?.request_id || null, body: payload, keyPool: tavilyAvailabilitySnapshot() }; return error; }
function isTavilyPlanLimitError(error) {
  const status = Number(error?.status || 0);
  // Tavily dùng 429 cho giới hạn tốc độ tạm thời (phải tôn trọng Retry-After),
  // còn 432/433 mới là lỗi hạn mức/credit. Không được loại key vĩnh viễn chỉ vì 429.
  if (status) return status === 432 || status === 433;
  return /exceeds your plan(?:'s)? set usage limit|upgrade your plan|support@tavily\.com|insufficient credits?|monthly limit|plan limit/i.test(String(error?.message || ''));
}
function isApiQuotaExhaustedError(error) {
  // Chỉ dừng toàn bộ phiên khi lớp quản lý key đã xác nhận cả pool thực sự hết lượt.
  // Không dò chuỗi lỗi ghép vì câu "Tavily lỗi (...); Firecrawl ... credit" từng bị
  // nhận nhầm thành Tavily hết hạn mức dù toàn bộ Tavily key vẫn còn ở trạng thái ready.
  return error?.quotaExhausted === true && ['Tavily', 'Firecrawl'].includes(error?.provider);
}
async function verifyTavilyKeyQuota(key, dispatcher = null) {
  try {
    const response = await trackedFetch('https://api.tavily.com/usage', {
      method: 'GET',
      headers: { Authorization: `Bearer ${decryptAdminSetting(key.secret)}`, Connection: 'close' },
      dispatcher,
      signal: AbortSignal.timeout(20000)
    }, null, { category: 'external', direction: 'Server -> Tavily usage verification', tavilyKeyId: key.id, tavilyKeyName: key.name });
    const payload = await response.json().catch(() => ({}));
    if (!response.ok) return { confirmed: false, status: response.status, payload };
    const { keyUsage, keyLimit, planUsage, planLimit, paygoUsage, paygoLimit, exhausted } = tavilyUsageQuotaState(payload);
    key.reportedUsage = { keyUsage, keyLimit, planUsage, planLimit, paygoUsage, paygoLimit, checkedAt: new Date().toISOString() };
    return { confirmed: exhausted, status: response.status, payload };
  } catch (error) {
    return { confirmed: false, status: Number(error?.status || 0), error: error?.message || String(error) };
  }
}
function tavilyUsageQuotaState(payload) {
  const keyUsage = Number(payload?.key?.usage);
  const keyLimit = Number(payload?.key?.limit);
  const planUsage = Number(payload?.account?.plan_usage);
  const planLimit = Number(payload?.account?.plan_limit);
  const paygoUsage = Number(payload?.account?.paygo_usage);
  const paygoLimit = Number(payload?.account?.paygo_limit);
  const keyExhausted = Number.isFinite(keyUsage) && Number.isFinite(keyLimit) && keyLimit > 0 && keyUsage >= keyLimit;
  const planExhausted = Number.isFinite(planUsage) && Number.isFinite(planLimit) && planLimit > 0 && planUsage >= planLimit;
  const paygoAvailable = Number.isFinite(paygoLimit) && paygoLimit > 0 && (!Number.isFinite(paygoUsage) || paygoUsage < paygoLimit);
  return { keyUsage, keyLimit, planUsage, planLimit, paygoUsage, paygoLimit, keyExhausted, planExhausted, paygoAvailable, exhausted: keyExhausted || (planExhausted && !paygoAvailable) };
}
function isNonRetryableFetchError(error) { return error?.nonRetryable === true || /failed to fetch|error fetching content|không tải được nội dung|không phân giải được dns|không thể kết nối trước khi gọi api/i.test(String(error?.message || '')); }
async function waitForTavilyKey(maxWaitMs = 90000) {
  const startedAt = Date.now();
  let refreshedExhaustedPool = false;
  while (true) {
    const key = acquireTavilyKey(); if (key) return key;
    const now = Date.now();
    const eligible = nicheSettings().tavilyKeys.map(normalizeTavilyMonthlyUsage).filter(item => item.active !== false && item.secret && !item.planLimitReached);
    const configured = nicheSettings().tavilyKeys.filter(item => item.active !== false && item.secret);
    if (!eligible.length && isTavilyPoolExhausted(configured)) {
      // Vercel có nhiều instance. Một instance cũ có thể vẫn giữ cờ hết quota dù Admin
      // vừa reset kho key ở instance khác. Luôn đối chiếu Supabase trước khi dừng toàn bộ job.
      if (USE_SUPABASE && !refreshedExhaustedPool) {
        refreshedExhaustedPool = true;
        await syncNicheSettingsFromSupabase();
        continue;
      }
      if (!await reconfirmTavilyPoolExhausted()) continue;
      const error = new Error('Tavily đã hết lượt sử dụng trên tất cả API key. Phiên phân loại đã dừng.');
      error.status = 503; error.quotaExhausted = true; error.provider = 'Tavily';
      error.tavilyResponse = { type: 'quota_exhausted', keyPool: tavilyAvailabilitySnapshot() };
      throw error;
    }
    if (!eligible.length) { const error = new Error('Không có Tavily API key khả dụng. Xem chi tiết trạng thái kho key.'); error.status = 503; error.tavilyResponse = { type: 'key_pool_unavailable', keyPool: tavilyAvailabilitySnapshot() }; throw error; }
    const earliest = Math.min(...eligible.map(item => Math.max(now, new Date(item.cooldownUntil || now).getTime())));
    const waitMs = Math.max(250, earliest - now + 150);
    if (Date.now() - startedAt + waitMs > maxWaitMs) { const error = new Error(`Tất cả Tavily key đang tạm nghỉ do giới hạn tần suất. Hãy quét lại sau ${Math.max(1, Math.ceil(waitMs / 1000))} giây.`); error.status = 429; error.tavilyResponse = { type: 'key_pool_cooldown', waitSeconds: Math.max(1, Math.ceil(waitMs / 1000)), keyPool: tavilyAvailabilitySnapshot() }; throw error; }
    await new Promise(resolve => setTimeout(resolve, waitMs));
  }
}
async function tavilyResearchDomain(domain, attemptedKeyIds = new Set()) {
  const key = await waitForTavilyKey();
  attemptedKeyIds.add(key.id);
  const proxyUrl = authStore.settings.proxyUrl;
  if (!proxyUrl) { releaseTavilyKey(key); throw new Error('Chưa cấu hình Proxy chung. Mọi request Tavily bắt buộc phải đi qua Proxy xoay.'); }
  const proxyRotation = ++tavilyProxyRotation;
  const dispatcher = getProxyDispatcher(proxyUrl, false);
  let failure = null;
  let retryWithNextKey = false;
  try {
      const response = await trackedFetch('https://api.tavily.com/extract', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${decryptAdminSetting(key.secret)}`, Connection: 'close' }, dispatcher, body: JSON.stringify({ urls: [`https://${domain}`], extract_depth: 'advanced', format: 'markdown', include_images: false }), signal: AbortSignal.timeout(45000) }, null, { category: 'external', direction: 'Server -> Tavily domain extract qua Proxy xoay', connection: `Proxy xoay riêng · request #${proxyRotation}`, proxyRotation, tavilyKeyId: key.id, tavilyKeyName: key.name, domain });
      const payload = await response.json().catch(() => ({})); if (!response.ok) throw nicheHttpError(response.status, payload, response);
      if (!Array.isArray(payload.results) || !payload.results.length) { const failed = payload.failed_results?.[0]; const detail = tavilyErrorText(failed, 'Tavily không trích xuất được nội dung website.'); const error = new Error(/error fetching content/i.test(detail) ? `Tavily không tải được nội dung từ https://${domain} (website có thể chặn trình thu thập, lỗi DNS/SSL hoặc tạm thời không phản hồi).` : detail); error.status = 422; error.tavilyResponse = { type: 'extract_failed', domain, failedResults: payload.failed_results || [] }; throw error; }
      key.usedCount = Number(key.usedCount || 0) + 1; key.lastUsedAt = new Date().toISOString(); key.lastError = ''; key.cooldownUntil = null; await saveAuthStore(); return { key: key.name, payload };
  } catch (error) {
      const planLimit = isTavilyPlanLimitError(error);
      const quotaCheck = planLimit ? await verifyTavilyKeyQuota(key, dispatcher) : null;
      const confirmedPlanLimit = quotaCheck?.confirmed === true;
      key.lastError = error.status === 422 ? '' : error.message; key.lastUsedAt = new Date().toISOString();
      if (confirmedPlanLimit) {
        // Không loại vĩnh viễn key dựa trên một phản hồi. Nút test có thể vẫn thành công và
        // hạn mức/rate limit phía Tavily có thể thay đổi; chỉ nghỉ key rồi luân chuyển key khác.
        key.planLimitReached = true;
        key.planLimitReachedAt = new Date().toISOString();
        key.cooldownUntil = null;
        retryWithNextKey = true;
      }
      else if (planLimit) {
        // 432/433 nhưng endpoint Usage chưa xác nhận hết lượt: không được loại key vĩnh viễn.
        key.cooldownUntil = new Date(Date.now() + 60000).toISOString();
        retryWithNextKey = true;
      }
      else if ([401, 403].includes(error.status)) {
        key.active = false;
        retryWithNextKey = true;
      }
      else if (error.status === 429) {
        key.cooldownUntil = new Date(Date.now() + Math.max(60, Number(error.retryAfter) || 60) * 1000).toISOString();
        retryWithNextKey = true;
      }
      else if (!error.status || error.status >= 500) {
        key.cooldownUntil = new Date(Date.now() + 60000).toISOString();
        retryWithNextKey = true;
      }
      await saveAuthStore();
      failure = error;
  } finally { if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {}); releaseTavilyKey(key); }
  // Key vừa bị Tavily xác nhận hết giới hạn gói đã bị loại khỏi pool.
  // Lấy key khả dụng kế tiếp và chạy lại đúng domain ngay trong tác vụ hiện tại.
  if (retryWithNextKey) {
    const activeKeys = nicheSettings().tavilyKeys.filter(item => item.active !== false && item.secret);
    if (isTavilyPoolExhausted(activeKeys)) return tavilyResearchDomain(domain, attemptedKeyIds);
    const configuredKeyCount = nicheSettings().tavilyKeys.filter(item => item.secret).length;
    if (attemptedKeyIds.size < configuredKeyCount) return tavilyResearchDomain(domain, attemptedKeyIds);
  }
  throw failure;
}
function tavilyContext(payload) {
  // Ưu tiên title để tiết kiệm token; nếu Tavily không có title thì vẫn tận
  // dụng content/raw_content thay vì báo thiếu dữ liệu sai.
  return (Array.isArray(payload?.results) ? payload.results : [])
    .map(row => String(row.title || row.content || row.raw_content || row.rawContent || row.rawcontent || '').trim())
    .filter(Boolean)
    .join('\n---\n')
    .slice(0, 12000);
}
async function firecrawlResearchDomain(domain, scrapeOptions = {}, attemptedKeyIds = new Set()) {
  const key = await waitForFirecrawlKey();
  attemptedKeyIds.add(key.id);
  const rawKey = key.isLegacy ? (decryptAdminSetting(key.secret) || key.secret) : decryptAdminSetting(key.secret);
  const proxyUrl = authStore.settings.proxyUrl;
  const dispatcher = proxyUrl ? getProxyDispatcher(proxyUrl, false) : null;
  const proxyRotation = ++firecrawlProxyRotation;
  let failure = null;
  let retryWithNextKey = false;
  try {
    const response = await trackedFetch('https://api.firecrawl.dev/v2/scrape', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${rawKey}`
      },
      dispatcher,
      body: JSON.stringify({
        url: `https://${domain}`,
        formats: Array.isArray(scrapeOptions.formats) && scrapeOptions.formats.length ? scrapeOptions.formats : ['markdown'],
        onlyMainContent: scrapeOptions.onlyMainContent ?? true
      }),
      signal: AbortSignal.timeout(45000)
    }, null, {
      category: 'external',
      direction: 'Server -> Firecrawl domain scrape qua Proxy xoay',
      connection: `Proxy xoay riêng · request #${proxyRotation}`,
      proxyRotation,
      firecrawlKeyId: key.id,
      firecrawlKeyName: key.name,
      domain
    });
    const payload = await response.json().catch(() => ({}));
    if (!response.ok || payload.success === false) {
      const msg = payload.error || payload.message || `Firecrawl HTTP ${response.status}`;
      const err = new Error(msg); err.status = response.status; throw err;
    }
    if (!key.isLegacy) {
      key.usedCount = Number(key.usedCount || 0) + 1; key.lastUsedAt = new Date().toISOString(); key.lastError = ''; key.planLimitReached = false; key.planLimitReachedAt = null; key.quotaConfirmedStatus = null; key.cooldownUntil = null; await saveAuthStore();
    }
    return { key: key.name, payload };
  } catch (error) {
    const isPlanLimit = isFirecrawlPlanLimitError(error);
    if (key.isLegacy && isPlanLimit) {
      error.quotaExhausted = true;
      error.provider = 'Firecrawl';
      error.message = `Firecrawl đã hết lượt sử dụng. Phiên phân loại đã dừng. (${error.message})`;
    }
    if (!key.isLegacy) {
      key.lastError = error.status === 422 ? '' : error.message; key.lastUsedAt = new Date().toISOString();
      if (isPlanLimit) {
        key.planLimitReached = true; key.planLimitReachedAt = new Date().toISOString(); key.quotaConfirmedStatus = 402; key.cooldownUntil = null; retryWithNextKey = true;
      } else if ([401, 403].includes(error.status)) {
        key.active = false;
        key.quotaConfirmedStatus = null;
        retryWithNextKey = true;
      } else if (error.status === 429) {
        key.cooldownUntil = new Date(Date.now() + 60000).toISOString(); retryWithNextKey = true;
      } else if (!error.status || error.status >= 500) {
        key.cooldownUntil = new Date(Date.now() + 60000).toISOString(); retryWithNextKey = true;
      }
      await saveAuthStore();
    }
    failure = error;
  } finally {
    if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {});
    releaseFirecrawlKey(key);
  }
  if (retryWithNextKey) {
    const configuredKeyCount = nicheSettings().firecrawlKeys.filter(item => item.secret).length;
    if (attemptedKeyIds.size < configuredKeyCount) return firecrawlResearchDomain(domain, scrapeOptions, attemptedKeyIds);
  }
  throw failure;
}
function firecrawlContext(payload) {
  const data = payload?.data || {};
  const meta = data.metadata || {};
  const parts = [];
  if (meta.title) parts.push(`Title: ${meta.title}`);
  const description = meta.description || meta.ogDescription || data.product?.description || data.summary || '';
  if (description) parts.push(`Description: ${description}`);
  if (data.markdown) parts.push(`Content:\n${data.markdown}`);
  return parts.join('\n\n').slice(0, 12000);
}
app.get('/api/niche/config', requireAdmin, async (req, res) => { try { if (USE_SUPABASE) await syncNicheSettingsFromSupabase(); const settings = nicheSettings(); const routerKey = decryptAdminSetting(settings.routerApiKey) || process.env.NINE_ROUTER_API_KEY || ''; res.json({ categories: settings.categories, model: settings.model, availableModels: ['deepseek-v4-flash', 'deepseek-v4-pro'], aiProvider: settings.aiProvider, routerBaseUrl: settings.routerBaseUrl, routerModel: settings.routerModel, routerApiKeyConfigured: Boolean(routerKey), systemPrompt: settings.systemPrompt, userPrompt: settings.userPrompt, endpoints: { deepseek: 'https://api.deepseek.com/chat/completions', router: openAiChatUrl(settings.routerBaseUrl), tavilyExtract: 'https://api.tavily.com/extract', firecrawlScrape: 'https://api.firecrawl.dev/v2/scrape' }, tavilyKeys: settings.tavilyKeys.map(publicTavilyKey), firecrawlKeys: (settings.firecrawlKeys || []).map(publicFirecrawlKey), deepseekConfigured: Boolean(decryptAdminSetting(settings.deepseekApiKey) || process.env.DEEPSEEK_API_KEY), firecrawlConfigured: Boolean(settings.firecrawlKeys.some(k => k.active !== false) || decryptAdminSetting(settings.firecrawlApiKey) || process.env.FIRECRAWL_API_KEY) }); } catch (error) { res.status(502).json({ error: `Không đồng bộ được kho API key: ${error.message}` }); } });
app.put('/api/niche/config', requireAdmin, async (req, res) => {
  const settings = nicheSettings(); const categories = parseNicheCategories(req.body?.categories); if (!categories.length) return res.status(400).json({ error: 'Cần ít nhất một ngách phân loại.' });
  const supplied = String(req.body?.deepseekApiKey || '').trim(); const firecrawlApiKey = String(req.body?.firecrawlApiKey || '').trim(); const routerApiKey = String(req.body?.routerApiKey || '').trim(); const systemPrompt = String(req.body?.systemPrompt || '').trim(); const userPrompt = String(req.body?.userPrompt || '').trim();
  if (!systemPrompt || !userPrompt) return res.status(400).json({ error: 'System Prompt và User Prompt không được để trống.' });
  const model = String(req.body?.model || ''); if (!['deepseek-v4-flash', 'deepseek-v4-pro'].includes(model)) return res.status(400).json({ error: 'Model DeepSeek không được hỗ trợ.' });
  const aiProvider = String(req.body?.aiProvider || 'deepseek'); if (!['deepseek', '9router'].includes(aiProvider)) return res.status(400).json({ error: 'Nguồn AI không hợp lệ.' });
  const routerBaseUrl = normalizeOpenAiBaseUrl(req.body?.routerBaseUrl); const routerModel = String(req.body?.routerModel || '').trim().slice(0, 160); if (aiProvider === '9router' && !routerModel) return res.status(400).json({ error: 'Cần nhập model 9Router.' });
  settings.categories = categories; settings.model = model; settings.aiProvider = aiProvider; settings.routerBaseUrl = routerBaseUrl; settings.routerModel = routerModel; settings.systemPrompt = systemPrompt.slice(0, 10000); settings.userPrompt = userPrompt.slice(0, 10000); if (supplied) settings.deepseekApiKey = encryptAdminSetting(supplied); if (firecrawlApiKey) settings.firecrawlApiKey = encryptAdminSetting(firecrawlApiKey); if (routerApiKey) settings.routerApiKey = encryptAdminSetting(routerApiKey); await saveAuthStore(); res.json({ ok: true, categories, model, aiProvider, routerBaseUrl, routerModel, deepseekConfigured: Boolean(decryptAdminSetting(settings.deepseekApiKey) || process.env.DEEPSEEK_API_KEY), firecrawlConfigured: Boolean(settings.firecrawlKeys.some(k => k.active !== false) || decryptAdminSetting(settings.firecrawlApiKey) || process.env.FIRECRAWL_API_KEY), routerApiKeyConfigured: Boolean(decryptAdminSetting(settings.routerApiKey) || process.env.NINE_ROUTER_API_KEY) });
});
app.post('/api/niche/firecrawl/test', requireAdmin, async (req, res) => {
  try {
    const apiKey = String(req.body?.apiKey || '').trim() || (acquireFirecrawlKey() ? decryptAdminSetting(acquireFirecrawlKey().secret) : null) || decryptAdminSetting(nicheSettings().firecrawlApiKey) || process.env.FIRECRAWL_API_KEY;
    if (!apiKey) return res.status(400).json({ error: 'Chưa có Firecrawl API Key để kiểm tra.' });
    const domain = 'attackshark.com';
    const proxyUrl = authStore.settings.proxyUrl;
    const dispatcher = proxyUrl ? getProxyDispatcher(proxyUrl, false) : null;
    const options = {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
      body: JSON.stringify({ url: `https://${domain}`, formats: ['markdown'], onlyMainContent: true }),
      signal: AbortSignal.timeout(45000)
    };
    if (dispatcher) options.dispatcher = dispatcher;
    const response = await trackedFetch('https://api.firecrawl.dev/v2/scrape', options, null, { category: 'external', direction: 'Server -> Firecrawl test (attackshark.com)' });
    if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {});
    const payload = await response.json().catch(() => ({}));
    if (!response.ok || payload.success === false) return res.status(response.status >= 400 && response.status < 600 ? response.status : 502).json({ error: payload.error || payload.message || `Firecrawl HTTP ${response.status}` });
    const meta = payload.data?.metadata || {};
    const title = meta.title || 'N/A';
    const description = meta.description || meta.ogDescription || payload.data?.product?.description || payload.data?.summary || 'N/A';
    res.json({ ok: true, message: `Firecrawl API Key hợp lệ. Scrape attackshark.com thành công: "${title}" - Description: ${description.slice(0, 150)}...` });
  } catch (error) { res.status(502).json({ error: error.message || 'Không kiểm tra được Firecrawl API Key.' }); }
});
app.post('/api/niche/firecrawl-keys/import', requireAdmin, async (req, res) => {
  if (USE_SUPABASE) await syncNicheSettingsFromSupabase();
  const settings = nicheSettings();
  if (!Array.isArray(settings.firecrawlKeys)) settings.firecrawlKeys = [];
  const lines = String(req.body?.keys || '').split(/[\r\n]+/);
  let added = 0, skipped = 0, reactivated = 0;
  for (const rawLine of lines) {
    const line = rawLine.trim(); if (!line) continue;
    const parts = line.split(/[,;\t]+/).map(item => item.trim()).filter(Boolean);
    let rawKey = '', nameCell = '';
    for (const part of parts) {
      if (/^fc-[A-Za-z0-9_-]{8,}/.test(part) || /^[A-Za-z0-9_-]{16,}$/.test(part)) rawKey = part;
      else if (!nameCell) nameCell = part;
    }
    if (!rawKey && parts.length === 1 && line.length >= 10) rawKey = line;
    if (!rawKey) { skipped++; continue; }
    const existing = settings.firecrawlKeys.find(item => decryptAdminSetting(item.secret) === rawKey);
    if (existing) {
      if (existing.active === false) { existing.active = true; reactivated++; } else skipped++;
      continue;
    }
    settings.firecrawlKeys.push({ id: crypto.randomUUID(), name: (nameCell || `Firecrawl ${settings.firecrawlKeys.length + 1}`).slice(0, 80), secret: encryptAdminSetting(rawKey), active: true, usedCount: 0, planLimitReached: false, planLimitReachedAt: null, usageMonth: firecrawlUsageMonth(), lastUsedAt: null, cooldownUntil: null, lastError: '' });
    added++;
  }
  await saveAuthStore();
  res.status(201).json({ ok: true, added, skipped, reactivated, firecrawlKeys: settings.firecrawlKeys.map(publicFirecrawlKey) });
});
app.patch('/api/niche/firecrawl-keys', requireAdmin, async (req, res) => {
  if (req.body?.action !== 'enable-all') return res.status(400).json({ error: 'Thao tác Firecrawl hàng loạt không hợp lệ.' });
  const keys = nicheSettings().firecrawlKeys; let enabled = 0;
  keys.forEach(key => { if (key.active === false) enabled++; key.active = true; key.planLimitReached = false; key.planLimitReachedAt = null; key.quotaConfirmedStatus = null; key.cooldownUntil = null; key.lastError = ''; });
  await saveAuthStore();
  res.json({ ok: true, enabled, total: keys.length, firecrawlKeys: keys.map(publicFirecrawlKey) });
});
app.patch('/api/niche/firecrawl-keys/:id', requireAdmin, async (req, res) => {
  const key = nicheSettings().firecrawlKeys.find(item => item.id === req.params.id);
  if (!key) return res.status(404).json({ error: 'Không tìm thấy Firecrawl key.' });
  if (req.body?.active !== undefined) key.active = Boolean(req.body.active);
  if (req.body?.reset) { key.usedCount = 0; key.planLimitReached = false; key.planLimitReachedAt = null; key.quotaConfirmedStatus = null; key.usageMonth = firecrawlUsageMonth(); key.cooldownUntil = null; key.lastError = ''; }
  await saveAuthStore();
  res.json({ ok: true, key: publicFirecrawlKey(key) });
});
app.post('/api/niche/firecrawl-keys/:id/test', requireAdmin, async (req, res) => {
  const key = nicheSettings().firecrawlKeys.find(item => item.id === req.params.id);
  if (!key) return res.status(404).json({ error: 'Không tìm thấy Firecrawl key.' });
  const testUrl = 'attackshark.com';
  try {
    const response = await fetch('https://api.firecrawl.dev/v2/scrape', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${decryptAdminSetting(key.secret)}` }, body: JSON.stringify({ url: `https://${testUrl}`, formats: ['markdown'], onlyMainContent: true }), signal: AbortSignal.timeout(45000) });
    const payload = await response.json().catch(() => ({}));
    key.lastUsedAt = new Date().toISOString();
    if (!response.ok || payload.success === false) {
      const errorText = payload.error || payload.message || `Firecrawl HTTP ${response.status}`;
      key.lastError = errorText;
      if (response.status === 429) key.cooldownUntil = new Date(Date.now() + 60000).toISOString();
      await saveAuthStore();
      return res.status(response.status >= 400 && response.status < 600 ? response.status : 502).json({ ok: false, error: errorText, testUrl });
    }
    key.usedCount = Number(key.usedCount || 0) + 1; key.active = true; key.planLimitReached = false; key.planLimitReachedAt = null; key.quotaConfirmedStatus = null; key.lastError = ''; key.cooldownUntil = null; await saveAuthStore();
    const meta = payload.data?.metadata || {};
    res.json({ ok: true, message: `Key còn hoạt động; Firecrawl Scrape thành công ${testUrl}: "${meta.title || 'N/A'}".`, status: response.status, testUrl });
  } catch (error) { key.lastError = error.message; key.lastUsedAt = new Date().toISOString(); await saveAuthStore(); res.status(502).json({ ok: false, error: `Không gọi được Firecrawl Scrape: ${error.message}`, testUrl }); }
});
app.delete('/api/niche/firecrawl-keys/:id', requireAdmin, async (req, res) => {
  const settings = nicheSettings(); const index = settings.firecrawlKeys.findIndex(item => item.id === req.params.id);
  if (index < 0) return res.status(404).json({ error: 'Không tìm thấy Firecrawl key.' });
  settings.firecrawlKeys.splice(index, 1); await saveAuthStore(); res.json({ ok: true });
});

// --- Affiliate Network Finder (Firecrawl Search v2) ---
function affiliateSettings() {
  if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = {};
  if (!authStore.settings.affiliateFinder || typeof authStore.settings.affiliateFinder !== 'object') {
    authStore.settings.affiliateFinder = {
      networks: ['Awin', 'GoAffPro', 'UpPromote', 'ShareASale', 'Impact', 'PartnerStack', 'CJ Affiliate', 'Everflow', 'Tapfiliate', 'ClickBank', 'Rakuten'],
      fallbackTemplates: [
        '{{project}} affiliate program goaff',
        '{{project}} affiliate program uppromote',
        '{{project}} affiliate program shareasale',
        '{{project}} affiliate program awin',
        '{{project}} affiliate program impact',
        '{{project}} affiliate program partnerstack'
      ]
    };
  }
  const settings = authStore.settings.affiliateFinder;
  if (!Array.isArray(settings.networks) || !settings.networks.length) {
    settings.networks = ['Awin', 'GoAffPro', 'UpPromote', 'ShareASale', 'Impact', 'PartnerStack', 'CJ Affiliate', 'Everflow', 'Tapfiliate', 'ClickBank', 'Rakuten'];
  }
  if (!Array.isArray(settings.fallbackTemplates) || !settings.fallbackTemplates.length) {
    settings.fallbackTemplates = [
      '{{project}} affiliate program goaff',
      '{{project}} affiliate program uppromote',
      '{{project}} affiliate program shareasale',
      '{{project}} affiliate program awin',
      '{{project}} affiliate program impact',
      '{{project}} affiliate program partnerstack'
    ];
  }
  return settings;
}

async function firecrawlSearch(query, limit = 10, context = null, excludedSources = new Set()) {
  let source = 'admin';
  if (context) {
    const available = [];
    const readyUserKeys = context.userKeys.filter(item => !context.disabledUserKeyIds.has(item.id));
    if (readyUserKeys.length && !excludedSources.has('user')) available.push('user');
    if (context.useAdmin && !excludedSources.has('admin')) available.push('admin');
    if (!available.length) throw new Error('Không còn Firecrawl API key khả dụng trong các nguồn đã chọn.');
    source = available.length > 1 ? available[(context.rotation++) % available.length] : available[0];
  }
  const readyUserKeys = context?.userKeys?.filter(item => !context.disabledUserKeyIds.has(item.id)) || [];
  const selectedUserKey = readyUserKeys.length ? readyUserKeys[(context.userRotation++) % readyUserKeys.length] : null;
  const key = source === 'user'
    ? { ...selectedUserKey, isLegacy: true, isUserKey: true }
    : await waitForFirecrawlKey();
  const rawKey = key.isUserKey ? key.secret : key.isLegacy ? (decryptAdminSetting(key.secret) || key.secret) : decryptAdminSetting(key.secret);
  const proxyUrl = authStore.settings.proxyUrl;
  const dispatcher = proxyUrl ? getProxyDispatcher(proxyUrl, false) : null;
  const proxyRotation = ++firecrawlProxyRotation;
  let failure = null;
  let retryWithNextKey = false;
  try {
    const response = await trackedFetch('https://api.firecrawl.dev/v2/search', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${rawKey}`
      },
      dispatcher,
      body: JSON.stringify({
        query,
        limit: Math.max(1, Math.min(20, limit)),
        sources: ['web']
      }),
      signal: AbortSignal.timeout(45000)
    }, null, {
      category: 'external',
      direction: 'Server -> Firecrawl search API v2 qua Proxy xoay',
      connection: `Proxy xoay riêng · request #${proxyRotation}`,
      proxyRotation,
      firecrawlKeyId: key.id,
      firecrawlKeyName: key.name,
      firecrawlKeySource: source,
      query
    });
    const payload = await response.json().catch(() => ({}));
    if (!response.ok || payload.success === false) {
      const msg = payload.error || payload.message || `Firecrawl Search HTTP ${response.status}`;
      const err = new Error(msg); err.status = response.status; throw err;
    }
    if (!key.isLegacy) {
      key.usedCount = Number(key.usedCount || 0) + 1; key.lastUsedAt = new Date().toISOString(); key.lastError = ''; key.cooldownUntil = null; await saveAuthStore();
    } else if (key.isUserKey) {
      const stored = authStore.users.find(item => item.id === context.userId)?.personalFirecrawlKeys?.find(item => item.id === key.id);
      if (stored) { stored.lastUsedAt = new Date().toISOString(); stored.lastError = ''; await saveAuthStore(); }
    }
    return { key: key.name, payload };
  } catch (error) {
    if (key.isUserKey) {
      context.disabledUserKeyIds.add(key.id);
      const stored = authStore.users.find(item => item.id === context.userId)?.personalFirecrawlKeys?.find(item => item.id === key.id);
      if (stored) { stored.lastUsedAt = new Date().toISOString(); stored.lastError = error.message || 'Firecrawl lỗi'; await saveAuthStore(); }
    }
    if (!key.isLegacy) {
      const isPlanLimit = /credit|quota|monthly limit|plan limit|usage limit|exceed/i.test(String(error.message || ''));
      key.lastError = error.status === 422 ? '' : error.message; key.lastUsedAt = new Date().toISOString();
      if (isPlanLimit) {
        key.planLimitReached = false; key.cooldownUntil = new Date(Date.now() + 60000).toISOString(); retryWithNextKey = true;
      } else if ([401, 403].includes(error.status)) {
        key.active = false;
      } else if (error.status === 429) {
        key.cooldownUntil = new Date(Date.now() + 60000).toISOString(); retryWithNextKey = true;
      }
      await saveAuthStore();
    }
    failure = error;
  } finally {
    if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {});
    releaseFirecrawlKey(key);
  }
  if (context && source === 'user') {
    if (context.userKeys.some(item => !context.disabledUserKeyIds.has(item.id))) return firecrawlSearch(query, limit, context, excludedSources);
    excludedSources.add('user');
    if (context.useAdmin) return firecrawlSearch(query, limit, context, excludedSources);
  }
  if (context && source === 'admin' && context.userKeys.some(item => !context.disabledUserKeyIds.has(item.id))) {
    excludedSources.add('admin');
    return firecrawlSearch(query, limit, context, excludedSources);
  }
  if (retryWithNextKey) return firecrawlSearch(query, limit, context, excludedSources);
  throw failure;
}

function affiliateComparableText(value) {
  return String(value || '')
    .normalize('NFD')
    .replace(/[\u0300-\u036f]/g, '')
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '');
}

function affiliateProjectKeys(project) {
  const raw = String(project || '').trim().replace(/^https?:\/\//i, '').replace(/\/.*$/, '').replace(/^www\./i, '');
  const hostname = raw.split(':')[0];
  const labels = hostname.split('.').filter(Boolean);
  const ignoredLabels = new Set(['www', 'shop', 'store', 'app', 'affiliate', 'affiliates', 'partners', 'partner', 'com', 'net', 'org', 'io', 'ai', 'co', 'uk', 'us', 'vn', 'de', 'fr', 'jp', 'au']);
  const keys = new Set([affiliateComparableText(raw), affiliateComparableText(hostname)]);
  const meaningfulLabels = labels.filter(label => !ignoredLabels.has(label.toLowerCase()));
  meaningfulLabels.forEach(label => {
    const normalized = affiliateComparableText(label);
    if (normalized.length >= 3) keys.add(normalized);
  });
  return [...keys].filter(key => key.length >= 3);
}

function affiliatePriority(value) {
  const normalized = affiliateComparableText(value);
  if (normalized.includes('goaffpro') || normalized.includes('goaff')) return 0;
  if (normalized.includes('uppromote')) return 1;
  return 2;
}

function affiliateProjectMatches(fullText, projectKeys) {
  const comparableFullText = affiliateComparableText(fullText);
  return projectKeys.some(key => comparableFullText.includes(key));
}

function affiliateStripHtml(value) {
  return String(value || '')
    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&(?:nbsp|amp|quot|#39|lt|gt);/gi, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

async function affiliateFetchHtml(url, maxLength = 1500000) {
  const proxyUrl = authStore.settings.proxyUrl;
  const dispatcher = proxyUrl ? getProxyDispatcher(proxyUrl, false) : null;
  try {
    const response = await trackedFetch(url, {
      headers: { 'User-Agent': 'Mozilla/5.0 (compatible; AffiliateDiscovery/1.0)' },
      dispatcher,
      redirect: 'follow',
      signal: AbortSignal.timeout(20000)
    }, null, { category: 'external', direction: 'Server -> Website affiliate discovery', target: url });
    if (!response.ok) throw new Error(`Website HTTP ${response.status}`);
    const contentType = String(response.headers.get('content-type') || '');
    if (contentType && !/html|text/i.test(contentType)) return { html: '', finalUrl: response.url || url };
    return { html: (await response.text()).slice(0, maxLength), finalUrl: response.url || url };
  } finally {
    if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {});
  }
}

async function discoverAffiliateFromHomepage(project, configuredNetworks) {
  if (!String(project).includes('.')) return { evidence: [] };
  const homepageUrl = `https://${String(project).replace(/^www\./i, '')}`;
  let homepage = { html: '', finalUrl: homepageUrl };
  try {
    homepage = await affiliateFetchHtml(homepageUrl);
  } catch (error) {
    console.warn(`[Affiliate Homepage Discovery] ${homepageUrl}: ${error.message}`);
  }

  const candidates = [];
  const addCandidate = (href, label = '') => {
    try {
      const absoluteUrl = new URL(String(href || '').trim(), homepage.finalUrl || homepageUrl).href;
      if (/^https?:/i.test(absoluteUrl) && !candidates.some(item => item.url === absoluteUrl)) candidates.push({ url: absoluteUrl, label: affiliateStripHtml(label) });
    } catch (_) {}
  };
  const anchorPattern = /<a\b[^>]*href\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
  let anchor;
  while ((anchor = anchorPattern.exec(homepage.html)) && candidates.length < 12) {
    const href = String(anchor[1] || '').trim();
    const label = affiliateStripHtml(anchor[2]);
    const context = `${href} ${label}`;
    if (!/(affiliate|affiliates|partner\s*program|referral|ambassador|influencer|creator\s*program|goaff|uppromote|shareasale|awin|impact\.com)/i.test(context)) continue;
    addCandidate(href, label);
  }

  // Direct HTTP may be blocked or may receive HTML without JS-rendered footer.
  // Firecrawl fallback explicitly includes the whole page and its discovered links.
  if (!candidates.length) {
    try {
      const scraped = await firecrawlResearchDomain(String(project).replace(/^www\./i, ''), {
        formats: ['markdown', 'links'],
        onlyMainContent: false
      });
      const data = scraped.payload?.data || {};
      const markdown = String(data.markdown || '');
      const markdownLinkPattern = /\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/gi;
      let markdownLink;
      while ((markdownLink = markdownLinkPattern.exec(markdown)) && candidates.length < 12) {
        const context = `${markdownLink[1]} ${markdownLink[2]}`;
        if (/(affiliate|affiliates|partner\s*program|referral|ambassador|influencer|creator\s*program|goaff|uppromote|shareasale|awin|impact\.com)/i.test(context)) addCandidate(markdownLink[2], markdownLink[1]);
      }
      const links = Array.isArray(data.links) ? data.links : [];
      for (const link of links) {
        const href = typeof link === 'string' ? link : String(link?.url || link?.href || '');
        const label = typeof link === 'object' ? String(link?.text || link?.title || '') : '';
        if (/(affiliate|affiliates|partner|referral|ambassador|goaff|uppromote|shareasale|awin|impact\.com)/i.test(`${href} ${label}`)) addCandidate(href, label);
        if (candidates.length >= 12) break;
      }
    } catch (error) {
      console.warn(`[Affiliate Firecrawl Homepage Fallback] ${homepageUrl}: ${error.message}`);
    }
  }

  const syntheticItems = candidates.map(candidate => ({
    url: candidate.url,
    title: candidate.label || 'Affiliate link trên website chính thức',
    description: `${project} ${candidate.label} Link được phát hiện trên website chính thức ${homepage.finalUrl || homepageUrl}`
  }));

  // Branded/custom affiliate portals may hide the NET from their URL. Follow those
  // candidate links once and inspect page HTML, scripts and redirect destination.
  for (const candidate of candidates.slice(0, 6)) {
    if (matchAffiliateNetworks({ data: { web: [syntheticItems.find(item => item.url === candidate.url)] } }, configuredNetworks, project).networks.length) continue;
    try {
      const target = await affiliateFetchHtml(candidate.url, 800000);
      syntheticItems.push({
        url: target.finalUrl || candidate.url,
        title: candidate.label || 'Affiliate registration portal',
        description: `${project} ${candidate.label} ${target.html}`
      });
    } catch (error) {
      console.warn(`[Affiliate Portal Discovery] ${candidate.url}: ${error.message}`);
    }
  }

  return {
    evidence: syntheticItems.map(item => ({
      url: item.url,
      title: item.title,
      description: item.description,
      source: 'official-homepage'
    }))
  };
}

function matchAffiliateNetworks(searchPayload, configuredNetworks, project) {
  const rawItems = Array.isArray(searchPayload?.data?.web) ? searchPayload.data.web : (Array.isArray(searchPayload?.data) ? searchPayload.data : []);
  const matches = [];
  const foundNetworksSet = new Set();
  const projectKeys = affiliateProjectKeys(project);

  for (const item of rawItems) {
    const url = String(item.url || item.link || '').trim();
    const title = String(item.title || '').trim();
    const description = String(item.description || item.snippet || item.markdown || '').trim();
    const fullText = `${url} ${title} ${description}`.toLowerCase();
    const projectMatched = affiliateProjectMatches(fullText, projectKeys);

    // A NET homepage often appears merely because its name is in the fallback query.
    // Only accept evidence when this same result also identifies the requested project.
    if (!projectMatched) continue;

    for (const net of configuredNetworks) {
      const netClean = String(net).trim();
      if (!netClean) continue;
      const netKey = netClean.toLowerCase();
      let matched = false;

      if (netKey === 'goaffpro' || netKey === 'goaff') {
        matched = /goaffpro|goaff/i.test(fullText);
      } else if (netKey === 'cj affiliate' || netKey === 'cj') {
        matched = /\bcj\.com\b|cj affiliate|commission junction/i.test(fullText);
      } else if (netKey === 'rakuten' || netKey === 'linkshare') {
        matched = /rakuten|linkshare/i.test(fullText);
      } else {
        matched = fullText.includes(netKey);
      }

      if (matched) {
        foundNetworksSet.add(netClean);
        matches.push({
          network: netClean,
          title: title || netClean,
          url: url || `https://${netKey}.com`,
          snippet: description.slice(0, 200)
        });
      }
    }
  }

  const uniqueMatches = [];
  const seen = new Set();
  for (const m of matches) {
    const key = `${m.network}:${m.url}`;
    if (!seen.has(key)) {
      seen.add(key);
      uniqueMatches.push(m);
    }
  }

  return { networks: Array.from(foundNetworksSet), matches: uniqueMatches };
}

function affiliateSearchEvidence(searchPayload, query) {
  const rows = Array.isArray(searchPayload?.data?.web) ? searchPayload.data.web : (Array.isArray(searchPayload?.data) ? searchPayload.data : []);
  return rows.slice(0, 10).map(row => ({
    url: String(row.url || row.link || '').trim(),
    title: String(row.title || '').trim(),
    description: String(row.description || row.snippet || row.markdown || '').trim(),
    source: 'firecrawl-search',
    query
  })).filter(row => row.url);
}

function affiliateCanonicalUrl(value) {
  try {
    const parsed = new URL(String(value || '').trim());
    parsed.hash = '';
    return parsed.toString().replace(/\/$/, '').toLowerCase();
  } catch (_) {
    return String(value || '').trim().replace(/\/$/, '').toLowerCase();
  }
}

function affiliateIsGenericNetworkPage(value) {
  try {
    const parsed = new URL(String(value || '').trim());
    const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
    const path = parsed.pathname.replace(/\/+$/, '').toLowerCase() || '/';
    const exactGenericPages = new Set([
      'goaffpro.com/',
      'uppromote.com/',
      'marketplace.uppromote.com/',
      'awin.com/',
      'shareasale.com/',
      'impact.com/',
      'partnerstack.com/',
      'cj.com/',
      'rakutenadvertising.com/'
    ]);
    if (exactGenericPages.has(`${host}${path}`)) return true;
    const genericHosts = new Set(['goaffpro.com', 'uppromote.com', 'marketplace.uppromote.com', 'awin.com', 'shareasale.com', 'impact.com', 'partnerstack.com', 'cj.com', 'rakutenadvertising.com']);
    const genericPaths = new Set(['/about', '/features', '/pricing', '/login', '/register', '/signup', '/marketplace', '/affiliate-marketing', '/solutions']);
    return genericHosts.has(host) && genericPaths.has(path);
  } catch (_) {
    return false;
  }
}

function affiliateNetworkTerms(network) {
  const key = affiliateComparableText(network);
  if (key === 'goaffpro' || key === 'goaff') return ['goaffpro', 'goaff'];
  if (key === 'uppromote') return ['uppromote'];
  if (key === 'awin') return ['awin.com', 'ui.awin.com', 'awin affiliate', 'awin partner'];
  if (key === 'shareasale') return ['shareasale', 'share a sale'];
  if (key === 'impact') return ['impact.com', 'impact radius', 'impact partnership cloud', 'impact affiliate'];
  if (key === 'cjaffiliate' || key === 'cj') return ['cj.com', 'cj affiliate', 'commission junction'];
  if (key === 'rakuten' || key === 'linkshare') return ['rakuten advertising', 'linkshare'];
  if (key === 'partnerstack') return ['partnerstack'];
  if (key === 'everflow') return ['everflow'];
  if (key === 'tapfiliate') return ['tapfiliate'];
  if (key === 'clickbank') return ['clickbank'];
  return [String(network || '').trim().toLowerCase()].filter(Boolean);
}

function affiliateEvidenceSupportsNetwork(evidence, network) {
  const text = `${evidence?.url || ''} ${evidence?.title || ''} ${evidence?.description || ''}`.toLowerCase();
  return affiliateNetworkTerms(network).some(term => text.includes(term));
}

function affiliateEvidenceExcerpt(value, configuredNetworks) {
  const text = String(value || '').replace(/\0/g, ' ');
  if (text.length <= 1200) return text;
  const lower = text.toLowerCase();
  const terms = [...new Set(configuredNetworks.flatMap(affiliateNetworkTerms).concat(['affiliate', 'partner program', 'referral']))];
  const chunks = [text.slice(0, 350)];
  for (const term of terms) {
    const index = lower.indexOf(term);
    if (index < 0) continue;
    chunks.push(text.slice(Math.max(0, index - 180), Math.min(text.length, index + term.length + 360)));
    if (chunks.join('\n---\n').length >= 1500) break;
  }
  return chunks.join('\n---\n').slice(0, 1600);
}

function affiliateIsUntrustedDirectory(value) {
  try {
    const host = new URL(String(value || '')).hostname.toLowerCase().replace(/^www\./, '');
    return new Set(['affilitizer.com', 'affiliateprogramdb.com', 'affiliate-program.amazon.com']).has(host);
  } catch (_) {
    return false;
  }
}

async function analyzeAffiliateEvidenceWithDeepSeek(project, configuredNetworks, rawEvidence) {
  const settings = nicheSettings();
  const apiKey = decryptAdminSetting(settings.deepseekApiKey) || process.env.DEEPSEEK_API_KEY || '';
  if (!apiKey) {
    const error = new Error('Admin chưa cấu hình DeepSeek API key để phân tích Affiliate.');
    error.status = 503;
    throw error;
  }

  const evidence = [];
  const seenUrls = new Map();
  for (const row of rawEvidence) {
    const url = String(row?.url || '').trim();
    if (!url || affiliateIsGenericNetworkPage(url) || affiliateIsUntrustedDirectory(url)) continue;
    const canonical = affiliateCanonicalUrl(url);
    const existingIndex = seenUrls.get(canonical);
    if (existingIndex !== undefined) {
      const existing = evidence[existingIndex];
      const candidateDescription = affiliateEvidenceExcerpt(row.description, configuredNetworks);
      if (candidateDescription.length > existing.description.length) existing.description = candidateDescription;
      if (row.source === 'official-homepage') existing.source = 'official-homepage';
      if (row.query && !existing.queries.includes(row.query)) existing.queries.push(row.query);
      continue;
    }
    if (evidence.length >= 80) break;
    seenUrls.set(canonical, evidence.length);
    evidence.push({
      id: evidence.length + 1,
      url,
      title: String(row.title || '').slice(0, 300),
      description: affiliateEvidenceExcerpt(row.description, configuredNetworks),
      source: row.source || 'firecrawl-search',
      queries: row.query ? [row.query] : []
    });
  }

  if (!evidence.length) return { networks: [], matches: [], evidenceCount: 0 };
  const prompt = [
    `Project/domain to verify: ${project}`,
    `Allowed affiliate networks (return names exactly as written): ${JSON.stringify(configuredNetworks)}`,
    'Analyze every evidence item below and identify ALL valid affiliate networks used by this project.',
    'Important rules:',
    '- A network name appearing only because it was included in a search query is NOT evidence.',
    '- Confirm that the evidence actually refers to this project; brand aliases and shortened brand names are allowed when context supports the relationship.',
    '- Evidence from source official-homepage is strong because its link was extracted from the project website.',
    '- Reject generic network homepages, marketplace landing pages, pricing/features/login/register pages, directories, aggregators, SEO articles, and third-party listings that do not establish a direct relationship between this project and the network.',
    '- Examples of invalid evidence by themselves: https://goaffpro.com/, https://marketplace.uppromote.com/, or a generic Awin/Impact/ShareASale landing page with no project identity.',
    '- A network-owned merchant/profile/register page is valid only when its URL, title, content, or merchant identity clearly refers to this project or a well-supported brand alias.',
    '- Classify sourceLevel as A for a direct official-homepage link or an official network merchant/register/profile page; B for an official project page/article explicitly announcing its affiliate program; C for directories, aggregators, SEO articles, generic pages, or uncertain evidence.',
    '- Return a network only when sourceLevel is A or B and confidence is at least 80.',
    '- Return every valid network, not only the first or strongest one.',
    '- Every returned network MUST have at least one evidence ID whose URL/title/description specifically supports that exact network. Do not attach an Awin-only page as evidence for ShareASale, Impact, or another network.',
    '- The selected evidence must contain a recognizable fingerprint of that exact network in its URL, title, description, branding, script, or redirect destination. A generic word such as "impact" in ordinary prose is not an Impact.com fingerprint.',
    '- Never invent a network or URL. Use only allowed network names and evidence IDs supplied below.',
    '- If none are valid, return an empty matches array.',
    'Return only JSON in this exact structure:',
    '{"matches":[{"network":"Allowed network name","evidenceIds":[1],"sourceLevel":"A","confidence":95,"reason":"short reason"}]}',
    `Evidence: ${JSON.stringify(evidence)}`
  ].join('\n');

  const response = await trackedFetch('https://api.deepseek.com/chat/completions', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
    body: JSON.stringify({
      model: settings.model,
      thinking: { type: 'disabled' },
      messages: [
        { role: 'system', content: 'You are a strict affiliate-network evidence analyst. Return valid JSON only and follow the supplied evidence constraints.' },
        { role: 'user', content: prompt }
      ],
      response_format: { type: 'json_object' },
      temperature: 0,
      max_tokens: 1600,
      stream: false
    }),
    signal: AbortSignal.timeout(90000)
  }, null, { category: 'external', direction: 'Server -> DeepSeek affiliate evidence analysis', project, evidenceCount: evidence.length });
  const payload = await response.json().catch(() => ({}));
  if (!response.ok) {
    const error = new Error(payload?.error?.message || payload?.error || `DeepSeek HTTP ${response.status}`);
    error.status = response.status;
    throw error;
  }

  const parsed = parseAiJsonContent(payload.choices?.[0]?.message?.content);
  const allowedNetworks = new Map(configuredNetworks.map(network => [affiliateComparableText(network), network]));
  const evidenceById = new Map(evidence.map(item => [item.id, item]));
  const matches = [];
  for (const selection of Array.isArray(parsed.matches) ? parsed.matches : []) {
    const network = allowedNetworks.get(affiliateComparableText(selection?.network));
    if (!network) continue;
    const sourceLevel = String(selection?.sourceLevel || '').toUpperCase();
    const confidence = Number(selection?.confidence || 0);
    if (!['A', 'B'].includes(sourceLevel) || confidence < 80) continue;
    const selectedEvidence = [...new Set(Array.isArray(selection.evidenceIds) ? selection.evidenceIds.map(Number) : [])]
      .map(id => evidenceById.get(id))
      .filter(item => item && affiliateEvidenceSupportsNetwork(item, network));
    for (const item of selectedEvidence) {
      matches.push({ network, title: item.title || network, url: item.url, snippet: item.description.slice(0, 200), source: item.source, sourceLevel, confidence, reason: String(selection.reason || '').slice(0, 300) });
    }
  }
  const uniqueMatches = matches.filter((match, index, rows) => rows.findIndex(row => row.network === match.network && affiliateCanonicalUrl(row.url) === affiliateCanonicalUrl(match.url)) === index);
  const networks = [...new Set(uniqueMatches.map(match => match.network))].sort((a, b) => affiliatePriority(a) - affiliatePriority(b));
  return { networks, matches: uniqueMatches, evidenceCount: evidence.length, model: payload.model || settings.model };
}

async function findProjectAffiliateNetworks(projectNameInput, username = null, firecrawlContext = null) {
  const project = String(projectNameInput || '').trim().replace(/^https?:\/\//i, '').replace(/\/.*$/, '');
  if (!project) {
    const error = new Error('Tên dự án hoặc domain không hợp lệ.');
    error.status = 400;
    throw error;
  }
  const settings = affiliateSettings();
  const networks = settings.networks
    .map((network, index) => ({ network, index }))
    .sort((a, b) => affiliatePriority(a.network) - affiliatePriority(b.network) || a.index - b.index)
    .map(item => item.network);
  const fallbackTemplates = settings.fallbackTemplates
    .map((template, index) => ({ template, index }))
    .sort((a, b) => affiliatePriority(a.template) - affiliatePriority(b.template) || a.index - b.index)
    .map(item => item.template);
  const queriesUsed = [];
  const allEvidence = [];
  const searchErrors = [];
  let successfulSearches = 0;

  const primaryQuery = `${project} affiliate program`;
  queriesUsed.push(primaryQuery);

  try {
    const search1 = await firecrawlSearch(primaryQuery, 10, firecrawlContext);
    successfulSearches++;
    allEvidence.push(...affiliateSearchEvidence(search1.payload, primaryQuery));
  } catch (err) {
    searchErrors.push(`${primaryQuery}: ${err.message}`);
    console.error(`[Affiliate Finder Primary Error] Query "${primaryQuery}":`, err.message);
  }

  const homepageResult = await discoverAffiliateFromHomepage(project, networks);
  allEvidence.push(...(homepageResult.evidence || []));

  if (fallbackTemplates.length > 0) {
    for (const template of fallbackTemplates) {
      const fallbackQuery = template.replace(/\{\{project\}\}/g, project);
      if (!fallbackQuery || queriesUsed.includes(fallbackQuery)) continue;
      queriesUsed.push(fallbackQuery);
      try {
        const searchFallback = await firecrawlSearch(fallbackQuery, 10, firecrawlContext);
        successfulSearches++;
        allEvidence.push(...affiliateSearchEvidence(searchFallback.payload, fallbackQuery));
      } catch (err) {
        searchErrors.push(`${fallbackQuery}: ${err.message}`);
        console.error(`[Affiliate Finder Fallback Error] Query "${fallbackQuery}":`, err.message);
      }
    }
  }

  if (!successfulSearches && !allEvidence.length) {
    const error = new Error(searchErrors[0] || 'Firecrawl Search không trả về kết quả hợp lệ.');
    error.status = 502;
    throw error;
  }

  const aiAnalysis = await analyzeAffiliateEvidenceWithDeepSeek(project, networks, allEvidence);
  const sortedMatches = aiAnalysis.matches
    .sort((a, b) => Number(b.source === 'official-homepage') - Number(a.source === 'official-homepage') || affiliatePriority(a.network) - affiliatePriority(b.network));
  const primaryEvidence = aiAnalysis.networks.map(network => sortedMatches.find(match => match.network === network)).filter(Boolean);
  const primaryKeys = new Set(primaryEvidence.map(match => `${match.network}:${affiliateCanonicalUrl(match.url)}`));
  const displayedMatches = [...primaryEvidence, ...sortedMatches.filter(match => !primaryKeys.has(`${match.network}:${affiliateCanonicalUrl(match.url)}`))].slice(0, 15);

  return {
    ok: true,
    id: crypto.randomUUID(),
    project,
    networks: aiAnalysis.networks,
    matches: displayedMatches,
    analyzedBy: 'deepseek',
    aiModel: aiAnalysis.model,
    evidenceCount: aiAnalysis.evidenceCount,
    queriesUsed,
    searchedBy: username,
    createdAt: new Date().toISOString(),
    updatedAt: new Date().toISOString()
  };
}

const affiliateJobs = new Map();
function publicAffiliateJob(job, since = 0) {
  return {
    id: job.id,
    status: job.status,
    total: job.projects.length,
    completed: job.completed,
    stopped: job.stopped,
    results: job.results.slice(Math.max(0, since)),
    nextCursor: job.results.length,
    firecrawlSource: job.firecrawlSource || 'admin',
    createdAt: job.createdAt,
    finishedAt: job.finishedAt || null
  };
}
async function runAffiliateJob(job, concurrency, username, firecrawlContext = null) {
  let cursor = 0;
  const workers = Array.from({ length: Math.min(concurrency, job.projects.length) }, async () => {
    while (cursor < job.projects.length && !job.stopped) {
      const inputOrder = cursor++;
      const project = job.projects[inputOrder];
      try {
        const row = await findProjectAffiliateNetworks(project, username, firecrawlContext);
        job.results.push({ inputOrder, row: { ...row, inputOrder } });
      } catch (error) {
        job.results.push({
          inputOrder,
          row: {
            project,
            inputOrder,
            networks: [],
            matches: [],
            error: error.message || 'Không thể tìm kiếm thông tin Affiliate.',
            updatedAt: new Date().toISOString()
          }
        });
      }
      job.completed++;
      job.updatedAt = new Date().toISOString();
    }
  });
  await Promise.all(workers);
  job.status = job.stopped ? 'stopped' : 'completed';
  job.finishedAt = new Date().toISOString();
  setTimeout(() => affiliateJobs.delete(job.id), 6 * 60 * 60 * 1000).unref?.();
}

function affiliateFirecrawlContext(user, requestedSource = '') {
  if (!Array.isArray(user.personalFirecrawlKeys)) user.personalFirecrawlKeys = [];
  const legacyKey = decryptAdminSetting(user?.personalFirecrawlApiKey);
  if (legacyKey && !user.personalFirecrawlKeys.length) {
    user.personalFirecrawlKeys.push({ id: crypto.randomUUID(), name: 'Firecrawl cá nhân 1', secret: user.personalFirecrawlApiKey, active: true, addedAt: new Date().toISOString() });
    user.personalFirecrawlApiKey = '';
  }
  const userKeys = user.personalFirecrawlKeys.filter(item => item.active !== false && decryptAdminSetting(item.secret)).map(item => ({ id: item.id, name: item.name, secret: decryptAdminSetting(item.secret) }));
  const canUseAdmin = user?.role === 'admin' || Boolean(user?.canUseAdminFirecrawl);
  const fallbackSource = ['user', 'admin', 'both'].includes(user?.affiliateFirecrawlSource)
    ? user.affiliateFirecrawlSource
    : canUseAdmin ? 'admin' : 'user';
  const source = ['user', 'admin', 'both'].includes(requestedSource) ? requestedSource : fallbackSource;
  if ((source === 'admin' || source === 'both') && !canUseAdmin) {
    const error = new Error('Tài khoản chưa được Admin cấp quyền dùng Firecrawl API key của Admin.'); error.status = 403; throw error;
  }
  if ((source === 'user' || source === 'both') && !userKeys.length) {
    const error = new Error('Chưa cấu hình Firecrawl API key cá nhân trong Settings.'); error.status = 400; throw error;
  }
  return { source, userId: user.id, userKeys: source === 'admin' ? [] : userKeys, useAdmin: source !== 'user', rotation: 0, userRotation: 0, disabledUserKeyIds: new Set() };
}

function publicPersonalFirecrawlKey(key) {
  return { id: key.id, name: key.name, active: key.active !== false, addedAt: key.addedAt || null, lastUsedAt: key.lastUsedAt || null, lastError: key.lastError || '' };
}

app.get('/api/affiliate/firecrawl-settings', requireAffiliate, (req, res) => {
  const user = req.user;
  if (!Array.isArray(user.personalFirecrawlKeys)) user.personalFirecrawlKeys = [];
  const legacyKey = decryptAdminSetting(user.personalFirecrawlApiKey);
  if (legacyKey && !user.personalFirecrawlKeys.length) {
    user.personalFirecrawlKeys.push({ id: crypto.randomUUID(), name: 'Firecrawl cá nhân 1', secret: user.personalFirecrawlApiKey, active: true, addedAt: new Date().toISOString() });
    user.personalFirecrawlApiKey = '';
    saveAuthStore().catch(() => {});
  }
  const personalKeys = user.personalFirecrawlKeys.filter(item => decryptAdminSetting(item.secret));
  const canUseAdmin = user.role === 'admin' || Boolean(user.canUseAdminFirecrawl);
  const source = ['user', 'admin', 'both'].includes(user.affiliateFirecrawlSource) ? user.affiliateFirecrawlSource : canUseAdmin ? 'admin' : 'user';
  res.setHeader('Cache-Control', 'no-store, private');
  res.json({
    canUseAdmin,
    personalKeyConfigured: personalKeys.some(item => item.active !== false),
    personalKeys: personalKeys.map(publicPersonalFirecrawlKey),
    source
  });
});

app.put('/api/affiliate/firecrawl-settings', requireAffiliate, async (req, res) => {
  const user = req.user;
  const source = String(req.body?.source || '').trim();
  const rawKeys = Array.isArray(req.body?.apiKeys) ? req.body.apiKeys : String(req.body?.apiKeys || req.body?.apiKey || '').split(/[\r\n,;]+/);
  if (!['user', 'admin', 'both'].includes(source)) return res.status(400).json({ error: 'Nguồn Firecrawl không hợp lệ.' });
  if ((source === 'admin' || source === 'both') && user.role !== 'admin' && !user.canUseAdminFirecrawl) return res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền dùng Firecrawl API key của Admin.' });
  if (!Array.isArray(user.personalFirecrawlKeys)) user.personalFirecrawlKeys = [];
  if (req.body?.clearPersonalKey === true) { user.personalFirecrawlApiKey = ''; user.personalFirecrawlKeys = []; }
  else {
    for (const apiKey of rawKeys.map(item => String(item).trim()).filter(Boolean)) {
      if (apiKey.length < 10) return res.status(400).json({ error: 'Có Firecrawl API key cá nhân không hợp lệ.' });
      if (user.personalFirecrawlKeys.some(item => decryptAdminSetting(item.secret) === apiKey)) continue;
      user.personalFirecrawlKeys.push({ id: crypto.randomUUID(), name: `Firecrawl cá nhân ${user.personalFirecrawlKeys.length + 1}`, secret: encryptAdminSetting(apiKey), active: true, addedAt: new Date().toISOString(), lastUsedAt: null, lastError: '' });
    }
  }
  if (req.body?.clearPersonalKey !== true && (source === 'user' || source === 'both') && !user.personalFirecrawlKeys.some(item => item.active !== false && decryptAdminSetting(item.secret))) return res.status(400).json({ error: 'Hãy nhập ít nhất một Firecrawl API key cá nhân trước khi chọn nguồn này.' });
  user.affiliateFirecrawlSource = source;
  await saveAuthStore();
  res.json({ ok: true, source, canUseAdmin: user.role === 'admin' || Boolean(user.canUseAdminFirecrawl), personalKeyConfigured: user.personalFirecrawlKeys.some(item => item.active !== false), personalKeys: user.personalFirecrawlKeys.map(publicPersonalFirecrawlKey) });
});

app.patch('/api/affiliate/firecrawl-settings/keys/:id', requireAffiliate, async (req, res) => {
  const key = (req.user.personalFirecrawlKeys || []).find(item => item.id === req.params.id);
  if (!key) return res.status(404).json({ error: 'Không tìm thấy Firecrawl key cá nhân.' });
  if (req.body?.active !== undefined) key.active = Boolean(req.body.active);
  await saveAuthStore(); res.json({ ok: true, key: publicPersonalFirecrawlKey(key) });
});
app.delete('/api/affiliate/firecrawl-settings/keys/:id', requireAffiliate, async (req, res) => {
  const keys = req.user.personalFirecrawlKeys || []; const index = keys.findIndex(item => item.id === req.params.id);
  if (index < 0) return res.status(404).json({ error: 'Không tìm thấy Firecrawl key cá nhân.' });
  keys.splice(index, 1); await saveAuthStore(); res.json({ ok: true });
});

app.get('/api/affiliate/config', requireAdmin, (req, res) => {
  const settings = affiliateSettings();
  res.json({ networks: settings.networks, fallbackTemplates: settings.fallbackTemplates });
});

app.put('/api/affiliate/config', requireAdmin, async (req, res) => {
  const settings = affiliateSettings();
  const rawNetworks = Array.isArray(req.body?.networks) ? req.body.networks : String(req.body?.networks || '').split(/[\r\n,;]+/);
  const rawTemplates = Array.isArray(req.body?.fallbackTemplates) ? req.body.fallbackTemplates : String(req.body?.fallbackTemplates || '').split(/[\r\n]+/);

  const networks = [...new Set(rawNetworks.map(item => String(item).trim()).filter(Boolean))].slice(0, 100);
  const fallbackTemplates = [...new Set(rawTemplates.map(item => String(item).trim()).filter(Boolean))].slice(0, 20);

  if (!networks.length) return res.status(400).json({ error: 'Cần ít nhất một nền tảng Affiliate (NET).' });

  settings.networks = networks;
  settings.fallbackTemplates = fallbackTemplates;
  await saveAuthStore();
  res.json({ ok: true, networks, fallbackTemplates });
});

app.post('/api/affiliate/test-search', requireAdmin, async (req, res) => {
  try {
    const project = String(req.body?.project || 'attackshark').trim();
    const result = await findProjectAffiliateNetworks(project, req.user?.username);
    res.json({ ok: true, result });
  } catch (error) {
    res.status(502).json({ error: error.message || 'Không thể tìm kiếm Affiliate.' });
  }
});

app.post('/api/affiliate/find', requireAffiliate, async (req, res) => {
  try {
    const context = affiliateFirecrawlContext(req.user, String(req.body?.firecrawlSource || ''));
    res.json(await findProjectAffiliateNetworks(req.body?.project, req.user?.username || null, context));
  } catch (error) {
    res.status(error.status && error.status >= 400 && error.status < 600 ? error.status : 502).json({ error: error.message || 'Không thể tìm kiếm Affiliate.' });
  }
});

app.post('/api/affiliate/jobs', requireAffiliate, (req, res) => {
  const rawProjects = Array.isArray(req.body?.projects) ? req.body.projects : String(req.body?.projects || '').split(/[\r\n]+/);
  const projects = [...new Set(rawProjects.map(p => String(p).trim().replace(/^https?:\/\//i, '').replace(/\/.*$/, '')).filter(Boolean))].slice(0, 10000);
  if (!projects.length) return res.status(400).json({ error: 'Cần ít nhất một dự án hợp lệ.' });
  const concurrency = Math.max(1, Math.min(5, Number(req.body?.concurrency) || 2));
  let firecrawlContext;
  try { firecrawlContext = affiliateFirecrawlContext(req.user, String(req.body?.firecrawlSource || '')); }
  catch (error) { return res.status(error.status || 400).json({ error: error.message }); }
  const job = {
    id: crypto.randomUUID(),
    ownerId: req.user.id,
    projects,
    results: [],
    completed: 0,
    stopped: false,
    status: 'running',
    createdAt: new Date().toISOString(),
    firecrawlSource: firecrawlContext.source
  };
  affiliateJobs.set(job.id, job);
  runAffiliateJob(job, concurrency, req.user?.username || null, firecrawlContext).catch(error => {
    job.status = 'failed';
    job.error = error.message;
    job.finishedAt = new Date().toISOString();
  });
  res.status(202).json(publicAffiliateJob(job));
});

app.get('/api/affiliate/jobs/:id', requireAffiliate, (req, res) => {
  const job = affiliateJobs.get(req.params.id);
  if (!job || job.ownerId !== req.user.id) return res.status(404).json({ error: 'Không tìm thấy phiên tìm kiếm Affiliate.' });
  res.json(publicAffiliateJob(job, Number(req.query.since) || 0));
});

app.post('/api/affiliate/jobs/:id/stop', requireAffiliate, (req, res) => {
  const job = affiliateJobs.get(req.params.id);
  if (!job || job.ownerId !== req.user.id) return res.status(404).json({ error: 'Không tìm thấy phiên tìm kiếm Affiliate.' });
  job.stopped = true;
  job.updatedAt = new Date().toISOString();
  res.json(publicAffiliateJob(job));
});
app.post('/api/niche/deepseek/test', requireAdmin, async (req, res) => {
  try {
    const apiKey = String(req.body?.apiKey || '').trim() || decryptAdminSetting(nicheSettings().deepseekApiKey) || process.env.DEEPSEEK_API_KEY;
    if (!apiKey) return res.status(400).json({ error: 'Chưa có DeepSeek API key để kiểm tra.' });
    const model = String(req.body?.model || nicheSettings().model); if (!['deepseek-v4-flash', 'deepseek-v4-pro'].includes(model)) return res.status(400).json({ error: 'Model DeepSeek không được hỗ trợ.' }); const response = await fetch('https://api.deepseek.com/chat/completions', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` }, body: JSON.stringify({ model, thinking: { type: 'disabled' }, messages: [{ role: 'user', content: 'Trả lời đúng một từ: OK' }], max_tokens: 8, temperature: 0 }), signal: AbortSignal.timeout(20000) });
    const payload = await response.json().catch(() => ({})); if (!response.ok) return res.status(response.status).json({ error: payload?.error?.message || `DeepSeek HTTP ${response.status}` });
    res.json({ ok: true, model: payload.model || 'deepseek-v4-flash', message: 'DeepSeek API key hợp lệ.' });
  } catch (error) { res.status(502).json({ error: error.message || 'Không kiểm tra được DeepSeek API key.' }); }
});
app.post('/api/niche/9router/test', requireAdmin, async (req, res) => {
  try {
    const settings = nicheSettings(); const baseUrl = normalizeOpenAiBaseUrl(req.body?.baseUrl || settings.routerBaseUrl); const model = String(req.body?.model || settings.routerModel || '').trim();
    const apiKey = String(req.body?.apiKey || '').trim() || decryptAdminSetting(settings.routerApiKey) || process.env.NINE_ROUTER_API_KEY || '';
    if (!model) return res.status(400).json({ error: 'Chưa nhập model 9Router.' });
    const headers = { 'Content-Type': 'application/json' }; if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
    const response = await fetch(openAiChatUrl(baseUrl), { method: 'POST', headers, body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Trả lời đúng một từ: OK' }], max_tokens: 16, temperature: 0, stream: false }), signal: AbortSignal.timeout(30000) });
    const payload = await response.json().catch(() => ({})); if (!response.ok) return res.status(response.status).json({ error: payload?.error?.message || payload?.error || `9Router HTTP ${response.status}` });
    res.json({ ok: true, model: payload.model || model, message: 'Kết nối 9Router thành công.' });
  } catch (error) { res.status(502).json({ error: error.message || 'Không kết nối được 9Router.' }); }
});
app.post('/api/niche/keys/import', requireAdmin, async (req, res) => {
  const lines = String(req.body?.keys || '').split(/\r?\n/).map(line => line.trim()).filter(Boolean); const settings = nicheSettings(); let added = 0; let skipped = 0;
  let reactivated = 0;
  for (const line of lines) {
    const match = line.match(/tvly-[A-Za-z0-9_-]{10,}/); const rawKey = match?.[0] || '';
    const prefix = match ? line.slice(0, match.index).replace(/[\s,;|"']+$/g, '').trim() : '';
    const nameCell = prefix.split(/[,;|\t]/).map(item => item.replace(/^["']|["']$/g, '').trim()).filter(Boolean).pop();
    if (!rawKey) { skipped++; continue; }
    const existing = settings.tavilyKeys.find(item => decryptAdminSetting(item.secret) === rawKey);
    if (existing) {
      if (existing.active === false) reactivated++;
      existing.active = true; existing.planLimitReached = false; existing.planLimitReachedAt = null; existing.cooldownUntil = null; existing.lastError = '';
      skipped++;
      continue;
    }
    settings.tavilyKeys.push({ id: crypto.randomUUID(), name: (nameCell || `Tavily ${settings.tavilyKeys.length + 1}`).slice(0, 80), secret: encryptAdminSetting(rawKey), active: true, usedCount: 0, planLimitReached: false, planLimitReachedAt: null, usageMonth: tavilyUsageMonth(), lastUsedAt: null, cooldownUntil: null, lastError: '' }); added++;
  }
  await saveAuthStore(); res.status(201).json({ ok: true, added, skipped, reactivated, tavilyKeys: settings.tavilyKeys.map(publicTavilyKey) });
});
app.patch('/api/niche/keys', requireAdmin, async (req, res) => {
  if (req.body?.action !== 'enable-all') return res.status(400).json({ error: 'Thao tác Tavily hàng loạt không hợp lệ.' });
  const keys = nicheSettings().tavilyKeys; let enabled = 0;
  keys.forEach(key => {
    if (key.active === false) enabled++;
    key.active = true; key.planLimitReached = false; key.planLimitReachedAt = null; key.cooldownUntil = null; key.lastError = '';
  });
  await saveAuthStore();
  res.json({ ok: true, enabled, total: keys.length, tavilyKeys: keys.map(publicTavilyKey) });
});
app.patch('/api/niche/keys/:id', requireAdmin, async (req, res) => { const key = nicheSettings().tavilyKeys.find(item => item.id === req.params.id); if (!key) return res.status(404).json({ error: 'Không tìm thấy Tavily key.' }); if (req.body?.active !== undefined) key.active = Boolean(req.body.active); if (req.body?.reset) { key.usedCount = 0; key.planLimitReached = false; key.planLimitReachedAt = null; key.usageMonth = tavilyUsageMonth(); key.cooldownUntil = null; key.lastError = ''; } await saveAuthStore(); res.json({ ok: true, key: publicTavilyKey(key) }); });
app.post('/api/niche/keys/:id/test', requireAdmin, async (req, res) => {
  const key = nicheSettings().tavilyKeys.find(item => item.id === req.params.id);
  if (!key) return res.status(404).json({ error: 'Không tìm thấy Tavily key.' });
  const testUrl = 'https://weikav.com';
  try {
    const response = await fetch('https://api.tavily.com/extract', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${decryptAdminSetting(key.secret)}` }, body: JSON.stringify({ urls: [testUrl], extract_depth: 'advanced', format: 'markdown', include_images: false }), signal: AbortSignal.timeout(45000) });
    const payload = await response.json().catch(() => ({}));
    key.lastUsedAt = new Date().toISOString();
    if (!response.ok) {
      const errorText = tavilyErrorText(payload, `Tavily HTTP ${response.status}`);
      key.lastError = errorText;
      if (isTavilyPlanLimitError({ status: response.status, message: errorText })) { key.planLimitReached = true; key.planLimitReachedAt = new Date().toISOString(); key.cooldownUntil = null; }
      else if (response.status === 429) key.cooldownUntil = new Date(Date.now() + 60000).toISOString();
      await saveAuthStore();
      return res.status(response.status).json({ ok: false, error: errorText, status: response.status, planLimit: isTavilyPlanLimitError({ status: response.status, message: errorText }), testUrl });
    }
    key.usedCount = Number(key.usedCount || 0) + 1; key.active = true; key.planLimitReached = false; key.planLimitReachedAt = null; key.lastError = ''; key.cooldownUntil = null; await saveAuthStore();
    res.json({ ok: true, message: `Key còn hoạt động; Tavily đã nhận yêu cầu Extract ${testUrl}.`, status: response.status, resultCount: Array.isArray(payload.results) ? payload.results.length : 0, failedCount: Array.isArray(payload.failed_results) ? payload.failed_results.length : 0, testUrl });
  } catch (error) { key.lastError = error.message; key.lastUsedAt = new Date().toISOString(); await saveAuthStore(); res.status(502).json({ ok: false, error: `Không gọi được Tavily Extract: ${error.message}`, testUrl }); }
});
app.delete('/api/niche/keys/:id', requireAdmin, async (req, res) => { const settings = nicheSettings(); const index = settings.tavilyKeys.findIndex(item => item.id === req.params.id); if (index < 0) return res.status(404).json({ error: 'Không tìm thấy Tavily key.' }); settings.tavilyKeys.splice(index, 1); await saveAuthStore(); res.json({ ok: true }); });
const domainNicheCache = new Map();
async function preloadDomainNiches(domains) {
  const candidates = [...new Set((domains || []).map(normalizeProjectDomain).filter(Boolean))];
  if (!USE_SUPABASE || !candidates.length || traceDomainTableAvailable === false) return;
  await ensureLegacyTraceDomainsMigrated();
  if (traceDomainTableAvailable === false) return;
  for (let index = 0; index < candidates.length; index += 200) {
    const batch = candidates.slice(index, index + 200);
    const params = new URLSearchParams({ select: 'domain,niche,niche_updated_at,niche_model', domain: `in.(${batch.map(domain => `"${domain.replace(/["(),]/g, '')}"`).join(',')})` });
    const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?${params}`, { headers: supabaseTraceHeaders(), signal: AbortSignal.timeout(15000) });
    if (response.status === 404) { traceDomainTableAvailable = false; return; }
    if (!response.ok) {
      // Cho phép server tiếp tục quét trong thời gian migration chưa được chạy.
      if ([400, 422].includes(response.status)) return;
      throw new Error(`Không thể đọc ngách từ Kho Domain: Supabase ${response.status}`);
    }
    const found = new Set();
    (await response.json()).forEach(row => {
      const domain = normalizeProjectDomain(row.domain);
      if (domain && row.niche) { found.add(domain); domainNicheCache.set(domain, { category: String(row.niche), updatedAt: row.niche_updated_at || null, model: row.niche_model || '' }); }
    });
    batch.forEach(domain => { if (!found.has(domain) && !domainNicheCache.has(domain)) domainNicheCache.set(domain, null); });
  }
}
async function storedDomainNiche(domain) {
  if (domainNicheCache.has(domain)) return domainNicheCache.get(domain);
  await preloadDomainNiches([domain]);
  return domainNicheCache.get(domain) || null;
}
async function saveDomainNiche(domain, category, model) {
  const record = { category, updatedAt: new Date().toISOString(), model: model || '' };
  domainNicheCache.set(domain, record);
  if (!USE_SUPABASE || traceDomainTableAvailable === false) return record;
  const payload = { domain, niche: category, niche_updated_at: record.updatedAt, niche_model: record.model || null };
  const response = await fetch(`${SUPABASE_URL}/rest/v1/trace_domains?on_conflict=domain`, { method: 'POST', headers: supabaseTraceHeaders('resolution=merge-duplicates,return=minimal'), body: JSON.stringify(payload), signal: AbortSignal.timeout(15000) });
  if (!response.ok) throw new Error(`Phân loại thành công nhưng không lưu được ngách vào Kho Domain: Supabase ${response.status}: ${await response.text()}`);
  traceDomainTableAvailable = true;
  return record;
}
const nicheDomainPreflightCache = new Map();
function isPrivatePreflightAddress(address) {
  const value = String(address || '').toLowerCase();
  if (value.includes(':')) return value === '::1' || value.startsWith('fc') || value.startsWith('fd') || value.startsWith('fe8') || value.startsWith('fe9') || value.startsWith('fea') || value.startsWith('feb');
  const parts = value.split('.').map(Number);
  if (parts.length !== 4 || parts.some(part => !Number.isInteger(part) || part < 0 || part > 255)) return true;
  return parts[0] === 10 || parts[0] === 127 || parts[0] === 0 || (parts[0] === 169 && parts[1] === 254) || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) || (parts[0] === 192 && parts[1] === 168);
}
async function preflightNicheDomain(domain) {
  const cached = nicheDomainPreflightCache.get(domain);
  if (cached && cached.expiresAt > Date.now()) return cached.result;
  let addresses;
  try { addresses = await dns.lookup(domain, { all: true }); }
  catch (error) {
    const result = { ok: false, reason: `Không phân giải được DNS (${error.code || error.message}).` };
    nicheDomainPreflightCache.set(domain, { result, expiresAt: Date.now() + 15 * 60 * 1000 }); return result;
  }
  if (!addresses.length || addresses.some(item => isPrivatePreflightAddress(item.address))) {
    const result = { ok: false, reason: 'Domain không trỏ tới địa chỉ Internet công khai.' };
    nicheDomainPreflightCache.set(domain, { result, expiresAt: Date.now() + 15 * 60 * 1000 }); return result;
  }
  let lastError = '';
  for (const protocol of ['https', 'http']) {
    try {
      // Mọi mã HTTP (kể cả 403/404/405) đều chứng minh máy chủ có phản hồi.
      await fetch(`${protocol}://${domain}/`, { method: 'HEAD', redirect: 'manual', signal: AbortSignal.timeout(10000), headers: { 'User-Agent': 'CheckGGAdsAll-Domain-Preflight/1.0' } });
      const result = { ok: true, protocol };
      nicheDomainPreflightCache.set(domain, { result, expiresAt: Date.now() + 15 * 60 * 1000 }); return result;
    } catch (error) { lastError = error?.cause?.code || error?.name || error?.message || 'connection_failed'; }
  }
  const result = { ok: false, reason: `Không thể kết nối trước khi gọi API (${lastError}).` };
  nicheDomainPreflightCache.set(domain, { result, expiresAt: Date.now() + 5 * 60 * 1000 }); return result;
}
async function classifyNicheDomain(domainInput, username = null, scraperEngine = 'tavily') {
    const domain = normalizeProjectDomain(domainInput); if (!domain) { const error = new Error('Domain không hợp lệ.'); error.status = 400; throw error; }
    const storedNiche = await storedDomainNiche(domain);
    if (storedNiche?.category) return { ok: true, id: crypto.randomUUID(), domain, category: storedNiche.category, product_group: storedNiche.category, tavilyKey: 'Kho Domain', scraperUsed: 'Kho Domain (tái sử dụng)', model: storedNiche.model || '', cached: true, sources: [], classifiedBy: username, createdAt: storedNiche.updatedAt || new Date().toISOString(), updatedAt: storedNiche.updatedAt || new Date().toISOString() };
    const preflight = await preflightNicheDomain(domain);
    if (!preflight.ok) { const error = new Error(`Bỏ qua trước khi dùng API key: ${preflight.reason}`); error.status = 422; error.nonRetryable = true; throw error; }
    const settings = nicheSettings(); const useRouter = settings.aiProvider === '9router'; const aiApiKey = useRouter ? (decryptAdminSetting(settings.routerApiKey) || process.env.NINE_ROUTER_API_KEY || '') : (decryptAdminSetting(settings.deepseekApiKey) || process.env.DEEPSEEK_API_KEY); if (!useRouter && !aiApiKey) { const error = new Error('Admin chưa cấu hình DeepSeek API key.'); error.status = 503; throw error; } if (useRouter && !settings.routerModel) { const error = new Error('Admin chưa cấu hình model 9Router.'); error.status = 503; throw error; }
    
    let context = '';
    let scraperUsed = 'Tavily';
    let tavilyKeyName = null;
    let sources = [];

    if (scraperEngine === 'firecrawl') {
        const firecrawl = await firecrawlResearchDomain(domain);
        context = firecrawlContext(firecrawl.payload);
        if (!context) {
            const error = new Error('Firecrawl không trả về dữ liệu nội dung hoặc description cho domain này.');
            error.status = 422;
            throw error;
        }
        scraperUsed = `Firecrawl (${firecrawl.key})`;
        const meta = firecrawl.payload?.data?.metadata || {};
        sources = [{ title: meta.title || domain, url: meta.url || `https://${domain}` }];
    } else {
        try {
            const tavily = await tavilyResearchDomain(domain);
            context = tavilyContext(tavily.payload);
            tavilyKeyName = tavily.key;
            scraperUsed = `Tavily (${tavily.key})`;
            sources = (tavily.payload?.results || []).slice(0, 5).map(item => ({ title: item.title || '', url: item.url || '' }));
            if (!context) throw new Error('Tavily không trả về title, content hoặc raw_content để phân loại domain này.');
        } catch (tavilyError) {
            const isFetchError = isNonRetryableFetchError(tavilyError) || tavilyError.status === 422 || /failed to fetch|error fetching content|không tải được nội dung/i.test(tavilyError.message);
            const firecrawlConfigured = (settings.firecrawlKeys || []).some(k => k.active !== false) || Boolean(decryptAdminSetting(settings.firecrawlApiKey) || process.env.FIRECRAWL_API_KEY);
            
            if (isFetchError && firecrawlConfigured) {
                try {
                    const firecrawl = await firecrawlResearchDomain(domain);
                    context = firecrawlContext(firecrawl.payload);
                    if (!context) throw new Error('Firecrawl cũng không trả về dữ liệu nội dung cho domain này.');
                    scraperUsed = 'Tavily -> Firecrawl (Fallback)';
                    tavilyKeyName = `Firecrawl (${firecrawl.key})`;
                    const meta = firecrawl.payload?.data?.metadata || {};
                    sources = [{ title: meta.title || domain, url: meta.url || `https://${domain}` }];
                } catch (firecrawlErr) {
                    const fallbackError = new Error(`Tavily lỗi (${tavilyError.message}); Fallback Firecrawl thất bại (${firecrawlErr.message})`);
                    fallbackError.status = firecrawlErr.status || tavilyError.status;
                    fallbackError.tavilyResponse = tavilyError.tavilyResponse || null;
                    fallbackError.firecrawlResponse = firecrawlErr.firecrawlResponse || null;
                    if (firecrawlErr.quotaExhausted === true) {
                        fallbackError.quotaExhausted = true;
                        fallbackError.provider = 'Firecrawl';
                    }
                    throw fallbackError;
                }
            } else {
                throw tavilyError;
            }
        }
    }

    const promptValues = { categories: settings.categories.join(', '), domain, context };
    const outputRule = '\n\nQUY TẮC ĐẦU RA BẮT BUỘC: chỉ trả đúng JSON {"category":"một nhãn trong danh sách"}. Không thêm confidence, reason hay bất kỳ trường/văn bản nào khác.';
    const aiUrl = useRouter ? openAiChatUrl(settings.routerBaseUrl) : 'https://api.deepseek.com/chat/completions'; const aiModel = useRouter ? settings.routerModel : settings.model; const aiHeaders = { 'Content-Type': 'application/json' }; if (aiApiKey) aiHeaders.Authorization = `Bearer ${aiApiKey}`; const aiBody = { model: aiModel, max_tokens: 24, messages: [{ role: 'system', content: renderNichePrompt(settings.systemPrompt, promptValues) + outputRule }, { role: 'user', content: renderNichePrompt(settings.userPrompt, promptValues) }], temperature: 0, stream: false }; if (!useRouter) { aiBody.thinking = { type: 'disabled' }; aiBody.response_format = { type: 'json_object' }; }
    const response = await trackedFetch(aiUrl, { method: 'POST', headers: aiHeaders, body: JSON.stringify(aiBody), signal: AbortSignal.timeout(60000) }, null, { category: 'external', direction: useRouter ? 'Server -> 9Router local niche classification' : 'Server -> DeepSeek niche classification', domain, aiProvider: settings.aiProvider });
    const result = await response.json().catch(() => ({})); if (!response.ok) { const error = new Error(result?.error?.message || result?.error || `${useRouter ? '9Router' : 'DeepSeek'} HTTP ${response.status}`); error.status = response.status; throw error; } const classification = parseAiJsonContent(result.choices?.[0]?.message?.content);
    const rawCategory = String(classification.product_group || classification.category || '').trim(); const category = settings.categories.includes(rawCategory) ? rawCategory : (settings.categories.includes('Khác') ? 'Khác' : settings.categories[0]); const now = new Date().toISOString(); await saveDomainNiche(domain, category, aiModel); return { ok: true, id: crypto.randomUUID(), domain, category, product_group: category, tavilyKey: tavilyKeyName || scraperUsed, scraperUsed, model: aiModel, aiProvider: settings.aiProvider, sources, classifiedBy: username, createdAt: now, updatedAt: now };
}
app.post('/api/niche/classify', requireNiche, async (req, res) => {
  try {
    res.json(await classifyNicheDomain(req.body?.domain, req.user?.username || null, req.body?.scraperEngine || 'tavily'));
  } catch (error) {
    if (error.quotaExhausted === true && error.provider === 'Firecrawl') {
      const recent = monitorStore.notifications.some(item => item.ownerId === req.user.id && item.title === 'Firecrawl đã hết lượt' && Date.now() - new Date(item.detectedAt || 0).getTime() < 60000);
      if (!recent) { monitorStore.notifications.unshift({ id: monitorId('notification'), ownerId: req.user.id, type: 'system', severity: 'warning', title: 'Firecrawl đã hết lượt', message: 'Tất cả Firecrawl API key đã hết lượt. Phiên phân loại đã tạm dừng.', detectedAt: new Date().toISOString(), read: false }); monitorStore.notifications = monitorStore.notifications.slice(0, 1000); await saveMonitorStore(); }
    }
    res.status(error.status && error.status >= 400 && error.status < 600 ? error.status : 502).json({ error: error.message || 'Không thể phân loại domain.', quotaExhausted: error.quotaExhausted === true, provider: error.provider || '', tavilyResponse: error.tavilyResponse || null, firecrawlResponse: error.firecrawlResponse || null });
  }
});

async function findOfficialProjectWebsite(projectName) {
  const project = String(projectName || '').trim().slice(0, 300);
  if (!project) { const error = new Error('Tên dự án không được để trống.'); error.status = 400; throw error; }
  const key = await waitForTavilyKey();
  const proxyUrl = authStore.settings.proxyUrl;
  if (!proxyUrl) { releaseTavilyKey(key); const error = new Error('Admin chưa cấu hình Proxy chung cho Tavily.'); error.status = 503; throw error; }
  const dispatcher = getProxyDispatcher(proxyUrl, false);
  let searchPayload;
  try {
    const response = await trackedFetch('https://api.tavily.com/search', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${decryptAdminSetting(key.secret)}`, Connection: 'close' }, dispatcher, body: JSON.stringify({ query: `${project} official website`, search_depth: 'advanced', max_results: 10, include_answer: false, include_images: false }), signal: AbortSignal.timeout(45000) }, null, { category: 'external', direction: 'Server -> Tavily project website search', project, tavilyKeyId: key.id, tavilyKeyName: key.name });
    searchPayload = await response.json().catch(() => ({}));
    if (!response.ok) throw nicheHttpError(response.status, searchPayload, response);
    key.usedCount = Number(key.usedCount || 0) + 1; key.lastUsedAt = new Date().toISOString(); key.lastError = ''; key.cooldownUntil = null; await saveAuthStore();
  } catch (error) {
    key.lastError = error.message; key.lastUsedAt = new Date().toISOString();
    if ([401, 403].includes(error.status)) key.active = false;
    else if (error.status === 429 || isTavilyPlanLimitError(error)) key.cooldownUntil = new Date(Date.now() + 60000).toISOString();
    await saveAuthStore(); throw error;
  } finally { if (dispatcher && typeof dispatcher.close === 'function') await dispatcher.close().catch(() => {}); releaseTavilyKey(key); }
  const candidates = (searchPayload.results || []).map(row => ({ title: String(row.title || ''), url: String(row.url || ''), content: String(row.content || '').slice(0, 500) })).filter(row => normalizeProjectDomain(row.url));
  if (!candidates.length) { const error = new Error('Tavily không tìm thấy website phù hợp.'); error.status = 404; throw error; }
  const settings = nicheSettings(); const useRouter = settings.aiProvider === '9router';
  const apiKey = useRouter ? (decryptAdminSetting(settings.routerApiKey) || process.env.NINE_ROUTER_API_KEY || '') : (decryptAdminSetting(settings.deepseekApiKey) || process.env.DEEPSEEK_API_KEY || '');
  if ((!useRouter && !apiKey) || (useRouter && !settings.routerModel)) { const domain = normalizeProjectDomain(candidates[0].url); return { project, domain, finalUrl: `https://${domain}`, source: 'Tavily', tavilyKey: key.name, candidates }; }
  const aiUrl = useRouter ? openAiChatUrl(settings.routerBaseUrl) : 'https://api.deepseek.com/chat/completions'; const headers = { 'Content-Type': 'application/json' }; if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
  const body = { model: useRouter ? settings.routerModel : settings.model, temperature: 0, max_tokens: 100, stream: false, messages: [{ role: 'system', content: 'Chọn website chính thức phù hợp nhất với tên dự án. Ưu tiên domain gốc và loại URL mạng xã hội, thư mục, bài viết. Chỉ trả JSON {"url":"https://example.com"}.' }, { role: 'user', content: `Tên dự án: ${project}\nKết quả tìm kiếm:\n${candidates.map((row, index) => `${index + 1}. ${row.title}\n${row.url}\n${row.content}`).join('\n\n')}` }] }; if (!useRouter) { body.thinking = { type: 'disabled' }; body.response_format = { type: 'json_object' }; }
  const response = await trackedFetch(aiUrl, { method: 'POST', headers, body: JSON.stringify(body), signal: AbortSignal.timeout(60000) }, null, { category: 'external', direction: useRouter ? 'Server -> 9Router project URL selection' : 'Server -> DeepSeek project URL selection', project });
  const payload = await response.json().catch(() => ({})); if (!response.ok) { const error = new Error(payload?.error?.message || `${useRouter ? '9Router' : 'DeepSeek'} HTTP ${response.status}`); error.status = response.status; throw error; }
  const selected = parseAiJsonContent(payload.choices?.[0]?.message?.content); const domain = normalizeProjectDomain(selected.url);
  if (!domain) { const error = new Error('AI không chọn được website hợp lệ.'); error.status = 422; throw error; }
  return { project, domain, finalUrl: `https://${domain}`, source: `${useRouter ? '9Router' : 'DeepSeek'} + Tavily`, tavilyKey: key.name, candidates };
}

app.post('/api/project-traffic/resolve-website', async (req, res) => {
  if (req.user.role !== 'admin' && !req.user.canUseTraffic) return res.status(403).json({ error: 'Tài khoản chưa được cấp quyền Quét Traffic.' });
  try { res.json(await findOfficialProjectWebsite(req.body?.project)); }
  catch (error) { res.status(error.status && error.status >= 400 && error.status < 600 ? error.status : 502).json({ error: error.message || 'Không tìm được website dự án.', tavilyResponse: error.tavilyResponse || null }); }
});

// Tác vụ chạy hoàn toàn trên server nên không phụ thuộc tab trình duyệt có đang
// được hiển thị hay không. Kết quả được giao diện lấy dần bằng API trạng thái.
const nicheJobs = new Map();
function publicNicheJob(job, since = 0) { return { id: job.id, status: job.status, total: job.domains.length, completed: job.completed, stopped: job.stopped, stopReason: job.stopReason || '', quotaExhausted: Boolean(job.quotaExhausted), provider: job.provider || '', results: job.results.slice(Math.max(0, since)), nextCursor: job.results.length, createdAt: job.createdAt, finishedAt: job.finishedAt || null }; }
async function runNicheJob(job, concurrency, username, scraperEngine = 'tavily') {
  await preloadDomainNiches(job.domains);
  let cursor = 0;
  const workers = Array.from({ length: Math.min(concurrency, job.domains.length) }, async () => {
    while (cursor < job.domains.length && !job.stopped) {
      const inputOrder = cursor++; const domain = job.domains[inputOrder];
      try { job.results.push({ inputOrder, row: await classifyNicheDomain(domain, username, scraperEngine) }); }
      catch (error) {
        const quotaExhausted = isApiQuotaExhaustedError(error);
        job.results.push({ inputOrder, row: { domain, inputOrder, error: error.message || 'Không thể phân loại domain.', interrupted: quotaExhausted, nonRetryable: isNonRetryableFetchError(error), tavilyResponse: error.tavilyResponse || error.firecrawlResponse || null, updatedAt: new Date().toISOString() } });
        if (quotaExhausted) {
          job.stopped = true;
          job.quotaExhausted = true;
          job.provider = error.provider;
          job.stopReason = error.message;
        }
      }
      job.completed++; job.updatedAt = new Date().toISOString();
    }
  });
  await Promise.all(workers); job.status = job.stopped ? 'stopped' : 'completed'; job.finishedAt = new Date().toISOString();
  if (job.quotaExhausted) {
    const title = `${job.provider} đã hết lượt`;
    const recent = monitorStore.notifications.some(item => item.ownerId === job.ownerId && item.title === title && Date.now() - new Date(item.detectedAt || 0).getTime() < 60000);
    if (!recent) {
      monitorStore.notifications.unshift({ id: monitorId('notification'), ownerId: job.ownerId, type: 'system', severity: 'warning', title, message: `${job.provider} đã xác nhận hết lượt trên tất cả API key. Phiên phân loại đã tạm dừng tại ${job.completed}/${job.domains.length} domain.`, detectedAt: new Date().toISOString(), read: false });
      monitorStore.notifications = monitorStore.notifications.slice(0, 1000); await saveMonitorStore();
    }
  }
  setTimeout(() => nicheJobs.delete(job.id), 6 * 60 * 60 * 1000).unref?.();
}
app.post('/api/niche/jobs', requireNiche, async (req, res) => {
  // Không dùng snapshot kho key có thể đã cũ trên một Vercel instance khác.
  if (USE_SUPABASE) await syncNicheSettingsFromSupabase();
  const domains = [...new Set((Array.isArray(req.body?.domains) ? req.body.domains : []).map(normalizeProjectDomain).filter(Boolean))].slice(0, 100000);
  if (!domains.length) return res.status(400).json({ error: 'Cần ít nhất một domain hợp lệ.' });
  const scraperEngine = String(req.body?.scraperEngine || 'tavily');
  const customFirecrawlKey = String(req.body?.firecrawlApiKey || '').trim();
  const job = { id: crypto.randomUUID(), ownerId: req.user.id, domains, results: [], completed: 0, stopped: false, status: 'running', createdAt: new Date().toISOString() };
  nicheJobs.set(job.id, job); runNicheJob(job, Math.max(1, Math.min(5, Number(req.body?.concurrency) || 2)), req.user?.username || null, scraperEngine, customFirecrawlKey).catch(error => { job.status = 'failed'; job.error = error.message; job.finishedAt = new Date().toISOString(); });
  res.status(202).json(publicNicheJob(job));
});
app.get('/api/niche/jobs/:id', requireNiche, (req, res) => { const job = nicheJobs.get(req.params.id); if (!job || job.ownerId !== req.user.id) return res.status(404).json({ error: 'Không tìm thấy phiên phân loại.' }); res.json(publicNicheJob(job, Number(req.query.since) || 0)); });
app.post('/api/niche/jobs/:id/stop', requireNiche, (req, res) => { const job = nicheJobs.get(req.params.id); if (!job || job.ownerId !== req.user.id) return res.status(404).json({ error: 'Không tìm thấy phiên phân loại.' }); job.stopped = true; res.json({ ok: true }); });

app.post('/api/admin/users', requireAdmin, async (req, res) => {
  const username = String(req.body?.username || '').trim().toLowerCase();
  const password = String(req.body?.password || '');
  if (!/^[a-z0-9._-]{3,40}$/.test(username) || password.length < 8) return res.status(400).json({ error: 'Username từ 3 ký tự; mật khẩu tối thiểu 8 ký tự.' });
  if (authStore.users.some(item => item.username === username)) return res.status(409).json({ error: 'Username đã tồn tại.' });
  const secured = hashPassword(password);
  const user = { id: crypto.randomUUID(), username, displayName: String(req.body?.displayName || username).trim(), role: 'user', active: true, canUseProxy: Boolean(req.body?.canUseProxy), canUseAdvertiserCount: Boolean(req.body?.canUseAdvertiserCount), canUseTraffic: Boolean(req.body?.canUseTraffic), canUseTrace: Boolean(req.body?.canUseTrace), canUseNiche: Boolean(req.body?.canUseNiche), canUseDomainDuplicate: Boolean(req.body?.canUseDomainDuplicate), canUseAffiliate: Boolean(req.body?.canUseAffiliate), canUseAdminFirecrawl: Boolean(req.body?.canUseAdminFirecrawl), affiliateFirecrawlSource: req.body?.canUseAdminFirecrawl ? 'admin' : 'user', sessionVersion: 1, passwordSalt: secured.salt, passwordHash: secured.hash, createdAt: new Date().toISOString() };
  user.canUseDomainStore = Boolean(req.body?.canUseDomainStore);
  authStore.users.push(user); await saveAuthStore(); res.status(201).json({ ok: true, user: publicUser(user) });
});
app.patch('/api/admin/users/:id', requireAdmin, async (req, res) => {
  const user = authStore.users.find(item => item.id === req.params.id);
  if (!user) return res.status(404).json({ error: 'Không tìm thấy user.' });
  if (req.body?.displayName !== undefined) user.displayName = String(req.body.displayName).trim() || user.username;
  if (req.body?.active !== undefined && user.id !== req.user.id) {
    const nextActive = Boolean(req.body.active);
    if (nextActive !== user.active) user.sessionVersion = Number(user.sessionVersion || 1) + 1;
    user.active = nextActive;
  }
  if (req.body?.canUseProxy !== undefined && user.role !== 'admin') user.canUseProxy = Boolean(req.body.canUseProxy);
  if (req.body?.canUseAdvertiserCount !== undefined && user.role !== 'admin') user.canUseAdvertiserCount = Boolean(req.body.canUseAdvertiserCount);
  if (req.body?.canUseTraffic !== undefined && user.role !== 'admin') user.canUseTraffic = Boolean(req.body.canUseTraffic);
  if (req.body?.canUseTrace !== undefined && user.role !== 'admin') user.canUseTrace = Boolean(req.body.canUseTrace);
  if (req.body?.canUseDomainStore !== undefined && user.role !== 'admin') user.canUseDomainStore = Boolean(req.body.canUseDomainStore);
  if (req.body?.canUseNiche !== undefined && user.role !== 'admin') {
    user.canUseNiche = Boolean(req.body.canUseNiche);
    if (!user.canUseNiche) for (const job of nicheJobs.values()) if (job.ownerId === user.id && job.status === 'running') { job.stopped = true; job.stopReason = 'Admin đã thu hồi quyền Phân loại ngách.'; }
  }
  if (req.body?.canUseDomainDuplicate !== undefined && user.role !== 'admin') user.canUseDomainDuplicate = Boolean(req.body.canUseDomainDuplicate);
  if (req.body?.canUseAdminFirecrawl !== undefined && user.role !== 'admin') {
    user.canUseAdminFirecrawl = Boolean(req.body.canUseAdminFirecrawl);
    if (!user.canUseAdminFirecrawl) {
      if (['admin', 'both'].includes(user.affiliateFirecrawlSource)) user.affiliateFirecrawlSource = 'user';
      for (const job of affiliateJobs.values()) if (job.ownerId === user.id && job.status === 'running' && ['admin', 'both'].includes(job.firecrawlSource)) {
        job.stopped = true;
        job.error = 'Admin đã thu hồi quyền sử dụng Firecrawl API key của Admin.';
      }
    }
  }
  if (req.body?.canUseAffiliate !== undefined && user.role !== 'admin') {
    user.canUseAffiliate = Boolean(req.body.canUseAffiliate);
    if (!user.canUseAffiliate) for (const job of affiliateJobs.values()) if (job.ownerId === user.id && job.status === 'running') { job.stopped = true; job.error = 'Admin đã thu hồi quyền Tìm nền tảng Affiliate.'; }
  }
  if (String(req.body?.password || '').length) {
    if (String(req.body.password).length < 8) return res.status(400).json({ error: 'Mật khẩu tối thiểu 8 ký tự.' });
    const secured = hashPassword(req.body.password); user.passwordSalt = secured.salt; user.passwordHash = secured.hash; user.sessionVersion = Number(user.sessionVersion || 1) + 1;
  }
  await saveAuthStore(); res.json({ ok: true, user: publicUser(user) });
});

app.use((req, res, next) => {
  if (req.path.startsWith('/api/') && req.method !== 'GET') {
    const adminCurl = parseAdminCurl(authStore.settings.curlCommand);
    if (adminCurl) {
      req.body.headers = adminCurl.headers;
      const requestedFilters = req.body.fReq?.['3'] || {};
      req.body.fReq = JSON.parse(JSON.stringify(adminCurl.fReq || {}));
      if (!req.body.fReq['3']) req.body.fReq['3'] = {};
      if (requestedFilters['6'] !== undefined) req.body.fReq['3']['6'] = requestedFilters['6'];
      if (requestedFilters['7'] !== undefined) req.body.fReq['3']['7'] = requestedFilters['7'];
      if (requestedFilters['8'] !== undefined) req.body.fReq['3']['8'] = requestedFilters['8'];
    } else req.body.headers = {};
    if (req.body?.traceMode === true && ['/api/extract', '/api/advertiser-creatives'].includes(req.path)) {
      const filters = getDefaultSearchTextFilters();
      if (!req.body.fReq || typeof req.body.fReq !== 'object') req.body.fReq = {};
      if (!req.body.fReq['3']) req.body.fReq['3'] = {};
      req.body.fReq['3']['4'] = filters.platform;
      req.body.fReq['3']['6'] = filters.startDate;
      req.body.fReq['3']['7'] = filters.endDate;
      req.body.fReq['3']['14'] = filters.format;
    }
    const needsCurl = ['/api/extract', '/api/advertiser-creatives', '/api/competitor-monitor'].includes(req.path)
      || /^\/api\/competitor-monitor\/[^/]+\/(run|client-scan)$/.test(req.path);
    if (needsCurl && !adminCurl) return res.status(503).json({ error: 'Admin chưa cấu hình xác thực Google. Vui lòng liên hệ Admin.' });
    const canProxy = req.user.role === 'admin' || Boolean(req.user.canUseProxy);
    if (!canProxy) { req.body.proxy = 'NONE'; req.body.disableProxy = true; }
    else if (req.body && req.body.disableProxy !== true) req.body.proxy = authStore.settings.proxyUrl || 'NONE';
  }
  next();
});
app.use(express.static(path.join(__dirname, 'public')));

const API_CATALOG = [
  { method: 'GET', path: '/api', name: 'Danh sách API', description: 'Trả về thông tin cơ bản và danh sách endpoint của dự án.', parameters: [] },
  { method: 'POST', path: '/api/extract', name: 'Quét theo tên miền', description: 'Quét Google Ads Transparency theo domain và trả tiến trình dạng NDJSON.', parameters: ['domain', 'headers', 'proxy?', 'fReq?', 'limit? (mặc định 5000)', 'scanId?'], responseType: 'application/x-ndjson' },
  { method: 'POST', path: '/api/extract/:scanId/pause', name: 'Tạm dừng quét domain', description: 'Tạm dừng một phiên quét sau khi trang hiện tại hoàn tất.', parameters: ['scanId'], responseType: 'application/json' },
  { method: 'POST', path: '/api/extract/:scanId/resume', name: 'Tiếp tục quét domain', description: 'Tiếp tục một phiên quét đang tạm dừng.', parameters: ['scanId'], responseType: 'application/json' },
  { method: 'POST', path: '/api/extract/:scanId/finish', name: 'Dừng và xử lý dữ liệu', description: 'Dừng lấy thêm và trả kết quả đã thu thập qua luồng NDJSON hiện tại.', parameters: ['scanId'], responseType: 'application/json' },
  { method: 'POST', path: '/api/advertiser-creatives', name: 'Mẫu quảng cáo theo nhà QC', description: 'Lấy danh sách mẫu quảng cáo theo advertiser ID.', parameters: ['advId', 'headers?', 'proxy?', 'fReq?'], responseType: 'application/x-ndjson' },
  { method: 'POST', path: '/api/ocr', name: 'Nhận dạng nội dung quảng cáo', description: 'OCR ảnh hoặc trích xuất chữ và domain từ trang xem trước.', parameters: ['imageUrl? | previewUrl?'], responseType: 'application/json' },
  { method: 'GET', path: '/api/ad-preview-html', name: 'HTML xem trước quảng cáo', description: 'Tải và làm sạch HTML từ URL content.js của mẫu quảng cáo.', parameters: ['url'], responseType: 'text/html' },
  { method: 'GET', path: '/api/image-proxy', name: 'Proxy hình ảnh', description: 'Tải ảnh qua server để tránh lỗi CORS hoặc trình chặn quảng cáo.', parameters: ['url'], responseType: 'image/*' },
  { method: 'POST', path: '/api/domain-summary', name: 'Tổng quan tên miền', description: 'Tính số nhà quảng cáo đang hoạt động và mẫu chạy lâu nhất.', parameters: ['domain', 'headers?', 'proxy?'], responseType: 'application/json' }
  ,{ method: 'POST', path: '/api/keyword-metrics', name: 'Volume và CPC từ khóa', description: 'Lấy khối lượng tìm kiếm, CPC và độ cạnh tranh theo từ khóa.', parameters: ['keyword', 'country?', 'currency?'], responseType: 'application/json' }
];

const apiCallHistory = [];
let apiCallSequence = 0;
const REDIS_REST_URL = process.env.UPSTASH_REDIS_REST_URL
  || process.env.UPSTASH_REDIS_REST_KV_REST_API_URL
  || process.env.KV_REST_API_URL
  || '';
const REDIS_REST_TOKEN = process.env.UPSTASH_REDIS_REST_TOKEN
  || process.env.UPSTASH_REDIS_REST_KV_REST_API_TOKEN
  || process.env.KV_REST_API_TOKEN
  || '';
const API_HISTORY_KEY = 'check-gg-ads:api-call-history';
const VISITOR_HISTORY_KEY = 'check-gg-ads:visitor-history';
const API_HISTORY_LIMIT = 200;
const API_HISTORY_FIELD_MAX_CHARS = 10000;
const VISITOR_HISTORY_LIMIT = 200;
const hasPersistentHistory = !USE_SUPABASE && Boolean(REDIS_REST_URL && REDIS_REST_TOKEN);
const visitorHistory = [];
let apiHistorySyncQueue = Promise.resolve();
let apiHistorySyncTimer = null;
let apiHistorySyncDirty = false;

function mergeApiHistoryRows(...groups) {
  const rowsById = new Map();
  groups.flat().filter(Boolean).forEach(row => {
    if (row.id && !rowsById.has(row.id)) {
      rowsById.set(row.id, {
        ...row,
        payload: compactApiHistoryField(row.payload),
        responseBody: compactApiHistoryField(row.responseBody),
        errorResponse: compactApiHistoryField(row.errorResponse)
      });
    }
  });
  return Array.from(rowsById.values())
    .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
    .slice(0, API_HISTORY_LIMIT);
}

function compactApiHistoryField(value) {
  if (value === null || value === undefined) return value;
  try {
    const serialized = JSON.stringify(value);
    if (serialized.length <= API_HISTORY_FIELD_MAX_CHARS) return value;
    return {
      truncated: true,
      originalCharacters: serialized.length,
      preview: serialized.slice(0, API_HISTORY_FIELD_MAX_CHARS)
    };
  } catch (error) {
    return String(value).slice(0, API_HISTORY_FIELD_MAX_CHARS);
  }
}

function flushApiHistoryToSupabase() {
  if (!USE_SUPABASE || !apiHistorySyncDirty) return apiHistorySyncQueue;
  apiHistorySyncDirty = false;
  const operation = apiHistorySyncQueue.then(async () => {
    const remoteRows = await supabaseStateRequest('api_history');
    const mergedRows = mergeApiHistoryRows(apiCallHistory, Array.isArray(remoteRows) ? remoteRows : []);
    apiCallHistory.splice(0, apiCallHistory.length, ...mergedRows);
    await supabaseStateRequest('api_history', mergedRows);
  });
  apiHistorySyncQueue = operation.catch(error => console.error('Supabase API history sync failed:', error.message));
  apiHistorySyncQueue.finally(() => {
    if (apiHistorySyncDirty && !apiHistorySyncTimer) {
      apiHistorySyncTimer = setTimeout(() => {
        apiHistorySyncTimer = null;
        flushApiHistoryToSupabase();
      }, 5000);
    }
  });
  return apiHistorySyncQueue;
}

function scheduleApiHistorySync() {
  if (!USE_SUPABASE) return;
  apiHistorySyncDirty = true;
  if (apiHistorySyncTimer) return;
  apiHistorySyncTimer = setTimeout(() => {
    apiHistorySyncTimer = null;
    flushApiHistoryToSupabase();
  }, 5000);
}

async function redisCommand(command) {
  const response = await fetch(REDIS_REST_URL, {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${REDIS_REST_TOKEN}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify(command),
    signal: AbortSignal.timeout(5000)
  });
  const data = await response.json();
  if (!response.ok || data.error) {
    throw new Error(data.error || `Redis HTTP ${response.status}`);
  }
  return data.result;
}

async function addApiHistory(entry) {
  const context = apiRequestContext.getStore() || {};
  const compactEntry = {
    ...entry,
    payload: compactApiHistoryField(entry.payload),
    responseBody: compactApiHistoryField(entry.responseBody),
    errorResponse: compactApiHistoryField(entry.errorResponse)
  };
  const record = {
    id: `${Date.now()}-${++apiCallSequence}-${Math.random().toString(36).slice(2, 8)}`,
    timestamp: new Date().toISOString(),
    ownerId: entry.ownerId ?? context.ownerId ?? null,
    username: entry.username ?? context.username ?? 'system',
    requestIp: entry.requestIp ?? context.requestIp ?? null,
    ...compactEntry
  };
  apiCallHistory.unshift(record);
  if (apiCallHistory.length > API_HISTORY_LIMIT) apiCallHistory.length = API_HISTORY_LIMIT;
  if (USE_SUPABASE) scheduleApiHistorySync();

  if (hasPersistentHistory) {
    try {
      await redisCommand(['LPUSH', API_HISTORY_KEY, JSON.stringify(record)]);
      await redisCommand(['LTRIM', API_HISTORY_KEY, 0, API_HISTORY_LIMIT - 1]);
    } catch (error) {
      console.error('Persistent API history write failed:', error.message);
    }
  }
}

async function getApiHistory() {
  if (USE_SUPABASE) {
    // Không chờ hàng đợi Supabase: nhật ký phải phản hồi tức thì từ RAM.
    // Dữ liệu remote đã được hydrate nền khi server khởi động.
    return apiCallHistory;
  }
  if (!hasPersistentHistory) return apiCallHistory;
  try {
    const rows = await redisCommand(['LRANGE', API_HISTORY_KEY, 0, API_HISTORY_LIMIT - 1]);
    return (rows || []).map(row => JSON.parse(row)).filter(Boolean);
  } catch (error) {
    console.error('Persistent API history read failed:', error.message);
    return apiCallHistory;
  }
}

async function clearApiHistory() {
  if (apiHistorySyncTimer) clearTimeout(apiHistorySyncTimer);
  apiHistorySyncTimer = null;
  apiHistorySyncDirty = false;
  await apiHistorySyncQueue;
  apiCallHistory.length = 0;
  if (USE_SUPABASE) await supabaseStateRequest('api_history', []);
  if (hasPersistentHistory) {
    await redisCommand(['DEL', API_HISTORY_KEY]);
  }
}

async function addVisitorHistory(entry) {
  const record = {
    id: `${Date.now()}-${Math.random().toString(36).slice(2, 9)}`,
    timestamp: new Date().toISOString(),
    ...entry
  };
  visitorHistory.unshift(record);
  if (visitorHistory.length > 200) visitorHistory.length = 200;
  persistSupabaseState('visitor_history', visitorHistory);
  if (hasPersistentHistory) {
    await redisCommand(['LPUSH', VISITOR_HISTORY_KEY, JSON.stringify(record)]);
      await redisCommand(['LTRIM', VISITOR_HISTORY_KEY, 0, VISITOR_HISTORY_LIMIT - 1]);
  }
  return record;
}

async function getVisitorHistory(ownerId) {
  let rows = visitorHistory;
  try {
    if (hasPersistentHistory) {
      const storedRows = await redisCommand(['LRANGE', VISITOR_HISTORY_KEY, 0, VISITOR_HISTORY_LIMIT - 1]);
      rows = (storedRows || []).map(row => JSON.parse(row)).filter(Boolean);
    }
  } catch (error) {
    console.error('Visitor history read failed:', error.message);
  }

  const uniqueVisitors = new Map();
  rows.filter(visitor => visitor.ownerId === ownerId).forEach(visitor => {
    const requestIp = visitor.requestIp || '';
    const isLoopback = requestIp === '::1'
      || requestIp === '127.0.0.1'
      || requestIp === '::ffff:127.0.0.1';
    const webrtcIp = visitor.webrtc && Array.isArray(visitor.webrtc.ips)
      ? visitor.webrtc.ips[0]
      : null;
    const identity = isLoopback && webrtcIp
      ? `webrtc:${webrtcIp}`
      : requestIp
        ? `request:${requestIp}`
        : webrtcIp
          ? `webrtc:${webrtcIp}`
          : `unknown:${visitor.id}`;

    if (!uniqueVisitors.has(identity)) {
      uniqueVisitors.set(identity, {
        ...visitor,
        identity,
        accessCount: 1,
        latestAccess: visitor.timestamp,
        firstAccess: visitor.timestamp
      });
      return;
    }

    const existing = uniqueVisitors.get(identity);
    existing.accessCount++;
    if (visitor.timestamp < existing.firstAccess) existing.firstAccess = visitor.timestamp;
  });

  return Array.from(uniqueVisitors.values());
}

async function clearVisitorHistory(ownerId) {
  const remaining = visitorHistory.filter(item => item.ownerId !== ownerId);
  visitorHistory.splice(0, visitorHistory.length, ...remaining);
  persistSupabaseState('visitor_history', visitorHistory);
  if (hasPersistentHistory) {
    await redisCommand(['DEL', VISITOR_HISTORY_KEY]);
    for (const record of remaining.slice().reverse()) await redisCommand(['LPUSH', VISITOR_HISTORY_KEY, JSON.stringify(record)]);
    if (remaining.length) await redisCommand(['LTRIM', VISITOR_HISTORY_KEY, 0, VISITOR_HISTORY_LIMIT - 1]);
  }
}

function decodeHeader(value) {
  if (!value) return null;
  try {
    return decodeURIComponent(value);
  } catch (error) {
    return value;
  }
}

function safeHistoryRequestData(value, depth = 0) {
  if (value === null || value === undefined) return value;
  if (depth > 8) return '[Dữ liệu lồng quá sâu]';
  if (Array.isArray(value)) return value.map(item => safeHistoryRequestData(item, depth + 1));
  if (typeof value !== 'object') return value;

  const safe = {};
  Object.entries(value).forEach(([key, item]) => {
    if (/authorization|cookie|token|secret|password|api[-_]?key|client[-_]?id|signature|^sign$/i.test(key)) {
      safe[key] = '[Đã ẩn thông tin xác thực]';
    } else if (/^headers?$/i.test(key)) {
      safe[key] = '[Đã ẩn cookie và thông tin xác thực]';
    } else if (/^proxy$/i.test(key) && item) {
      safe[key] = 'Có sử dụng';
    } else {
      safe[key] = safeHistoryRequestData(item, depth + 1);
    }
  });
  return safe;
}

function safeRequestPayload(req) {
  const query = safeHistoryRequestData(req.query || {});
  const body = safeHistoryRequestData(req.body || {});
  const payload = {};
  if (Object.keys(query).length) payload.query = query;
  if (Object.keys(body).length) payload.body = body;
  return Object.keys(payload).length ? payload : null;
}

app.use('/api', (req, res, next) => {
  if (
    req.path === '/'
    || req.path === '/presence'
    || req.path === '/call-history'
    || req.path === '/visitor-log'
    || req.path === '/visitor-history'
  ) return next();
  const startedAt = Date.now();
  let responseBody = null;
  let historySaved = false;
  const originalJson = res.json.bind(res);
  res.json = (body) => {
    responseBody = body;
    return originalJson(body);
  };
  const originalEnd = res.end.bind(res);
  res.end = (chunk, encoding, callback) => {
    const result = originalEnd(chunk, encoding, callback);
    if (!historySaved) {
      historySaved = true;
      const historyEntry = {
        category: 'internal',
        direction: 'Ứng dụng → API nội bộ',
        method: req.method,
        target: req.originalUrl,
        status: res.statusCode,
        durationMs: Date.now() - startedAt,
        payload: safeRequestPayload(req),
        responseBody,
        error: res.statusCode >= 400
          ? (responseBody && (responseBody.error || responseBody.message)) || `HTTP ${res.statusCode}`
          : null,
        errorResponse: res.statusCode >= 400 ? responseBody : null
      };
      addApiHistory(historyEntry).catch(error => console.error('API history write failed:', error.message));
    }
    return result;
  };
  next();
});

app.get('/api/call-history', requireAdmin, async (req, res) => {
  const calls = await getApiHistory();
  res.setHeader('Cache-Control', 'no-store, max-age=0');
  res.json({
    total: calls.length,
    calls,
    storage: USE_SUPABASE ? 'supabase' : (hasPersistentHistory ? 'redis' : 'memory'),
    persistent: USE_SUPABASE || hasPersistentHistory
  });
});

app.delete('/api/call-history', requireAdmin, async (req, res) => {
  try {
    await clearApiHistory();
    res.json({ success: true, message: 'Đã xóa toàn bộ lịch sử API call.' });
  } catch (error) {
    res.status(500).json({ error: `Không thể xóa lịch sử API: ${error.message}` });
  }
});

app.post('/api/visitor-log', async (req, res) => {
  try {
    const client = req.body && typeof req.body === 'object' ? req.body : {};
    const requestIp = req.get('x-vercel-forwarded-for')
      || req.get('x-forwarded-for')
      || req.get('x-real-ip')
      || req.socket.remoteAddress
      || null;
    const record = await addVisitorHistory({
      ownerId: req.user.id,
      username: req.user.username,
      requestIp,
      ipSources: {
        xVercelForwardedFor: req.get('x-vercel-forwarded-for') || null,
        xForwardedFor: req.get('x-forwarded-for') || null,
        xRealIp: req.get('x-real-ip') || null,
        socketAddress: req.socket.remoteAddress || null
      },
      webrtc: {
        supported: Boolean(client.webrtc && client.webrtc.supported),
        status: client.webrtc && client.webrtc.status ? client.webrtc.status : 'unknown',
        ips: Array.isArray(client.webrtc && client.webrtc.ips) ? client.webrtc.ips.slice(0, 10) : [],
        candidates: Array.isArray(client.webrtc && client.webrtc.candidates)
          ? client.webrtc.candidates.slice(0, 20)
          : []
      },
      geo: {
        continent: req.get('x-vercel-ip-continent') || null,
        country: req.get('x-vercel-ip-country') || null,
        region: decodeHeader(req.get('x-vercel-ip-country-region')),
        city: decodeHeader(req.get('x-vercel-ip-city')),
        latitude: req.get('x-vercel-ip-latitude') || null,
        longitude: req.get('x-vercel-ip-longitude') || null,
        timezone: decodeHeader(req.get('x-vercel-ip-timezone')),
        postalCode: decodeHeader(req.get('x-vercel-ip-postal-code'))
      },
      request: {
        referer: req.get('referer') || null,
        origin: req.get('origin') || null,
        host: req.get('host') || null,
        forwardedHost: req.get('x-forwarded-host') || null,
        forwardedProto: req.get('x-forwarded-proto') || req.protocol,
        vercelId: req.get('x-vercel-id') || null,
        deploymentUrl: req.get('x-vercel-deployment-url') || null,
        pageUrl: typeof client.pageUrl === 'string' ? client.pageUrl.slice(0, 2000) : null
      },
      browser: {
        userAgent: req.get('user-agent') || null,
        acceptLanguage: req.get('accept-language') || null,
        secChUa: req.get('sec-ch-ua') || null,
        secChUaMobile: req.get('sec-ch-ua-mobile') || null,
        secChUaPlatform: req.get('sec-ch-ua-platform') || null,
        dnt: req.get('dnt') || null,
        language: client.language || null,
        languages: Array.isArray(client.languages) ? client.languages.slice(0, 10) : [],
        platform: client.platform || null,
        timezone: client.timezone || null,
        viewport: client.viewport || null,
        screen: client.screen || null,
        devicePixelRatio: client.devicePixelRatio || null,
        hardwareConcurrency: client.hardwareConcurrency || null,
        deviceMemory: client.deviceMemory || null,
        cookieEnabled: typeof client.cookieEnabled === 'boolean' ? client.cookieEnabled : null,
        online: typeof client.online === 'boolean' ? client.online : null
      }
    });
    res.json({ success: true, id: record.id });
  } catch (error) {
    res.status(500).json({ error: `Không thể lưu lịch sử truy cập: ${error.message}` });
  }
});

app.get('/api/visitor-history', async (req, res) => {
  const visitors = await getVisitorHistory(req.user.id);
  res.setHeader('Cache-Control', 'no-store, max-age=0');
  res.json({
    total: visitors.length,
    totalAccesses: visitors.reduce((sum, visitor) => sum + visitor.accessCount, 0),
    visitors,
    storage: hasPersistentHistory ? 'redis' : 'memory',
    persistent: hasPersistentHistory
  });
});

app.delete('/api/visitor-history', async (req, res) => {
  try {
    await clearVisitorHistory(req.user.id);
    res.json({ success: true, message: 'Đã xóa lịch sử IP của tài khoản.' });
  } catch (error) {
    res.status(500).json({ error: `Không thể xóa lịch sử IP: ${error.message}` });
  }
});

async function trackedFetch(url, options = {}, payload = null, metadata = {}) {
  const startedAt = Date.now();
  const method = (options.method || 'GET').toUpperCase();
  const actualPayload = method === 'GET' ? null : payload;
  const category = url.includes('api.ipify.org') ? 'proxy' : 'external';
  const direction = category === 'proxy'
    ? 'Server → API kiểm tra Proxy/IP'
    : 'Server → API Google/Dữ liệu ngoài';
  try {
    const response = await fetch(url, options);
    let responseBody = null;
    try {
      const contentType = String(response.headers.get('content-type') || '');
      if (contentType.includes('json') || contentType.includes('text') || contentType.includes('javascript')) {
        const clone = response.clone();
        const text = await clone.text();
        const cleanText = text.replace(/\u0000/g, '');
        try { responseBody = JSON.parse(cleanText); } catch (e) { responseBody = cleanText.slice(0, 1000); }
      }
    } catch (e) {}

    const historyMetadata = { ...metadata };
    delete historyMetadata.captureResponseIp;
    if (metadata.captureResponseIp) {
      const responseIpCandidate = [
        response.headers.get('x-proxy-ip'),
        response.headers.get('x-client-ip'),
        response.headers.get('cf-connecting-ip'),
        response.headers.get('x-forwarded-for'),
        responseBody?.proxy_ip,
        responseBody?.proxyIp,
        responseBody?.client_ip,
        responseBody?.clientIp,
        responseBody?.ip
      ].find(Boolean);
      const responseIp = String(responseIpCandidate || '').split(',')[0].trim();
      if (/^(?:\d{1,3}\.){3}\d{1,3}$|^[0-9a-f:]{2,}$/i.test(responseIp)) {
        historyMetadata.proxyIpChecked = responseIp;
        historyMetadata.proxyIpSource = 'TrafficLens response';
      } else {
        historyMetadata.proxyIpUnavailable = true;
      }
    }
    // Bản ghi được thêm vào RAM đồng bộ ngay trong addApiHistory; Redis/Supabase
    // tiếp tục lưu nền để không giữ response ngoài chỉ vì kho nhật ký chậm.
    addApiHistory({
      category,
      direction,
      method,
      target: url,
      status: response.status,
      durationMs: Date.now() - startedAt,
      payload: actualPayload,
      responseBody,
      ...historyMetadata
    }).catch(error => console.error('API history background save failed:', error.message));
    return response;
  } catch (error) {
    const causeDetails = [error.message, error.cause?.message, error.cause?.code]
      .filter(Boolean)
      .filter((value, index, values) => values.indexOf(value) === index)
      .join(' | ');
    addApiHistory({
      category,
      direction,
      method,
      target: url,
      status: 'ERROR',
      durationMs: Date.now() - startedAt,
      payload: actualPayload,
      error: causeDetails || 'Lỗi kết nối không xác định',
      ...metadata
    }).catch(historyError => console.error('API error history background save failed:', historyError.message));
    throw error;
  }
}

app.get('/api', (req, res) => {
  const forwardedProto = req.get('x-forwarded-proto');
  const protocol = forwardedProto ? forwardedProto.split(',')[0].trim() : req.protocol;
  const baseUrl = `${protocol}://${req.get('host')}`;

  res.json({
    name: 'Google Ads Transparency Scraper API',
    version: '1.0.0',
    environment: process.env.VERCEL ? 'vercel' : 'local',
    baseUrl,
    endpoints: API_CATALOG.map(endpoint => ({ ...endpoint, url: `${baseUrl}${endpoint.path}` }))
  });
});

// Simple sleep utility
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

/**
 * Determine Proxy Agent or direct connection based on proxy parameters
 */
function getProxyDispatcher(proxy, disableProxy) {
  const isProxyDisabled = disableProxy === true ||
    proxy === 'NONE' ||
    proxy === 'DIRECT' ||
    proxy === false ||
    proxy === 'false' ||
    proxy === 'OFF';

  if (isProxyDisabled) {
    console.log('⚡ Proxy is DISABLED — Sending requests via Direct Server IP');
    return undefined;
  }

  const proxyUrl = normalizeProxyUrl((typeof proxy === 'string' && proxy.trim()) ? proxy.trim() : DEFAULT_PROXY_URL);
  if (!proxyUrl) return undefined;

  const safeUrl = proxyUrl.replace(/:[^:@]+@/, ':****@');
  console.log(`🛡️ Using Proxy Dispatcher: ${safeUrl}`);
  return new ProxyAgent({
    uri: proxyUrl,
    keepAliveTimeout: 15000,
    keepAliveMaxTimeout: 30000
  });
}

const googleProxyCircuit = new Map();
const GOOGLE_PROXY_COOLDOWN_MS = 10 * 60 * 1000;
const GOOGLE_PROXY_NETWORK_RETRIES = 10;
function googleProxyCircuitKey(proxy) {
  return crypto.createHash('sha256').update(String(proxy || '')).digest('hex');
}
function isGoogleProxyCircuitOpen(proxy) {
  const key = googleProxyCircuitKey(proxy);
  const blockedUntil = googleProxyCircuit.get(key) || 0;
  if (blockedUntil <= Date.now()) {
    googleProxyCircuit.delete(key);
    return false;
  }
  return true;
}
function openGoogleProxyCircuit(proxy) {
  googleProxyCircuit.set(googleProxyCircuitKey(proxy), Date.now() + GOOGLE_PROXY_COOLDOWN_MS);
}
function closeGoogleProxyCircuit(proxy) {
  googleProxyCircuit.delete(googleProxyCircuitKey(proxy));
}

// Check Current IP Address (Direct or via Proxy with multi-service fallback)
app.get('/api/check-ip', async (req, res) => {
  const canProxy = req.user.role === 'admin' || Boolean(req.user.canUseProxy);
  const proxyParam = canProxy ? authStore.settings.proxyUrl : '';
  const useProxy = canProxy && req.query.useProxy !== 'false' && Boolean(proxyParam);
  const dispatcher = useProxy ? getProxyDispatcher(proxyParam) : undefined;

  const ipServices = [
    'https://api.ipify.org?format=json',
    'https://ipinfo.io/json',
    'https://icanhazip.com'
  ];

  let detectedIp = null;
  let lastError = null;

  for (const serviceUrl of ipServices) {
    try {
      const fetchOptions = {
        signal: AbortSignal.timeout(4000)
      };
      if (dispatcher) {
        fetchOptions.dispatcher = dispatcher;
      }

      const response = await trackedFetch(serviceUrl, fetchOptions, null, {
        proxyIpChecked: Boolean(dispatcher)
      });

      if (response.ok) {
        const text = await response.text();
        try {
          const json = JSON.parse(text);
          detectedIp = json.ip || json.ip_address;
        } catch (e) {
          detectedIp = text.trim();
        }

        if (detectedIp && detectedIp.length >= 7) {
          break;
        }
      }
    } catch (err) {
      lastError = err.message;
    }
  }

  if (detectedIp) {
    res.json({
      success: true,
      ip: detectedIp,
      isProxy: Boolean(dispatcher),
      proxyUsed: Boolean(dispatcher)
    });
  } else {
    res.status(500).json({
      success: false,
      error: `Không thể kiểm tra IP: ${lastError || 'Timeout'}`,
      isProxy: Boolean(dispatcher)
    });
  }
});

// ==========================================
// FEATURE: Persistent Tesseract OCR Worker Pool
// ==========================================
const OCR_POOL_SIZE = Math.max(2, Math.min(6, Number(process.env.OCR_POOL_SIZE) || 4));
const ocrWorkerPool = [];
let ocrWorkerIndex = 0;
let isOcrPoolInitializing = false;

async function getOcrWorker() {
  if (ocrWorkerPool.length > 0) {
    const worker = ocrWorkerPool[ocrWorkerIndex % ocrWorkerPool.length];
    ocrWorkerIndex++;
    return worker;
  }
  
  console.log('⚡ Initializing fallback Tesseract worker...');
  const worker = await createWorker('eng');
  await worker.setParameters({
    tessjs_create_hocr: '0',
    tessjs_create_tsv: '0',
    tessjs_create_box: '0',
    tessjs_create_unlv: '0',
    tessjs_create_osd: '0'
  });
  ocrWorkerPool.push(worker);
  return worker;
}

async function initOcrWorkerPool() {
  if (isOcrPoolInitializing) return;
  isOcrPoolInitializing = true;
  console.log(`⚡ Pre-warming ${OCR_POOL_SIZE} persistent Tesseract OCR workers for maximum speed...`);
  try {
    for (let i = 0; i < OCR_POOL_SIZE; i++) {
      const worker = await createWorker('eng');
      await worker.setParameters({
        tessjs_create_hocr: '0',
        tessjs_create_tsv: '0',
        tessjs_create_box: '0',
        tessjs_create_unlv: '0',
        tessjs_create_osd: '0'
      });
      ocrWorkerPool.push(worker);
    }
    console.log(`🚀 ${OCR_POOL_SIZE} Tesseract OCR workers are ready & cached! OCR speed is now 10x-50x faster!`);
  } catch (err) {
    console.error('Lỗi khởi tạo OCR Worker Pool:', err);
  }
}

// Pre-warm OCR workers in background
// OCR ảnh được thực hiện trên trình duyệt người dùng; server không khởi tạo worker OCR.

/**
 * Fetch external IP address using the designated proxy dispatcher
 */
async function getExternalIp(dispatcher) {
  try {
    const res = await trackedFetch('https://api.ipify.org?format=json', {
      headers: { 'connection': 'close' },
      dispatcher: dispatcher || undefined,
      signal: AbortSignal.timeout(4000)
    });
    if (res.ok) {
      const data = await res.json();
      return data.ip || 'Không rõ';
    }
  } catch (e) {
    console.error('Error fetching active proxy/server IP:', e.message);
  }
  return 'Không lấy được IP';
}

/**
 * Clean and parse Google RPC response.
 * Strips the safety prefix )]}'\n if present.
 */
function parseGoogleRPCResponse(text) {
  if (text.startsWith(")]}'\n")) {
    text = text.substring(5);
  }
  return JSON.parse(text);
}

/**
 * Format timestamp strings/nanoseconds into a clean readable date
 */
function formatRPCDate(timestampObj) {
  if (!timestampObj || !timestampObj['1']) return null;
  const seconds = parseInt(timestampObj['1'], 10);
  return new Date(seconds * 1000).toISOString().split('T')[0];
}

function getDefaultSearchTextFilters() {
  const endDate = new Date();
  const startDate = new Date(endDate);
  startDate.setUTCDate(startDate.getUTCDate() - 7);
  const toNumber = date => parseInt(date.toISOString().slice(0, 10).replace(/-/g, ''), 10);
  return {
    platform: 1,
    format: [3],
    startDate: toNumber(startDate),
    endDate: toNumber(endDate)
  };
}

const DEFAULT_DOMAIN_CREATIVE_LIMIT = 5000;
const MAX_DOMAIN_CREATIVE_LIMIT = 1000000;
const domainScanControls = new Map();

function parseDomainCreativeLimit(value) {
  if (value === undefined || value === null || value === '') {
    return DEFAULT_DOMAIN_CREATIVE_LIMIT;
  }
  const parsed = Number(value);
  if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > MAX_DOMAIN_CREATIVE_LIMIT) {
    return null;
  }
  return parsed;
}

function normalizeProjectDomain(value) {
  let input = String(value || '').replace(/[\u200B-\u200D\u2060\uFEFF]/g, '').replace(/&amp;/gi, '&').trim();
  if (!input) return '';
  input = input.replace(/^(https?):\/(?!\/)/i, '$1://').replace(/([a-z0-9-])\s*\.\s*(?=[a-z0-9-])/gi, '$1.');
  const annotatedDomain = /^((?:[a-z0-9-]+\.)+[a-z]{2,63})(?:\s+\([^)]*\))$/i.exec(input)?.[1];
  if (annotatedDomain) input = annotatedDomain;

  const hyperlinkFormulaUrl = /^=HYPERLINK\(\s*["']([^"']+)["']/i.exec(input)?.[1];
  const markdownUrl = /^\s*\[[^\]]*\]\(\s*([^\s)]+)\s*\)\s*$/i.exec(input)?.[1];
  if (hyperlinkFormulaUrl) input = hyperlinkFormulaUrl;
  else if (markdownUrl) input = markdownUrl;
  else {
    const embeddedUrl = /https?:\/\/[^\s<>{}\[\]"']+/i.exec(input)?.[0];
    if (embeddedUrl) input = embeddedUrl.replace(/[),.;!?]+$/, '');
  }
  input = input.trim().replace(/^["'`(<\[]+|["'`)>\]]+$/g, '');
  input = input.replace(/^(https?):\/(?!\/)/i, '$1://').replace(/([a-z0-9-])\s*\.\s*(?=[a-z0-9-])/gi, '$1.');
  const labeledDomain = /^((?:[a-z0-9-]+\.)+[a-z]{2,63})(?:\s+\([^)]*\))$/i.exec(input)?.[1];
  if (labeledDomain) input = labeledDomain;

  try {
    const parsed = new URL(/^https?:\/\//i.test(input) ? input : `https://${input}`);
    const hostname = parsed.hostname.toLowerCase().replace(/^www\./i, '').replace(/\.$/, '');
    return hostname.includes('.') && !/\s/.test(hostname) ? hostname : '';
  } catch (error) {
    return '';
  }
}

// Kho Domain tracks projects/brands, not country-specific storefronts. Keep this
// separate from normalizeProjectDomain because the crawler still needs the exact
// host while traversing a site.
const COUNTRY_SECOND_LEVEL_SUFFIXES = new Set([
  'ac','asn','biz','co','com','edu','firm','gen','go','gov','id','ind','info','ltd','me','mil','net','ne','nom','or','org','plc','pro','sch','web'
]);
const AFFILIATE_MERCHANT_SUBDOMAIN_HOSTS = new Set(['sjv.io', 'pxf.io', 'evyy.net']);
const SHARED_AFFILIATE_TRACKING_HOSTS = new Set([...AFFILIATE_MERCHANT_SUBDOMAIN_HOSTS, 'jdoqocy.com', 'kqzyfj.com', 'anrdoezrs.net', 'tkqlhce.com', 'dpbolvw.net', 'linksynergy.com', 'awin1.com', 'tidd.ly', 'prf.hn', 'rstyle.me', 'shopstyle.it', 'skimresources.com']);
const GENERIC_AFFILIATE_SUBDOMAINS = new Set(['www', 'go', 'click', 'track', 'tracking', 'link', 'links', 'affiliate', 'affiliates']);
function projectDomainIdentity(value) {
  const hostname = normalizeProjectDomain(value);
  if (!hostname) return '';
  const labels = hostname.split('.').filter(Boolean);
  if (labels.length < 2) return '';
  // Some shared affiliate hosts encode the merchant in the left-most
  // subdomain. Never collapse these links into fake projects such as sjv.com
  // or pxf.com. Generic tracker URLs without a merchant are not projects.
  const affiliateBase = labels.slice(-2).join('.');
  if (labels.length >= 3 && AFFILIATE_MERCHANT_SUBDOMAIN_HOSTS.has(affiliateBase)) {
    const merchant = labels[0].toLowerCase();
    if (/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(merchant) && !GENERIC_AFFILIATE_SUBDOMAINS.has(merchant)) return `${merchant}.com`;
  }
  if (SHARED_AFFILIATE_TRACKING_HOSTS.has(affiliateBase)) return '';
  const tld = labels.at(-1);
  const hasCountrySuffix = tld.length === 2 && labels.length >= 3 && COUNTRY_SECOND_LEVEL_SUFFIXES.has(labels.at(-2));
  const brand = labels.at(hasCountrySuffix ? -3 : -2);
  return /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i.test(brand || '') ? `${brand}.com` : '';
}

function uniqueProjectDomains(values) {
  return [...new Set((Array.isArray(values) ? values : []).map(projectDomainIdentity).filter(Boolean))];
}

function updateDomainScanControl(req, res, action) {
  const control = domainScanControls.get(req.params.scanId);
  if (!control) {
    return res.status(404).json({ error: 'Phiên quét không tồn tại hoặc đã kết thúc.' });
  }

  if (action === 'pause') control.paused = true;
  if (action === 'resume') {
    control.paused = false;
    control.resumeWaiters.splice(0).forEach(resolve => resolve());
  }
  if (action === 'finish') {
    control.finishRequested = true;
    control.paused = false;
    control.resumeWaiters.splice(0).forEach(resolve => resolve());
  }

  res.json({
    success: true,
    scanId: req.params.scanId,
    status: control.finishRequested ? 'finishing' : control.paused ? 'paused' : 'running'
  });
}

app.post('/api/extract/:scanId/pause', (req, res) => updateDomainScanControl(req, res, 'pause'));
app.post('/api/extract/:scanId/resume', (req, res) => updateDomainScanControl(req, res, 'resume'));
app.post('/api/extract/:scanId/finish', (req, res) => updateDomainScanControl(req, res, 'finish'));

// API endpoint to run extraction with real-time progress streaming
app.post('/api/extract', async (req, res) => {
  const { headers, proxy, fReq } = req.body;
  const allTime = req.body.dateRangePreset === 'all';
  const domain = normalizeProjectDomain(req.body.domain);
  const creativeLimit = parseDomainCreativeLimit(req.body.limit);
  const scanId = typeof req.body.scanId === 'string' && req.body.scanId.trim()
    ? req.body.scanId.trim()
    : `scan-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;

  if (!domain) {
    return res.status(400).json({ error: 'Domain is required.' });
  }
  if (creativeLimit === null) {
    return res.status(400).json({ error: `limit phải là số nguyên từ 1 đến ${MAX_DOMAIN_CREATIVE_LIMIT}.` });
  }
  if (!headers || Object.keys(headers).length === 0) {
    return res.status(400).json({ error: 'Thiếu Headers/Cookies xác thực Google.' });
  }
  if (domainScanControls.has(scanId)) {
    return res.status(409).json({ error: 'scanId đang được sử dụng bởi một phiên quét khác.' });
  }

  const scanControl = { paused: false, finishRequested: false, resumeWaiters: [] };
  domainScanControls.set(scanId, scanControl);

  // Set up headers for NDJSON streaming
  res.writeHead(200, {
    'Content-Type': 'application/x-ndjson',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive'
  });

  const sendChunk = (data) => {
    res.write(JSON.stringify(data) + '\n');
  };

  const advertisersMap = {};
  let overallCreativesCount = 0;
  let totalEstimatedDomainAds = 0;
  let pageNum = 0;
  const serializeAdvertisers = () => Object.values(advertisersMap).map(adv => ({
    ...adv,
    creativeSet: undefined
  })).sort((a, b) => b.creativeCount - a.creativeCount);

  try {
    const { disableProxy } = req.body;
    const dispatcher = getProxyDispatcher(proxy, disableProxy);

    const activeIp = await getExternalIp(dispatcher);
    sendChunk({ type: 'ip', ip: activeIp });

    // Map of aggregated advertisers
    sendChunk({ type: 'scan', scanId, limit: creativeLimit });
    sendChunk({ type: 'status', message: `Đang bắt đầu quét tên miền dự án (giới hạn ${creativeLimit.toLocaleString('vi-VN')} mẫu)...` });

    // Single phase domain level scan
    let domainPageToken = null;
    let isDomainFinished = false;
    const proxyConfiguredAtStart = !disableProxy && proxy && !['NONE', 'DIRECT', 'OFF', 'false'].includes(String(proxy));
    let proxyTransportUnavailable = proxyConfiguredAtStart && isGoogleProxyCircuitOpen(proxy);
    if (proxyTransportUnavailable) {
      sendChunk({ type: 'status', message: 'Proxy đang trong thời gian tạm nghỉ vì các lần kiểm tra gần nhất không truy cập được Google. Lượt quét này sẽ dùng kết nối trực tiếp ngay.' });
    }

    while (!isDomainFinished) {
      if (scanControl.finishRequested) break;
      if (scanControl.paused) {
        sendChunk({ type: 'paused', scanId, message: 'Đã tạm dừng. Có thể tiếp tục hoặc dừng và xử lý dữ liệu hiện có.' });
        await new Promise(resolve => scanControl.resumeWaiters.push(resolve));
        if (scanControl.finishRequested) break;
        sendChunk({ type: 'resumed', scanId, message: 'Đã tiếp tục quét.' });
      }

      pageNum++;
      const defaultFilters = getDefaultSearchTextFilters();

      const reqPayload = {
        "2": 100, // Fetch 100 creatives per page request
        "3": {
          "4": defaultFilters.platform,
          "12": { "1": domain, "2": true },
          "14": defaultFilters.format
        },
        "7": { "1": 1, "2": 22, "3": 2704 }
      };
      if (!allTime) {
        reqPayload['3']['6'] = defaultFilters.startDate;
        reqPayload['3']['7'] = defaultFilters.endDate;
      }

      if (fReq && fReq['3']) {
        if (fReq['3']['4'] !== undefined) reqPayload['3']['4'] = fReq['3']['4'];
        if (!allTime && fReq['3']['6'] !== undefined) reqPayload['3']['6'] = fReq['3']['6'];
        if (!allTime && fReq['3']['7'] !== undefined) reqPayload['3']['7'] = fReq['3']['7'];
        if (Array.isArray(fReq['3']['8']) && fReq['3']['8'].length) reqPayload['3']['8'] = fReq['3']['8'];
        if (fReq['3']['14'] !== undefined) reqPayload['3']['14'] = fReq['3']['14'];
      }
      if (domainPageToken) reqPayload["4"] = domainPageToken;

      // Extract parameter details for descriptive logging
      const dateRangeStr = (reqPayload['3']['6'] && reqPayload['3']['7']) 
        ? `${reqPayload['3']['6']} - ${reqPayload['3']['7']}` 
        : (allTime ? 'Tất cả thời gian' : 'Theo cấu hình xác thực');
      
      const platformStr = reqPayload['3']['4'] === 1 ? 'Search' : (reqPayload['3']['4'] === 2 ? 'YouTube' : (reqPayload['3']['4'] === 4 ? 'Display' : 'Khác'));
      
      const formatsArr = reqPayload['3']['14'];
      const formatStr = Array.isArray(formatsArr) 
        ? formatsArr.map(f => f === 1 ? 'Hình ảnh' : (f === 3 ? 'Chữ' : (f === 2 ? 'Video' : 'Khác'))).join(', ') 
        : 'Tất cả';

      const tokenStr = reqPayload["4"] ? reqPayload["4"] : 'Không có (Trang đầu)';

      sendChunk({ 
        type: 'status', 
        message: `[Yêu cầu Trang ${pageNum}]: Lọc số lượng: 100 | Ngày: ${dateRangeStr} | Nền tảng: ${platformStr} | Định dạng: ${formatStr} | Token: ${tokenStr}` 
      });

      const postData = new URLSearchParams();
      postData.append('f.req', JSON.stringify(reqPayload));
      const proxyConfiguredForScan = !disableProxy && proxy && !['NONE', 'DIRECT', 'OFF', 'false'].includes(String(proxy));
      const proxyEnabled = proxyConfiguredForScan && !proxyTransportUnavailable;
      // Khi Proxy circuit đang mở, IP trực tiếp có thể bị Google rate-limit
      // ngắn hạn. Cho đủ thời gian hạ nhiệt thay vì dừng sau khoảng 6 giây.
      const maxAttempts = proxyEnabled ? 30 : 8;
      let response = null;
      let lastFetchError = null;
      for (let attempt = 0; attempt < maxAttempts; attempt++) {
        if (scanControl.finishRequested) break;
        if (scanControl.paused) {
          sendChunk({ type: 'paused', scanId, message: 'Đã tạm dừng trong lúc thử lại kết nối Proxy.' });
          await new Promise(resolve => scanControl.resumeWaiters.push(resolve));
          if (scanControl.finishRequested) break;
          sendChunk({ type: 'resumed', scanId, message: 'Đã tiếp tục quét.' });
        }
        let requestDispatcher = proxyEnabled ? dispatcher : undefined;
        let requestProxyIp = proxyEnabled ? activeIp : null;
        if (proxyEnabled && attempt > 0) {
          requestDispatcher = getProxyDispatcher(proxy, false);
          requestProxyIp = await getExternalIp(requestDispatcher);
          const retryLimitLabel = lastFetchError ? GOOGLE_PROXY_NETWORK_RETRIES : maxAttempts;
          sendChunk({ type: 'status', message: `Proxy đã tạo kết nối/IP mới, tiếp tục lại trang ${pageNum} (lần ${attempt + 1}/${retryLimitLabel}) • IP: ${requestProxyIp}` });
        }
        try {
          response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
            method: 'POST',
            headers: { ...headers, 'content-type': 'application/x-www-form-urlencoded' },
            body: postData.toString(),
            dispatcher: requestDispatcher || undefined
          }, { 'f.req': reqPayload }, { proxyIpChecked: requestProxyIp });
          if (proxyEnabled) closeGoogleProxyCircuit(proxy);
          lastFetchError = null;
        } catch (error) {
          lastFetchError = error;
          response = null;
          const proxyErrorDetail = String(error.cause?.message || error.message || '');
          const proxyTransportCancelled = /request was cancelled|request aborted|proxy connect|socket closed/i.test(proxyErrorDetail);
          if (proxyEnabled && (proxyTransportCancelled || attempt >= GOOGLE_PROXY_NETWORK_RETRIES - 1)) {
            proxyTransportUnavailable = true;
            openGoogleProxyCircuit(proxy);
            const reason = proxyTransportCancelled
              ? `Proxy đã hủy kết nối tới Google (${proxyErrorDetail})`
              : `Proxy đã đổi ${GOOGLE_PROXY_NETWORK_RETRIES} kết nối nhưng không truy cập được Google (${proxyErrorDetail})`;
            sendChunk({ type: 'status', message: `${reason}. Đang tự chuyển sang kết nối trực tiếp để không làm gián đoạn lượt quét...` });
            try {
              response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
                method: 'POST',
                headers: { ...headers, 'content-type': 'application/x-www-form-urlencoded' },
                body: postData.toString()
              }, { 'f.req': reqPayload }, { proxyFallback: true });
              lastFetchError = null;
              // Trước đây response 429 vẫn bị báo "kết nối thành công" rồi break,
              // khiến nhánh retry 429 bên dưới không bao giờ được chạy.
              if (response.status === 429) {
                const directRetryLimit = 5;
                for (let directAttempt = 1; directAttempt < directRetryLimit && response.status === 429; directAttempt++) {
                  try { await response.body?.cancel(); } catch (cancelError) {}
                  const delay = Math.min(10000, 1500 * directAttempt);
                  sendChunk({ type: 'status', message: `Kết nối trực tiếp tới Google trả về 429. Đang chờ ${Math.ceil(delay / 1000)} giây và thử lại ${directAttempt + 1}/${directRetryLimit}...` });
                  await sleep(delay);
                  response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
                    method: 'POST',
                    headers: { ...headers, 'content-type': 'application/x-www-form-urlencoded' },
                    body: postData.toString()
                  }, { 'f.req': reqPayload }, { proxyFallback: true, direct429Retry: directAttempt + 1 });
                }
              }
              if (response.ok) {
                sendChunk({ type: 'status', message: 'Kết nối trực tiếp tới Google thành công. Tiếp tục quét và vẫn giữ toàn bộ dữ liệu đã lấy.' });
              } else {
                sendChunk({ type: 'status', message: `Kết nối trực tiếp tới Google trả về HTTP ${response.status} ${response.statusText || ''}.` });
              }
              break;
            } catch (directError) {
              lastFetchError = directError;
              throw new Error(`Proxy không truy cập được Google và kết nối trực tiếp cũng thất bại: ${directError.cause?.message || directError.message}`);
            }
          }
          if (attempt === maxAttempts - 1) break;
          const retryMessage = proxyEnabled
            ? `Kết nối Proxy tới Google bị gián đoạn: ${error.cause?.message || error.message}. Đang thử lại kết nối ${Math.min(attempt + 2, GOOGLE_PROXY_NETWORK_RETRIES)}/${GOOGLE_PROXY_NETWORK_RETRIES}...`
            : `Kết nối trực tiếp tới Google bị gián đoạn: ${error.cause?.message || error.message}. Đang thử lại ${attempt + 2}/${maxAttempts}...`;
          sendChunk({ type: 'status', message: retryMessage });
          await sleep(Math.min(8000, 1200 * (attempt + 1)));
          continue;
        }
        if (response.status !== 429) break;
        try { await response.body?.cancel(); } catch (error) {}
        if (!proxyEnabled && attempt === maxAttempts - 1) break;
        const retryAfterSeconds = Number(response.headers.get('retry-after'));
        const delay = proxyEnabled
          ? Math.min(5000, 700 + attempt * 350)
          : Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
            ? Math.min(60000, retryAfterSeconds * 1000)
            : Math.min(15000, 2000 * (attempt + 1));
        sendChunk({ type: 'status', message: `Google trả về 429. ${proxyEnabled ? 'Đang tự đổi kết nối Proxy/IP' : 'Đang thử lại'} sau ${Math.ceil(delay / 1000)} giây; dữ liệu đã lấy vẫn được giữ.` });
        await sleep(delay);
      }

      if (scanControl.finishRequested) break;
      if (!response && lastFetchError) throw lastFetchError;
      if (!response) throw new Error('Không nhận được phản hồi từ Google.');

      if (!response.ok) {
        throw new Error(`Google API returned status code ${response.status}: ${response.statusText}`);
      }

      const responseText = await response.text();
      let data;
      try {
        data = parseGoogleRPCResponse(responseText);
      } catch (parseErr) {
        throw new Error('Không thể phân tích dữ liệu JSON trả về từ Google. Có thể phiên đăng nhập / cookie của bạn đã hết hạn.');
      }

      const creatives = data['1'] || [];
      const pageTokenNew = data['2'] || null;
      if (data['5']) totalEstimatedDomainAds = parseInt(data['5'], 10);

      for (const creative of creatives) {
        if (overallCreativesCount >= creativeLimit) {
          isDomainFinished = true;
          break;
        }
        const creativeAdvId = creative['1'];
        if (!creativeAdvId) continue;

        // Image check
        let imageUrl = null;
        try {
          const html = creative['3']['3']['2'];
          if (html) {
            const match = html.match(/src="([^"]+)"/);
            if (match) imageUrl = match[1];
          }
        } catch (e) {}

        let previewUrl = null;
        if (!imageUrl) {
          try {
            const preview = creative['3']['1']['4'];
            if (preview && preview.startsWith('http')) {
              if (preview.includes('displayads-formats') || preview.includes('content.js')) {
                previewUrl = preview;
              } else {
                imageUrl = preview;
              }
            }
          } catch (e) {}
        }

        const creativeId = creative['2'];
        const rawAdvName = creative['12'] ? creative['12'].trim() : 'Nhà quảng cáo không xác định';
        const groupKey = rawAdvName.toLowerCase();
        const creativeFormat = creative['4'];
        const firstSeen = formatRPCDate(creative['6']);
        const lastSeen = formatRPCDate(creative['7']);

        if (!advertisersMap[groupKey]) {
          advertisersMap[groupKey] = {
            id: creativeAdvId,
            allIds: [creativeAdvId],
            name: rawAdvName,
            creativeCount: 0,
            formats: {},
            firstSeen: firstSeen,
            lastSeen: lastSeen,
            creativesList: [],
            creativeSet: new Set()
          };
        } else {
          if (!advertisersMap[groupKey].allIds.includes(creativeAdvId)) {
            advertisersMap[groupKey].allIds.push(creativeAdvId);
          }
        }

        const advObj = advertisersMap[groupKey];
        if (!advObj.creativeSet.has(creativeId)) {
          advObj.creativeSet.add(creativeId);
          advObj.creativeCount++;
          advObj.formats[creativeFormat] = (advObj.formats[creativeFormat] || 0) + 1;
          overallCreativesCount++;

          const dStart = firstSeen ? new Date(firstSeen) : null;
          const dEnd = lastSeen ? new Date(lastSeen) : null;
          const daysActive = (dStart && dEnd) ? (Math.ceil(Math.abs(dEnd - dStart) / (1000 * 60 * 60 * 24)) + 1) : 0;

          advObj.creativesList.push({
            creativeId,
            advertiserId: creativeAdvId,
            format: creativeFormat,
            firstSeen,
            lastSeen,
            daysActive,
            imageUrl,
            previewUrl
          });
        }

        if (firstSeen && (!advObj.firstSeen || firstSeen < advObj.firstSeen)) {
          advObj.firstSeen = firstSeen;
        }
        if (lastSeen && (!advObj.lastSeen || lastSeen > advObj.lastSeen)) {
          advObj.lastSeen = lastSeen;
        }
      }

      sendChunk({
        type: 'status',
        message: `-> Trang ${pageNum}: Nhận và xử lý ${creatives.length} QC. ${pageTokenNew ? 'Mã phân trang tiếp theo: ' + pageTokenNew : '(Không còn mã phân trang - Đã quét hết)'}`
      });

      sendChunk({
        type: 'progress',
        page: pageNum,
        pageCreativesCount: creatives.length,
        accumulatedCreativesCount: overallCreativesCount,
        accumulatedAdvertisersCount: Object.keys(advertisersMap).length,
        totalEstimatedAds: totalEstimatedDomainAds,
        limit: creativeLimit
      });
      sendChunk({ type: 'checkpoint', advertisers: serializeAdvertisers(), totalCreatives: overallCreativesCount });

      if (overallCreativesCount >= creativeLimit) {
        isDomainFinished = true;
        sendChunk({ type: 'status', message: `Đã đạt giới hạn ${creativeLimit.toLocaleString('vi-VN')} mẫu quảng cáo cho domain này.` });
      } else if (scanControl.finishRequested) {
        isDomainFinished = true;
      } else if (!pageTokenNew || pageTokenNew === domainPageToken || pageNum >= 10000) {
        isDomainFinished = true;
      } else {
        domainPageToken = pageTokenNew;
        await sleep(250 + Math.random() * 250);
      }
    }

    // Convert Sets to JSON friendly arrays before sending to client
    const advertisersList = serializeAdvertisers();

    sendChunk({
      type: 'done',
      message: 'Hoàn thành!',
      reason: scanControl.finishRequested ? 'user_finished' : overallCreativesCount >= creativeLimit ? 'limit_reached' : 'completed',
      limit: creativeLimit,
      scanId,
      totalCreatives: overallCreativesCount,
      totalAdvertisers: advertisersList.length,
      advertisers: advertisersList
    });

  } catch (error) {
    console.error('Extraction Error:', error);
    const partialAdvertisers = serializeAdvertisers();
    sendChunk({ type: 'error', error: error.message, advertisers: partialAdvertisers, totalCreatives: overallCreativesCount, recoverable: partialAdvertisers.length > 0 });
  } finally {
    domainScanControls.delete(scanId);
    res.end();
  }
});

// Endpoint: Fetch creatives for specific Advertiser ID
app.post('/api/advertiser-creatives', async (req, res) => {
  const { advId, headers, proxy, fReq, disableProxy, monitorMode } = req.body;
  const allTime = req.body.dateRangePreset === 'all';
  if (!advId) {
    return res.status(400).json({ error: 'Thiếu ID nhà quảng cáo (advId)' });
  }

  res.writeHead(200, {
    'Content-Type': 'application/json; charset=utf-8',
    'Transfer-Encoding': 'chunked',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive'
  });

  const sendChunk = (data) => {
    res.write(JSON.stringify(data) + '\n');
  };

  try {
    const dispatcher = getProxyDispatcher(proxy, disableProxy);

    const activeIp = await getExternalIp(dispatcher);
    sendChunk({ type: 'ip', ip: activeIp });

    const activeHeaders = headers || {};

    let pageToken = null;
    let pageNum = 0;
    let totalEstimated = 0;
    let totalFetched = 0;
    let isFinished = false;

    sendChunk({ type: 'status', message: `Khởi tạo quét chi tiết mẫu quảng cáo cho nhà QC ID: ${advId}...` });

    let skippedTotal = 0;

    while (!isFinished) {
      pageNum++;
      const defaultFilters = getDefaultSearchTextFilters();

      const reqPayload = {
        "2": 100,
        "3": {
          "4": defaultFilters.platform,
          "12": { "1": "", "2": true },
          "13": { "1": [ advId ] },
          "14": defaultFilters.format
        },
        "7": { "1": 1, "2": 22, "3": 2704 }
      };
      if (!allTime) {
        reqPayload['3']['6'] = defaultFilters.startDate;
        reqPayload['3']['7'] = defaultFilters.endDate;
      }

      if (fReq && fReq['3']) {
        if (!allTime && fReq['3']['6'] !== undefined) reqPayload['3']['6'] = fReq['3']['6'];
        if (!allTime && fReq['3']['7'] !== undefined) reqPayload['3']['7'] = fReq['3']['7'];
        if (Array.isArray(fReq['3']['8']) && fReq['3']['8'].length) reqPayload['3']['8'] = fReq['3']['8'];
      }
      // Advertiser lookup is always restricted to Search platform and TEXT format.
      reqPayload['3']['4'] = 1;
      reqPayload['3']['14'] = [3];
      if (pageToken) reqPayload["4"] = pageToken;

      // Extract parameter details for descriptive logging
      const dateRangeStr = (reqPayload['3']['6'] && reqPayload['3']['7']) 
        ? `${reqPayload['3']['6']} - ${reqPayload['3']['7']}` 
        : (allTime ? 'Tất cả thời gian' : 'Theo cấu hình xác thực');
      
      const platformStr = reqPayload['3']['4'] === 1 ? 'Search' : (reqPayload['3']['4'] === 2 ? 'YouTube' : (reqPayload['3']['4'] === 4 ? 'Display' : 'Khác'));
      
      const formatsArr = reqPayload['3']['14'];
      const formatStr = Array.isArray(formatsArr) 
        ? formatsArr.map(f => f === 1 ? 'Hình ảnh' : (f === 3 ? 'Chữ' : (f === 2 ? 'Video' : 'Khác'))).join(', ') 
        : 'Tất cả';

      const tokenStr = reqPayload["4"] ? reqPayload["4"] : 'Không có (Trang đầu)';

      sendChunk({ 
        type: 'status', 
        message: `[Yêu cầu Trang ${pageNum}]: Lọc số lượng: 100 | Ngày: ${dateRangeStr} | Nền tảng: ${platformStr} | Định dạng: ${formatStr} | Token: ${tokenStr}` 
      });

      const postData = new URLSearchParams();
      postData.append('f.req', JSON.stringify(reqPayload));
      const proxyIpChecked = activeIp;

      let response;
      let lastFetchError = null;
      let networkRetries = 0;
      const usedProxyIps = new Set(proxyIpChecked ? [proxyIpChecked] : []);
      const proxyConfiguredForScan = !disableProxy && proxy && !['NONE', 'DIRECT', 'OFF', 'false'].includes(String(proxy));
      let proxyTransportUnavailable = proxyConfiguredForScan && isGoogleProxyCircuitOpen(proxy);
      if (proxyTransportUnavailable) {
        sendChunk({ type: 'status', message: 'Proxy đang trong thời gian tạm nghỉ. Lượt quét này sẽ tự động dùng kết nối trực tiếp.' });
      }
      const proxyEnabled = proxyConfiguredForScan && !proxyTransportUnavailable;
      const maxAttempts = proxyEnabled ? 30 : 3;

      for (let attempt = 0; attempt < maxAttempts; attempt++) {
        let requestDispatcher = proxyEnabled ? dispatcher : undefined;
        let requestProxyIp = proxyEnabled ? activeIp : null;
        if (proxyEnabled && attempt > 0) {
          for (let rotationTry = 0; rotationTry < 3; rotationTry++) {
            requestDispatcher = getProxyDispatcher(proxy, false);
            requestProxyIp = await getExternalIp(requestDispatcher);
            if (!usedProxyIps.has(requestProxyIp) || rotationTry === 2) break;
          }
          usedProxyIps.add(requestProxyIp);
          sendChunk({ type: 'status', message: `Đã đổi kết nối Proxy cho vòng xử lý #${attempt + 1} • IP: ${requestProxyIp}` });
        }
        try {
          response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
            method: 'POST',
            headers: {
              ...activeHeaders,
              'content-type': 'application/x-www-form-urlencoded'
            },
            body: postData.toString(),
            dispatcher: requestDispatcher || undefined
          }, { 'f.req': reqPayload }, { proxyIpChecked: requestProxyIp });
          if (proxyEnabled) closeGoogleProxyCircuit(proxy);
          lastFetchError = null;
        } catch (error) {
          lastFetchError = error;
          response = null;
          const proxyErrorDetail = String(error.cause?.message || error.message || '');
          const proxyTransportCancelled = /request was cancelled|request aborted|proxy connect|socket closed|fetch failed/i.test(proxyErrorDetail);
          if (proxyEnabled && (proxyTransportCancelled || networkRetries >= 2)) {
            proxyTransportUnavailable = true;
            openGoogleProxyCircuit(proxy);
            const reason = proxyTransportCancelled
              ? `Kết nối Proxy gián đoạn (${proxyErrorDetail})`
              : `Proxy không phản hồi sau ${networkRetries + 1} lần thử`;
            sendChunk({ type: 'status', message: `${reason}. Đang tự chuyển sang kết nối trực tiếp để không làm gián đoạn lượt quét...` });
            try {
              response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
                method: 'POST',
                headers: {
                  ...activeHeaders,
                  'content-type': 'application/x-www-form-urlencoded'
                },
                body: postData.toString()
              }, { 'f.req': reqPayload }, { proxyFallback: true });
              lastFetchError = null;
              sendChunk({ type: 'status', message: 'Kết nối trực tiếp tới Google thành công. Tiếp tục thu thập mẫu quảng cáo.' });
              break;
            } catch (directError) {
              lastFetchError = directError;
            }
          }
          if (attempt === maxAttempts - 1) break;
          networkRetries++;
          const retryDelay = proxyEnabled ? Math.min(5000, 1200 * networkRetries) : 1500 * networkRetries;
          sendChunk({ type: 'status', message: `Kết nối Google bị gián đoạn (${error.message}). Thử lại ${networkRetries}/5 sau ${Math.ceil(retryDelay / 1000)} giây...` });
          await sleep(retryDelay);
          continue;
        }
        if (response.status !== 429) break;
        try { await response.body?.cancel(); } catch (error) {}
        const retryDelay = proxyEnabled
          ? Math.min(5000, 700 + attempt * 350) + Math.random() * 500
          : 2000 * Math.pow(2, attempt) + Math.random() * 1000;
        sendChunk({ type: 'status', message: `Google giới hạn tần suất (429). ${proxyEnabled ? 'Đang tự đổi kết nối Proxy/IP' : 'Đang thử lại'} và tiếp tục đúng trang hiện tại sau ${Math.ceil(retryDelay / 1000)} giây...` });
        await sleep(retryDelay);
      }

      if (!response && lastFetchError) throw lastFetchError;
      if (!response.ok) {
        throw new Error(`Google API returned status code ${response.status}: ${response.statusText}`);
      }

      const responseText = await response.text();
      let data;
      try {
        data = parseGoogleRPCResponse(responseText);
      } catch (parseErr) {
        throw new Error('Lỗi giải mã JSON. Có thể cookie xác thực đã hết hạn.');
      }

      const creatives = data['1'] || [];
      const pageTokenNew = data['2'] || null;
      if (data['5']) totalEstimated = parseInt(data['5'], 10);

      totalFetched += creatives.length;
      let pageSkipped = 0;
      const processedCreatives = [];
      const filteredCreatives = [];

      creatives.forEach(c => {
        const creativeId = c['2'];
        const format = c['4'];
        const firstSeen = formatRPCDate(c['6']);
        const lastSeen = formatRPCDate(c['7']);
        const rawName = c['12'] ? c['12'].trim() : 'Không rõ';
        const domain = c['14'] || null;

        // Image extraction
        let imageUrl = null;
        try {
          const html = c['3']['3']['2'];
          if (html) {
            const match = html.match(/src="([^"]+)"/);
            if (match) imageUrl = match[1];
          }
        } catch (e) {}

        let previewUrl = null;
        if (!imageUrl) {
          try {
            const preview = c['3']['1']['4'];
            if (preview && preview.startsWith('http')) {
              if (preview.includes('displayads-formats') || preview.includes('content.js')) {
                previewUrl = preview;
              } else {
                imageUrl = preview;
              }
            }
          } catch (e) {}
        }

        // Skip dynamic displayads previews
        if (!imageUrl && previewUrl) {
          pageSkipped++;
          skippedTotal++;
          filteredCreatives.push({
            creativeId,
            advertiserId: advId,
            format,
            firstSeen,
            lastSeen,
            rawName,
            domain,
            previewUrl
          });
          return;
        }

        processedCreatives.push({ creativeId, advertiserId: advId, format, firstSeen, lastSeen, rawName, domain, imageUrl, previewUrl });
      });

      sendChunk({
        type: 'status',
        message: `-> Trang ${pageNum}: Nhận được ${creatives.length} QC (Đã lọc bỏ ${pageSkipped} mẫu văn bản/Search, tích lũy bỏ qua: ${skippedTotal}). ${pageTokenNew ? 'Mã phân trang tiếp theo: ' + pageTokenNew : '(Không còn mã phân trang - Đã quét hết)'}`
      });

      sendChunk({
        type: 'creatives',
        page: pageNum,
        creatives: processedCreatives,
        filteredCreatives,
        pageSkipped,
        skippedTotal,
        totalFetched,
        totalEstimated
      });

      if (!pageTokenNew || pageTokenNew === pageToken || pageNum >= 100) {
        isFinished = true;
      } else {
        pageToken = pageTokenNew;
        await sleep(monitorMode ? 2500 + Math.random() * 2500 : 250 + Math.random() * 250);
      }
    }

    sendChunk({ type: 'done', message: 'Tải dữ liệu hoàn tất!' });

  } catch (error) {
    console.error('Advertiser Creative Fetch Error:', error);
    sendChunk({ type: 'error', error: error.message });
  } finally {
    res.end();
  }
});

// Endpoint: Perform OCR on ad image URL or extract text from search ad preview
app.post('/api/ocr', async (req, res) => {
  if (req.body?.imageUrl) {
    return res.status(410).json({ error: 'OCR image is processed only in the user browser.' });
  }
  const { imageUrl, previewUrl } = req.body;
  if (!imageUrl && !previewUrl) {
    return res.status(400).json({ error: 'Thiếu tham số imageUrl hoặc previewUrl' });
  }

  try {
    let text = '';

    if (previewUrl) {
      const jsResponse = await trackedFetch(previewUrl, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        }
      }, null, { direction: 'Server → Tải nội dung quảng cáo phục vụ OCR' });
      if (!jsResponse.ok) {
        throw new Error(`Lỗi tải mã nguồn xem trước: ${jsResponse.statusText}`);
      }
      const rawScript = await jsResponse.text();
      
      const decodedScript = rawScript
        .replace(/\\x([0-9a-fA-F]{2})/g, (match, hex) => String.fromCharCode(parseInt(hex, 16)))
        .replace(/\\u([0-9a-fA-F]{4})/g, (match, hex) => String.fromCharCode(parseInt(hex, 16)));

      const htmlRegex = /'((?:<!DOCTYPE html>)?<html[\s\S]*?<\/html>)'/i;
      const match = decodedScript.match(htmlRegex);
      let pageHtml = match ? match[1] : decodedScript;
      
      let cleanHtml = pageHtml.replace(/<style[\s\S]*?<\/style>/gi, ' ');
      cleanHtml = cleanHtml.replace(/<script[\s\S]*?<\/script>/gi, ' ');
      text = cleanHtml.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
    } else {
      const imgResponse = await trackedFetch(imageUrl, {
        headers: {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        }
      }, null, { direction: 'Server → Tải ảnh quảng cáo phục vụ OCR' });

      if (!imgResponse.ok) {
        throw new Error(`Lỗi tải ảnh: ${imgResponse.statusText}`);
      }

      const arrayBuf = await imgResponse.arrayBuffer();
      const imageBuffer = Buffer.from(arrayBuf);

      try {
        const worker = await getOcrWorker();
        const { data } = await worker.recognize(imageBuffer);
        text = data.text || '';
      } catch (ocrErr) {
        console.error('Tesseract OCR error:', ocrErr);
        throw new Error(`Định dạng ảnh không được hỗ trợ hoặc bị lỗi: ${ocrErr.message}`);
      }
    }

    const cleanValue = (val) => {
      if (!val) return null;
      return val.replace(/[`*®]/g, '').trim();
    };

    const cleanUrl = (url) => {
      let cleaned = cleanValue(url);
      if (!cleaned) return null;

      // Remove http://, https://, www. and trailing slash/paths
      const domainMatch = cleaned.match(/(?:https?:\/\/)?(?:www\.)?([^\/:\s]+)/i);
      return domainMatch ? domainMatch[1].toLowerCase() : cleaned.toLowerCase();
    };

    const domainRegex = /[a-zA-Z0-9-]+([._][a-zA-Z0-9-]+)*\.[a-zA-Z]{2,10}(\/.*)?$/;

    const foundDomains = [];
    const tokens = text.split(/[\s\n\r\t]+/);

    tokens.forEach(rawToken => {
      const cleanedToken = cleanValue(rawToken);
      if (!cleanedToken) return;

      if (domainRegex.test(cleanedToken)) {
        const finalDomain = cleanUrl(cleanedToken);
        if (
          finalDomain &&
          finalDomain.includes('.') &&
          !foundDomains.includes(finalDomain) &&
          !finalDomain.endsWith('.js') &&
          !finalDomain.endsWith('.png') &&
          !finalDomain.endsWith('.jpg') &&
          !finalDomain.endsWith('.jpeg') &&
          !finalDomain.endsWith('.gif') &&
          !finalDomain.endsWith('.css') &&
          !finalDomain.endsWith('.svg') &&
          finalDomain !== 'google.com' &&
          finalDomain !== 'gstatic.com' &&
          finalDomain !== 'googleusercontent.com' &&
          finalDomain !== 'googlesyndication.com'
        ) {
          foundDomains.push(finalDomain);
        }
      }
    });

    res.json({ text, domains: foundDomains });
  } catch (error) {
    console.error('OCR Process Error:', error);
    res.status(500).json({ error: error.message });
  }
});

// Endpoint: Fetch and extract HTML ad template from preview content.js
app.get('/api/ad-preview-html', async (req, res) => {
  const { url } = req.query;
  if (!url) {
    return res.status(400).send('Missing url parameter');
  }

  try {
    const response = await trackedFetch(url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
      }
    });
    if (!response.ok) {
      throw new Error(`Failed to fetch preview script: ${response.status}`);
    }

    const text = await response.text();
    
    const decoded = text
      .replace(/\\x([0-9a-fA-F]{2})/g, (match, hex) => String.fromCharCode(parseInt(hex, 16)))
      .replace(/\\u([0-9a-fA-F]{4})/g, (match, hex) => String.fromCharCode(parseInt(hex, 16)));

    const htmlRegex = /'((?:<!DOCTYPE html>)?<html[\s\S]*?<\/html>)'/i;
    const match = decoded.match(htmlRegex);
    
    let pageHtml = '';
    if (match) {
      pageHtml = match[1];
    } else {
      const htmlRegex2 = /((?:<!DOCTYPE html>)?<html[\s\S]*?<\/html>)/i;
      const match2 = decoded.match(htmlRegex2);
      if (match2) {
        pageHtml = match2[1];
      } else {
        throw new Error('HTML content not found in preview script');
      }
    }

    // Strip scripts to prevent code leaking or executing in the iframe
    const cleanHtml = pageHtml.replace(/<script[\s\S]*?<\/script>/gi, '');

    res.setHeader('Content-Type', 'text/html; charset=utf-8');
    res.send(cleanHtml);
  } catch (err) {
    console.error('Ad Preview HTML Error:', err);
    res.status(500).send(`Failed to generate ad preview: ${err.message}`);
  }
});

// Endpoint: Proxy images to bypass client-side AdBlockers (net::ERR_BLOCKED_BY_CLIENT)
app.get('/api/image-proxy', async (req, res) => {
  const { url } = req.query;
  if (!url) {
    return res.status(400).send('Missing url parameter');
  }

  try {
    const response = await trackedFetch(url);
    if (!response.ok) {
      throw new Error(`Failed to fetch image: ${response.status}`);
    }

    const contentType = response.headers.get('content-type') || 'image/jpeg';
    res.setHeader('Content-Type', contentType);
    
    const buffer = await response.arrayBuffer();
    res.send(Buffer.from(buffer));
  } catch (err) {
    console.error('Image Proxy Error:', err);
    res.status(500).send('Failed to load image');
  }
});

/**
 * Endpoint: POST /api/domain-summary
 * Calculates:
 * 1. activeAdvertisersCount: Số lượng đối thủ đang chạy dự án đó cho đến hiện tại.
 * 2. maxDaysActive: Số ngày chạy nhiều nhất của 1 nhà quảng cáo ĐANG CHẠY DỰ ÁN ĐÓ ĐẾN HIỆN TẠI.
 */
app.post('/api/domain-summary', async (req, res) => {
  if (req.user.role !== 'admin' && !req.user.canUseAdvertiserCount) {
    return res.status(403).json({ error: 'Tài khoản chưa được Admin cấp quyền Quét Nhà quảng cáo.' });
  }
  const { domain, headers, proxy } = req.body;
  if (!domain) {
    return res.status(400).json({ error: 'Domain parameter is required.' });
  }
  const diagnostics = [];
  try {
    const { disableProxy, requireProxy } = req.body;
    const proxyConfigured = !disableProxy && proxy && !['NONE', 'DIRECT', 'OFF', 'false'].includes(String(proxy));
    if (requireProxy && !proxyConfigured) {
      return res.status(400).json({ error: 'Chức năng này yêu cầu Proxy thật nhưng chưa có Proxy hợp lệ.' });
    }
    let usingProxy = Boolean(proxyConfigured && (requireProxy || !isGoogleProxyCircuitOpen(proxy)));
    let proxyFallbackUsed = false;

    const allCreativesList = [];
    let pageToken = null;
    let isFinished = false;
    let pageNum = 0;

    const adminCurl = parseAdminCurl(authStore.settings.curlCommand);
    const requestHeaders = headers || adminCurl?.headers || {
      'accept': '*/*',
      'accept-language': 'vi,en;q=0.9',
      'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    };

    while (!isFinished) {
      pageNum++;
      const defaultFilters = getDefaultSearchTextFilters();
      const reqPayload = {
        "2": 100,
        "3": {
          "4": defaultFilters.platform,
          "6": defaultFilters.startDate,
          "7": defaultFilters.endDate,
          "12": { "1": domain, "2": true },
          "14": defaultFilters.format
        },
        "7": { "1": 1, "2": 22, "3": 2704 }
      };
      if (pageToken) reqPayload["4"] = pageToken;

      const postData = new URLSearchParams();
      postData.append('f.req', JSON.stringify(reqPayload));
      let response = null;
      let lastRequestError = null;
      let successfulDispatcher = null;
      for (let attempt = 0; attempt < 5; attempt++) {
        // Tạo tunnel hoàn toàn mới cho mỗi lần thử. Không gọi ipify bằng
        // dispatcher này vì proxy xoay có thể hủy session khi đổi hostname.
        const requestDispatcher = usingProxy ? getProxyDispatcher(proxy, false) : undefined;
        try {
          response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
            method: 'POST',
            headers: { ...requestHeaders, 'content-type': 'application/x-www-form-urlencoded' },
            body: postData.toString(),
            dispatcher: requestDispatcher,
            signal: AbortSignal.timeout(45000)
          }, { 'f.req': reqPayload }, { proxyConnection: usingProxy ? 'Proxy xoay' : 'Trực tiếp', retryAttempt: attempt + 1, proxyFallback: proxyFallbackUsed });
          if (response.ok) {
            successfulDispatcher = requestDispatcher;
            diagnostics.push({ page: pageNum, attempt: attempt + 1, connection: usingProxy ? 'Proxy xoay' : 'Trực tiếp', status: response.status, result: 'success' });
            if (usingProxy) closeGoogleProxyCircuit(proxy);
            break;
          }
          diagnostics.push({ page: pageNum, attempt: attempt + 1, connection: usingProxy ? 'Proxy xoay' : 'Trực tiếp', status: response.status, result: 'http-error' });
          lastRequestError = new Error(`Google API trả về HTTP ${response.status}`);
        } catch (requestError) {
          const cause = requestError?.cause;
          const rootCause = cause?.cause;
          const detail = rootCause?.message || rootCause?.code || cause?.message || cause?.code || requestError?.message || 'fetch failed';
          diagnostics.push({
            page: pageNum,
            attempt: attempt + 1,
            connection: usingProxy ? 'Proxy xoay' : 'Trực tiếp',
            result: 'network-error',
            error: requestError?.message || 'fetch failed',
            cause: cause?.message || null,
            code: cause?.code || null,
            rootCause: rootCause?.message || null,
            rootCode: rootCause?.code || null
          });
          lastRequestError = new Error(`Kết nối Proxy/Google thất bại: ${detail}`);
          response = null;

          // Đồng bộ cơ chế của "Quét tên miền dự án": proxy không tạo được
          // HTTPS tunnel tới Google thì giữ nguyên payload/trang và thử trực tiếp.
          const proxyTransportRejected = usingProxy && /request was cancelled|request aborted|proxy response|http tunneling|und_err_aborted|405/i.test(
            [requestError?.message, cause?.message, cause?.code, rootCause?.message, rootCause?.code].filter(Boolean).join(' ')
          );
          if (proxyTransportRejected) {
            openGoogleProxyCircuit(proxy);
            diagnostics.push({
              page: pageNum,
              attempt: attempt + 1,
              connection: 'Chuyển Proxy → Trực tiếp',
              result: 'fallback',
              reason: detail
            });
            try {
              response = await trackedFetch('https://adstransparency.google.com/anji/_/rpc/SearchService/SearchCreatives?authuser=0', {
                method: 'POST',
                headers: { ...requestHeaders, 'content-type': 'application/x-www-form-urlencoded' },
                body: postData.toString(),
                signal: AbortSignal.timeout(45000)
              }, { 'f.req': reqPayload }, { proxyFallback: true, retryAttempt: attempt + 1 });
              if (response.ok) {
                usingProxy = false;
                proxyFallbackUsed = true;
                lastRequestError = null;
                diagnostics.push({ page: pageNum, attempt: attempt + 1, connection: 'Trực tiếp', status: response.status, result: 'fallback-success' });
                if (requestDispatcher) {
                  try { await requestDispatcher.close(); } catch (closeError) {
                    try { await requestDispatcher.destroy(); } catch (destroyError) {}
                  }
                }
                break;
              }
              diagnostics.push({ page: pageNum, attempt: attempt + 1, connection: 'Trực tiếp', status: response.status, result: 'fallback-http-error' });
              lastRequestError = new Error(`Kết nối trực tiếp trả về HTTP ${response.status}`);
            } catch (directError) {
              const directCause = directError?.cause?.cause || directError?.cause;
              diagnostics.push({
                page: pageNum,
                attempt: attempt + 1,
                connection: 'Trực tiếp',
                result: 'fallback-network-error',
                error: directError?.message || 'fetch failed',
                cause: directCause?.message || null,
                code: directCause?.code || null
              });
              lastRequestError = new Error(`Proxy bị từ chối và kết nối trực tiếp cũng thất bại: ${directCause?.message || directError.message}`);
            }
          }
        }

        if (requestDispatcher) {
          try { await requestDispatcher.close(); } catch (closeError) {
            try { await requestDispatcher.destroy(); } catch (destroyError) {}
          }
        }
        if (usingProxy) openGoogleProxyCircuit(proxy);
        if (attempt < 4) await sleep(1000 * (attempt + 1));
      }

      if (!response || !response.ok) {
        const finalError = lastRequestError || new Error('Google API không phản hồi.');
        finalError.diagnostics = diagnostics;
        throw finalError;
      }

      const responseText = await response.text();
      if (successfulDispatcher) {
        try { await successfulDispatcher.close(); } catch (closeError) {
          try { await successfulDispatcher.destroy(); } catch (destroyError) {}
        }
      }
      const data = parseGoogleRPCResponse(responseText);

      const creatives = data['1'] || [];
      const pageTokenNew = data['2'] || null;

      creatives.forEach(creative => {
        const creativeAdvId = creative['1'];
        const creativeId = creative['2'];
        if (!creativeAdvId || !creativeId) return;

        const rawAdvName = creative['12'] ? creative['12'].trim() : 'Nhà quảng cáo không xác định';
        const firstSeen = formatRPCDate(creative['6']);
        const lastSeen = formatRPCDate(creative['7']);

        allCreativesList.push({
          creativeId,
          advertiserId: creativeAdvId,
          advertiserName: rawAdvName,
          firstSeen,
          lastSeen
        });
      });

      if (!pageTokenNew || pageTokenNew === pageToken || pageNum >= 100) {
        isFinished = true;
      } else {
        pageToken = pageTokenNew;
        await sleep(200 + Math.random() * 200);
      }
    }

    if (allCreativesList.length === 0) {
      return res.json({
        domain,
        proxyActive: usingProxy,
        proxyIp: usingProxy ? 'Proxy xoay' : 'IP trực tiếp',
        proxyFallbackUsed,
        activeAdvertisersCount: 0,
        maxDaysActive: 0,
        maxDaysCreative: null,
        message: 'Không tìm thấy mẫu quảng cáo nào cho tên miền này.'
      });
    }

    // Determine the latest active date found in the scan
    const todayStr = new Date().toISOString().split('T')[0];
    let maxDomainLastSeen = todayStr;
    allCreativesList.forEach(c => {
      if (c.lastSeen && c.lastSeen > maxDomainLastSeen) {
        maxDomainLastSeen = c.lastSeen;
      }
    });

    // Subtract 1 day from maxDomainLastSeen to allow for Google Ads feed update lag (1-day delay tolerance)
    const thresholdDateObj = new Date(maxDomainLastSeen);
    thresholdDateObj.setDate(thresholdDateObj.getDate() - 1);
    const thresholdDateStr = thresholdDateObj.toISOString().split('T')[0];

    // Filter individual creatives that are still active as of thresholdDate / maxDomainLastSeen
    const activeCreatives = allCreativesList.filter(c => {
      if (!c.lastSeen) return false;
      return c.lastSeen >= thresholdDateStr;
    }).map(c => {
      const dStart = new Date(c.firstSeen);
      const dEnd = new Date(c.lastSeen);
      const diffTime = Math.abs(dEnd - dStart);
      const daysActive = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
      return { ...c, daysActive };
    });

    // 1. Number of active advertisers currently running as of today
    const uniqueActiveAdvIds = new Set(activeCreatives.map(c => c.advertiserId));
    const activeAdvertisersCount = uniqueActiveAdvIds.size;

    // 2. Find the single individual creative that has been running for the longest days
    let maxDaysActive = 0;
    let maxDaysCreative = null;

    activeCreatives.forEach(c => {
      if (c.daysActive > maxDaysActive) {
        maxDaysActive = c.daysActive;
        maxDaysCreative = c;
      }
    });

    res.json({
      success: true,
      domain,
      proxyActive: usingProxy,
      proxyIp: usingProxy ? 'Proxy xoay' : 'IP trực tiếp',
      proxyFallbackUsed,
      asOfDate: maxDomainLastSeen,
      activeAdvertisersCount,
      maxDaysActive,
      maxDaysCreative
    });

  } catch (error) {
    console.error('Domain Summary API Error:', error);
    res.status(500).json({
      error: error.message,
      details: error.diagnostics || diagnostics
    });
  }
});

function encryptAdminSetting(value) {
  const key = crypto.createHash('sha256').update(`admin-setting:${SESSION_SECRET}`).digest();
  const iv = crypto.randomBytes(12);
  const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
  const encrypted = Buffer.concat([cipher.update(String(value), 'utf8'), cipher.final()]);
  return ['v1', iv.toString('base64url'), cipher.getAuthTag().toString('base64url'), encrypted.toString('base64url')].join('.');
}

function decryptAdminSetting(value) {
  if (!value) return '';
  const [version, iv, tag, encrypted] = String(value).split('.');
  if (version !== 'v1' || !iv || !tag || !encrypted) return '';
  try {
    const key = crypto.createHash('sha256').update(`admin-setting:${SESSION_SECRET}`).digest();
    const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(iv, 'base64url'));
    decipher.setAuthTag(Buffer.from(tag, 'base64url'));
    return Buffer.concat([decipher.update(Buffer.from(encrypted, 'base64url')), decipher.final()]).toString('utf8');
  } catch (_) { return ''; }
}

function googleOAuthRuntimeConfig() {
  const stored = authStore.settings?.googleOAuth || {};
  const rawTokenEncryptionKey = decryptAdminSetting(stored.tokenEncryptionKey) || ENV_GOOGLE_TOKEN_ENCRYPTION_KEY || SESSION_SECRET;
  return {
    clientId: String(stored.clientId || ENV_GOOGLE_OAUTH_CLIENT_ID || '').trim(),
    clientSecret: decryptAdminSetting(stored.clientSecret) || ENV_GOOGLE_OAUTH_CLIENT_SECRET,
    redirectUri: String(stored.redirectUri || ENV_GOOGLE_OAUTH_REDIRECT_URI || '').trim(),
    rawTokenEncryptionKey,
    tokenEncryptionKeyConfigured: Boolean(decryptAdminSetting(stored.tokenEncryptionKey) || ENV_GOOGLE_TOKEN_ENCRYPTION_KEY),
    tokenEncryptionKey: crypto.createHash('sha256').update(rawTokenEncryptionKey).digest()
  };
}

function encryptGoogleToken(value) {
  const iv = crypto.randomBytes(12);
  const cipher = crypto.createCipheriv('aes-256-gcm', googleOAuthRuntimeConfig().tokenEncryptionKey, iv);
  const encrypted = Buffer.concat([cipher.update(String(value), 'utf8'), cipher.final()]);
  return ['v1', iv.toString('base64url'), cipher.getAuthTag().toString('base64url'), encrypted.toString('base64url')].join('.');
}

function decryptGoogleToken(value) {
  const [version, iv, tag, encrypted] = String(value || '').split('.');
  if (version !== 'v1' || !iv || !tag || !encrypted) throw new Error('Refresh token Google đã lưu không hợp lệ.');
  const decipher = crypto.createDecipheriv('aes-256-gcm', googleOAuthRuntimeConfig().tokenEncryptionKey, Buffer.from(iv, 'base64url'));
  decipher.setAuthTag(Buffer.from(tag, 'base64url'));
  return Buffer.concat([decipher.update(Buffer.from(encrypted, 'base64url')), decipher.final()]).toString('utf8');
}

function googleOAuthRedirectUri(req) {
  const configuredRedirect = googleOAuthRuntimeConfig().redirectUri;
  if (configuredRedirect) return configuredRedirect;
  const protocol = String(req.get('x-forwarded-proto') || req.protocol || 'https').split(',')[0].trim();
  return `${protocol}://${req.get('host')}/api/google-sheets/oauth/callback`;
}

function signGoogleOAuthState(payload) {
  const encoded = Buffer.from(JSON.stringify(payload)).toString('base64url');
  const signature = crypto.createHmac('sha256', SESSION_SECRET).update(`google-oauth:${encoded}`).digest('base64url');
  return `${encoded}.${signature}`;
}

function verifyGoogleOAuthState(value) {
  const [encoded, signature] = String(value || '').split('.');
  if (!encoded || !signature) return null;
  const expected = crypto.createHmac('sha256', SESSION_SECRET).update(`google-oauth:${encoded}`).digest('base64url');
  if (signature.length !== expected.length || !crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) return null;
  try {
    const payload = JSON.parse(Buffer.from(encoded, 'base64url').toString('utf8'));
    return payload.exp > Date.now() ? payload : null;
  } catch (_) { return null; }
}

async function googleTokenRequest(params) {
  const response = await fetch('https://oauth2.googleapis.com/token', {
    method: 'POST', headers: { 'content-type': 'application/x-www-form-urlencoded' },
    body: new URLSearchParams(params), signal: AbortSignal.timeout(20000)
  });
  const result = await response.json().catch(() => ({}));
  if (!response.ok || result.error) throw new Error(result.error_description || result.error || `Google OAuth HTTP ${response.status}`);
  return result;
}

async function googleAccessTokenForUser(user) {
  const config = googleOAuthRuntimeConfig();
  if (!config.clientId || !config.clientSecret) throw new Error('Admin chưa cấu hình Google OAuth trên server.');
  if (!user.googleSheetsOAuth?.refreshToken) throw new Error('Tài khoản chưa kết nối Google Sheets.');
  return googleTokenRequest({
    client_id: config.clientId, client_secret: config.clientSecret,
    refresh_token: decryptGoogleToken(user.googleSheetsOAuth.refreshToken), grant_type: 'refresh_token'
  });
}

app.get('/api/google-sheets/oauth/status', (req, res) => {
  const config = googleOAuthRuntimeConfig();
  res.json({ configured: Boolean(config.clientId && config.clientSecret), connected: Boolean(req.user.googleSheetsOAuth?.refreshToken), email: req.user.googleSheetsOAuth?.email || null });
});

app.get('/api/google-sheets/oauth/start', (req, res) => {
  const config = googleOAuthRuntimeConfig();
  if (!config.clientId || !config.clientSecret) return res.status(503).json({ error: 'Admin chưa cấu hình GOOGLE_OAUTH_CLIENT_ID và GOOGLE_OAUTH_CLIENT_SECRET.' });
  const redirectUri = googleOAuthRedirectUri(req);
  const returnTo = req.query.returnTo === '/admin' && req.user.role === 'admin' ? '/admin' : '/';
  const state = signGoogleOAuthState({ userId: req.user.id, redirectUri, returnTo, nonce: crypto.randomUUID(), exp: Date.now() + 10 * 60 * 1000 });
  const params = new URLSearchParams({
    client_id: config.clientId, redirect_uri: redirectUri, response_type: 'code',
    scope: 'openid email https://www.googleapis.com/auth/spreadsheets',
    access_type: 'offline', prompt: 'consent', state
  });
  res.json({ url: `https://accounts.google.com/o/oauth2/v2/auth?${params}` });
});

app.get('/api/google-sheets/oauth/callback', async (req, res) => {
  const state = verifyGoogleOAuthState(req.query.state);
  if (!state || state.userId !== req.user.id || req.query.error) return res.redirect('/?googleSheets=error');
  try {
    const config = googleOAuthRuntimeConfig();
    const token = await googleTokenRequest({
      code: String(req.query.code || ''), client_id: config.clientId,
      client_secret: config.clientSecret, redirect_uri: state.redirectUri,
      grant_type: 'authorization_code'
    });
    const userInfoResponse = await fetch('https://openidconnect.googleapis.com/v1/userinfo', { headers: { authorization: `Bearer ${token.access_token}` }, signal: AbortSignal.timeout(10000) });
    const userInfo = await userInfoResponse.json().catch(() => ({}));
    const user = authStore.users.find(item => item.id === req.user.id);
    if (!user || !token.refresh_token) throw new Error('Google không trả refresh token. Hãy thu hồi quyền ứng dụng rồi kết nối lại.');
    user.googleSheetsOAuth = { refreshToken: encryptGoogleToken(token.refresh_token), email: userInfo.email || null, connectedAt: new Date().toISOString() };
    await saveAuthStore();
    setSession(res, user);
    res.redirect(`${state.returnTo || '/'}?googleSheets=connected`);
  } catch (error) {
    console.error('Google OAuth callback failed:', error.message);
    res.redirect('/?googleSheets=error');
  }
});

app.delete('/api/google-sheets/oauth', async (req, res) => {
  const user = authStore.users.find(item => item.id === req.user.id);
  if (user) { delete user.googleSheetsOAuth; await saveAuthStore(); }
  res.json({ ok: true });
});

function parseGoogleSpreadsheetId(value) {
  const text = String(value || '').trim();
  const match = /\/spreadsheets\/d\/([a-zA-Z0-9-_]+)/.exec(text);
  const id = match ? match[1] : text;
  return /^[a-zA-Z0-9-_]{20,}$/.test(id) ? id : '';
}

const googleSheetDomainCache = new Map();
app.post('/api/google-sheets/domains', requireDomainDuplicate, async (req, res) => {
  try {
    const spreadsheetId = parseGoogleSpreadsheetId(req.body?.spreadsheetUrl || req.body?.spreadsheetId);
    if (!spreadsheetId) return res.status(400).json({ error: 'Link Google Sheet không hợp lệ.' });
    const sheetName = String(req.body?.sheetName || 'Competitor Ads').trim().slice(0, 100);
    const requestedDomainColumn = String(req.body?.domainColumnName || '').trim().slice(0, 100);
    const requestedRegistrantColumn = String(req.body?.registrantColumnName || '').trim().slice(0, 100);
    const registrantColumnSupplied = Object.prototype.hasOwnProperty.call(req.body || {}, 'registrantColumnName');
    const cacheKey = `${req.user.id}:${spreadsheetId}:${sheetName}:${requestedDomainColumn}:${requestedRegistrantColumn}`;
    const cached = googleSheetDomainCache.get(cacheKey);
    if (!req.body?.refresh && cached && Date.now() - cached.createdAt < 5 * 60 * 1000) return res.json({ ...cached.response, cached: true });
    const token = await googleAccessTokenForUser(req.user);
    const googleHeaders = { authorization: `Bearer ${token.access_token}` };
    const metadataResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}?fields=properties.title,sheets.properties.title`, { headers: googleHeaders, signal: AbortSignal.timeout(20000) });
    const metadata = await metadataResponse.json().catch(() => ({}));
    if (!metadataResponse.ok) {
      const connectedEmail = req.user.googleSheetsOAuth?.email || 'tài khoản Google đang kết nối';
      if (metadataResponse.status === 404) throw new Error(`Không tìm thấy file hoặc ${connectedEmail} chưa được cấp quyền xem Google Sheet này. Hãy chia sẻ file cho đúng tài khoản rồi thử lại.`);
      throw new Error(metadata.error?.message || `Google Sheets HTTP ${metadataResponse.status}`);
    }
    const availableSheets = (metadata.sheets || []).map(item => String(item.properties?.title || '')).filter(Boolean);
    const resolvedSheetName = availableSheets.find(title => title === sheetName) || availableSheets.find(title => title.toLocaleLowerCase('vi') === sheetName.toLocaleLowerCase('vi'));
    if (!resolvedSheetName) throw new Error(`Không tìm thấy tab “${sheetName}” trong file “${metadata.properties?.title || 'Google Sheet'}”. Các tab hiện có: ${availableSheets.join(', ') || 'không có tab nào'}.`);
    const range = `'${resolvedSheetName.replace(/'/g, "''")}'!A:ZZ`;
    const response = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?majorDimension=ROWS`, { headers: { authorization: `Bearer ${token.access_token}` }, signal: AbortSignal.timeout(30000) });
    const payload = await response.json().catch(() => ({}));
    if (!response.ok) throw new Error(payload.error?.message || `Google Sheets HTTP ${response.status}`);
    const rows = Array.isArray(payload.values) ? payload.values : [];
    const rawHeaders = (rows[0] || []).map(value => String(value || '').trim());
    const normalizeHeader = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').replace(/đ/gi, 'd').replace(/\s+/g, ' ').trim().toLowerCase();
    const headers = rawHeaders.map(normalizeHeader);
    const columnLetterIndex = value => {
      const letters = String(value || '').trim().toUpperCase();
      if (!/^[A-Z]{1,2}$/.test(letters)) return -1;
      return [...letters].reduce((index, letter) => index * 26 + letter.charCodeAt(0) - 64, 0) - 1;
    };
    const domainColumnByLetter = columnLetterIndex(requestedDomainColumn);
    const registrantColumnByLetter = columnLetterIndex(requestedRegistrantColumn);
    let domainColumn = domainColumnByLetter >= 0 ? domainColumnByLetter : requestedDomainColumn ? headers.indexOf(normalizeHeader(requestedDomainColumn)) : headers.findIndex(value => /^(domain|domain du an|link du an|link domain|ten mien|website|url)$/i.test(value));
    if (requestedDomainColumn && domainColumn < 0) throw new Error(`Không tìm thấy cột domain “${requestedDomainColumn}”. Hãy nhập tên tiêu đề hoặc ký tự cột như D, F, AA. Các cột hiện có: ${rawHeaders.filter(Boolean).join(', ') || 'không có tiêu đề'}.`);
    if (domainColumn < 0) domainColumn = 0;
    let registrantColumn = registrantColumnByLetter >= 0 ? registrantColumnByLetter : requestedRegistrantColumn ? headers.indexOf(normalizeHeader(requestedRegistrantColumn)) : registrantColumnSupplied ? -1 : headers.findIndex(value => /^(nhan su|nguoi dang ky|nguoi phu trach|nguoi so huu|owner|assignee|registered by|registrant)$/i.test(value));
    if (requestedRegistrantColumn && registrantColumn < 0) throw new Error(`Không tìm thấy cột người đăng ký “${requestedRegistrantColumn}”. Hãy nhập tên tiêu đề hoặc ký tự cột như D, F, AA. Các cột hiện có: ${rawHeaders.filter(Boolean).join(', ') || 'không có tiêu đề'}.`);
    if (!registrantColumnSupplied && registrantColumn < 0 && domainColumn !== 1 && (rows[0] || []).length > 1) registrantColumn = 1;
    const registrationMap = new Map();
    const firstDomainRow = new Map();
    const duplicateDetails = [];
    const invalidDetails = [];
    let validRows = 0;
    let invalidRows = 0;
    const dataStartsAtFirstRow = domainColumnByLetter >= 0;
    rows.slice(dataStartsAtFirstRow ? 0 : 1).forEach((row, index) => {
      const rowNumber = index + (dataStartsAtFirstRow ? 1 : 2);
      const rawDomain = String(row[domainColumn] || '').trim();
      const sourceDomain = normalizeProjectDomain(rawDomain);
      const domain = projectDomainIdentity(sourceDomain);
      if (!domain) { invalidRows++; invalidDetails.push({ rowNumber, value: rawDomain || '(trống)' }); return; }
      validRows++;
      if (firstDomainRow.has(domain)) duplicateDetails.push({ domain, sourceDomain, rowNumber, firstRowNumber: firstDomainRow.get(domain), value: rawDomain });
      else firstDomainRow.set(domain, rowNumber);
      const registrant = registrantColumn >= 0 ? String(row[registrantColumn] || '').trim() : '';
      if (!registrationMap.has(domain)) registrationMap.set(domain, new Set());
      if (registrant) registrationMap.get(domain).add(registrant);
    });
    const registrations = [...registrationMap].map(([domain, names]) => ({ domain, registrants: [...names], registrant: [...names].join(', ') }));
    const domains = registrations.map(item => item.domain);
    const responseBody = { domains, registrations, count: domains.length, totalRows: Math.max(0, rows.length - (dataStartsAtFirstRow ? 0 : 1)), validRows, duplicateRows: duplicateDetails.length, invalidRows, duplicateDetails, invalidDetails, domainColumn: domainColumn + 1, domainHeader: domainColumnByLetter >= 0 ? `Cột ${requestedDomainColumn.toUpperCase()}` : String((rows[0] || [])[domainColumn] || ''), registrantColumn: registrantColumn >= 0 ? registrantColumn + 1 : null, registrantHeader: registrantColumnByLetter >= 0 ? `Cột ${requestedRegistrantColumn.toUpperCase()}` : registrantColumn >= 0 ? String((rows[0] || [])[registrantColumn] || '') : '', sheetName: resolvedSheetName, spreadsheetTitle: metadata.properties?.title || '' };
    googleSheetDomainCache.set(cacheKey, { response: responseBody, createdAt: Date.now() });
    res.json({ ...responseBody, cached: false });
  } catch (error) { res.status(502).json({ error: `Không thể đọc domain từ Google Sheet: ${error.message}` }); }
});

async function exportViaGoogleSheetsApi(req, res, spreadsheetId, sheetName, columns, records) {
  const token = await googleAccessTokenForUser(req.user);
  const headers = { authorization: `Bearer ${token.access_token}`, 'content-type': 'application/json' };
  const metadataResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}?fields=sheets.properties.title`, { headers, signal: AbortSignal.timeout(20000) });
  const metadata = await metadataResponse.json().catch(() => ({}));
  if (!metadataResponse.ok) throw new Error(metadata.error?.message || `Google Sheets HTTP ${metadataResponse.status}`);
  const targetSheet = String(sheetName || 'Competitor Ads').slice(0, 100);
  if (!(metadata.sheets || []).some(sheet => sheet.properties?.title === targetSheet)) {
    const createResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}:batchUpdate`, {
      method: 'POST', headers, body: JSON.stringify({ requests: [{ addSheet: { properties: { title: targetSheet } } }] }), signal: AbortSignal.timeout(20000)
    });
    const createResult = await createResponse.json().catch(() => ({}));
    if (!createResponse.ok) throw new Error(createResult.error?.message || `Không thể tạo tab Sheet (${createResponse.status}).`);
  }
  const range = `'${targetSheet.replace(/'/g, "''")}'!A:ZZ`;
  const headerRange = `'${targetSheet.replace(/'/g, "''")}'!1:1`;
  const headerResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(headerRange)}`, {
    headers, signal: AbortSignal.timeout(20000)
  });
  const headerResult = await headerResponse.json().catch(() => ({}));
  if (!headerResponse.ok) throw new Error(headerResult.error?.message || `Không thể kiểm tra tiêu đề Sheet (${headerResponse.status}).`);
  const sheetIsEmpty = !Array.isArray(headerResult.values) || !headerResult.values[0]?.length;
  const values = records.map(record => columns.map(column => record[column.key] ?? ''));
  if (sheetIsEmpty) values.unshift(columns.map(column => column.label || column.key));
  const appendResponse = await fetch(`https://sheets.googleapis.com/v4/spreadsheets/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}:append?valueInputOption=USER_ENTERED&insertDataOption=INSERT_ROWS`, {
    method: 'POST', headers, body: JSON.stringify({ majorDimension: 'ROWS', values }), signal: AbortSignal.timeout(30000)
  });
  const appendResult = await appendResponse.json().catch(() => ({}));
  if (!appendResponse.ok) throw new Error(appendResult.error?.message || `Google Sheets append HTTP ${appendResponse.status}`);
  return res.json({ ok: true, mode: 'oauth', inserted: records.length, updated: 0, updatedRange: appendResult.updates?.updatedRange || null });
}

app.post('/api/google-sheet-export', async (req, res) => {
  try {
    const { spreadsheetUrl, spreadsheetId: rawSpreadsheetId, webAppUrl, sheetName, secret, columns, records } = req.body || {};
    if (!Array.isArray(records) || records.length === 0) return res.status(400).json({ ok: false, error: 'Thiếu dữ liệu kết quả.' });
    if (!Array.isArray(columns) || columns.length === 0) return res.status(400).json({ ok: false, error: 'Chưa chọn cột dữ liệu cần xuất.' });
    if (records.length > 20000) return res.status(413).json({ ok: false, error: 'Mỗi lần chỉ gửi tối đa 20.000 dòng.' });
    const spreadsheetId = parseGoogleSpreadsheetId(rawSpreadsheetId || spreadsheetUrl);
    if (spreadsheetId) return await exportViaGoogleSheetsApi(req, res, spreadsheetId, sheetName, Array.isArray(columns) ? columns : [], records);
    let targetUrl;
    try {
      targetUrl = new URL(String(webAppUrl || ''));
    } catch (error) {
      return res.status(400).json({ ok: false, error: 'Web App URL không hợp lệ.' });
    }

    if (targetUrl.protocol !== 'https:' || targetUrl.hostname !== 'script.google.com' || !targetUrl.pathname.startsWith('/macros/s/')) {
      return res.status(400).json({ ok: false, error: 'Chỉ chấp nhận Google Apps Script Web App URL.' });
    }
    if (!secret) {
      return res.status(400).json({ ok: false, error: 'Thiếu secret key hoặc dữ liệu kết quả.' });
    }
    const upstream = await fetch(targetUrl, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ sheetName, secret, columns: Array.isArray(columns) ? columns : [], records }),
      redirect: 'follow'
    });
    const text = await upstream.text();
    let result;
    try { result = JSON.parse(text); } catch (error) {
      return res.status(502).json({ ok: false, error: 'Apps Script không trả về JSON hợp lệ.' });
    }
    return res.status(upstream.ok && result.ok ? 200 : 502).json(result);
  } catch (error) {
    return res.status(500).json({ ok: false, error: error.message });
  }
});

app.post('/api/keyword-metrics', async (req, res) => {
  try {
    const keyword = String(req.body?.keyword || '').trim().slice(0, 200);
    const country = String(req.body?.country || '').trim().slice(0, 10);
    const currency = String(req.body?.currency || 'usd').trim().toLowerCase().slice(0, 8);
    if (!keyword) {
      return res.status(400).json({ success: false, error: 'Keyword là bắt buộc.' });
    }

    const upstream = await fetch('https://tool-keyword-planner.vercel.app/api/volume', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ keyword, country, currency })
    });
    const text = await upstream.text();
    let result;
    try { result = JSON.parse(text); } catch (error) {
      return res.status(502).json({ success: false, error: 'API Volume không trả về JSON hợp lệ.' });
    }
    if (!upstream.ok || !result.success) {
      return res.status(upstream.ok ? 502 : upstream.status).json({
        success: false,
        error: result.error || `API Volume trả về HTTP ${upstream.status}`
      });
    }
    return res.json({
      success: true,
      keyword,
      volume: result.volume ?? null,
      cpcValue: result.cpc_value ?? null,
      cpcCurrency: result.cpc_currency || '$',
      competition: result.competition ?? null,
      noData: Boolean(result.no_data)
    });
  } catch (error) {
    return res.status(502).json({ success: false, error: error.message });
  }
});

let nextTrafficProfile = 1;
function takeNextTrafficProfile() {
  const profile = nextTrafficProfile;
  nextTrafficProfile = profile >= 10 ? 1 : profile + 1;
  return profile;
}

function isWorkfastTrafficPayload(payload) {
  return Boolean(payload && typeof payload === 'object' && (
    payload.Engagments
    || payload.Engagements
    || payload.EstimatedMonthlyVisits
    || payload.TrafficSources
    || payload.TopCountryShares
  ));
}

function trafficSessionExpired(status, payload) {
  // Workfast sometimes sends a non-2xx HTTP status together with a complete
  // traffic result. The data itself is authoritative in that case.
  if (isWorkfastTrafficPayload(payload)) return false;
  if ([401, 403, 419].includes(Number(status))) return true;

  const text = payload && typeof payload === 'object'
    ? [payload.error, payload.message, payload.detail, payload.description, payload.Description]
        .filter(value => typeof value === 'string')
        .join(' ')
    : String(payload || '');

  return /unauthori[sz]ed|authentication\s+(?:failed|required)|(?:token|session|timestamp)\s+(?:is\s+)?(?:invalid|expired)|(?:invalid|expired)\s+(?:token|session|timestamp)|signature\s+(?:is\s+)?(?:invalid|expired)|(?:invalid|expired)\s+signature|đăng nhập|hết hạn/i.test(text);
}

function maskTrafficCredential(value) {
  const text = String(value || '');
  if (!text) return '';
  if (text.length <= 10) return `${text.slice(0, 2)}••••${text.slice(-2)}`;
  return `${text.slice(0, 6)}••••••${text.slice(-4)}`;
}

function buildTrafficDiagnostics(auth, domain, status = null) {
  const query = new URLSearchParams({
    domain,
    timestamp: String(auth?.timestamp || ''),
    source: 'extension',
    clientId: maskTrafficCredential(auth?.clientId),
    sign: maskTrafficCredential(auth?.sign)
  });
  return {
    profile: auth?.profile || null,
    profileName: auth?.profileName || (auth?.profile ? `Profile ${auth.profile}` : null),
    variablesReceived: {
      timestamp: String(auth?.timestamp || ''),
      clientId: maskTrafficCredential(auth?.clientId),
      sign: maskTrafficCredential(auth?.sign),
      complete: Boolean(auth?.timestamp && auth?.clientId && auth?.sign)
    },
    request: { method: 'GET', url: `https://traffic.workfast.cc/?${query}` },
    status
  };
}

let currentProfileIndex = 0;
function enabledTrafficProfiles() {
  const configured = Array.isArray(authStore.settings.trafficProfiles) ? authStore.settings.trafficProfiles : [];
  return configured.filter(item => item && item.enabled !== false && String(item.id || '').trim());
}
function takeNextTrafficProfile() {
  const profiles = enabledTrafficProfiles();
  if (!profiles.length) return null;
  const profile = profiles[currentProfileIndex % profiles.length];
  currentProfileIndex = (currentProfileIndex + 1) % profiles.length;
  return profile;
}

let cachedTrafficAuth = null; // Stores { sign, clientId, timestamp, profile, fetchedAt }
let pendingAuthPromise = null; // Shared promise to guarantee ONLY 1 Ngrok get-sign request is sent for all threads
const TRAFFIC_AUTH_REJECTION_COOLDOWN_MS = 60 * 1000;

async function getSharedTrafficAuth(forceRefresh = false, errors = []) {
  if (cachedTrafficAuth?.rejectedAt && Date.now() - cachedTrafficAuth.rejectedAt < TRAFFIC_AUTH_REJECTION_COOLDOWN_MS) {
    errors.push('Sign vừa bị Workfast từ chối; tạm ngừng gọi get-sign trong 60 giây để tránh gọi liên tục.');
    return cachedTrafficAuth;
  }
  if (!forceRefresh && cachedTrafficAuth && !cachedTrafficAuth.rejectedAt && (Date.now() - cachedTrafficAuth.fetchedAt < 15 * 60 * 1000)) {
    return cachedTrafficAuth;
  }

  if (pendingAuthPromise) {
    return pendingAuthPromise;
  }

  pendingAuthPromise = (async () => {
    const profiles = enabledTrafficProfiles();
    for (let attempt = 0; attempt < profiles.length; attempt++) {
      const profileConfig = takeNextTrafficProfile();
      if (!profileConfig) break;
      const profile = profileConfig.id;
      try {
        const signUrl = `https://implant-buddy-bring.ngrok-free.dev/get-sign?profile=${encodeURIComponent(profile)}`;
        const signResponse = await trackedFetch(signUrl, {
          headers: { 'ngrok-skip-browser-warning': 'true', accept: 'application/json' },
          signal: AbortSignal.timeout(60000) // Timeout 60 giây (1 phút) như yêu cầu
        }, null, { category: 'external', direction: `Server → Ngrok Get-Sign (${profileConfig.name})`, trafficProfile: profile, trafficProfileName: profileConfig.name });

        const signText = await signResponse.text();
        let credentials;
        try {
          credentials = JSON.parse(signText);
        } catch (e) {
          const isNgrokHtml = /ngrok|offline|tunnel|ERR_NGROK/i.test(signText);
          errors.push(`Profile ${profile}: ${isNgrokHtml ? 'Ngrok Tunnel đang Offline' : 'Không trả về JSON hợp lệ'}`);
          continue;
        }

        if (signResponse.ok && credentials && credentials.sign && credentials.clientId && credentials.timestamp) {
          cachedTrafficAuth = {
            sign: String(credentials.sign),
            clientId: String(credentials.clientId),
            timestamp: String(credentials.timestamp),
            profile: credentials.profile || profile,
            profileName: profileConfig.name,
            fetchedAt: Date.now()
          };
          return cachedTrafficAuth;
        } else {
          errors.push(`Profile ${profile}: Ngrok trả về ${credentials?.error || credentials?.message || `HTTP ${signResponse.status}`}`);
        }
      } catch (err) {
        errors.push(`Profile ${profile} get-sign: ${err.message}`);
      }
    }
    return cachedTrafficAuth;
  })().finally(() => {
    pendingAuthPromise = null;
  });

  return pendingAuthPromise;
}

async function rotateSharedTrafficAuthAfterWorkfastRateLimit(failedAuth, errors = []) {
  const cacheIsFailedAuth = Boolean(cachedTrafficAuth
    && String(cachedTrafficAuth.profile) === String(failedAuth?.profile)
    && cachedTrafficAuth.sign === failedAuth?.sign);
  if (cachedTrafficAuth && !cacheIsFailedAuth) return cachedTrafficAuth;
  if (pendingAuthPromise) return pendingAuthPromise;
  if (cacheIsFailedAuth) cachedTrafficAuth = null;
  return getSharedTrafficAuth(true, errors);
}

let trafficLensProxySlot = null;
let trafficLensProxyGeneration = 0;

function resetTrafficLensProxySlot() {
  const previous = trafficLensProxySlot;
  trafficLensProxySlot = null;
  if (previous?.dispatcher && typeof previous.dispatcher.close === 'function') {
    Promise.resolve(previous.dispatcher.close()).catch(() => {});
  }
}

function acquireTrafficLensProxyLease() {
  const proxyUrl = authStore.settings.proxyUrl;
  if (!proxyUrl) throw new Error('Chưa cấu hình Proxy chung cho TrafficLens.');
  // TrafficLens: tuyệt đối một request trên một ProxyAgent/tunnel riêng.
  // Rotating endpoint của nhà cung cấp sẽ cấp IP mới theo từng request.
  return {
    generation: ++trafficLensProxyGeneration,
    dispatcher: getProxyDispatcher(proxyUrl, false),
    issued: 1,
    completed: 0
  };
}

function releaseTrafficLensProxyLease(slot) {
  if (!slot) return;
  slot.completed++;
  // Đóng ngay sau đúng một request để không request nào tái sử dụng tunnel/IP này.
  if (slot.completed >= 1 && slot.dispatcher && typeof slot.dispatcher.close === 'function') {
    Promise.resolve(slot.dispatcher.close()).catch(() => {});
  }
}

async function requestTrafficLensAttempt(domain, attempt) {
  const sessionCookie = decryptAdminSetting(authStore.settings.trafficLens?.sessionCookie);
  const lease = acquireTrafficLensProxyLease();
  try {
    const proxyIp = await getExternalIp(lease.dispatcher);
    const form = new FormData();
    form.append('url', domain);
    const headers = {
      accept: '*/*',
      origin: 'https://trafficlens.space',
      referer: 'https://trafficlens.space/',
      'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36'
    };
    if (sessionCookie) headers.cookie = `session=${sessionCookie}`;
    const response = await trackedFetch('https://trafficlens.space/api/try', {
      method: 'POST',
      dispatcher: lease.dispatcher,
      headers,
      body: form,
      signal: AbortSignal.timeout(15000)
    }, null, {
      category: 'external',
      direction: 'Server → TrafficLens API',
      connection: `Proxy xoay riêng · request #${lease.generation} · lần thử ${attempt}/3`,
      trafficProvider: 'trafficlens',
      trafficDomain: domain,
      proxyRotation: lease.generation,
      proxyCallInRotation: 1,
      trafficAttempt: attempt,
      proxyIpChecked: proxyIp,
      proxyIpSource: 'Preflight ipify'
    });
    const text = await response.text();
    let payload = null;
    try { payload = text ? JSON.parse(text) : null; } catch (_) {}
    if (!response.ok || !payload || String(payload.status || '').toUpperCase() !== 'OK') {
      const detail = payload?.error || payload?.message || payload?.status || text.slice(0, 300) || `HTTP ${response.status}`;
      const error = new Error(`TrafficLens API báo lỗi: ${detail}`);
      error.statusCode = response.status || 502;
      error.details = payload || text.slice(0, 1000);
      throw error;
    }
    const visits = Number(String(payload.total_visits ?? '').replace(/[^0-9.-]/g, ''));
    const bounceRate = Number(String(payload.bounce_rate ?? '').replace(/[^0-9.-]/g, ''));
    return {
      success: true,
      provider: 'trafficlens',
      domain: normalizeProjectDomain(payload.domain) || domain,
      visits: Number.isFinite(visits) ? visits : null,
      latestMonth: payload.data_month || null,
      monthlyVisits: payload.data_month && Number.isFinite(visits) ? { [payload.data_month]: visits } : {},
      engagements: null,
      bounceRate: Number.isFinite(bounceRate) ? bounceRate : null,
      timeOnSite: null,
      pagesPerVisit: null,
      globalRank: payload.global_rank ?? null,
      countryRank: payload.country_rank ?? null,
      categoryRank: null,
      category: null,
      topCountryShares: null,
      trafficSources: null,
      snapshotDate: payload.data_month || null,
      change: payload.change || null,
      teaser: Boolean(payload.teaser),
      remaining: payload.remaining ?? null,
      limit: payload.limit ?? null,
      proxyRotation: lease.generation,
      proxyCallInRotation: 1,
      proxyIpChecked: proxyIp,
      trafficAttempt: attempt,
      authDiagnostics: {
        provider: 'TrafficLens',
        profileName: `Proxy rotation ${lease.generation}`,
        variablesReceived: { complete: true, session: sessionCookie ? `${'•'.repeat(12)}${sessionCookie.slice(-6)}` : 'Không dùng cookie' },
        request: { method: 'POST', url: 'https://trafficlens.space/api/try', form: { url: domain }, proxyCall: '1/1', proxyRequestId: lease.generation, proxyIpChecked: proxyIp, attempt },
        status: response.status
      },
      raw: payload
    };
  } finally {
    releaseTrafficLensProxyLease(lease);
  }
}

async function requestTrafficLens(domain) {
  const attempts = [];
  let lastError = null;
  for (let attempt = 1; attempt <= 3; attempt++) {
    try {
      const result = await requestTrafficLensAttempt(domain, attempt);
      result.attempts = attempts;
      return result;
    } catch (error) {
      lastError = error;
      attempts.push({ attempt, error: error.message, status: error.statusCode || null, details: error.details || null });
      if (attempt < 3) await sleep(350 * attempt);
    }
  }
  lastError.details = { attempts, lastResponse: lastError.details || null };
  throw lastError;
}

const TRAFFICLENS_API_LIMIT = 60;
const trafficLensApiLocalWindows = new Map();

async function reserveTrafficLensApiQuota(apiKey) {
  const now = Date.now();
  const minute = Math.floor(now / 60000);
  const retryAfterMs = (minute + 1) * 60000 - now + 150;
  const keyHash = crypto.createHash('sha256').update(apiKey).digest('hex').slice(0, 20);
  if (hasPersistentHistory) {
    try {
      const redisKey = `trafficlens:rate:${keyHash}:${minute}`;
      const count = Number(await redisCommand(['INCR', redisKey]));
      if (count === 1) redisCommand(['EXPIRE', redisKey, 75]).catch(() => {});
      return { allowed: count <= TRAFFICLENS_API_LIMIT, count, remaining: Math.max(0, TRAFFICLENS_API_LIMIT - count), retryAfterMs, storage: 'redis' };
    } catch (error) {
      console.warn('TrafficLens distributed rate limiter fallback:', error.message);
    }
  }
  const localKey = `${keyHash}:${minute}`;
  const count = (trafficLensApiLocalWindows.get(localKey) || 0) + 1;
  trafficLensApiLocalWindows.set(localKey, count);
  for (const key of trafficLensApiLocalWindows.keys()) {
    if (!key.endsWith(`:${minute}`)) trafficLensApiLocalWindows.delete(key);
  }
  return { allowed: count <= TRAFFICLENS_API_LIMIT, count, remaining: Math.max(0, TRAFFICLENS_API_LIMIT - count), retryAfterMs, storage: 'memory' };
}

function numberFromTrafficValue(value) {
  if (value === null || value === undefined || value === '') return null;
  const numeric = Number(String(value).replace(/[^0-9.-]/g, ''));
  return Number.isFinite(numeric) ? numeric : null;
}

async function requestTrafficLensApi(domain) {
  const apiKey = decryptAdminSetting(authStore.settings.trafficLensApi?.apiKey);
  if (!apiKey) {
    const error = new Error('Chưa cấu hình X-API-Key của TrafficLens API trong Admin.');
    error.statusCode = 503;
    throw error;
  }
  const failures = [];
  let lastError = null;
  for (let attempt = 1; attempt <= 3; attempt++) {
    const quota = await reserveTrafficLensApiQuota(apiKey);
    if (!quota.allowed) {
      const error = new Error(`TrafficLens API đã đạt giới hạn ${TRAFFICLENS_API_LIMIT} request/phút.`);
      error.statusCode = 429;
      error.rateLimited = true;
      error.retryAfterMs = quota.retryAfterMs;
      error.details = { limit: TRAFFICLENS_API_LIMIT, remaining: 0, retryAfterMs: quota.retryAfterMs, limiter: quota.storage };
      throw error;
    }
    try {
      const endpoint = `https://api.trafficlens.space/v1/traffic?domain=${encodeURIComponent(domain)}`;
      const response = await trackedFetch(endpoint, {
        headers: { accept: 'application/json', 'X-API-Key': apiKey },
        signal: AbortSignal.timeout(15000)
      }, null, {
        category: 'external',
        direction: 'Server → TrafficLens Official API',
        connection: 'IP server trực tiếp · API Key',
        trafficProvider: 'trafficlens_api',
        trafficDomain: domain,
        trafficAttempt: attempt,
        rateLimitRemaining: quota.remaining,
        rateLimitStorage: quota.storage
      });
      const text = await response.text();
      let payload = null;
      try { payload = text ? JSON.parse(text) : null; } catch (_) {}
      if (!response.ok) {
        const detail = payload?.error || payload?.message || text.slice(0, 500) || `HTTP ${response.status}`;
        const error = new Error(`TrafficLens Official API báo lỗi: ${detail}`);
        error.statusCode = response.status;
        error.details = payload || text.slice(0, 1000);
        if (response.status === 429) {
          error.rateLimited = true;
          error.retryAfterMs = Math.max(1000, Number(response.headers.get('retry-after') || 0) * 1000 || quota.retryAfterMs);
        }
        throw error;
      }
      const data = payload?.data && typeof payload.data === 'object' ? payload.data : payload;
      if (!data || typeof data !== 'object') throw new Error('TrafficLens Official API không trả về JSON hợp lệ.');
      if (data.error || (data.status && String(data.status).toUpperCase() !== 'OK')) {
        const error = new Error(`TrafficLens Official API báo lỗi: ${data.error || data.message || data.status}`);
        error.statusCode = 502;
        error.details = data;
        throw error;
      }
      const trafficData = data.traffic && typeof data.traffic === 'object' ? data.traffic : data;
      const rankData = data.rank && typeof data.rank === 'object' ? data.rank : data;
      const engagementData = data.engagement && typeof data.engagement === 'object' ? data.engagement : (data.engagements || data.Engagments || data.Engagements || {});
      const metaData = data.meta && typeof data.meta === 'object' ? data.meta : data;
      const monthlyRows = Array.isArray(trafficData.monthly) ? trafficData.monthly : [];
      const monthlyVisits = monthlyRows.length
        ? Object.fromEntries(monthlyRows.map(item => [String(item?.month || '').slice(0, 7), numberFromTrafficValue(item?.visits)]).filter(([month]) => /^\d{4}-\d{2}$/.test(month)))
        : (trafficData.monthly_visits || trafficData.monthlyVisits || data.EstimatedMonthlyVisits || {});
      const latestMonth = metaData.data_month || data.latestMonth || Object.keys(monthlyVisits).sort().pop() || null;
      const visits = numberFromTrafficValue(trafficData.visits ?? trafficData.total_visits ?? trafficData.totalVisits ?? trafficData.Visits ?? (latestMonth ? monthlyVisits[latestMonth] : null));
      const bounceRate = numberFromTrafficValue(engagementData.bounce_rate ?? engagementData.bounceRate ?? engagementData.BounceRate);
      return {
        success: true,
        provider: 'trafficlens_api',
        domain: normalizeProjectDomain(data.domain) || domain,
        visits,
        latestMonth,
        monthlyVisits,
        engagements: Object.keys(engagementData).length ? engagementData : null,
        bounceRate,
        timeOnSite: numberFromTrafficValue(engagementData.time_on_site ?? engagementData.timeOnSite ?? engagementData.TimeOnSite),
        pagesPerVisit: numberFromTrafficValue(engagementData.pages_per_visit ?? engagementData.pagesPerVisit ?? engagementData.PagesPerVisit),
        globalRank: rankData.global ?? rankData.global_rank ?? rankData.globalRank ?? null,
        countryRank: rankData.country ?? rankData.country_rank ?? rankData.countryRank ?? null,
        categoryRank: rankData.category ?? rankData.category_rank ?? rankData.categoryRank ?? null,
        category: metaData.category ?? data.category ?? null,
        topCountryShares: data.top_country_shares || data.topCountryShares || null,
        trafficSources: data.traffic_sources || data.trafficSources || null,
        snapshotDate: metaData.data_month || data.snapshotDate || null,
        change: trafficData.change ?? data.change ?? null,
        whois: data.whois || null,
        meta: data.meta || null,
        keywords: Array.isArray(data.keywords) ? data.keywords : [],
        rateLimit: { limit: TRAFFICLENS_API_LIMIT, remaining: quota.remaining, storage: quota.storage },
        trafficAttempt: attempt,
        attempts: failures,
        authDiagnostics: {
          provider: 'TrafficLens Official API',
          profileName: 'API Key',
          variablesReceived: { complete: true, apiKeyHeaderSent: true, headerName: 'X-API-Key', apiKey: `${apiKey.slice(0, 8)}${'•'.repeat(10)}${apiKey.slice(-4)}` },
          request: { method: 'GET', url: endpoint, domain, attempt, headers: { 'X-API-Key': `${apiKey.slice(0, 8)}${'•'.repeat(10)}${apiKey.slice(-4)}`, accept: 'application/json' } },
          status: response.status
        },
        raw: payload
      };
    } catch (error) {
      lastError = error;
      if (error.rateLimited) throw error;
      failures.push({ attempt, status: error.statusCode || null, error: error.message, details: error.details || null });
      if (attempt < 3) await sleep(350 * attempt);
    }
  }
  lastError.details = { attempts: failures, lastResponse: lastError.details || null };
  throw lastError;
}

async function requestSiteTrafficApi(domain) {
  const apiKey = decryptAdminSetting(authStore.settings.siteTrafficApi?.apiKey);
  if (!apiKey) {
    const error = new Error('Ch\u01b0a c\u1ea5u h\u00ecnh API key c\u1ee7a SiteTraffic trong Admin.');
    error.statusCode = 503;
    throw error;
  }
  const endpoint = `https://api.goanyapi.com/api/v1/traffic?domain=${encodeURIComponent(domain)}`;
  const response = await trackedFetch(endpoint, {
    headers: { accept: 'application/json', authorization: `Bearer ${apiKey}` },
    signal: AbortSignal.timeout(20000)
  }, null, {
    category: 'external',
    direction: 'Server -> SiteTraffic API (GoAnyAPI)',
    connection: 'IP server truc tiep - Bearer API Key',
    trafficProvider: 'sitetraffic_api',
    trafficDomain: domain
  });
  const text = await response.text();
  let payload = null;
  try { payload = text ? JSON.parse(text) : null; } catch (_) {}
  if (!response.ok || !payload || String(payload.code || '').toLowerCase() !== 'ok') {
    const detail = payload?.message || payload?.error || text.slice(0, 500) || `HTTP ${response.status}`;
    const error = new Error(`SiteTraffic API b\u00e1o l\u1ed7i: ${detail}`);
    error.statusCode = response.status || 502;
    error.details = payload || text.slice(0, 1000);
    error.rateLimited = response.status === 429;
    error.retryAfterMs = error.rateLimited ? Math.max(1000, Number(response.headers.get('retry-after') || 0) * 1000 || 60000) : null;
    throw error;
  }
  const data = payload.data;
  if (!data || typeof data !== 'object') {
    const error = new Error('SiteTraffic API kh\u00f4ng tr\u1ea3 v\u1ec1 d\u1eef li\u1ec7u h\u1ee3p l\u1ec7.');
    error.statusCode = 502;
    throw error;
  }
  const monthlyVisits = data.EstimatedMonthlyVisits && typeof data.EstimatedMonthlyVisits === 'object' ? data.EstimatedMonthlyVisits : {};
  const latestMonthKey = Object.keys(monthlyVisits).sort().pop() || null;
  const engagements = data.Engagments || data.Engagements || null;
  const engagementMonth = engagements?.Year && engagements?.Month ? `${engagements.Year}-${String(engagements.Month).padStart(2, '0')}` : null;
  return {
    success: true,
    provider: 'sitetraffic_api',
    domain: normalizeProjectDomain(data.SiteName || data.query?.domain) || domain,
    visits: numberFromTrafficValue(engagements?.Visits ?? (latestMonthKey ? monthlyVisits[latestMonthKey] : null)),
    latestMonth: engagementMonth || latestMonthKey?.slice(0, 7) || null,
    monthlyVisits,
    engagements,
    bounceRate: numberFromTrafficValue(engagements?.BounceRate),
    timeOnSite: numberFromTrafficValue(engagements?.TimeOnSite),
    pagesPerVisit: numberFromTrafficValue(engagements?.PagePerVisit),
    globalRank: data.GlobalRank?.Rank ?? data.GlobalRank ?? null,
    countryRank: data.CountryRank?.Rank ?? data.CountryRank ?? null,
    categoryRank: data.CategoryRank?.Rank ?? data.CategoryRank ?? null,
    category: data.Category || data.CategoryRank?.Category || null,
    topCountryShares: data.TopCountryShares || null,
    trafficSources: data.TrafficSources || null,
    snapshotDate: engagementMonth || latestMonthKey || null,
    change: null,
    whois: data.DateData || null,
    meta: { title: data.Title || null, description: data.Description || null, policy: data.Policy ?? null },
    keywords: Array.isArray(data.TopKeywords) ? data.TopKeywords : [],
    costCredits: numberFromTrafficValue(data.costCredits),
    remainingCredits: numberFromTrafficValue(data.remainingCredits),
    authDiagnostics: {
      provider: 'SiteTraffic API',
      profileName: 'GoAnyAPI Bearer Key',
      variablesReceived: { complete: true, authorizationHeaderSent: true, apiKey: `${apiKey.slice(0, 6)}${'\u2022'.repeat(10)}${apiKey.slice(-4)}` },
      request: { method: 'GET', url: endpoint, domain, headers: { authorization: `Bearer ${apiKey.slice(0, 6)}${'\u2022'.repeat(10)}${apiKey.slice(-4)}`, accept: 'application/json' } },
      status: response.status
    },
    raw: payload
  };
}

app.get('/api/domain-traffic', async (req, res) => {
  if (req.user.role !== 'admin' && !req.user.canUseTraffic) {
    return res.status(403).json({ success: false, error: 'Tài khoản chưa được Admin cấp quyền Quét Traffic.' });
  }
  const domain = normalizeProjectDomain(req.query.domain);
  if (!domain) return res.status(400).json({ success: false, error: 'Domain không hợp lệ.' });
  const requestedProvider = String(req.query.provider || '').trim().toLowerCase();
  if (requestedProvider && !['workfast', 'trafficlens', 'trafficlens_api', 'sitetraffic_api', 'workfast_sitetraffic'].includes(requestedProvider)) {
    return res.status(400).json({ success: false, error: 'Nguồn API Traffic không hợp lệ.' });
  }
  const trafficProvider = requestedProvider || authStore.settings.trafficProvider || 'workfast';
  if (trafficProvider === 'workfast_sitetraffic') {
    const workfastAttempts = [];
    const maxAttempts = Math.max(1, enabledTrafficProfiles().length);
    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
      const workfastResponse = await fetch(`${INTERNAL_BASE_URL}/api/domain-traffic?domain=${encodeURIComponent(domain)}&provider=workfast`, { headers: { 'x-internal-key': INTERNAL_API_KEY }, signal: AbortSignal.timeout(70000) });
      const workfastResult = await workfastResponse.json().catch(() => ({}));
      if (workfastResponse.ok && workfastResult?.success) {
        return res.json({ ...workfastResult, provider: 'workfast', requestedProvider: 'workfast_sitetraffic', fallbackUsed: false });
      }
      workfastAttempts.push({ attempt, status: workfastResponse.status, error: workfastResult?.error || `HTTP ${workfastResponse.status}`, profile: workfastResult?.profile || null });
      const quotaLimited = workfastResponse.status === 429 && (workfastResult?.retryable || workfastResult?.rateLimited || /h\u1ebft h\u1ea1n m\u1ee9c|rate\s*limit|too\s*many/i.test(String(workfastResult?.error || '')));
      if (!quotaLimited) return res.status(workfastResponse.status || 502).json({ ...workfastResult, requestedProvider: 'workfast_sitetraffic', fallbackUsed: false });
      if (!workfastResult?.nextProfile) break;
    }
    try {
      const fallback = await requestSiteTrafficApi(domain);
      return res.json({ ...fallback, requestedProvider: 'workfast_sitetraffic', fallbackUsed: true, fallbackFrom: 'workfast', workfastAttempts });
    } catch (error) {
      return res.status(Number(error.statusCode) >= 400 ? Number(error.statusCode) : 502).json({ success: false, provider: 'sitetraffic_api', requestedProvider: 'workfast_sitetraffic', fallbackUsed: true, fallbackFrom: 'workfast', error: error.message || 'Workfast h\u1ebft h\u1ea1n m\u1ee9c v\u00e0 SiteTraffic c\u0169ng kh\u00f4ng th\u1ec3 x\u1eed l\u00fd.', workfastAttempts, details: error.details || null });
    }
  }
  if (trafficProvider === 'sitetraffic_api') {
    try {
      return res.json(await requestSiteTrafficApi(domain));
    } catch (error) {
      return res.status(Number(error.statusCode) >= 400 ? Number(error.statusCode) : 502).json({
        success: false,
        provider: 'sitetraffic_api',
        error: error.message || 'Kh\u00f4ng th\u1ec3 k\u1ebft n\u1ed1i SiteTraffic API.',
        retryable: Boolean(error.rateLimited),
        rateLimited: Boolean(error.rateLimited),
        retryAfterMs: error.retryAfterMs || null,
        details: error.details || null
      });
    }
  }
  if (trafficProvider === 'trafficlens_api') {
    try {
      return res.json(await requestTrafficLensApi(domain));
    } catch (error) {
      return res.status(Number(error.statusCode) >= 400 ? Number(error.statusCode) : 502).json({
        success: false,
        provider: 'trafficlens_api',
        error: error.message || 'Không thể kết nối TrafficLens Official API.',
        retryable: Boolean(error.rateLimited),
        rateLimited: Boolean(error.rateLimited),
        retryAfterMs: error.retryAfterMs || null,
        details: error.details || null
      });
    }
  }
  if (trafficProvider === 'trafficlens') {
    try {
      return res.json(await requestTrafficLens(domain));
    } catch (error) {
      return res.status(Number(error.statusCode) >= 400 ? Number(error.statusCode) : 502).json({
        success: false,
        provider: 'trafficlens',
        error: error.message || 'Không thể kết nối TrafficLens.',
        details: error.details || null
      });
    }
  }
  const errors = [];

  // Lần 1: Dùng chữ ký dùng chung (chỉ gọi Ngrok 1 lần duy nhất với timeout 60s)
  let auth = await getSharedTrafficAuth(false, errors);

  if (!auth) {
    return res.status(502).json({
      success: false,
      error: 'Chưa lấy được chữ ký xác thực từ Ngrok (implant-buddy-bring.ngrok-free.dev). Vui lòng kiểm tra lại Ngrok tunnel.',
      details: errors
    });
  }

  const attemptedProfiles = [];
  const trafficAttempt = 0;
  try {
    const params = new URLSearchParams({
      domain,
      timestamp: auth.timestamp,
      source: 'extension',
      clientId: auth.clientId,
      sign: auth.sign
    });
    const trafficUrl = `https://traffic.workfast.cc/?${params}`;
    const trafficResponse = await trackedFetch(trafficUrl, {
      headers: { accept: 'application/json' },
      signal: AbortSignal.timeout(30000)
    }, null, {
      category: 'external',
      direction: `Server → Workfast Traffic API (${auth.profileName || `Profile ${auth.profile}`})`,
      connection: 'IP server trực tiếp',
      trafficAttempt: trafficAttempt + 1,
      trafficProfile: auth.profile,
      trafficProfileName: auth.profileName || `Profile ${auth.profile}`,
      trafficDomain: domain
    });

    const trafficText = await trafficResponse.text();
    let traffic;
    try { traffic = JSON.parse(trafficText); } catch (error) { traffic = null; }

    const hasTrafficData = isWorkfastTrafficPayload(traffic);
    if (traffic && (hasTrafficData || (trafficResponse.ok && !trafficSessionExpired(trafficResponse.status, traffic)))) {
      const monthlyVisits = traffic.EstimatedMonthlyVisits && typeof traffic.EstimatedMonthlyVisits === 'object'
        ? traffic.EstimatedMonthlyVisits
        : {};
      const latestMonth = Object.keys(monthlyVisits).sort().pop() || null;
      const visits = Number(traffic.Engagments?.Visits ?? (latestMonth ? monthlyVisits[latestMonth] : null));
      const engagements = traffic.Engagments || traffic.Engagements || {};
      const bounceRate = engagements.BounceRate ?? engagements.Bounce_Rate ?? traffic.BounceRate ?? null;
      const timeOnSite = engagements.TimeOnSite ?? engagements.AvgVisitDuration ?? engagements.VisitDuration ?? traffic.TimeOnSite ?? null;
      const pagesPerVisit = engagements.PageViews ?? engagements.PagesPerVisit ?? engagements.PagePerVisit
        ?? engagements.PagesPerSession ?? traffic.PagesPerVisit ?? traffic.PageViews ?? null;

      return res.json({
        success: true,
        domain,
        reusedAuth: true,
        visits: Number.isFinite(visits) ? visits : null,
        latestMonth,
        monthlyVisits,
        engagements: Object.keys(engagements).length ? engagements : null,
        bounceRate: bounceRate === null ? null : Number(bounceRate),
        timeOnSite: timeOnSite === null ? null : Number(timeOnSite),
        pagesPerVisit: pagesPerVisit === null ? null : Number(pagesPerVisit),
        globalRank: traffic.GlobalRank?.Rank ?? null,
        countryRank: traffic.CountryRank?.Rank ?? null,
        categoryRank: traffic.CategoryRank?.Rank ?? null,
        category: traffic.Category || traffic.CategoryName || null,
        topCountryShares: traffic.TopCountryShares || traffic.Geo || null,
        trafficSources: traffic.TrafficSources || null,
        snapshotDate: traffic.SnapshotDate || null,
        profile: auth.profile,
        profileName: auth.profileName || `Profile ${auth.profile}`,
        attemptedProfiles,
        authDiagnostics: buildTrafficDiagnostics(auth, domain, trafficResponse.status),
        raw: traffic
      });
    } else {
      const detailMsg = traffic?.message || traffic?.error || (typeof traffic === 'object' ? JSON.stringify(traffic) : (trafficText ? trafficText.slice(0, 150) : 'Session expired'));
      const isRateLimited = trafficResponse.status === 429 || /rate\s*limit|too\s*many\s*requests/i.test(detailMsg);
      const isProfileUnavailable = isRateLimited || trafficResponse.status === 502;
      if (isProfileUnavailable) {
        attemptedProfiles.push({ id: auth.profile, name: auth.profileName || `Profile ${auth.profile}` });
        errors.push(`Profile ${auth.profile}: ${isRateLimited ? 'Rate limit exceeded' : `HTTP 502 - ${detailMsg}`}`);
        const nextAuth = isRateLimited ? await rotateSharedTrafficAuthAfterWorkfastRateLimit(auth, errors) : null;
        return res.status(isRateLimited ? 429 : 502).json({
          success: false,
          error: isRateLimited
            ? `Profile ${auth.profile} đã hết hạn mức Workfast. Đã chuẩn bị bộ sign kế tiếp để dùng chung.`
            : `Workfast trả về 502 - ${detailMsg}. Không gọi lại /get-sign vì đây không phải lỗi hết hạn mức.`,
          retryable: isRateLimited,
          nextProfile: nextAuth ? { id: nextAuth.profile, name: nextAuth.profileName || `Profile ${nextAuth.profile}` } : null,
          profileCount: enabledTrafficProfiles().length,
          attemptedProfiles,
          authDiagnostics: buildTrafficDiagnostics(auth, domain, trafficResponse.status),
          details: errors
        });
      }
      if ([401, 403, 419].includes(trafficResponse.status)) {
        if (cachedTrafficAuth && cachedTrafficAuth.sign === auth.sign) cachedTrafficAuth.rejectedAt = Date.now();
      }
      return res.status(502).json({
        success: false,
        error: `Workfast Traffic API báo lỗi: ${detailMsg}`,
        status: trafficResponse.status,
        profile: auth.profile,
        attemptedProfiles,
        authDiagnostics: buildTrafficDiagnostics(auth, domain, trafficResponse.status),
        connection: 'IP server trực tiếp'
      });
    }
  } catch (error) {
    return res.status(502).json({
      success: false,
      error: `Kết nối Workfast Traffic API thất bại: ${error.message}`,
      retryable: false,
      authDiagnostics: buildTrafficDiagnostics(auth, domain, 'NETWORK_ERROR'),
      details: {
        connection: 'IP server trực tiếp',
        attemptedProfiles,
        cause: error.cause?.message || null,
        code: error.cause?.code || error.code || null
      }
    });
  }
});

async function scanAdvertiserDomainsForMonitor(schedule, advertiserId) {
  const adminCurl = parseAdminCurl(authStore.settings.curlCommand);
  if (!adminCurl) throw new Error('Admin chưa cấu hình xác thực Google.');
  schedule.headers = adminCurl.headers;
  schedule.fReq = adminCurl.fReq || {};
  let response;
  try {
    response = await fetch(`${INTERNAL_BASE_URL}/api/advertiser-creatives`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', 'x-internal-key': INTERNAL_API_KEY },
      body: JSON.stringify({
        advId: advertiserId,
        headers: schedule.headers || {},
        fReq: schedule.fReq || {},
        proxy: schedule.useProxy ? authStore.settings.proxyUrl : '',
        disableProxy: !schedule.useProxy,
        monitorMode: true
      }),
      signal: AbortSignal.timeout(5 * 60 * 1000)
    });
  } catch (error) {
    throw new Error(`Không kết nối được dịch vụ quét nội bộ: ${error.cause?.code || error.message}`);
  }
  if (!response.ok) throw new Error(`HTTP ${response.status}`);
  const text = await response.text();
  const ocrCandidates = [];
  for (const line of text.split('\n')) {
    if (!line.trim()) continue;
    const chunk = JSON.parse(line);
    if (chunk.type === 'error') throw new Error(chunk.error);
    if (chunk.type !== 'creatives') continue;
    for (const creative of [...(chunk.creatives || []), ...(chunk.filteredCreatives || [])]) {
      const detail = {
        creativeId: creative.creativeId || '',
        advertiserId,
        adUrl: creative.creativeId
          ? `https://adstransparency.google.com/advertiser/${encodeURIComponent(advertiserId)}/creative/${encodeURIComponent(creative.creativeId)}?region=anywhere`
          : `https://adstransparency.google.com/advertiser/${encodeURIComponent(advertiserId)}?region=anywhere`
      };
      if (creative.imageUrl || creative.previewUrl) {
        ocrCandidates.push({ detail, imageUrl: creative.imageUrl || '', previewUrl: creative.previewUrl || '' });
      }
    }
  }
  return ocrCandidates;
}

async function runCompetitorMonitor(schedule) {
  if (!schedule || schedule.running) return;
  schedule.awaitingClientOcr = true;
  schedule.lastError = '';
  saveMonitorStore();
}

async function completeCompetitorMonitor(schedule, submittedResults) {
  if (!schedule) return;
  try {
    schedule.snapshot = schedule.snapshot || {};
    for (const advertiserId of schedule.advertiserIds) {
      const found = new Map();
      for (const item of submittedResults?.[advertiserId] || []) {
        const domain = normalizeProjectDomain(item?.domain);
        if (domain && !found.has(domain)) found.set(domain, item.detail || {});
      }
      const previous = new Set(Array.isArray(schedule.snapshot[advertiserId]) ? schedule.snapshot[advertiserId] : []);
      const newDomains = [];
      const knownDomains = [];
      for (const [domain, detail] of found) {
        (previous.has(domain) ? knownDomains : newDomains).push({ domain, ...detail });
      }
      if (schedule.lastRunAt) {
        for (const { domain, ...detail } of newDomains) {
          monitorStore.notifications.unshift({
            id: monitorId('notification'),
            scheduleId: schedule.id,
            ownerId: schedule.ownerId,
            advertiserId,
            domain,
            creativeId: detail.creativeId,
            adUrl: detail.adUrl,
            detectedAt: new Date().toISOString(),
            read: false
          });
        }
        if (newDomains.length) {
          try {
            await sendTelegramNotification(schedule, advertiserId, newDomains);
            schedule.lastTelegramError = '';
          } catch (error) {
            schedule.lastTelegramError = error.message;
          }
        }
      }
      schedule.snapshot[advertiserId] = Array.from(found.keys());
      schedule.lastScanResults = schedule.lastScanResults || {};
      schedule.lastScanResults[advertiserId] = {
        scannedAt: new Date().toISOString(),
        totalDomains: found.size,
        newDomains: schedule.lastRunAt ? newDomains : [],
        knownDomains: schedule.lastRunAt ? knownDomains : Array.from(found, ([domain, detail]) => ({ domain, ...detail }))
      };
    }
    schedule.lastRunAt = new Date().toISOString();
    schedule.awaitingClientOcr = false;
  } catch (error) {
    schedule.lastError = error.message;
  } finally {
    schedule.running = false;
    schedule.ocrDeviceStartedAt = null;
    schedule.ocrDeviceId = null;
    schedule.scanProgress = null;
    const intervalMinutes = Number(schedule.intervalMinutes || (schedule.intervalHours * 60) || 1440);
    schedule.nextRunAt = new Date(Date.now() + intervalMinutes * 60000).toISOString();
    monitorStore.notifications = monitorStore.notifications.slice(0, 1000);
    saveMonitorStore();
  }
}

app.get('/api/competitor-monitor', async (req, res) => {
  try { await refreshSupabaseMonitorState(); }
  catch (error) { console.warn('Using cached monitor state:', error.message); }
  let changed = false;
  monitorStore.schedules.forEach(schedule => {
    const belongsToUser = req.user.role === 'admin' || schedule.ownerId === req.user.id;
    if (!belongsToUser) return;
    if (schedule.running) {
      const startedAt = new Date(schedule.ocrDeviceStartedAt || 0).getTime();
      const staleDeviceLock = !schedule.ocrDeviceId || !Number.isFinite(startedAt) || startedAt <= 0 || Date.now() - startedAt > 60 * 60 * 1000;
      if (staleDeviceLock) {
        schedule.running = false;
        schedule.ocrDeviceStartedAt = null;
        schedule.ocrDeviceId = null;
        schedule.scanProgress = null;
        schedule.lastError = 'Phiên OCR trước đã ngắt kết nối; thiết bị hiện tại sẽ tự động nhận lại.';
        changed = true;
      }
    }
    if (!schedule.running && !schedule.awaitingClientOcr && new Date(schedule.nextRunAt || 0).getTime() <= Date.now()) {
      schedule.awaitingClientOcr = true;
      schedule.lastError = '';
      changed = true;
    }
  });
  if (changed) await saveMonitorStore();
  const visibleSchedules = req.user.role === 'admin' ? monitorStore.schedules : monitorStore.schedules.filter(item => item.ownerId === req.user.id);
  const visibleNotifications = req.user.role === 'admin' ? monitorStore.notifications : monitorStore.notifications.filter(item => item.ownerId === req.user.id);
  const schedules = visibleSchedules.map(({ headers, fReq, proxy, telegram, ...safe }) => {
    const ownerTelegram = getUserTelegram(safe.ownerId);
    return {
      ...safe,
      telegramEnabled: Boolean(ownerTelegram.enabled),
      telegramConfigured: Boolean(ownerTelegram.botToken && ownerTelegram.chatId)
    };
  });
  const telegram = getUserTelegram(req.user.id);
  const token = String(telegram.botToken || '');
  const tokenHint = token
    ? `${token.slice(0, Math.min(6, token.length))}${token.length > 10 ? '••••••' : '•••'}${token.slice(-4)}`
    : '';
  res.json({
    schedules, notifications: visibleNotifications,
    unread: visibleNotifications.filter(item => !item.read).length,
    curlConfigured: Boolean(parseAdminCurl(authStore.settings.curlCommand)),
    telegram: {
      enabled: Boolean(telegram.enabled), configured: Boolean(telegram.botToken && telegram.chatId),
      chatId: telegram.chatId || '', tokenHint
    }
  });
});

app.put('/api/competitor-monitor/telegram', async (req, res) => {
  const current = getUserTelegram(req.user.id);
  const enabled = Boolean(req.body?.enabled);
  const botToken = String(req.body?.botToken || '').trim() || current.botToken || '';
  const chatId = String(req.body?.chatId || '').trim() || current.chatId || '';
  if (enabled && (!/^\d+:[A-Za-z0-9_-]+$/.test(botToken) || !chatId)) {
    return res.status(400).json({ ok: false, error: 'Cần Bot Token và Chat ID hợp lệ.' });
  }
  monitorStore.telegramByUser[req.user.id] = { enabled, botToken, chatId };
  await saveMonitorStore();
  res.json({ ok: true, enabled, configured: Boolean(botToken && chatId), chatId });
});

app.post('/api/competitor-monitor/telegram/test', async (req, res) => {
  try {
    const saved = getUserTelegram(req.user.id);
    const botToken = String(req.body?.botToken || '').trim() || saved.botToken || '';
    const chatId = String(req.body?.chatId || '').trim() || saved.chatId || '';
    const result = await sendTelegramTest(botToken, chatId);
    const chat = result.chat;
    res.json({ ok: true, botUsername: result.bot.username, chatId: chat.id, chatTitle: chat.title || chat.username || chat.first_name || String(chat.id || '') });
  } catch (error) {
    res.status(400).json({ ok: false, error: error.message });
  }
});

app.post('/api/competitor-monitor', async (req, res) => {
  const advertiserIds = [...new Set((req.body?.advertiserIds || []).map(value => String(value).trim()).filter(Boolean))];
  const intervalMinutes = Number(req.body?.intervalMinutes || (Number(req.body?.intervalHours) * 60));
  if (!advertiserIds.length || !Number.isFinite(intervalMinutes) || intervalMinutes < 1) {
    return res.status(400).json({ error: 'Cần ít nhất một Advertiser ID và chu kỳ từ 1 phút.' });
  }
  const scheduleFReq = JSON.parse(JSON.stringify(req.body.fReq || {}));
  if (scheduleFReq['3']) {
    delete scheduleFReq['3']['6'];
    delete scheduleFReq['3']['7'];
  }
  const schedule = {
    id: monitorId('schedule'), ownerId: req.user.id, advertiserIds, intervalMinutes,
    headers: req.body.headers || {}, fReq: scheduleFReq,
    useProxy: (req.user.role === 'admin' || Boolean(req.user.canUseProxy)) && req.body.disableProxy !== true,
    snapshot: {}, createdAt: new Date().toISOString(),
    lastRunAt: null, nextRunAt: new Date().toISOString(), lastError: '', lastTelegramError: '', running: false
  };
  monitorStore.schedules.push(schedule);
  saveMonitorStore();
  runCompetitorMonitor(schedule);
  res.status(201).json({ ok: true, id: schedule.id });
});

app.post('/api/competitor-monitor/:id/run', (req, res) => {
  const schedule = monitorStore.schedules.find(item => item.id === req.params.id && (req.user.role === 'admin' || item.ownerId === req.user.id));
  if (!schedule) return res.status(404).json({ error: 'Không tìm thấy lịch.' });
  if (req.body?.headers && Object.keys(req.body.headers).length) schedule.headers = req.body.headers;
  if (req.body?.disableProxy !== undefined) schedule.useProxy = (req.user.role === 'admin' || Boolean(req.user.canUseProxy)) && !Boolean(req.body.disableProxy);
  saveMonitorStore();
  runCompetitorMonitor(schedule);
  res.json({ ok: true });
});

app.post('/api/competitor-monitor/:id/client-scan', async (req, res) => {
  const schedule = monitorStore.schedules.find(item => item.id === req.params.id && (req.user.role === 'admin' || item.ownerId === req.user.id));
  if (!schedule) return res.status(404).json({ error: 'Không tìm thấy lịch.' });
  if (!schedule.awaitingClientOcr) return res.status(409).json({ error: 'Lịch này chưa đến hạn OCR.' });
  const deviceId = String(req.body?.deviceId || '').trim().slice(0, 100);
  if (!deviceId) return res.status(400).json({ error: 'Thiếu mã thiết bị OCR.' });
  if (schedule.running && schedule.ocrDeviceId !== deviceId) return res.status(409).json({ error: 'Một thiết bị khác đang xử lý lịch này.' });
  schedule.running = true;
  schedule.lastError = '';
  schedule.ocrDeviceId = deviceId;
  schedule.ocrDeviceStartedAt = new Date().toISOString();
  schedule.scanProgress = { stage: 'preparing', message: 'Đang tải danh sách mẫu quảng cáo…', processed: 0, total: 0, percent: 0, updatedAt: new Date().toISOString() };
  await saveMonitorStore();
  try {
    const candidates = {};
    for (let index = 0; index < schedule.advertiserIds.length; index++) {
      const advertiserId = schedule.advertiserIds[index];
      schedule.scanProgress = { stage: 'preparing', message: `Đang tải mẫu của ${advertiserId}`, processed: index, total: schedule.advertiserIds.length, percent: Math.round(index / schedule.advertiserIds.length * 100), updatedAt: new Date().toISOString() };
      await saveMonitorStore();
      candidates[advertiserId] = await scanAdvertiserDomainsForMonitor(schedule, advertiserId);
    }
    const totalCandidates = Object.values(candidates).reduce((sum, items) => sum + items.length, 0);
    schedule.scanProgress = { stage: 'ocr', message: `Đã chuẩn bị ${totalCandidates} mẫu, bắt đầu OCR`, processed: 0, total: totalCandidates, percent: 0, updatedAt: new Date().toISOString() };
    await saveMonitorStore();
    res.json({ ok: true, scheduleId: schedule.id, candidates });
  } catch (error) {
    schedule.running = false;
    schedule.ocrDeviceStartedAt = null;
    schedule.ocrDeviceId = null;
    schedule.scanProgress = null;
    schedule.lastError = error.message;
    saveMonitorStore();
    res.status(502).json({ error: error.message });
  }
});

app.post('/api/competitor-monitor/:id/client-progress', async (req, res) => {
  const schedule = monitorStore.schedules.find(item => item.id === req.params.id && (req.user.role === 'admin' || item.ownerId === req.user.id));
  if (!schedule) return res.status(404).json({ error: 'Không tìm thấy lịch.' });
  if (!schedule.running || String(req.body?.deviceId || '') !== String(schedule.ocrDeviceId || '')) return res.status(409).json({ error: 'Phiên OCR không còn hoạt động trên thiết bị này.' });
  const processed = Math.max(0, Number(req.body?.processed) || 0);
  const total = Math.max(0, Number(req.body?.total) || 0);
  schedule.scanProgress = {
    stage: 'ocr',
    message: total ? `Đang OCR mẫu ${Math.min(processed, total)}/${total}` : 'Đang OCR dữ liệu quảng cáo',
    processed,
    total,
    percent: total ? Math.min(100, Math.round(processed / total * 100)) : 0,
    updatedAt: new Date().toISOString()
  };
  schedule.ocrDeviceStartedAt = new Date().toISOString();
  await saveMonitorStore();
  res.json({ ok: true, progress: schedule.scanProgress });
});

app.post('/api/competitor-monitor/:id/client-results', async (req, res) => {
  const schedule = monitorStore.schedules.find(item => item.id === req.params.id && (req.user.role === 'admin' || item.ownerId === req.user.id));
  if (!schedule) return res.status(404).json({ error: 'Không tìm thấy lịch.' });
  if (!schedule.running || !schedule.awaitingClientOcr) return res.status(409).json({ error: 'Không có phiên OCR trình duyệt đang chờ kết quả.' });
  if (String(req.body?.deviceId || '') !== String(schedule.ocrDeviceId || '')) return res.status(409).json({ error: 'Phiên OCR thuộc về thiết bị khác.' });
  await completeCompetitorMonitor(schedule, req.body?.results || {});
  res.json({ ok: true, lastRunAt: schedule.lastRunAt, nextRunAt: schedule.nextRunAt });
});

app.delete('/api/competitor-monitor/:id', (req, res) => {
  monitorStore.schedules = monitorStore.schedules.filter(item => item.id !== req.params.id || (req.user.role !== 'admin' && item.ownerId !== req.user.id));
  saveMonitorStore();
  res.json({ ok: true });
});

app.patch('/api/competitor-monitor/:id', (req, res) => {
  const schedule = monitorStore.schedules.find(item => item.id === req.params.id && (req.user.role === 'admin' || item.ownerId === req.user.id));
  if (!schedule) return res.status(404).json({ error: 'Không tìm thấy lịch.' });
  const intervalMinutes = Number(req.body?.intervalMinutes);
  if (!Number.isFinite(intervalMinutes) || intervalMinutes < 1) {
    return res.status(400).json({ error: 'Chu kỳ phải từ 1 phút.' });
  }
  schedule.intervalMinutes = intervalMinutes;
  delete schedule.intervalHours;
  schedule.nextRunAt = new Date(Date.now() + intervalMinutes * 60000).toISOString();
  saveMonitorStore();
  res.json({ ok: true, intervalMinutes, nextRunAt: schedule.nextRunAt });
});

app.patch('/api/competitor-monitor/notifications/:id', (req, res) => {
  if (req.params.id === 'all') monitorStore.notifications.forEach(item => { if (req.user.role === 'admin' || item.ownerId === req.user.id) item.read = true; });
  else {
    const item = monitorStore.notifications.find(notification => notification.id === req.params.id && (req.user.role === 'admin' || notification.ownerId === req.user.id));
    if (item) item.read = req.body?.read !== false;
  }
  saveMonitorStore();
  res.json({ ok: true });
});

async function ensureSupabaseHydrated() {
  if (!USE_SUPABASE) {
    if (REQUIRE_SUPABASE) throw new Error('Production chưa có SUPABASE_URL và SUPABASE_SERVICE_ROLE_KEY.');
    return;
  }
  if (!stateHydrationPromise) stateHydrationPromise = (async () => {
    const remoteAuth = await supabaseStateRequest('auth', undefined, 12000);
    if (remoteAuth?.users) {
      authStore = remoteAuth;
      if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = {};
      if (!['workfast', 'trafficlens', 'trafficlens_api', 'sitetraffic_api', 'workfast_sitetraffic'].includes(authStore.settings.trafficProvider)) authStore.settings.trafficProvider = 'workfast';
      if (!authStore.settings.siteTrafficApi || typeof authStore.settings.siteTrafficApi !== 'object') authStore.settings.siteTrafficApi = { apiKey: '' };
      if (!authStore.settings.extensionBridge || typeof authStore.settings.extensionBridge !== 'object') authStore.settings.extensionBridge = { token: '' };
      if (!Array.isArray(authStore.extensionJobs)) authStore.extensionJobs = [];
      if (!authStore.settings.traceGoogleSheet || typeof authStore.settings.traceGoogleSheet !== 'object') authStore.settings.traceGoogleSheet = { spreadsheetUrl: '', sheetName: 'Data', lastSync: null };
      if (!Array.isArray(authStore.loginHistory)) authStore.loginHistory = [];
      if (authStore.loginHistory.length > 500) {
        authStore.loginHistory.length = 500;
        persistSupabaseState('auth', authStore);
      }
    }
    else persistSupabaseState('auth', authStore);
    lastSupabaseRefreshAt = Date.now();
    if (authStore.settings?.proxyUrl) {
      const storedProxy = authStore.settings.proxyUrl;
      try { authStore.settings.proxyUrl = normalizeProxyUrl(storedProxy); } catch (error) { authStore.settings.proxyUrl = ''; }
      if (authStore.settings.proxyUrl !== storedProxy) persistSupabaseState('auth', authStore);
    }
    Promise.all([
      supabaseStateRequest('api_history'), supabaseStateRequest('visitor_history')
    ]).then(([remoteApiHistory, remoteVisitorHistory]) => {
      if (Array.isArray(remoteApiHistory)) {
        apiCallHistory.splice(0, apiCallHistory.length, ...mergeApiHistoryRows(remoteApiHistory));
        // Ghi lại bản legacy đã giới hạn/compact để cold start sau không còn
        // phải tải blob lịch sử hàng nghìn response nguyên khối.
        scheduleApiHistorySync();
      }
      else persistSupabaseState('api_history', apiCallHistory);
      if (Array.isArray(remoteVisitorHistory)) visitorHistory.splice(0, visitorHistory.length, ...remoteVisitorHistory.slice(0, VISITOR_HISTORY_LIMIT));
      else persistSupabaseState('visitor_history', visitorHistory);
    }).catch(error => console.warn('Background Supabase hydration failed:', error.message));
    console.log('Supabase authentication storage connected.');
  })().catch(error => { stateHydrationPromise = null; throw error; });
  return stateHydrationPromise;
}

async function refreshSupabaseSharedState(force = false, authOnly = false) {
  if (!USE_SUPABASE) return;
  await ensureSupabaseHydrated();
  if (!force && Date.now() - lastSupabaseRefreshAt < 30000) return;
  if (supabaseRefreshPromise) return supabaseRefreshPromise;
  supabaseRefreshPromise = (async () => {
    await supabaseWriteQueue;
    const remoteAuth = await supabaseStateRequest('auth', undefined, 8000);
    if (remoteAuth?.users) {
      authStore = remoteAuth;
      if (!authStore.settings || typeof authStore.settings !== 'object') authStore.settings = {};
      if (!authStore.settings.traceGoogleSheet || typeof authStore.settings.traceGoogleSheet !== 'object') authStore.settings.traceGoogleSheet = { spreadsheetUrl: '', sheetName: 'Data', lastSync: null };
      if (!Array.isArray(authStore.loginHistory)) authStore.loginHistory = [];
      if (authStore.loginHistory.length > 500) authStore.loginHistory.length = 500;
    }
    if (!authOnly) {
      const remoteMonitor = await supabaseStateRequest('monitor');
      if (remoteMonitor?.schedules) monitorStore = remoteMonitor;
    }
    lastSupabaseRefreshAt = Date.now();
  })().finally(() => { supabaseRefreshPromise = null; });
  return supabaseRefreshPromise;
}

async function refreshSupabaseMonitorState() {
  if (!USE_SUPABASE) return;
  await supabaseWriteQueue;
  const remoteMonitor = await supabaseStateRequest('monitor');
  if (remoteMonitor?.schedules) monitorStore = remoteMonitor;
}

function protectAsyncLayer(layer) {
  if (layer.route?.stack) {
    layer.route.stack.forEach(protectAsyncLayer);
    return;
  }
  if (!layer?.handle || layer.handle.length === 4 || layer.handle.__asyncProtected) return;
  const original = layer.handle;
  const protectedHandle = function (req, res, next) {
    try {
      const result = original.call(this, req, res, next);
      if (result && typeof result.catch === 'function') result.catch(next);
      return result;
    } catch (error) {
      return next(error);
    }
  };
  protectedHandle.__asyncProtected = true;
  layer.handle = protectedHandle;
}

if (app._router?.stack) app._router.stack.forEach(protectAsyncLayer);

app.use((error, req, res, next) => {
  console.error(`[Server error] ${req.method} ${req.originalUrl}:`, error?.stack || error);
  if (res.headersSent) return res.end();
  res.status(error?.status || 500).json({
    error: error?.message || 'Server gặp lỗi ngoài dự kiến. Vui lòng thử lại.'
  });
});

process.on('unhandledRejection', reason => {
  console.error('[Unhandled promise rejection]', reason?.stack || reason);
});

if (require.main === module) {
  const server = app.listen(PORT, () => {
    console.log(`Server is running at http://localhost:${PORT}`);
  });
  server.keepAliveTimeout = 65000;
  server.headersTimeout = 66000;
  server.requestTimeout = 10 * 60 * 1000;
  server.on('error', error => console.error('[HTTP server error]', error));
}

app.locals.nicheQuotaTestHooks = { isTavilyPlanLimitError, isApiQuotaExhaustedError, isTavilyPoolExhausted, tavilyUsageQuotaState };
module.exports = app;
