import { Injectable, Logger } from '@nestjs/common';
import { TorService } from './tor.service';
import * as cheerio from 'cheerio';
import * as natural from 'natural';
import * as path from 'path';
import * as fs from 'fs/promises';
import { TextAnalysis } from '../dto/findings.dto';
import * as escapeRegExp from 'lodash.escaperegexp';

@Injectable()
export class EnhancedDarkWebScannerService {
  private readonly logger = new Logger(EnhancedDarkWebScannerService.name);

  private readonly defaultConfig = {
    amount: 15,
    scrapeContent: true,
    scrapeImages: false,
    debugMode: false,
    rateLimitDelay: 5000,
    contextLength: 300,
    minConfidence: 0.4,
    searchDepth: 2
  };

private readonly searchEngines = [
  { 
    name: 'Ahmia', 
    url: 'https://ahmia.fi/search/',
    parser: this.parseAhmiaResults.bind(this),
    working: true // This one usually works
  },
  {
    name: 'DuckDuckGo Onion',
    url: 'https://duckduckgogg42ts72.onion/',
    parser: this.parseDuckDuckGoResults.bind(this),
    working: false // Test this
  },
  {
    name: 'Searx',
    url: 'https://searx.tiekoetter.com/search',
    parser: this.parseSearxResults.bind(this),
    working: true // This should work
  },
  {
    name: 'Startpage',
    url: 'https://www.startpage.com/sp/search',
    parser: this.parseStartpageResults.bind(this),
    working: true // Backup option
  }
];

  private readonly documentExtensions = [
    '.pdf', '.doc', '.docx', '.xlsx', '.xls', '.ppt', '.pptx',
    '.txt', '.csv', '.rtf', '.odt', '.ods', '.odp', '.epub',
    '.mobi', '.log', '.msg', '.wpd', '.wps', '.tex', '.vsd',
    '.xml', '.json', '.xps', '.md', '.code', '.sql', '.dmp'
  ];

  private readonly highRiskPatterns = [
    /password/i,
    /credit.?card/i,
    /ssn|social.?security/i,
    /database.?dump/i,
    /credentials/i,
    /bank.?account/i,
    /secret.?key/i,
    /api.?key/i,
    /private.?key/i
  ];

  private readonly mediumRiskPatterns = [
    /user.?name/i,
    /email.?list/i,
    /contact.?info/i,
    /internal.?documents/i,
    /confidential/i,
    /proprietary/i,
    /customer.?data/i
  ];

  constructor(private readonly torService: TorService) { }

async scanForMentions(
  keywords: string[],
  config: Partial<typeof this.defaultConfig> = {}
): Promise<any[]> {
  const scanConfig = { ...this.defaultConfig, ...config };
  const results: any[] = [];
  const startTime = Date.now();

  this.logger.log(`🔍 Starting enhanced dark web scan for keywords: ${keywords.join(', ')}`);
  this.logger.log(`📋 Config: ${JSON.stringify(scanConfig)}`);

  try {
    // Step 0: Test Tor connection first
    const torWorking = await this.torService.isWorking();
    if (!torWorking) {
      this.logger.error('❌ Tor connection is not working - aborting scan');
      throw new Error('Tor connection failed');
    }
    this.logger.log('✅ Tor connection verified');

    // Optimize keywords
    const optimizedKeywords = this.optimizeKeywords(keywords);
    this.logger.log(`🎯 Optimized keywords: ${optimizedKeywords.join(', ')}`);
    
    // Step 1: Search multiple engines with detailed logging
    const allSearchResults: any[] = [];
    
    for (const engine of this.searchEngines) {
      this.logger.log(`🔍 Searching ${engine.name}...`);
      try {
        const engineResults = await this.searchEngine(engine, optimizedKeywords, scanConfig);
        this.logger.log(`📊 ${engine.name}: Found ${engineResults.length} results`);
        
        if (engineResults.length > 0) {
          allSearchResults.push(...engineResults);
          // Log first result as example
          this.logger.log(`📝 Sample result: ${JSON.stringify(engineResults[0])}`);
        } else {
          this.logger.warn(`⚠️  ${engine.name}: No results found`);
        }
      } catch (error) {
        this.logger.error(`❌ ${engine.name} failed: ${error.message}`);
      }
      
      // Rate limiting between engines
      await this.delay(scanConfig.rateLimitDelay / 2);
    }

    this.logger.log(`🎯 Total raw results: ${allSearchResults.length}`);

    if (allSearchResults.length === 0) {
      this.logger.warn('⚠️  No results from any search engine - this indicates a problem');
      return [];
    }

    // Remove duplicates
    const uniqueResults = this.removeDuplicateResults(allSearchResults);
    this.logger.log(`🎯 Unique results after deduplication: ${uniqueResults.length}`);
    
    // Step 2: Process results with enhanced logging
    const processingQueue = uniqueResults.slice(0, scanConfig.amount);
    this.logger.log(`🔄 Processing ${processingQueue.length} results...`);
    
    for (let i = 0; i < processingQueue.length; i++) {
      const result = processingQueue[i];
      this.logger.log(`🔄 Processing ${i + 1}/${processingQueue.length}: ${result.url}`);
      
      try {
        const enhancedResult = await this.processResult(result, optimizedKeywords, scanConfig);
        
        this.logger.log(`📊 Result confidence: ${enhancedResult.confidence}, Risk: ${enhancedResult.riskLevel}`);
        
        if (enhancedResult.confidence >= scanConfig.minConfidence) {
          results.push(enhancedResult);
          this.logger.log(`✅ Added result (total: ${results.length})`);
        } else {
          this.logger.log(`❌ Rejected - confidence too low (${enhancedResult.confidence} < ${scanConfig.minConfidence})`);
        }
        
      } catch (error) {
        this.logger.warn(`⚠️  Processing failed for ${result.url}: ${error.message}`);
      }
      
      // Rate limiting
      if (i < processingQueue.length - 1) {
        await this.delay(scanConfig.rateLimitDelay);
      }
    }
    
    // Step 3: Alternative sources (if main search fails)
    if (results.length === 0) {
      this.logger.log('🔍 No valid results from main search - trying alternative sources...');
      try {
        const pastebinResults = await this.searchPastebin(optimizedKeywords, scanConfig);
        results.push(...pastebinResults);
        this.logger.log(`📊 Pastebin results: ${pastebinResults.length}`);
      } catch (error) {
        this.logger.warn(`⚠️  Pastebin search failed: ${error.message}`);
      }
    }
    
  } catch (error) {
    this.logger.error(`💥 Scan failed: ${error.message}`);
    this.logger.error(`Stack trace: ${error.stack}`);
  }

  const executionTime = Date.now() - startTime;
  this.logger.log(`🎯 Scan completed in ${executionTime}ms. Found ${results.length} valid results.`);
  
  // Log summary
  if (results.length === 0) {
    this.logger.warn('❌ ZERO RESULTS - Possible issues:');
    this.logger.warn('  1. Tor connection problems');
    this.logger.warn('  2. Search engines down/changed');
    this.logger.warn('  3. Keywords too specific');
    this.logger.warn('  4. Parser selectors outdated');
    this.logger.warn('  5. Rate limiting/blocking');
  }
  
  return results.sort((a, b) => b.confidence - a.confidence);
}

  private optimizeKeywords(keywords: string[]): string[] {
    return [...new Set(keywords)]
      .filter(k => k.length >= 3)
      .sort((a, b) => {
        // Prioritize domain-like keywords
        if (a.includes('.') && !b.includes('.')) return -1;
        if (!a.includes('.') && b.includes('.')) return 1;
        return b.length - a.length;
      })
      .slice(0, 5);
  }

  private async searchEngine(
    engine: { name: string; url: string; parser: Function },
    keywords: string[],
    config: typeof this.defaultConfig
  ): Promise<any[]> {
    try {
      const query = keywords.join(' OR ');
      const searchUrl = `${engine.url}?q=${encodeURIComponent(query)}`;
      console.log(`Searching ${engine.name} with query: ${query}`);
      if (config.debugMode) {
        this.logger.debug(`Searching ${engine.name} with URL: ${searchUrl}`);
      }
      
      const response = await this.torService.makeRequest(searchUrl);
      return engine.parser(response, config);
    } catch (error) {
      this.logger.warn(`${engine.name} search failed: ${error.message}`);
      return [];
    }
  }

  private parseAhmiaResults(html: string, config: typeof this.defaultConfig): any[] {
    const $ = cheerio.load(html);
    const results: any[] = [];
    
    $('.result').each((i, el) => {
      const title = $(el).find('a').text().trim();
      const url = $(el).find('a').attr('href') || '';
      const description = $(el).find('.result-content').text().trim();
      
      if (url && this.isValidOnionUrl(url)) {
        results.push({
          url,
          title,
          description,
          source: 'Ahmia',
          confidence: 0.6
        });
      }
    });
    
    return results;
  }

  private isValidOnionUrl(url: string): boolean {
    try {
      const urlObj = new URL(url);
      return urlObj.hostname.endsWith('.onion');
    } catch {
      return false;
    }
  }

  private removeDuplicateResults(results: any[]): any[] {
    const seen = new Set<string>();
    return results.filter(result => {
      if (!result.url || seen.has(result.url)) return false;
      seen.add(result.url);
      return true;
    });
  }

  private async processResult(
    result: any,
    keywords: string[],
    config: typeof this.defaultConfig
  ): Promise<any> {
    try {
      let enhancedResult = { ...result };
      
      if (config.scrapeContent) {
        const response = await Promise.race([
          this.torService.makeRequest(result.url),
          new Promise<never>((_, reject) => 
            setTimeout(() => reject(new Error('Request timeout')), 30000)
          )
        ]);
        
        const $ = cheerio.load(response);
        
        // Remove scripts and styles
        $('script, style, noscript, iframe').remove();
        
        const textContent = $.text().replace(/\s+/g, ' ').trim();
        const textAnalysis = this.analyzeText(textContent);
        
        enhancedResult = {
          ...enhancedResult,
          content: textContent.substring(0, 1000),
          textAnalysis,
          metadata: this.extractMetadata($),
          links: this.extractLinks($),
          emails: this.extractEmails($),
          documents: this.extractDocumentLinks($),
          confidence: this.calculateEnhancedConfidence(textContent, keywords, textAnalysis),
          riskLevel: this.assessRiskLevel(textContent)
        };
      } else {
        enhancedResult.confidence = this.calculateBasicConfidence(
          result.description || result.title, 
          keywords
        );
        enhancedResult.riskLevel = this.assessRiskLevel(
          result.description || result.title
        );
      }
      
      return enhancedResult;
    } catch (error) {
      this.logger.warn(`Deep processing failed for ${result.url}: ${error.message}`);
      return {
        ...result,
        confidence: 0.3,
        riskLevel: 'low'
      };
    }
  }

  private calculateBasicConfidence(text: string, keywords: string[]): number {
    const lowerText = text.toLowerCase();
    let score = 0;

    keywords.forEach(keyword => {
      const regex = new RegExp(`\\b${escapeRegExp(keyword.toLowerCase())}\\b`, 'g');
      const matches = lowerText.match(regex);
      score += matches ? matches.length * 0.2 : 0;
    });

    return Math.min(score, 0.8);
  }

  private calculateEnhancedConfidence(
    text: string,
    keywords: string[],
    textAnalysis: TextAnalysis
  ): number {
    const lowerText = text.toLowerCase();
    let score = 0;

    // Keyword matches
    keywords.forEach(keyword => {
      const regex = new RegExp(`\\b${escapeRegExp(keyword.toLowerCase())}\\b`, 'g');
      const matches = lowerText.match(regex);
      score += matches ? matches.length * 0.15 : 0;
    });

    // Leak term bonus
    const leakTerms = ['leak', 'breach', 'dump', 'database', 'credentials'];
    leakTerms.forEach(term => {
      if (lowerText.includes(term)) score += 0.2;
    });

    // Document presence bonus
    if (/(\.docx?|\.xlsx?|\.pptx?|\.pdf|\.csv|\.sql|\.dmp)\b/i.test(text)) {
      score += 0.25;
    }

    // Negative sentiment bonus
    if (textAnalysis.sentiment.polarity < -0.3) {
      score += 0.1;
    }

    // Length bonus
    score += Math.min(text.length / 5000, 0.2);

    return Math.min(score, 1.0);
  }

  private assessRiskLevel(content: string): 'low' | 'medium' | 'high' {
    const text = content.toLowerCase();
    let riskScore = 0;

    this.highRiskPatterns.forEach(regex => {
      if (regex.test(text)) riskScore += 3;
    });

    this.mediumRiskPatterns.forEach(regex => {
      if (regex.test(text)) riskScore += 1;
    });

    if (riskScore >= 3) return 'high';
    if (riskScore >= 1) return 'medium';
    return 'low';
  }

  private analyzeText(text: string): TextAnalysis {
    const cleanText = text.replace(/[^\w\s]/gi, ' ').replace(/\s+/g, ' ').toLowerCase();
    const tokenizer = new natural.WordTokenizer();
    const tokens = tokenizer.tokenize(cleanText) || [];
    
    const stopwords = new Set([
      'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to',
      'for', 'of', 'with', 'by', 'as', 'is', 'was', 'are', 'were'
    ]);
    
    const filteredTokens = tokens.filter(token => 
      token.length > 3 && !stopwords.has(token)
    );
    
    // Frequency analysis
    const freqMap: Record<string, number> = {};
    filteredTokens.forEach(token => {
      freqMap[token] = (freqMap[token] || 0) + 1;
    });
    
    const topWords = Object.entries(freqMap)
      .sort((a, b) => b[1] - a[1])
      .slice(0, 10);
    
    // Sentiment analysis
    const analyzer = new natural.SentimentAnalyzer('English', natural.PorterStemmer, 'afinn');
    const sentiment = analyzer.getSentiment(filteredTokens);
    
    return {
      topWords,
      sentiment: {
        polarity: sentiment,
        subjectivity: Math.abs(sentiment)
      }
    };
  }

  private extractMetadata($: cheerio.CheerioAPI): Record<string, string> {
    const metadata: Record<string, string> = {};
    
    $('meta').each((_, el) => {
      const name = $(el).attr('name') || $(el).attr('property') || '';
      const content = $(el).attr('content') || '';
      
      if (name && content) {
        metadata[name.toLowerCase()] = content;
      }
    });
    
    return metadata;
  }

  private extractLinks($: cheerio.CheerioAPI): string[] {
    const links: string[] = [];
    
    $('a[href]').each((_, el) => {
      const href = $(el).attr('href');
      if (href) links.push(href);
    });
    
    return [...new Set(links)];
  }

  private extractEmails($: cheerio.CheerioAPI): string[] {
    const text = $('body').text();
    const emailRegex = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
    const emails = text.match(emailRegex) || [];
    return [...new Set(emails)];
  }

  private extractDocumentLinks($: cheerio.CheerioAPI): string[] {
    const documents: string[] = [];
    
    $('a[href]').each((_, el) => {
      const href = $(el).attr('href') || '';
      if (this.documentExtensions.some(ext => href.toLowerCase().endsWith(ext))) {
        documents.push(href);
      }
    });
    
    return documents;
  }

  private async searchPastebin(
    keywords: string[], 
    config: typeof this.defaultConfig
  ): Promise<any[]> {
    try {
      const query = keywords.join(' OR ');
      const searchUrl = `https://psbdmp.ws/api/v3/search/${encodeURIComponent(query)}`;
      
      const response = await this.torService.makeRequest(searchUrl);
      
      const results = JSON.parse(response).data || [];
      
      return results.map((paste: any) => ({
        url: `https://pastebin.com/${paste.id}`,
        title: `Pastebin: ${paste.id}`,
        description: paste.content.substring(0, 200) + '...',
        source: 'Pastebin',
        confidence: 0.85,
        riskLevel: 'high',
        content: paste.content
      }));
    } catch (error) {
      this.logger.warn(`Pastebin search failed: ${error.message}`);
      return [];
    }
  }

  private delay(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  async healthCheck(): Promise<boolean> {
    return this.torService.isWorking();
  }

  private parseDuckDuckGoResults(html: string, config: any): any[] {
  const $ = cheerio.load(html);
  const results: any[] = [];
  
  $('.web-result, .result').each((i, el) => {
    const title = $(el).find('.result__title a, h2 a').text().trim();
    const url = $(el).find('.result__title a, h2 a').attr('href') || '';
    const description = $(el).find('.result__snippet, .result__body').text().trim();
    
    if (url && title) {
      results.push({
        url: url.startsWith('http') ? url : `https://duckduckgo.com${url}`,
        title,
        description,
        source: 'DuckDuckGo',
        confidence: 0.7
      });
    }
  });
  
  return results;
}

private parseSearxResults(html: string, config: any): any[] {
  const $ = cheerio.load(html);
  const results: any[] = [];
  
  $('.result').each((i, el) => {
    const title = $(el).find('h3 a, .result-title a').text().trim();
    const url = $(el).find('h3 a, .result-title a').attr('href') || '';
    const description = $(el).find('.content, .result-content').text().trim();
    
    if (url && title) {
      results.push({
        url,
        title,
        description,
        source: 'Searx',
        confidence: 0.65
      });
    }
  });
  
  return results;
}

private parseStartpageResults(html: string, config: any): any[] {
  const $ = cheerio.load(html);
  const results: any[] = [];
  
  $('.w-gl__result').each((i, el) => {
    const title = $(el).find('.w-gl__result-title').text().trim();
    const url = $(el).find('.w-gl__result-url').text().trim();
    const description = $(el).find('.w-gl__description').text().trim();
    
    if (url && title) {
      results.push({
        url,
        title,
        description,
        source: 'Startpage',
        confidence: 0.6
      });
    }
  });
  
  return results;
}
}