import { Injectable, Logger } from '@nestjs/common';
import axios from 'axios';
import * as cheerio from 'cheerio';
import { URL } from 'url';
import { cyberKeywords } from './keywords/crawler.keywords';

@Injectable()
export class CrawlerService {
  private readonly logger = new Logger(CrawlerService.name);
  private visitedUrls: Set<string> = new Set();
  private validLinks: Set<string> = new Set();
  private baseUrl: string;
  private baseHostname: string;

  // Configurable patterns to exclude
  private excludePatterns: RegExp[] = [
    /\/docs\//i,
    /\/blog\//i,
    /\/guides\//i,
    /\/guide\//i,
    /\/api\//i,
    /\.pdf$/i,
    /\.zip$/i,
    /\.jpg$/i,
    /\.png$/i,
    /\.gif$/i,
    /\.css$/i,
    /\.js$/i,
    /\/logout\//i,
    /\/terms-of-service\//i,
    /\/privacy-policy\//i,
    /\/feed\//i,
    /\/rss\//i,
    /\/tag\//i,
    /\/category\//i,
    /\/author\//i,
    /\/page\//i,
    /\/static\//i,
    /\/assets\//i,
  ];

  /**
   * Start crawling a website and find all valid links within the same domain
   * @param startUrl The URL to start crawling from
   * @param maxDepth Maximum depth to crawl (default: 3)
   * @param maxUrls Maximum number of URLs to process (default: 500)
   * @returns An array of valid URLs discovered
   */
  async crawlWebsite(startUrl: string, maxDepth: number = 2, maxUrls: number = 20): Promise<string[]> {
    this.clearCrawlData();
    try {
      let normalizedUrl = startUrl;
      if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) {
        normalizedUrl = `https://${normalizedUrl}`;
        this.logger.log(`Added https:// protocol to URL: ${normalizedUrl}`);
      }

      // Validate and normalize the start URL
      const url = new URL(normalizedUrl);
      this.baseUrl = `${url.protocol}//${url.hostname}`;
      this.baseHostname = url.hostname;

      this.logger.log(`Starting crawl from ${normalizedUrl}`);
      this.logger.log(`Base URL: ${this.baseUrl}`);
      this.logger.log(`Max URLs to crawl: ${maxUrls}`);

      // Begin the recursive crawl
      await this.crawl(normalizedUrl, 0, maxDepth, maxUrls);

      // Convert set to array for return
      const results = Array.from(this.validLinks);

      this.logger.log(`Crawl completed. Found ${results.length} valid links. Visited ${this.visitedUrls.size} URLs.`);
      return results;
    } catch (error) {
      this.logger.error(`Error starting crawl: ${error.message}`);

      // Provide more helpful error message
      if (error.code === 'ERR_INVALID_URL') {
        throw new Error(`Invalid URL format: ${startUrl}. Please ensure it's a valid URL (e.g., example.com or https://example.com)`);
      }

      throw error;
    }
  }

  /**
   * Recursive function to crawl pages
   */
  private async crawl(url: string, currentDepth: number, maxDepth: number, maxUrls: number): Promise<void> {
    // Check if we've reached the maximum number of URLs before processing this URL
    if (this.visitedUrls.size >= maxUrls) {
      return;
    }

    // Stop conditions
    if (currentDepth > maxDepth || this.visitedUrls.has(url)) {
      return;
    }

    // Mark this URL as visited BEFORE processing it
    this.visitedUrls.add(url);
    this.logger.debug(`Crawling: ${url} (Depth: ${currentDepth}) [${this.visitedUrls.size}/${maxUrls}]`);

    try {
      // Make the HTTP request with a timeout
      const response = await axios.get(url, {
        timeout: 10000,
        headers: {
          'User-Agent': 'NestJS-Crawler/1.0',
        },
        validateStatus: status => status < 400, // Only accept 2xx and 3xx status codes
      });

      // Only process HTML responses
      const contentType = response.headers['content-type'];
      if (!contentType || !contentType.includes('text/html')) {
        return;
      }

      // Use cheerio to parse the HTML
      const $ = cheerio.load(response.data);

      // Extract all links
      const links = new Set<string>();
      $('a').each((_, element) => {
        const href = $(element).attr('href');
        if (href) {
          try {
            // Skip href values that are clearly not URLs
            if (href.startsWith('#') || href.startsWith('javascript:') || href.startsWith('mailto:') || href.startsWith('tel:')) {
              return;
            }

            // Normalize the URL
            const resolvedUrl = new URL(href, url).href;
            links.add(resolvedUrl);
          } catch (error) {
            // Invalid URL, skip it
            this.logger.debug(`Invalid href: ${href}`);
          }
        }
      });

      // Process each link
      for (const link of links) {
        // Skip if we've already visited this URL or reached maxUrls
        if (this.visitedUrls.has(link) || this.visitedUrls.size >= maxUrls) {
          continue;
        }

        try {
          const linkUrl = new URL(link);

          // Check if the link is within the same hostname
          if (linkUrl.hostname === this.baseHostname) {
            // Check if excluded by pattern
            const shouldExclude = this.excludePatterns.some(pattern => pattern.test(linkUrl.pathname));

            if (shouldExclude) continue;

            // // Check if link is cyber-related using keywords
            // const isCyberRelated = cyberKeywords.some(keyword => linkUrl.href.toLowerCase().includes(keyword));

            // if (isCyberRelated) {
            this.validLinks.add(link);

            if (this.visitedUrls.size < maxUrls) {
              await this.crawl(link, currentDepth + 1, maxDepth, maxUrls);
            }
            // }
          }
        } catch (error) {
          // Invalid URL, skip it
          this.logger.debug(`Invalid URL: ${link}`);
        }
      }
    } catch (error) {
      // Handle different types of errors
      if (error.code === 'ECONNREFUSED') {
        this.logger.warn(`Connection refused for ${url}`);
      } else if (error.code === 'ENOTFOUND') {
        this.logger.warn(`Domain not found for ${url}`);
      } else if (error.response && error.response.status) {
        this.logger.warn(`Error status ${error.response.status} for ${url}`);
      } else {
        this.logger.warn(`Error crawling ${url}: ${error.message}`);
      }
    }
  }

  /**
   * Clear previous crawl data
   */
  clearCrawlData(): void {
    this.visitedUrls.clear();
    this.validLinks.clear();
    this.logger.log('Crawl data cleared');
  }

  /**
   * Update exclude patterns for customized filtering
   */
  updateExcludePatterns(patterns: RegExp[]): void {
    this.excludePatterns = patterns;
    this.logger.log('Exclude patterns updated');
  }
}