export interface ScrapedData {
  title: string
  metaDescription: string
  content: string
  links: string[]
  images: string[]
}

/**
 * Scrape a URL using the Jina AI scraper service
 * @param url URL to scrape
 * @returns ScrapedData object with extracted content
 */
export async function scrapeUrl(url: string): Promise<ScrapedData> {
  try {
    // Use Jina AI scraper service
    const jinaUrl = `https://r.jina.ai/${encodeURIComponent(url)}`

    // Fetch from Jina service
    const response = await fetch(jinaUrl)

    if (!response.ok) {
      if (response.status === 429) {
        throw new Error(
          "Rate limited by Jina service (429 Too Many Requests). Try again later."
        )
      }
      if (response.status === 403) {
        throw new Error("Access forbidden by Jina service (403 Forbidden).")
      }
      if (response.status === 404) {
        throw new Error(
          "Page not found (404 Not Found). Check if the URL is correct."
        )
      }
      throw new Error(
        `Failed to fetch URL via Jina: ${response.status} ${response.statusText}`
      )
    }

    // Jina returns HTML content that we need to extract data from
    const html = await response.text()

    // For now, we're creating a simplified response
    // In a production environment, you might want to implement more sophisticated parsing
    // or check if Jina provides an API that returns structured data directly
    return {
      title: extractTitle(html),
      metaDescription: extractMetaDescription(html),
      content: extractMainContent(html),
      links: extractLinks(html, url),
      images: extractImages(html, url),
    }
  } catch (error) {
    console.error("Error scraping URL:", error)
    throw new Error(
      `Failed to scrape website: ${
        error instanceof Error ? error.message : "Unknown error"
      }`
    )
  }
}

/**
 * Extract title from HTML
 */
function extractTitle(html: string): string {
  const titleMatch = html.match(/<title[^>]*>(.*?)<\/title>/i)
  return titleMatch ? titleMatch[1].trim() : ""
}

/**
 * Extract meta description from HTML
 */
function extractMetaDescription(html: string): string {
  const metaMatch = html.match(
    /<meta\s+name="description"\s+content="([^"]*)"[^>]*>/i
  )
  return metaMatch ? metaMatch[1].trim() : ""
}

/**
 * Extract main content from HTML (simplified)
 */
function extractMainContent(html: string): string {
  // Simple content extraction - remove HTML tags and trim
  const content = html
    .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
    .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
    .replace(/<[^>]+>/g, " ")
    .replace(/\s+/g, " ")
    .trim()

  return content.substring(0, 5000) // Limit content length
}

/**
 * Extract links from HTML
 */
function extractLinks(html: string, baseUrl: string): string[] {
  const links: string[] = []
  const linkRegex = /<a\s+(?:[^>]*?\s+)?href="([^"]*)"[^>]*>/gi
  let match

  while ((match = linkRegex.exec(html)) !== null) {
    const href = match[1]
    if (href && !href.startsWith("#") && !href.startsWith("javascript:")) {
      try {
        const absoluteUrl = new URL(href, baseUrl).toString()
        links.push(absoluteUrl)
      } catch {
        // Skip invalid URLs
      }
    }
  }

  return [...new Set(links)].slice(0, 20) // Remove duplicates and limit to 20
}

/**
 * Extract images from HTML
 */
function extractImages(html: string, baseUrl: string): string[] {
  const images: string[] = []
  const imgRegex = /<img\s+(?:[^>]*?\s+)?src="([^"]*)"[^>]*>/gi
  let match

  while ((match = imgRegex.exec(html)) !== null) {
    const src = match[1]
    if (src) {
      try {
        const absoluteSrc = new URL(src, baseUrl).toString()
        images.push(absoluteSrc)
      } catch {
        // Skip invalid URLs
      }
    }
  }

  return [...new Set(images)].slice(0, 10) // Remove duplicates and limit to 10
}
jina-scraper
