Clean content from any webpage

import { Stack0 } from '@stack0/sdk'
 
const stack0 = new Stack0({ apiKey: process.env.STACK0_API_KEY })
 
// Extract clean content from webpage
export async function extractContent(url: string) {
  const { extractedData } = await stack0.extraction.extractAndWait({
    url,
    mode: 'content',
    options: {
      format: 'markdown',
      includeImages: true,
      includeLinks: true,
      removeNavigation: true,
      removeAds: true,
    },
  })
 
  return {
    title: extractedData.title,
    content: extractedData.markdown,
    wordCount: extractedData.wordCount,
    images: extractedData.images,
  }
}
 
// Batch extract for knowledge base
export async function buildKnowledgeBase(urls: string[]) {
  const { results } = await stack0.extraction.batchAndWait({
    urls,
    config: {
      mode: 'content',
      options: { format: 'markdown' },
    },
  })
 
  // Chunk for vector database
  return results.flatMap(r =>
    chunkMarkdown(r.extractedData.markdown, 500)
      .map(chunk => ({
        source: r.url,
        content: chunk,
      }))
  )
}