repo_description: A Powerful web scraper powered by LLM | OpenAI, Gemini & Ollama
repo_topics: ai-scraping gemini-api llm llm-scraper openai scraper web-scraper webscraping
code: import asyncio
  import streamlit as st
  from src.web_extractor import WebExtractor
  from src.scrapers.playwright_scraper import ScraperConfig
  import os
  
  class StreamlitWebScraperChat:
      def __init__(self, model_name, scraper_config: ScraperConfig = None):
          self.web_extractor = WebExtractor(model_name=model_name, scraper_config=scraper_config)
  
      def process_message(self, message: str, conversation_history: list[dict] | None = None) -> str:
          async def process_with_progress():
              progress_placeholder = st.empty()
              progress_placeholder.text("Processing...")
              result = await self.web_extractor.process_query(
                  message,
                  conversation_history=conversation_history,
                  progress_callback=progress_placeholder.text
              )
              progress_placeholder.empty()
              return result
  
          return asyncio.run(process_with_progress())