Summarizing RSS Articles

2025-06-0411 turns38,467 charsgpt-4-1-mini, gpt-4o⑂ 1 fork(s)

rss-automation llm-summarization topic-clustering

Summary

The user is building an automated RSS pipeline that fetches articles, clusters them by topic, and generates summaries using Mistral LLMs with TTS output.

Messages

⑂ thread forked here

# === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"  # Change voice if desired
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm  # Progress bar

# Load feed URLs from YAML configuration
def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

# Fetch and parse articles from RSS feeds
def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A')
            })
    return articles

# Use Ollama to summarize text
def summarize_with_ollama(text, model=SUMMARY_MODEL):
    prompt = (
        "Summarize the following news article in 3-5 sentences, focusing on the key facts, context, "
        "and implications. Avoid speculation and opinion.\n\n"
        f"{text}\n\nSummary:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Extract and summarize article content using Ollama
def summarize_articles(articles, model=SUMMARY_MODEL):
    summaries = []
    print("\nSummarizing articles...\n")
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_ollama(text, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    joined_summaries = "\n\n".join(
        f"Title: {s['title']}\nSummary: {s['summary']}" for s in summaries
    )
    prompt = (
        "You are a professional news anchor. Create a coherent news broadcast script based on the following article summaries. "
        "Weave them together into a flowing narrative, grouping related topics and keeping it informative and neutral:\n\n"
        f"{joined_summaries}\n\nBroadcast:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Save full broadcast with timestamped filename
def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename  # return path for TTS to use

# Convert broadcast to speech with timestamped filename
async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

# Main workflow
def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)

    # Save digest and get timestamped filename
    digest_path = save_digest(broadcast)

    # Create matching timestamped mp3 path
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main(). -------- I am thinking about implementing a way so that each of the calls to the LLM for summarization of each article would take into consideration meta data in the construction of each prompt using f strings to populate each prompt with the contextual awareness of the news sources which would include things like the commonality of coverage of a topic or story and thus the stories reported by more sources would have preference and you could use values on a scale which are quantified in order to have the numbers adjusted dynamically

You're thinking in exactly the right direction: by enriching the prompts with metadata (like story frequency, source reliability, or topic clustering), you can guide the LLM to produce **more contextually aware** and **prioritized** summaries. Here's how you can approach this step by step:

---

## 🧠

gpt-4o

This is a fantastic idea — adding dynamic contextual metadata about each article’s coverage frequency and source characteristics into the prompt for summarization would definitely improve the quality and objectivity of summaries, making them more aware of the overall news landscape.

---

## How you could approach this enhancement:

### 1. Quantify and Collect Metadata

- **Count how many sources cover each story or topic:**  
  For example, if the same story or key event appears in multiple feeds, that story’s importance/weight should be higher.

- **Assign weights or scores to sources** based on their reliability, bias, regional focus, or editorial stance — these can be static or dynamically adjusted.

- **Track common topics or keywords** across articles to identify recurring themes.

### 2. Structure your metadata for each article

You could build a dictionary for each article like:

```python
{
  "title": "...",
  "link": "...",
  "published": "...",
  "coverage_score": 3,  # e.g., how many sources mention it
  "source_weight": 0.8,  # e.g., a reliability/bias score
  "topic_relevance": 0.9,  # how central the article is to a trending topic
}
```

### 3. Modify the summarization prompt to include this metadata

You can format the prompt like:

```python
prompt = f"""
Summarize the following news article in 3-5 sentences, focusing on the key facts, context, and implications.
This article has a coverage score of {coverage_score} (number of sources reporting on it) and a source reliability score of {source_weight}.
Consider that stories covered by more sources should be given preference and be more detailed.
Avoid speculation and opinion.

Article text:
{text}

Summary:
"""
```

### 4. How to detect story commonality / coverage?

- **Naive approach:** Match article titles or main keywords across feeds.  
- **Better:** Use semantic similarity (e.g., embeddings) on article summaries or titles to cluster stories.  
- **Simpler alternative:** You could hash normalized titles or key sentences and count duplicates.

### 5. Implementation Outline

```python
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

def cluster_similar_articles(articles, threshold=0.7):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    similarity_matrix = cosine_similarity(vectorizer)
    
    coverage_scores = defaultdict(int)
    
    # For each article, count how many others are similar above threshold
    for i in range(len(articles)):
        count = sum(similarity_matrix[i] > threshold)
        coverage_scores[i] = count
    
    return coverage_scores
```

Then when summarizing:

```python
coverage_scores = cluster_similar_articles(articles)

for idx, article in enumerate(articles):
    coverage_score = coverage_scores[idx]
    prompt = f"""
    Summarize the following news article in 3-5 sentences, focusing on the key facts, context, and implications.
    This article is reported by approximately {coverage_score} sources.
    Please emphasize stories reported by multiple sources.
    
    Article text:
    {article_text}
    
    Summary:
    """
    # pass prompt to Ollama
```

---

## Summary

- **Add a coverage metric** by detecting story commonality across feeds  
- **Add a source metadata weight** for bias or reliability (you could manually define this in your `feeds.yaml`)  
- **Dynamically generate prompts embedding this metadata** to provide the LLM more context  
- This encourages the LLM to prioritize stories with broader coverage and reflect news importance more accurately

---

If you want, I can help you integrate this into your existing codebase or create a sample implementation for you! Would you like that?

gpt-4-1-mini

modify this to do so: # === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"  # Change voice if desired
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm  # Progress bar

# Load feed URLs from YAML configuration
def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

# Fetch and parse articles from RSS feeds
def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A')
            })
    return articles

# Use Ollama to summarize text
def summarize_with_ollama(text, model=SUMMARY_MODEL):
    prompt = (
        "Summarize the following news article in 3-5 sentences, focusing on the key facts, context, "
        "and implications. Avoid speculation and opinion.\n\n"
        f"{text}\n\nSummary:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Extract and summarize article content using Ollama
def summarize_articles(articles, model=SUMMARY_MODEL):
    summaries = []
    print("\nSummarizing articles...\n")
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_ollama(text, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    joined_summaries = "\n\n".join(
        f"Title: {s['title']}\nSummary: {s['summary']}" for s in summaries
    )
    prompt = (
        "You are a professional news anchor. Create a coherent news broadcast script based on the following article summaries. "
        "Weave them together into a flowing narrative, grouping related topics and keeping it informative and neutral:\n\n"
        f"{joined_summaries}\n\nBroadcast:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Save full broadcast with timestamped filename
def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename  # return path for TTS to use

# Convert broadcast to speech with timestamped filename
async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

# Main workflow
def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)

    # Save digest and get timestamped filename
    digest_path = save_digest(broadcast)

    # Create matching timestamped mp3 path
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main()

Here’s a modified version of your script that adds **contextual metadata** to each article before summarization, including:

- `coverage_score`: how many other articles are similar (based on TF-IDF similarity).
- `source_name`: extracted from the feed URL.
- This metadata is then **injected into the summarization prompt**.

---

### ✅ Changes made:
- Added metadata extraction to count similar articles.
- Injected metadata into the prompt.
- Refactored summarization to support contextual prompts.

Here’s the complete, modified script:

```python
# === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"  # Change voice if desired
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm
from urllib.parse import urlparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Load feed URLs from YAML configuration
def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

# Fetch and parse articles from RSS feeds
def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        source_name = urlparse(url).netloc.replace('www.', '')
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A'),
                'source': source_name
            })
    return articles

# Assign a coverage score based on title similarity
def compute_coverage_scores(articles, threshold=0.7):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    similarity_matrix = cosine_similarity(vectorizer)
    
    for i in range(len(articles)):
        score = sum(similarity_matrix[i] > threshold) - 1  # exclude self
        articles[i]['coverage_score'] = score
    return articles

# Use Ollama to summarize text with contextual metadata
def summarize_with_context(text, metadata, model=SUMMARY_MODEL):
    prompt = f"""
You are summarizing a news article from the source: {metadata['source']}.
This article is reported by approximately {metadata['coverage_score']} other sources.

Summarize the article in 3–5 sentences, focusing on the key facts, context, and implications.
Avoid speculation or opinion.

Article content:
{text}

Summary:
"""
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Extract and summarize article content using Ollama
def summarize_articles(articles, model=SUMMARY_MODEL):
    print("\nSummarizing articles...\n")
    summaries = []
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_context(text, article, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    joined_summaries = "\n\n".join(
        f"Title: {s['title']}\nSummary: {s['summary']}" for s in summaries
    )
    prompt = (
        "You are a professional news anchor. Create a coherent news broadcast script based on the following article summaries. "
        "Weave them together into a flowing narrative, grouping related topics and keeping it informative and neutral:\n\n"
        f"{joined_summaries}\n\nBroadcast:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Save full broadcast with timestamped filename
def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename

# Convert broadcast to speech with timestamped filename
async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

# Main workflow
def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    articles = compute_coverage_scores(articles)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)
    digest_path = save_digest(broadcast)
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main()
```

---

## ✅ Benefits:
- Summaries now **weigh article significance** based on broader coverage.
- Prompts **embed source metadata** to allow model grounding.
- Modular structure makes it easy to extend with bias metrics or clustering by topic.

Would you like me to also integrate **source credibility or bias scoring** next?

gpt-4o

I want the final prompt to take these things into consideration as well

Great — to extend **contextual awareness into the final broadcast prompt**, we’ll inject **metadata such as**:

- `source` of each article  
- `coverage_score` (e.g., "widely reported" vs "less reported")  
- Optionally: `publication date`

This gives the broadcast LLM richer material to group stories by **topic importance, coverage volume**, or **source credibility**.

---

### ✅ Updated Sections:
Here are the **two key modified sections**:

---

### 1. 🧠 **`summarize_articles()`** (already modified above)

This part already adds:
- `source`  
- `coverage_score`

We’ll **retain** this structure.

---

### 2. 🎙️ **`generate_broadcast()`** — Modify to use metadata

```python
# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    enriched_summaries = "\n\n".join(
        f"Title: {s['title']}\n"
        f"Source: {s.get('source', 'unknown')} | "
        f"Coverage Score: {s.get('coverage_score', 0)} | "
        f"Published: {s.get('published', 'N/A')}\n"
        f"Summary: {s['summary']}"
        for s in summaries
    )

    prompt = (
        "You are a professional news anchor. Use the metadata below to determine importance and flow of the stories:\n"
        "- Stories with higher coverage scores are more widely reported and should be prioritized.\n"
        "- Consider source reputation and group related topics.\n"
        "- Keep tone factual, concise, and smoothly flowing.\n\n"
        "Here are the article summaries and metadata:\n\n"
        f"{enriched_summaries}\n\n"
        "Now write the final broadcast script:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']
```

---

### 🧩 Final Notes:

✅ This method:
- Promotes high-coverage stories to lead the broadcast  
- Helps LLM logically group content  
- Adds source transparency

You could later:
- Assign weights based on domain reputation
- Add `bias rating` if you’re scoring that

Would you like to also:
- Log these enriched summaries to a CSV or JSON file?
- Add tags or clusters based on topic similarity?

Let me know and I’ll help you automate that part too.

gpt-4o

rewrite to do so: # === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"  # Change voice if desired
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm
from urllib.parse import urlparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Load feed URLs from YAML configuration
def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

# Fetch and parse articles from RSS feeds
def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        source_name = urlparse(url).netloc.replace('www.', '')
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A'),
                'source': source_name
            })
    return articles

# Assign a coverage score based on title similarity
def compute_coverage_scores(articles, threshold=0.7):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    similarity_matrix = cosine_similarity(vectorizer)
    
    for i in range(len(articles)):
        score = sum(similarity_matrix[i] > threshold) - 1  # exclude self
        articles[i]['coverage_score'] = score
    return articles

# Use Ollama to summarize text with contextual metadata
def summarize_with_context(text, metadata, model=SUMMARY_MODEL):
    prompt = f"""
You are summarizing a news article from the source: {metadata['source']}.
This article is reported by approximately {metadata['coverage_score']} other sources.

Summarize the article in 3–5 sentences, focusing on the key facts, context, and implications.
Avoid speculation or opinion.

Article content:
{text}

Summary:
"""
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Extract and summarize article content using Ollama
def summarize_articles(articles, model=SUMMARY_MODEL):
    print("\nSummarizing articles...\n")
    summaries = []
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_context(text, article, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    joined_summaries = "\n\n".join(
        f"Title: {s['title']}\nSummary: {s['summary']}" for s in summaries
    )
    prompt = (
        "You are a professional news anchor. Create a coherent news broadcast script based on the following article summaries. "
        "Weave them together into a flowing narrative, grouping related topics and keeping it informative and neutral:\n\n"
        f"{joined_summaries}\n\nBroadcast:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Save full broadcast with timestamped filename
def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename

# Convert broadcast to speech with timestamped filename
async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

# Main workflow
def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    articles = compute_coverage_scores(articles)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)
    digest_path = save_digest(broadcast)
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main()

Here’s the full and properly rewritten version of your script, integrating source and coverage metadata into the final `generate_broadcast()` prompt as requested:

---

```python
# === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"  # Change voice if desired
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm
from urllib.parse import urlparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Load feed URLs from YAML configuration
def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

# Fetch and parse articles from RSS feeds
def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        source_name = urlparse(url).netloc.replace('www.', '')
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A'),
                'source': source_name
            })
    return articles

# Assign a coverage score based on title similarity
def compute_coverage_scores(articles, threshold=0.7):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    similarity_matrix = cosine_similarity(vectorizer)

    for i in range(len(articles)):
        score = sum(similarity_matrix[i] > threshold) - 1  # exclude self
        articles[i]['coverage_score'] = score
    return articles

# Use Ollama to summarize text with contextual metadata
def summarize_with_context(text, metadata, model=SUMMARY_MODEL):
    prompt = f"""
You are summarizing a news article from the source: {metadata['source']}.
This article is reported by approximately {metadata['coverage_score']} other sources.

Summarize the article in 3–5 sentences, focusing on the key facts, context, and implications.
Avoid speculation or opinion.

Article content:
{text}

Summary:
"""
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Extract and summarize article content using Ollama
def summarize_articles(articles, model=SUMMARY_MODEL):
    print("\nSummarizing articles...\n")
    summaries = []
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_context(text, article, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'source': article['source'],
                'coverage_score': article['coverage_score'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

# Use Ollama to generate a cohesive news broadcast from all summaries
def generate_broadcast(summaries, model=BROADCAST_MODEL):
    enriched_summaries = "\n\n".join(
        f"Title: {s['title']}\n"
        f"Source: {s['source']} | Coverage Score: {s['coverage_score']} | Published: {s['published']}\n"
        f"Summary: {s['summary']}"
        for s in summaries
    )

    prompt = (
        "You are a professional news anchor. Use the metadata below to determine the importance and order of the stories:\n"
        "- Prioritize stories with higher coverage scores (more widely reported).\n"
        "- Mention the source when relevant.\n"
        "- Group related topics and transitions naturally between them.\n"
        "- Maintain a clear, informative, and neutral tone.\n\n"
        "Article summaries with metadata:\n\n"
        f"{enriched_summaries}\n\n"
        "Now write the full broadcast script:"
    )
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

# Save full broadcast with timestamped filename
def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename

# Convert broadcast to speech with timestamped filename
async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

# Main workflow
def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    articles = compute_coverage_scores(articles)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)
    digest_path = save_digest(broadcast)
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main()
```

---

### ✅ Key Enhancements:

- Each article summary now carries:
  - `source`
  - `coverage_score`
  - `published date`
- `generate_broadcast()` uses these for flow, prioritization, and tone.
- Prompt gives clear instructions to the LLM on how to use this metadata.

Would you like to also:
- Cache articles locally?
- Cluster by topic using LLM or TF-IDF before summarization?

Let me know if you want a version that supports longer history or daily digests too.

gpt-4o

cluster by topic

To **cluster articles by topic** before generating the broadcast, we can use a TF-IDF vectorizer + KMeans clustering to group similar articles. Then, we structure the final prompt so articles are **grouped and narrated together**.

Here’s the full updated script with **topic clustering**, along with integration into the `generate_broadcast()` prompt:

---

### ✅ Changes:
- Add `cluster_by_topic()` using KMeans
- Group summaries by cluster
- Update `generate_broadcast()` to narrate by topic clusters

---

```python
# === CONFIGURABLE MODELS ===
SUMMARY_MODEL = 'mistral:latest'
BROADCAST_MODEL = 'mistral-small:24b-instruct-2501-q8_0'
TTS_VOICE = "en-US-GuyNeural"
NUM_TOPICS = 3  # Number of clusters
# ===========================

import os
import yaml
import feedparser
from newspaper import Article
from datetime import datetime
import asyncio
import edge_tts
import ollama
from tqdm import tqdm
from urllib.parse import urlparse
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans

def load_feeds(config_path='feeds.yaml'):
    with open(config_path, 'r') as file:
        config = yaml.safe_load(file)
    return config.get('feeds', [])

def fetch_articles(feed_urls, max_articles=1):
    articles = []
    print("Fetching and parsing RSS feeds...\n")
    for url in tqdm(feed_urls, desc="Scraping feeds", unit="feed"):
        feed = feedparser.parse(url)
        source_name = urlparse(url).netloc.replace('www.', '')
        for entry in feed.entries[:max_articles]:
            articles.append({
                'title': entry.title,
                'link': entry.link,
                'published': entry.get('published', 'N/A'),
                'source': source_name
            })
    return articles

def compute_coverage_scores(articles, threshold=0.7):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    similarity_matrix = cosine_similarity(vectorizer)
    for i in range(len(articles)):
        score = sum(similarity_matrix[i] > threshold) - 1
        articles[i]['coverage_score'] = score
    return articles

def cluster_by_topic(articles, num_topics=NUM_TOPICS):
    titles = [a['title'] for a in articles]
    vectorizer = TfidfVectorizer(stop_words='english').fit_transform(titles)
    km = KMeans(n_clusters=min(num_topics, len(articles)), random_state=42)
    labels = km.fit_predict(vectorizer)
    for i, label in enumerate(labels):
        articles[i]['topic_cluster'] = label
    return articles

def summarize_with_context(text, metadata, model=SUMMARY_MODEL):
    prompt = f"""
You are summarizing a news article from the source: {metadata['source']}.
This article is reported by approximately {metadata['coverage_score']} other sources.

Summarize the article in 3–5 sentences, focusing on the key facts, context, and implications.
Avoid speculation or opinion.

Article content:
{text}

Summary:
"""
    response = ollama.chat(model=model, messages=[{"role": "user", "content": prompt}])
    return response['message']['content']

def summarize_articles(articles, model=SUMMARY_MODEL):
    print("\nSummarizing articles...\n")
    summaries = []
    for article in tqdm(articles, desc="Summarizing", unit="article"):
        try:
            news_article = Article(article['link'])
            news_article.download()
            news_article.parse()
            text = news_article.text[:2000]
            summary = summarize_with_context(text, article, model=model)
            summaries.append({
                'title': article['title'],
                'link': article['link'],
                'published': article['published'],
                'source': article['source'],
                'coverage_score': article['coverage_score'],
                'topic_cluster': article['topic_cluster'],
                'summary': summary
            })
        except Exception as e:
            print(f"Error processing article: {article['link']}\n{e}")
    return summaries

def generate_broadcast(summaries, model=BROADCAST_MODEL):
    # Group summaries by cluster
    cluster_groups = {}
    for s in summaries:
        cluster_groups.setdefault(s['topic_cluster'], []).append(s)

    prompt_parts = []
    for cluster_id, cluster in sorted(cluster_groups.items()):
        topic_heading = f"Topic Cluster {cluster_id + 1}:"
        cluster_text = "\n\n".join(
            f"Title: {s['title']}\n"
            f"Source: {s['source']} | Coverage Score: {s['coverage_score']} | Published: {s['published']}\n"
            f"Summary: {s['summary']}"
            for s in cluster
        )
        prompt_parts.append(f"{topic_heading}\n{cluster_text}")

    final_prompt = (
        "You are a professional news anchor. Create a coherent broadcast script from the following topic-based summaries.\n"
        "For each topic cluster, introduce the theme briefly, then cover each article.\n"
        "Prioritize higher coverage scores and maintain a neutral, informative tone.\n\n"
        f"{'\n\n'.join(prompt_parts)}\n\nBroadcast script:"
    )

    response = ollama.chat(model=model, messages=[{"role": "user", "content": final_prompt}])
    return response['message']['content']

def save_digest(digest_text, output_dir='.'):
    timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    filename = os.path.join(output_dir, f'digest_{timestamp}.md')
    with open(filename, 'w') as file:
        file.write(digest_text)
    return filename

async def text_to_speech(text, output_path, voice=TTS_VOICE):
    communicate = edge_tts.Communicate(text, voice=voice)
    await communicate.save(output_path)

def main():
    feed_urls = load_feeds()
    articles = fetch_articles(feed_urls, max_articles=1)
    articles = compute_coverage_scores(articles)
    articles = cluster_by_topic(articles)
    summaries = summarize_articles(articles)
    broadcast = generate_broadcast(summaries)

    digest_path = save_digest(broadcast)
    mp3_path = digest_path.replace('.md', '.mp3')
    asyncio.run(text_to_speech(broadcast, output_path=mp3_path))

if __name__ == "__main__":
    main()
```

---

### 🧠 Example Summary Prompt Structure for LLM:
```plaintext
Topic Cluster 1:
Title: Inflation ticks up
Source: reuters.com | Coverage Score: 4 | Published: June 1, 2025
Summary: ...

Topic Cluster 2:
...
```

Let me know if you'd like to **name topics automatically** (e.g. "Politics", "Tech") using keyword extraction or classification — it’s very doable.

gpt-4o