From d48fa20b7dad294ef26aff215bdbcb79e3cfa1ad Mon Sep 17 00:00:00 2001 From: Abdeladim Fadheli Date: Mon, 13 Jul 2026 07:35:52 +0000 Subject: [PATCH 1/3] Add RankBits AI visibility tracker tutorial code --- .../ai_visibility_tracker.py | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 general/rankbits-ai-visibility/ai_visibility_tracker.py diff --git a/general/rankbits-ai-visibility/ai_visibility_tracker.py b/general/rankbits-ai-visibility/ai_visibility_tracker.py new file mode 100644 index 00000000..e3cf32c4 --- /dev/null +++ b/general/rankbits-ai-visibility/ai_visibility_tracker.py @@ -0,0 +1,344 @@ +""" +Track Your AI Visibility with Python & RankBits API. + +This script demonstrates the full workflow: +1. Check your RankBits account and plan +2. Create an AI visibility scan for any domain +3. Poll until the scan completes +4. Parse the results and generate visualizations + +Requirements: + pip install requests matplotlib + +Usage: + export RANKBITS_TOKEN="rb_your_token_here" + python ai_visibility_tracker.py +""" + +import os +import sys +import time +import json +from datetime import datetime + +import requests +import matplotlib.pyplot as plt +import matplotlib.ticker as mticker + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +TOKEN = os.environ.get("RANKBITS_TOKEN", "rb_your_token_here") +BASE_URL = "https://rankbits.com/v1" +HEADERS = { + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", +} + +# The domain you want to scan +TARGET_URL = "https://thepythoncode.com" + +# Free engines to use (omit "paid" providers like openai_pro, claude_pro, gemini_pro) +ENGINES = ["openai", "gemini", "perplexity", "claude", "google_ai_mode"] + +# Number of AI-generated prompts (plan caps apply) +PROMPT_COUNT = 5 + + +# --------------------------------------------------------------------------- +# Helper: pretty-print JSON +# --------------------------------------------------------------------------- + +def print_json(obj: dict, title: str = "") -> None: + """Print a dictionary as formatted JSON.""" + if title: + print(f"\n{'=' * 60}\n{title}\n{'=' * 60}") + print(json.dumps(obj, indent=2, default=str)) + + +# --------------------------------------------------------------------------- +# Step 1 – Check your account +# --------------------------------------------------------------------------- + +def check_account() -> dict: + """Fetch plan info and credit usage from /v1/me.""" + resp = requests.get(f"{BASE_URL}/me", headers=HEADERS) + resp.raise_for_status() + data = resp.json() + plan = data["plan"] + resp_info = plan["responses"] + + print("šŸ”‘ Account") + print(f" Plan: {plan['label']} (${plan['price_usd']}/mo)") + print(f" Monthly: {resp_info['used']}/{resp_info['monthly_limit']} responses") + print(f" Credits: {resp_info['purchased_remaining']} purchased remaining") + print(f" Engines: {len(plan['allowed_provider_keys'])} available") + return data + + +# --------------------------------------------------------------------------- +# Step 2 – Create a scan +# --------------------------------------------------------------------------- + +def create_scan( + url: str, + prompt_count: int = 5, + providers: list[str] | None = None, +) -> dict: + """Submit an async scan and return the public ID.""" + payload: dict = {"url": url, "prompt_count": prompt_count} + if providers: + payload["providers"] = providers + + resp = requests.post(f"{BASE_URL}/scans", headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + + scan = data["scan"] + print(f"\nšŸš€ Scan created") + print(f" ID: {scan['public_id']}") + print(f" Domain: {scan['domain']}") + print(f" Status: {scan['status']}") + print(f" View live: https://rankbits.com{data['links']['app']}") + return data + + +# --------------------------------------------------------------------------- +# Step 3 – Poll until done +# --------------------------------------------------------------------------- + +def poll_scan(public_id: str, poll_seconds: float = 3.0, max_wait: float = 300.0) -> dict: + """Poll /v1/scans/{id} until status is 'done' or timeout.""" + url = f"{BASE_URL}/scans/{public_id}" + start = time.time() + last_completed = 0 + + print(f"\nā³ Polling scan {public_id} ...") + while True: + elapsed = time.time() - start + if elapsed > max_wait: + raise TimeoutError(f"Scan did not complete within {max_wait}s") + + resp = requests.get(url, headers=HEADERS) + resp.raise_for_status() + data = resp.json() + + status = data["scan"]["status"] + progress = data.get("progress", {}) + completed = progress.get("completed_results", 0) + expected = progress.get("expected_results", 0) + + # Print progress when it changes + if completed != last_completed: + pct = (completed / expected * 100) if expected else 0 + print(f" [{status}] {completed}/{expected} ({pct:.0f}%)") + last_completed = completed + + if status == "done": + print(" āœ… Scan complete!") + return data + if status in ("error", "failed"): + raise RuntimeError(f"Scan failed: {data}") + + time.sleep(poll_seconds) + + +# --------------------------------------------------------------------------- +# Step 4 – Parse & display results +# --------------------------------------------------------------------------- + +def summarize_results(data: dict) -> None: + """Print a human-readable summary of scan results.""" + aggregate = data.get("aggregate", {}) + overall = aggregate.get("overall", {}) + providers = aggregate.get("providers", {}) + results = data.get("results", []) + prompts = data.get("prompts", []) + + # ---- 4a. Overview ---- + print(f"\nšŸ“Š Visibility Summary for {data['scan']['domain']}") + print(f" Overall score: {overall.get('score', 'N/A')}") + print(f" Mention rate: {overall.get('mention_rate', 0):.1f}%") + print(f" Citation rate: {overall.get('citation_rate', 0):.1f}%") + print(f" Total results: {len(results)} rows") + + # ---- 4b. Per-engine breakdown ---- + print(f"\nšŸ¤– Engine Breakdown") + print(f" {'Engine':<20s} {'Score':>7s} {'Mention%':>9s} {'Citation%':>10s}") + print(f" {'-'*46}") + for key, pdata in sorted(providers.items(), key=lambda x: -x[1].get("score", 0)): + print( + f" {key:<20s} {pdata.get('score', 0):>7.1f} " + f"{pdata.get('mention_rate', 0):>8.1f}% {pdata.get('citation_rate', 0):>9.1f}%" + ) + + # ---- 4c. Prompts used ---- + print(f"\nšŸ’¬ Prompts ({len(prompts)})") + for p in prompts: + print(f" • {p['text']}") + + # ---- 4d. Share of voice (top 5) ---- + sov = aggregate.get("share_of_voice", []) + if sov: + print(f"\nšŸ”— Top Cited Domains (Share of Voice)") + for entry in sov[:5]: + print(f" {entry['domain']:40s} {entry.get('citation_count', 0)} citations") + + # ---- 4e. Where we were found ---- + found = [r for r in results if r.get("brand_mentioned") or r.get("brand_cited")] + if found: + print(f"\nāœ… Where {data['scan']['domain']} Appeared ({len(found)}/{len(results)})") + for r in found: + mentioned = "āœ…" if r["brand_mentioned"] else "āŒ" + cited = "āœ…" if r["brand_cited"] else "āŒ" + print(f" [{r['provider']:20s}] Mentioned: {mentioned} Cited: {cited}") + print(f" Prompt: {r['prompt'][:100]}") + else: + print(f"\nāš ļø {data['scan']['domain']} was NOT mentioned or cited in any result!") + print(" Time to improve your AI visibility! → https://rankbits.com") + + +# --------------------------------------------------------------------------- +# Step 5 – Generate charts +# --------------------------------------------------------------------------- + +def generate_charts(data: dict, output_dir: str = ".") -> None: + """Create matplotlib charts from scan results.""" + aggregate = data.get("aggregate", {}) + providers = aggregate.get("providers", {}) + domain = data["scan"]["domain"] + + if not providers: + print("āš ļø No provider data to chart.") + return + + # Sort engines by score descending + engines = sorted(providers.items(), key=lambda x: -x[1].get("score", 0)) + names = [e[0].replace("_", " ").title() for e in engines] + scores = [e[1].get("score", 0) for e in engines] + mention_rates = [e[1].get("mention_rate", 0) for e in engines] + citation_rates = [e[1].get("citation_rate", 0) for e in engines] + + # Colors + bar_color = "#7c3aed" + mention_color = "#10b981" + citation_color = "#f59e0b" + + # ---- Chart 1: Scores by engine ---- + fig1, ax1 = plt.subplots(figsize=(8, 5)) + bars = ax1.barh(names, scores, color=bar_color, edgecolor="white", linewidth=0.5, height=0.5) + ax1.set_xlabel("Visibility Score (0–100)", fontsize=11) + ax1.set_title(f"AI Visibility Score by Engine — {domain}", fontsize=13, fontweight="bold") + ax1.invert_yaxis() + ax1.xaxis.set_major_formatter(mticker.FormatStrFormatter("%.0f")) + for bar, val in zip(bars, scores): + ax1.text(bar.get_width() + 0.5, bar.get_y() + bar.get_height() / 2, + f"{val:.1f}", va="center", fontsize=10, fontweight="semibold") + ax1.set_xlim(0, max(scores) * 1.3 + 5 if max(scores) > 0 else 30) + plt.tight_layout() + fig1.savefig(f"{output_dir}/engine_scores.png", dpi=150) + print(f"\nšŸ“ˆ Chart saved: {output_dir}/engine_scores.png") + + # ---- Chart 2: Mention vs Citation rates ---- + fig2, ax2 = plt.subplots(figsize=(8, 5)) + x = range(len(names)) + width = 0.35 + ax2.bar([i - width / 2 for i in x], mention_rates, width, label="Mention Rate %", + color=mention_color, edgecolor="white", linewidth=0.5) + ax2.bar([i + width / 2 for i in x], citation_rates, width, label="Citation Rate %", + color=citation_color, edgecolor="white", linewidth=0.5) + ax2.set_xticks(x) + ax2.set_xticklabels(names, fontsize=9) + ax2.set_ylabel("Percentage (%)", fontsize=11) + ax2.set_title(f"Mention vs Citation Rate — {domain}", fontsize=13, fontweight="bold") + ax2.legend(fontsize=10, loc="upper right") + ax2.set_ylim(0, max(max(mention_rates), max(citation_rates)) * 1.4 + 5) + plt.tight_layout() + fig2.savefig(f"{output_dir}/mention_vs_citation.png", dpi=150) + print(f"šŸ“ˆ Chart saved: {output_dir}/mention_vs_citation.png") + + # ---- Chart 3: Results grid (heatmap-style table) ---- + results = data.get("results", []) + if results: + # Build a matrix: rows=prompts, cols=engines + prompt_texts = sorted({r["prompt"][:60] for r in results}) + engine_names = sorted({r["provider"] for r in results}) + + matrix = [] + for pt in prompt_texts: + row = [] + for eng in engine_names: + match = [r for r in results if r["prompt"].startswith(pt[:30]) and r["provider"] == eng] + if match: + m = match[0] + if m["brand_cited"]: + row.append(2) # cited (best) + elif m["brand_mentioned"]: + row.append(1) # mentioned + else: + row.append(0) # absent + else: + row.append(0) + matrix.append(row) + + fig3, ax3 = plt.subplots(figsize=(max(8, len(engine_names) * 1.2), + max(5, len(prompt_texts) * 0.6))) + cmap = plt.cm.RdYlGn + im = ax3.imshow(matrix, cmap=cmap, aspect="auto", vmin=0, vmax=2) + + ax3.set_xticks(range(len(engine_names))) + ax3.set_xticklabels([e.replace("_", " ").title() for e in engine_names], + rotation=30, ha="right", fontsize=9) + ax3.set_yticks(range(len(prompt_texts))) + ax3.set_yticklabels(prompt_texts, fontsize=8) + + # Add text in each cell + for i in range(len(prompt_texts)): + for j in range(len(engine_names)): + val = matrix[i][j] + symbol = {0: "ā—‹", 1: "ā–²", 2: "ā˜…"}[val] + ax3.text(j, i, symbol, ha="center", va="center", + fontsize=14, color="black" if val == 2 else "white") + + ax3.set_title(f"Presence Grid — {domain}\nā—‹ Absent ā–² Mentioned ā˜… Cited", + fontsize=12, fontweight="bold") + plt.tight_layout() + fig3.savefig(f"{output_dir}/presence_grid.png", dpi=150) + print(f"šŸ“ˆ Chart saved: {output_dir}/presence_grid.png") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> None: + if TOKEN == "rb_your_token_here": + print("āŒ Set your RANKBITS_TOKEN environment variable first.") + print(" Get one at: https://rankbits.com/signup") + sys.exit(1) + + print(f"šŸŽÆ Tracking AI visibility for: {TARGET_URL}") + print(f" Engines: {', '.join(ENGINES)}") + + # 1. Check account + check_account() + + # 2. Start scan + scan_data = create_scan(TARGET_URL, prompt_count=PROMPT_COUNT, providers=ENGINES) + public_id = scan_data["scan"]["public_id"] + + # 3. Poll until complete + results = poll_scan(public_id) + + # 4. Summarize + summarize_results(results) + + # 5. Charts + generate_charts(results) + + print("\n✨ Done! Track ongoing visibility at https://rankbits.com") + + +if __name__ == "__main__": + main() From 2a1c47a4c13758f06f1861336e0a4ca2a434f870 Mon Sep 17 00:00:00 2001 From: Rockikz Date: Wed, 30 Sep 2026 12:24:22 +0100 Subject: [PATCH 2/3] Fix youtube-extractor for 2026 YouTube page structure - Add Accept-Language header so YouTube returns English output regardless of visitor locale - Fix likes extraction (compositeVideoPrimaryInfoRenderer -> segmentedLikeDislikeButtonViewModel) - Fix tags: keywords come from ytInitialPlayerResponse.videoDetails, not ytInitialData.metadata - Fix duration via videoDetails.lengthSeconds (regex kept as fallback) - Fix UnicodeEncodeError crash on Windows consoles when printing descriptions --- .../youtube-extractor/extract_video_info.py | 96 ++++++++++++------- 1 file changed, 60 insertions(+), 36 deletions(-) diff --git a/web-scraping/youtube-extractor/extract_video_info.py b/web-scraping/youtube-extractor/extract_video_info.py index bed184b0..29ecf426 100644 --- a/web-scraping/youtube-extractor/extract_video_info.py +++ b/web-scraping/youtube-extractor/extract_video_info.py @@ -1,15 +1,36 @@ +import sys import requests from bs4 import BeautifulSoup import re import json import argparse +def _find_key(obj, key): + """Recursively search a nested dict/list structure for the first occurrence of `key`.""" + if isinstance(obj, dict): + if key in obj: + return obj[key] + for value in obj.values(): + found = _find_key(value, key) + if found is not None: + return found + elif isinstance(obj, list): + for item in obj: + found = _find_key(item, key) + if found is not None: + return found + return None + + def get_video_info(url): """ Extract video information from YouTube using modern approach """ headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + # Without this, YouTube serves the page in the visitor's locale (e.g. Arabic), + # so views/dates/subscribers come back in the wrong language + 'Accept-Language': 'en-US,en;q=0.9', } try: @@ -17,19 +38,21 @@ def get_video_info(url): response = requests.get(url, headers=headers) response.raise_for_status() - # Create beautiful soup object to parse HTML - soup = BeautifulSoup(response.text, "html.parser") - # Initialize the result result = {} - # Extract ytInitialData which contains all the video information + # Extract ytInitialData which contains the video/watch page information data_match = re.search(r'var ytInitialData = ({.*?});', response.text) if not data_match: raise Exception("Could not find ytInitialData in page") data_json = json.loads(data_match.group(1)) + # Extract ytInitialPlayerResponse (video metadata: keywords, duration, view count) + player_match = re.search(r'var ytInitialPlayerResponse = ({.*?});', response.text) + player_json = json.loads(player_match.group(1)) if player_match else {} + video_details = player_json.get('videoDetails', {}) + # Get the main content sections contents = data_json['contents']['twoColumnWatchNextResults']['results']['results']['contents'] @@ -77,43 +100,40 @@ def get_video_info(url): else: result["description"] = "Description not available" - # Try to extract video duration from player overlay - # This is a fallback approach since the original method doesn't work - duration_match = re.search(r'"approxDurationMs":"(\d+)"', response.text) - if duration_match: - duration_ms = int(duration_match.group(1)) - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 + # Extract video duration from the player response (accurate, e.g. "19" seconds), + # with a fallback regex on the page HTML + if 'lengthSeconds' in video_details: + duration_s = int(video_details['lengthSeconds']) + minutes = duration_s // 60 + seconds = duration_s % 60 result["duration"] = f"{minutes}:{seconds:02d}" else: - result["duration"] = "Duration not available" + duration_match = re.search(r'"approxDurationMs":"(\d+)"', response.text) + if duration_match: + duration_ms = int(duration_match.group(1)) + minutes = duration_ms // 60000 + seconds = (duration_ms % 60000) // 1000 + result["duration"] = f"{minutes}:{seconds:02d}" + else: + result["duration"] = "Duration not available" - # Extract video tags if available - video_tags = [] - if 'keywords' in data_json.get('metadata', {}).get('videoMetadataRenderer', {}): - video_tags = data_json['metadata']['videoMetadataRenderer']['keywords'] + # Extract video tags (they live in ytInitialPlayerResponse.videoDetails.keywords, + # NOT in ytInitialData.metadata like the old code assumed) + video_tags = video_details.get('keywords', []) result["tags"] = ', '.join(video_tags) if video_tags else "No tags available" - # Extract likes (modern approach) + # Extract likes (2026 structure): + # videoActions.menuRenderer.topLevelButtons[0].segmentedLikeDislikeButtonViewModel + # .likeButtonViewModel.likeButtonViewModel.toggleButtonViewModel + # .toggleButtonViewModel.defaultButtonViewModel.buttonViewModel.title result["likes"] = "Likes count not available" - result["dislikes"] = "UNKNOWN" # YouTube no longer shows dislikes + if 'videoPrimaryInfoRenderer' in contents[0]: + button = _find_key(contents[0]['videoPrimaryInfoRenderer'], 'buttonViewModel') + if button and button.get('iconName') == 'LIKE' and 'title' in button: + result["likes"] = button['title'].replace('\xa0', ' ') - # Try to find likes in the new structure - for content in contents: - if 'compositeVideoPrimaryInfoRenderer' in content: - composite = content['compositeVideoPrimaryInfoRenderer'] - if 'likeButton' in composite: - like_button = composite['likeButton'] - if 'toggleButtonRenderer' in like_button: - toggle = like_button['toggleButtonRenderer'] - if 'defaultText' in toggle: - default_text = toggle['defaultText'] - if 'accessibility' in default_text: - accessibility = default_text['accessibility'] - if 'accessibilityData' in accessibility: - label = accessibility['accessibilityData']['label'] - if 'like' in label.lower(): - result["likes"] = label + # Dislikes are not published by YouTube anymore + result["dislikes"] = "UNKNOWN" return result @@ -121,6 +141,10 @@ def get_video_info(url): raise Exception(f"Error extracting video info: {str(e)}") if __name__ == "__main__": + # Avoid UnicodeEncodeError crashes on Windows consoles (cp1252) + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding='utf-8', errors='replace') + parser = argparse.ArgumentParser(description="YouTube Video Data Extractor") parser.add_argument("url", help="URL of the YouTube video") @@ -147,4 +171,4 @@ def get_video_info(url): except Exception as e: print(f"Error: {e}") - print("\nNote: YouTube frequently changes its structure, so this script may need updates.") \ No newline at end of file + print("\nNote: YouTube frequently changes its structure, so this script may need updates.") From bfba0e959fb63a9a543e1e2af85b8f6b208ec111 Mon Sep 17 00:00:00 2001 From: Rockikz Date: Wed, 30 Sep 2026 12:41:03 +0100 Subject: [PATCH 3/3] Harden YouTube extractor metadata parsing --- .../youtube-extractor/extract_video_info.py | 154 ++++++++++-------- .../youtube-extractor/requirements.txt | 3 +- .../test_extract_video_info.py | 100 ++++++++++++ 3 files changed, 191 insertions(+), 66 deletions(-) create mode 100644 web-scraping/youtube-extractor/test_extract_video_info.py diff --git a/web-scraping/youtube-extractor/extract_video_info.py b/web-scraping/youtube-extractor/extract_video_info.py index 29ecf426..c58f3df0 100644 --- a/web-scraping/youtube-extractor/extract_video_info.py +++ b/web-scraping/youtube-extractor/extract_video_info.py @@ -1,27 +1,35 @@ import sys import requests -from bs4 import BeautifulSoup import re import json import argparse -def _find_key(obj, key): - """Recursively search a nested dict/list structure for the first occurrence of `key`.""" +def _find_button_by_icon(obj, icon_name): + """Find the first nested button view model with the requested icon.""" if isinstance(obj, dict): - if key in obj: - return obj[key] + if obj.get('iconName') == icon_name and 'title' in obj: + return obj for value in obj.values(): - found = _find_key(value, key) - if found is not None: - return found + button = _find_button_by_icon(value, icon_name) + if button is not None: + return button elif isinstance(obj, list): for item in obj: - found = _find_key(item, key) - if found is not None: - return found + button = _find_button_by_icon(item, icon_name) + if button is not None: + return button return None +def _format_duration(total_seconds): + """Format a duration as M:SS or H:MM:SS.""" + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + if hours: + return f"{hours}:{minutes:02d}:{seconds:02d}" + return f"{total_seconds // 60}:{seconds:02d}" + + def get_video_info(url): """ Extract video information from YouTube using modern approach @@ -35,7 +43,7 @@ def get_video_info(url): try: # Download HTML code - response = requests.get(url, headers=headers) + response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() # Initialize the result @@ -50,48 +58,63 @@ def get_video_info(url): # Extract ytInitialPlayerResponse (video metadata: keywords, duration, view count) player_match = re.search(r'var ytInitialPlayerResponse = ({.*?});', response.text) - player_json = json.loads(player_match.group(1)) if player_match else {} + player_json = {} + if player_match: + try: + player_json = json.loads(player_match.group(1)) + except json.JSONDecodeError: + pass + if not isinstance(player_json, dict): + player_json = {} video_details = player_json.get('videoDetails', {}) + if not isinstance(video_details, dict): + video_details = {} - # Get the main content sections - contents = data_json['contents']['twoColumnWatchNextResults']['results']['results']['contents'] - - # Extract video information from videoPrimaryInfoRenderer - if 'videoPrimaryInfoRenderer' in contents[0]: - primary = contents[0]['videoPrimaryInfoRenderer'] - - # Video title - result["title"] = primary['title']['runs'][0]['text'] - - # Video views - result["views"] = primary['viewCount']['videoViewCountRenderer']['viewCount']['simpleText'] - - # Date published - result["date_published"] = primary['dateText']['simpleText'] + # Get the main content sections, and locate renderers by type rather than position. + try: + contents = data_json['contents']['twoColumnWatchNextResults']['results']['results']['contents'] + except (KeyError, TypeError): + raise ValueError("Could not find video details; the video may be unavailable") + if not isinstance(contents, list): + raise ValueError("Could not find video details; the video may be unavailable") + + primary = next( + (item['videoPrimaryInfoRenderer'] for item in contents + if isinstance(item, dict) and 'videoPrimaryInfoRenderer' in item), + None, + ) + if primary is None: + raise ValueError("Could not find video details; the video may be unavailable") + + # Video title, views, and publication date + result["title"] = primary['title']['runs'][0]['text'] + result["views"] = primary['viewCount']['videoViewCountRenderer']['viewCount']['simpleText'] + result["date_published"] = primary['dateText']['simpleText'] - # Extract channel information from videoSecondaryInfoRenderer - secondary = None - if 'videoSecondaryInfoRenderer' in contents[1]: - secondary = contents[1]['videoSecondaryInfoRenderer'] + # Extract channel information from videoSecondaryInfoRenderer, if present. + secondary = next( + (item['videoSecondaryInfoRenderer'] for item in contents + if isinstance(item, dict) and 'videoSecondaryInfoRenderer' in item), + None, + ) + if secondary: owner = secondary['owner']['videoOwnerRenderer'] - - # Channel name channel_name = owner['title']['runs'][0]['text'] - - # Channel ID channel_id = owner['navigationEndpoint']['browseEndpoint']['browseId'] - - # Channel URL - FIXED with proper /channel/ path channel_url = f"https://www.youtube.com/channel/{channel_id}" - - # Number of subscribers channel_subscribers = owner['subscriberCountText']['accessibility']['accessibilityData']['label'] - + result['channel'] = { - 'name': channel_name, - 'url': channel_url, + 'name': channel_name, + 'url': channel_url, 'subscribers': channel_subscribers } + else: + result['channel'] = { + 'name': 'Channel not available', + 'url': 'Channel not available', + 'subscribers': 'Channel subscribers not available' + } # Extract video description if secondary and 'attributedDescription' in secondary: @@ -100,36 +123,39 @@ def get_video_info(url): else: result["description"] = "Description not available" - # Extract video duration from the player response (accurate, e.g. "19" seconds), - # with a fallback regex on the page HTML - if 'lengthSeconds' in video_details: - duration_s = int(video_details['lengthSeconds']) - minutes = duration_s // 60 - seconds = duration_s % 60 - result["duration"] = f"{minutes}:{seconds:02d}" + # Live streams report elapsed time in lengthSeconds, not media duration. + if video_details.get('isLiveContent'): + result["duration"] = "LIVE" else: - duration_match = re.search(r'"approxDurationMs":"(\d+)"', response.text) - if duration_match: - duration_ms = int(duration_match.group(1)) - minutes = duration_ms // 60000 - seconds = (duration_ms % 60000) // 1000 - result["duration"] = f"{minutes}:{seconds:02d}" + try: + duration_s = int(video_details['lengthSeconds']) + except (KeyError, TypeError, ValueError): + duration_s = None + + if duration_s is not None: + result["duration"] = _format_duration(duration_s) else: - result["duration"] = "Duration not available" + duration_match = re.search(r'"approxDurationMs":"(\d+)"', response.text) + if duration_match: + duration_ms = int(duration_match.group(1)) + result["duration"] = _format_duration(duration_ms // 1000) + else: + result["duration"] = "Duration not available" # Extract video tags (they live in ytInitialPlayerResponse.videoDetails.keywords, # NOT in ytInitialData.metadata like the old code assumed) video_tags = video_details.get('keywords', []) result["tags"] = ', '.join(video_tags) if video_tags else "No tags available" - # Extract likes (2026 structure): - # videoActions.menuRenderer.topLevelButtons[0].segmentedLikeDislikeButtonViewModel - # .likeButtonViewModel.likeButtonViewModel.toggleButtonViewModel - # .toggleButtonViewModel.defaultButtonViewModel.buttonViewModel.title + # Extract the exact count from the LIKE button's accessibility label when available. result["likes"] = "Likes count not available" - if 'videoPrimaryInfoRenderer' in contents[0]: - button = _find_key(contents[0]['videoPrimaryInfoRenderer'], 'buttonViewModel') - if button and button.get('iconName') == 'LIKE' and 'title' in button: + button = _find_button_by_icon(primary, 'LIKE') + if button: + accessibility_text = button.get('accessibilityText', '') + count_match = re.search(r'([\d,]+)\s+other people', accessibility_text, re.IGNORECASE) + if count_match: + result["likes"] = count_match.group(1).replace(',', '') + else: result["likes"] = button['title'].replace('\xa0', ' ') # Dislikes are not published by YouTube anymore diff --git a/web-scraping/youtube-extractor/requirements.txt b/web-scraping/youtube-extractor/requirements.txt index 73a4de92..f2293605 100644 --- a/web-scraping/youtube-extractor/requirements.txt +++ b/web-scraping/youtube-extractor/requirements.txt @@ -1,2 +1 @@ -requests_html -bs4 \ No newline at end of file +requests diff --git a/web-scraping/youtube-extractor/test_extract_video_info.py b/web-scraping/youtube-extractor/test_extract_video_info.py new file mode 100644 index 00000000..4b895f05 --- /dev/null +++ b/web-scraping/youtube-extractor/test_extract_video_info.py @@ -0,0 +1,100 @@ +import json +import unittest +from unittest.mock import Mock, patch + +from extract_video_info import get_video_info + + +def make_html(player_response, contents=None): + primary = { + "title": {"runs": [{"text": "Test video"}]}, + "viewCount": {"videoViewCountRenderer": {"viewCount": {"simpleText": "1,234 views"}}}, + "dateText": {"simpleText": "Jan 1, 2026"}, + # An unrelated button appears first; the extractor must find the LIKE button. + "topLevelButtons": [ + {"buttonViewModel": {"iconName": "SHARE", "title": "Share"}}, + { + "buttonViewModel": { + "iconName": "LIKE", + "title": "19M", + "accessibilityText": "like this video along with 19,962,782 other people", + } + }, + ], + } + secondary = { + "owner": { + "videoOwnerRenderer": { + "title": {"runs": [{"text": "Test channel"}]}, + "navigationEndpoint": {"browseEndpoint": {"browseId": "UC-test"}}, + "subscriberCountText": {"accessibility": {"accessibilityData": {"label": "10 subscribers"}}}, + } + }, + "attributedDescription": {"content": "A test description"}, + } + if contents is None: + contents = [ + {"unrelatedRenderer": {}}, + {"videoPrimaryInfoRenderer": primary}, + {"videoSecondaryInfoRenderer": secondary}, + ] + data = { + "contents": { + "twoColumnWatchNextResults": { + "results": {"results": {"contents": contents}} + } + } + } + player = json.dumps(player_response) if isinstance(player_response, dict) else player_response + return f"var ytInitialData = {json.dumps(data)}; var ytInitialPlayerResponse = {player};" + + +class ExtractVideoInfoTests(unittest.TestCase): + def extract(self, html): + response = Mock() + response.text = html + response.raise_for_status.return_value = None + with patch("extract_video_info.requests.get", return_value=response) as get: + result = get_video_info("https://www.youtube.com/watch?v=test") + get.assert_called_once() + self.assertEqual(get.call_args.kwargs["timeout"], 15) + return result + + def test_exact_likes_and_hour_duration(self): + result = self.extract(make_html({ + "videoDetails": { + "lengthSeconds": "7322", + "keywords": ["python", "test"], + } + })) + + self.assertEqual(result["likes"], "19962782") + self.assertEqual(result["duration"], "2:02:02") + self.assertEqual(result["tags"], "python, test") + self.assertEqual(result["channel"]["name"], "Test channel") + + def test_live_stream_is_not_reported_as_elapsed_duration(self): + result = self.extract(make_html({ + "videoDetails": { + "lengthSeconds": "121601512", + "isLiveContent": True, + } + })) + + self.assertEqual(result["duration"], "LIVE") + + def test_malformed_optional_player_response_keeps_page_metadata(self): + result = self.extract(make_html("{invalid json}")) + + self.assertEqual(result["title"], "Test video") + self.assertEqual(result["likes"], "19962782") + self.assertEqual(result["duration"], "Duration not available") + self.assertEqual(result["tags"], "No tags available") + + def test_missing_watch_page_structure_fails_with_clear_error(self): + with self.assertRaisesRegex(Exception, "video may be unavailable"): + self.extract(make_html({}, contents=[])) + + +if __name__ == "__main__": + unittest.main()