Building A YouTube Comment Sentiment Analyzer App Using Scrapingdog & Lovable

Copy
1Title: YouTube Comment Sentiment (1‑Paragraph Summary)
2 
3Goal:
4Build a minimal Next.js app that:
5 
6Accepts a YouTube video ID (not the full URL) and two API keys entered on the frontend (Scrapingdog + OpenRouter).
7 
8Fetches all comments via Scrapingdog (10 per page with next_page_token).
9 
10Uses OpenRouter → OpenAI gpt‑4o / 4o‑mini to produce exactly one paragraph (≤120 words) summarizing overall sentiment.
11 
12No DB, no Supabase, no persistence; do not log keys.
13 
14Key UX:
15 
16Inputs: videoId, scrapingdogApiKey, openrouterApiKey, model (default openai/gpt-4o-mini, option openai/gpt-4o).
17 
18Button: Analyze Sentiment.
19 
20Show simple "Analyzing…" progress text while running.
21 
22Output: one paragraph only (no bullets, no tables, no per‑comment output).
23 
24Implementation:
25 
26Use Next.js App Router.
27 
28Frontend sends keys to backend per request; backend uses them only in memory.
29 
30Backend performs hierarchical summarization (chunk → mini summaries → final synthesis).
31 
32Basic backoff/retry, safe chunk size (~4000 chars).
33 
34Validate video ID (not URL).
35 
36Clamp final text to a single paragraph (≤120 words).
37 
38Create these files
391) app/page.tsx
40tsx
41Copy
42Edit
43"use client";
44import { useState } from "react";
45 
46const YT_ID_RE = /^[A-Za-z0-9_-]{11}$/; // Standard YouTube ID length is 11
47 
48export default function Home() {
49  const [videoId, setVideoId] = useState("");
50  const [scrapingdogApiKey, setScrapingdogApiKey] = useState("");
51  const [openrouterApiKey, setOpenrouterApiKey] = useState("");
52  const [model, setModel] = useState("openai/gpt-4o-mini");
53  const [loading, setLoading] = useState(false);
54  const [summary, setSummary] = useState<string>("");
55  const [error, setError] = useState<string>("");
56 
57  async function handleRun() {
58    setError("");
59    setSummary("");
60    if (!YT_ID_RE.test(videoId)) {
61      setError("Enter a valid YouTube video ID (11 characters, not a full URL).");
62      return;
63    }
64    if (!scrapingdogApiKey || !openrouterApiKey) {
65      setError("Both API keys are required.");
66      return;
67    }
68    setLoading(true);
69    try {
70      const res = await fetch("/api/run", {
71        method: "POST",
72        headers: { "Content-Type": "application/json" },
73        body: JSON.stringify({ videoId, scrapingdogApiKey, openrouterApiKey, model }),
74      });
75      const data = await res.json();
76      if (!res.ok) throw new Error(data?.error || "Failed");
77      setSummary(data.summaryParagraph || "");
78    } catch (e: any) {
79      setError(e?.message || "Something went wrong");
80    } finally {
81      setLoading(false);
82    }
83  }
84 
85  return (
86    <main className="max-w-xl mx-auto p-6 space-y-4">
87      <h1 className="text-2xl font-semibold">YouTube Comment Sentiment</h1>
88      <p className="text-sm text-gray-600">
89        Enter the <b>video ID</b> (not the full URL), add your API keys, and get a single-paragraph sentiment summary.
90      </p>
91
92      <label className="block">
93        <span className="text-sm">YouTube Video ID</span>
94        <input
95          className="w-full border rounded p-2"
96          placeholder="e.g. dQw4w9WgXcQ"
97          value={videoId}
98          onChange={(e) => setVideoId(e.target.value.trim())}
99        />
100      </label>
101
102      <label className="block">
103        <span className="text-sm">Scrapingdog API Key</span>
104        <input
105          className="w-full border rounded p-2"
106          placeholder="sk_live_..."
107          value={scrapingdogApiKey}
108          onChange={(e) => setScrapingdogApiKey(e.target.value)}
109          type="password"
110        />
111      </label>
112
113      <label className="block">
114        <span className="text-sm">OpenRouter API Key</span>
115        <input
116          className="w-full border rounded p-2"
117          placeholder="sk-or-v1-..."
118          value={openrouterApiKey}
119          onChange={(e) => setOpenrouterApiKey(e.target.value)}
120          type="password"
121        />
122      </label>
123
124      <label className="block">
125        <span className="text-sm">Model</span>
126        <select
127          className="w-full border rounded p-2"
128          value={model}
129          onChange={(e) => setModel(e.target.value)}
130        >
131          <option value="openai/gpt-4o-mini">openai/gpt-4o-mini (cheaper)</option>
132          <option value="openai/gpt-4o">openai/gpt-4o</option>
133        </select>
134      </label>
135
136      <button
137        onClick={handleRun}
138        disabled={loading}
139        className="w-full rounded bg-black text-white py-2 disabled:opacity-60"
140      >
141        {loading ? "Analyzing…" : "Analyze Sentiment"}
142      </button>
143
144      {error && <p className="text-red-600 text-sm">{error}</p>}
145      {summary && (
146        <div className="border rounded p-3 bg-gray-50">
147          <p className="text-sm leading-6">{summary}</p>
148        </div>
149      )}
150    </main>
151  );
152}
1532) app/api/run/route.ts
154ts
155Copy
156Edit
157import { NextRequest, NextResponse } from "next/server";
158 
159export const dynamic = "force-dynamic";
160 
161type Comment = { id: string; text: string; author?: string; published_at?: string };
162 
163const OPENROUTER_BASE = "https://openrouter.ai/api/v1";
164const DEFAULT_MODEL = "openai/gpt-4o-mini"; // switch to "openai/gpt-4o" via UI if desired
165const YT_ID_RE = /^[A-Za-z0-9_-]{11}$/;     // require ID not URL
166 
167// 1) Fetch ALL comments via Scrapingdog (10 per page) using next_page_token
168async function fetchAllComments(
169  videoId: string,
170  scrapingdogApiKey: string
171): Promise<Comment[]> {
172  const base = "https://api.scrapingdog.com/youtube/comments";
173  let token: string | undefined = undefined;
174  const out: Comment[] = [];
175 
176  // Page through until next token is exhausted
177  while (true) {
178    const params = new URLSearchParams({ api_key: scrapingdogApiKey, video_id: videoId });
179    if (token) params.set("next_page_token", token);
180    const url = `${base}?${params.toString()}`;
181 
182    const res = await fetch(url, { method: "GET", cache: "no-store" });
183    if (!res.ok) {
184      const txt = await res.text().catch(() => "");
185      throw new Error(`Scrapingdog ${res.status}: ${txt || res.statusText}`);
186    }
187    const data = await res.json();
188 
189    const items: any[] = data?.comments || [];
190    for (const c of items) {
191      const id = c?.comment_id ?? c?.id;
192      const text = c?.text_original ?? c?.text ?? "";
193      if (!id || !text) continue;
194      out.push({
195        id: String(id),
196        text: String(text),
197        author: c?.author_display_name,
198        published_at: c?.published_at,
199      });
200    }
201 
202    token = data?.scrapingdog_pagination?.next ?? data?.next_page_token ?? undefined;
203 
204    // gentle pacing
205    await new Promise((r) => setTimeout(r, 120));
206 
207    if (!token) break;
208  }
209 
210  // Deduplicate by comment_id
211  return Array.from(new Map(out.map((c) => [c.id, c])).values());
212}
213 
214// 2) OpenRouter helper
215async function callOpenRouter({
216  apiKey,
217  model,
218  system,
219  user,
220  temperature = 0,
221}: {
222  apiKey: string;
223  model: string;
224  system: string;
225  user: string;
226  temperature?: number;
227}): Promise<string> {
228  const res = await fetch(`${OPENROUTER_BASE}/chat/completions`, {
229    method: "POST",
230    headers: {
231      Authorization: `Bearer ${apiKey}`,
232      "Content-Type": "application/json",
233      "HTTP-Referer": "https://yourapp.local", // attribution (any string)
234      "X-Title": "YouTube Sentiment",
235    },
236    body: JSON.stringify({
237      model,
238      temperature,
239      messages: [
240        { role: "system", content: system },
241        { role: "user", content: user },
242      ],
243    }),
244  });
245 
246  if (!res.ok) {
247    const txt = await res.text().catch(() => "");
248    throw new Error(`OpenRouter ${res.status}: ${txt || res.statusText}`);
249  }
250  const json = await res.json();
251  return json?.choices?.[0]?.message?.content?.trim() || "";
252}
253 
254// 3) Chunk comments and summarize (hierarchical)
255function chunkCommentsByChars(comments: Comment[], maxChars = 4000): string[] {
256  const chunks: string[] = [];
257  let buf = "";
258 
259  for (const c of comments) {
260    const line = (c.text || "").replace(/\s+/g, " ").trim();
261    if (!line) continue;
262    if ((buf.length + line.length + 4) > maxChars) {
263      if (buf) chunks.push(buf);
264      buf = line;
265    } else {
266      buf = buf ? `${buf}\n---\n${line}` : line;
267    }
268  }
269  if (buf) chunks.push(buf);
270  return chunks;
271}
272 
273const CHUNK_SYSTEM = `You are a precise sentiment summarizer. Output plain text only. No lists, no headings, no JSON.`;
274const CHUNK_USER_TEMPLATE = (commentsPlain: string) => `
275You will see a batch of YouTube comments (raw text).
276Write a concise 2–3 sentence summary of the sentiment and main themes in neutral, professional tone.
277Handle slang, emojis, sarcasm, and mixed opinions.
278Output plain text only. No bullets, no labels, no JSON.
279
280Comments:
281${commentsPlain}
282`.trim();
283 
284const FINAL_SYSTEM = `You are a precise sentiment summarizer. Output exactly one paragraph (<=120 words). No lists, no headings, no JSON.`;
285const FINAL_USER_TEMPLATE = (miniSummaries: string[]) => `
286You will see multiple short summaries, each representing a subset of YouTube comments from the same video.
287Synthesize them into EXACTLY ONE PARAGRAPH (<=120 words) that states the overall sentiment (positive, neutral, or negative), briefly explains why, and notes any prominent themes or controversies.
288Consider sarcasm, emojis, and mixed opinions.
289Output one paragraph only. No bullets, no titles, no JSON.
290
291Batch Summaries:
292${miniSummaries.join("\n")}
293`.trim();
294 
295export async function POST(req: NextRequest) {
296  try {
297    const { videoId, scrapingdogApiKey, openrouterApiKey, model } = await req.json();
298 
299    // Validate inputs
300    if (!videoId || !YT_ID_RE.test(videoId)) {
301      return NextResponse.json({ error: "Please provide a valid YouTube video ID (11 chars, not a full URL)." }, { status: 400 });
302    }
303    if (!scrapingdogApiKey) {
304      return NextResponse.json({ error: "scrapingdogApiKey is required." }, { status: 400 });
305    }
306    if (!openrouterApiKey) {
307      return NextResponse.json({ error: "openrouterApiKey is required." }, { status: 400 });
308    }
309 
310    const modelToUse = (model && typeof model === "string") ? model : DEFAULT_MODEL;
311 
312    // 1) Collect all comments
313    const comments = await fetchAllComments(videoId, scrapingdogApiKey);
314    if (!comments.length) {
315      return NextResponse.json({
316        summaryParagraph: "No comments were found for this video, so there is no sentiment to summarize."
317      });
318    }
319 
320    // 2) Chunk -> mini summaries
321    const chunks = chunkCommentsByChars(comments, 4000);
322    const miniSummaries: string[] = [];
323    for (const chunk of chunks) {
324      try {
325        const mini = await callOpenRouter({
326          apiKey: openrouterApiKey,
327          model: modelToUse,
328          system: CHUNK_SYSTEM,
329          user: CHUNK_USER_TEMPLATE(chunk),
330          temperature: 0,
331        });
332        if (mini) miniSummaries.push(mini);
333        await new Promise((r) => setTimeout(r, 200)); // gentle pacing
334      } catch {
335        // one retry
336        const mini = await callOpenRouter({
337          apiKey: openrouterApiKey,
338          model: modelToUse,
339          system: CHUNK_SYSTEM,
340          user: CHUNK_USER_TEMPLATE(chunk),
341          temperature: 0,
342        });
343        if (mini) miniSummaries.push(mini);
344      }
345    }
346 
347    // 3) Final synthesis -> EXACTLY one paragraph (<=120 words)
348    const finalParagraph = await callOpenRouter({
349      apiKey: openrouterApiKey,
350      model: modelToUse,
351      system: FINAL_SYSTEM,
352      user: FINAL_USER_TEMPLATE(miniSummaries),
353      temperature: 0,
354    });
355 
356    // Ensure one paragraph, clamp length
357    const oneLine = finalParagraph.replace(/\n+/g, " ").trim();
358    const words = oneLine.split(/\s+/);
359    const clamped = words.length <= 130 ? oneLine : words.slice(0, 120).join(" ") + "…";
360 
361    return NextResponse.json({ summaryParagraph: clamped });
362  } catch (err: any) {
363    return NextResponse.json({ error: err?.message || "Unexpected error" }, { status: 500 });
364  }
365}
366 
367Notes & Next Steps
368No DB: Keys are provided by the user each run; nothing is stored.