From eb57b845207f97de530f3effe9caa1d5a734415c Mon Sep 17 00:00:00 2001 From: oiov Date: Wed, 7 Aug 2024 16:47:36 +0800 Subject: [PATCH] feat: add meta scraping --- app/(protected)/dashboard/scrape/loading.tsx | 11 ++ .../dashboard/scrape/meta-scraping.tsx | 123 ++++++++++++++++++ app/(protected)/dashboard/scrape/page.tsx | 30 +++++ app/api/scraping/{ => meta}/route.ts | 37 +++++- components/shared/icons.tsx | 2 + config/dashboard.ts | 1 + package.json | 1 + pnpm-lock.yaml | 16 +++ 8 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 app/(protected)/dashboard/scrape/loading.tsx create mode 100644 app/(protected)/dashboard/scrape/meta-scraping.tsx create mode 100644 app/(protected)/dashboard/scrape/page.tsx rename app/api/scraping/{ => meta}/route.ts (59%) diff --git a/app/(protected)/dashboard/scrape/loading.tsx b/app/(protected)/dashboard/scrape/loading.tsx new file mode 100644 index 0000000..bb440cf --- /dev/null +++ b/app/(protected)/dashboard/scrape/loading.tsx @@ -0,0 +1,11 @@ +import { Skeleton } from "@/components/ui/skeleton"; +import { DashboardHeader } from "@/components/dashboard/header"; + +export default function DashboardRecordsLoading() { + return ( + <> + + + + ); +} diff --git a/app/(protected)/dashboard/scrape/meta-scraping.tsx b/app/(protected)/dashboard/scrape/meta-scraping.tsx new file mode 100644 index 0000000..6f38152 --- /dev/null +++ b/app/(protected)/dashboard/scrape/meta-scraping.tsx @@ -0,0 +1,123 @@ +"use client"; + +import { useState } from "react"; +import JsonView from "@uiw/react-json-view"; +import { githubLightTheme } from "@uiw/react-json-view/githubLight"; +import { vscodeTheme } from "@uiw/react-json-view/vscode"; +import { useTheme } from "next-themes"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +export default function MetaScraping() { + const { theme } = useTheme(); + const [currentLink, setCurrentLink] = useState(""); + const [protocol, setProtocol] = useState("https"); + const [metaInfo, setMetaInfo] = useState({ + title: "", + description: "", + image: "", + icon: "", + url: "", + lang: "", + author: "", + publisher: "", + date: "", + }); + const [isScraping, setIsScraping] = useState(false); + + const handleScraping = async () => { + if (currentLink) { + setIsScraping(true); + const res = await fetch( + `/api/scraping/meta?url=${protocol}://${currentLink}`, + ); + if (!res.ok || res.status !== 200) { + toast.error(res.statusText); + } else { + const data = await res.json(); + setMetaInfo(data); + toast.success("Success!"); + } + setIsScraping(false); + } + }; + return ( + <> + + + Website Meta Scrape + Scrape the meta data of a website. + + +
+ + setCurrentLink(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + handleScraping(); + } + }} + /> + +
+ +
+ +
+ + + + ); +} diff --git a/app/(protected)/dashboard/scrape/page.tsx b/app/(protected)/dashboard/scrape/page.tsx new file mode 100644 index 0000000..5d2feca --- /dev/null +++ b/app/(protected)/dashboard/scrape/page.tsx @@ -0,0 +1,30 @@ +import { redirect } from "next/navigation"; + +import { getCurrentUser } from "@/lib/session"; +import { constructMetadata } from "@/lib/utils"; +import { DashboardHeader } from "@/components/dashboard/header"; + +import MetaScraping from "./meta-scraping"; + +export const metadata = constructMetadata({ + title: "Scraping API - WR.DO", + description: "Quickly extract valuable structured website data", +}); + +export default async function DashboardPage() { + const user = await getCurrentUser(); + + if (!user?.id) redirect("/login"); + + return ( + <> + + + + ); +} diff --git a/app/api/scraping/route.ts b/app/api/scraping/meta/route.ts similarity index 59% rename from app/api/scraping/route.ts rename to app/api/scraping/meta/route.ts index 453a9ac..7b83764 100644 --- a/app/api/scraping/route.ts +++ b/app/api/scraping/meta/route.ts @@ -2,6 +2,7 @@ import cheerio from "cheerio"; import { checkUserStatus } from "@/lib/dto/user"; import { getCurrentUser } from "@/lib/session"; +import { isLink } from "@/lib/utils"; export const revalidate = 60; @@ -11,11 +12,11 @@ export async function GET(req: Request) { if (user instanceof Response) return user; const url = new URL(req.url); - const link = url.searchParams.get("link"); - if (!link) { - return Response.json("link is required", { + const link = url.searchParams.get("url"); + if (!link || !isLink(link)) { + return Response.json("Url is required", { status: 400, - statusText: "link is required", + statusText: "Url is required", }); } @@ -28,7 +29,7 @@ export async function GET(req: Request) { } const html = await res.text(); - console.log(html); + // console.log(html); const $ = cheerio.load(html); const title = @@ -44,8 +45,32 @@ export async function GET(req: Request) { $("meta[name='og:image']").attr("content") || $("meta[property='twitter:image']").attr("content") || $("meta[name='twitter:image']").attr("content"); + const icon = + $("link[rel='icon']").attr("href") || + $("link[rel='apple-touch-icon']").attr("href"); + const lang = + $("html").attr("lang") || + $("html").attr("xml:lang") || + $("body").attr("lang") || + $("body").attr("xml:lang"); + const author = + $("meta[name='author']").attr("content") || + $("meta[property='author']").attr("content"); + const publisher = + $("meta[name='publisher']").attr("content") || + $("meta[property='publisher']").attr("content"); - return Response.json({ title, description, image }); + return Response.json({ + title, + description, + image, + icon, + url: link, + lang, + author, + publisher, + date: new Date(), + }); } catch (error) { console.log(error); return Response.json(error?.statusText || error, { diff --git a/components/shared/icons.tsx b/components/shared/icons.tsx index 8c5ea2e..6349a02 100644 --- a/components/shared/icons.tsx +++ b/components/shared/icons.tsx @@ -3,6 +3,7 @@ import { ArrowRight, ArrowUpRight, BookOpen, + Bug, Check, ChevronLeft, ChevronRight, @@ -127,4 +128,5 @@ export const Icons = { globe: Globe, link: Link, mail: Mail, + bug: Bug, }; diff --git a/config/dashboard.ts b/config/dashboard.ts index d390469..496cb9e 100644 --- a/config/dashboard.ts +++ b/config/dashboard.ts @@ -11,6 +11,7 @@ export const sidebarLinks: SidebarNavItem[] = [ { href: "/dashboard", icon: "dashboard", title: "Dashboard" }, { href: "/dashboard/records", icon: "globeLock", title: "DNS Records" }, { href: "/dashboard/urls", icon: "link", title: "Short Urls" }, + { href: "/dashboard/scrape", icon: "bug", title: "Scraping API" }, ], }, { diff --git a/package.json b/package.json index 691b7ad..af7172b 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@react-email/html": "0.0.8", "@t3-oss/env-nextjs": "^0.11.0", "@typescript-eslint/parser": "^7.16.1", + "@uiw/react-json-view": "2.0.0-alpha.26", "@unovis/react": "^1.4.3", "@unovis/ts": "^1.4.3", "@vercel/analytics": "^1.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bab6ebe..a519c06 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: '@typescript-eslint/parser': specifier: ^7.16.1 version: 7.16.1(eslint@8.57.0)(typescript@5.5.3) + '@uiw/react-json-view': + specifier: 2.0.0-alpha.26 + version: 2.0.0-alpha.26(@babel/runtime@7.24.4)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@unovis/react': specifier: ^1.4.3 version: 1.4.3(@unovis/ts@1.4.3)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -2896,6 +2899,13 @@ packages: resolution: {integrity: sha512-Qlzzx4sE4u3FsHTPQAAQFJFNOuqtuY0LFrZHwQ8IHK705XxBiWOFkfKRWu6niB7hwfgnwIpO4jTC75ozW1PHWg==} engines: {node: ^18.18.0 || >=20.0.0} + '@uiw/react-json-view@2.0.0-alpha.26': + resolution: {integrity: sha512-i3uph/hjjT+RwUMjC3lpQUFwWWP+/muipo5Hj22XlhreFrcnhgydorq3PFBnmu4nZxNzkwGar07M6/f6+5b+XQ==} + peerDependencies: + '@babel/runtime': '>=7.10.0' + react: '>=18.0.0' + react-dom: '>=18.0.0' + '@ungap/structured-clone@1.2.0': resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} @@ -9508,6 +9518,12 @@ snapshots: '@typescript-eslint/types': 7.16.1 eslint-visitor-keys: 3.4.3 + '@uiw/react-json-view@2.0.0-alpha.26(@babel/runtime@7.24.4)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@babel/runtime': 7.24.4 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + '@ungap/structured-clone@1.2.0': {} '@unovis/dagre-layout@0.8.8-2':