ouyang/wr.do
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add meta scraping

Browse Source
This commit is contained in:
oiov
2024-08-07 16:47:36 +08:00
parent 9fe8e01745
commit eb57b84520
8 changed files with 215 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/(protected)/dashboard/scrape/loading.tsx
+11
View File
@@ -0,0 +1,11 @@
import { Skeleton } from "@/components/ui/skeleton";
import { DashboardHeader } from "@/components/dashboard/header";
export default function DashboardRecordsLoading() {
return (
<>
<DashboardHeader heading="DNS Records" text="" />
<Skeleton className="h-full w-full rounded-lg" />
</>
);
}
app/(protected)/dashboard/scrape/meta-scraping.tsx
+123
View File
@@ -0,0 +1,123 @@
"use client";
import { useState } from "react";
import JsonView from "@uiw/react-json-view";
import { githubLightTheme } from "@uiw/react-json-view/githubLight";
import { vscodeTheme } from "@uiw/react-json-view/vscode";
import { useTheme } from "next-themes";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
export default function MetaScraping() {
const { theme } = useTheme();
const [currentLink, setCurrentLink] = useState("");
const [protocol, setProtocol] = useState("https");
const [metaInfo, setMetaInfo] = useState({
title: "",
description: "",
image: "",
icon: "",
url: "",
lang: "",
author: "",
publisher: "",
date: "",
});
const [isScraping, setIsScraping] = useState(false);
const handleScraping = async () => {
if (currentLink) {
setIsScraping(true);
const res = await fetch(
`/api/scraping/meta?url=${protocol}://${currentLink}`,
);
if (!res.ok || res.status !== 200) {
toast.error(res.statusText);
} else {
const data = await res.json();
setMetaInfo(data);
toast.success("Success!");
}
setIsScraping(false);
}
};
return (
<>
<Card>
<CardHeader>
<CardTitle>Website Meta Scrape</CardTitle>
<CardDescription>Scrape the meta data of a website.</CardDescription>
</CardHeader>
<CardContent>
<div className="flex items-center">
<Select
onValueChange={(value: string) => {
setProtocol(value);
}}
name="protocol"
defaultValue={"https"}
>
<SelectTrigger className="h-10 w-24 rounded-r-none shadow-inner">
<SelectValue placeholder="Protocol" />
</SelectTrigger>
<SelectContent>
<SelectItem key="https" value="https">
https
</SelectItem>
<SelectItem key="http" value="http">
http
</SelectItem>
</SelectContent>
</Select>
<Input
type="text"
placeholder="www.example.com"
className="h-10 rounded-none border focus:border-primary active:border-primary"
value={currentLink}
size={100}
onChange={(e) => setCurrentLink(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
handleScraping();
}
}}
/>
<Button
variant="blue"
onClick={handleScraping}
disabled={isScraping}
className="rounded-l-none"
>
{isScraping ? "Scraping..." : "Send"}
</Button>
</div>
<div className="mt-4 rounded-md border p-3">
<JsonView
style={theme === "dark" ? vscodeTheme : githubLightTheme}
value={metaInfo}
displayObjectSize={false}
displayDataTypes={false}
/>
</div>
</CardContent>
</Card>
</>
);
}
app/(protected)/dashboard/scrape/page.tsx
+30
View File
@@ -0,0 +1,30 @@
import { redirect } from "next/navigation";
import { getCurrentUser } from "@/lib/session";
import { constructMetadata } from "@/lib/utils";
import { DashboardHeader } from "@/components/dashboard/header";
import MetaScraping from "./meta-scraping";
export const metadata = constructMetadata({
title: "Scraping API - WR.DO",
description: "Quickly extract valuable structured website data",
});
export default async function DashboardPage() {
const user = await getCurrentUser();
if (!user?.id) redirect("/login");
return (
<>
<DashboardHeader
heading="Scraping&nbsp;&nbsp;API"
text="Quickly extract valuable structured website data."
link="/docs/scraping-api"
linkText="Scraping API."
/>
<MetaScraping />
</>
);
}
app/api/scraping/route.ts → app/api/scraping/meta/route.ts
+31 -6
View File
@@ -2,6 +2,7 @@ import cheerio from "cheerio";
import { checkUserStatus } from "@/lib/dto/user";
import { getCurrentUser } from "@/lib/session";
import { isLink } from "@/lib/utils";
export const revalidate = 60;
@@ -11,11 +12,11 @@ export async function GET(req: Request) {
if (user instanceof Response) return user;
const url = new URL(req.url);
const link = url.searchParams.get("link");
if (!link) {
return Response.json("link is required", {
const link = url.searchParams.get("url");
if (!link || !isLink(link)) {
return Response.json("Url is required", {
status: 400,
statusText: "link is required",
statusText: "Url is required",
});
}
@@ -28,7 +29,7 @@ export async function GET(req: Request) {
}
const html = await res.text();
console.log(html);
// console.log(html);
const $ = cheerio.load(html);
const title =
@@ -44,8 +45,32 @@ export async function GET(req: Request) {
$("meta[name='og:image']").attr("content") ||
$("meta[property='twitter:image']").attr("content") ||
$("meta[name='twitter:image']").attr("content");
const icon =
$("link[rel='icon']").attr("href") ||
$("link[rel='apple-touch-icon']").attr("href");
const lang =
$("html").attr("lang") ||
$("html").attr("xml:lang") ||
$("body").attr("lang") ||
$("body").attr("xml:lang");
const author =
$("meta[name='author']").attr("content") ||
$("meta[property='author']").attr("content");
const publisher =
$("meta[name='publisher']").attr("content") ||
$("meta[property='publisher']").attr("content");
return Response.json({ title, description, image });
return Response.json({
title,
description,
image,
icon,
url: link,
lang,
author,
publisher,
date: new Date(),
});
} catch (error) {
console.log(error);
return Response.json(error?.statusText || error, {
components/shared/icons.tsx
+2
View File
@@ -3,6 +3,7 @@ import {
ArrowRight,
ArrowUpRight,
BookOpen,
Bug,
Check,
ChevronLeft,
ChevronRight,
@@ -127,4 +128,5 @@ export const Icons = {
globe: Globe,
link: Link,
mail: Mail,
bug: Bug,
};
config/dashboard.ts
+1
View File
@@ -11,6 +11,7 @@ export const sidebarLinks: SidebarNavItem[] = [
{ href: "/dashboard", icon: "dashboard", title: "Dashboard" },
{ href: "/dashboard/records", icon: "globeLock", title: "DNS Records" },
{ href: "/dashboard/urls", icon: "link", title: "Short Urls" },
{ href: "/dashboard/scrape", icon: "bug", title: "Scraping API" },
],
},
{
package.json
+1
View File
@@ -56,6 +56,7 @@
"@react-email/html": "0.0.8",
"@t3-oss/env-nextjs": "^0.11.0",
"@typescript-eslint/parser": "^7.16.1",
"@uiw/react-json-view": "2.0.0-alpha.26",
"@unovis/react": "^1.4.3",
"@unovis/ts": "^1.4.3",
"@vercel/analytics": "^1.3.1",
pnpm-lock.yaml
Generated
+16
View File
@@ -119,6 +119,9 @@ importers:
'@typescript-eslint/parser':
specifier: ^7.16.1
version: 7.16.1(eslint@8.57.0)(typescript@5.5.3)
'@uiw/react-json-view':
specifier: 2.0.0-alpha.26
version: 2.0.0-alpha.26(@babel/runtime@7.24.4)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@unovis/react':
specifier: ^1.4.3
version: 1.4.3(@unovis/ts@1.4.3)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -2896,6 +2899,13 @@ packages:
resolution: {integrity: sha512-Qlzzx4sE4u3FsHTPQAAQFJFNOuqtuY0LFrZHwQ8IHK705XxBiWOFkfKRWu6niB7hwfgnwIpO4jTC75ozW1PHWg==}
engines: {node: ^18.18.0 || >=20.0.0}
'@uiw/react-json-view@2.0.0-alpha.26':
resolution: {integrity: sha512-i3uph/hjjT+RwUMjC3lpQUFwWWP+/muipo5Hj22XlhreFrcnhgydorq3PFBnmu4nZxNzkwGar07M6/f6+5b+XQ==}
peerDependencies:
'@babel/runtime': '>=7.10.0'
react: '>=18.0.0'
react-dom: '>=18.0.0'
'@ungap/structured-clone@1.2.0':
resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
@@ -9508,6 +9518,12 @@ snapshots:
'@typescript-eslint/types': 7.16.1
eslint-visitor-keys: 3.4.3
'@uiw/react-json-view@2.0.0-alpha.26(@babel/runtime@7.24.4)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
dependencies:
'@babel/runtime': 7.24.4
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
'@ungap/structured-clone@1.2.0': {}
'@unovis/dagre-layout@0.8.8-2':