add axios and add url param as well

apps/studio-next/src/app/api/crawler/route.tsx

@@ -6,7 +6,7 @@ import axios from "axios";
 export async function GET(request: NextRequest) {
   const Base64searchParams = request.nextUrl.searchParams.get('base64');
   const URLsearchParams = request.nextUrl.searchParams.get('url');
-  console.log(URLsearchParams);
+
   try {
     if (!Base64searchParams && !URLsearchParams) return new NextResponse(null, { status: 200 });
     let info: DocumentInfo | null = null;
@@ -20,8 +20,7 @@ export async function GET(request: NextRequest) {
       try {
           const response = await axios.get(URLsearchParams);
           if (response.status === 200) {
-            console.log(response.data);
-            info = await parseURL(response.data);
+            info = response.data;
           } else {
             return new NextResponse("Not a valid URL", { status: 500 });
           }

apps/studio-next/src/middleware.ts

@@ -5,10 +5,11 @@ export async function middleware(request: NextRequest) {
   const userAgents = crawlers.map(crawler => crawler.pattern);
   const requestInfo = userAgent(request);
   const res = NextResponse.next();
-  for (const ua of userAgents) {
-    if (requestInfo.ua.toLowerCase().includes(ua.toLowerCase())) {
+  const documentURL = request.nextUrl.searchParams.get("url");
+  console.log(documentURL);
+  // for (const ua of userAgents) {
+  //   if (requestInfo.ua.toLowerCase().includes(ua.toLowerCase())) {
       const encodedDocument = request.nextUrl.searchParams.get("base64");
-      const documentURL = request.nextUrl.searchParams.get("url");
       if (!encodedDocument && !documentURL) {
         return res;
       }
@@ -18,10 +19,10 @@ export async function middleware(request: NextRequest) {
       if (documentURL) {
         return NextResponse.rewrite(new URL(`/api/crawler?url=${documentURL}`, request.url));
       }
+      return res;
     }
-  }
-  return res;
-}
+  //}
+//}
 
 export const config = {
   matcher: ['/:base64', '/:url'],