- Multithreaded requests are implemented

example.js

@@ -1,6 +1,7 @@
 const async = require("async");
 const { ParseAudios } = require("./dist/index");
 require("dotenv").config();
+const fs = require("fs");
 
 // basic
 (async () => {
@@ -15,9 +16,12 @@ require("dotenv").config();
     headless: false
   });
   await parse.launch();
-  const tracks = await parse.run(377897606, 3);
+  const tracks = await parse.run(41697780);
   await parse.exit();
-  console.log(tracks);
+  fs.writeFile("tracks.json", JSON.stringify(tracks), err => {
+    if (err) throw err;
+    console.log("The file tracks.json has been created!");
+  });
 })().catch(err => {
   console.log("❌ " + err.message);
   process.exit(1);

src/get-encypted-url.ts

@@ -1,28 +1,13 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import axios from "axios";
 
-function delay(time: number) {
-  return new Promise(function(resolve) {
-    setTimeout(resolve, time);
-  });
-}
-
-async function asyncForEach(array: any[], callback: Function) {
-  for (let index = 0; index < array.length; index++) {
-    await callback(array[index], index, array);
-  }
-}
-
 async function getEncryptedUrl(cookie: string, ids: string[]) {
   const result: any[] = [];
-  const spliceIds = ids;
 
-  await asyncForEach(spliceIds, async () => {
-    const newIds = spliceIds.splice(0, 6).join(",");
-
-    const response = await axios.post(
+  const promises = ids.map(async id => {
+    return await axios.post(
       "https://vk.com/al_audio.php",
-      `act=reload_audio&al=1&ids=${newIds}`,
+      `act=reload_audio&al=1&ids=${id}`,
       {
         headers: {
           "Content-Type": "application/x-www-form-urlencoded",
@@ -34,19 +19,23 @@ async function getEncryptedUrl(cookie: string, ids: string[]) {
         }
       }
     );
+  });
 
-    try {
-      response.data.payload[1][0].forEach((item: any) => {
-        if (!Array.isArray(item) && !item.length) return;
-        result.push(item[2]);
-      });
-    } catch (err) {
-      console.error(err);
-      throw new Error("HTTP Request is failed, please send issue to repo");
-    }
-
-    console.log(`⚡ Finished ${result.length} tracks`);
-    await delay(500);
+  await Promise.all(promises).then(responses => {
+    responses.forEach(track => {
+      try {
+        const data = track.data.payload[1][0];
+        if (typeof data !== "undefined") {
+          data.forEach((item: any) => {
+            if (!Array.isArray(item) && !item.length) return;
+            result.push(item[2]);
+          });
+        }
+      } catch (err) {
+        console.error(err);
+        throw new Error("HTTP Request is failed, please restart");
+      }
+    });
   });
 
   return result;

src/get-scrolled-down-page.ts

@@ -14,14 +14,11 @@ async function getScrolledDownPage(page: Page) {
     };
 
     while (scrollY !== prevScrollY) {
-      console.log(`⏬ Scrolled to - ${scrollY}y`);
       await scrollTo();
       await page.waitFor(100);
     }
   };
 
-
-
   await scrollDown();
   await page.waitFor(1000);
 }

src/index.ts

@@ -29,13 +29,6 @@ interface ParseConfig {
   headless?: boolean;
 }
 
-// interface Track {
-//   mp3: string;
-//   title: string;
-//   author: string;
-//   cover?: string;
-// }
-
 class ParseAudios {
   private browser!: Browser;
 
@@ -69,11 +62,10 @@ class ParseAudios {
       if (max) {
         return data.splice(0, max);
       }
+
       return data;
     }, max);
 
-    console.log(`${audiosDataset.length} - tracks ready to parse`);
-
     let tracks = audiosDataset.map(track => ({
       mp3: "",
       title: track[3],
@@ -84,7 +76,13 @@ class ParseAudios {
     await page.close();
 
     const ids = audiosDataset.map(dataset => takeId(dataset));
-    const encryptedUrls = await getEncryptedUrl(cookie, ids);
+
+    const slicedIds: string[] = [];
+    for (let i = 0; i <= ids.length; i += 7) {
+      slicedIds.push(ids.slice(i, i + 7).join(","));
+    }
+
+    const encryptedUrls = await getEncryptedUrl(cookie, slicedIds);
 
     tracks = encryptedUrls.map((item, index) => ({
       ...tracks[index],
@@ -96,12 +94,14 @@ class ParseAudios {
       return { ...item, mp3: encodedMp3 };
     });
 
-    console.log(`✅ id${id} — done`);
+    console.log(
+      `⚡ Finished ${tracks.length} / ${audiosDataset.length} tracks`
+    );
     return tracks;
   }
 
   async makeAuthBrowser() {
-    console.log("...login");
+    console.log("login...");
     const page = await this.browser.newPage();
     await page.goto("https://vk.com/feed");
     await page.waitFor("input[id=email]");
@@ -119,11 +119,12 @@ class ParseAudios {
     await page.waitFor("div.top_profile_name", { timeout: 10000 }).catch(() => {
       throw new Error(
         "Timeout 10s \n Failed to auth, possible reasons: \n 1. Incorrect login or password \n 2. Showed captcha \n 3." +
-          " Slowly internet"
+          " Slowly internet \n" +
+          "4. Slowly computer"
       );
     });
     await page.close();
-    console.log(`✅ - Auth`);
+    console.log(`✅ Auth`);
   }
 }