-
Notifications
You must be signed in to change notification settings - Fork 1
/
index.js
92 lines (70 loc) · 2.35 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
const puppeteer = require("puppeteer");
const xlsx = require("xlsx");
(async () => {
const URL =
"https://www.amazon.com/s?k=programmer+socks&crid=2GGN7SIBWWR17&sprefix=programmer+socks%2Caps%2C161&ref=nb_sb_ss_pltr-xclick_1_16";
const proxyURL = 'gw.dataimpulse.com:823'
const username = ''
const password = ''
const browser = await puppeteer.launch({
headless: false,
args: [
`--proxy-server=${proxyURL}`
]
});
const page = await browser.newPage();
await page.authenticate({
username,
password,
})
await page.goto(URL, { waitUntil: "networkidle2" });
const title = await page.title();
console.log(`Titulo de la pagina: ${title}`);
let products = [];
let nextPage = true;
while (nextPage) {
const newProducts = await page.evaluate(() => {
const products = Array.from(
document.querySelectorAll(".puis-card-container.s-card-container")
);
return products.map((product) => {
const title = product.querySelector(".a-text-normal")?.innerText;
const priceWhole = product.querySelector(".a-price-whole")?.innerText;
const priceFraction =
product.querySelector(".a-price-fraction")?.innerText;
if (!priceWhole || !priceFraction) {
return {
title,
price: "N/A",
}
}
const priceWholeCleaned = priceWhole.replace(/\n/g, "").trim()
const priceFractionCleaned = priceFraction.replace(/\n/g, "").trim()
return {
title,
price: `${priceWholeCleaned}${priceFractionCleaned}`
}
});
});
products = [...products, ...newProducts];
nextPage = await page.evaluate(() => {
const nextButton = document.querySelector(".s-pagination-next");
if (
nextButton &&
!nextButton.classList.contains("s-pagination-disabled")
) {
nextButton.click();
return true;
}
return false;
});
await new Promise((resolve) => setTimeout(resolve, 2000));
}
console.log(products)
const wb = xlsx.utils.book_new();
const ws = xlsx.utils.json_to_sheet(products);
const path = "products.xlsx";
xlsx.utils.book_append_sheet(wb, ws, "Products");
xlsx.writeFile(wb, path);
// await browser.close();
})();