ScrapeCart/scrapper.js at main · SpaceTesla/ScrapeCart · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
const puppeteer = require("puppeteer");
const fs = require("fs").promises;

const scrapeFlipkart = async (q) => {
  let products = [];

  const browser = await puppeteer.launch({
    headless: false,
    defaultViewport: false,
    userDataDir: "./tmpF",
  });

  const page = await browser.newPage();
  await page.goto(`https://www.flipkart.com/search?q=${q}`);

  const productHandles = await page.$$("div._4ddWXP");

  for (const productHandle of productHandles) {
    try {
      const title = await page.evaluate(
        (el) => el.querySelector("a.s1Q9rs").title,
        productHandle
      );

      const link = await page.evaluate(
        (el) => el.querySelector("a.s1Q9rs").href,
        productHandle
      );

      const price = await page.evaluate(
        (el) => el.querySelector("._30jeq3").textContent.substring(1),
        productHandle
      );

      const img = await page.evaluate(
        (el) => el.querySelector(".CXW8mj > img").src,
        productHandle
      );

      products.push({ img: img, title: title, link: link, price: price });
    } catch (err) {
      console.log(err);
    }
  }
  console.log(products);
  console.log(products.length);
  const fileName = __filename.split("\\").pop().split(".")[0];
  await fs.writeFile(`flipkart.json`, JSON.stringify(products, null, 2));

  await browser.close();

  return products;
};

const scrapeAmazon = async (q) => {
  let products = [];

  const browser = await puppeteer.launch({
    headless: false,
    defaultViewport: false,
    userDataDir: "./tmp",
  });
  const page = await browser.newPage();
  await page.goto(`https://www.amazon.in/s?k=${q}`);

  const productHandles = await page.$$(
    "div.s-main-slot.s-result-list.s-search-results.sg-row > div.s-result-item"
  );

  for (const productHandle of productHandles) {
    try {
      const title = await page.evaluate(
        (el) => el.querySelector("h2 > a > span").textContent,
        productHandle
      );

      const link = await page.evaluate(
        (el) => el.querySelector("h2 > a").href,
        productHandle
      );

      const price = await page.evaluate(
        (el) => el.querySelector(".a-price-whole").textContent,
        productHandle
      );

      const img = await page.evaluate(
        (el) => el.querySelector("div > span > a > div > img").src,
        productHandle
      );

      products.push({ img: img, title: title, link: link, price: price });
    } catch (err) {
      console.log(err);
    }
  }
  console.log(products);
  console.log(products.length);
  const fileName = __filename.split("\\").pop().split(".")[0];
  await fs.writeFile(`amazon.json`, JSON.stringify(products, null, 2));

  await browser.close();

  return products;
};

module.exports = { scrapeAmazon, scrapeFlipkart };