ScrapeCart/index.js at main · SpaceTesla/ScrapeCart · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
const express = require("express");
const path = require("path");
const puppeteer = require("puppeteer");
const fs = require("fs").promises;

const { scrapeAmazon, scrapeFlipkart } = require("./scrapper");

const app = express();
const port = 8080;

app.use(express.urlencoded({ extended: true }));

app.set("view engine", "ejs");
app.set("views", path.join(__dirname, "views"));
app.use(express.static(path.join(__dirname, "public")));

// app.get("/search", async (req, res) => {
//   const { q } = req.query;
//   let products = await scrape(q);
//   res.render("index", { products: products });
// });

let cacheA = {};
app.get("/amazon/search", async (req, res) => {
  const { q } = req.query;

  // If the query is in the cache, use the cached data
  if (cacheA[q]) {
    res.render("search", { products: cacheA[q] });
  } else {
    // Otherwise, scrape the data and store it in the cache
    const products = await scrapeAmazon(q);
    cacheA[q] = products;
    res.render("search", { products: products, domain: "Amazon" });
  }
});

let cacheF = {};

app.get("/", (req, res) => {
  res.render("index");
});

app.get("/flipkart/search", async (req, res) => {
  const { q } = req.query;
  console.log(q);
  // If the query is in the cache, use the cached data
  if (cacheF[q]) {
    res.render("search", { products: cacheF[q] });
  } else {
    // Otherwise, scrape the data and store it in the cache
    const products = await scrapeFlipkart(q);
    cacheF[q] = products;
    res.render("search", { products: products, domain: "Flipkart" });
  }
});

app.listen(port, () => {
  console.log(`Listening to port: ${port}`);
});

// const scrape = async (q) => {
//   let products = [];
//   const browser = await puppeteer.launch({
//     headless: false,
//     defaultViewport: false,
//     userDataDir: "./tmp",
//   });
//   const page = await browser.newPage();
//   await page.goto(`https://www.amazon.in/s?k=${q}`);

//   const productHandles = await page.$$(
//     "div.s-main-slot.s-result-list.s-search-results.sg-row > div.s-result-item"
//   );

//   // console.log(productHandles);

//   for (const productHandle of productHandles) {
//     try {
//       const title = await page.evaluate(
//         (el) => el.querySelector("h2 > a > span").textContent,
//         productHandle
//       );

//       const link = await page.evaluate(
//         (el) => el.querySelector("h2 > a").href,
//         productHandle
//       );

//       const price = await page.evaluate(
//         (el) => el.querySelector(".a-price-whole").textContent,
//         productHandle
//       );

//       const img = await page.evaluate(
//         (el) => el.querySelector("div > span > a > div > img").src,
//         productHandle
//       );

//       // console.log(img);

//       // console.log(price);
//       products.push({ img: img, title: title, link: link, price: price });
//     } catch (err) {
//       // console.log(err);
//     }
//   }
//   console.log(products);
//   console.log(products.length);
//   const fileName = __filename.split("\\").pop().split(".")[0];
//   await fs.writeFile(`${fileName}.json`, JSON.stringify(products, null, 2));

//   await browser.close();

//   return products;
// };

// scrape();