使用puppeteer抓取谷歌地圖時滾動不起作用

我正在抓取谷歌地圖位置資料，但發生的是它只回傳用戶評論的前 10 個結果，而不是在那之后。我認為滾動功能存在一些問題。

const puppeteer = require('puppeteer');

function extractItems() {
  const extractedElements = document.querySelectorAll('.MyEned span.wiI7pd');
  const items = [];
  for (let element of extractedElements) {
    items.push(element.innerText);
  }
  return items;
}

async function scrapeItems(
  page,
  extractItems,
  itemCount,
  scrollDelay = 2000,
) {
  let items = [];
  try {
    let previousHeight;
    while (items.length < itemCount) {
      items = await page.evaluate(extractItems);
      previousHeight = await page.evaluate('div.m6QErb.DxyBCb.scrollHeight');//selector for scroller
      await page.evaluate('window.scrollTo(0, div.m6QErb.DxyBCb.scrollHeight)');
      await page.waitForFunction(`div.m6QErb.DxyBCb.scrollHeight > ${previousHeight}`);
      await page.waitForTimeout(scrollDelay);
    }
  } catch(e) { }
  return items;
}

(async () => {
  let browser = await puppeteer.connect();
  browser = await puppeteer.launch({
    headless: false,
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });
  const [page] = await browser.pages();
  page.setViewport({ width: 1280, height: 926 });

  await page.goto('https://www.google.com/maps/place/Ace Florist & Flower Delivery/@40.8265438,-73.5011026,15z/data=!4m7!3m6!1s0x0:0x9062074cae10c10f!8m2!3d40.8265438!4d-73.5011026!9m1!1b1');

  // Auto-scroll and extract desired items from the page. Currently set to extract eight items.
  const items = await scrapeItems(page, extractItems, 30);

  console.log(items)

  await browser.close();
})();

uj5u.com熱心網友回復：

所以我剛剛發現我必須document.querySelector在評估滾動高度以及檢查滾動高度大于以前的高度時添加。

    items = await page.evaluate(extractItems);
    previousHeight = page.evaluate('document.querySelector("div.m6QErb.DxyBCb").scrollHeight');
    await page.evaluate(`document.querySelector("div.m6QErb.DxyBCb").scrollTo(0, ${previousHeight[0]})`);
    await page.waitForFunction(`document.querySelector("div.m6QErb.DxyBCb").scrollHeight > ${previousHeight[0]}`);
    await page.waitForTimeout(scrollDelay);

uj5u.com熱心網友回復：

此代碼作業正常：

'use strict'

const puppeteer = require('puppeteer');
function extractItems() {
  const extractedElements = document.querySelectorAll('.MyEned span.wiI7pd');
  const items = [];
  for (let element of extractedElements) {
    items.push(element.innerText);
  }
  return items;
}
async function scrapeItems(
  page,
  extractItems,
  itemCount,
  scrollDelay = 2000,
) {
  let items = [];
  try {
    let previousHeight;
    while (items.length < itemCount) {
      console.log(`items.length: ${items.length} itemCount: ${itemCount}`)
      
      items = await page.evaluate(extractItems);

      previousHeight = await page.evaluate(() => { 
        const scroller = document.querySelector('div.m6QErb.DxyBCb') 
        return scroller.scrollHeight  
      })

      await page.evaluate(`document.querySelector("div.m6QErb.DxyBCb").scrollTo(0, ${previousHeight})`);
      await page.waitForFunction(`document.querySelector("div.m6QErb.DxyBCb").scrollHeight > ${previousHeight}`);
      await page.waitForTimeout(scrollDelay);

    }
  } catch(e) { }
  return items;
}


(async () => {
  const browser = await puppeteer.launch({
    headless: false,
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });
  const [page] = await browser.pages();
  page.setViewport({ width: 1280, height: 926 });

  await page.goto('https://www.google.com/maps/place/Ace Florist & Flower Delivery/@40.8265438,-73.5011026,15z/data=!4m7!3m6!1s0x0:0x9062074cae10c10f!8m2!3d40.8265438!4d-73.5011026!9m1!1b1');

  // Auto-scroll and extract desired items from the page. Currently set to extract eight items.
  const items = await scrapeItems(page, extractItems, 30);

  console.log(items)

await browser.close();
})();

標籤：javascript 网页抓取 傀儡师

上一篇：在R中抓取評論

下一篇：如何使用自定義標記顏色將每個句子包裝在標簽中？