The getAllUntil function is an asynchronous function that retrieves and filters data from the DOM using Selenium WebDriver, allowing for conditional scrolling and optional filtering. It recursively retrieves data, filters it based on a callback function, and returns the collected data when no more data is found or the scrolling condition is met.

Run example

npm run import -- "all elements until"

all elements until

var importer = require('../Core');
var scrollClient = importer.import("scroll specific element")
const selectDom = importer.import("selenium select")

async function getAllUntil(driver, scrollableSelector,
    dataSelector,
    compare = (a, b) => a === b,
    cb = (i) => i < 3,
    set = [],
    up = false,
    i = 0) {
    //let body = await driver.findElement(By.css('body'))
    let result = await selectDom(driver, dataSelector)
    //let result = await driver.executeScript('return (function main() {\n return 1;\n})()')

    let filtered = []
    let newPosts = ((typeof result === 'string' ? [result] : result) || [])
    for (let i = 0; i < newPosts.length; i++) {
        let found = false
        for (let j = 0; j < set.length; j++) {
            if (await compare(newPosts[i], set[j])) {
                found = true
            }
        }
        if (!found) {
            filtered.push(newPosts[i])
        }
    }
    set = set.concat(filtered);
    // TODO: something with page timing is causing this to fail, continue to call until cb
    if (filtered.length > 0 && await cb(i, set)) {
        await scrollClient(driver, scrollableSelector, up)
        await new Promise(resolve => setTimeout(resolve, 2500))
        await scrollClient(driver, scrollableSelector, up)
        await new Promise(resolve => setTimeout(resolve, 2500))
        return await getAllUntil(driver, scrollableSelector, dataSelector, compare, cb, set, up, i + 1)
    }
    return set
}


module.exports = getAllUntil;

const { By } = require('selenium-webdriver');
const { scrollSpecificElement } = require('../Core');
const { seleniumSelect } = require('../Core');

async function getAllUntil(driver, scrollableSelector, dataSelector, compare = (a, b) => a === b, cb = (i, set) => i < 3, set = [], up = false, i = 0) {
    /**
     * Recursive function to retrieve data from a scrollable element until a condition is met.
     *
     * @param {object} driver - Selenium WebDriver instance.
     * @param {string} scrollableSelector - CSS selector for the scrollable element.
     * @param {string} dataSelector - CSS selector for the data to retrieve.
     * @param {function} compare - Function to compare elements (default: equality).
     * @param {function} cb - Callback function to determine if retrieval should continue (default: less than 3 iterations).
     * @param {array} set - Set of retrieved elements.
     * @param {boolean} up - Scroll direction (default: false).
     * @param {number} i - Current iteration (default: 0).
     * @returns {array} Set of retrieved elements.
     */
    const result = await seleniumSelect(driver, dataSelector);
    const filtered = result.filter((element, index) =>!set.some((setElement) => compare(element, setElement)));

    set = [...set,...filtered];
    if (filtered.length > 0 && await cb(i, set)) {
        await scrollSpecificElement(driver, scrollableSelector, up);
        await new Promise(resolve => setTimeout(resolve, 2500));
        await scrollSpecificElement(driver, scrollableSelector, up);
        await new Promise(resolve => setTimeout(resolve, 2500));
        return await getAllUntil(driver, scrollableSelector, dataSelector, compare, cb, set, up, i + 1);
    }
    return set;
}

module.exports = getAllUntil;

Function getAllUntil

Overview

getAllUntil is an asynchronous function that retrieves and filters data from the DOM using Selenium WebDriver.

Parameters

• driver: Selenium WebDriver instance
• scrollableSelector: CSS selector for the scrollable element
• dataSelector: CSS selector for the data elements
• compare: Optional callback function for filtering data (default: (a, b) => a === b)
• cb: Optional callback function for conditionally scrolling (default: (i) => i < 3)
• set: Optional array of collected data (default: [])
• up: Optional flag indicating whether to scroll up (default: false)
• i: Optional index for the callback function (default: 0)

Behavior

1. It retrieves the data from the DOM using selectDom.
2. It filters the retrieved data using the compare callback function.
3. It adds the filtered data to the set array.
4. If conditionally scrolling is enabled, it scrolls the scrollable element and waits for 2.5 seconds before recursing.
5. It returns the set array when no more data is found or the condition for conditionally scrolling is not met.

Modules

• importer: Requires the ../Core module.
• scrollClient: Imports the scroll specific element module from importer.
• selectDom: Imports the selenium select module from importer.

Export

getAllUntil is exported as a module.