disney-cruise-prices/index.ts

import puppeteer, {Page} from 'puppeteer';
import * as fs from "fs";
import {createCsvFromJson, createCsvFromJsonArray} from "./flatten-data";

(async () => {
    const browser = await puppeteer.launch({headless: false, devtools: true});
    const page = await browser.newPage();

    await page.goto('https://disneycruise.disney.go.com/cruises-destinations/list/#september-2023,october-2023,january-2024,february-2024,march-2024,april-2024,may-2024,alaska-cruises,bahamas-cruises,bermuda-cruises,canada-cruises,caribbean-cruises,mexico-cruises,pacific-coast-cruises,5-to-6');

    // Set screen size
    await page.setViewport({width: 1519, height: 1024});

    // Type into search box
    // await page.type('.search-box__input', 'automate beyond recorder');

    // Wait and click on first result
    const showDatesButtonSelector = '.product-card-footer-wrapper__btn';
    await page.waitForSelector(showDatesButtonSelector);
    // await page.click(showDatesButtonSelector);

    // Scroll down
    // should scroll down until the button count stops going up
    let buttonCountChanged = true;
    let buttonCount = 0;
    let buttonElements = await page.$$(showDatesButtonSelector);
    buttonCount = buttonElements.length;
    // while (buttonCountChanged) {
    //     await page.evaluate(() => {
    //         window.scrollBy(0, window.innerHeight)
    //         return Promise.resolve();
    //     });
    //     buttonElements = await page.$$(showDatesButtonSelector);
    //     if (buttonElements.length > buttonCount) {
    //         buttonCount = buttonElements.length;
    //     } else {
    //         buttonCountChanged = false;
    //     }
    // }
    //
    // for (let i = 0; i < buttonElements.length; i++) {
    //     console.log(buttonElements[i]);
    //     await buttonElements[i].click();
    //     // await page.waitForSelector('wdpr-price');
    //     // const price = await page.$eval('wdpr-price', (el) => el.textContent);
    //     // console.log('price', price);
    // }

    const data = await getAvailableProducts(page);

    // console.log('data');
    // console.log(JSON.stringify(data, null, ' '));
    fs.writeFileSync('products.json', JSON.stringify(data, null, ' '));
    createCsvFromJsonArray(data.products, 'products.csv');

    let availableSailings = await Promise.all(data.products.map(async (product: { productId: string; itineraries: { itineraryId: string; }[]; }) => {
        return getAvailableSailings(page, product.productId, product.itineraries[0].itineraryId).then(({sailings}): any[] => {
          return sailings;
      });
    })).then((results) => {
        let sailings: any[] = [];
        results.forEach((result) => {
            sailings = sailings.concat(result);
        });
        return sailings;
    });

    console.log('availableSailings');
    console.log(JSON.stringify(availableSailings, null, ' '));
    fs.writeFileSync('sailings.json', JSON.stringify(availableSailings, null, ' '));
    createCsvFromJsonArray(availableSailings, 'sailings.csv');

    console.log('buttonCount', buttonCount);

    // // Locate the full title with a unique string
    // const textSelector = await page.waitForSelector(
    //     'text/Customize and automate'
    // );
    // // @ts-ignore
    // const fullTitle = await textSelector.evaluate(el => el.textContent);
    //
    // // Print the full title
    // console.log('The title of this blog post is "%s".', fullTitle);

    await browser.close();
})();

// function getButtonCount(page: Page) {
//     return page.$$('.product-card-footer-wrapper__btn').then((buttonElements) => {
//         return buttonElements.length;
//     });
// }

function getAvailableProducts(page: Page): Promise<any> {
    return page.evaluate(() => {
        return getProductsPage(1).then((data) => {
            let pageCount = data.totalPages;
            let promises = [];
            for (let i = 2; i <= pageCount; i++) {
                promises.push(getProductsPage(i));
            }
            return Promise.all(promises).then((results) => {
                results.forEach((result) => {
                    data.products = data.products.concat(result.products);
                });
                return data;
            });
        });

        function getProductsPage(pageCount = 1): Promise<any> {
            return fetch('/dcl-apps-productavail-vas/available-products/', {
                body: JSON.stringify({
                    "currency": "USD",
                    "filters": ["2023-09;filterType=date", "2023-10;filterType=date", "2024-01;filterType=date", "2024-02;filterType=date", "2024-03;filterType=date", "2024-04;filterType=date", "2024-05;filterType=date", "ALASKA;filterType=destination", "BAHAMAS;filterType=destination", "BERMUDA;filterType=destination", "CANADA;filterType=destination", "CARIBBEAN;filterType=destination", "MEXICAN RIVIERA;filterType=destination", "CALIFORNIA COAST;filterType=destination", "5-6;filterType=night"],
                    "partyMix": [{
                        "accessible": false,
                        "adultCount": 2,
                        "childCount": 2,
                        "nonAdultAges": [{"age": 7, "ageUnit": "YEAR"}, {"age": 9, "ageUnit": "YEAR"}],
                        "partyMixId": "0"
                    }],
                    "region": "INTL",
                    "storeId": "DCL",
                    "affiliations": [],
                    "page": pageCount,
                    "pageHistory": false,
                    "includeAdvancedBookingPrices": true,
                    "exploreMorePage": 1,
                    "exploreMorePageHistory": false
                }),
                headers: {
                    'Content-Type': 'application/json'
                },
                method: 'POST'
            }).then((response) => {
                return response.json();
            }).then((data) => {
                return data;
            });
        }
    });
}

async function getAvailableSailings(page: Page, productID: string, itineraryId: string): Promise<any> {
    return await page.evaluate((productID, itineraryId) => {
        return fetch('/dcl-apps-productavail-vas/available-sailings/', {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json'
            },
            body: JSON.stringify({
                "currency": "USD",
                "filters": [
                    "2023-09;filterType=date",
                    "2023-10;filterType=date",
                    "2024-01;filterType=date",
                    "2024-02;filterType=date",
                    "2024-03;filterType=date",
                    "2024-04;filterType=date",
                    "2024-05;filterType=date",
                    "ALASKA;filterType=destination",
                    "BAHAMAS;filterType=destination",
                    "BERMUDA;filterType=destination",
                    "CANADA;filterType=destination",
                    "CARIBBEAN;filterType=destination",
                    "MEXICAN RIVIERA;filterType=destination",
                    "CALIFORNIA COAST;filterType=destination",
                    "5-6;filterType=night"
                ],
                "partyMix": [
                    {
                        "accessible": false,
                        "adultCount": 2,
                        "childCount": 2,
                        "nonAdultAges": [
                            {
                                "age": 7,
                                "ageUnit": "YEAR"
                            },
                            {
                                "age": 9,
                                "ageUnit": "YEAR"
                            }
                        ],
                        "partyMixId": "0"
                    }
                ],
                "region": "INTL",
                "storeId": "DCL",
                "affiliations": [],
                "itineraryId": itineraryId,
                "productId": productID,
                "includeAdvancedBookingPrices": true
            })
        }).then((response) => {
            return response.json();
        }).then((data) => {
            return data;
        });
    }, productID, itineraryId);
}