import puppeteer, {Page} from 'puppeteer'; import * as fs from "fs"; import {createCsvFromJson, createCsvFromJsonArray} from "./flatten-data"; (async () => { const browser = await puppeteer.launch({headless: false, devtools: true}); const page = await browser.newPage(); await page.goto('https://disneycruise.disney.go.com/cruises-destinations/list/#september-2023,october-2023,january-2024,february-2024,march-2024,april-2024,may-2024,alaska-cruises,bahamas-cruises,bermuda-cruises,canada-cruises,caribbean-cruises,mexico-cruises,pacific-coast-cruises,5-to-6'); // Set screen size await page.setViewport({width: 1519, height: 1024}); // Type into search box // await page.type('.search-box__input', 'automate beyond recorder'); // Wait and click on first result const showDatesButtonSelector = '.product-card-footer-wrapper__btn'; await page.waitForSelector(showDatesButtonSelector); // await page.click(showDatesButtonSelector); // Scroll down // should scroll down until the button count stops going up let buttonCountChanged = true; let buttonCount = 0; let buttonElements = await page.$$(showDatesButtonSelector); buttonCount = buttonElements.length; // while (buttonCountChanged) { // await page.evaluate(() => { // window.scrollBy(0, window.innerHeight) // return Promise.resolve(); // }); // buttonElements = await page.$$(showDatesButtonSelector); // if (buttonElements.length > buttonCount) { // buttonCount = buttonElements.length; // } else { // buttonCountChanged = false; // } // } // // for (let i = 0; i < buttonElements.length; i++) { // console.log(buttonElements[i]); // await buttonElements[i].click(); // // await page.waitForSelector('wdpr-price'); // // const price = await page.$eval('wdpr-price', (el) => el.textContent); // // console.log('price', price); // } const data = await getAvailableProducts(page); // console.log('data'); // console.log(JSON.stringify(data, null, ' ')); fs.writeFileSync('products.json', JSON.stringify(data, null, ' ')); createCsvFromJsonArray(data.products, 'products.csv'); let availableSailings = await Promise.all(data.products.map(async (product: { productId: string; itineraries: { itineraryId: string; }[]; }) => { return getAvailableSailings(page, product.productId, product.itineraries[0].itineraryId).then(({sailings}): any[] => { return sailings; }); })).then((results) => { let sailings: any[] = []; results.forEach((result) => { sailings = sailings.concat(result); }); return sailings; }); console.log('availableSailings'); console.log(JSON.stringify(availableSailings, null, ' ')); fs.writeFileSync('sailings.json', JSON.stringify(availableSailings, null, ' ')); createCsvFromJsonArray(availableSailings, 'sailings.csv'); console.log('buttonCount', buttonCount); // // Locate the full title with a unique string // const textSelector = await page.waitForSelector( // 'text/Customize and automate' // ); // // @ts-ignore // const fullTitle = await textSelector.evaluate(el => el.textContent); // // // Print the full title // console.log('The title of this blog post is "%s".', fullTitle); await browser.close(); })(); // function getButtonCount(page: Page) { // return page.$$('.product-card-footer-wrapper__btn').then((buttonElements) => { // return buttonElements.length; // }); // } function getAvailableProducts(page: Page): Promise { return page.evaluate(() => { return getProductsPage(1).then((data) => { let pageCount = data.totalPages; let promises = []; for (let i = 2; i <= pageCount; i++) { promises.push(getProductsPage(i)); } return Promise.all(promises).then((results) => { results.forEach((result) => { data.products = data.products.concat(result.products); }); return data; }); }); function getProductsPage(pageCount = 1): Promise { return fetch('/dcl-apps-productavail-vas/available-products/', { body: JSON.stringify({ "currency": "USD", "filters": ["2023-09;filterType=date", "2023-10;filterType=date", "2024-01;filterType=date", "2024-02;filterType=date", "2024-03;filterType=date", "2024-04;filterType=date", "2024-05;filterType=date", "ALASKA;filterType=destination", "BAHAMAS;filterType=destination", "BERMUDA;filterType=destination", "CANADA;filterType=destination", "CARIBBEAN;filterType=destination", "MEXICAN RIVIERA;filterType=destination", "CALIFORNIA COAST;filterType=destination", "5-6;filterType=night"], "partyMix": [{ "accessible": false, "adultCount": 2, "childCount": 2, "nonAdultAges": [{"age": 7, "ageUnit": "YEAR"}, {"age": 9, "ageUnit": "YEAR"}], "partyMixId": "0" }], "region": "INTL", "storeId": "DCL", "affiliations": [], "page": pageCount, "pageHistory": false, "includeAdvancedBookingPrices": true, "exploreMorePage": 1, "exploreMorePageHistory": false }), headers: { 'Content-Type': 'application/json' }, method: 'POST' }).then((response) => { return response.json(); }).then((data) => { return data; }); } }); } async function getAvailableSailings(page: Page, productID: string, itineraryId: string): Promise { return await page.evaluate((productID, itineraryId) => { return fetch('/dcl-apps-productavail-vas/available-sailings/', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ "currency": "USD", "filters": [ "2023-09;filterType=date", "2023-10;filterType=date", "2024-01;filterType=date", "2024-02;filterType=date", "2024-03;filterType=date", "2024-04;filterType=date", "2024-05;filterType=date", "ALASKA;filterType=destination", "BAHAMAS;filterType=destination", "BERMUDA;filterType=destination", "CANADA;filterType=destination", "CARIBBEAN;filterType=destination", "MEXICAN RIVIERA;filterType=destination", "CALIFORNIA COAST;filterType=destination", "5-6;filterType=night" ], "partyMix": [ { "accessible": false, "adultCount": 2, "childCount": 2, "nonAdultAges": [ { "age": 7, "ageUnit": "YEAR" }, { "age": 9, "ageUnit": "YEAR" } ], "partyMixId": "0" } ], "region": "INTL", "storeId": "DCL", "affiliations": [], "itineraryId": itineraryId, "productId": productID, "includeAdvancedBookingPrices": true }) }).then((response) => { return response.json(); }).then((data) => { return data; }); }, productID, itineraryId); }