initial commit - get basic sailing info
This commit is contained in:
201
index.ts
Normal file
201
index.ts
Normal file
@ -0,0 +1,201 @@
|
||||
import puppeteer, {Page} from 'puppeteer';
|
||||
import * as fs from "fs";
|
||||
import {createCsvFromJson, createCsvFromJsonArray} from "./flatten-data";
|
||||
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({headless: false, devtools: true});
|
||||
const page = await browser.newPage();
|
||||
|
||||
await page.goto('https://disneycruise.disney.go.com/cruises-destinations/list/#september-2023,october-2023,january-2024,february-2024,march-2024,april-2024,may-2024,alaska-cruises,bahamas-cruises,bermuda-cruises,canada-cruises,caribbean-cruises,mexico-cruises,pacific-coast-cruises,5-to-6');
|
||||
|
||||
// Set screen size
|
||||
await page.setViewport({width: 1519, height: 1024});
|
||||
|
||||
// Type into search box
|
||||
// await page.type('.search-box__input', 'automate beyond recorder');
|
||||
|
||||
// Wait and click on first result
|
||||
const showDatesButtonSelector = '.product-card-footer-wrapper__btn';
|
||||
await page.waitForSelector(showDatesButtonSelector);
|
||||
// await page.click(showDatesButtonSelector);
|
||||
|
||||
// Scroll down
|
||||
// should scroll down until the button count stops going up
|
||||
let buttonCountChanged = true;
|
||||
let buttonCount = 0;
|
||||
let buttonElements = await page.$$(showDatesButtonSelector);
|
||||
buttonCount = buttonElements.length;
|
||||
// while (buttonCountChanged) {
|
||||
// await page.evaluate(() => {
|
||||
// window.scrollBy(0, window.innerHeight)
|
||||
// return Promise.resolve();
|
||||
// });
|
||||
// buttonElements = await page.$$(showDatesButtonSelector);
|
||||
// if (buttonElements.length > buttonCount) {
|
||||
// buttonCount = buttonElements.length;
|
||||
// } else {
|
||||
// buttonCountChanged = false;
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// for (let i = 0; i < buttonElements.length; i++) {
|
||||
// console.log(buttonElements[i]);
|
||||
// await buttonElements[i].click();
|
||||
// // await page.waitForSelector('wdpr-price');
|
||||
// // const price = await page.$eval('wdpr-price', (el) => el.textContent);
|
||||
// // console.log('price', price);
|
||||
// }
|
||||
|
||||
const data = await getAvailableProducts(page);
|
||||
|
||||
// console.log('data');
|
||||
// console.log(JSON.stringify(data, null, ' '));
|
||||
fs.writeFileSync('products.json', JSON.stringify(data, null, ' '));
|
||||
createCsvFromJsonArray(data.products, 'products.csv');
|
||||
|
||||
let availableSailings = await Promise.all(data.products.map(async (product: { productId: string; itineraries: { itineraryId: string; }[]; }) => {
|
||||
return getAvailableSailings(page, product.productId, product.itineraries[0].itineraryId).then(({sailings}): any[] => {
|
||||
return sailings;
|
||||
});
|
||||
})).then((results) => {
|
||||
let sailings: any[] = [];
|
||||
results.forEach((result) => {
|
||||
sailings = sailings.concat(result);
|
||||
});
|
||||
return sailings;
|
||||
});
|
||||
|
||||
console.log('availableSailings');
|
||||
console.log(JSON.stringify(availableSailings, null, ' '));
|
||||
fs.writeFileSync('sailings.json', JSON.stringify(availableSailings, null, ' '));
|
||||
createCsvFromJsonArray(availableSailings, 'sailings.csv');
|
||||
|
||||
console.log('buttonCount', buttonCount);
|
||||
|
||||
// // Locate the full title with a unique string
|
||||
// const textSelector = await page.waitForSelector(
|
||||
// 'text/Customize and automate'
|
||||
// );
|
||||
// // @ts-ignore
|
||||
// const fullTitle = await textSelector.evaluate(el => el.textContent);
|
||||
//
|
||||
// // Print the full title
|
||||
// console.log('The title of this blog post is "%s".', fullTitle);
|
||||
|
||||
await browser.close();
|
||||
})();
|
||||
|
||||
// function getButtonCount(page: Page) {
|
||||
// return page.$$('.product-card-footer-wrapper__btn').then((buttonElements) => {
|
||||
// return buttonElements.length;
|
||||
// });
|
||||
// }
|
||||
|
||||
function getAvailableProducts(page: Page): Promise<any> {
|
||||
return page.evaluate(() => {
|
||||
return getProductsPage(1).then((data) => {
|
||||
let pageCount = data.totalPages;
|
||||
let promises = [];
|
||||
for (let i = 2; i <= pageCount; i++) {
|
||||
promises.push(getProductsPage(i));
|
||||
}
|
||||
return Promise.all(promises).then((results) => {
|
||||
results.forEach((result) => {
|
||||
data.products = data.products.concat(result.products);
|
||||
});
|
||||
return data;
|
||||
});
|
||||
});
|
||||
|
||||
function getProductsPage(pageCount = 1): Promise<any> {
|
||||
return fetch('/dcl-apps-productavail-vas/available-products/', {
|
||||
body: JSON.stringify({
|
||||
"currency": "USD",
|
||||
"filters": ["2023-09;filterType=date", "2023-10;filterType=date", "2024-01;filterType=date", "2024-02;filterType=date", "2024-03;filterType=date", "2024-04;filterType=date", "2024-05;filterType=date", "ALASKA;filterType=destination", "BAHAMAS;filterType=destination", "BERMUDA;filterType=destination", "CANADA;filterType=destination", "CARIBBEAN;filterType=destination", "MEXICAN RIVIERA;filterType=destination", "CALIFORNIA COAST;filterType=destination", "5-6;filterType=night"],
|
||||
"partyMix": [{
|
||||
"accessible": false,
|
||||
"adultCount": 2,
|
||||
"childCount": 2,
|
||||
"nonAdultAges": [{"age": 7, "ageUnit": "YEAR"}, {"age": 9, "ageUnit": "YEAR"}],
|
||||
"partyMixId": "0"
|
||||
}],
|
||||
"region": "INTL",
|
||||
"storeId": "DCL",
|
||||
"affiliations": [],
|
||||
"page": pageCount,
|
||||
"pageHistory": false,
|
||||
"includeAdvancedBookingPrices": true,
|
||||
"exploreMorePage": 1,
|
||||
"exploreMorePageHistory": false
|
||||
}),
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
method: 'POST'
|
||||
}).then((response) => {
|
||||
return response.json();
|
||||
}).then((data) => {
|
||||
return data;
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function getAvailableSailings(page: Page, productID: string, itineraryId: string): Promise<any> {
|
||||
return await page.evaluate((productID, itineraryId) => {
|
||||
return fetch('/dcl-apps-productavail-vas/available-sailings/', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
"currency": "USD",
|
||||
"filters": [
|
||||
"2023-09;filterType=date",
|
||||
"2023-10;filterType=date",
|
||||
"2024-01;filterType=date",
|
||||
"2024-02;filterType=date",
|
||||
"2024-03;filterType=date",
|
||||
"2024-04;filterType=date",
|
||||
"2024-05;filterType=date",
|
||||
"ALASKA;filterType=destination",
|
||||
"BAHAMAS;filterType=destination",
|
||||
"BERMUDA;filterType=destination",
|
||||
"CANADA;filterType=destination",
|
||||
"CARIBBEAN;filterType=destination",
|
||||
"MEXICAN RIVIERA;filterType=destination",
|
||||
"CALIFORNIA COAST;filterType=destination",
|
||||
"5-6;filterType=night"
|
||||
],
|
||||
"partyMix": [
|
||||
{
|
||||
"accessible": false,
|
||||
"adultCount": 2,
|
||||
"childCount": 2,
|
||||
"nonAdultAges": [
|
||||
{
|
||||
"age": 7,
|
||||
"ageUnit": "YEAR"
|
||||
},
|
||||
{
|
||||
"age": 9,
|
||||
"ageUnit": "YEAR"
|
||||
}
|
||||
],
|
||||
"partyMixId": "0"
|
||||
}
|
||||
],
|
||||
"region": "INTL",
|
||||
"storeId": "DCL",
|
||||
"affiliations": [],
|
||||
"itineraryId": itineraryId,
|
||||
"productId": productID,
|
||||
"includeAdvancedBookingPrices": true
|
||||
})
|
||||
}).then((response) => {
|
||||
return response.json();
|
||||
}).then((data) => {
|
||||
return data;
|
||||
});
|
||||
}, productID, itineraryId);
|
||||
}
|
||||
Reference in New Issue
Block a user