initial commit - get basic sailing info

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

.idea/disney-cruise-prices.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="Go" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.idea/copilot/chatSessions" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/disney-cruise-prices.iml" filepath="$PROJECT_DIR$/.idea/disney-cruise-prices.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

flatten-data.ts

@@ -0,0 +1,82 @@
+import * as fs from 'fs';
+import * as json2csv from 'json2csv';
+
+interface FlattenData {
+    [key: string]: string | number | FlattenData;
+}
+
+function flattenJson(data: any, parentKey = '', separator = '.'): FlattenData {
+    const flattenedData: FlattenData = {};
+    for (const key in data) {
+        if (data.hasOwnProperty(key)) {
+            const newKey = parentKey ? `${parentKey}${separator}${key}` : key;
+            if (typeof data[key] === 'object' && !Array.isArray(data[key])) {
+                const nestedData = flattenJson(data[key], newKey, separator);
+                Object.assign(flattenedData, nestedData);
+            } else {
+                flattenedData[newKey] = data[key];
+            }
+        }
+    }
+    return flattenedData;
+}
+
+export function createCsvFromJson(jsonData: string, csvFilePath: string): void {
+    // Load JSON data
+    const data = JSON.parse(jsonData);
+
+    // Flatten JSON data
+    const flattenedData = flattenJson(data);
+
+    // Get headers from flattened data
+    const headers = Object.keys(flattenedData);
+
+    // Convert flattened data to array of objects for json2csv
+    const rows = [flattenedData];
+
+    // Create CSV file
+    const csv = json2csv.parse(rows, { fields: headers });
+    fs.writeFileSync(csvFilePath, csv);
+    console.log(`CSV file created at: ${csvFilePath}`);
+}
+
+export function createCsvFromJsonArray(jsonDataArray: any[], csvFilePath: string): void {
+    // Flatten JSON data in array
+    const flattenedDataArray = jsonDataArray.map((data) => flattenJson(data));
+
+    // Get headers from flattened data
+    const headers = Array.from(
+        new Set(flattenedDataArray.flatMap((data) => Object.keys(data)))
+    );
+
+    // Convert flattened data to array of objects for json2csv
+    const rows = flattenedDataArray.map((data) => headers.reduce((obj, key) => {
+        // @ts-ignore
+        obj[key] = data[key] || '';
+        return obj;
+    }, {}));
+
+    // Create CSV file
+    const csv = json2csv.parse(rows, { fields: headers });
+    fs.writeFileSync(csvFilePath, csv);
+    console.log(`CSV file created at: ${csvFilePath}`);
+}
+
+// Example usage
+// const jsonData = `{
+//   "name": "John",
+//   "age": 30,
+//   "address": {
+//     "street": "123 Main St",
+//     "city": "New York",
+//     "state": "NY",
+//     "zip": "10001"
+//   },
+//   "phone": {
+//     "home": "123-456-7890",
+//     "work": "987-654-3210"
+//   }
+// }`;
+
+// const csvFilePath = 'output.csv';
+// createCsvFromJson(jsonData, csvFilePath);

go.mod

@@ -0,0 +1,3 @@
+module disney-cruise-prices
+
+go 1.19

index.ts

@@ -0,0 +1,201 @@
+import puppeteer, {Page} from 'puppeteer';
+import * as fs from "fs";
+import {createCsvFromJson, createCsvFromJsonArray} from "./flatten-data";
+
+(async () => {
+    const browser = await puppeteer.launch({headless: false, devtools: true});
+    const page = await browser.newPage();
+
+    await page.goto('https://disneycruise.disney.go.com/cruises-destinations/list/#september-2023,october-2023,january-2024,february-2024,march-2024,april-2024,may-2024,alaska-cruises,bahamas-cruises,bermuda-cruises,canada-cruises,caribbean-cruises,mexico-cruises,pacific-coast-cruises,5-to-6');
+
+    // Set screen size
+    await page.setViewport({width: 1519, height: 1024});
+
+    // Type into search box
+    // await page.type('.search-box__input', 'automate beyond recorder');
+
+    // Wait and click on first result
+    const showDatesButtonSelector = '.product-card-footer-wrapper__btn';
+    await page.waitForSelector(showDatesButtonSelector);
+    // await page.click(showDatesButtonSelector);
+
+    // Scroll down
+    // should scroll down until the button count stops going up
+    let buttonCountChanged = true;
+    let buttonCount = 0;
+    let buttonElements = await page.$$(showDatesButtonSelector);
+    buttonCount = buttonElements.length;
+    // while (buttonCountChanged) {
+    //     await page.evaluate(() => {
+    //         window.scrollBy(0, window.innerHeight)
+    //         return Promise.resolve();
+    //     });
+    //     buttonElements = await page.$$(showDatesButtonSelector);
+    //     if (buttonElements.length > buttonCount) {
+    //         buttonCount = buttonElements.length;
+    //     } else {
+    //         buttonCountChanged = false;
+    //     }
+    // }
+    //
+    // for (let i = 0; i < buttonElements.length; i++) {
+    //     console.log(buttonElements[i]);
+    //     await buttonElements[i].click();
+    //     // await page.waitForSelector('wdpr-price');
+    //     // const price = await page.$eval('wdpr-price', (el) => el.textContent);
+    //     // console.log('price', price);
+    // }
+
+    const data = await getAvailableProducts(page);
+
+    // console.log('data');
+    // console.log(JSON.stringify(data, null, ' '));
+    fs.writeFileSync('products.json', JSON.stringify(data, null, ' '));
+    createCsvFromJsonArray(data.products, 'products.csv');
+
+    let availableSailings = await Promise.all(data.products.map(async (product: { productId: string; itineraries: { itineraryId: string; }[]; }) => {
+        return getAvailableSailings(page, product.productId, product.itineraries[0].itineraryId).then(({sailings}): any[] => {
+          return sailings;
+      });
+    })).then((results) => {
+        let sailings: any[] = [];
+        results.forEach((result) => {
+            sailings = sailings.concat(result);
+        });
+        return sailings;
+    });
+
+    console.log('availableSailings');
+    console.log(JSON.stringify(availableSailings, null, ' '));
+    fs.writeFileSync('sailings.json', JSON.stringify(availableSailings, null, ' '));
+    createCsvFromJsonArray(availableSailings, 'sailings.csv');
+
+    console.log('buttonCount', buttonCount);
+
+    // // Locate the full title with a unique string
+    // const textSelector = await page.waitForSelector(
+    //     'text/Customize and automate'
+    // );
+    // // @ts-ignore
+    // const fullTitle = await textSelector.evaluate(el => el.textContent);
+    //
+    // // Print the full title
+    // console.log('The title of this blog post is "%s".', fullTitle);
+
+    await browser.close();
+})();
+
+// function getButtonCount(page: Page) {
+//     return page.$$('.product-card-footer-wrapper__btn').then((buttonElements) => {
+//         return buttonElements.length;
+//     });
+// }
+
+function getAvailableProducts(page: Page): Promise<any> {
+    return page.evaluate(() => {
+        return getProductsPage(1).then((data) => {
+            let pageCount = data.totalPages;
+            let promises = [];
+            for (let i = 2; i <= pageCount; i++) {
+                promises.push(getProductsPage(i));
+            }
+            return Promise.all(promises).then((results) => {
+                results.forEach((result) => {
+                    data.products = data.products.concat(result.products);
+                });
+                return data;
+            });
+        });
+
+        function getProductsPage(pageCount = 1): Promise<any> {
+            return fetch('/dcl-apps-productavail-vas/available-products/', {
+                body: JSON.stringify({
+                    "currency": "USD",
+                    "filters": ["2023-09;filterType=date", "2023-10;filterType=date", "2024-01;filterType=date", "2024-02;filterType=date", "2024-03;filterType=date", "2024-04;filterType=date", "2024-05;filterType=date", "ALASKA;filterType=destination", "BAHAMAS;filterType=destination", "BERMUDA;filterType=destination", "CANADA;filterType=destination", "CARIBBEAN;filterType=destination", "MEXICAN RIVIERA;filterType=destination", "CALIFORNIA COAST;filterType=destination", "5-6;filterType=night"],
+                    "partyMix": [{
+                        "accessible": false,
+                        "adultCount": 2,
+                        "childCount": 2,
+                        "nonAdultAges": [{"age": 7, "ageUnit": "YEAR"}, {"age": 9, "ageUnit": "YEAR"}],
+                        "partyMixId": "0"
+                    }],
+                    "region": "INTL",
+                    "storeId": "DCL",
+                    "affiliations": [],
+                    "page": pageCount,
+                    "pageHistory": false,
+                    "includeAdvancedBookingPrices": true,
+                    "exploreMorePage": 1,
+                    "exploreMorePageHistory": false
+                }),
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                method: 'POST'
+            }).then((response) => {
+                return response.json();
+            }).then((data) => {
+                return data;
+            });
+        }
+    });
+}
+
+async function getAvailableSailings(page: Page, productID: string, itineraryId: string): Promise<any> {
+    return await page.evaluate((productID, itineraryId) => {
+        return fetch('/dcl-apps-productavail-vas/available-sailings/', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                "currency": "USD",
+                "filters": [
+                    "2023-09;filterType=date",
+                    "2023-10;filterType=date",
+                    "2024-01;filterType=date",
+                    "2024-02;filterType=date",
+                    "2024-03;filterType=date",
+                    "2024-04;filterType=date",
+                    "2024-05;filterType=date",
+                    "ALASKA;filterType=destination",
+                    "BAHAMAS;filterType=destination",
+                    "BERMUDA;filterType=destination",
+                    "CANADA;filterType=destination",
+                    "CARIBBEAN;filterType=destination",
+                    "MEXICAN RIVIERA;filterType=destination",
+                    "CALIFORNIA COAST;filterType=destination",
+                    "5-6;filterType=night"
+                ],
+                "partyMix": [
+                    {
+                        "accessible": false,
+                        "adultCount": 2,
+                        "childCount": 2,
+                        "nonAdultAges": [
+                            {
+                                "age": 7,
+                                "ageUnit": "YEAR"
+                            },
+                            {
+                                "age": 9,
+                                "ageUnit": "YEAR"
+                            }
+                        ],
+                        "partyMixId": "0"
+                    }
+                ],
+                "region": "INTL",
+                "storeId": "DCL",
+                "affiliations": [],
+                "itineraryId": itineraryId,
+                "productId": productID,
+                "includeAdvancedBookingPrices": true
+            })
+        }).then((response) => {
+            return response.json();
+        }).then((data) => {
+            return data;
+        });
+    }, productID, itineraryId);
+}

output.csv

@@ -0,0 +1,2 @@
+"name","age","address.street","address.city","address.state","address.zip","phone.home","phone.work"
+"John",30,"123 Main St","New York","NY","10001","123-456-7890","987-654-3210"

package.json

@@ -0,0 +1,19 @@
+{
+  "name": "disney-cruise-prices",
+  "version": "1.0.0",
+  "description": "Collects the prices available on https://disneycruise.disney.go.com/",
+  "main": "dist/index.js",
+  "scripts": {
+    "start": "node dist/index.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Mason Payne <mason@masonitestudios.com>",
+  "license": "MIT",
+  "dependencies": {
+    "json2csv": "^6.0.0-alpha.2",
+    "puppeteer": "^22.6.1"
+  },
+  "devDependencies": {
+    "@types/json2csv": "^5.0.3"
+  }
+}

tsconfig.json

@@ -0,0 +1,100 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Enable incremental compilation */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./",                          /* Specify the folder for .tsbuildinfo incremental compilation files. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2021",                                     /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */
+    // "reactNamespace": "",                             /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like `./node_modules/@types`. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files */
+    // "noResolve": true,                                /* Disallow `import`s, `require`s or `<reference>`s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */
+    "outDir": "./dist",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have `@internal` in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like `__extends` in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing `const enum` declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied `any` type.. */
+    // "strictNullChecks": true,                         /* When type checking, take into account `null` and `undefined`. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when `this` is given the type `any`. */
+    // "useUnknownInCatchVariables": true,               /* Type catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when a local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Include 'undefined' in index signature results */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  }
+}