diff --git a/README.md b/README.md index 0f9b7681a..d03cbc744 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,4 @@ The workspaces within this monorepo are: 3. **api-client:** A typescript client responsible for providing a streamlined and typed interface to interact with our API. The frontend uses this client to send request to our API. -4. **scrapers-v2**: A typescript web scraper repsonsible for scraping [Northeastern's course catalog website](https://catalog.northeastern.edu/undergraduate/) for the latest graduation requirements for majors at Northeastern. - -5. **common**: All common types and logic used by our frontend, api and scrapers. +4. **common**: All common types and logic used by our frontend, api and scrapers. diff --git a/packages/scrapers-v2/package.json b/packages/scrapers-v2/package.json deleted file mode 100644 index 5370eb45c..000000000 --- a/packages/scrapers-v2/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@graduate/scrapers-v2", - "version": "1.0.0", - "description": "scrapers used to generate essential data such as majors and plans", - "private": true, - "dependencies": { - "@graduate/common": "*", - "cheerio": "1.0.0-rc.3", - "ts-node": "^10.8.0", - "undici": "^5.8.2" - }, - "devDependencies": { - "@types/cheerio": "0.22.13", - "@types/jest": "^27", - "@types/node": "^16", - "jest": "^27.5.1", - "ts-jest": "^27.1.3", - "typescript": "^4.6.2" - }, - "scripts": { - "test": "jest", - "scrape": "ts-node ./src/main.ts" - }, - "jest": { - "moduleFileExtensions": [ - "ts", - "tsx", - "js" - ], - "transform": { - "\\.(ts|tsx)$": "ts-jest" - }, - "testEnvironment": "node" - } -} diff --git a/packages/scrapers-v2/src/classify/README.md b/packages/scrapers-v2/src/classify/README.md deleted file mode 100644 index 8fb7b48a9..000000000 --- a/packages/scrapers-v2/src/classify/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Classify - -This folder contains code for the catalog entry classification stage of the scraper pipeline. In this stage, we classify catalog entries and annotate them with their type. - -## Design - -This strategy was mostly born through testing against the entire catalog, made easier by the existence of the `StatsLogger` class and the runtime. - -## Classification Strategy - -We try three different ways of categorizing an entry, in order. The first that returns a non-unknown type is returned. Here is a simple outline representing the strategy priority (in order): - -1. if the entry name contains a comma and the name after comma matches one of the following: - - is `minor` -> minor - - starts with `bs` -> major - - starts with `b` and ends with `a` -> major - - starts with `ba` -> major - - else try strategy 2 -2. has tabs, and second tab text is: - - `program requirements` -> major - - caveat: some minors have `program requirements`, but all the ones that do should be categorized by strategy 1. - - `minor requirements` -> minor - - `concentration requirements` -> concentration - - else try strategy 3 -3. if exactly 1 element exist with id ending in `programrequirementstext` and matches one of the following: - - begins `program` -> major - - begins `minor` -> minor - - begins `concentration` -> concentration - - else return unknown - -### How it does - -The vast majority of entries should be able to be categorized by strategy 1. There are ~17 that require strategy two (for example, the business concentrations), and finally there is only 1 that requires strategy 3 (the global business strategy business concentration, for some reason). - -Finally, the only unclassified entries (8 of them, at the time of writing) are the accelerated degree programs, and the first year engineering program. - -### Examples - -- categorized by name: - - [comp sci - major](https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/bscs/) - - [business - minor](https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/accounting-advisory-services-minor/) - - [behavioral neuroscience - major](https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-philosophy-bs/) - - this one is special: the "BS" is capitalized - - [math and polysci - major](https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-political-science-bs/) - - also special: has no tabs! thankfully good ol' strategy 1 comes first -- categorized by tabs: - - no commas in the title: - - [business analytics - concentration](https://catalog.northeastern.edu/undergraduate/business/concentrations/business-analytics/) - - [fintech - concentration](https://catalog.northeastern.edu/undergraduate/business/concentrations/fintech/) - - has a comma, but ending doesn't match one of the expected: - - [pharmd - major](https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/pharmacy-pharmd/) -- categorized by containerId: - - https://catalog.northeastern.edu/undergraduate/business/concentrations/global-business-strategy/ - - has no comma in the title, AND ALSO has no tabs! wowzers. good thing we have backup strategy # 3 -- unknowns - - [cs accelerated degree program](https://catalog.northeastern.edu/undergraduate/computer-information-science/accelerated-bachelor-graduate-degree-programs/) - - [first year engineering](https://catalog.northeastern.edu/undergraduate/engineering/first-year-engineering/) diff --git a/packages/scrapers-v2/src/classify/classify.ts b/packages/scrapers-v2/src/classify/classify.ts deleted file mode 100644 index f2c3c9a42..000000000 --- a/packages/scrapers-v2/src/classify/classify.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { ensureLengthAtLeast, loadHTML, parseText } from "../utils"; -import { CatalogEntryType, TypedCatalogEntry } from "./types"; - -export const addTypeToUrl = async (url: URL): Promise => { - const type = getUrlType(await loadHTML(url.href)); - return { url, type }; -}; - -// try to get the type from each strategy, in order (name, tabs, container) -const getUrlType = ($: CheerioStatic) => { - const typeFromName = getTypeFromNameEnding($); - if (typeFromName !== CatalogEntryType.Unknown) { - return typeFromName; - } - const typeFromTabs = getTypeFromTabs($); - if (typeFromTabs !== CatalogEntryType.Unknown) { - return typeFromTabs; - } - const typeFromContainer = getTypeFromContainer($); - if (typeFromContainer !== CatalogEntryType.Unknown) { - return typeFromContainer; - } - return CatalogEntryType.Unknown; -}; - -const getTypeFromNameEnding = ($: CheerioStatic) => { - const nameEnding = getNameEnding($); - if (nameEnding && isMajorEnding(nameEnding)) { - return CatalogEntryType.Major; - } else if (nameEnding?.toLowerCase() === "minor") { - return CatalogEntryType.Minor; - } - return CatalogEntryType.Unknown; -}; - -const getTypeFromTabs = ($: CheerioStatic) => { - const tabsContainer = $("#contentarea #tabs"); - if (tabsContainer.length === 0) { - return CatalogEntryType.Unknown; - } else if (tabsContainer.length === 1) { - return getTypeFromTabText(tabsContainer.find("ul > li").toArray().map($)); - } - throw new Error( - `Expected 0 or 1 tab container, but found ${tabsContainer.length}.` - ); -}; - -const getTypeFromContainer = ($: CheerioStatic) => { - const container = $("[id$='requirementstextcontainer']"); - if (container.length === 1) { - const id = container.attr("id"); - if (id === "minorrequirementstextcontainer") { - return CatalogEntryType.Minor; - } else if (id === "programrequirementstextcontainer") { - return CatalogEntryType.Major; - } else if (id === "concentrationrequirementstextcontainer") { - return CatalogEntryType.Concentration; - } - } - return CatalogEntryType.Unknown; -}; - -const getNameEnding = ($: CheerioStatic) => { - const name = parseText($("#site-title").find("h1")); - const degree = name.lastIndexOf(","); - if (degree !== -1) { - // assume ", " - return name - .substring(degree + 2) - .toLowerCase() - .trim(); - } - return null; -}; - -const isMajorEnding = (ending: string) => { - const bs = ending.substring(0, 2) === "bs"; - const ba = ending[0] + ending[ending.length - 1] === "ba"; - // very uncommon ending - const ba1 = ending.substring(0, 2) === "ba"; - return bs || ba || ba1; -}; - -const getTypeFromTabText = (tabs: Cheerio[]) => { - // most entries have 3 tabs, but some have 2 or rarely 4 - const [, tab] = ensureLengthAtLeast(2, tabs); - const tabText = parseText(tab); - - switch (tabText.toLowerCase()) { - case "minor requirements": - return CatalogEntryType.Minor; - case "concentration requirements": - return CatalogEntryType.Concentration; - case "program requirements": - return CatalogEntryType.Major; - default: - return CatalogEntryType.Unknown; - } -}; diff --git a/packages/scrapers-v2/src/classify/types.ts b/packages/scrapers-v2/src/classify/types.ts deleted file mode 100644 index 7cda48fef..000000000 --- a/packages/scrapers-v2/src/classify/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export enum CatalogEntryType { - Major = "Major", - Minor = "Minor", - Concentration = "Concentration", - Unknown = "Unknown", -} - -export type TypedCatalogEntry = { url: URL; type: CatalogEntryType }; diff --git a/packages/scrapers-v2/src/constants.ts b/packages/scrapers-v2/src/constants.ts deleted file mode 100644 index 384f7f0fa..000000000 --- a/packages/scrapers-v2/src/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const BASE_URL = "https://catalog.northeastern.edu"; diff --git a/packages/scrapers-v2/src/main.ts b/packages/scrapers-v2/src/main.ts deleted file mode 100644 index bc854ed80..000000000 --- a/packages/scrapers-v2/src/main.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { runPipeline } from "./runtime/pipeline"; - -runPipeline(2021, 2022); diff --git a/packages/scrapers-v2/src/runtime/README.md b/packages/scrapers-v2/src/runtime/README.md deleted file mode 100644 index d02379390..000000000 --- a/packages/scrapers-v2/src/runtime/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Runtime/pipeline - -This folder contains code to glue all the individual stages of the scraper pipeline together and make it run. This also includes code for error handling and logging, to make the scrapers easier to debug. - -## Design constraints - -The pipeline was constructed with several goals in mind. the full planning doc can be found [here](https://www.notion.so/sandboxnu/Scraper-Brainstorming-61261181144c4dac82488255cfa34744). - -high level goals: - -1. be confident my changes are not breaking anything -2. be able to test changes on the full catalog -3. be able to hack on compiler with minimal compilers experience - -this manifested itself in running the pipeline for each catalog entry independently (so that one entry can't break the entire pipeline), and also putting a lot of effort in making logging good (diagnostics and aggregate info across entries). - -## High level strategy - -There are roughly three steps to running the pipeline: - -1. scrape URLs to all the entries -2. for each URL, run the pipeline -3. log a summary of what happened - -Each step is described more in detail in the next section, but at a high level, this is all that the entire pipeline does. - -The main entrypoint for the scrapers can be found in `pipeline.ts` in a function called `runPipeline`. - -## Implementation Details - -### step 1: URL scraping - -The first step is the only step that cannot occur for each entry separately: scraping the URLs of all the entries. However, so that a single URL failing does not break the whole scrape, we ignore URLs that fail to fetch. - -Eventually, it may be beneficial to add retry logic for the URLs that fail. - -Also, when making a many requests, it is helpful to utilize TCP socket `keepAlive` property to re-use sockets between requests (can google for more info). To do this, we install an agent. - -### step 2: PIPELINE - -At a high level, we want to take an input and apply a series of functions to it in order. however, at each point, if the function fails, we want to error out, and skip the rest of the functions. - -The way we put the pipeline together is by using Promises: we can very easily apply a series of asynchronous functions using the `.then()` method of a promise. - -#### addPhase - -To do this, we use the `addPhase` function. It essentially takes a function and wraps it in a try/catch so that errors don't break the whole scraper. - -The types for this function are a little hard to read, but these exist just to make sure everything is type-checked properly. - -It also skips the stage if there was already an error earlier on in the pipeline. - -It also allows for labelling stages with a `Phase`, to record what stages the pipeline got to via a trace. - -#### `Pipeline` datatype - -The `Pipeline` datatype is just to give us some handy bookkeeping, namely an identifier (URL to the entry), the aforementioned trace of the phases completed, and the current value in the pipeline as a result (either ok or error). - -### step 3: logging - -there are two steps to logging: first showing progress, so that the developer can know that the scraper is actually running/doing something. this is done via the `logProgress` function. as each individual entry pipeline completes, its status is logged. `.` for success, `-` for filtered out, or a # representing the stage it errored on. - -the second step is logging metrics and aggregations. this is done with the `logResults` method, in tandem with the `StatsLogger` class. - -#### StatsLogger - -the `StatsLogger` class allows for "recording" fields (and errors) to print a breakdown of the different values at the end. - -to record a field, you specify the field label and its value. the number of occurrences of each value is tallied as the field is recorded. - -for example, let's say we want to record the label "status". if we call the method with "filtered" three times, "ok" once, and "error" twice, and then call `logger.print()`, the logger will print out the following (note, will sort field value in order of # of occurrences): - -``` -status: - filtered: 3 - error: 2 - ok: 1 -``` - -to record errors, it's basically a variation of the same thing (record and tally occurrences). the only difference is that we also want to print out the entry IDs (url) associated with each error, so we track that as well. - -#### logResuts - -this function pretty much just uses the StatsLogger to record certain information, and should be straightforward enough to read through. - -## Running - -to run the scraper, cd into the `packages/scrapers-v2` directory and run `yarn scrape`. - -## Future Work - -See the bottom of the notion page for an up-to-date list of future work. - -- save stage output(s) to file -- show diff at each stage -- specify select URLs to run on for faster dev feedback loop -- specify select stages to run for faster dev feedback loop diff --git a/packages/scrapers-v2/src/runtime/axios.ts b/packages/scrapers-v2/src/runtime/axios.ts deleted file mode 100644 index d5f6372b4..000000000 --- a/packages/scrapers-v2/src/runtime/axios.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { getGlobalDispatcher, Pool, setGlobalDispatcher } from "undici"; -import { BASE_URL } from "../constants"; - -/** - * The scrapers (by default) try to make a lot of HTTP requests, and too many at - * once cause some to start failing with weird errors. To fix this, we share TCP - * sockets between requests and utilize `keepAlive` by installing an agent. - */ -export const createAgent = () => { - setGlobalDispatcher( - new Pool(BASE_URL, { - pipelining: 10, - connections: 25, - }) - ); - return () => getGlobalDispatcher().destroy(); -}; diff --git a/packages/scrapers-v2/src/runtime/logger.ts b/packages/scrapers-v2/src/runtime/logger.ts deleted file mode 100644 index 452e8e04d..000000000 --- a/packages/scrapers-v2/src/runtime/logger.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { Pipeline, StageLabel } from "./types"; -import { ResultType } from "@graduate/common"; -import { FilterError } from "./pipeline"; -import { CatalogEntryType } from "../classify/types"; -import { HDocument } from "../tokenize/types"; - -/** - * Logs the progress of the scrape so the developer knows the scraper isn't deadlocked. - * - * As each individual entry pipeline completes, its status is logged. `.` for - * success, `-` for filtered out, or a # representing the stage it errored on. - * - * @param pipelines The in-progress pipelines - */ -export const logProgress = async ( - pipelines: Array>> -) => { - // set handlers to log the result of each pipeline - for (const promise of pipelines) { - promise.then(({ result, trace }) => { - if (result.type === ResultType.Ok) { - process.stdout.write("."); - } else if (result.err[0] instanceof FilterError) { - process.stdout.write("-"); - } else { - process.stdout.write(String(trace.length)); - } - }); - } - - const awaited = await Promise.all(pipelines); - process.stdout.write("\n"); - return awaited; -}; - -/** - * Logs scrape summary information, error messages, and the URLs of the catalog - * entries that errored. - * - * Note: To display fewer stacktraces, the logger will try to aggregate them. - * However, this doesn't always work with async stacktraces, so there might - * sometimes appear two of the same stacktrace. - * - * @param results The completed pipelines - */ -export const logResults = ( - results: Pipeline<{ - tokenized: HDocument; - type: CatalogEntryType; - url: URL; - }>[] -) => { - const stats = new StatsLogger(); - - for (const { result, trace, id } of results) { - if (result.type === ResultType.Ok) { - logOkResult(stats, result, id); - } else { - logErrResult(stats, id, trace, result.err); - } - } - - stats.print(); -}; - -const logOkResult = ( - stats: StatsLogger, - result: { ok: { tokenized: HDocument; type: CatalogEntryType } }, - id: URL -) => { - stats.recordField("status", "ok"); - - // record OK values - const { tokenized, type } = result.ok; - stats.recordField("status", "ok"); - stats.recordField("entry type", type); - if (type === CatalogEntryType.Major && tokenized.programRequiredHours <= 0) { - // only applies to majors, because concentrations and minors don't have hours requirement - stats.recordError(new Error("major with hours <= 0"), id); - } -}; - -const logErrResult = ( - stats: StatsLogger, - id: URL, - trace: StageLabel[], - errors: unknown[] -) => { - // special case the filter error - if (errors[0] instanceof FilterError) { - stats.recordField("status", "filtered"); - stats.recordField("filtered", errors[0].actual); - if (errors[0].actual === CatalogEntryType.Unknown) { - console.log("entry with unknown type:", id.toString()); - } - return; - } - - stats.recordField("status", "error"); - stats.recordField("stage failures", trace[trace.length - 1]); - - for (const err of errors) { - if (err instanceof Error) { - stats.recordError(err, id); - } else { - stats.recordError(new Error(`non-error value: ${err}`), id); - } - } -}; - -/** - * Allows for "recording" fields (and errors) to print a breakdown of the - * different values at the end. - * - * Each field value will be added to a tally. The # of occurrences of each value - * for a field are then displayed when `print()` is called. - * - * Errors are also tallied. Error grouping is done by comparing stacktrace. This - * doesn't quite work for async stacktraces, so sometimes two of the same error - * are displayed separately. - */ -class StatsLogger { - // field -> value -> count - private fields: Record> = {}; - // message -> list -> stacktrace - private errors: Map< - string, - Array<{ err: Error; count: number; annot: string; entryIds: URL[] }> - > = new Map(); - - /** - * Records a field and its value, with the goal of printing the counts for - * each different value the field has at the end. - * - * @param field The name of the field - * @param value The value of the field - */ - recordField(field: string, value: any) { - if (field === "errors") { - throw new Error( - "cannot use 'errors' as a field key, use a different name" - ); - } - this.record(field, value); - } - - /** - * Records an error for a specific entry. Error uniqueness is determined by - * stack trace. This doesn't quite work for async stacktraces, so sometimes - * two of the same error are displayed separately. - * - * @param err The error - * @param entryId The entry URL - */ - recordError(err: Error, entryId: URL) { - const key = err.message ?? "had no stacktrace"; - const storedErrors = this.errors.get(key) ?? []; - for (const stored of storedErrors) { - // if the stacktrace matches, increment the count - if (err.stack === stored.err.stack) { - stored.count += 1; - stored.entryIds.push(entryId); - this.record("errors", stored.annot); - return; - } - } - const id = storedErrors.length === 0 ? "" : ` #${storedErrors.length}`; - const annot = `${err.message}${id}`; - // stacktrace didn't match, so add a new stacktrace entry for this error message - storedErrors.push({ err, count: 1, annot, entryIds: [entryId] }); - this.errors.set(key, storedErrors); - this.record("errors", annot); - } - - /** - * Increments the count for a field, stored in this.fields, or adds it if it - * doesn't exist. Compares values with reference equality - * - * @param field The field name - * @param value The value - */ - private record(field: string, value: any) { - if (!(field in this.fields)) { - this.fields[field] = new Map(); - } - const map = this.fields[field]; - map.set(value, (map.get(value) ?? 0) + 1); - } - - /** - * Prints field and error information. - * - * Prints stacktraces (with URLs) first, then aggregate information. Also - * prints in order of # of occurrences. - */ - print() { - // log errors with stacktraces - const errors = Array.from(this.errors.values()) - .flat() - .sort((a, b) => b.count - a.count); - for (const { err, count, annot, entryIds } of errors) { - console.log(annot, count); - console.error(err); - console.log(entryIds.map((url) => url.toString())); - } - - // log normal metrics (including error aggregates) - for (const [field, map] of Object.entries(this.fields)) { - console.log(field, ":"); - const entries = Array.from(map.entries()).sort((a, b) => b[1] - a[1]); - for (const entry of entries) { - console.log("\t", ...entry); - } - } - } -} diff --git a/packages/scrapers-v2/src/runtime/pipeline.ts b/packages/scrapers-v2/src/runtime/pipeline.ts deleted file mode 100644 index 18e32006a..000000000 --- a/packages/scrapers-v2/src/runtime/pipeline.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { fetchAndTokenizeHTML } from "../tokenize/tokenize"; -import { addTypeToUrl } from "../classify/classify"; -import { scrapeMajorLinks } from "../urls/urls"; -import { CatalogEntryType, TypedCatalogEntry } from "../classify/types"; -import { Err, Ok, ResultType } from "@graduate/common"; -import { Pipeline, StageLabel } from "./types"; -import { createAgent } from "./axios"; -import { logProgress, logResults } from "./logger"; - -/** - * Runs a full scrape of the catalog, logging the results to the console. - * Currently, does nothing with the output. To run from cli, run `yarn scrape` - * in `scrapers-v2` dir. Also see `main.ts`. - */ -export const runPipeline = async (yearStart: number, yearEnd: number) => { - const unregisterAgent = createAgent(); - const { entries, unfinished } = await scrapeMajorLinks(yearStart, yearEnd); - if (unfinished.length > 0) { - console.log("didn't finish searching some entries", ...unfinished); - } - - // can use for debugging logging throughout the stages - // const stats = new StatsLogger(); - const pipelines = entries.map((entry) => { - return createPipeline(entry) - .then(addPhase(StageLabel.Classify, addTypeToUrl)) - .then( - addPhase(StageLabel.Filter, filterEntryType, [ - CatalogEntryType.Minor, - CatalogEntryType.Major, - CatalogEntryType.Concentration, - ]) - ) - .then(addPhase(StageLabel.Tokenize, tokenizeEntry)); - }); - const results = await logProgress(pipelines); - await unregisterAgent(); - logResults(results); -}; - -// convenience constructor for making a pipeline -const createPipeline = (input: URL): Promise> => { - return Promise.resolve({ - id: input, - trace: [], - result: Ok(input), - }); -}; - -/** - * Wraps the provided function with a try/catch so that errors don't break the - * whole scraper. - * - * @param phase The identifier for the stage, to be recorded in pipeline trace. - * @param next The function representing this stage. the first argument of this - * function must be the primary entry input. - * @param args Any additional arguments the stage function requires. - */ -const addPhase = ( - phase: StageLabel, - next: - | ((...args: [Input, ...Args]) => Promise) - | ((...args: [Input, ...Args]) => Output), - ...args: Args -) => { - return async (input: Pipeline): Promise> => { - const { id, trace, result } = input; - if (result.type === ResultType.Err) { - return { id, trace, result }; - } - const newTrace = [...trace, phase]; - try { - const applied = await next(result.ok, ...args); - return { id, trace: newTrace, result: Ok(applied) }; - } catch (e) { - return { id, trace: newTrace, result: Err([e]) }; - } - }; -}; - -const filterEntryType = ( - entry: TypedCatalogEntry, - types: CatalogEntryType[] -) => { - if (types.includes(entry.type)) { - return entry; - } - throw new FilterError(entry.type, types); -}; - -export class FilterError { - actual; - allowed; - - constructor(actual: CatalogEntryType, allowed: CatalogEntryType[]) { - this.actual = actual; - this.allowed = allowed; - } -} - -const tokenizeEntry = async (entry: TypedCatalogEntry) => { - const tokenized = await fetchAndTokenizeHTML(entry.url); - return { ...entry, tokenized }; -}; diff --git a/packages/scrapers-v2/src/runtime/types.ts b/packages/scrapers-v2/src/runtime/types.ts deleted file mode 100644 index 231a50503..000000000 --- a/packages/scrapers-v2/src/runtime/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Result } from "@graduate/common"; - -/** Represents the label for a stage in the scraper pipeline */ -export enum StageLabel { - Classify = "Classify", - Filter = "Filter", - Tokenize = "Tokenize", -} - -/** - * Represents a pipeline value, at some point in the scraper pipeline. Contains - * a trace of the stages that have been run, as well as the resulting value - * (either an error, or OK), and the URL the pipeline was run on. - */ -export type Pipeline = { - id: URL; - trace: StageLabel[]; - result: Result; -}; diff --git a/packages/scrapers-v2/src/tokenize/README.md b/packages/scrapers-v2/src/tokenize/README.md deleted file mode 100644 index ce4ce5c16..000000000 --- a/packages/scrapers-v2/src/tokenize/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Tokenize - -This folder contains code for the **tokenize** (aka lexing) stage of the scraper, which handles converting the HTML of a course catalog page into distinct recognized pieces, known as "tokens". - -If the tokenizer finds a token that it does not recognize, it should explicitly error, to make the devs aware of an unhandled case (so that they can later add support for it). - -## Catalog Page Structure - -For more specific docs on what shape we expect the catalog entry pages to exist in, please see the `types.ts` file. - -At a high level, each catalog page consists of a bunch of `sections`. - -Each section then contains a list of `rows`. A section also has a text descriptor. - -There are a lot of different types of rows. Here's a breakdown: - -- Row - - TextRow - - AreaHeader (description, hour) - - Comment (description, hour) - - Subheader (description, hour) - - CourseRow - - CourseRow (description, subject, hour, classId) - - OrCourseRow (description, subject, hour, classId) - - MultiCourseRow - - AndCourseRow (hour, list of (classId, subject, ...)) - - OrOfAndCourseRow (hour, list of (classId, subject, ...)) - - RangeRow - - boundedRangeRow (hour, subject, startId, endId) - - unboundedRangeRow (hour, subj) - - lowerBoundedRangeRow (hour, subject, startId) - - lowerBoundedRangeRowWithExceptions (hour, subject, startId, exceptionsIds[]) - -### Edge cases - -Some catalog entries do not have tabs, so instead of finding the courses container via the tabs, we just look throughout the page for an element with id ending in `requirementstextcontainer`. - -Some courses are also an OR of ANDs, for example, [this one](https://catalog.northeastern.edu/undergraduate/engineering/bioengineering/bioengineering-biochemistry-bsbioe/#programrequirementstext) - search "phys 1155". In this case, it is an OR of the previous row, but also is an AND, having its own sub-courses. Additionally, rows of this type have no column for hours. - -#### Program required hours - -to find the hours, we look for the **program requirements** header, one of "program requirements", "program requirement", and "program credit requirement". - -following the header, there is paragraphs in number >= 1. Either the first or last paragraph should begin with text "minimum of ..." of " total credits ...", so we look for that. diff --git a/packages/scrapers-v2/src/tokenize/constants.ts b/packages/scrapers-v2/src/tokenize/constants.ts deleted file mode 100644 index 94cb23219..000000000 --- a/packages/scrapers-v2/src/tokenize/constants.ts +++ /dev/null @@ -1,27 +0,0 @@ -// matches subject and courseId, ex: XXXX9999 -export const COURSE_REGEX = /([A-Z]{2,4})\s(\d{4})/g; - -// just subject -export const SUBJECT_REGEX = /([A-Z]{2,4})/g; - -// Range regexes, used to match on text to determine range type -// For full documentation, see notion doc - -// Case 1 -export const RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_1 = - /([A-Z]{2,4})\s(\d{4}) or higher/; - -// Case 2 -export const RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_2 = - /((Select from any)|(Complete \w+)) [A-Z]{2,4} courses? numbered \d{4} or above/; - -// for parsing the text of the above (to pull out course info) -export const RANGE_LOWER_BOUNDED_PARSE = - /([A-Z]{2,4})(( courses? numbered )|(\s))([0-9]{4})/g; - -// Case 3 and 4 -export const RANGE_BOUNDED_MAYBE_EXCEPTIONS = - /([A-Z]{2,4})\s(\d{4})(( to )|(-)|(–))([A-Z]{2,4})\s(\d{4})/; - -// Case 5 and 6 -export const RANGE_UNBOUNDED = /([A-Z]{2,4}, ){2,}and ([A-Z]{2,4})/; diff --git a/packages/scrapers-v2/src/tokenize/tokenize.ts b/packages/scrapers-v2/src/tokenize/tokenize.ts deleted file mode 100644 index c6eba0cf3..000000000 --- a/packages/scrapers-v2/src/tokenize/tokenize.ts +++ /dev/null @@ -1,499 +0,0 @@ -import { assertUnreachable } from "@graduate/common"; -import { - ensureLength, - ensureLengthAtLeast, - loadHTML, - parseText, -} from "../utils"; -import { - COURSE_REGEX, - RANGE_BOUNDED_MAYBE_EXCEPTIONS, - RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_1, - RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_2, - RANGE_LOWER_BOUNDED_PARSE, - RANGE_UNBOUNDED, - SUBJECT_REGEX, -} from "./constants"; -import { - CourseRow, - HDocument, - HRow, - HRowType, - HSection, - MultiCourseRow, - RangeBoundedRow, - RangeLowerBoundedRow, - RangeUnboundedRow, - TextRow, - WithExceptions, -} from "./types"; -import { join } from "path"; -import { BASE_URL } from "../constants"; - -/** - * Fetch html for page and convert into intermediate representation (IR) - * - * @param url The url of the page to tokenize - */ -export const fetchAndTokenizeHTML = async (url: URL): Promise => { - return await tokenizeHTML(await loadHTML(url.href)); -}; - -/** - * Tokenize scraped html into intermediate representation (IR) - * - * @param $ The cheerio static for the page to tokenize - */ -export const tokenizeHTML = async ($: CheerioStatic): Promise => { - const majorName: string = parseText($("#site-title").find("h1")); - const catalogYear: string = parseText($("#edition")).split(" ")[0]; - const yearVersion: number = parseInt(catalogYear.split("-")[0]); - - const requirementsContainer = getRequirementsContainer($); - const sections = await tokenizeSections($, requirementsContainer); - - const programRequiredHours = getProgramRequiredHours( - $, - requirementsContainer - ); - - return { - programRequiredHours, - yearVersion, - majorName, - sections: sections, - }; -}; - -/** - * Retrieves the cheerio container containing the degree requirements. If there - * are no tabs, tries to look for ID ending in 'requirementstextcontainer', - * otherwise tries to find second tab href. - * - * @param $ - */ -const getRequirementsContainer = ($: CheerioStatic) => { - const tabsContainer = $("#contentarea #tabs"); - if (tabsContainer.length === 0) { - // had no tabs, so just look for id ending in "requirementstextcontainer" - const container = $("[id$='requirementstextcontainer']"); - if (container.length === 1) { - return container; - } - throw new Error(`unexpected # of matching ids: ${container.length}`); - } else if (tabsContainer.length === 1) { - const tabsArr = tabsContainer.find("ul > li > a").toArray().map($); - const [, requirementsTab] = ensureLengthAtLeast(2, tabsArr); - const containerId = requirementsTab.attr("href"); - return $(containerId); - } - throw new Error("unable to find a requirementstextcontainer"); -}; - -/** - * Retrieves the # of required course-hours for this degree. Looks for a heading - * with text "program requirements" (ish), and then checks first and last line, - * first and third word, if it matches a number. if so, returns that #, else 0. - * - * @param $ - * @param requirementsContainer - */ -const getProgramRequiredHours = ( - $: CheerioStatic, - requirementsContainer: Cheerio -) => { - const programRequiredHeading = requirementsContainer - .find("h2") - .filter((_, element) => { - const text = parseText($(element)).toLowerCase(); - // "program requirement", "program requirements", or "program credit requirements" - return /program (\w+ )?requirement(s?)/.test(text); - }); - - const nextAll = programRequiredHeading.nextAll().toArray().map($); - if (nextAll.length >= 1) { - for (const next of [nextAll[0], nextAll[nextAll.length - 1]]) { - // keep if matches "minimum of " or "" - // regex matches space characters (\x) and non-breaking space (\xa0) - const parts = parseText(next).split(/[\s\xa0]+/); - // regex matches digits (\d) groups of at least 1 (+) - if (/\d+/.test(parts[0])) { - return Number(parts[0]); - } else if (/\d+/.test(parts[2])) { - return Number(parts[2]); - } - } - } - - return 0; -}; - -/** - * Produces overall HSections for each HTML table in the page - * - * @param $ - * @param requirementsContainer - */ -const tokenizeSections = async ( - $: CheerioStatic, - requirementsContainer: Cheerio -): Promise => { - // use a stack to keep track of the course list title and description - const descriptions: string[] = []; - const courseList: HSection[] = []; - - for (const element of requirementsContainer.children().toArray()) { - if (element.name === "h2" || element.name === "h3") { - // element is h2 or h3 means it's a header text - descriptions.push(parseText($(element))); - } else if ( - element.name === "table" && - element.attribs["class"] === "sc_courselist" - ) { - // class "sc_courselist" signifies that this table is a list of courses - // => parse the table's rows - const tableDesc = descriptions.pop() || ""; - const courseTable = { - description: tableDesc, - entries: tokenizeRows($, element), - }; - courseList.push(courseTable); - } else if ( - // only necessary for business concentrations - element.name === "ul" && - parseText($(element).prev()).includes("concentration") - ) { - // if we encounter an unordered list and preceding element contains text "concentration", - // assume the list is of links for business concentrations. - // only applies to: - // https://catalog.northeastern.edu/undergraduate/business/business-administration-bsba/#programrequirementstext - const links = constructNestedLinks($, element); - const pages = await Promise.all(links.map(loadHTML)); - const containerId = "#concentrationrequirementstextcontainer"; - const concentrations = await Promise.all( - pages.map((concentrationPage) => - tokenizeSections(concentrationPage, concentrationPage(containerId)) - ) - ); - courseList.push(...concentrations.flat()); - } - } - - return courseList; -}; - -/** - * Finds and fetches nested links, for majors with concentration requirements on - * separate pages. - * - * @param $ - * @param element - */ -const constructNestedLinks = ($: CheerioStatic, element: CheerioElement) => { - // TODO: add support to non-current catalogs - return $(element) - .find("li > a") - .toArray() - .map((link) => $(link).attr("href")) - .map((path) => join(BASE_URL, path)); -}; - -/** - * Converts tables rows into a list of HRows - * - * @param $ - * @param table - */ -const tokenizeRows = ($: CheerioStatic, table: CheerioElement): HRow[] => { - const courseTable: HRow[] = []; - - for (const tr of $(table).find("tbody > tr").toArray()) { - // different row type - const tds = $(tr).find("td").toArray().map($); - const type = getRowType($, tr, tds); - const row = constructRow($, tds, type); - courseTable.push(row); - } - - return courseTable; -}; - -/** - * Pre-parses the row to determine its type - * - * @param $ - * @param tr - * @param tds - */ -const getRowType = ($: CheerioStatic, tr: CheerioElement, tds: Cheerio[]) => { - const trClasses = new Set(tr.attribs["class"].split(" ")); - const td = tds[0]; - const tdClasses = new Set(td.attr("class")?.split(" ")); - - if (tdClasses.size > 0) { - if (tdClasses.has("codecol")) { - if (trClasses.has("orclass") !== tdClasses.has("orclass")) { - throw new Error("td and tr orclass were not consistent"); - } - const hasMultipleCourses = td.find(".code").toArray().length > 1; - if (tdClasses.has("orclass")) { - if (hasMultipleCourses) { - return HRowType.OR_OF_AND_COURSE; - } - return HRowType.OR_COURSE; - } else if (hasMultipleCourses) { - return HRowType.AND_COURSE; - } - return HRowType.PLAIN_COURSE; - } - throw Error(`td class was not "codecol": "${tdClasses}"`); - } - - if (trClasses.has("subheader")) { - return HRowType.SUBHEADER; - } else if (trClasses.has("areaheader")) { - return HRowType.HEADER; - } - - const tdText = parseText(td); - // Different range types - if ( - RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_1.test(tdText) || - RANGE_LOWER_BOUNDED_MAYBE_EXCEPTIONS_2.test(tdText) - ) { - return HRowType.RANGE_LOWER_BOUNDED; - } else if (RANGE_BOUNDED_MAYBE_EXCEPTIONS.test(tdText)) { - return HRowType.RANGE_BOUNDED; - } else if (RANGE_UNBOUNDED.test(tdText)) { - return HRowType.RANGE_UNBOUNDED; - } - - return HRowType.COMMENT; -}; - -/** - * Converts a single row based on the passed-in type (determined by {@link getRowType} - * - * @param $ - * @param tds - * @param type - */ -const constructRow = ( - $: CheerioStatic, - tds: Cheerio[], - type: HRowType -): HRow => { - switch (type) { - case HRowType.HEADER: - case HRowType.SUBHEADER: - case HRowType.COMMENT: - return constructTextRow($, tds, type); - case HRowType.OR_COURSE: - return constructOrCourseRow($, tds); - case HRowType.PLAIN_COURSE: - return constructPlainCourseRow($, tds); - case HRowType.AND_COURSE: - case HRowType.OR_OF_AND_COURSE: - return constructMultiCourseRow($, tds, type); - case HRowType.RANGE_LOWER_BOUNDED: - case HRowType.RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS: - return constructRangeLowerBoundedMaybeExceptions($, tds); - case HRowType.RANGE_BOUNDED: - case HRowType.RANGE_BOUNDED_WITH_EXCEPTIONS: - return constructRangeBoundedMaybeExceptions($, tds); - case HRowType.RANGE_UNBOUNDED: - return constructRangeUnbounded($, tds); - - default: - return assertUnreachable(type); - } -}; - -const constructTextRow = ( - $: CheerioStatic, - tds: Cheerio[], - type: T -): TextRow => { - if (tds.length !== 2) { - throw new Error(tds.toString()); - } - const [c1, c2] = ensureLength(2, tds); - const description = parseText(c1); - const hour = parseHour(c2); - return { hour, description, type }; -}; - -const constructPlainCourseRow = ( - $: CheerioStatic, - tds: Cheerio[] -): CourseRow => { - const [code, desc, hourCol] = ensureLength(3, tds); - const { subject, classId } = parseCourseTitle(parseText(code)); - const description = parseText(desc); - const hour = parseHour(hourCol); - return { hour, description, type: HRowType.PLAIN_COURSE, subject, classId }; -}; - -const constructOrCourseRow = ( - $: CheerioStatic, - tds: Cheerio[] -): CourseRow => { - const [code, desc] = ensureLength(2, tds); - // remove "or " - const { subject, classId } = parseCourseTitle( - parseText(code).substring(3).trim() - ); - const description = parseText(desc); - // there may be multiple courses in the OR, so we can't backtrack - return { hour: 0, description, type: HRowType.OR_COURSE, subject, classId }; -}; - -const constructMultiCourseRow = ( - $: CheerioStatic, - tds: Cheerio[], - type: HRowType.AND_COURSE | HRowType.OR_OF_AND_COURSE -): - | MultiCourseRow - | MultiCourseRow => { - // some ORs of ANDs don't have a third cell for hour column - const [code, desc, hourCol] = ensureLengthAtLeast(2, tds); - const titles = code - .find(".code") - .toArray() - .map($) - .map(parseText) - .map(parseCourseTitle); - const firstDescription = parseText(desc.contents().first()); - const restDescriptions = desc - .children(".blockindent") - .toArray() - // ignore the first four characters, "and " - .map((c) => parseText($(c)).substring(4).trim()); - const descriptions = [firstDescription, ...restDescriptions]; - if (titles.length !== descriptions.length) { - const msg = `found titles: ${titles.length} !== found descs: ${descriptions.length}`; - throw new Error(msg + titles + descriptions); - } - const courses = titles.map(({ subject, classId }, i) => ({ - subject, - classId, - description: descriptions[i], - })); - const hour = hourCol ? parseHour(hourCol) : 0; - return { - hour, - type, - description: descriptions.join(" and "), - courses, - }; -}; - -const constructRangeLowerBoundedMaybeExceptions = ( - $: CheerioStatic, - tds: Cheerio[] -): - | WithExceptions< - RangeLowerBoundedRow - > - | RangeLowerBoundedRow => { - const [desc, hourCol] = ensureLength(2, tds); - const hour = parseHour(hourCol); - // text should match one of the following: - // - CS 9999 or higher[, except CS 9999, CS 9999, CS 3999,... ] - // - Select from any HIST course numbered 3000 or above. - // - Complete three HIST courses numbered 2303 or above. Cluster is subject to Department approval. - const text = parseText(desc); - // should match the form [["CS 9999", "CS", "9999"], [...]] - const matches = Array.from(text.matchAll(RANGE_LOWER_BOUNDED_PARSE)); - const [[, subject, , , , id], ...exceptions] = ensureLengthAtLeast( - 1, - matches - ); - if (exceptions.length > 0) { - return { - type: HRowType.RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS, - hour, - subject, - classIdStart: Number(id), - exceptions: exceptions.map(([, subject, , , , id]) => ({ - subject, - classId: Number(id), - })), - }; - } - return { - type: HRowType.RANGE_LOWER_BOUNDED, - hour, - subject, - classIdStart: Number(id), - }; -}; - -const constructRangeBoundedMaybeExceptions = ( - $: CheerioStatic, - tds: Cheerio[] -): - | RangeBoundedRow - | WithExceptions> => { - const [desc, hourCol] = ensureLength(2, tds); - const hour = parseHour(hourCol); - // text should match the form: - // 1. CS 1000 to CS 5999 - // 2. CS 1000-CS 5999 - const text = parseText(desc); - // should match the form [["CS 9999", "CS", "9999"], [...]] - const matches = Array.from(text.matchAll(COURSE_REGEX)); - const [[, subject, classIdStart], [, , classIdEnd], ...exceptions] = - ensureLengthAtLeast(2, matches); - const result = { - hour, - subject, - classIdStart: Number(classIdStart), - classIdEnd: Number(classIdEnd), - }; - if (exceptions.length > 0) { - return { - ...result, - type: HRowType.RANGE_BOUNDED_WITH_EXCEPTIONS, - exceptions: exceptions.map(([, subject, id]) => ({ - subject, - classId: Number(id), - })), - }; - } - return { - ...result, - type: HRowType.RANGE_BOUNDED, - }; -}; - -const constructRangeUnbounded = ( - $: CheerioStatic, - tds: Cheerio[] -): RangeUnboundedRow => { - const [desc, hourCol] = ensureLength(2, tds); - const hour = parseHour(hourCol); - // text should match one of the following: - // - Any course in ARTD, ARTE, ARTF, ARTG, ARTH, and GAME subject areas as long as prerequisites have been met. - // - BIOE, CHME, CIVE, EECE, ME, IE, MEIE, and ENGR to Department approval. - const text = parseText(desc); - const matches = Array.from(text.match(SUBJECT_REGEX) ?? []); - const subjects = ensureLengthAtLeast(3, matches); - return { - type: HRowType.RANGE_UNBOUNDED, - hour, - subjects, - }; -}; - -const parseHour = (td: Cheerio) => { - const hourText = td.text(); - return parseInt(hourText.split("-")[0]) || 0; -}; -const parseCourseTitle = (parsedCourse: string) => { - const [subject, classId] = ensureLength(2, parsedCourse.split(" ")); - return { - subject, - classId: Number(classId), - }; -}; diff --git a/packages/scrapers-v2/src/tokenize/types.ts b/packages/scrapers-v2/src/tokenize/types.ts deleted file mode 100644 index a14e117dc..000000000 --- a/packages/scrapers-v2/src/tokenize/types.ts +++ /dev/null @@ -1,116 +0,0 @@ -/** - * An HTML document (catalog page) has a few identifiable features, along with a - * bunch of sections. - */ -export type HDocument = { - yearVersion: number; - majorName: string; - programRequiredHours: number; - sections: HSection[]; -}; - -/** An HTML section (of a document) has a description, and a list of rows that it contains. */ -export type HSection = { - description: string; - entries: HRow[]; -}; - -/** - * An HTML row (abbreviated HRow) consists of four main different types of row: - * - * - TextRow: a row containing text. either an areaHeader, comment, or subHeader. - * - CourseRow: a single course, that may have an "OR" annotation - * - MultiCourseRow: multiple courses (2+). currently only AND courses appear as - * multiCourseRows - * - RangeRow: either bounded, unbounded, or only bounded on the bottom (sometimes - * with exceptions) - */ -export type HRow = - // text rows - | TextRow - | TextRow - | TextRow - // course rows - | CourseRow - | CourseRow - // multi course rows - | MultiCourseRow - | MultiCourseRow - // range rows - | RangeLowerBoundedRow - | WithExceptions< - RangeLowerBoundedRow - > - | RangeBoundedRow - | WithExceptions> - | RangeUnboundedRow; - -// an enum to give a unique discriminator to each of the above cases -// the different outputs we have, by TYPE -export enum HRowType { - HEADER = "HEADER", - SUBHEADER = "SUBHEADER", - COMMENT = "COMMENT", - OR_COURSE = "OR_COURSE", - AND_COURSE = "AND_COURSE", - OR_OF_AND_COURSE = "OR_OF_AND_COURSE", - PLAIN_COURSE = "PLAIN_COURSE", - - RANGE_LOWER_BOUNDED = "RANGE_LOWER_BOUNDED", - RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS = "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - - RANGE_BOUNDED = "RANGE_BOUNDED", - RANGE_BOUNDED_WITH_EXCEPTIONS = "RANGE_BOUNDED_WITH_EXCEPTIONS", - - RANGE_UNBOUNDED = "RANGE_UNBOUNDED", -} - -export interface TextRow { - type: T; - description: string; - hour: number; -} - -export interface CourseRow { - type: T; - description: string; - hour: number; - subject: string; - classId: number; -} - -export interface MultiCourseRow { - type: T; - description: string; - hour: number; - // may contain duplicates - courses: Array<{ subject: string; classId: number; description: string }>; -} - -export interface RangeBoundedRow { - type: T; - hour: number; - subject: string; - classIdStart: number; - classIdEnd: number; -} - -export interface RangeUnboundedRow { - type: T; - hour: number; - subjects: Array; -} - -export interface RangeLowerBoundedRow { - type: T; - hour: number; - subject: string; - classIdStart: number; -} - -export type WithExceptions = S & { - exceptions: Array<{ - subject: string; - classId: number; - }>; -}; diff --git a/packages/scrapers-v2/src/urls/README.md b/packages/scrapers-v2/src/urls/README.md deleted file mode 100644 index 6a92e8a06..000000000 --- a/packages/scrapers-v2/src/urls/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# URL scraper - -This folder contains code for scraping the URLs of entries in the catalog, and is based on the sidebar _hierarchy_ that the catalog uses to describe the locations of catalog entries. - -You can think of the catalog sidebar as a bunch of entries, with each entry potentially containing more sub-entries (hence _hierarchical_ in nature). - -## Scraping Strategy - -We hardcode a set of "top level entries" to scrape, which are the colleges. They are listed underneath [undergraduate](https://catalog.northeastern.edu/undergraduate/), and are just the colleges: CAMD, D'Amore McKim, Khoury, COE, Bouvé, COE, and CSSH. - -When you click on one of the colleges above, (from the undergraduate link) you can see the sidebar will expand, with more entries. We then add all of these sub-entries to a queue of links to visit, and begin by visiting the first (think, BFS). - -Visiting the first one, "School of Architecture" (under CAMD), we see it yields more links. Rinse and repeat the above process. Visiting the first of these, we see that this time, it does _not_ yield more sub-links. Therefore, we have a "leaf" of the hierarchy "tree", and we store the link to this entry. - -## Misc - -When actually scraping, we just store a list of paths (each path to a catalog entry). However because of the hierarchical nature of the catalog, we figured it may be useful to later be able to have a hierarchy, so we convert it to be a hierarchy. - -Maybe in the future this is used, or maybe it is not needed and can be removed. - -## Support for old catalogs - -All the old catalogs can be found here: https://registrar.northeastern.edu/group/academic-catalogs/ - -However, only the last ~7 or so catalogs are actually stored in HTML form. we don't know if earlier versions were in HTML at one point, but the server was taken offline, or if there was only ever a PDF of the catalog available. - -Because of this, in the future, it may be worth it to download old versions of the catalog for safekeeping. - -Also because of this, we cannot scrape older versions of the catalog (older than 7 years), because the scraper cannot function on non-HTML versions of the catalog. - -The ticket for adding old catalog support can be found here: https://trello.com/c/7aFTWtMU/452-scrapers-add-old-catalog-support diff --git a/packages/scrapers-v2/src/urls/types.ts b/packages/scrapers-v2/src/urls/types.ts deleted file mode 100644 index f2215a982..000000000 --- a/packages/scrapers-v2/src/urls/types.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* -Master list of old catalogs: -https://registrar.northeastern.edu/group/academic-catalogs/ - -Catalogs that are scrape-able (HTML): -2021–2022 Catalog and Course Descriptions (current year) -2020–2021 Catalog and Course Descriptions -2019–2020 Catalog and Course Descriptions -2018–2019 Catalog and Course Descriptions -2017–2018 Catalog and Course Descriptions -2016–2017 Catalog and Course Descriptions -*/ - -export enum College { - ARTS_MEDIA_DESIGN = "arts-media-design", - BUSINESS = "business", - KHOURY = "computer-information-science", - ENGINEERING = "engineering", - HEALTH_SCIENCES = "health-sciences", - SCIENCE = "science", - SOCIAL_SCIENCES_HUMANITIES = "social-sciences-humanities", -} - -/** - * Represents the result of an attempted catalog URL scrape (to find URLs to entries). - * - * Produces a list of entry URLs, as well a queue of unfinished URLs that still need - * to be searched. Will produce unfinished URLs if visiting a URL fails. - */ -export type CatalogURLResult = { - entries: URL[]; - unfinished: Array; -}; - -export type EntryError = { url: URL; error: unknown }; diff --git a/packages/scrapers-v2/src/urls/urls.ts b/packages/scrapers-v2/src/urls/urls.ts deleted file mode 100644 index 86c81fe68..000000000 --- a/packages/scrapers-v2/src/urls/urls.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { loadHtmlWithUrl } from "../utils"; -import { CatalogURLResult, College } from "./types"; -import { ResultType } from "@graduate/common"; -import { join } from "path"; -import { BASE_URL } from "../constants"; - -/** - * Scrapes all catalog entries underneath the colleges for the specified catalog - * year (given in the form of two numbers to avoid ambiguity: ex, 2021-2022). - * - * @param start Starting year (must be end year - 1) - * @param end Ending year - * @returns A hierarchy of catalog entry links - */ -export const scrapeMajorLinks = async ( - start: number, - end: number -): Promise => { - if (start !== end - 1) { - throw new Error("start should == end-1"); - } - - if (start < 2016) { - // this is because there is no HTML version of those catalogs - throw new Error("scraping for years before 2016-2017 are not supported"); - } - - if (start !== 2021) { - // todo: implement generic scraping from overall archive - // https://registrar.northeastern.edu/group/academic-catalogs/ - throw new Error("only current year is supported"); - } - - return scrapeMajorLinksForUrl(BASE_URL, "undergraduate"); -}; - -/** - * Given a baseUrl and a path, attempts to scrape the major catalog based on the - * sidebar hierarchy. - * - * Assumes that the provided baseURL + path have direct sub-entries for each of - * the colleges. - * - * @param baseUrl The base url of the major catalog. should look something like - * "https://catalog.northeastern.edu" - * @param path The path of the major catalog. something like - * "/undergraduate/" or "/archive/2018-2019/undergraduate". - */ -export const scrapeMajorLinksForUrl = async ( - baseUrl: string, - path: string -): Promise => { - const initQueue = Object.values(College).map( - (college) => new URL(join(baseUrl, path, college, "/")) - ); - return await scrapeLinks(baseUrl, initQueue); -}; - -/** - * Retrieves all sub-entries of the given initial queue in BFS fashion using the - * catalog sidebar hierarchy. - * - * @param baseUrl The base catalog URL, i.e. https://catalog.northeastern.edu - * @param initQueue A queue of parent entries - * @returns A flat list of all the last level children catalog entries - */ -const scrapeLinks = async ( - baseUrl: string, - initQueue: URL[] -): Promise => { - const entries: URL[] = []; - const unfinished = []; - - // there are multiple links in the sidebar to the same entry - // keep a set to avoid visiting the same entry twice - const seen = new Set(initQueue.map((url) => url.href)); - let queue = initQueue; - while (queue.length > 0) { - const { ok, errors } = await getUrlHtmls(queue); - unfinished.push(...errors); - const nextQueue: URL[] = []; - for (const { $, url } of ok) { - const children = getChildrenForPathId($, url).toArray().map($); - for (const element of children) { - const path = getLinkForEl(element); - const url = new URL(join(baseUrl, path)); - if (!seen.has(url.href)) { - const bucket = isParent(element) ? nextQueue : entries; - bucket.push(url); - seen.add(url.href); - } - } - } - queue = nextQueue; - } - - return { entries, unfinished }; -}; - -const isParent = (el: Cheerio) => { - return el.hasClass("isparent"); -}; - -const getLinkForEl = (element: Cheerio) => { - const aTag = element.find("a"); - if (aTag.length === 0) { - const msg = "Catalog is missing a link for a parent element."; - throw new Error(msg); - } - - return aTag.attr("href"); -}; - -const getChildrenForPathId = ($: CheerioStatic, url: URL) => { - // The catalog entries have an ID equal to the path, with a trailing slash - // We select the element via its ID - // Note: for getElementById, forward slashes need to be escaped - const id = url.pathname.replaceAll("/", "\\/"); - const current = $(`#${id}`); - return current.children(); -}; - -const getUrlHtmls = async (queue: URL[]) => { - const fetchResults = await Promise.all(queue.map(loadHtmlWithUrl)); - - const ok = []; - const errors = []; - for (const { url, result } of fetchResults) { - if (result.type === ResultType.Ok) { - ok.push({ $: result.ok, url }); - continue; - } - errors.push({ error: result.err, url }); - } - return { ok, errors }; -}; diff --git a/packages/scrapers-v2/src/utils.ts b/packages/scrapers-v2/src/utils.ts deleted file mode 100644 index 4086c5ee0..000000000 --- a/packages/scrapers-v2/src/utils.ts +++ /dev/null @@ -1,85 +0,0 @@ -import * as cheerio from "cheerio"; -import { Err, Ok, Result } from "@graduate/common"; -import { existsSync } from "fs"; -import { mkdir, readFile, writeFile } from "fs/promises"; -import undici from "undici"; - -export const loadHtmlWithUrl = async ( - url: URL -): Promise<{ url: URL; result: Result }> => { - let result: Result; - try { - result = Ok(await loadHTML(url.href)); - } catch (error) { - result = Err(error); - } - return { url, result }; -}; - -/** - * Whether to cache the response bodies of requests. if set to true, - * `cachedGetRequest` will cache requests. - */ -const USE_CACHE = false; -export const loadHTML = async (url: string): Promise => { - const data = await cachedGetRequest(url); - return cheerio.load(data); -}; - -/** - * If use cache is true, will attempt to look for request body in - * `./catalogCache` before fetching. If does not exist, will save the response - * in `./catalogCache` before returning response. - * - * @param url - */ -const cachedGetRequest = async (url: string) => { - if (!USE_CACHE) { - return await wrappedGetRequest(url); - } - - if (!existsSync("./catalogCache")) { - await mkdir("./catalogCache"); - } - - // https://stackoverflow.com/questions/35511331/how-to-make-a-valid-filename-from-an-arbitrary-string-in-javascript - const path = `./catalogCache/${url.replaceAll(/[\/|\\:*?"<>]/g, "-")}`; - if (existsSync(path)) { - return await readFile(path); - } - - const data = await wrappedGetRequest(url); - await writeFile(path, data); - return data; -}; - -const wrappedGetRequest = async (url: string) => { - const response = await undici.request(url, { maxRedirections: 1 }); - if (response.statusCode !== 200) { - throw new Error(`non-ok status code: ${response.statusCode}, url: ${url}`); - } - return await response.body.text(); -}; - -export const ensureLength = (n: number, l: T[]) => { - const length = l.length; - if (length !== n) { - const msg = `expected text row to contain exactly ${n} cells, found ${length}`; - throw new Error(msg); - } - return l; -}; - -export const ensureLengthAtLeast = (n: number, l: T[]) => { - const length = l.length; - if (length < n) { - const msg = `expected text row to contain at least ${n} cells, found ${length}`; - throw new Error(msg); - } - return l; -}; - -export const parseText = (td: Cheerio) => { - // replace &NBSP with space - return td.text().replaceAll("\xa0", " ").trim(); -}; diff --git a/packages/scrapers-v2/test/__snapshots__/tokenize.test.ts.snap b/packages/scrapers-v2/test/__snapshots__/tokenize.test.ts.snap deleted file mode 100644 index aa6fcdb0b..000000000 --- a/packages/scrapers-v2/test/__snapshots__/tokenize.test.ts.snap +++ /dev/null @@ -1,7862 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`scraper v2 snapshot tests 3 classes per AND (physics) 1`] = ` -Object { - "majorName": "Physics, BS", - "programRequiredHours": 133, - "sections": Array [ - Object { - "description": "Physics Major Requirements", - "entries": Array [ - Object { - "description": "Introductory Physics", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Physics 1", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 5, - "type": "COMMENT", - }, - Object { - "courses": Array [ - Object { - "classId": 1161, - "description": "Physics 1", - "subject": "PHYS", - }, - Object { - "classId": 1162, - "description": "Lab for PHYS 1161", - "subject": "PHYS", - }, - ], - "description": "Physics 1 and Lab for PHYS 1161", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "Physics for Engineering 1", - "subject": "PHYS", - }, - Object { - "classId": 1152, - "description": "Lab for PHYS 1151", - "subject": "PHYS", - }, - Object { - "classId": 1153, - "description": "Interactive Learning Seminar for PHYS 1151", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 1 and Lab for PHYS 1151 and Interactive Learning Seminar for PHYS 1151", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Physics 2", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 5, - "type": "COMMENT", - }, - Object { - "courses": Array [ - Object { - "classId": 1165, - "description": "Physics 2", - "subject": "PHYS", - }, - Object { - "classId": 1166, - "description": "Lab for PHYS 1165", - "subject": "PHYS", - }, - ], - "description": "Physics 2 and Lab for PHYS 1165", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1155, - "description": "Physics for Engineering 2", - "subject": "PHYS", - }, - Object { - "classId": 1156, - "description": "Lab for PHYS 1155", - "subject": "PHYS", - }, - Object { - "classId": 1157, - "description": "Interactive Learning Seminar for PHYS 1155", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 2 and Lab for PHYS 1155 and Interactive Learning Seminar for PHYS 1155", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Intermediate Physics", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2303, - "description": "Modern Physics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2371, - "description": "Electronics", - "subject": "PHYS", - }, - Object { - "classId": 2372, - "description": "Lab for PHYS 2371", - "subject": "PHYS", - }, - ], - "description": "Electronics and Lab for PHYS 2371", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "description": "Advanced Physics", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3600, - "description": "Advanced Physics Laboratory", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3601, - "description": "Classical Dynamics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3602, - "description": "Electricity and Magnetism 1", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3603, - "description": "Electricity and Magnetism 2", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4115, - "description": "Quantum Mechanics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4305, - "description": "Thermodynamics and Statistical Mechanics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Elective Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 4606, - "description": "Mathematical and Computational Methods for Physics", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classIdEnd": 7999, - "classIdStart": 3500, - "hour": 0, - "subject": "PHYS", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Experiential Learning", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course in experiential learning. See department for approved courses. Note: The experiential learning requirement is waived following a student presentation connected with a co-op and/or research experience. The requirement is often fulfilled by a talk at a Society of Physics Students meeting but can be fulfilled by an adequately documented presentation at a professional meeting or at an appropriate campus event. Contact your faculty advisor for additional information.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Senior Capstone", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 5318, - "description": "Principles of Experimental Physics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses", - "entries": Array [ - Object { - "description": "Mathematics", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2321, - "description": "Calculus 3 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2341, - "description": "Differential Equations and Linear Algebra for Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2331, - "description": "Linear Algebra", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3081, - "description": "Probability and Statistics", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computational Methods", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1211, - "description": "Computational Problem Solving in Physics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1130, - "description": "Computing, Data, and Science", - "hour": 0, - "subject": "PHYS", - "type": "OR_COURSE", - }, - Object { - "classId": 1111, - "description": "Engineering Problem Solving and Computation", - "hour": 0, - "subject": "GE", - "type": "OR_COURSE", - }, - Object { - "description": "Chemistry", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1211, - "description": "General Chemistry 1", - "subject": "CHEM", - }, - Object { - "classId": 1212, - "description": "Lab for CHEM 1211", - "subject": "CHEM", - }, - ], - "description": "General Chemistry 1 and Lab for CHEM 1211", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "description": "Technical Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete 8 semester hours from the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2280, - "description": "Statistics and Software", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2321, - "hour": 0, - "subject": "MATH", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 7999, - "classIdStart": 2303, - "hour": 0, - "subject": "PHYS", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2311, - "hour": 0, - "subject": "CHEM", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2301, - "hour": 0, - "subject": "BIOL", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2300, - "hour": 0, - "subject": "ENVR", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 4900, - "classIdStart": 2990, - "hour": 0, - "subject": "CS", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 4699, - "classIdStart": 2001, - "hour": 0, - "subject": "CHME", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 4699, - "classIdStart": 2001, - "hour": 0, - "subject": "CIVE", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2001, - "hour": 0, - "subject": "EECE", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 4699, - "classIdStart": 2001, - "hour": 0, - "subject": "ME", - "type": "RANGE_BOUNDED", - }, - Object { - "classIdEnd": 4699, - "classIdStart": 2001, - "hour": 0, - "subject": "IE", - "type": "RANGE_BOUNDED", - }, - ], - }, - Object { - "description": "Astrophysics Concentration (Optional)", - "entries": Array [ - Object { - "description": "Astrophysics Core", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1111, - "description": "Astronomy", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "The following courses may be counted toward major electives, if they are listed as applicable options in the approved curriculum:", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 3111, - "description": "Astrophysical Processes: Decoding the Universe", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4111, - "description": "Multimessenger Astrophysics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5117, - "description": "Advanced Astrophysics Topics", - "hour": 4, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5118, - "description": "General Relativity and Cosmology", - "hour": 0, - "subject": "PHYS", - "type": "OR_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests CS & Game Dev matches snapshot 1`] = ` -Object { - "majorName": "Computer Science and Game Development, BS", - "programRequiredHours": 133, - "sections": Array [ - Object { - "description": "Computer Science Courses", - "entries": Array [ - Object { - "description": "Computer Science Overview", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "First Year Seminar", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1000, - "description": "Art and Design at Northeastern", - "hour": 0, - "subject": "ARTF", - "type": "OR_COURSE", - }, - Object { - "classId": 1210, - "description": "Professional Development for Khoury Co-op", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2000, - "description": "Professional Development for Co-op", - "hour": 0, - "subject": "EEAM", - "type": "OR_COURSE", - }, - Object { - "description": "Computer Science Fundamental Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1800, - "description": "Discrete Structures", - "subject": "CS", - }, - Object { - "classId": 1802, - "description": "Seminar for CS 1800", - "subject": "CS", - }, - ], - "description": "Discrete Structures and Seminar for CS 1800", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2500, - "description": "Fundamentals of Computer Science 1", - "subject": "CS", - }, - Object { - "classId": 2501, - "description": "Lab for CS 2500", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 1 and Lab for CS 2500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2510, - "description": "Fundamentals of Computer Science 2", - "subject": "CS", - }, - Object { - "classId": 2511, - "description": "Lab for CS 2510", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 2 and Lab for CS 2510", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "description": "Computer Science Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3000, - "description": "Algorithms and Data", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3500, - "description": "Object-Oriented Design", - "subject": "CS", - }, - Object { - "classId": 3501, - "description": "Lab for CS 3500", - "subject": "CS", - }, - ], - "description": "Object-Oriented Design and Lab for CS 3500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 3520, - "description": "Programming in C++ (Integrative course)", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3540, - "description": "Game Programming (Integrative course)", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3650, - "description": "Computer Systems", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3700, - "description": "Networks and Distributed Systems", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4300, - "description": "Computer Graphics (Integrative course)", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4500, - "description": "Software Development", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4530, - "description": "Fundamentals of Software Engineering", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "classId": 4850, - "description": "Building Game Engines (Integrative course)", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computer Science Elective Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4150, - "description": "Game Artificial Intelligence (Integrative course)", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4300, - "description": "Human Computer Interaction", - "hour": 0, - "subject": "IS", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Game Design Courses", - "entries": Array [ - Object { - "description": "Game Design Required", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1110, - "description": "Games and Society", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2500, - "description": "Foundations of Game Design", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2950, - "description": "Game Studio", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3400, - "description": "Level Design and Game Architecture", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3700, - "description": "Rapid Idea Prototyping for Games", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3800, - "description": "Game Concept Development", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4700, - "description": "Game Design Capstone", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Game Design Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classId": 1850, - "description": "Experimental Game Design", - "hour": 4, - "subject": "GAME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3300, - "description": "Game Interface Design", - "hour": 0, - "subject": "GAME", - "type": "OR_COURSE", - }, - Object { - "classId": 4000, - "description": "Topics in Game Design", - "hour": 0, - "subject": "GAME", - "type": "OR_COURSE", - }, - Object { - "description": "Khoury/Game-Related Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three of the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "hour": 0, - "subjects": Array [ - "ARTD", - "ARTE", - "ARTF", - "ARTG", - "ARTH", - "GAME", - ], - "type": "RANGE_UNBOUNDED", - }, - Object { - "description": "If GAME 4000 (or any other topics course in the subjects listed above) is completed more than once, the additional completions may be allowed toward the Game Design electives.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 5010, - "subject": "CS", - }, - ], - "hour": 0, - "subject": "CS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4930, - "subject": "CY", - }, - ], - "hour": 0, - "subject": "CY", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "DS", - }, - ], - "hour": 0, - "subject": "DS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "IS", - }, - ], - "hour": 0, - "subject": "IS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2331, - "description": "Linear Algebra", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2350, - "description": "Statistics", - "hour": 0, - "subject": "ECON", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2320, - "description": "Statistics in Psychological Research", - "hour": 0, - "subject": "PSYC", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses", - "entries": Array [ - Object { - "description": "Psychology", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1101, - "description": "Foundations of Psychology", - "hour": 4, - "subject": "PSYC", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Mathematics", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two courses from the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 1260, - "description": "Math Fundamentals for Games (Integrative course)", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classIdStart": 1342, - "hour": 0, - "subject": "MATH", - "type": "RANGE_LOWER_BOUNDED", - }, - ], - }, - Object { - "description": "Computer Science Writing Requirement", - "entries": Array [ - Object { - "description": "College Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Advanced Writing in the Disciplines", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3302, - "description": "Advanced Writing in the Technical Professions", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3314, - "description": "Advanced Writing in the Arts, Media, and Design", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - Object { - "classId": 3315, - "description": "Interdisciplinary Advanced Writing in the Disciplines", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 12 semester hours of general electives.", - "hour": 12, - "type": "COMMENT", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test NO tabs (architecture and english) 1`] = ` -Object { - "majorName": "Architecture and English, BS", - "programRequiredHours": 128, - "sections": Array [ - Object { - "description": "Architecture Requirements", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1110, - "description": "Fundamental Architectural Representation", - "hour": 4, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1120, - "description": "Fundamental Architectural Design", - "hour": 6, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1310, - "description": "Buildings and Cities, A Global History", - "hour": 4, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1450, - "description": "Understanding Design", - "hour": 4, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2130, - "description": "Site, Space, and Program", - "hour": 6, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2240, - "description": "Architectonic Systems", - "hour": 4, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Architecure History Electives", - "hour": 8, - "type": "HEADER", - }, - Object { - "classIdEnd": 2399, - "classIdStart": 2300, - "hour": 0, - "subject": "ARCH", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 3370, - "description": "Advanced Topics in Architectural History", - "hour": 0, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3450, - "description": "Advanced Architectural Communication", - "hour": 0, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5310, - "description": "Design Tactics and Operations", - "hour": 0, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "English Requirements", - "entries": Array [ - Object { - "description": "English Course-Level Requirement", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "In addition to the capstone, two of the courses completed from the lists below must be numbered 3000–4999.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Introduction to College", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1000, - "description": "English at Northeastern", - "hour": 1, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Foundational Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1400, - "description": "Introduction to Literary Studies", - "hour": 4, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1160, - "description": "Introduction to Rhetoric", - "hour": 4, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1410, - "description": "Introduction to Research on Writing", - "hour": 0, - "subject": "ENGL", - "type": "OR_COURSE", - }, - Object { - "description": "Diversity", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following. This course may also be used to fulfill an additional English requirement below:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2296, - "description": "Early African-American Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2450, - "description": "Postcolonial Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2455, - "description": "American Women Writers", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2760, - "description": "Writing in Global Contexts", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3663, - "description": "The African American Novel", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3678, - "description": "Bedrooms and Battlefields: Hebrew Bible and the Origins of Sex, Gender, and Ethnicity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3685, - "description": "Modern and Contemporary Jewish Literature", - "hour": 0, - "subject": "ENGL/JWSS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Pre–Nineteenth-Century Literature", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1600, - "description": "Introduction to Shakespeare", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1700, - "description": "Global Literatures 1", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2240, - "description": "17th-Century British Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2296, - "description": "Early African-American Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3678, - "description": "Bedrooms and Battlefields: Hebrew Bible and the Origins of Sex, Gender, and Ethnicity", - "hour": 0, - "subject": "ENGL/JWSS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Nineteenth-, Twentieth-, and Twenty-First-Century Literature", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2330, - "description": "The American Renaissance", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3720, - "description": "19th-Century Major Figure", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2301, - "description": "The Graphic Novel", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2440, - "description": "The Modern Bestseller", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2600, - "description": "Irish Literary Culture (Abroad)", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2610, - "description": "Contemporary Israeli Literature and Art (Abroad)", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3685, - "description": "Modern and Contemporary Jewish Literature", - "hour": 0, - "subject": "ENGL/JWSS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3730, - "description": "20th- and 21st-Century Major Figure", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Theories and Methods", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1140, - "description": "Grammar: The Architecture of English", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1160, - "description": "Introduction to Rhetoric", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1410, - "description": "Introduction to Research on Writing", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3325, - "description": "Rhetoric of Law", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3340, - "description": "Technologies of Text", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3381, - "description": "The Practice and Theory of Teaching Writing", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3700, - "description": "Narrative Medicine", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1150, - "description": "Introduction to Language and Linguistics", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2350, - "description": "Linguistic Analysis", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3450, - "description": "Syntax", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3452, - "description": "Semantics", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3454, - "description": "History of English", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3456, - "description": "Language and Gender", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3458, - "description": "Topics in Linguistics", - "hour": 0, - "subject": "LING", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Comparative Literature", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1450, - "description": "Reading and Writing in the Digital Age", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1502, - "description": "American Literature to 1865", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2420, - "description": "Contemporary Poetry", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2430, - "description": "Contemporary Fiction", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2450, - "description": "Postcolonial Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2455, - "description": "American Women Writers", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2510, - "description": "Horror Fiction", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2520, - "description": "Science Fiction", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2600, - "description": "Irish Literary Culture (Abroad)", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2620, - "description": "What Is Nature?", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2690, - "description": "Boston in Literature", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3487, - "description": "Film and Text (Abroad)", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3663, - "description": "The African American Novel", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2700, - "description": "Creative Writing", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2710, - "description": "Style and Editing", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2740, - "description": "Writing and Community Engagement", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2760, - "description": "Writing in Global Contexts", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2770, - "description": "Writing to Heal", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2780, - "description": "Visual Writing", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2850, - "description": "Writing for Social Media: Theory and Practice", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3375, - "description": "Writing Boston", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3376, - "description": "Creative Nonfiction", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3377, - "description": "Poetry Workshop", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3378, - "description": "Fiction Workshop", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3380, - "description": "Writing Seminar", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3382, - "description": "Publishing in the 21st Century", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3384, - "description": "The Writer’s Marketplace", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Capstone", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4710, - "description": "Capstone Seminar", - "hour": 4, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4720, - "description": "Capstone Project", - "hour": 0, - "subject": "ENGL", - "type": "OR_COURSE", - }, - Object { - "description": "English Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two additional ENGL electives.", - "hour": 8, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Integrative Requirement", - "entries": Array [ - Object { - "classId": 2330, - "description": "Architecture and the City in the Nineteenth Century", - "hour": 4, - "subject": "ARCH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2340, - "description": "Modern Architecture", - "hour": 0, - "subject": "ARCH", - "type": "OR_COURSE", - }, - Object { - "classId": 3375, - "description": "Writing Boston", - "hour": 4, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test OR of ANDs (bioengineering biochemistry) 1`] = ` -Object { - "majorName": "Bioengineering and Biochemistry, BSBioE", - "programRequiredHours": 139, - "sections": Array [ - Object { - "description": "Engineering", - "entries": Array [ - Object { - "description": "Required Engineering", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2350, - "description": "Biomechanics", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2355, - "description": "Quantitative Physiology for Bioengineers", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2365, - "description": "Bioengineering Measurement, Experimentation, and Statistics", - "subject": "BIOE", - }, - Object { - "classId": 2366, - "description": "Lab for BIOE 2365", - "subject": "BIOE", - }, - ], - "description": "Bioengineering Measurement, Experimentation, and Statistics and Lab for BIOE 2365", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 3210, - "description": "Bioelectricity", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3310, - "description": "Transport and Fluids for Bioengineers", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3380, - "description": "Biomolecular Dynamics and Control", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5410, - "description": "Molecular Bioengineering", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5420, - "description": "Cellular Engineering", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5430, - "description": "Principles and Applications of Tissue Engineering", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Bioengineering Capstone", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4790, - "description": "Capstone Design 1", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4792, - "description": "Capstone Design 2", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Supplemental Credit", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "2 semester hours from the following course count toward the engineering requirement:", - "hour": 2, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "3 semester hours from the following course count toward the engineering requirement:", - "hour": 3, - "type": "COMMENT", - }, - Object { - "classId": 1502, - "description": "Cornerstone of Engineering 2 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Mathematics/Science Requirement", - "entries": Array [ - Object { - "description": "Required Mathematics/Science", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1115, - "description": "General Biology 1 for Engineers", - "subject": "BIOL", - }, - Object { - "classId": 1116, - "description": "Lab for BIOL 1115", - "subject": "BIOL", - }, - ], - "description": "General Biology 1 for Engineers and Lab for BIOL 1115", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1111, - "description": "General Biology 1", - "subject": "BIOL", - }, - Object { - "classId": 1112, - "description": "Lab for BIOL 1111", - "subject": "BIOL", - }, - ], - "description": "General Biology 1 and Lab for BIOL 1111", - "hour": 0, - "type": "OR_OF_AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2301, - "description": "Genetics and Molecular Biology", - "subject": "BIOL", - }, - Object { - "classId": 2302, - "description": "Lab for BIOL 2301", - "subject": "BIOL", - }, - ], - "description": "Genetics and Molecular Biology and Lab for BIOL 2301", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3611, - "description": "Biochemistry", - "subject": "BIOL", - }, - Object { - "classId": 3612, - "description": "Lab for BIOL 3611", - "subject": "BIOL", - }, - ], - "description": "Biochemistry and Lab for BIOL 3611", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "General Chemistry for Engineers", - "subject": "CHEM", - }, - Object { - "classId": 1153, - "description": "Recitation for CHEM 1151", - "subject": "CHEM", - }, - ], - "description": "General Chemistry for Engineers and Recitation for CHEM 1151", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2311, - "description": "Organic Chemistry 1", - "subject": "CHEM", - }, - Object { - "classId": 2312, - "description": "Lab for CHEM 2311", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 1 and Lab for CHEM 2311", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2313, - "description": "Organic Chemistry 2", - "subject": "CHEM", - }, - Object { - "classId": 2314, - "description": "Lab for CHEM 2313", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 2 and Lab for CHEM 2313", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2321, - "description": "Calculus 3 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2341, - "description": "Differential Equations and Linear Algebra for Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1171, - "description": "Physics 1 for Bioscience and Bioengineering", - "subject": "PHYS", - }, - Object { - "classId": 1172, - "description": "Lab for PHYS 1171", - "subject": "PHYS", - }, - Object { - "classId": 1173, - "description": "Interactive Learning Seminar for PHYS 1171", - "subject": "PHYS", - }, - ], - "description": "Physics 1 for Bioscience and Bioengineering and Lab for PHYS 1171 and Interactive Learning Seminar for PHYS 1171", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "Physics for Engineering 1", - "subject": "PHYS", - }, - Object { - "classId": 1152, - "description": "Lab for PHYS 1151", - "subject": "PHYS", - }, - Object { - "classId": 1153, - "description": "Interactive Learning Seminar for PHYS 1151", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 1 and Lab for PHYS 1151 and Interactive Learning Seminar for PHYS 1151", - "hour": 0, - "type": "OR_OF_AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1175, - "description": "Physics 2 for Bioscience and Bioengineering", - "subject": "PHYS", - }, - Object { - "classId": 1176, - "description": "Lab for PHYS 1175", - "subject": "PHYS", - }, - Object { - "classId": 1177, - "description": "Interactive Learning Seminar for PHYS 1175", - "subject": "PHYS", - }, - ], - "description": "Physics 2 for Bioscience and Bioengineering and Lab for PHYS 1175 and Interactive Learning Seminar for PHYS 1175", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1155, - "description": "Physics for Engineering 2", - "subject": "PHYS", - }, - Object { - "classId": 1156, - "description": "Lab for PHYS 1155", - "subject": "PHYS", - }, - Object { - "classId": 1157, - "description": "Interactive Learning Seminar for PHYS 1155", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 2 and Lab for PHYS 1155 and Interactive Learning Seminar for PHYS 1155", - "hour": 0, - "type": "OR_OF_AND_COURSE", - }, - Object { - "description": "Advanced Biology Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course in the following range:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2311, - "hour": 0, - "subject": "BIOL", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Advanced Chemistry Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course in the following range:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2310, - "hour": 0, - "subject": "CHEM", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Supplemental Credit", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "1 semester hour from the following course counts toward the mathematics/science requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Professional Development", - "entries": Array [ - Object { - "description": "Professional Development", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1000, - "description": "First-Year Seminar", - "hour": 1, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2000, - "description": "Introduction to Engineering Co-op Education", - "hour": 1, - "subject": "ENCP", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3000, - "description": "Professional Issues in Engineering", - "hour": 1, - "subject": "ENCP", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "1 semester hour from the following course counts toward the professional development requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "1 semester hour from the following course counts toward the professional development requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1502, - "description": "Cornerstone of Engineering 2 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Writing Requirements", - "entries": Array [ - Object { - "description": "A grade of C or higher is required:", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Advanced Writing in the Technical Professions", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3307, - "description": "Advanced Writing in the Sciences", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - Object { - "classId": 3315, - "description": "Interdisciplinary Advanced Writing in the Disciplines", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 12 semester hours of academic, nonremedial, nonrepetitive courses.", - "hour": 12, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Integrative Course", - "entries": Array [ - Object { - "description": "This course is already required above and also fulfills the integrative requirement.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classId": 4790, - "description": "Capstone Design 1", - "hour": 4, - "subject": "BIOE", - "type": "PLAIN_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test range bounded (history) 1`] = ` -Object { - "majorName": "Media and Screen Studies and History, BA", - "programRequiredHours": 130, - "sections": Array [ - Object { - "description": "Media and Screen Studies Requirements", - "entries": Array [ - Object { - "description": "Media and Screen Studies Common Requirements", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1220, - "description": "Media, Culture, and Society", - "hour": 4, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1320, - "description": "Media and Social Change", - "hour": 4, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1420, - "description": "Media History", - "hour": 0, - "subject": "MSCR", - "type": "OR_COURSE", - }, - Object { - "description": "Foundation Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1230, - "description": "Introduction to Film Production", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2220, - "description": "Understanding Media", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Diversity or Globalization Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3392, - "description": "Gender and Film", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2325, - "description": "Global Media", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2505, - "description": "Digital Feminisms", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3437, - "description": "Media and Identity", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Writing-Intensive", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2505, - "description": "Digital Feminisms", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3600, - "description": "Film Theory", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3420, - "description": "Digital Media Culture", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3422, - "description": "Media Audiences", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3700, - "description": "Queer Media", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4208, - "description": "TV History", - "hour": 0, - "subject": "MSCR", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Media and Screen Studies Electives", - "entries": Array [ - Object { - "description": "Complete three courses from MSCR or from the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 2380, - "description": "Video Basics", - "hour": 0, - "subject": "ARTD", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3480, - "description": "Video: Sound and Image", - "hour": 0, - "subject": "ARTD", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3485, - "description": "Experimental Video", - "hour": 0, - "subject": "ARTD", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1450, - "description": "Sound Production for Digital Media", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2550, - "description": "Television Field Production", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2655, - "description": "Television Studio Production", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3655, - "description": "Digital Editing for TV", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3750, - "description": "Special Effects and Postproduction for Television", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4755, - "description": "Production Capstone", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "History Requirements", - "entries": Array [ - Object { - "description": "History Colloquium", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "Historical Research and Writing", - "hour": 1, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1201, - "description": "First-Year Seminar", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Introductory-Level Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course in the following range, not used to fulfill another requirement:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classIdEnd": 1999, - "classIdStart": 1001, - "hour": 0, - "subject": "HIST", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "History Seminar and Historical Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "The History Seminar", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2302, - "description": "Historical Writing", - "hour": 1, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Pre-1800 History Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course from the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1187, - "description": "Introduction to Latin American History", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1218, - "description": "Pirates, Planters, and Patriots: Making the Americas, 1492–1804", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1252, - "description": "Japanese Literature and Culture", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1272, - "description": "Europe in the Middle Ages, 500–1500", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1285, - "description": "Introduction to Russian Civilization", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2330, - "description": "Colonial and Revolutionary America", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2370, - "description": "Renaissance to Enlightenment", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Intermediate/Advanced History Cluster", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three courses in the following range:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 2303, - "hour": 0, - "subject": "HIST", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Advanced History", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course in the following range:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classIdEnd": 5999, - "classIdStart": 3000, - "hour": 0, - "subject": "HIST", - "type": "RANGE_BOUNDED", - }, - Object { - "description": "Capstone", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4701, - "description": "Capstone Seminar", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Integrative Requirement", - "entries": Array [ - Object { - "classId": 1357, - "description": "History of Information in the United States: Media, Technology, Law", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test range bounded with exceptions (cs and math) 1`] = ` -Object { - "majorName": "Computer Science and Mathematics, BS", - "programRequiredHours": 132, - "sections": Array [ - Object { - "description": "Computer Science Courses", - "entries": Array [ - Object { - "description": "Computer Science Overview", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "First Year Seminar", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1210, - "description": "Professional Development for Khoury Co-op", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computer Science Fundamental Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1800, - "description": "Discrete Structures", - "subject": "CS", - }, - Object { - "classId": 1802, - "description": "Seminar for CS 1800", - "subject": "CS", - }, - ], - "description": "Discrete Structures and Seminar for CS 1800", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2500, - "description": "Fundamentals of Computer Science 1", - "subject": "CS", - }, - Object { - "classId": 2501, - "description": "Lab for CS 2500", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 1 and Lab for CS 2500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2510, - "description": "Fundamentals of Computer Science 2", - "subject": "CS", - }, - Object { - "classId": 2511, - "description": "Lab for CS 2510", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 2 and Lab for CS 2510", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 2800, - "description": "Logic and Computation", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computer Science Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3000, - "description": "Algorithms and Data", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3500, - "description": "Object-Oriented Design", - "subject": "CS", - }, - Object { - "classId": 3501, - "description": "Lab for CS 3500", - "subject": "CS", - }, - ], - "description": "Object-Oriented Design and Lab for CS 3500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 3800, - "description": "Theory of Computation", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4300, - "description": "Computer Graphics", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4100, - "description": "Artificial Intelligence", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "classId": 4500, - "description": "Software Development", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4530, - "description": "Fundamentals of Software Engineering", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "description": "Khoury Elective Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "With adviser approval, a directed study, research, project study, or appropriate graduate-level course may also be taken as a computer science elective.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Complete eight semester hours of CS, CY, DS, or IS classes that are not already required. Choose courses within the following ranges:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 5010, - "subject": "CS", - }, - ], - "hour": 0, - "subject": "CS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4930, - "subject": "CY", - }, - ], - "hour": 0, - "subject": "CY", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "DS", - }, - ], - "hour": 0, - "subject": "DS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "IS", - }, - ], - "hour": 0, - "subject": "IS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - ], - }, - Object { - "description": "Mathematics Courses", - "entries": Array [ - Object { - "description": "Calculus Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2321, - "description": "Calculus 3 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Mathematics Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2331, - "description": "Linear Algebra", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2341, - "description": "Differential Equations and Linear Algebra for Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3081, - "description": "Probability and Statistics", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3175, - "description": "Group Theory", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3527, - "description": "Number Theory 1", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Mathematics Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three courses in the following range:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classIdEnd": 4999, - "classIdStart": 3001, - "exceptions": Array [ - Object { - "classId": 4000, - "subject": "MATH", - }, - ], - "hour": 0, - "subject": "MATH", - "type": "RANGE_BOUNDED_WITH_EXCEPTIONS", - }, - ], - }, - Object { - "description": "Supporting Course", - "entries": Array [ - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2600, - "description": "Issues in Race, Science, and Technology", - "hour": 0, - "subject": "AFAM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4170, - "description": "The Law, Ethics, and Policy of Data and Digital Technologies", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5240, - "description": "Cyberlaw: Privacy, Ethics, and Digital Rights", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2220, - "description": "History of Technology", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2102, - "description": "Bostonography: The City through Data, Texts, Maps, and Networks", - "hour": 0, - "subject": "INSH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "PHIL", - "type": "OR_COURSE", - }, - Object { - "classId": 1145, - "description": "Technology and Human Values", - "hour": 0, - "subject": "PHIL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1280, - "description": "The Twenty-First-Century Workplace", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2485, - "description": "Environment, Technology, and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4528, - "description": "Computers and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Computer Science Writing Requirement", - "entries": Array [ - Object { - "description": "College Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Advanced Writing in the Disciplines", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3302, - "description": "Advanced Writing in the Technical Professions", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3307, - "description": "Advanced Writing in the Sciences", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - Object { - "classId": 3315, - "description": "Interdisciplinary Advanced Writing in the Disciplines", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 28 semester hours of general electives.", - "hour": 28, - "type": "COMMENT", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test range lower bounded (cs & history) 1`] = ` -Object { - "majorName": "Computer Science and History, BS", - "programRequiredHours": 135, - "sections": Array [ - Object { - "description": "Computer Science Courses", - "entries": Array [ - Object { - "description": "Computer Science Overview", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "First Year Seminar", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1000, - "description": "History at Northeastern", - "hour": 0, - "subject": "HIST", - "type": "OR_COURSE", - }, - Object { - "classId": 1210, - "description": "Professional Development for Khoury Co-op", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2000, - "description": "Professional Development for Co-op", - "hour": 0, - "subject": "EESH", - "type": "OR_COURSE", - }, - Object { - "description": "Computer Science Fundamental Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1800, - "description": "Discrete Structures", - "subject": "CS", - }, - Object { - "classId": 1802, - "description": "Seminar for CS 1800", - "subject": "CS", - }, - ], - "description": "Discrete Structures and Seminar for CS 1800", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2500, - "description": "Fundamentals of Computer Science 1", - "subject": "CS", - }, - Object { - "classId": 2501, - "description": "Lab for CS 2500", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 1 and Lab for CS 2500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2510, - "description": "Fundamentals of Computer Science 2", - "subject": "CS", - }, - Object { - "classId": 2511, - "description": "Lab for CS 2510", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 2 and Lab for CS 2510", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "description": "Computer Science Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3000, - "description": "Algorithms and Data", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3200, - "description": "Database Design", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3500, - "description": "Object-Oriented Design", - "subject": "CS", - }, - Object { - "classId": 3501, - "description": "Lab for CS 3500", - "subject": "CS", - }, - ], - "description": "Object-Oriented Design and Lab for CS 3500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 2000, - "description": "Principles of Information Science", - "hour": 4, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Khoury Elective Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "With adviser approval, directed study, research, project study, and appropriate graduate-level courses may also be taken as upper-division electives.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Complete 16 credits of upper-division CS, CY, DS, or IS courses that are not already required. Choose courses within the following ranges:", - "hour": 16, - "type": "COMMENT", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 5010, - "subject": "CS", - }, - ], - "hour": 0, - "subject": "CS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4930, - "subject": "CY", - }, - ], - "hour": 0, - "subject": "CY", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "DS", - }, - ], - "hour": 0, - "subject": "DS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "IS", - }, - ], - "hour": 0, - "subject": "IS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - ], - }, - Object { - "description": "History Courses", - "entries": Array [ - Object { - "description": "History Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "Historical Research and Writing", - "hour": 1, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1201, - "description": "First-Year Seminar", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2301, - "description": "The History Seminar", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2302, - "description": "Historical Writing", - "hour": 1, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "description": "History Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one introductory course from the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1150, - "description": "East Asian Studies", - "hour": 0, - "subject": "ASNS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1120, - "description": "Public History, Public Memory", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1130, - "description": "Introduction to the History of the United States", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1170, - "description": "Europe: Empires, Revolutions, Wars, and Their Aftermath", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1185, - "description": "Introduction to Middle Eastern History", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1103, - "description": "Introduction to Women’s, Gender, and Sexuality Studies", - "hour": 0, - "subject": "WMNS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete one course from the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1218, - "description": "Pirates, Planters, and Patriots: Making the Americas, 1492–1804", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1252, - "description": "Japanese Literature and Culture", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1285, - "description": "Introduction to Russian Civilization", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1389, - "description": "History of Espionage 1: Antiquity to World War II", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2330, - "description": "Colonial and Revolutionary America", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Intermediate/Advanced History Cluster", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classIdStart": 2303, - "hour": 0, - "subject": "HIST", - "type": "RANGE_LOWER_BOUNDED", - }, - Object { - "description": "Complete one advanced-level course:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classIdStart": 3000, - "hour": 0, - "subject": "HIST", - "type": "RANGE_LOWER_BOUNDED", - }, - Object { - "description": "History Capstone Seminar or Senior Project", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4701, - "description": "Capstone Seminar", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Integrative Course Requirement", - "entries": Array [ - Object { - "classId": 2211, - "description": "The World Since 1945", - "hour": 4, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses", - "entries": Array [ - Object { - "description": "Research Methods", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course from the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2350, - "description": "Statistics", - "hour": 0, - "subject": "ECON", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3300, - "description": "Geographic Information Systems", - "hour": 0, - "subject": "ENVR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5260, - "description": "Geographical Information Systems", - "hour": 0, - "subject": "ENVR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2400, - "description": "Quantitative Techniques", - "hour": 0, - "subject": "POLS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2320, - "description": "Statistics in Psychological Research", - "hour": 0, - "subject": "PSYC", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computing and Social Issues", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2600, - "description": "Issues in Race, Science, and Technology", - "hour": 0, - "subject": "AFAM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4170, - "description": "The Law, Ethics, and Policy of Data and Digital Technologies", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5240, - "description": "Cyberlaw: Privacy, Ethics, and Digital Rights", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2220, - "description": "History of Technology", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2102, - "description": "Bostonography: The City through Data, Texts, Maps, and Networks", - "hour": 0, - "subject": "INSH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "PHIL", - "type": "OR_COURSE", - }, - Object { - "classId": 1145, - "description": "Technology and Human Values", - "hour": 0, - "subject": "PHIL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1280, - "description": "The Twenty-First-Century Workplace", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2485, - "description": "Environment, Technology, and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4528, - "description": "Computers and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Computer Science Writing Requirement", - "entries": Array [ - Object { - "description": "College Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1102, - "description": "First-Year Writing for Multilingual Writers", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - Object { - "description": "Advanced Writing in the Disciplines", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "This requirement is satisfied by HIST 2302 taken in conjunction with HIST 2301.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 32 credits of general electives.", - "hour": 32, - "type": "COMMENT", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests Test range unbounded (chemical engineering) 1`] = ` -Object { - "majorName": "Chemical Engineering, BSChE", - "programRequiredHours": 134, - "sections": Array [ - Object { - "description": "Engineering", - "entries": Array [ - Object { - "classId": 2308, - "description": "Conservation Principles in Chemical Engineering", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2310, - "description": "Transport Processes 1", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2320, - "description": "Chemical Engineering Thermodynamics 1", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3312, - "description": "Transport Processes 2", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3315, - "description": "Chemical Engineering Experimental Design 1", - "subject": "CHME", - }, - Object { - "classId": 3316, - "description": "Recitation for CHME 3315", - "subject": "CHME", - }, - ], - "description": "Chemical Engineering Experimental Design 1 and Recitation for CHME 3315", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "classId": 3322, - "description": "Chemical Engineering Thermodynamics 2", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 4315, - "description": "Chemical Engineering Experimental Design 2", - "subject": "CHME", - }, - Object { - "classId": 4316, - "description": "Recitation for CHME 4315", - "subject": "CHME", - }, - ], - "description": "Chemical Engineering Experimental Design 2 and Recitation for CHME 4315", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "classId": 4510, - "description": "Chemical Engineering Kinetics", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4512, - "description": "Chemical Engineering Process Control", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Chemical Engineering Capstone", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4701, - "description": "Separations and Process Analysis", - "hour": 4, - "subject": "CHME", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 4703, - "description": "Chemical Process Design Capstone", - "subject": "CHME", - }, - Object { - "classId": 4705, - "description": "Recitation for CHME 4703", - "subject": "CHME", - }, - ], - "description": "Chemical Process Design Capstone and Recitation for CHME 4703", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "description": "Advanced Engineering Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one course numbered between 4000 and 5999 in any of the following subject areas:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "hour": 0, - "subjects": Array [ - "BIOE", - "CHME", - "CIVE", - "EECE", - "ME", - "IE", - "MEIE", - "ENGR", - ], - "type": "RANGE_UNBOUNDED", - }, - Object { - "description": "Supplemental Credit", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "2 semester hours from the following course count toward the engineering requirement:", - "hour": 2, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "3 semester hours from the following course count toward the engineering requirement:", - "hour": 3, - "type": "COMMENT", - }, - Object { - "classId": 1502, - "description": "Cornerstone of Engineering 2 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses: Mathematics/Science", - "entries": Array [ - Object { - "description": "Required Mathematics/Science", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "General Chemistry for Engineers", - "subject": "CHEM", - }, - Object { - "classId": 1153, - "description": "Recitation for CHEM 1151", - "subject": "CHEM", - }, - ], - "description": "General Chemistry for Engineers and Recitation for CHEM 1151", - "hour": 4, - "type": "AND_COURSE", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2321, - "description": "Calculus 3 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2341, - "description": "Differential Equations and Linear Algebra for Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "Physics for Engineering 1", - "subject": "PHYS", - }, - Object { - "classId": 1152, - "description": "Lab for PHYS 1151", - "subject": "PHYS", - }, - Object { - "classId": 1153, - "description": "Interactive Learning Seminar for PHYS 1151", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 1 and Lab for PHYS 1151 and Interactive Learning Seminar for PHYS 1151", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1111, - "description": "General Biology 1", - "hour": 0, - "subject": "BIOL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1115, - "description": "General Biology 1 for Engineers", - "hour": 0, - "subject": "BIOL", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1155, - "description": "Physics for Engineering 2", - "subject": "PHYS", - }, - Object { - "classId": 1156, - "description": "Lab for PHYS 1155", - "subject": "PHYS", - }, - Object { - "classId": 1157, - "description": "Interactive Learning Seminar for PHYS 1155", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 2 and Lab for PHYS 1155 and Interactive Learning Seminar for PHYS 1155", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Supplemental Credit", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "1 semester hour from the following course counts toward the mathematics/science requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses: Advanced Science", - "entries": Array [ - Object { - "description": "Complete one of the following pairs:", - "hour": 5, - "type": "COMMENT", - }, - Object { - "courses": Array [ - Object { - "classId": 2311, - "description": "Organic Chemistry 1", - "subject": "CHEM", - }, - Object { - "classId": 2312, - "description": "Lab for CHEM 2311", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 1 and Lab for CHEM 2311", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2315, - "description": "Organic Chemistry 1 for Chemistry Majors", - "subject": "CHEM", - }, - Object { - "classId": 2316, - "description": "Lab for CHEM 2315", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 1 for Chemistry Majors and Lab for CHEM 2315", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Complete one of the following pairs:", - "hour": 5, - "type": "COMMENT", - }, - Object { - "courses": Array [ - Object { - "classId": 2313, - "description": "Organic Chemistry 2", - "subject": "CHEM", - }, - Object { - "classId": 2314, - "description": "Lab for CHEM 2313", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 2 and Lab for CHEM 2313", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2317, - "description": "Organic Chemistry 2 for Chemistry Majors", - "subject": "CHEM", - }, - Object { - "classId": 2318, - "description": "Lab for CHEM 2317", - "subject": "CHEM", - }, - ], - "description": "Organic Chemistry 2 for Chemistry Majors and Lab for CHEM 2317", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2301, - "description": "Genetics and Molecular Biology", - "hour": 0, - "subject": "BIOL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2327, - "description": "Human Parasitology", - "hour": 0, - "subject": "BIOL", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3421, - "description": "Microbiology", - "subject": "BIOL", - }, - Object { - "classId": 3422, - "description": "Lab for BIOL 3421", - "subject": "BIOL", - }, - ], - "description": "Microbiology and Lab for BIOL 3421", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "classId": 3603, - "description": "Mammalian Systems Physiology", - "hour": 0, - "subject": "BIOL", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3611, - "description": "Biochemistry", - "subject": "BIOL", - }, - Object { - "classId": 3612, - "description": "Lab for BIOL 3611", - "subject": "BIOL", - }, - ], - "description": "Biochemistry and Lab for BIOL 3611", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2321, - "description": "Analytical Chemistry", - "subject": "CHEM", - }, - Object { - "classId": 2322, - "description": "Lab for CHEM 2321", - "subject": "CHEM", - }, - ], - "description": "Analytical Chemistry and Lab for CHEM 2321", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3403, - "description": "Quantum Chemistry and Spectroscopy", - "subject": "CHEM", - }, - Object { - "classId": 3404, - "description": "Lab for CHEM 3403", - "subject": "CHEM", - }, - ], - "description": "Quantum Chemistry and Spectroscopy and Lab for CHEM 3403", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3431, - "description": "Physical Chemistry", - "subject": "CHEM", - }, - Object { - "classId": 3432, - "description": "Lab for CHEM 3431", - "subject": "CHEM", - }, - ], - "description": "Physical Chemistry and Lab for CHEM 3431", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3501, - "description": "Inorganic Chemistry", - "subject": "CHEM", - }, - Object { - "classId": 3502, - "description": "Lab for CHEM 3501", - "subject": "CHEM", - }, - Object { - "classId": 3503, - "description": "Recitation for CHEM 3501", - "subject": "CHEM", - }, - ], - "description": "Inorganic Chemistry and Lab for CHEM 3501 and Recitation for CHEM 3501", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 4628, - "description": "Introduction to Spectroscopy of Organic Compounds", - "subject": "CHEM", - }, - Object { - "classId": 4629, - "description": "Identification of Organic Compounds", - "subject": "CHEM", - }, - ], - "description": "Introduction to Spectroscopy of Organic Compounds and Identification of Organic Compounds", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2302, - "description": "Ecology", - "subject": "EEMB", - }, - Object { - "classId": 2303, - "description": "Lab for EEMB 2302", - "subject": "EEMB", - }, - ], - "description": "Ecology and Lab for EEMB 2302", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "classId": 3460, - "description": "Conservation Biology", - "hour": 0, - "subject": "EEMB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1211, - "description": "Computational Problem Solving in Physics", - "hour": 0, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2303, - "description": "Modern Physics", - "hour": 0, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2371, - "description": "Electronics", - "subject": "PHYS", - }, - Object { - "classId": 2372, - "description": "Lab for PHYS 2371", - "subject": "PHYS", - }, - ], - "description": "Electronics and Lab for PHYS 2371", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "classId": 3601, - "description": "Classical Dynamics", - "hour": 0, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3602, - "description": "Electricity and Magnetism 1", - "hour": 0, - "subject": "PHYS", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Professional Development", - "entries": Array [ - Object { - "description": "Professional Development", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1000, - "description": "First-Year Seminar", - "hour": 1, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2000, - "description": "Introduction to Engineering Co-op Education", - "hour": 1, - "subject": "ENCP", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3000, - "description": "Professional Issues in Engineering", - "hour": 1, - "subject": "ENCP", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "1 semester hour from the following course counts toward the professional development requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1501, - "description": "Cornerstone of Engineering 1 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - Object { - "description": "1 semester hour from the following course counts toward the professional development requirement:", - "hour": 1, - "type": "COMMENT", - }, - Object { - "classId": 1502, - "description": "Cornerstone of Engineering 2 1", - "hour": 0, - "subject": "GE", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Writing Requirements", - "entries": Array [ - Object { - "description": "A grade of C or higher is required:", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Advanced Writing in the Technical Professions", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Interdisciplinary Advanced Writing in the Disciplines", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 24 semester hours of academic, nonremedial, nonrepetitive courses.", - "hour": 24, - "type": "COMMENT", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests multiple of the same class per AND (cs) 1`] = ` -Object { - "majorName": "Computer Science, BSCS", - "programRequiredHours": 134, - "sections": Array [ - Object { - "description": "Computer Science Requirements", - "entries": Array [ - Object { - "description": "Computer Science Overview", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1200, - "description": "First Year Seminar", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1210, - "description": "Professional Development for Khoury Co-op", - "hour": 1, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computer Science Fundamental Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1800, - "description": "Discrete Structures", - "subject": "CS", - }, - Object { - "classId": 1802, - "description": "Seminar for CS 1800", - "subject": "CS", - }, - ], - "description": "Discrete Structures and Seminar for CS 1800", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2500, - "description": "Fundamentals of Computer Science 1", - "subject": "CS", - }, - Object { - "classId": 2501, - "description": "Lab for CS 2500", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 1 and Lab for CS 2500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2510, - "description": "Fundamentals of Computer Science 2", - "subject": "CS", - }, - Object { - "classId": 2511, - "description": "Lab for CS 2510", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 2 and Lab for CS 2510", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 2810, - "description": "Mathematics of Data Models", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computer Science Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3000, - "description": "Algorithms and Data", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3500, - "description": "Object-Oriented Design", - "subject": "CS", - }, - Object { - "classId": 3501, - "description": "Lab for CS 3500", - "subject": "CS", - }, - ], - "description": "Object-Oriented Design and Lab for CS 3500", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "classId": 3650, - "description": "Computer Systems", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3800, - "description": "Theory of Computation", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4500, - "description": "Software Development", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4530, - "description": "Fundamentals of Software Engineering", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "description": "Security Required Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2550, - "description": "Foundations of Cybersecurity", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3740, - "description": "Systems Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4740, - "description": "Network Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Presentation Requirement", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Choose one:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1112, - "description": "Public Speaking", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1113, - "description": "Business and Professional Speaking", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1210, - "description": "Persuasion and Rhetoric", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1511, - "description": "Communication and Storytelling", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1125, - "description": "Improvisation", - "hour": 0, - "subject": "THTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1130, - "description": "Introduction to Acting", - "hour": 0, - "subject": "THTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1180, - "description": "The Dynamic On-Screen Presenter", - "hour": 0, - "subject": "THTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2345, - "description": "Acting for the Camera", - "hour": 0, - "subject": "THTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Khoury Elective Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "With adviser approval, directed study, research, project study, and appropriate graduate-level courses may also be taken as upper-division electives.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Complete 8 credits of CS, CY, DS, or IS classes that are not already required. Choose courses within the following ranges:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 5010, - "subject": "CS", - }, - ], - "hour": 0, - "subject": "CS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4930, - "subject": "CY", - }, - ], - "hour": 0, - "subject": "CY", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2500, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "DS", - }, - ], - "hour": 0, - "subject": "DS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - Object { - "classIdStart": 2000, - "exceptions": Array [ - Object { - "classId": 4900, - "subject": "IS", - }, - ], - "hour": 0, - "subject": "IS", - "type": "RANGE_LOWER_BOUNDED_WITH_EXCEPTIONS", - }, - ], - }, - Object { - "description": "Supporting Courses", - "entries": Array [ - Object { - "description": "Mathematics Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1365, - "description": "Introduction to Mathematical Reasoning", - "hour": 4, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Computing and Social Issues", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2600, - "description": "Issues in Race, Science, and Technology", - "hour": 0, - "subject": "AFAM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4170, - "description": "The Law, Ethics, and Policy of Data and Digital Technologies", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5240, - "description": "Cyberlaw: Privacy, Ethics, and Digital Rights", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2150, - "description": "Literature and Digital Diversity", - "hour": 0, - "subject": "ENGL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2220, - "description": "History of Technology", - "hour": 0, - "subject": "HIST", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2102, - "description": "Bostonography: The City through Data, Texts, Maps, and Networks", - "hour": 0, - "subject": "INSH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1300, - "description": "Knowledge in a Digital World", - "hour": 0, - "subject": "PHIL", - "type": "OR_COURSE", - }, - Object { - "classId": 1145, - "description": "Technology and Human Values", - "hour": 0, - "subject": "PHIL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1280, - "description": "The Twenty-First-Century Workplace", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2485, - "description": "Environment, Technology, and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4528, - "description": "Computers and Society", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electrical Engineering", - "hour": 0, - "type": "HEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 2322, - "description": "Fundamentals of Digital Design and Computer Organization", - "subject": "EECE", - }, - Object { - "classId": 2323, - "description": "Lab for EECE 2322", - "subject": "EECE", - }, - ], - "description": "Fundamentals of Digital Design and Computer Organization and Lab for EECE 2322", - "hour": 5, - "type": "AND_COURSE", - }, - Object { - "description": "Science Requirement", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two courses (and any required labs) from the following science categories:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "description": "Biology", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1111, - "description": "General Biology 1", - "subject": "BIOL", - }, - Object { - "classId": 1112, - "description": "Lab for BIOL 1111", - "subject": "BIOL", - }, - ], - "description": "General Biology 1 and Lab for BIOL 1111", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1113, - "description": "General Biology 2", - "subject": "BIOL", - }, - Object { - "classId": 1114, - "description": "Lab for BIOL 1113", - "subject": "BIOL", - }, - ], - "description": "General Biology 2 and Lab for BIOL 1113", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2301, - "description": "Genetics and Molecular Biology", - "subject": "BIOL", - }, - Object { - "classId": 2302, - "description": "Lab for BIOL 2301", - "subject": "BIOL", - }, - ], - "description": "Genetics and Molecular Biology and Lab for BIOL 2301", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Chemistry", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1211, - "description": "General Chemistry 1", - "subject": "CHEM", - }, - Object { - "classId": 1212, - "description": "Lab for CHEM 1211", - "subject": "CHEM", - }, - Object { - "classId": 1213, - "description": "Recitation for CHEM 1211", - "subject": "CHEM", - }, - ], - "description": "General Chemistry 1 and Lab for CHEM 1211 and Recitation for CHEM 1211", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1214, - "description": "General Chemistry 2", - "subject": "CHEM", - }, - Object { - "classId": 1215, - "description": "Lab for CHEM 1214", - "subject": "CHEM", - }, - Object { - "classId": 1216, - "description": "Recitation for CHEM 1214", - "subject": "CHEM", - }, - ], - "description": "General Chemistry 2 and Lab for CHEM 1214 and Recitation for CHEM 1214", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Geology/Environmental Science", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1200, - "description": "Dynamic Earth", - "subject": "ENVR", - }, - Object { - "classId": 1201, - "description": "Lab for ENVR 1200", - "subject": "ENVR", - }, - ], - "description": "Dynamic Earth and Lab for ENVR 1200", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1202, - "description": "History of Earth and Life", - "subject": "ENVR", - }, - Object { - "classId": 1203, - "description": "Interpreting Earth History", - "subject": "ENVR", - }, - ], - "description": "History of Earth and Life and Interpreting Earth History", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1200, - "description": "Dynamic Earth", - "subject": "ENVR", - }, - Object { - "classId": 1201, - "description": "Lab for ENVR 1200", - "subject": "ENVR", - }, - ], - "description": "Dynamic Earth and Lab for ENVR 1200", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2310, - "description": "Earth Materials", - "subject": "ENVR", - }, - Object { - "classId": 2311, - "description": "Lab for ENVR 2310", - "subject": "ENVR", - }, - ], - "description": "Earth Materials and Lab for ENVR 2310", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2340, - "description": "Earth Landforms and Processes", - "subject": "ENVR", - }, - Object { - "classId": 2341, - "description": "Lab for ENVR 2340", - "subject": "ENVR", - }, - ], - "description": "Earth Landforms and Processes and Lab for ENVR 2340", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3300, - "description": "Geographic Information Systems", - "subject": "ENVR", - }, - Object { - "classId": 3301, - "description": "Lab for ENVR 3300", - "subject": "ENVR", - }, - ], - "description": "Geographic Information Systems and Lab for ENVR 3300", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 4500, - "description": "Applied Hydrogeology", - "subject": "ENVR", - }, - Object { - "classId": 4501, - "description": "Lab for ENVR 4500", - "subject": "ENVR", - }, - ], - "description": "Applied Hydrogeology and Lab for ENVR 4500", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1202, - "description": "History of Earth and Life", - "subject": "ENVR", - }, - Object { - "classId": 1203, - "description": "Interpreting Earth History", - "subject": "ENVR", - }, - ], - "description": "History of Earth and Life and Interpreting Earth History", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 5242, - "description": "Ancient Marine Life", - "subject": "ENVR", - }, - Object { - "classId": 5243, - "description": "Lab for ENVR 5242", - "subject": "ENVR", - }, - ], - "description": "Ancient Marine Life and Lab for ENVR 5242", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "description": "Mathematics", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2331, - "description": "Linear Algebra", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3081, - "description": "Probability and Statistics", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Physics", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "courses": Array [ - Object { - "classId": 1145, - "description": "Physics for Life Sciences 1", - "subject": "PHYS", - }, - Object { - "classId": 1146, - "description": "Lab for PHYS 1145", - "subject": "PHYS", - }, - ], - "description": "Physics for Life Sciences 1 and Lab for PHYS 1145", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1147, - "description": "Physics for Life Sciences 2", - "subject": "PHYS", - }, - Object { - "classId": 1148, - "description": "Lab for PHYS 1147", - "subject": "PHYS", - }, - ], - "description": "Physics for Life Sciences 2 and Lab for PHYS 1147", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1151, - "description": "Physics for Engineering 1", - "subject": "PHYS", - }, - Object { - "classId": 1152, - "description": "Lab for PHYS 1151", - "subject": "PHYS", - }, - Object { - "classId": 1153, - "description": "Interactive Learning Seminar for PHYS 1151", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 1 and Lab for PHYS 1151 and Interactive Learning Seminar for PHYS 1151", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1155, - "description": "Physics for Engineering 2", - "subject": "PHYS", - }, - Object { - "classId": 1156, - "description": "Lab for PHYS 1155", - "subject": "PHYS", - }, - Object { - "classId": 1157, - "description": "Interactive Learning Seminar for PHYS 1155", - "subject": "PHYS", - }, - ], - "description": "Physics for Engineering 2 and Lab for PHYS 1155 and Interactive Learning Seminar for PHYS 1155", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1161, - "description": "Physics 1", - "subject": "PHYS", - }, - Object { - "classId": 1162, - "description": "Lab for PHYS 1161", - "subject": "PHYS", - }, - Object { - "classId": 1163, - "description": "Recitation for PHYS 1161", - "subject": "PHYS", - }, - ], - "description": "Physics 1 and Lab for PHYS 1161 and Recitation for PHYS 1161", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 1165, - "description": "Physics 2", - "subject": "PHYS", - }, - Object { - "classId": 1166, - "description": "Lab for PHYS 1165", - "subject": "PHYS", - }, - Object { - "classId": 1167, - "description": "Recitation for PHYS 1165", - "subject": "PHYS", - }, - ], - "description": "Physics 2 and Lab for PHYS 1165 and Recitation for PHYS 1165", - "hour": 0, - "type": "AND_COURSE", - }, - ], - }, - Object { - "description": "Computer Science Writing Requirement", - "entries": Array [ - Object { - "description": "College Writing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1111, - "description": "First-Year Writing", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Advanced Writing in the Disciplines", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3302, - "description": "Advanced Writing in the Technical Professions", - "hour": 4, - "subject": "ENGW", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Interdisciplinary Advanced Writing in the Disciplines", - "hour": 0, - "subject": "ENGW", - "type": "OR_COURSE", - }, - ], - }, - Object { - "description": "Required General Electives", - "entries": Array [ - Object { - "description": "Complete 28 credits of general electives.", - "hour": 28, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Artificial Intelligence", - "entries": Array [ - Object { - "classId": 4100, - "description": "Artificial Intelligence", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4400, - "description": "Machine Learning and Data Mining 1", - "hour": 4, - "subject": "DS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete two of the following courses not already taken:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 4120, - "description": "Natural Language Processing", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4150, - "description": "Game Artificial Intelligence", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Robotic Science and Systems", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4200, - "description": "Information Retrieval", - "hour": 0, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4420, - "description": "Machine Learning and Data Mining 2", - "hour": 0, - "subject": "DS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3466, - "description": "Cognition", - "hour": 0, - "subject": "PSYC", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Foundations", - "entries": Array [ - Object { - "description": "Complete two of the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2800, - "description": "Logic and Computation", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4820, - "description": "Computer-Aided Reasoning", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "classId": 4805, - "description": "Fundamentals of Complexity Theory", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4810, - "description": "Advanced Algorithms", - "hour": 0, - "subject": "CS", - "type": "OR_COURSE", - }, - Object { - "description": "Complete two of the following courses not already taken:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 4805, - "description": "Fundamentals of Complexity Theory", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4810, - "description": "Advanced Algorithms", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4820, - "description": "Computer-Aided Reasoning", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4830, - "description": "System Specification, Verification, and Synthesis", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 3950, - "description": "Introduction to Computer Science Research", - "subject": "CS", - }, - Object { - "classId": 4950, - "description": "Computer Science Research Seminar", - "subject": "CS", - }, - Object { - "classId": 4950, - "description": "Computer Science Research Seminar", - "subject": "CS", - }, - ], - "description": "Introduction to Computer Science Research and Computer Science Research Seminar and Computer Science Research Seminar", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "classId": 4770, - "description": "Cryptography", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Human-Centered Computing", - "entries": Array [ - Object { - "classId": 4300, - "description": "Human Computer Interaction", - "hour": 4, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4800, - "description": "Empirical Research Methods", - "hour": 4, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete two of the following courses not already taken:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2000, - "description": "Principles of Information Science", - "hour": 0, - "subject": "IS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4120, - "description": "Natural Language Processing", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4550, - "description": "Web Development", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4200, - "description": "Information Presentation and Visualization", - "hour": 0, - "subject": "DS", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Software", - "entries": Array [ - Object { - "classId": 2800, - "description": "Logic and Computation", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3700, - "description": "Networks and Distributed Systems", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4400, - "description": "Programming Languages", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete one of the following courses not already taken:", - "hour": 0, - "type": "COMMENT", - }, - Object { - "classId": 3520, - "description": "Programming in C++", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3620, - "description": "Building Extensible Systems", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4240, - "description": "Large-Scale Parallel Data Processing", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4410, - "description": "Compilers", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4550, - "description": "Web Development", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4820, - "description": "Computer-Aided Reasoning", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4830, - "description": "System Specification, Verification, and Synthesis", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Systems", - "entries": Array [ - Object { - "classId": 3700, - "description": "Networks and Distributed Systems", - "hour": 4, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete one of the following courses not already taken:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3740, - "description": "Systems Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4740, - "description": "Network Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Complete two of the following courses not already taken:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 3520, - "description": "Programming in C++", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4300, - "description": "Computer Graphics", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3740, - "description": "Systems Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Robotic Science and Systems", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4710, - "description": "Mobile and Wireless Systems", - "hour": 0, - "subject": "CS", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4740, - "description": "Network Security", - "hour": 0, - "subject": "CY", - "type": "PLAIN_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; - -exports[`scraper v2 snapshot tests nested linked concentration pages (business) 1`] = ` -Object { - "majorName": "Bachelor of Science in Business Administration, BSBA", - "programRequiredHours": 128, - "sections": Array [ - Object { - "description": "Business Requirements", - "entries": Array [ - Object { - "description": "Accounting", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1201, - "description": "Financial Accounting and Reporting", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2301, - "description": "Managerial Accounting", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Finance", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2201, - "description": "Financial Management", - "hour": 4, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Marketing", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2201, - "description": "Introduction to Marketing", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Management Information Systems", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Management Information Systems", - "hour": 4, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Operations Management and Supply Chain Management", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Supply Chain and Operations Management", - "hour": 4, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Organizational Behavior", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3201, - "description": "Organizational Behavior", - "hour": 4, - "subject": "ORGB", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Strategy in Action", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4501, - "description": "Strategy in Action", - "hour": 4, - "subject": "STRT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "International Business/Social Responsibility", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1203, - "description": "International Business and Global Social Responsibility", - "hour": 4, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Statistics", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Business Statistics", - "hour": 4, - "subject": "MGSC", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Accounting", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3401, - "description": "Financial Reporting and Analysis 1", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4501, - "description": "Financial Reporting and Analysis 2", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 3403, - "description": "Advisory Services and Emerging Accounting Systems", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3416, - "description": "Strategic Cost Analysis for Decision Making", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4412, - "description": "Auditing and Other Assurance Services", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4414, - "description": "Income Tax Determination and Planning", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Accounting and Advisory Services", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3402, - "description": "Financial Reporting and Financial Statement Analysis", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3403, - "description": "Advisory Services and Emerging Accounting Systems", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 5220, - "description": "Data Analytics for Advisory Services", - "hour": 4, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "", - "entries": Array [ - Object { - "description": "Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3416, - "description": "Strategic Cost Analysis for Decision Making", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4414, - "description": "Income Tax Determination and Planning", - "hour": 0, - "subject": "ACCT", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Brand Management", - "entries": Array [ - Object { - "description": "Required Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3720, - "description": "Brand Management", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three electives, with at least 8 credits from MKTG courses:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 2301, - "description": "Innovation!", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3530, - "description": "Project Management", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2720, - "description": "Enabling Technologies for Consumer Engagement", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3401, - "description": "Marketing Research", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3402, - "description": "Gaining Insights from Consumer Data", - "hour": 0, - "subject": "MKTG", - "type": "OR_COURSE", - }, - Object { - "classId": 4502, - "description": "Managing Customer Engagement in a Service World", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4504, - "description": "Advertising and Brand Promotion", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4506, - "description": "Consumer Behavior", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4510, - "description": "New Product Development", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3330, - "description": "Design Thinking for Startups", - "hour": 0, - "subject": "ENTR", - "type": "OR_COURSE", - }, - Object { - "classId": 3335, - "description": "Product Innovation and Portfolio Management", - "hour": 0, - "subject": "ENTR", - "type": "OR_COURSE", - }, - Object { - "classId": 4720, - "description": "Understanding the Platform Economy", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "description": "A maximum of one course may be applied to requirements of a second concentration.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Business Analytics", - "entries": Array [ - Object { - "description": "Only one course may double count between another concentration or minor.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2510, - "description": "Fundamentals of Information Analytics", - "hour": 4, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3403, - "description": "Data Management in the Enterprise", - "hour": 4, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3405, - "description": "Data Wrangling for Business Analytics", - "hour": 0, - "subject": "MISM", - "type": "OR_COURSE", - }, - Object { - "classId": 3501, - "description": "Information Visualization for Business", - "hour": 4, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3515, - "description": "Data Mining for Business", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3525, - "description": "Modeling for Business Analytics", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Management Information Systems", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Corporate Innovation", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Innovation!", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4504, - "description": "Corporate Innovation Seminar", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two electives, one of which may come from the additional electives list:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2304, - "description": "Industry Disruption and Corporate Transformation", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3335, - "description": "Product Innovation and Portfolio Management", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4225, - "description": "Growth, Acquisitions, and Alliances", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Electives", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 2206, - "description": "Global Social Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2215, - "description": "Understanding Family Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2303, - "description": "Marketing Strategies for Startups", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Entrepreneurship", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Entrepreneurial Finance and Private Equity", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3340, - "description": "Healthcare Management, Innovation, and Design", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3530, - "description": "Project Management", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4301, - "description": "Strategic Analysis and Decision Making", - "hour": 0, - "subject": "STRT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "One course from an approved Dialogue may count toward a concentration elective.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Entrepreneurial Startups", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Innovation!", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4501, - "description": "Entrepreneurship Seminar", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two electives, one of which may come from the Additional Electives list:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2303, - "description": "Marketing Strategies for Startups", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3305, - "description": "Business Model Innovation for Entrepreneurs", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3330, - "description": "Design Thinking for Startups", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Electives", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 2206, - "description": "Global Social Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2215, - "description": "Understanding Family Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2304, - "description": "Industry Disruption and Corporate Transformation", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4505, - "description": "Entrepreneurial Venture Growth Strategies", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Entrepreneurship", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Entrepreneurial Finance and Private Equity", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "One course from an approved Dialogue may count toward an elective.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Family Business", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Innovation!", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4503, - "description": "Owner and Family Managed Business Seminar", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two electives, one of which may come from the Additional Elective list:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2215, - "description": "Understanding Family Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Managing and Growing the Family Business", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3401, - "description": "Consulting Operations Growth in SMEs", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Electives", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 2206, - "description": "Global Social Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2303, - "description": "Marketing Strategies for Startups", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2304, - "description": "Industry Disruption and Corporate Transformation", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Entrepreneurship", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Entrepreneurial Finance and Private Equity", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Students may use one course from an approved Dialogue experience toward the elective area of the concentration.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Finance", - "entries": Array [ - Object { - "description": "Required Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3301, - "description": "Corporate Finance", - "hour": 4, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3303, - "description": "Investments", - "hour": 0, - "subject": "FINA", - "type": "OR_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three of the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 3520, - "description": "Impact Investing and Social Finance", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2720, - "description": "Sustainability in the Business Environment", - "hour": 0, - "subject": "FINA", - "type": "OR_COURSE", - }, - Object { - "classId": 2730, - "description": "Fintech and Financial Innovation", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3301, - "description": "Corporate Finance (if not selected as a required course)", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3303, - "description": "Investments (if not selected as a required course)", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4219, - "description": "Portfolio Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4220, - "description": "Behavioral Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4310, - "description": "Working Capital Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4312, - "description": "Issues in Corporate Governance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4320, - "description": "International Financial Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4330, - "description": "Emerging Financial Markets", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4340, - "description": "Blockchain Applications in Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4350, - "description": "Applied Financial Econometrics", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4370, - "description": "Financial Modeling", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4380, - "description": "Financial Data Analytics", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4390, - "description": "Machine Learning in Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4410, - "description": "Valuation and Value Creation", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4412, - "description": "Personal Financial Planning", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4420, - "description": "Mergers and Acquisitions", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4512, - "description": "Financial Risk Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4514, - "description": "Investment Banking", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4516, - "description": "Real Estate Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4524, - "description": "Credit Analysis", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4526, - "description": "Core Topics in Alternative Investments", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4602, - "description": "Turnaround Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4604, - "description": "Fixed-Income Securities", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4608, - "description": "Advanced Financial Strategy", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Entrepreneurial Finance and Private Equity", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Fintech", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3303, - "description": "Investments", - "hour": 4, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3301, - "description": "Corporate Finance", - "hour": 0, - "subject": "FINA", - "type": "OR_COURSE", - }, - Object { - "classId": 4380, - "description": "Financial Data Analytics", - "hour": 4, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2730, - "description": "Fintech and Financial Innovation", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4340, - "description": "Blockchain Applications in Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4350, - "description": "Applied Financial Econometrics", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4390, - "description": "Machine Learning in Finance", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Healthcare Management and Consulting", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3340, - "description": "Healthcare Management, Innovation, and Design", - "hour": 4, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Managing Healthcare Operations and Supply Chain", - "hour": 4, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Implementation/Consulting Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3110, - "description": "The Consulting Environment", - "hour": 0, - "subject": "BUSN", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3220, - "description": "International Entrepreneurship and Innovation Consulting", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3501, - "description": "Information Visualization for Business", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3515, - "description": "Data Mining for Business", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Managing Organizational Change and Disruption", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3530, - "description": "Project Management", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3420, - "description": "Managing Human Capital", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4410, - "description": "Workforce Analytics", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4550, - "description": "Management Consulting in Organizations", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Interdisciplinary Healthcare Focus", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3201, - "description": "Health Communication", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4533, - "description": "Consultation Skills", - "hour": 0, - "subject": "COMM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2970, - "description": "Research Methods for Human Services", - "hour": 0, - "subject": "HUSV", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3500, - "description": "Introduction to Healthcare Systems Engineering", - "hour": 0, - "subject": "IE", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2515, - "description": "Healthcare Policy and Administration", - "hour": 0, - "subject": "PHTH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4515, - "description": "Critical Issues in Health and Public-Health Policy", - "hour": 0, - "subject": "PHTH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3440, - "description": "Sociology of Human Service Organizations", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3441, - "description": "Sociology of Health and Illness", - "hour": 0, - "subject": "SOCL", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in International Business", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1203, - "description": "International Business and Global Social Responsibility", - "hour": 4, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1209, - "description": "International Business and Global Social Responsibility", - "hour": 0, - "subject": "INTB", - "type": "OR_COURSE", - }, - Object { - "description": "Complete one or both of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 2501, - "description": "Competing to Win in Emerging Markets", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3310, - "description": "Cultural Aspects of International Business", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following courses (one if both courses above selected).", - "hour": 4, - "type": "COMMENT", - }, - Object { - "description": "Note: One course can also count toward a different business concentration.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Managing Internationally", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 3205, - "description": "Understanding and Managing Cultural Differences", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3320, - "description": "International Business Management and Environment", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4202, - "description": "Executing Global Strategy", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in International Business", - "hour": 0, - "subject": "INTB", - "type": "PLAIN_COURSE", - }, - Object { - "description": "International Functional Knowledge", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 2206, - "description": "Global Social Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3306, - "description": "Global Entrepreneurship", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4320, - "description": "International Financial Management", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4512, - "description": "International Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3301, - "description": "Global Supply Chain Strategy", - "hour": 0, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives not listed may be approved via the undergraduate dean’s office.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Management", - "entries": Array [ - Object { - "description": "Required Course", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 4550, - "description": "Management Consulting in Organizations", - "hour": 4, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Note: Only one non-MGMT course may be used as an elective.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Complete three of the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 2215, - "description": "Understanding Family Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2414, - "description": "Social Responsibility of Business in an Age of Inequality", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4225, - "description": "Growth, Acquisitions, and Alliances", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3305, - "description": "Power and Influence", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Managing Organizational Change and Disruption", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3340, - "description": "Healthcare Management, Innovation, and Design", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3350, - "description": "Managing a Diverse Workforce", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3360, - "description": "Law and the Legal Process", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3380, - "description": "Leading with Character", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3420, - "description": "Managing Human Capital", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3435, - "description": "Social Networks and Organizations", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3530, - "description": "Project Management", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4310, - "description": "The Management Practices of Great Organizations", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4410, - "description": "Workforce Analytics", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Management", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Management Information Systems", - "entries": Array [ - Object { - "description": "Only one course may double count between another concentration or minor.", - "hour": 0, - "type": "COMMENT", - }, - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3403, - "description": "Data Management in the Enterprise", - "hour": 4, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three of the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 2510, - "description": "Fundamentals of Information Analytics", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3501, - "description": "Information Visualization for Business", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3515, - "description": "Data Mining for Business", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4501, - "description": "Business Systems Integration", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Management Information Systems", - "hour": 0, - "subject": "MISM", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Marketing", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2602, - "description": "Quantitative Analysis of Consumer Data", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3401, - "description": "Marketing Research", - "hour": 0, - "subject": "MKTG", - "type": "OR_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete three of the following:", - "hour": 12, - "type": "COMMENT", - }, - Object { - "classId": 2301, - "description": "Marketing and Society", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2602, - "description": "Quantitative Analysis of Consumer Data (if not taken as a required course)", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2720, - "description": "Enabling Technologies for Consumer Engagement", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3301, - "description": "Marketing Management", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3401, - "description": "Marketing Research (if not taken as a required course)", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3402, - "description": "Gaining Insights from Consumer Data", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3501, - "description": "Marketing Analytics", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3720, - "description": "Brand Management", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4120, - "description": "Undergraduate Research Practicum in Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4420, - "description": "Sales Management", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4502, - "description": "Managing Customer Engagement in a Service World", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4504, - "description": "Advertising and Brand Promotion", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4506, - "description": "Consumer Behavior", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4508, - "description": "Digital Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4510, - "description": "New Product Development", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4512, - "description": "International Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4520, - "description": "Business-to-Business Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4720, - "description": "Understanding the Platform Economy", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Marketing", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Marketing Analytics", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2602, - "description": "Quantitative Analysis of Consumer Data", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3402, - "description": "Gaining Insights from Consumer Data", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3501, - "description": "Marketing Analytics", - "hour": 4, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "courses": Array [ - Object { - "classId": 2500, - "description": "Fundamentals of Computer Science 1", - "subject": "CS", - }, - Object { - "classId": 2501, - "description": "Lab for CS 2500", - "subject": "CS", - }, - ], - "description": "Fundamentals of Computer Science 1 and Lab for CS 2500", - "hour": 0, - "type": "AND_COURSE", - }, - Object { - "courses": Array [ - Object { - "classId": 2000, - "description": "Programming with Data", - "subject": "DS", - }, - Object { - "classId": 2001, - "description": "Data Science Programming Practicum", - "subject": "DS", - }, - ], - "description": "Programming with Data and Data Science Programming Practicum", - "hour": 0, - "type": "AND_COURSE", - }, - ], - }, - Object { - "description": "Concentration in Social Innovation and Entrepreneurship", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 2301, - "description": "Innovation!", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4506, - "description": "Seminar in Social Innovation and Entrepreneurship", - "hour": 4, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Electives", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete two of the following, one of which may come from the additional elective list:", - "hour": 8, - "type": "COMMENT", - }, - Object { - "classId": 2206, - "description": "Global Social Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2414, - "description": "Social Responsibility of Business in an Age of Inequality", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3520, - "description": "Impact Investing and Social Finance", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Additional Electives", - "hour": 0, - "type": "SUBHEADER", - }, - Object { - "classId": 2215, - "description": "Understanding Family Enterprise", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2303, - "description": "Marketing Strategies for Startups", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2304, - "description": "Industry Disruption and Corporate Transformation", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Entrepreneurship", - "hour": 0, - "subject": "ENTR", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4610, - "description": "Entrepreneurial Finance and Private Equity", - "hour": 0, - "subject": "FINA", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3302, - "description": "Negotiating in Business", - "hour": 0, - "subject": "MGMT", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 2301, - "description": "Marketing and Society", - "hour": 0, - "subject": "MKTG", - "type": "PLAIN_COURSE", - }, - Object { - "description": "One course from an approved Dialogue may count toward an elective.", - "hour": 0, - "type": "COMMENT", - }, - ], - }, - Object { - "description": "Concentration in Supply Chain Management", - "entries": Array [ - Object { - "description": "Required Courses", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 3301, - "description": "Global Supply Chain Strategy", - "hour": 4, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3305, - "description": "Sourcing, Procurement, and Negotiation", - "hour": 4, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3310, - "description": "Logistics and Transportation Management", - "hour": 4, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Elective", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 3308, - "description": "Supply Chain Analytics and Emerging Technologies", - "hour": 0, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 3315, - "description": "Managing Healthcare Operations and Supply Chain", - "hour": 0, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4401, - "description": "Contemporary Topics in Supply Chain Management", - "hour": 0, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 4983, - "description": "Special Topics in Supply Chain Management", - "hour": 0, - "subject": "SCHM", - "type": "PLAIN_COURSE", - }, - ], - }, - Object { - "description": "Supporting Courses", - "entries": Array [ - Object { - "description": "Mathematics", - "hour": 0, - "type": "HEADER", - }, - Object { - "description": "Complete one of the following:", - "hour": 4, - "type": "COMMENT", - }, - Object { - "classId": 1231, - "description": "Calculus for Business and Economics", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1241, - "description": "Calculus 1", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1242, - "description": "Calculus 2", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1251, - "description": "Calculus and Differential Equations for Biology 1", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1252, - "description": "Calculus and Differential Equations for Biology 2", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1340, - "description": "Intensive Calculus for Engineers", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1341, - "description": "Calculus 1 for Science and Engineering", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1342, - "description": "Calculus 2 for Science and Engineering", - "hour": 0, - "subject": "MATH", - "type": "PLAIN_COURSE", - }, - Object { - "description": "Macroeconomics and Microeconomics", - "hour": 0, - "type": "HEADER", - }, - Object { - "classId": 1115, - "description": "Principles of Macroeconomics", - "hour": 4, - "subject": "ECON", - "type": "PLAIN_COURSE", - }, - Object { - "classId": 1116, - "description": "Principles of Microeconomics", - "hour": 4, - "subject": "ECON", - "type": "PLAIN_COURSE", - }, - ], - }, - ], - "yearVersion": 2022, -} -`; diff --git a/packages/scrapers-v2/test/__snapshots__/urls.test.ts.snap b/packages/scrapers-v2/test/__snapshots__/urls.test.ts.snap deleted file mode 100644 index a00d46615..000000000 --- a/packages/scrapers-v2/test/__snapshots__/urls.test.ts.snap +++ /dev/null @@ -1,472 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`url scraper scrape 1`] = ` -Array [ - "https://catalog.northeastern.edu/https:/nextcatalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architectural-design-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architectural-history-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architectural-science-systems-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architectural-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architectural-studies-design-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architecture-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/architecture-english-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/landscape-architecture-bla/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/architecture/urban-landscape-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/animation-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/art-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/art-history-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/art-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/creative-computing-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/creative-fabrication-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/design-bfa/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/experience-design-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/game-art-animation-bfa/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/game-art-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/game-design-bfa/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/game-design-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/game-design-music-concentration-music-technology-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/graphic-and-information-design-mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/graphic-and-information-design-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/immersive-media-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/interaction-design-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/media-arts-bfa/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/media-arts-communication-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/photography-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/studio-art-bfa/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/art-design/video-arts-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/argumentation-law-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/cinema-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication--media-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-graphic-information-design-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-sociology-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-speech-language-pathology-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/communication-studies-theatre-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/digital-communications-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/film-production-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/film-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/human-communication-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/improvisation-storytelling-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-production-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-english-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-history-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-journalism-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-media-art-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-sociology-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-theatre-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/media-screen-studies-theatre-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/oratory-public-speaking-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/political-communication-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/rhetoric-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/social-activism-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/sports-media-communication-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/interdisciplinary/creativity-theory-practice-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-english-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-interaction-design-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-practice-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/journalism-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/photojournalism-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/journalism/public-relations-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/ethnomusicology-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-communication-studies-concentration-industry/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-composition-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-concentration-industry-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-concentration-music-technology-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-industry-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-performance-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-recording-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/music-technology-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/music/songwriting-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/global-fashion-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/performing-arts-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/playwriting-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-interaction-design-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-interaction-design-bs/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-journalism-ba/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatre-performance-social-change-minor/", - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/theatrical-design-minor/", - "https://catalog.northeastern.edu/undergraduate/business/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/business/business-administration-bsba/", - "https://catalog.northeastern.edu/undergraduate/business/business-administration-combined-majors/business-administration-communication-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/business/business-administration-combined-majors/business-administration-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/business/business-administration-combined-majors/design-bs/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/accounting-advisory-services/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/accounting/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/brand-management/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/business-analytics/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/corporate-innovation/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/entrepreneurial-startups/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/family-business/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/finance/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/fintech/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/global-business-strategy/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/healthcare-management-consulting/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/international-business/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/management-information-systems/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/management/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/marketing-analytics/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/marketing/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/social-innovation-entrepreneurship/", - "https://catalog.northeastern.edu/undergraduate/business/concentrations/supply-chain-management/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/accounting-advisory-services-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/brand-management-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/business-administration-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/business-analytics-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/consulting-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/corporate-innovation-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/emerging-markets-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/entrepreneurial-startups-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/family-business-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/leadership-human-capital-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/management-information-systems-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/marketing-analytics-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/marketing-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/social-innovation-entrepreneurship-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/strategy-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/supply-chain-management-minor/", - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/sustainable-business-practices-minor/", - "https://catalog.northeastern.edu/undergraduate/business/international-business-bsib/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-behavioral-neuroscience-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-cognitive-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-communication-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-concentration-music-composition-technology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-criminal-justice-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-design-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-english-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-environmental-sustainability-sciences-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-game-development-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-history-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-linguistics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-media-arts-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-physics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-political-science-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-theatre-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/cybersecurity-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-behavioral-neuroscience-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-biochemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-chemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-ecology-evolutionary-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-environmental-sustainability-sciences-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-health-science-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-international-affairs-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-linguistics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-physics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/data-science-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-information-science-combined-majors/economics-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/bacs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/bscs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/cybersecurity-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/cybersecurity-criminal-justice-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/computer-science/minor/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/cybersecurity/cybersecurity-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/data-science/data-science-bs/", - "https://catalog.northeastern.edu/undergraduate/computer-information-science/data-science/data-science-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/engineering/bioengineering/bioengineering-biochemistry-bsbioe/", - "https://catalog.northeastern.edu/undergraduate/engineering/bioengineering/bioengineering-bsbioe/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/biochemical-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-biochemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-bioengineering-bsche/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-bsche/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-computer-science-bsche/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-environmental-engineering-bsche/", - "https://catalog.northeastern.edu/undergraduate/engineering/chemical/chemical-engineering-physics-bsche/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/architectural-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/civil-engineering-architectural-studies-bsce/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/civil-engineering-bsce/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/civil-engineering-computer-science-bsce/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/civil-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/environmental-engineering-bsenve/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/environmental-engineering-health-sciences-bsenve/", - "https://catalog.northeastern.edu/undergraduate/engineering/civil-environmental/environmental-engineering-landscape-architecture-bsenve/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/biomedical-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/computational-data-analytics-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/computer-engineering-bscompe/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/computer-engineering-computer-science-bscompe/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/computer-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/computer-engineering-physics-bscompe/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/electrical-computer-engineering-bsee-bscompe/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/electrical-engineering-bsee/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/electrical-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/electrical-engineering-music-concentration-music-technology-bsee/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/electrical-engineering-physics-bsee/", - "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/robotics-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/first-year-engineering/", - "https://catalog.northeastern.edu/undergraduate/engineering/interdisciplinary-minors/design-inovation-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/interdisciplinary-minors/entrepreneurial-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/interdisciplinary-minors/materials-science-and-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/interdisciplinary-minors/sustainable-energy-systems-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/aerospace-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/biomechanical-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/bsie/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/bsme/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/healthcare-system-operations-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/industrial-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/mechanical-engineering-bioengineering-bsme/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/mechanical-engineering-design-bsme/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/mechanical-engineering-history-bsme/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/mechanical-engineering-minor/", - "https://catalog.northeastern.edu/undergraduate/engineering/mechanical-industrial/mechanical-engineering-physics-bsme/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/communication-sciences-disorders-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/early-intervention-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/human-movement-science-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/speech-language-pathology-audiology-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/speech-language-pathology-audiology-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/exercise-science-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/global-health-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-psychology-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-science-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-science-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-science-communication-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-science-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/health-science-sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/mindfulness-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/nutrition-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/public-health-ba/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/public-health-journalism-ba/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/community-health-behavioral-sciences/public-health-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/nursing/accelerated-second-degree-students-bsn/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/nursing/bsn/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/nursing/nursing-bsn-transfer-track/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/nursing/wellness-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/health-sciences-entrepreneurship-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/pharmaceutical-sciences-bs/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/pharmaceutical-sciences-minor/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/pharmacy-pharmd/", - "https://catalog.northeastern.edu/undergraduate/health-sciences/pharmacy/pharmacy-pharmd/#text", - "https://catalog.northeastern.edu/undergraduate/science/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-bs/", - "https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-design-bs/", - "https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-minor/", - "https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biochemistry/biochemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biochemistry/biochemistry-minor/", - "https://catalog.northeastern.edu/undergraduate/science/biology/biology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biology/biology-english-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biology/biology-mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biology/biology-minor/", - "https://catalog.northeastern.edu/undergraduate/science/biology/biology-political-science-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biology/cell-molecular-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/biology/cell-molecular-biology-minor/", - "https://catalog.northeastern.edu/undergraduate/science/chemistry-chemical-biology/chemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/science/chemistry-chemical-biology/chemistry-minor/", - "https://catalog.northeastern.edu/undergraduate/science/chemistry-chemical-biology/environmental-chemistry-minor/", - "https://catalog.northeastern.edu/undergraduate/science/interdisciplinary/network-science-minor/", - "https://catalog.northeastern.edu/undergraduate/science/interdisciplinary/scientific-commercialization-minor/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-communication-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-cultural-anthropology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-english-ba/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-minor/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/linguistics/linguistics-speech-language-pathology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/ecology-evolutionary-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/ecology-evolutionary-biology-minor/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-history-ba/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-studies-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-chemistry-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-landscape-architecture-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/environmental-sustainability-sciences-minor/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/geosciences-minor/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/marine-biology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/marine-environmental/marine-sciences-minor/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-ba/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-minor/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-physics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-political-science-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/mathematics/mathematics-sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/physics/applied-physics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/physics/astrophysics-minor/", - "https://catalog.northeastern.edu/undergraduate/science/physics/biomedical-physics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/physics/physics-bs/", - "https://catalog.northeastern.edu/undergraduate/science/physics/physics-minor/", - "https://catalog.northeastern.edu/undergraduate/science/physics/physics-music-bs/", - "https://catalog.northeastern.edu/undergraduate/science/physics/physics-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/science/psychology/psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/science/psychology/psychology-minor/", - "https://catalog.northeastern.edu/undergraduate/science/psychology/psychology-music-bs/", - "https://catalog.northeastern.edu/undergraduate/science/psychology/psychology-theatre-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/accelerated-bachelor-graduate-degree-programs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/asian-studies/asian-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/asian-studies/east-asian-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-political-science-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/criminology-criminal-justice/criminal-justice-sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/african-american-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/african-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-english-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-media-and-screen-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/africana-studies-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-english-interpreting-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-human-services-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-linguistics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/american-sign-language-theatre-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/arabic-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/black-feminist-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/chinese-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/film-and-international-culture-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/french-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/german-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/history-culture-law-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/italian-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/japanese-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/portuguese-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/russian-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/spanish-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/spanish-international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/spanish-linguistics-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/spanish-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-human-services-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-international-business-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-journalism-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-mathematics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/economics/economics-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-communication-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-criminal-justice-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-cultural-anthropology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-graphic-information-design-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/english-theatre-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/english/writing-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-asian-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-criminal-justice-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-cultural-anthropology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-economics-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-english-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/history/history-religious-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-communications-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-criminal-justice-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-psychology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-sociology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/human-services/human-services-sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/computational-social-sciences-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/digital-methods-in-the-humanities-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/food-systems-sustainability-health-equity-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/health-humanities-society-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/latino-latina-latin-american-caribbean-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/law-public-policy-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/politics-philosophy-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/urban-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/interdisciplinary/womens-gender-sexuality-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/african-studies-concentration-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/asian-studies-concentration-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/cultural-anthropology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/economics-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/european-studies-concentration-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/international-affairs-criminal-justice-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/international-affairs-history-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/international-affairs-international-business-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/international-affairs-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/latin-american-studies-concentration-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/middle-east-studies-concentration-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/middle-east-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/international-affairs/religious-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/jewish-studies/jewish-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/jewish-studies/jewish-studies-religion-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/ethics-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/information-ethics-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/philosophy--ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/philosophy-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/religious-studies-africana-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/religious-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/philosophy-religion/religious-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/american-political-institutions-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/international-security-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-business-administration-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-communication-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-communication-studies-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-economics-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-economics-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-human-services-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-human-services-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/political-science/political-science-philosophy-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-religious-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/cultural-anthropology-theatre-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/science-technology-studies-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-cultural-anthropology-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-cultural-anthropology-bs/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-environmental-studies-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-international-affairs-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-minor/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-philosophy-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-political-science-ba/", - "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/sociology-anthropology/sociology-religious-studies-ba/", -] -`; diff --git a/packages/scrapers-v2/test/classify.test.ts b/packages/scrapers-v2/test/classify.test.ts deleted file mode 100644 index d2e12446b..000000000 --- a/packages/scrapers-v2/test/classify.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { addTypeToUrl } from "../src/classify/classify"; -import { - ACCEL_DEG_PROG_CAMD, - ACCEL_DEG_PROG_COS, - ACCEL_DEG_PROG_KHOURY, - ACCOUNTING_MINOR, - AEROSPACE_MINOR, - ARCH_ENGLISH, - ARCHITECTURE_MINOR, - BSCS, - BUSINESS, - CHEMICAL_ENG, - CS_GAME_DEV, - CS_HISTORY, - CS_MUSIC_WITH_TECH_MAJOR, - DIGITAL_METHODS_HUMANITIES_MINOR, - FINTECH_CONCENTRATION, - GLOBAL_BUS_STRATEGY_MINOR, - MATH_POLYSCI, - MEDIA_SCREEN_STUDIES_HISTORY, - PHARMD, - WOMEN_GENDER_SEXUALITY_MINOR, -} from "./testUrls"; -import { CatalogEntryType } from "../src/classify/types"; - -const MAJORS = [ - BSCS, - CS_HISTORY, - MEDIA_SCREEN_STUDIES_HISTORY, - CS_MUSIC_WITH_TECH_MAJOR, - BUSINESS, - CHEMICAL_ENG, - CS_GAME_DEV, - - // no tabs - MATH_POLYSCI, - ARCH_ENGLISH, - - // ending is in uppercase - new URL( - "https://catalog.northeastern.edu/undergraduate/science/behavioral-neuroscience/behavioral-neuroscience-philosophy-bs/" - ), - - // non standard ending (ends in "pharmd") - PHARMD, -]; - -const CONCENTRATIONS = [ - FINTECH_CONCENTRATION, - - // no tabs - GLOBAL_BUS_STRATEGY_MINOR, - - // no comma in title - new URL( - "https://catalog.northeastern.edu/undergraduate/business/concentrations/accounting" - ), - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/brand-management', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/business-analytics', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/corporate-innovation-venture', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/entrepreneurial-startups', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/family-business', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/fintech', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/finance', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/international-business', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/healthcare-management-consulting', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/management', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/management-information-systems', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/marketing', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/marketing-analytics', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/social-innovation-entrepreneurship', - // 'https://catalog.northeastern.edu/undergraduate/business/concentrations/supply-chain-management' -]; - -const MINORS = [ - ACCOUNTING_MINOR, - ARCHITECTURE_MINOR, - AEROSPACE_MINOR, - WOMEN_GENDER_SEXUALITY_MINOR, - - // no tabs - DIGITAL_METHODS_HUMANITIES_MINOR, - - // has tabs, but with second tab text "program requirements" (not "minor requirements") - new URL( - "https://catalog.northeastern.edu/undergraduate/arts-media-design/communication-studies/sports-media-communication-minor" - ), - new URL( - "https://catalog.northeastern.edu/undergraduate/arts-media-design/theatre/performing-arts-minor" - ), - new URL( - "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/corporate-innovation-venture-minor" - ), - // "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/consulting-minor", - // "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/entrepreneurial-startups-minor", - // "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/family-business-minor", - // "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/social-innovation-entrepreneurship-minor", - // "https://catalog.northeastern.edu/undergraduate/business/interdisciplinary-minors/strategy-minor", - // "https://catalog.northeastern.edu/undergraduate/engineering/electrical-computer/robotics-minor", - // "https://catalog.northeastern.edu/undergraduate/health-sciences/clinical-rehabilitation-sciences/human-movement-science-minor", - // "https://catalog.northeastern.edu/undergraduate/social-sciences-humanities/cultures-societies-global-studies/german-minor", -]; - -const UNKNOWN = [ - // no tabs - // no elem w id ending in "requirementstextcontainer" - ACCEL_DEG_PROG_KHOURY, - ACCEL_DEG_PROG_CAMD, - ACCEL_DEG_PROG_COS, - // ACCEL_DEG_PROG_BUSINESS, - // ACCEL_DEG_PROG_ENG, - // ACCEL_DEG_PROG_CSSH, - // ACCEL_DEG_PROG_CSSH, -]; - -const testUrlsMatchType = (type: CatalogEntryType, urls: URL[]) => { - describe(type, () => { - for (const url of urls) { - test(url.href, async () => { - const { type } = await addTypeToUrl(url); - expect(type).toBe(type); - }); - } - }); -}; - -describe("classify", () => { - testUrlsMatchType(CatalogEntryType.Unknown, UNKNOWN); - testUrlsMatchType(CatalogEntryType.Major, MAJORS); - testUrlsMatchType(CatalogEntryType.Minor, MINORS); - testUrlsMatchType(CatalogEntryType.Concentration, CONCENTRATIONS); -}); diff --git a/packages/scrapers-v2/test/testUrls.ts b/packages/scrapers-v2/test/testUrls.ts deleted file mode 100644 index f586b4a86..000000000 --- a/packages/scrapers-v2/test/testUrls.ts +++ /dev/null @@ -1,93 +0,0 @@ -export const format = (path: string) => - new URL(`https://catalog.northeastern.edu${path}`); -export const CS_GAME_DEV = format( - "/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-game-development-bs" -); -export const BUSINESS = format( - "/undergraduate/business/business-administration-bsba" -); -export const PHYSICS = format("/undergraduate/science/physics/physics-bs"); -export const BSCS = format( - "/undergraduate/computer-information-science/computer-science/bscs" -); -export const MEDIA_SCREEN_STUDIES_HISTORY = format( - "/undergraduate/arts-media-design/communication-studies/media-screen-studies-history-ba" -); - -export const CS_HISTORY = format( - "/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-history-bs" -); -export const CHEMICAL_ENG = format( - "/undergraduate/engineering/chemical/chemical-engineering-bsche" -); -export const ACCOUNTING_MINOR = format( - "/undergraduate/business/interdisciplinary-minors/accounting-advisory-services-minor" -); -export const ARCHITECTURE_MINOR = format( - "/undergraduate/arts-media-design/architecture/architectural-design-minor" -); -export const FINTECH_CONCENTRATION = format( - "/undergraduate/business/concentrations/fintech" -); -export const AEROSPACE_MINOR = format( - "/undergraduate/engineering/mechanical-industrial/aerospace-minor" -); -export const ACCEL_DEG_PROG_KHOURY = format( - "/undergraduate/computer-information-science/accelerated-bachelor-graduate-degree-programs" -); -export const CS_MUSIC_WITH_TECH_MAJOR = format( - "/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-concentration-music-composition-technology-bs" -); -export const WOMEN_GENDER_SEXUALITY_MINOR = format( - "/undergraduate/social-sciences-humanities/interdisciplinary/womens-gender-sexuality-studies-minor" -); -export const CS_MATH = format( - "/undergraduate/computer-information-science/computer-information-science-combined-majors/computer-science-mathematics-bs" -); -export const DIGITAL_METHODS_HUMANITIES_MINOR = format( - "/undergraduate/social-sciences-humanities/interdisciplinary/digital-methods-in-the-humanities-minor" -); -export const MATH_POLYSCI = format( - "/undergraduate/science/mathematics/mathematics-political-science-bs/" -); -export const GLOBAL_BUS_STRATEGY_MINOR = format( - "/undergraduate/business/concentrations/global-business-strategy/" -); -// no tabs -export const ARCH_ENGLISH = format( - "/undergraduate/arts-media-design/architecture/architecture-english-bs/" -); -export const ACCEL_DEG_PROG_CAMD = format( - "/undergraduate/arts-media-design/accelerated-bachelor-graduate-degree-programs" -); -export const ACCEL_DEG_PROG_BUSINESS = format( - "/undergraduate/business/accelerated-bachelor-graduate-degree-programs" -); -export const ACCEL_DEG_PROG_ENG = format( - "/undergraduate/engineering/accelerated-bachelor-graduate-degree-programs" -); -export const ACCEL_DEG_PROG_HEALTH = format( - "/undergraduate/health-sciences/accelerated-bachelor-graduate-degree-programs" -); -export const ACCEL_DEG_PROG_COS = format( - "/undergraduate/science/accelerated-bachelor-graduate-degree-programs" -); -export const ACCEL_DEG_PROG_CSSH = format( - "/undergraduate/social-sciences-humanities/accelerated-bachelor-graduate-degree-programs" -); -// OR of ANDs -export const BIOENG_BIOCHEM = format( - "/undergraduate/engineering/bioengineering/bioengineering-biochemistry-bsbioe" -); -export const PUBLIC_HEALTH_BA = format( - "/undergraduate/health-sciences/community-health-behavioral-sciences/public-health-ba" -); -export const PHARM_SCI_BS = format( - "/undergraduate/health-sciences/pharmacy/pharmaceutical-sciences-bs" -); -export const PHARM_STUDIES_BS = format( - "/undergraduate/health-sciences/pharmacy/pharmacy-studies-bs" -); -export const PHARMD = format( - "/undergraduate/health-sciences/pharmacy/pharmacy-pharmd" -); diff --git a/packages/scrapers-v2/test/tokenize.test.ts b/packages/scrapers-v2/test/tokenize.test.ts deleted file mode 100644 index 4bc4956bc..000000000 --- a/packages/scrapers-v2/test/tokenize.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { fetchAndTokenizeHTML } from "../src/tokenize/tokenize"; -import { - ARCH_ENGLISH, - BIOENG_BIOCHEM, - BSCS, - BUSINESS, - CHEMICAL_ENG, - CS_GAME_DEV, - CS_HISTORY, - CS_MATH, - MEDIA_SCREEN_STUDIES_HISTORY, - PHARM_SCI_BS, - PHARMD, - PHYSICS, - PUBLIC_HEALTH_BA, -} from "./testUrls"; - -describe("scraper v2 snapshot tests", () => { - test("CS & Game Dev matches snapshot", async () => { - expect(await fetchAndTokenizeHTML(CS_GAME_DEV)).toMatchSnapshot(); - }); - test("nested linked concentration pages (business)", async () => { - expect(await fetchAndTokenizeHTML(BUSINESS)).toMatchSnapshot(); - }); - test("3 classes per AND (physics)", async () => { - expect(await fetchAndTokenizeHTML(PHYSICS)).toMatchSnapshot(); - }); - // CS 4950 is in the same AND twice for the Foundations concentration - test("multiple of the same class per AND (cs) ", async () => { - expect(await fetchAndTokenizeHTML(BSCS)).toMatchSnapshot(); - }); - // Range bounded - test("Test range bounded (history)", async () => { - expect( - await fetchAndTokenizeHTML(MEDIA_SCREEN_STUDIES_HISTORY) - ).toMatchSnapshot(); - }); - // Range bounded with exceptions - test("Test range bounded with exceptions (cs and math)", async () => { - expect(await fetchAndTokenizeHTML(CS_MATH)).toMatchSnapshot(); - }); - // Range lower bounded - test("Test range lower bounded (cs & history)", async () => { - expect(await fetchAndTokenizeHTML(CS_HISTORY)).toMatchSnapshot(); - }); - // Range unbounded - test("Test range unbounded (chemical engineering)", async () => { - expect(await fetchAndTokenizeHTML(CHEMICAL_ENG)).toMatchSnapshot(); - }); - // Or of ands - test("Test OR of ANDs (bioengineering biochemistry)", async () => { - expect(await fetchAndTokenizeHTML(BIOENG_BIOCHEM)).toMatchSnapshot(); - }); - // no tabs - test("Test NO tabs (architecture and english)", async () => { - expect(await fetchAndTokenizeHTML(ARCH_ENGLISH)).toMatchSnapshot(); - }); - describe("weird program requirement hours text placement", () => { - const get = (url: URL) => - fetchAndTokenizeHTML(url).then((h) => h.programRequiredHours); - test("Minimum of x hours", async () => { - expect(await get(PUBLIC_HEALTH_BA)).toBeGreaterThan(0); - }); - test("paragraph in front of `x total hrs`", async () => { - expect(await get(PHARMD)).toBeGreaterThan(0); - }); - test("paragraph in front of `Minimum of x hrs`", async () => { - expect(await get(PHARM_SCI_BS)).toBeGreaterThan(0); - }); - }); -}); diff --git a/packages/scrapers-v2/test/urls.test.ts b/packages/scrapers-v2/test/urls.test.ts deleted file mode 100644 index db72cbcae..000000000 --- a/packages/scrapers-v2/test/urls.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { scrapeMajorLinks } from "../src/urls/urls"; - -describe("url scraper", () => { - test("scrape", async () => { - const result = await scrapeMajorLinks(2021, 2022); - - expect(result.unfinished).toHaveLength(0); - const urlStrings = result.entries.map((url) => url.href); - expect(urlStrings.length).toEqual(new Set(urlStrings).size); - expect(urlStrings.sort()).toMatchSnapshot(); - }, 15000); -}); diff --git a/packages/scrapers-v2/tsconfig.json b/packages/scrapers-v2/tsconfig.json deleted file mode 100644 index 13b56b9be..000000000 --- a/packages/scrapers-v2/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "compilerOptions": { - "target": "es2017", - "allowJs": true, - "skipLibCheck": true, - "esModuleInterop": true, - "allowSyntheticDefaultImports": true, - "strict": true, - "forceConsistentCasingInFileNames": true, - "module": "commonjs", - "moduleResolution": "node", - "resolveJsonModule": true, - "outDir": "./build", - "noImplicitAny": true, - "strictNullChecks": true, - "lib": ["dom", "dom.iterable", "esnext"], - "isolatedModules": true, - "noEmit": true, - "jsx": "react", - "strictPropertyInitialization": false, - "experimentalDecorators": true - }, - "include": ["src", "test"] -} diff --git a/yarn.lock b/yarn.lock index 95028fe4b..47ce9276c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3065,23 +3065,6 @@ __metadata: languageName: unknown linkType: soft -"@graduate/scrapers-v2@workspace:packages/scrapers-v2": - version: 0.0.0-use.local - resolution: "@graduate/scrapers-v2@workspace:packages/scrapers-v2" - dependencies: - "@graduate/common": "*" - "@types/cheerio": 0.22.13 - "@types/jest": ^27 - "@types/node": ^16 - cheerio: 1.0.0-rc.3 - jest: ^27.5.1 - ts-jest: ^27.1.3 - ts-node: ^10.8.0 - typescript: ^4.6.2 - undici: ^5.8.2 - languageName: unknown - linkType: soft - "@humanwhocodes/config-array@npm:^0.10.5": version: 0.10.5 resolution: "@humanwhocodes/config-array@npm:0.10.5" @@ -4001,15 +3984,6 @@ __metadata: languageName: node linkType: hard -"@types/cheerio@npm:0.22.13": - version: 0.22.13 - resolution: "@types/cheerio@npm:0.22.13" - dependencies: - "@types/node": "*" - checksum: a7b34a497910afd1fd5cfccd48c429256b0a6f1039d110caea32b4afbbdd687e15539049d8de355978e2ff81a476ccec1304726e0a21062ad8a57046ee8c625b - languageName: node - linkType: hard - "@types/connect@npm:*": version: 3.4.35 resolution: "@types/connect@npm:3.4.35" @@ -4144,7 +4118,7 @@ __metadata: languageName: node linkType: hard -"@types/jest@npm:^27, @types/jest@npm:^27.0.1": +"@types/jest@npm:^27.0.1": version: 27.5.2 resolution: "@types/jest@npm:27.5.2" dependencies: @@ -4246,7 +4220,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^16, @types/node@npm:^16.0.0": +"@types/node@npm:^16.0.0": version: 16.11.60 resolution: "@types/node@npm:16.11.60" checksum: 8c9670801f736a06f5ec8dd4ade00cb8d822f1dd168ad255909af6f6439a50abc231fa684418810002fdcd4c282804f56ceee6e7d3cd3209736c4fad115f4b2c @@ -5490,13 +5464,6 @@ __metadata: languageName: node linkType: hard -"boolbase@npm:~1.0.0": - version: 1.0.0 - resolution: "boolbase@npm:1.0.0" - checksum: 3e25c80ef626c3a3487c73dbfc70ac322ec830666c9ad915d11b701142fab25ec1e63eff2c450c74347acfd2de854ccde865cd79ef4db1683f7c7b046ea43bb0 - languageName: node - linkType: hard - "brace-expansion@npm:^1.1.7": version: 1.1.11 resolution: "brace-expansion@npm:1.1.11" @@ -5744,20 +5711,6 @@ __metadata: languageName: node linkType: hard -"cheerio@npm:1.0.0-rc.3": - version: 1.0.0-rc.3 - resolution: "cheerio@npm:1.0.0-rc.3" - dependencies: - css-select: ~1.2.0 - dom-serializer: ~0.1.1 - entities: ~1.1.1 - htmlparser2: ^3.9.1 - lodash: ^4.15.0 - parse5: ^3.0.1 - checksum: 90163e8f360d3a9ac27d7ee83edd891236cad63df75e4fde5efcc27269996716a3f8c8dfcefaa2e77ddd6a21c8e54ed6169138096c869913e571abe2264f36fe - languageName: node - linkType: hard - "chokidar@npm:3.5.3, chokidar@npm:^3.4.0, chokidar@npm:^3.5.3": version: 3.5.3 resolution: "chokidar@npm:3.5.3" @@ -6236,25 +6189,6 @@ __metadata: languageName: node linkType: hard -"css-select@npm:~1.2.0": - version: 1.2.0 - resolution: "css-select@npm:1.2.0" - dependencies: - boolbase: ~1.0.0 - css-what: 2.1 - domutils: 1.5.1 - nth-check: ~1.0.1 - checksum: 607cca60d2f5c56701fe5f800bbe668b114395c503d4e4808edbbbe70b8be3c96a6407428dc0227fcbdf335b20468e6a9e7fd689185edfb57d402e1e4837c9b7 - languageName: node - linkType: hard - -"css-what@npm:2.1": - version: 2.1.3 - resolution: "css-what@npm:2.1.3" - checksum: a52d56c591a7e1c37506d0d8c4fdef72537fb8eb4cb68711485997a88d76b5a3342b73a7c79176268f95b428596c447ad7fa3488224a6b8b532e2f1f2ee8545c - languageName: node - linkType: hard - "cssom@npm:^0.4.4": version: 0.4.4 resolution: "cssom@npm:0.4.4" @@ -6536,40 +6470,6 @@ __metadata: languageName: node linkType: hard -"dom-serializer@npm:0": - version: 0.2.2 - resolution: "dom-serializer@npm:0.2.2" - dependencies: - domelementtype: ^2.0.1 - entities: ^2.0.0 - checksum: 376344893e4feccab649a14ca1a46473e9961f40fe62479ea692d4fee4d9df1c00ca8654811a79c1ca7b020096987e1ca4fb4d7f8bae32c1db800a680a0e5d5e - languageName: node - linkType: hard - -"dom-serializer@npm:~0.1.1": - version: 0.1.1 - resolution: "dom-serializer@npm:0.1.1" - dependencies: - domelementtype: ^1.3.0 - entities: ^1.1.1 - checksum: 4f6a3eff802273741931cfd3c800fab4e683236eed10628d6605f52538a6bc0ce4770f3ca2ad68a27412c103ae9b6cdaed3c0a8e20d2704192bde497bc875215 - languageName: node - linkType: hard - -"domelementtype@npm:1, domelementtype@npm:^1.3.0, domelementtype@npm:^1.3.1": - version: 1.3.1 - resolution: "domelementtype@npm:1.3.1" - checksum: 7893da40218ae2106ec6ffc146b17f203487a52f5228b032ea7aa470e41dfe03e1bd762d0ee0139e792195efda765434b04b43cddcf63207b098f6ae44b36ad6 - languageName: node - linkType: hard - -"domelementtype@npm:^2.0.1": - version: 2.3.0 - resolution: "domelementtype@npm:2.3.0" - checksum: ee837a318ff702622f383409d1f5b25dd1024b692ef64d3096ff702e26339f8e345820f29a68bcdcea8cfee3531776b3382651232fbeae95612d6f0a75efb4f6 - languageName: node - linkType: hard - "domexception@npm:^2.0.1": version: 2.0.1 resolution: "domexception@npm:2.0.1" @@ -6579,35 +6479,6 @@ __metadata: languageName: node linkType: hard -"domhandler@npm:^2.3.0": - version: 2.4.2 - resolution: "domhandler@npm:2.4.2" - dependencies: - domelementtype: 1 - checksum: 49bd70c9c784f845cd047e1dfb3611bd10891c05719acfc93f01fc726a419ed09fbe0b69f9064392d556a63fffc5a02010856cedae9368f4817146d95a97011f - languageName: node - linkType: hard - -"domutils@npm:1.5.1": - version: 1.5.1 - resolution: "domutils@npm:1.5.1" - dependencies: - dom-serializer: 0 - domelementtype: 1 - checksum: 800d1f9d1c2e637267dae078ff6e24461e6be1baeb52fa70f2e7e7520816c032a925997cd15d822de53ef9896abb1f35e5c439d301500a9cd6b46a395f6f6ca0 - languageName: node - linkType: hard - -"domutils@npm:^1.5.1": - version: 1.7.0 - resolution: "domutils@npm:1.7.0" - dependencies: - dom-serializer: 0 - domelementtype: 1 - checksum: f60a725b1f73c1ae82f4894b691601ecc6ecb68320d87923ac3633137627c7865725af813ae5d188ad3954283853bcf46779eb50304ec5d5354044569fcefd2b - languageName: node - linkType: hard - "dotenv-expand@npm:5.1.0": version: 5.1.0 resolution: "dotenv-expand@npm:5.1.0" @@ -6715,20 +6586,6 @@ __metadata: languageName: node linkType: hard -"entities@npm:^1.1.1, entities@npm:~1.1.1": - version: 1.1.2 - resolution: "entities@npm:1.1.2" - checksum: d537b02799bdd4784ffd714d000597ed168727bddf4885da887c5a491d735739029a00794f1998abbf35f3f6aeda32ef5c15010dca1817d401903a501b6d3e05 - languageName: node - linkType: hard - -"entities@npm:^2.0.0": - version: 2.2.0 - resolution: "entities@npm:2.2.0" - checksum: 19010dacaf0912c895ea262b4f6128574f9ccf8d4b3b65c7e8334ad0079b3706376360e28d8843ff50a78aabcb8f08f0a32dbfacdc77e47ed77ca08b713669b3 - languageName: node - linkType: hard - "env-paths@npm:^2.2.0": version: 2.2.1 resolution: "env-paths@npm:2.2.1" @@ -8160,20 +8017,6 @@ __metadata: languageName: node linkType: hard -"htmlparser2@npm:^3.9.1": - version: 3.10.1 - resolution: "htmlparser2@npm:3.10.1" - dependencies: - domelementtype: ^1.3.1 - domhandler: ^2.3.0 - domutils: ^1.5.1 - entities: ^1.1.1 - inherits: ^2.0.1 - readable-stream: ^3.1.1 - checksum: 6875f7dd875aa10be17d9b130e3738cd8ed4010b1f2edaf4442c82dfafe9d9336b155870dcc39f38843cbf7fef5e4fcfdf0c4c1fd4db3a1b91a1e0ee8f6c3475 - languageName: node - linkType: hard - "http-cache-semantics@npm:^4.1.0": version: 4.1.0 resolution: "http-cache-semantics@npm:4.1.0" @@ -9650,7 +9493,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:4.17.21, lodash@npm:^4.15.0, lodash@npm:^4.17.19, lodash@npm:^4.17.21, lodash@npm:^4.7.0": +"lodash@npm:4.17.21, lodash@npm:^4.17.19, lodash@npm:^4.17.21, lodash@npm:^4.7.0": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 @@ -10650,15 +10493,6 @@ __metadata: languageName: node linkType: hard -"nth-check@npm:~1.0.1": - version: 1.0.2 - resolution: "nth-check@npm:1.0.2" - dependencies: - boolbase: ~1.0.0 - checksum: 59e115fdd75b971d0030f42ada3aac23898d4c03aa13371fa8b3339d23461d1badf3fde5aad251fb956aaa75c0a3b9bfcd07c08a34a83b4f9dadfdce1d19337c - languageName: node - linkType: hard - "number-is-nan@npm:^1.0.0": version: 1.0.1 resolution: "number-is-nan@npm:1.0.1" @@ -10951,15 +10785,6 @@ __metadata: languageName: node linkType: hard -"parse5@npm:^3.0.1": - version: 3.0.3 - resolution: "parse5@npm:3.0.3" - dependencies: - "@types/node": "*" - checksum: 6a82d59d60496f4a8bba99daee37eda728adb403197b9c9a163dcc02e369758992bcc67f1618d4f1445f4b12e7651e001c2847e446b8376d4d706e1d571f570d - languageName: node - linkType: hard - "parse5@npm:^5.1.1": version: 5.1.1 resolution: "parse5@npm:5.1.1" @@ -11694,7 +11519,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0": +"readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0": version: 3.6.0 resolution: "readable-stream@npm:3.6.0" dependencies: @@ -12922,7 +12747,7 @@ __metadata: languageName: node linkType: hard -"ts-jest@npm:^27.0.3, ts-jest@npm:^27.1.3, ts-jest@npm:^27.1.4": +"ts-jest@npm:^27.0.3, ts-jest@npm:^27.1.4": version: 27.1.5 resolution: "ts-jest@npm:27.1.5" dependencies: @@ -12970,7 +12795,7 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^10.8.0, ts-node@npm:^10.9.1": +"ts-node@npm:^10.9.1": version: 10.9.1 resolution: "ts-node@npm:10.9.1" dependencies: @@ -13297,13 +13122,6 @@ __metadata: languageName: node linkType: hard -"undici@npm:^5.8.2": - version: 5.10.0 - resolution: "undici@npm:5.10.0" - checksum: 7ba2b71dccc74cd2bdf645b83e9aaef374ae04855943d0a2f42a3d0b9e5556f37cc9b5156fb5288277a2fa95fd46a56f3ae0d5cf73db3f008d75ec41104b136c - languageName: node - linkType: hard - "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0"