tweak: Make improvements to session cookie/crumb retrieval for Yahoo …

…Finance API
SierraSoftworks · Nov 25, 2023 · fc1bfb1 · fc1bfb1
1 parent a6a563b
commit fc1bfb1
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 2 deletions.
diff --git a/datasources/alphavantage.spec.ts b/datasources/alphavantage.spec.ts
@@ -3,6 +3,7 @@ import {AlphaVantage} from "./alphavantage"
 
 describe.skip("AlphaVantage Stock API", () => {
     const alphaVantage = new AlphaVantage(process.env.ALPHAVANTAGE_API_KEY)
+    alphaVantage.disableCache()
 
     describe("getCurrencyData", () => {
         it("should return currency data for a pair of currencies", async () => {

diff --git a/datasources/datasource.ts b/datasources/datasource.ts
@@ -10,8 +10,18 @@ export class DataSource {
     }
 
     private cacheDirectory: string
+    private cacheEnabled = true
+
+    disableCache() {
+        this.cacheEnabled = false
+    }
 
     protected async cached<T>(key: string, expiry: CacheExpiryTime, hydrate: () => Promise<T>): Promise<T> {
+        if (!this.cacheEnabled)
+        {
+            return await hydrate()
+        }
+
         await fs.promises.mkdir(this.cacheDirectory, { recursive: true })
         const cachePath = this.getCachePath(key, expiry)
 

diff --git a/datasources/tiingo.spec.ts b/datasources/tiingo.spec.ts
@@ -3,6 +3,7 @@ import {Tiingo} from "./tiingo"
 
 describe.skip("Tiingo Stock API", () => {
     const tiingo = new Tiingo(process.env.TIINGO_API_KEY)
+    tiingo.disableCache()
 
     describe("getCurrencyData", () => {
         it("should return currency data for a pair of currencies", async () => {

diff --git a/datasources/yahoo.spec.ts b/datasources/yahoo.spec.ts
@@ -3,6 +3,7 @@ import {Yahoo} from "./yahoo"
 
 describe("yahoo markets", () => {
     const yahoo = new Yahoo()
+    yahoo.disableCache()
 
     describe("getCurrencyData", () => {
         it("should return currency data for a pair of currencies", async () => {

diff --git a/datasources/yahoo.ts b/datasources/yahoo.ts
@@ -1,14 +1,24 @@
+import { randomInt } from "node:crypto"
 import {buildUrl, fetchSafe} from "../utils/http"
 import {CurrencyDataSource, DataSource, StockDataSource} from "./datasource"
 
+const validUserAgents = [
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/120.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.1; rv:109.0) Gecko/20100101 Firefox/120.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/120.0"
+]
+
 export class Yahoo extends DataSource implements StockDataSource, CurrencyDataSource {
     constructor() {
         super("yahoo")
     }
 
     private headers = {
         "Accept": "application/json, text/javascript, text/plain, */*; q=0.01",
-        "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36',
+        "User-Agent": validUserAgents[randomInt(validUserAgents.length)],
     }
 
     private cookie: string
@@ -70,7 +80,12 @@ export class Yahoo extends DataSource implements StockDataSource, CurrencyDataSo
         if (this.crumb) return
 
         this.cookie = await this.getSessionCookie()
+
+        if (!this.cookie) throw new Error("Failed to get session cookie")
+
         this.crumb = await this.getSessionCrumb(this.cookie)
+
+        if (!this.crumb) throw new Error("Failed to get session crumb")
     }
 
     private async getSessionCookie(): Promise<string> {
@@ -84,7 +99,7 @@ export class Yahoo extends DataSource implements StockDataSource, CurrencyDataSo
     }
 
     private async getSessionCrumb(sessionCookie: string): Promise<string> {
-        const response = await fetch("https://query2.finance.yahoo.com/v1/test/getcrumb", {
+        const response = await fetch("https://query1.finance.yahoo.com/v1/test/getcrumb", {
             headers: Object.assign({}, this.headers, {
                 "Cookie": sessionCookie
             }),