From bb8f3311a8793ab4a63b85b6dd851b6e71e868a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mr=C3=A1zek?= Date: Sun, 7 Jan 2024 16:59:41 +0100 Subject: [PATCH] Support extended components --- .github/workflows/update_components.yaml | 1 + jlcparts/datatables.py | 7 ++++- jlcparts/lcsc.py | 36 ++++++++++++++++++++++++ jlcparts/partLib.py | 23 ++++++++++++++- jlcparts/ui.py | 14 +++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_components.yaml b/.github/workflows/update_components.yaml index 6673674e31c..a4ff063bc1e 100644 --- a/.github/workflows/update_components.yaml +++ b/.github/workflows/update_components.yaml @@ -54,6 +54,7 @@ jobs: jlcparts getlibrary --age 10000 \ --limit 15000 \ parts.csv cache.sqlite3 + jlcparts updatepreferred cache.sqlite3 jlcparts buildtables --jobs 0 \ --ignoreoldstock 120 \ cache.sqlite3 web/build/data diff --git a/jlcparts/datatables.py b/jlcparts/datatables.py index ce922ad373d..4e290986e0c 100644 --- a/jlcparts/datatables.py +++ b/jlcparts/datatables.py @@ -193,8 +193,13 @@ def pullExtraAttributes(component): a dictionary """ status = "Discontinued" if component["extra"] == {} else "Active" + type = "Extended" + if component["basic"]: + type = "Basic" + if component["preferred"]: + type = "Preferred" return { - "Basic/Extended": "Basic" if component["basic"] else "Extended", + "Basic/Extended": type, "Package": component["package"], "Status": status } diff --git a/jlcparts/lcsc.py b/jlcparts/lcsc.py index 092f409d0fb..b382cac2b51 100644 --- a/jlcparts/lcsc.py +++ b/jlcparts/lcsc.py @@ -1,3 +1,4 @@ +import json import requests import os import time @@ -28,6 +29,41 @@ def makeLcscRequest(url, payload=None): return requests.get(url, params=newPayload) +def pullPreferredComponents(): + resp = requests.get("https://jlcpcb.com/api/overseas-pcb-order/v1/getAll") + token = resp.cookies.get_dict()["XSRF-TOKEN"] + + headers = { + "Content-Type": "application/json", + "X-XSRF-TOKEN": token, + } + PAGE_SIZE = 1000 + + currentPage = 1 + components = set() + while True: + body = { + "currentPage": currentPage, + "pageSize": PAGE_SIZE, + "preferredComponentFlag": True + } + + resp = requests.post( + "https://jlcpcb.com/api/overseas-pcb-order/v1/shoppingCart/smtGood/selectSmtComponentList", + headers=headers, + json=body + ) + + body = resp.json() + for c in [x["componentCode"] for x in body["data"]["componentPageInfo"]["list"]]: + components.add(c) + + if not body["data"]["componentPageInfo"]["hasNextPage"]: + break + currentPage += 1 + + return components + if __name__ == "__main__": r = makeLcscRequest("https://ips.lcsc.com/rest/wmsc2agent/product/info/C7063") print(r.json()) diff --git a/jlcparts/partLib.py b/jlcparts/partLib.py index a04a938fd10..61785a6d364 100755 --- a/jlcparts/partLib.py +++ b/jlcparts/partLib.py @@ -35,6 +35,8 @@ def dbToComp(comp): comp["lcsc"] = lcscFromDb(comp["lcsc"]) comp["price"] = json.loads(comp["price"]) comp["extra"] = json.loads(comp["extra"]) + comp["basic"] = bool(comp["basic"]) + comp["preferred"] = bool(comp["preferred"]) return comp class PartLibraryDb: @@ -54,6 +56,7 @@ def __init__(self, filepath=None): joints INTEGER NOT NULL, manufacturer_id INTEGER NOT NULL, basic INTEGER NOT NULL, + preferred INTEGER NOT NULL DEFAULT 0, description TEXT NOT NULL, datasheet TEXT NOT NULL, stock INTEGER NOT NULL, @@ -64,11 +67,22 @@ def __init__(self, filepath=None): )""") # Perform migration if we miss last on stock - columns = self.conn.execute("pragma table_info(components)") + migrated = False + columns = list(self.conn.execute("pragma table_info(components)")) if "last_on_stock" not in [x[1] for x in columns]: self.conn.execute(""" ALTER TABLE components ADD COLUMN last_on_stock INTEGER NOT NULL DEFAULT 0; """) + migrated = True + + # Perform migration if we miss the preferred flag + if "preferred" not in [x[1] for x in columns]: + self.conn.execute(""" + ALTER TABLE components ADD COLUMN preferred INTEGER NOT NULL DEFAULT 0; + """) + migrated = True + + if migrated: self.conn.execute("DROP VIEW v_components") self.conn.execute(""" @@ -104,6 +118,7 @@ def __init__(self, filepath=None): c.joints AS joints, m.name AS manufacturer, c.basic AS basic, + c.preferred as preferred, c.description AS description, c.datasheet AS datasheet, c.stock AS stock, @@ -292,6 +307,12 @@ def updateJlcPart(self, component, flag=None): """, data) self._commit() + def setPreferred(self, lcscSet): + cursor = self.conn.cursor() + cursor.execute("UPDATE components SET preferred = 0") + cursor.execute(f"UPDATE components SET preferred = 1 WHERE lcsc IN ({','.join(len(lcscSet) * ['?'])})", + [lcscToDb(x) for x in lcscSet]) + self._commit() def categories(self): res = {} diff --git a/jlcparts/ui.py b/jlcparts/ui.py index e3c4170de5e..4e6c80df4b3 100644 --- a/jlcparts/ui.py +++ b/jlcparts/ui.py @@ -4,6 +4,7 @@ import click from jlcparts.datatables import buildtables, normalizeAttribute +from jlcparts.lcsc import pullPreferredComponents from jlcparts.partLib import (PartLibrary, PartLibraryDb, getLcscExtraNew, loadJlcTable, loadJlcTableLazy) @@ -62,6 +63,18 @@ def getLibrary(source, db, age, limit): # db.vacuum() + +@click.command() +@click.argument("db", type=click.Path(dir_okay=False, writable=True)) +def updatePreferred(db): + """ + Download list of preferred components from JLC PCB and mark them into the DB. + """ + preferred = pullPreferredComponents() + lib = PartLibraryDb(db) + lib.setPreferred(preferred) + + @click.command() @click.argument("libraryFilename") def listcategories(libraryfilename): @@ -152,6 +165,7 @@ def cli(): cli.add_command(listcategories) cli.add_command(listattributes) cli.add_command(buildtables) +cli.add_command(updatePreferred) cli.add_command(fetchDetails) cli.add_command(fetchTable) cli.add_command(testComponent)