diff --git a/.gitignore b/.gitignore index 9f3182b..9af9000 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.venv .DS_Store */.DS_Store @@ -9,3 +10,7 @@ build/ .tox/ .env *.log + +# result files +data/spider_data/*.xlsx +data/spider_data/*.sdf diff --git a/data/spider_data/.files b/data/spider_data/.files new file mode 100644 index 0000000..e69de29 diff --git a/data/spider_data/Baizhu_disease.xlsx b/data/spider_data/Baizhu_disease.xlsx deleted file mode 100644 index 478a43e..0000000 Binary files a/data/spider_data/Baizhu_disease.xlsx and /dev/null differ diff --git a/data/spider_data/Baizhu_ingredients.xlsx b/data/spider_data/Baizhu_ingredients.xlsx deleted file mode 100644 index 3c2c355..0000000 Binary files a/data/spider_data/Baizhu_ingredients.xlsx and /dev/null differ diff --git a/data/spider_data/Baizhu_targets.xlsx b/data/spider_data/Baizhu_targets.xlsx deleted file mode 100644 index 736947c..0000000 Binary files a/data/spider_data/Baizhu_targets.xlsx and /dev/null differ diff --git a/data/spider_data/Chenpi_disease.xlsx b/data/spider_data/Chenpi_disease.xlsx deleted file mode 100644 index cb1c8cf..0000000 Binary files a/data/spider_data/Chenpi_disease.xlsx and /dev/null differ diff --git a/data/spider_data/Chenpi_ingredients.xlsx b/data/spider_data/Chenpi_ingredients.xlsx deleted file mode 100644 index e7e128e..0000000 Binary files a/data/spider_data/Chenpi_ingredients.xlsx and /dev/null differ diff --git a/data/spider_data/Chenpi_targets.xlsx b/data/spider_data/Chenpi_targets.xlsx deleted file mode 100644 index 86d2d30..0000000 Binary files a/data/spider_data/Chenpi_targets.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuang_disease.xlsx b/data/spider_data/Mahuang_disease.xlsx deleted file mode 100644 index c096c5a..0000000 Binary files a/data/spider_data/Mahuang_disease.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuang_ingredients.xlsx b/data/spider_data/Mahuang_ingredients.xlsx deleted file mode 100644 index e1adbb9..0000000 Binary files a/data/spider_data/Mahuang_ingredients.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuang_targets.xlsx b/data/spider_data/Mahuang_targets.xlsx deleted file mode 100644 index 67fa124..0000000 Binary files a/data/spider_data/Mahuang_targets.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuanggen_disease.xlsx b/data/spider_data/Mahuanggen_disease.xlsx deleted file mode 100644 index b9edd38..0000000 Binary files a/data/spider_data/Mahuanggen_disease.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuanggen_ingredients.xlsx b/data/spider_data/Mahuanggen_ingredients.xlsx deleted file mode 100644 index d8c8a8d..0000000 Binary files a/data/spider_data/Mahuanggen_ingredients.xlsx and /dev/null differ diff --git a/data/spider_data/Mahuanggen_targets.xlsx b/data/spider_data/Mahuanggen_targets.xlsx deleted file mode 100644 index a54b383..0000000 Binary files a/data/spider_data/Mahuanggen_targets.xlsx and /dev/null differ diff --git a/requirements.txt b/requirements.txt index a0424ba..e7201b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ beautifulsoup4==4.11.2 lxml==4.9.2 -pandas==1.5.2 +pandas requests==2.28.1 +rdkit +openpyxl +tqdm \ No newline at end of file diff --git a/src/get_all_data.py b/src/get_all_data.py index 449d91d..dfa11a9 100644 --- a/src/get_all_data.py +++ b/src/get_all_data.py @@ -7,7 +7,7 @@ def get_data(type): tcmsp = TcmspSpider() - url = f"https://tcmsp-e.com/browse.php?qc={type}" + url = f"https://old.tcmsp-e.com/browse.php?qc={type}" # 获取页面 html = tcmsp.get_response(url) diff --git a/src/tcmsp.py b/src/tcmsp.py index 5526e65..d12ec02 100644 --- a/src/tcmsp.py +++ b/src/tcmsp.py @@ -1,10 +1,10 @@ #!/usr/local/bin/python3 # -*- encoding: utf-8 -*- -''' +""" @Brief : TCMSP数据库爬虫 @Time : 2023/02/09 19:39:55 @Author : https://github.com/shujuecn -''' +""" import os import re @@ -13,12 +13,14 @@ import pandas as pd from bs4 import BeautifulSoup as bs import lxml.html - +from rdkit import Chem +import tempfile +from tqdm import tqdm class TcmspSpider: def __init__(self): - self.root_url = "https://www.tcmsp-e.com/tcmspsearch.php" + self.root_url = "https://old.tcmsp-e.com/tcmspsearch.php" self.headers = { "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; NCE-AL10 Build/HUAWEINCE-AL10; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", @@ -129,7 +131,7 @@ def get_herb_data(self, cn_name, en_name, pinyin_name): ingredients_data, file_path=f"{self.spider_file_path}", file_name=f"{pinyin_name}_ingredients", - index="MOL_ID" + index="MOL_ID", ) # 导出 Targets @@ -139,7 +141,7 @@ def get_herb_data(self, cn_name, en_name, pinyin_name): targets_data, file_path=f"{self.spider_file_path}", file_name=f"{pinyin_name}_targets", - index="MOL_ID" + index="MOL_ID", ) # 导出 Disease @@ -150,9 +152,11 @@ def get_herb_data(self, cn_name, en_name, pinyin_name): disease_data, file_path=f"{self.spider_file_path}", file_name=f"{pinyin_name}_disease", - index=False + index=False, ) + print("正在导出sdf文件，过程较慢，耐心等待...") + self.mol2sdf(ingredients_data, f"{self.spider_file_path}{pinyin_name}.sdf") print(f"{cn_name}下载完成！\n") def get_json_data(self, html, num, pattern): @@ -200,3 +204,36 @@ def text_to_excel(self, data, file_path, file_name, index): else: print(f"未查询到{file_name}的信息！") + + def mol2sdf(self, data, output_path): + df = pd.DataFrame(data) + mol_ids = df["MOL_ID"].to_list() + + with Chem.SDWriter(output_path) as writer: + for mol_id in tqdm(mol_ids, desc="导出进度"): + mol2_url = f"https://old.tcmsp-e.com/tcmspmol/{mol_id}.mol2" + response = requests.get(mol2_url) + if response.status_code == 200: + mol2_content = response.text + with tempfile.NamedTemporaryFile( + suffix=".mol2", delete=False + ) as temp_file: + temp_file.write(mol2_content.encode("utf-8")) + temp_file.flush() + temp_file_path = temp_file.name + + try: + mol = Chem.MolFromMol2File( + temp_file_path, sanitize=False, cleanupSubstructures=False + ) + if mol: + writer.write(mol) + else: + print(f"Failed to convert MOL2 to SDF for MOL_ID: {mol_id}") + except Exception as e: + print(f"Error processing MOL_ID {mol_id}: {e}") + finally: + os.remove(temp_file_path) + else: + print(f"Failed to download MOL2 file for MOL_ID: {mol_id}") + print(f"已保存：{output_path}")