Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

支持Mac/Win，修复小bug #1

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

支持Mac/Win，修复小bug #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ab7df39
修改了-a为指向整个项目
aw220 Jan 15, 2022
1af109a
可同时支持macos
aw220 Jan 15, 2022
7ee83a4
Update README.md
aw220 Jan 16, 2022
411647d
Update codeql_compile.py
aw220 Mar 1, 2022
e3ecafc
Update codeql_compile.py
aw220 Mar 1, 2022
e81b8eb
Update codeql_compile.py
aw220 Mar 1, 2022
c5f7288
Update README.md
aw220 Mar 1, 2022
f67fcd3
Update README.md
aw220 Mar 2, 2022
1af9166
Update README.md
aw220 Mar 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@

## 使用方法

1.在windows上使用
1.支持win和mac
2.第二步不是必须项，可以直接执行第一步，然后就可以开始创建数据库
3.最好限制下要分析的包范围，过大反而不利于分析。

### 1、反编译项目
默认情况下使用java-decompiler.jar进行反编译，会在项目源代码路径的父级目录创建以**项目名+_save+时间戳**命名的目录

参数`-a`：指定项目源代码路径
参数`-a`：指定项目根路径
参数`-d`：指定反编译代码的依赖包路径
```cmd
python3 codeql_compile.py -a D:\project\java\apps\2\cloud -d D:\project\java\apps\BOOT-INF\lib
Expand Down Expand Up @@ -44,6 +44,12 @@ python3 codeql_compile.py -a D:\project\java\apps\2\cloud -o D:\project\java\app
D:\codeql.exe database create D:\codeql\databases\demo-database --language="java" --source-root=D:\codeql\demo_save_1641018608 --command="run.cmd"
```



### 4、直接创建数据库
会在当前目录生成`项目名_database`的数据库
参数`-l`:指定语言，暂时支持java
参数`-i`:指定额外包含的文件，暂时只支持xml文件
```cmd
python3 codeql_compile.py -l java -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib" 普通创建
python3 codeql_compile.py -l java -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib -i xml" 包含xml文件
```

188 changes: 150 additions & 38 deletions codeql_compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
# coding=utf-8
import pathlib, zipfile, subprocess
import os, sys, time, re
import argparse
import platform
import shutil


# 指定自定义java-decompiler的路径
self_java_decompiler_path = r'D:\idea_v2019\IntelliJ IDEA 2019.3.5\plugins\java-decompiler\lib\java-decompiler.jar'
# 指定自定义java-decompiler路径
self_java_decompiler_path = r'/java-decompiler.jar'
# 指定自定义ecj的路径
self_ecj_path = r"D:\project\java\apps\ecj-4.6.1.jar"
self_ecj_path = r"/ecj-4.6.1.jar"
# 指定自定义procyon的路径
self_procyon_path = r"D:\project\java\apps\procyon-decompiler-0.6-prerelease.jar"
self_procyon_path = r"/procyon.jar"
# 指定自定义codeql的路径
codeql_path = r"/codeql"


def get_arch():
return platform.system().lower()


# 用来校验本地的ecj、java_decompiler路径是否正确
Expand All @@ -23,7 +30,9 @@ def verify(file_jar, my_path):

def java_decompiler_run():
# 搜索源项目中包含.jar的所有路径
_sub = subprocess.getstatusoutput('java -cp "{}" org.jetbrains.java.decompiler.main.decompiler.ConsoleDecompiler -dgs=true {} {}'.format(java_decompiler_path, app_path, save_path))
_sub = subprocess.getstatusoutput(
'java -cp "{}" org.jetbrains.java.decompiler.main.decompiler.ConsoleDecompiler -dgs=true {} {}'.format(
java_decompiler_path, app_path, save_path))
if _sub[0] != 0:
print(_sub[1])
sys.exit("java_decompiler 执行失败......")
Expand All @@ -39,38 +48,48 @@ def java_decompiler_run():

# 先尝试编译成class，定位错误文件再使用procyon反编译替换
def check():
_sub = subprocess.getstatusoutput('{}/run.cmd'.format(save_path))
_sub = subprocess.getstatusoutput('{0}/{1}'.format(save_path,run))
# 正则匹配错误文件路径
re_matchs = set(re.findall("ERROR in (.*)? \(at line", _sub[1]))

# 确认是否未编译生成class
for re_match in re_matchs:
is_file = pathlib.Path(re_match.replace(".java", ".class")).is_file()
if is_file:
re_matchs.remove(re_match)
try:
for re_match in re_matchs.copy():
is_file = pathlib.Path(re_match.replace(".java", ".class")).is_file()
if is_file:
re_matchs.remove(re_match)
except Exception as e:
print("waring:" + e)

app_jars_name = [jar_path.name.rstrip('.jar') for jar_path in pathlib.Path(app_path).glob('**/*.jar')]
error_jars = [app_jar_name for app_jar_name in app_jars_name for re_match in re_matchs if app_jar_name not in re_match]
error_jars = [app_jar_name for app_jar_name in app_jars_name for re_match in re_matchs if
app_jar_name not in re_match]
error_classes = re_matchs.difference(set(error_jars))
# 使用 procyon 反编译jar包
for app_jar_path in pathlib.Path(app_path).glob('**/*.jar'):
jar_folder = app_jar_path.name.rstrip('.jar')
if jar_folder in error_jars:
_sub = subprocess.getstatusoutput('java -jar "{}" {} -o {}/{}'.format(procyon_path, app_jar_path, save_path, jar_folder))
_sub = subprocess.getstatusoutput(
'java -jar "{}" {} -o {}/{}'.format(procyon_path, app_jar_path, save_path, jar_folder))

# 使用 procyon 反编译class文件
if not os.path.exists(save_path + "/procyon_class"):
os.mkdir(save_path + "/procyon_class")
for class_path in error_classes:
class_path = str(class_path).replace(save_path, app_path).replace(".java", ".class")
_sub = subprocess.getstatusoutput('java -jar "{}" {} -o {}/procyon_class'.format(procyon_path, class_path, save_path))
_sub = subprocess.getstatusoutput(
'java -jar "{}" {} -o {}/procyon_class'.format(procyon_path, class_path, save_path))
# 将反编译后的文件替换原先文件
for class_path in pathlib.Path(save_path+"/procyon_class").glob('**/*.java'):
to_class_path = [class_path for class_path in pathlib.Path(save_path).glob('**/{}'.format(class_path.relative_to("{}/procyon_class".format(save_path)))) if "procyon_class" not in str(class_path)]
for class_path in pathlib.Path(save_path + "/procyon_class").glob('**/*.java'):
to_class_path = [class_path for class_path in pathlib.Path(save_path).glob(
'**/{}'.format(class_path.relative_to("{}/procyon_class".format(save_path)))) if
"procyon_class" not in str(class_path)]
shutil.move(str(class_path), str(to_class_path[0]))

shutil.rmtree(save_path + "/procyon_class")


# 创建用来编译的脚本，再codeql创建数据库时使用
# 创建用来编译的脚本，在codeql创建数据库时使用
def compile_cmd_file_create():
# 准备待编译的jar包
with open("{}/file.txt".format(save_path), "w+") as f:
Expand All @@ -92,39 +111,132 @@ def compile_cmd_file_create():

with open("{}/run.sh".format(save_path), "w+") as f:
f.write(compile_cmd)
# 给予权限
if platform.system().lower() != 'windows':
os.system('chmod u+x {}/run.sh'.format(save_path))


def create_database(app_path, include=None, source=None):
if source is None:
source = app_path
if include is None:
os.system(
'{codeql} database create ../{name} -l \"java\" -s \"{source}\" -c \"{run}\"'.format(
app_path=app_path, codeql=codeql_path,
name="{0}_database".format(
pathlib.Path(
app_path).name),
source=source,
run=run))
else:
os.system("{codeql} database init -s {source} -l \"java\" {name}".format(app_path=app_path,
codeql=codeql_path,
name="{0}_database".format(
pathlib.Path(
app_path).name),
source=source))
os.system("{codeql} database trace-command {name} {run}".format(app_path=app_path,
codeql=codeql_path,
name="{0}_database".format(
pathlib.Path(
app_path).name),
run=run
))
extend = None
if include == "xml":
extend = "xml"
os.system(
"{codeql} database index-files --language {include} --include-extension .{extend} {name}".format(
app_path=app_path,
codeql=codeql_path,
include=include,
extend=extend,
name="{0}_database".format(
pathlib.Path(
app_path).name),
))
os.system(
"{codeql} database finalize {name}".format(
app_path=app_path,
codeql=codeql_path,
include=include,
extend=extend,
name="{0}_database".format(
pathlib.Path(
app_path).name),
))


epilog = r'''Example:
python3 codeql_compile.py -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib"
python3 codeql_compile.py -a D:\java\apps\cloud -o "D:\java\apps\cloud_save_1641018608 -c"
python3 codeql_compile.py -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib" 仅编译程序
python3 codeql_compile.py -l java -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib" 普通创建
python3 codeql_compile.py -l java -a D:\java\apps\cloud -d "D:\java\apps\cloud\lib -i xml" 包含xml文件
'''
parse = argparse.ArgumentParser(epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
parse.add_argument('-a', '--app', help='输入源项目路径')
parse.add_argument('-l', '--language', help='指定语言')
parse.add_argument('-i', '--include', help='指定要额外包含的文件类型')
parse.add_argument('-a', '--app', help='输入项目根路径')
parse.add_argument('-d', '--dep', help='输入依赖包路径')
parse.add_argument('-c', '--check', help='对java-decompiler反编译内容进行检测', action="store_true")
parse.add_argument('-o', '--out', help='输入刚刚已经反编译好的存放路径')
args = parse.parse_args()
# 指定语言
language = args.language
# 指定源项目路径，就是要被反编译的项目路径
app_path = args.app
# 指定依赖包路径，在编译代码时需要指定，否则代码中存在相关依赖会导致编译失败
dependencies_path = args.dep
save_path = args.out

if app_path is not None and dependencies_path is not None:
ecj_path = verify("ecj.jar", self_ecj_path)
java_decompiler_path = verify("java-decompiler.jar", self_java_decompiler_path)
if ecj_path is False or java_decompiler_path is False:
sys.exit("请在当前目录存放ecj.jar、java-decompiler.jar，或者通过self_java_decompiler_path、self_ecj_path指定自定义路径")
save_path = pathlib.Path.joinpath(pathlib.Path(app_path).parent, "{}_save_{}".format(pathlib.Path(app_path).name, int(time.time())))
save_path.mkdir()
java_decompiler_run()
compile_cmd_file_create()

elif args.check is not None and app_path is not None and save_path is not None:
procyon_path = verify("procyon.jar", self_procyon_path)
if procyon_path is False:
sys.exit("请在当前目录存放procyon.jar，或者通过self_procyon_path指定自定义路径")
check()
# 指定要包含的文件
include = args.include
# 脚本类型
run = None
if "windows" == get_arch():
run = "./run.sh"
else:
parse.print_help()
sys.exit()
run = "run.cmd"

if __name__ == '__main__':
if language == "java":
if app_path is not None and dependencies_path is not None:
ecj_path = verify("ecj.jar", self_ecj_path)
java_decompiler_path = verify("java-decompiler.jar", self_java_decompiler_path)
if ecj_path is False or java_decompiler_path is False:
sys.exit("请在当前目录存放ecj.jar、java-decompiler.jar，或者通过self_java_decompiler_path、self_ecj_path指定自定义路径")
save_path = pathlib.Path.joinpath(pathlib.Path(app_path).parent,
"{}_save_{}".format(pathlib.Path(app_path).name, int(time.time())))
save_path.mkdir()
java_decompiler_run()
compile_cmd_file_create()
create_database(app_path, include, save_path)
elif app_path is not None:
create_database(app_path, save_path)
elif args.check is not None and app_path is not None and save_path is not None:
procyon_path = verify("procyon.jar", self_procyon_path)
if procyon_path is False:
sys.exit("请在当前目录存放procyon.jar，或者通过self_procyon_path指定自定义路径")
check()
else:
parse.print_help()
sys.exit()
else:
if app_path is not None and dependencies_path is not None:
ecj_path = verify("ecj.jar", self_ecj_path)
java_decompiler_path = verify("java-decompiler.jar", self_java_decompiler_path)
if ecj_path is False or java_decompiler_path is False:
sys.exit("请在当前目录存放ecj.jar、java-decompiler.jar，或者通过self_java_decompiler_path、self_ecj_path指定自定义路径")
save_path = pathlib.Path.joinpath(pathlib.Path(app_path).parent,
"{}_save_{}".format(pathlib.Path(app_path).name, int(time.time())))
save_path.mkdir()
java_decompiler_run()
compile_cmd_file_create()
elif app_path is not None:
create_database(app_path, save_path)
elif args.check is not None and app_path is not None and save_path is not None:
procyon_path = verify("procyon.jar", self_procyon_path)
if procyon_path is False:
sys.exit("请在当前目录存放procyon.jar，或者通过self_procyon_path指定自定义路径")
check()
else:
parse.print_help()
sys.exit()