diff --git a/frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js b/frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js index 39784705083..d78ce67b543 100644 --- a/frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js +++ b/frontend/src/components/dialog/sysadmin-dialog/sysadmin-logs-export-excel-dialog.js @@ -2,6 +2,9 @@ import React from 'react'; import PropTypes from 'prop-types'; import { Button, Modal, ModalHeader, ModalBody, ModalFooter, FormGroup, Label, Input, Alert } from 'reactstrap'; import { gettext, siteRoot } from '../../../utils/constants'; +import { systemAdminAPI } from '../../../utils/system-admin-api'; +import toaster from '../../../components/toast'; +import { Utils } from '../../../utils/utils'; import moment from 'moment'; class LogsExportExcelDialog extends React.Component { @@ -13,6 +16,7 @@ class LogsExportExcelDialog extends React.Component { startDateStr: '', endDateStr: '', errMsg: '', + taskId: '', }; } @@ -20,25 +24,65 @@ class LogsExportExcelDialog extends React.Component { if (!this.isValidDateStr()) { return; } - let { startDateStr, endDateStr } = this.state; - let url = siteRoot; - switch (this.props.logType) { case 'login': - url += 'sys/loginadmin/export-excel/'; + this.sysExportLogs('loginadmin'); break; case 'fileAccess': - url += 'sys/log/fileaudit/export-excel/'; + this.sysExportLogs('fileaudit'); break; case 'fileUpdate': - url += 'sys/log/fileupdate/export-excel/'; + this.sysExportLogs('fileupdate'); break; case 'sharePermission': - url += 'sys/log/permaudit/export-excel/'; + this.sysExportLogs('permaudit'); break; } - location.href = url + '?start=' + startDateStr + '&end=' + endDateStr; - this.props.toggle(); + }; + + sysExportLogs = (logType) =>{ + let { startDateStr, endDateStr } = this.state; + let task_id = ''; + systemAdminAPI.sysAdminExportLogsExcel(startDateStr, endDateStr, logType).then(res => { + task_id = res.data.task_id; + this.setState({ + taskId: task_id + }); + this.props.toggle(); + return systemAdminAPI.queryAsyncOperationExportExcel(task_id); + }).then(res => { + if (res.data.is_finished === true){ + location.href = siteRoot + 'sys/log/export-excel/?task_id=' + task_id + '&log_type=' + logType; + } else { + this.timer = setInterval(() => { + systemAdminAPI.queryAsyncOperationExportExcel(task_id).then(res => { + if (res.data.is_finished === true){ + this.setState({isFinished: true}); + clearInterval(this.timer); + location.href = siteRoot + 'sys/log/export-excel/?task_id=' + task_id + '&log_type=' + logType; + } + }).catch(err => { + if (this.state.isFinished === false){ + clearInterval(this.timer); + toaster.danger(gettext('Failed to export. Please check whether the size of table attachments exceeds the limit.')); + } + }); + }, 1000); + } + }).catch(error => { + this.props.toggle(); + if (error.response && error.response.status === 500) { + const error_msg = error.response.data ? error.response.data['error_msg'] : null; + if (error_msg && error_msg !== 'Internal Server Error') { + toaster.danger(error_msg); + } else { + toaster.danger(gettext('Internal Server Error.')); + } + } else { + let errMessage = Utils.getErrorMsg(error); + toaster.danger(errMessage); + } + }); }; isValidDateStr = () => { diff --git a/frontend/src/utils/system-admin-api.js b/frontend/src/utils/system-admin-api.js index d69816f278e..1bf4c9b7d57 100644 --- a/frontend/src/utils/system-admin-api.js +++ b/frontend/src/utils/system-admin-api.js @@ -66,6 +66,21 @@ class SystemAdminAPI { return this.req.get(url, {params: params}); } + sysAdminExportLogsExcel(start, end, logType) { + const url = this.server + '/api/v2.1/admin/logs/export-excel/'; + const params = { + start: start, + end: end, + logType: logType + }; + return this.req.get(url, { params: params }); + } + + queryAsyncOperationExportExcel(task_id) { + const url = this.server + '/api/v2.1/query-export-status/?task_id=' + task_id; + return this.req.get(url); + } + } let systemAdminAPI = new SystemAdminAPI(); diff --git a/seahub/api2/endpoints/admin/logs_export.py b/seahub/api2/endpoints/admin/logs_export.py new file mode 100644 index 00000000000..dc36dd701c6 --- /dev/null +++ b/seahub/api2/endpoints/admin/logs_export.py @@ -0,0 +1,102 @@ +import os +import logging +import time +import json +from shutil import rmtree +from django.http import FileResponse +from rest_framework.authentication import SessionAuthentication +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework import status +from urllib.parse import quote + +from seahub.api2.authentication import TokenAuthentication +from seahub.api2.endpoints.utils import check_time_period_valid, export_logs_to_excel, event_export_status +from seahub.api2.permissions import IsProVersion +from seahub.api2.throttling import UserRateThrottle +from seahub.api2.utils import api_error +from seahub.auth.decorators import login_required +from seahub.base.decorators import sys_staff_required + + +class SysLogsExport(APIView): + authentication_classes = (TokenAuthentication, SessionAuthentication) + permission_classes = (IsAdminUser, IsProVersion) + throttle_classes = (UserRateThrottle,) + + def get(self, request): + start = request.GET.get('start', None) + end = request.GET.get('end', None) + log_type = request.GET.get('logType', None) + + if not check_time_period_valid(start, end): + error_msg = 'Failed to export excel, invalid start or end date.' + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + + task_id = export_logs_to_excel(start, end, log_type) + res_data = {'task_id': task_id} + return Response(res_data) + + +class FileLogsExportStatus(APIView): + authentication_classes = (TokenAuthentication, SessionAuthentication) + permission_classes = (IsAdminUser, IsProVersion) + throttle_classes = (UserRateThrottle,) + + def get(self, request): + """ + Get task status by task id + """ + task_id = request.GET.get('task_id', '') + if not task_id: + error_msg = 'task_id invalid.' + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + + resp = event_export_status(task_id) + if resp.status_code == 500: + logger.error('query export status error: %s, %s' % (task_id, resp.content)) + return api_error(500, 'Internal Server Error') + if not resp.status_code == 200: + return api_error(resp.status_code, resp.content) + + is_finished = json.loads(resp.content)['is_finished'] + + return Response({'is_finished': is_finished}) + + +@login_required +@sys_staff_required +def sys_log_export_excel(request): + task_id = request.GET.get('task_id', None) + log_type = request.GET.get('log_type', None) + + if not task_id: + error_msg = 'task_id invalid.' + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + + if log_type == 'loginadmin': + excel_name = 'login-logs.xlsx' + elif log_type == 'fileaudit': + excel_name = 'file-access-logs.xlsx' + elif log_type == 'fileupdate': + excel_name = 'file-update-logs.xlsx' + elif log_type == 'permaudit': + excel_name = 'perm-audit-logs.xlsx' + else: + error_msg = 'log_type invalid' + return api_error(status.HTTP_400_BAD_REQUEST, error_msg) + + target_dir = os.path.join('/tmp/seafile_events/', task_id) + tmp_excel_path = os.path.join(target_dir, excel_name) + if not os.path.isfile(tmp_excel_path): + return api_error(status.HTTP_400_BAD_REQUEST, excel_name + ' not found.') + + response = FileResponse(open(tmp_excel_path, 'rb'), content_type='application/ms-excel', as_attachment=True) + try: + rmtree(target_dir) + except OSError: + pass + response['Content-Disposition'] = 'attachment;filename*=UTF-8\'\'' + quote(excel_name) + + return response diff --git a/seahub/api2/endpoints/utils.py b/seahub/api2/endpoints/utils.py index 7c40756fa88..a83daf41d40 100644 --- a/seahub/api2/endpoints/utils.py +++ b/seahub/api2/endpoints/utils.py @@ -4,6 +4,7 @@ import time import logging import requests +import json import datetime import urllib.request import urllib.parse @@ -18,7 +19,8 @@ from seahub.base.templatetags.seahub_tags import email2nickname, email2contact_email from seahub.utils import get_log_events_by_time, is_pro_version, is_org_context -from seahub.settings import SEADOC_PRIVATE_KEY, FILE_CONVERTER_SERVER_URL +from seahub.settings import SEADOC_PRIVATE_KEY, FILE_CONVERTER_SERVER_URL, SECRET_KEY, \ + SEAFEVENTS_SERVER_URL try: from seahub.settings import MULTI_TENANCY @@ -282,3 +284,35 @@ def sdoc_export_to_docx(path, username, doc_uuid, download_token, resp = requests.post(url, json=params, headers=headers, timeout=30) return resp + + +def format_date(start, end): + start_struct_time = datetime.datetime.strptime(start, "%Y-%m-%d") + start_timestamp = time.mktime(start_struct_time.timetuple()) + + end_struct_time = datetime.datetime.strptime(end, "%Y-%m-%d") + end_timestamp = time.mktime(end_struct_time.timetuple()) + end_timestamp += 24 * 60 * 60 + return start_timestamp, end_timestamp + + +def export_logs_to_excel(start, end, log_type): + start_time, end_time = format_date(start, end) + payload = {'exp': int(time.time()) + 300, } + token = jwt.encode(payload, SECRET_KEY, algorithm='HS256') + headers = {"Authorization": "Token %s" % token} + url = urljoin(SEAFEVENTS_SERVER_URL, '/add-init-export-log-task') + params = {'start_time': start_time, 'end_time': end_time, 'log_type': log_type} + resp = requests.get(url, params=params, headers=headers) + return json.loads(resp.content)['task_id'] + + +def event_export_status(task_id): + payload = {'exp': int(time.time()) + 300, } + token = jwt.encode(payload, SECRET_KEY, algorithm='HS256') + headers = {"Authorization": "Token %s" % token} + url = urljoin(SEAFEVENTS_SERVER_URL, '/query-export-status') + params = {'task_id': task_id} + resp = requests.get(url, params=params, headers=headers) + + return resp diff --git a/seahub/sysadmin_extra/views.py b/seahub/sysadmin_extra/views.py deleted file mode 100644 index 00e0c86a991..00000000000 --- a/seahub/sysadmin_extra/views.py +++ /dev/null @@ -1,262 +0,0 @@ -# Copyright (c) 2012-2016 Seafile Ltd. -import os -import logging - -from django.shortcuts import render - -from django.utils.translation import gettext as _ -from django.contrib import messages -from django.http import HttpResponse, HttpResponseRedirect, Http404 -from django.core.exceptions import ValidationError - -from seahub.api2.endpoints.utils import check_time_period_valid, \ - get_log_events_by_type_and_time - -from seahub.base.decorators import sys_staff_required -from seahub.auth.decorators import login_required -from seahub.sysadmin_extra.models import UserLoginLog -from seahub.utils import EVENTS_ENABLED, get_file_audit_events, \ - get_file_update_events, get_perm_audit_events, \ - is_pro_version, generate_file_audit_event_type -from seahub.utils.timeutils import utc_to_local -from seahub.utils.ms_excel import write_xls -from seahub.settings import SITE_ROOT - -from seaserv import seafile_api, ccnet_api - -logger = logging.getLogger(__name__) - -@login_required -@sys_staff_required -def sys_login_admin_export_excel(request): - """ Export user login logs to excel. - """ - next_page = request.headers.get('referer', None) - if not next_page: - next_page = SITE_ROOT - - start = request.GET.get('start', None) - end = request.GET.get('end', None) - - if not check_time_period_valid(start, end): - messages.error(request, _('Failed to export excel, invalid start or end date')) - return HttpResponseRedirect(next_page) - - # Filtering a DateTimeField with dates won't include items on the last day, - # because the bounds are interpreted as '0am on the given date'. - end = end + ' 23:59:59' - - try: - user_login_logs = UserLoginLog.objects.filter(login_date__range=(start, end)) - except ValidationError as e: - logger.error(e) - messages.error(request, _('Failed to export excel, invalid start or end date')) - return HttpResponseRedirect(next_page) - - logs = list(user_login_logs) - head = [_("Name"), _("IP"), _("Status"), _("Time")] - data_list = [] - for log in logs: - login_time = log.login_date.strftime("%Y-%m-%d %H:%M:%S") - status = _('Success') if log.login_success else _('Failed') - row = [log.username, log.login_ip, status, login_time] - data_list.append(row) - - wb = write_xls('login-logs', head, data_list) - if not wb: - messages.error(request, _('Failed to export excel')) - return HttpResponseRedirect(next_page) - - response = HttpResponse(content_type='application/ms-excel') - response['Content-Disposition'] = 'attachment; filename=login-logs.xlsx' - wb.save(response) - return response - -@login_required -@sys_staff_required -def sys_log_file_audit_export_excel(request): - """ Export file access logs to excel. - """ - next_page = request.headers.get('referer', None) - if not next_page: - next_page = SITE_ROOT - - if not is_pro_version(): - messages.error(request, _('Failed to export excel, this feature is only in professional version.')) - return HttpResponseRedirect(next_page) - - start = request.GET.get('start', None) - end = request.GET.get('end', None) - if not check_time_period_valid(start, end): - messages.error(request, _('Failed to export excel, invalid start or end date')) - return HttpResponseRedirect(next_page) - - events = get_log_events_by_type_and_time('file_audit', start, end) - - head = [_("User"), _("Type"), _("IP"), _("Device"), _("Date"), - _("Library Name"), _("Library ID"), _("Library Owner"), _("File Path")] - data_list = [] - - events.sort(key=lambda x: x.timestamp, reverse=True) - for ev in events: - event_type, ev.show_device = generate_file_audit_event_type(ev) - - repo_id = ev.repo_id - repo = seafile_api.get_repo(repo_id) - if repo: - repo_name = repo.name - repo_owner = seafile_api.get_repo_owner(repo_id) or \ - seafile_api.get_org_repo_owner(repo_id) - else: - repo_name = _('Deleted') - repo_owner = '--' - - username = ev.user if ev.user else _('Anonymous User') - date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \ - ev.timestamp else '' - - row = [username, event_type, ev.ip, ev.show_device, - date, repo_name, ev.repo_id, repo_owner, ev.file_path] - data_list.append(row) - - wb = write_xls('file-access-logs', head, data_list) - if not wb: - messages.error(request, _('Failed to export excel')) - return HttpResponseRedirect(next_page) - - response = HttpResponse(content_type='application/ms-excel') - response['Content-Disposition'] = 'attachment; filename=file-access-logs.xlsx' - wb.save(response) - return response - -@login_required -@sys_staff_required -def sys_log_file_update_export_excel(request): - """ Export file update logs to excel. - """ - next_page = request.headers.get('referer', None) - if not next_page: - next_page = SITE_ROOT - - if not is_pro_version(): - messages.error(request, _('Failed to export excel, this feature is only in professional version.')) - return HttpResponseRedirect(next_page) - - start = request.GET.get('start', None) - end = request.GET.get('end', None) - if not check_time_period_valid(start, end): - messages.error(request, _('Failed to export excel, invalid start or end date')) - return HttpResponseRedirect(next_page) - - events = get_log_events_by_type_and_time('file_update', start, end) - - head = [_("User"), _("Date"), _("Library Name"), _("Library ID"), - _("Library Owner"), _("Action")] - data_list = [] - - events.sort(key=lambda x: x.timestamp, reverse=True) - for ev in events: - - repo_id = ev.repo_id - repo = seafile_api.get_repo(repo_id) - if repo: - repo_name = repo.name - repo_owner = seafile_api.get_repo_owner(repo_id) or \ - seafile_api.get_org_repo_owner(repo_id) - else: - repo_name = _('Deleted') - repo_owner = '--' - - username = ev.user if ev.user else _('Anonymous User') - date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \ - ev.timestamp else '' - - row = [username, date, repo_name, ev.repo_id, repo_owner, ev.file_oper.strip()] - data_list.append(row) - - wb = write_xls('file-update-logs', head, data_list) - if not wb: - messages.error(request, _('Failed to export excel')) - return HttpResponseRedirect(next_page) - - response = HttpResponse(content_type='application/ms-excel') - response['Content-Disposition'] = 'attachment; filename=file-update-logs.xlsx' - wb.save(response) - return response - -@login_required -@sys_staff_required -def sys_log_perm_audit_export_excel(request): - """ Export permission audit logs to excel. - """ - next_page = request.headers.get('referer', None) - if not next_page: - next_page = SITE_ROOT - - if not is_pro_version(): - messages.error(request, _('Failed to export excel, this feature is only in professional version.')) - return HttpResponseRedirect(next_page) - - start = request.GET.get('start', None) - end = request.GET.get('end', None) - if not check_time_period_valid(start, end): - messages.error(request, _('Failed to export excel, invalid start or end date')) - return HttpResponseRedirect(next_page) - - events = get_log_events_by_type_and_time('perm_audit', start, end) - - head = [_("From"), _("To"), _("Action"), _("Permission"), _("Library"), - _("Folder Path"), _("Date")] - data_list = [] - - events.sort(key=lambda x: x.timestamp, reverse=True) - for ev in events: - repo = seafile_api.get_repo(ev.repo_id) - repo_name = repo.repo_name if repo else _('Deleted') - - if '@' in ev.to: - to = ev.to - elif ev.to.isdigit(): - group = ccnet_api.get_group(int(ev.to)) - to = group.group_name if group else _('Deleted') - elif 'all' in ev.to: - to = _('Organization') - else: - to = '--' - - if 'add' in ev.etype: - action = _('Add') - elif 'modify' in ev.etype: - action = _('Modify') - elif 'delete' in ev.etype: - action = _('Delete') - else: - action = '--' - - if ev.permission == 'rw': - permission = _('Read-Write') - elif ev.permission == 'r': - permission = _('Read-Only') - else: - permission = '--' - - date = utc_to_local(ev.timestamp).strftime('%Y-%m-%d %H:%M:%S') if \ - ev.timestamp else '' - - row = [ev.from_user, to, action, permission, repo_name, - ev.file_path, date] - data_list.append(row) - - wb = write_xls('perm-audit-logs', head, data_list) - if not wb: - next_page = request.headers.get('referer', None) - if not next_page: - next_page = SITE_ROOT - - messages.error(request, _('Failed to export excel')) - return HttpResponseRedirect(next_page) - - response = HttpResponse(content_type='application/ms-excel') - response['Content-Disposition'] = 'attachment; filename=perm-audit-logs.xlsx' - wb.save(response) - return response diff --git a/seahub/urls.py b/seahub/urls.py index cc8454876dc..21b147cf022 100644 --- a/seahub/urls.py +++ b/seahub/urls.py @@ -121,6 +121,7 @@ from seahub.api2.endpoints.repo_upload_links import RepoUploadLinks, RepoUploadLink # Admin +from seahub.api2.endpoints.admin.logs_export import SysLogsExport, FileLogsExportStatus, sys_log_export_excel from seahub.api2.endpoints.admin.abuse_reports import AdminAbuseReportsView, AdminAbuseReportView from seahub.api2.endpoints.admin.revision_tag import AdminTaggedItemsView from seahub.api2.endpoints.admin.login_logs import LoginLogs, AdminLoginLogs @@ -886,21 +887,17 @@ from seahub.utils import is_pro_version if is_pro_version(): - from seahub.sysadmin_extra.views import \ - sys_login_admin_export_excel, sys_log_file_audit_export_excel, \ - sys_log_file_update_export_excel, sys_log_perm_audit_export_excel urlpatterns += [ re_path(r'^api/v2.1/admin/logs/login/$', LoginLogs.as_view(), name='api-v2.1-admin-logs-login'), - path('sys/loginadmin/export-excel/', sys_login_admin_export_excel, name='sys_login_admin_export_excel'), - re_path(r'^api/v2.1/admin/logs/file-audit/$', FileAudit.as_view(), name='api-v2.1-admin-logs-file-audit'), - path('sys/log/fileaudit/export-excel/', sys_log_file_audit_export_excel, name='sys_log_file_audit_export_excel'), - re_path(r'^api/v2.1/admin/logs/file-update/$', FileUpdate.as_view(), name='api-v2.1-admin-logs-file-update'), - path('sys/log/fileupdate/export-excel/', sys_log_file_update_export_excel, name='sys_log_file_update_export_excel'), - re_path(r'^api/v2.1/admin/logs/perm-audit/$', PermAudit.as_view(), name='api-v2.1-admin-logs-perm-audit'), - path('sys/log/permaudit/export-excel/', sys_log_perm_audit_export_excel, name='sys_log_perm_audit_export_excel'), + + re_path(r'^api/v2.1/admin/logs/export-excel/$', SysLogsExport.as_view(), name='api-v2.1-admin-logs-export-excel'), + re_path(r'^api/v2.1/query-export-status/$', FileLogsExportStatus.as_view(), name='api-v2.1-query-export-status'), + path('sys/log/export-excel/', sys_log_export_excel, name='sys_log_export_excel'), + + ] if getattr(settings, 'MULTI_TENANCY', False): diff --git a/seahub/utils/__init__.py b/seahub/utils/__init__.py index 6110c4fda3a..a958bbb2c57 100644 --- a/seahub/utils/__init__.py +++ b/seahub/utils/__init__.py @@ -1165,7 +1165,6 @@ def prepare_converted_html(raw_path, obj_id, doctype, ret_dict): try: add_office_convert_task(obj_id, doctype, raw_path) except Exception as e: - print(e) logging.exception('failed to add_office_convert_task: %s' % e) return _('Internal Server Error') return None