diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a24c0ef --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: +- '2.6' +- '2.7' +install: +- pip install pytest pep8 requests gi +script: +- pep8 --ignore=E402,E121,E123,E126,E226,E24,E704 client server +notifications: + email: false diff --git a/client/Makefile b/client/Makefile new file mode 100644 index 0000000..678ce75 --- /dev/null +++ b/client/Makefile @@ -0,0 +1,35 @@ +PYTHON=python2 +DESTDIR=/ +BUILDIR=$(CURDIR)/debian/dpcs +PROJECT=dpcs +VERSION=0.1.0 + +all: + @echo "make source - Create source package" + @echo "make install - Install on local system" + @echo "make buildrpm - Generate a rpm package" + @echo "make builddeb - Generate a deb package" + @echo "make clean - Get rid of scratch and byte files" + +source: + $(PYTHON) setup.py sdist $(COMPILE) + +install: + $(PYTHON) setup.py install --root $(DESTDIR) $(COMPILE) + +buildrpm: + $(PYTHON) setup.py bdist_rpm --post-install=rpm/postinstall --pre-uninstall=rpm/preuninstall + +builddeb: + # build the source package in the parent directory + # then rename it to project_version.orig.tar.gz + $(PYTHON) setup.py sdist $(COMPILE) --dist-dir=../ + rename -f 's/$(PROJECT)-(.*)\.tar\.gz/$(PROJECT)_$$1\.orig\.tar\.gz/' ../* + # build the package + dpkg-buildpackage -i -I -rfakeroot + +clean: + $(PYTHON) setup.py clean + $(MAKE) -f $(CURDIR)/debian/rules clean + rm -rf build/ MANIFEST + find . -name '*.pyc' -delete diff --git a/systemcheck/__init__.py b/client/configure old mode 100644 new mode 100755 similarity index 100% rename from systemcheck/__init__.py rename to client/configure diff --git a/client/configure.in b/client/configure.in new file mode 100644 index 0000000..5988637 --- /dev/null +++ b/client/configure.in @@ -0,0 +1 @@ +dnl autoconf stub diff --git a/client/debian/changelog b/client/debian/changelog new file mode 100644 index 0000000..006b83c --- /dev/null +++ b/client/debian/changelog @@ -0,0 +1,5 @@ +dpcs (0.1) UNRELEASED; urgency=low + + * Initial release. + + -- Marek Bardoński Mon, 14 Mar 2016 17:22:58 +0100 diff --git a/client/debian/compat b/client/debian/compat new file mode 100644 index 0000000..2542995 --- /dev/null +++ b/client/debian/compat @@ -0,0 +1 @@ ++7 diff --git a/client/debian/control b/client/debian/control new file mode 100644 index 0000000..64984bd --- /dev/null +++ b/client/debian/control @@ -0,0 +1,17 @@ +Source: dpcs +Section: python +Priority: optional +Maintainer: Marek Bardoński, +Build-Depends: debhelper (>= 7), + python (>= 2.6.6-3~), + python-requests (>= 2.2), + python-gi (>= 3.12) +Standards-Version: 3.9.2 +X-Python-Version: >= 2.6 + +Package: dpcs +Architecture: all +Section: python +Depends: ${misc:Depends}, ${python:Depends} +Description: An application, that would help Ubuntu users in effortless problem solving. + An application for automatic problem solving in Ubuntu OS. In short, our application would first gather large amount of data concerning crashes of application that Ubuntu users have encountered during their work. Next we would analyze those errors and find solutions. Based on the resulting database, a user would be proposed a solution for an error that he encounters. diff --git a/client/debian/copyright b/client/debian/copyright new file mode 100644 index 0000000..fe7dae8 --- /dev/null +++ b/client/debian/copyright @@ -0,0 +1,7 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: dpcs +Upstream-Contact: Marek Bardoński + +Files: * +Copyright: 2016, dpcs-team, +License: LGPL-3 diff --git a/client/debian/rules b/client/debian/rules new file mode 100755 index 0000000..31048c2 --- /dev/null +++ b/client/debian/rules @@ -0,0 +1,11 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +include /usr/share/cdbs/1/rules/debhelper.mk +include /usr/share/cdbs/1/class/python-autotools.mk + +DEB_PYTHON2_MODULE_PACKAGES = dpcs + +clean:: + rm -rf build build-stamp configure-stamp build/ MANIFEST + dh_clean diff --git a/client/dpcs_client/__init__.py b/client/dpcs_client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/client/dpcs_client/settings.py b/client/dpcs_client/settings.py new file mode 100644 index 0000000..53550e7 --- /dev/null +++ b/client/dpcs_client/settings.py @@ -0,0 +1,18 @@ +"""Utilities for handling settings.""" +import json +import os + +# config file path +DEFAULT_SERVER_ADDRESS = "http://private-a6e53-dpcs.apiary-mock.com/" +FILE = os.path.expanduser('~/.dpcs/.dpcsconfig') + + +def read_settings(): + with open(FILE, 'r') as f: + try: + settings = json.load(f) + if 'server_address' not in settings: + settings['server_address'] = DEFAULT_SERVER_ADDRESS + except ValueError: + settings = {'server_address': DEFAULT_SERVER_ADDRESS} + return settings diff --git a/systemcheck/systemcheck.py b/client/dpcs_client/systemcheck.py old mode 100755 new mode 100644 similarity index 87% rename from systemcheck/systemcheck.py rename to client/dpcs_client/systemcheck.py index 727ac75..9f0318e --- a/systemcheck/systemcheck.py +++ b/client/dpcs_client/systemcheck.py @@ -1,9 +1,7 @@ -#!/usr/bin/python - -import platform +"""Utilities for checking system's information.""" import os +import platform import re -import json def systemcheck(): @@ -17,7 +15,8 @@ def systemcheck(): # get installed packages from dpkg packages_raw_info = \ os.popen( - "dpkg-query -W -f='${binary:Package}\t${Version}\t${Status}\n' | grep \"install ok installed\"" + "dpkg-query -W -f='${binary:Package}\t${Version}\t${Status}\n'" + + " | grep \"install ok installed\"" ).read() packages_raw_info = packages_raw_info.split("\n") @@ -35,8 +34,3 @@ def systemcheck(): data["platform"] = platform_dict data["packages"] = package_list return data - - -if __name__ == "__main__": - - print(json.dumps(systemcheck())) diff --git a/client/main/client.py b/client/main/dpcs old mode 100644 new mode 100755 similarity index 93% rename from client/main/client.py rename to client/main/dpcs index 1ae85f9..af90c04 --- a/client/main/client.py +++ b/client/main/dpcs @@ -1,3 +1,4 @@ +#! /usr/bin/env python2 # -*- coding: utf-8 -*- from __future__ import print_function from os import mkdir @@ -13,12 +14,11 @@ from requests import post from requests import RequestException -from ...systemcheck.systemcheck import systemcheck +from dpcs_client.settings import read_settings +from dpcs_client.systemcheck import systemcheck EXIT_OK = 0 -SERVER_ADDRESS = "http://private-a6e53-dpcs.apiary-mock.com/" - def generate_report(exit_code, stderr_output): """Generate a report which will be sent to the DCPS server. @@ -143,8 +143,10 @@ def save_script(script): if __name__ == '__main__': if len(sys.argv) != 2: - print("Usage python client.py '[command to check]'") - exit(1) + print("Usage dpcs '[command to check]'") + sys.exit(1) + + server_address = read_settings()['server_address'] p = subprocess.Popen(sys.argv[1], stderr=subprocess.PIPE, @@ -156,7 +158,7 @@ def save_script(script): if code != EXIT_OK: - api_description_url = SERVER_ADDRESS + "vd1/paths/" + api_description_url = server_address + "vd1/paths/" try: response = get(api_description_url) except RequestException as e: @@ -166,7 +168,7 @@ def save_script(script): exit(2) api_paths = response.json() - crash_report_url = SERVER_ADDRESS + api_paths["crash-reports"] + crash_report_url = server_address + api_paths["crash-reports"] headers = { 'Content-Type': 'text/json' } diff --git a/client/settings/dpcs-settings b/client/settings/dpcs-settings new file mode 100755 index 0000000..a2e3bc0 --- /dev/null +++ b/client/settings/dpcs-settings @@ -0,0 +1,153 @@ +#!/usr/bin/env python2 + +import json +import pgi as gi +gi.install_as_gi() +gi.require_version('Gtk', '3.0') +from gi.repository import Gtk +import os.path + +from dpcs_client.settings import DEFAULT_SERVER_ADDRESS +from dpcs_client.settings import FILE +from dpcs_client.settings import read_settings + +# the window title +PANELTITLE = "DPCS Settings" + +# about DPCS - authors, basic info, licensing and such... +ABOUT_DPCS = """AI powered agent, running locally on your machine, +analyzing what's going on in the logs and overall on the system, +and advertizing what's wrong, then, hopefully, +offer an answer to help you fix it. +""" + + +# minimal window width and height +WIDTH = 600 +HEIGHT = 200 + + +def displaypanel(): + """ displays the prefs dialog + """ + win = SettingsPanel() + win.show_all() + Gtk.main() + + +class SettingsPanel(Gtk.Window): + """ The preferences dialog for DPCS""" + + def __init__(self): + Gtk.Window.__init__(self, title=PANELTITLE) + self.set_size_request(WIDTH, HEIGHT) + + maincontainer = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, + spacing=0) + self.add(maincontainer) + + notebook = Gtk.Notebook() + maincontainer.pack_start(notebook, True, True, 0) + + buttonscontainer = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL) + maincontainer.pack_start(buttonscontainer, False, False, 0) + + # settings tab + + settingspage = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=5) + + settings = read_settings() + settingspage.pack_start(self.addTextField('Server address', + settings['server_address']), + True, True, 0) + + # end of setting boxes + + notebook.append_page(settingspage, Gtk.Label('Settings')) + + # about DPCS tab + + aboutpage = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=5) + + acontent = Gtk.Label() + acontent.set_text(ABOUT_DPCS) + acontent.set_justify(Gtk.Justification.LEFT) + aboutpage.pack_start(acontent, True, True, 0) + + notebook.append_page(aboutpage, Gtk.Label('About DPCS')) + + # ok, apply and cancel buttons + + self.ok = Gtk.Button(label="Ok") + cancel = Gtk.Button(label="Cancel") + restore = Gtk.Button(label="Restore default") + applyb = Gtk.Button(label="Apply") + + buttonscontainer.pack_end(self.ok, False, False, 0) + buttonscontainer.pack_end(applyb, False, False, 0) + buttonscontainer.pack_end(restore, False, False, 0) + buttonscontainer.pack_end(cancel, False, False, 0) + + cancel.connect("clicked", Gtk.main_quit) + applyb.connect("clicked", self.applysettings) + restore.connect("clicked", self.restoresettings) + self.ok.connect("clicked", self.applysettings) + + # loads current config + self.loadconfig() + self.connect("delete-event", Gtk.main_quit) + + def addTextField(self, label, default_text): + self.entry = Gtk.Entry() + self.entry.set_text(default_text) + label_gtk = Gtk.Label(label) + table = Gtk.Table(1, 3, True) + table.attach(label_gtk, 0, 1, 0, 1) + table.attach(self.entry, 1, 3, 0, 1) + return table + + def applysettings(self, widget): + """ writes settings to dpcs.config, exits + if ok has been pressed + """ + conf_directory = os.path.dirname(FILE) + + if not os.path.exists(conf_directory): + os.makedirs(conf_directory) + + f = open(FILE, 'w') + + settings = {'server_address': self.entry.get_text()} + json.dump(settings, f, indent=2) + f.close() + if widget == self.ok: + Gtk.main_quit() + + def restoresettings(self, widget): + """Restore default settings.""" + conf_directory = os.path.dirname(FILE) + + if not os.path.exists(conf_directory): + os.makedirs(conf_directory) + + f = open(FILE, 'w') + + settings = {'server_address': DEFAULT_SERVER_ADDRESS} + json.dump(settings, f, indent=2) + f.close() + + self.entry.set_text(DEFAULT_SERVER_ADDRESS) + + def loadconfig(self): + """ loads current settings from dpcs.config + """ + with open(FILE, 'r') as f: + try: + settings = json.load(f) + if 'server_address' in settings: + self.entry.set_text(settings['server_address']) + except ValueError: + pass + +if __name__ == "__main__": + displaypanel() diff --git a/client/settings/dpcs-settingspanel/README.md b/client/settings/dpcs-settingspanel/README.md deleted file mode 100644 index 216b3b8..0000000 --- a/client/settings/dpcs-settingspanel/README.md +++ /dev/null @@ -1 +0,0 @@ -# The settings panel for the DPCS project diff --git a/client/settings/dpcs-settingspanel/settingspanel.py b/client/settings/dpcs-settingspanel/settingspanel.py deleted file mode 100755 index ff5b316..0000000 --- a/client/settings/dpcs-settingspanel/settingspanel.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 - -import json -import gi -gi.require_version('Gtk', '3.0') -from gi.repository import Gtk -import os.path - -# the window title -PANELTITLE = "DPCS Settings" - -# about DPCS - authors, basic info, licensing and such... -ABOUT_DPCS = """ To be written! -""" - -# minimal window width and height -WIDTH = 350 -HEIGHT = 400 -# settings dictionary -settings = {} -#config file path -FILE = os.path.expanduser('~/.dpcs/.dpcsconfig') - -buttonlist = [] - -def displaypanel(): - """ displays the prefs dialog - """ - win = SettingsPanel() - win.show_all() - Gtk.main() - -def addCheckButton(label, parent): - """ add checkbox with label @label to container @parent - """ - checkButton = Gtk.CheckButton.new_with_label(label) - parent.pack_start(checkButton, False, False, 10) - buttonlist.append(checkButton) - return checkButton - -def save_setting (checkButton): - """ saves checkbox value in settings - """ - if checkButton.get_active(): - value = 'true' - else: - value = 'false' - settings[checkButton.get_label()] = value - -def load_setting (checkButton): - """ loads initial value of @checkbutton - """ - if settings[checkButton.get_label()] == 'true' : - checkButton.set_active(True) - else: - checkButton.set_active(False) - -class SettingsPanel(Gtk.Window): - """ The preferences dialog for DPCS - """ - - def __init__(self): - Gtk.Window.__init__(self, title=PANELTITLE) - self.set_size_request(WIDTH,HEIGHT) - - maincontainer = Gtk.Box(orientation = Gtk.Orientation.VERTICAL, spacing = 0) - self.add(maincontainer) - - notebook = Gtk.Notebook() - maincontainer.pack_start(notebook, True, True, 0) - - buttonscontainer = Gtk.Box(orientation = Gtk.Orientation.HORIZONTAL) - maincontainer.pack_start(buttonscontainer, False, False, 0) - - # settings tab - - settingspage = Gtk.Box(orientation = Gtk.Orientation.VERTICAL, spacing = 5) - - # add setting boxes here by addCheckButton(Label, Box) - - eb = addCheckButton('Enable DPCS', settingspage) - ab = addCheckButton('test1', settingspage) - bb = addCheckButton('test2', settingspage) - gb = addCheckButton('test3', settingspage) - - # end of setting boxes - - notebook.append_page(settingspage, Gtk.Label('Settings')) - - # about DPCS tab - - aboutpage = Gtk.Box(orientation = Gtk.Orientation.VERTICAL, spacing = 5) - - acontent = Gtk.Label() - acontent.set_text(ABOUT_DPCS) - acontent.set_justify(Gtk.Justification.LEFT) - aboutpage.pack_start(acontent, True, True, 0) - - notebook.append_page(aboutpage, Gtk.Label('About DPCS')) - - # ok, apply and cancel buttons - - self.ok = Gtk.Button(label="Ok") - cancel = Gtk.Button(label="Cancel") - applyb = Gtk.Button(label="Apply") - - buttonscontainer.pack_end(self.ok, False, False, 0) - buttonscontainer.pack_end(applyb, False, False, 0) - buttonscontainer.pack_end(cancel, False, False, 0) - - cancel.connect("clicked", Gtk.main_quit) - applyb.connect("clicked", self.applysettings) - self.ok.connect("clicked", self.applysettings) - - # loads current config - self.loadconfig() - self.connect("delete-event", Gtk.main_quit) - - - def applysettings(self, widget): - """ writes settings to dpcs.config, exits - if ok has been pressed - """ - os.makedirs(os.path.dirname(FILE), exist_ok=True) - f = open(FILE, 'w') - - for s in buttonlist: - save_setting(s) - json.dump(settings, f, indent=2) - f.close() - if widget == self.ok: - Gtk.main_quit() - - def loadconfig(self): - """ loads current settings from dpcs.config - """ - if os.path.isfile(FILE): - f = open(FILE, 'r') - settings.update(json.load(f)) - for b in buttonlist: - load_setting(b) - - f.close() - -if __name__ == "__main__": - displaypanel() \ No newline at end of file diff --git a/client/setup.py b/client/setup.py new file mode 100644 index 0000000..1e30eb6 --- /dev/null +++ b/client/setup.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 3, as published +# by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranties of +# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +"""Setup file for dpcs-client.""" + +from setuptools import setup +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + errno = pytest.main(self.test_args) + raise SystemExit(errno) + +setup(classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: ' + + 'GNU Lesser General Public License v3 (LGPLv3)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet :: Log Analysis', + 'Topic :: System :: Monitoring', + 'Topic :: System :: Software Distribution', + 'Topic :: Utilities' + ], + scripts=['main/dpcs', 'settings/dpcs-settings'], + cmdclass={'test': PyTest}, + description='The client for the Data Powered Crash Solver', + install_requires=['requests', 'gi'], + keywords=['dpcs', 'ai', 'fix', 'error solving', 'log analysis'], + license='LGPLv3', + long_description=open('../readme.md', 'r').read(), + name='dpcs', + packages=['dpcs_client'], + tests_require=['pytest', 'coverage'], + url='https://github.com/DPCS-team/DPCS', + version='0.1.0' + ) diff --git a/docs/Comparing_lifelong_system_logs.pdf b/docs/Comparing_lifelong_system_logs.pdf new file mode 100644 index 0000000..2334663 Binary files /dev/null and b/docs/Comparing_lifelong_system_logs.pdf differ diff --git a/docs/Evaluating algorithm's efficiency b/docs/Evaluating algorithm's efficiency new file mode 100644 index 0000000..77cdc79 --- /dev/null +++ b/docs/Evaluating algorithm's efficiency @@ -0,0 +1,18 @@ +Evaluating algorithm's efficiency + +There are following issues which must be consider: + We probably want our models to have (and use) option 'don't know the solution' when there are low chances of success. + We may have very little test data (for example if model solution can be asses only by human). + Wrong decisions may have different scale of consequences – we don’t want to crash computer of our user. + +Taking it into account it may be necessary to: + Ask alpha users to give us more feedback than 'this (do not) solved my problem'. It may be possible because some of those will be people somehow interested in our work. + Prepare set of examples for which we would establish good solutions and than check our algorithms on those. + If our solvers will work not only by suggesting solutions but implementing it, than we could prepare some automatic tests. +The first of solutions seems easiest (although we probably should do at least some basic tests before giving solver to any user). + + +The other methods of assessment worth considering are: + Counting time from giving solution to the user to his (positive or negative) evaluation of results. The shortest time means that solution (even if wrong) was more understandable. + Counting users resigning from using given version of solver (but this may not be possible with alpha version) + diff --git a/docs/Initial_research.md b/docs/Initial_research.md new file mode 100644 index 0000000..0bd7dc7 --- /dev/null +++ b/docs/Initial_research.md @@ -0,0 +1,142 @@ +Title: Application error message clustering and classification in order to find an already existing solution, via an offline agent. Initial research. +-------------------------- + +Abstract +-------------- +*#PERSON Marek Bardoński #2ND* + +General usage: problem description, proposed solution + +Introduction, motivation +-------------------------- +*#PERSON Mateusz Macias* + +1) Where did the idea come from? (idea from Canonical + us, meeting) + +2) UW ML RG, to learn ML techniques, focus on teaching, large group of people, some of them are unexperienced + +3) Use cases (1) new linux users, 2) admins, manual scripting replacement, 3) others(google + stackoverflow)) + +4) Plans to incorporate the app in Ubuntu 17.04 + +Algorithms overview +-------------------------- +*#PERSON Mateusz Susik #2ND* + +1) Clustering - affinity propagation + +2) Classification - NN + +Techincal overview +-------------------------- +*#PERSON Mateusz Susik #2ND* + +1) Server + +a) Communication with client, security - detecting spam, loops, attacks, etc + +b) HDFS, canonical, Spark + +c) Cleaning the database + +2) Client + +a) REST, problems with reading from terminals + +b) offline classification + +3) Pipeline description + + + +Preprocessing +-------------------------- +*#PERSON Hubert Tarasiuk* + +1) Normalization of system paths (~home), /opt/bin, /bin/ etc - heuristics + +2) lowercase, 's, timestamps, PII (emails, passwords) removal (library?) + +3) Optional translation + +4) Stopwords (?) + + +Clustering +-------------------------- +*#PERSON Marek Bardoński Jacek Karwowski* + +Features + +1) Word count, word bigram, TF, IDF, package name, package version, basic system info (possible size limitations?) [src5] + +Main algorithm + +2) Affinity propagation [src1] [abstract generation - src needed] + +Supporting algorithms + +3) Spectral clustering [src needed] + +Heuristics + +4) thefuck [src4], microsoft [src3] + +Classification +-------------------------- +*#PERSON Piotr Wiśniewski* + +Main algorithm + +1) Neural networks + +Supporting algorithms + +2) Multiclass logistic regression + + +Solution matching approaches +-------------------------- +Stack overflow crawler + +*#PERSON Szymon Pajzert* + +Lifelong systemlogs + +*#PERSON Marek Bardonski* + +Examples +-------------------------- +*#PERSON Krystyna Gajczyk* + +Usecases + + +Similar projects +-------------------------- +*#PERSON Jakub Staroń* + +Red Hat Access + +Pulse + +Entropy + +Cluebox + + + +Bibliography +-------------------------- +[src1] http://www.psi.toronto.edu/affinitypropagation/FreyDueckScience07.pdf + +[src4] https://github.com/nvbn/thefuck + +[src3] http://research.microsoft.com/apps/pubs/default.aspx?id=81176 + +[src5] http://uu.diva-portal.org/smash/get/diva2:667650/FULLTEXT01.pdf + +Symbols +---------- +#2ND - second iteration, after completing other tasks +(?) - to the autor's discretion diff --git a/docs/SystemLifelongLogsApporach.pdf b/docs/SystemLifelongLogsApporach.pdf new file mode 100644 index 0000000..3795f0d Binary files /dev/null and b/docs/SystemLifelongLogsApporach.pdf differ diff --git a/readme.md b/readme.md index 8f756e4..4985453 100644 --- a/readme.md +++ b/readme.md @@ -6,3 +6,30 @@ There are 3 phases to get this done. First a "model" has to be trained. This requires a lot of data from running and failing systems, to identify issues, causes, and solutions Then an agent capable of using this model and run it locally has to be created (you don't want to ship your logs to the cloud permanently, everything has to be local) Then it needs to improve over time, and keep getting better as more and more problems arise, so it's kind of a ever-running system. + +Installing the client from repository: +-------------------------------------- + +``` +cd client +sudo python setup.py install +``` + +Installing the server from repository: +-------------------------------------- + +``` +curl -sSL https://get.docker.com/ | sh +sudo apt install python-pip +sudo pip install docker-compose +cd server/alpha +sudo docker-compose up +``` + +Creating the debian package: +---------------------------- + +``` +cd client +make builddeb +``` diff --git a/server/alpha/Dockerfile b/server/alpha/Dockerfile new file mode 100644 index 0000000..f48a4af --- /dev/null +++ b/server/alpha/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.5 +MAINTAINER Leonid Logvinov +RUN mkdir /dpcs +WORKDIR /dpcs +ADD . /dpcs +ENV DJANGO_SETTINGS_MODULE 'server_django.prod_settings' +RUN pip install -r requirements/requirements.common +RUN pip install -r requirements/requirements.prod +RUN python manage.py check --deploy +RUN python manage.py makemigrations +RUN python manage.py migrate +RUN python manage.py makemigrations alpha +RUN python manage.py migrate alpha +RUN python manage.py collectstatic -v0 --no-input +RUN echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com', 'pass')" | python manage.py shell + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + nginx \ + gettext-base \ + && rm -rf /var/lib/apt/lists/* + +COPY nginx.conf /etc/nginx/nginx.conf + +EXPOSE 80 + +RUN nginx + +CMD ["gunicorn", "-c", "gunicorn_conf.py", "--chdir", "/dpcs", "server_django.wsgi", "--reload"] \ No newline at end of file diff --git a/server/alpha/README.md b/server/alpha/README.md new file mode 100644 index 0000000..9748a50 --- /dev/null +++ b/server/alpha/README.md @@ -0,0 +1,18 @@ +# Now you can go to : + #1) localhost:8000/crash-reports/ + #2) localhost:8000/crash-groups/ + #3) localhost:8000/solutions/ +# to see lists of crash reports, crash groups, and solutions. +# Alternatively you can go to: + #4) localhost:8000/crash-reports/id/ + #5) localhost:8000/crash-groups/id/ + #6) localhost:8000/solutions/id/ +# where id is the ID of either crash report, crash group or a solution + +# You can also make other HTTP request to the sites above. +# If you want to create a new crash report you need to send HTTP POST request to #1) +# Similary you send HTTP POST request to create new crash-group or solution to #2) or #3) + +# If you want to change a specific crash report you send HTTP PUT request to #4). + +# If you want to delete a specific crash report, crash group or a solution, you send a HTTP DELETE request to either #4), #5), or #6). diff --git a/server/alpha/alpha/__init__.py b/server/alpha/alpha/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/alpha/alpha/admin.py b/server/alpha/alpha/admin.py new file mode 100644 index 0000000..0e482ea --- /dev/null +++ b/server/alpha/alpha/admin.py @@ -0,0 +1,9 @@ +from django.contrib import admin +from .models import CrashGroup, Application, CrashReport, SystemInfo, Solution +# Register your models here. + +admin.site.register(CrashGroup) +admin.site.register(Application) +admin.site.register(CrashReport) +admin.site.register(SystemInfo) +admin.site.register(Solution) \ No newline at end of file diff --git a/server/alpha/alpha/apps.py b/server/alpha/alpha/apps.py new file mode 100644 index 0000000..5cdce23 --- /dev/null +++ b/server/alpha/alpha/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AlphaConfig(AppConfig): + name = 'alpha' diff --git a/server/alpha/alpha/models.py b/server/alpha/alpha/models.py new file mode 100644 index 0000000..63524cf --- /dev/null +++ b/server/alpha/alpha/models.py @@ -0,0 +1,47 @@ +from django.db import models + + + +class CrashGroup(models.Model): + crash_group_id = models.IntegerField(primary_key=True) + def __str__ (self): + return str(self.crash_group_id) + +class Application(models.Model): + #id = models.AutoField(primary_key=True) + name = models.CharField(max_length=255) + version = models.CharField(max_length=255) + def __str__(self): + return "Application: " + self.name + " " + self.version + "\n" + +class SystemInfo(models.Model): + #id = models.AutoField(primary_key=True) + name = models.CharField(max_length=255, default="Ubuntu") + version = models.CharField(max_length=255) + def __str__(self): + return "System information: " + self.name + self.version + "\n" + +class CrashReport(models.Model): + crash_report_id = models.IntegerField(primary_key=True) + crash_group_id = models.ForeignKey(CrashGroup, related_name='crash_group_reports') + stderr_output = models.TextField() + exit_code = models.IntegerField() + systeminfo = models.ForeignKey(SystemInfo) + application = models.ForeignKey(Application) + def __str__ (self): + text = "Crash report: " + text += str(self.crash_report_id) + " " + text += str(self.crash_group_id) + " " + text += str(self.exit_code) + " " + text += self.stderr_output + "\n" + return text + +class Solution(models.Model): + solution_id = models.IntegerField(primary_key=True) + crash_group_id = models.ForeignKey(CrashGroup) + shell_script = models.TextField() + def __str__ (self): + text = "Solution: " + text += str(self.solution_id) + " " + text += str(self.crash_group_id) + " " + text += self.shell_script \ No newline at end of file diff --git a/server/alpha/alpha/serializers.py b/server/alpha/alpha/serializers.py new file mode 100644 index 0000000..6ce8bc2 --- /dev/null +++ b/server/alpha/alpha/serializers.py @@ -0,0 +1,38 @@ +from rest_framework import serializers +from alpha.models import SystemInfo, CrashReport, CrashGroup, Application, Solution + + +class ApplicationSerializer(serializers.ModelSerializer): + class Meta: + model = Application + fields = ('name', 'version') + + +class SystemInfoSerializer(serializers.ModelSerializer): + class Meta: + model = SystemInfo + fields = ('name', 'version') + + +class CrashReportSerializer(serializers.ModelSerializer): + systeminfo = SystemInfoSerializer(read_only=False) + application = ApplicationSerializer(read_only=False) + + class Meta: + model = CrashReport + fields = ('crash_report_id', 'crash_group_id', 'stderr_output', 'exit_code', 'application', 'systeminfo') + + +class CrashGroupSerializer(serializers.ModelSerializer): + crash_group_reports = CrashReportSerializer(many=True, required=False, read_only=True) + class Meta: + model = CrashGroup + fields = ('crash_group_id', 'crash_group_reports') + + def create(self, validated_data): + return CrashGroup.objects.create(**validated_data) + +class SolutionSerializer(serializers.ModelSerializer): + class Meta: + model = Solution + fields = ('solution_id', 'crash_group_id', 'shell_script') \ No newline at end of file diff --git a/server/alpha/alpha/templates/alpha/crash_group_add.html b/server/alpha/alpha/templates/alpha/crash_group_add.html new file mode 100644 index 0000000..947f330 --- /dev/null +++ b/server/alpha/alpha/templates/alpha/crash_group_add.html @@ -0,0 +1,28 @@ + + + + + Title + + +{{ msg }} +
+{% if groups.count > 0 %} + + {% for group in groups %} + + + + {% endfor %} +
{{ group.crash_group_id }}
+{% endif %} + +

Add new crash group

+
+ {% csrf_token %} + + +
+ + + \ No newline at end of file diff --git a/server/alpha/alpha/templates/alpha/crash_report_add.html b/server/alpha/alpha/templates/alpha/crash_report_add.html new file mode 100644 index 0000000..50b6f08 --- /dev/null +++ b/server/alpha/alpha/templates/alpha/crash_report_add.html @@ -0,0 +1,41 @@ + + + + + Title + + + +{{ msg }} +
+{% if reports.count > 0 %} + + {% for report in reports %} + + + + + + + + + {% endfor %} +
{{ report.crash_report_id }}{{ report.crash_group_id }}{{ report.stderr_output }}{{ report.exit_code }}{{ report.application }}{{ report.systeminfo }}
+{% endif %} + +

Add new crash report

+
+ {% csrf_token %} +
+
+
+
+
+
+
+ +
+ + + + \ No newline at end of file diff --git a/server/alpha/alpha/tests.py b/server/alpha/alpha/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/server/alpha/alpha/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/server/alpha/alpha/urls.py b/server/alpha/alpha/urls.py new file mode 100644 index 0000000..c64242f --- /dev/null +++ b/server/alpha/alpha/urls.py @@ -0,0 +1,16 @@ +from django.conf.urls import url +from alpha import views + +urlpatterns = [ + url(r'^crash-reports/$', views.crash_report_list), + url(r'^crash-reports/(?P[0-9]+)/$', views.crash_report_detail), + + url(r'^crash-groups/$', views.crash_group_list), + url(r'^crash-groups/(?P[0-9]+)/$', views.crash_group_detail), + + url(r'^crash-groups/add/$', views.crash_group_add), + url(r'^crash-reports/add/$', views.crash_report_add), + + url(r'^solutions/(?P[0-9]+)/$', views.solution_detail), + url(r'^solutions/$', views.solution_list), +] diff --git a/server/alpha/alpha/views.py b/server/alpha/alpha/views.py new file mode 100644 index 0000000..0594b8f --- /dev/null +++ b/server/alpha/alpha/views.py @@ -0,0 +1,297 @@ +from django.shortcuts import render_to_response +from django.template import RequestContext +from rest_framework import status +from rest_framework.decorators import api_view +from rest_framework.response import Response +from alpha.models import Application, SystemInfo, \ + CrashGroup, CrashReport, Solution +from alpha.serializers import CrashGroupSerializer, \ + CrashReportSerializer, SolutionSerializer + + +# Adds the crash report to the database. Returns crash report's id on success. +# Otherwise returns -1 +def add_crash_report(request): + if request.method == 'POST': + + # assume the data is passed correctly ... + cr_id = int(request.data['crash_report_id']) + cg_id = int(request.data['crash_group_id']) + std_err = request.data['stderr_output'] + ex_code = int(request.data['exit_code']) + ap = request.data['application'] + sysinfo = request.data['systeminfo'] + + try: + CrashReport.objects.get(pk=cr_id) + return -1 + except CrashReport.DoesNotExist: + pass + + try: + cg = CrashGroup.objects.get(crash_group_id=cg_id) + except CrashGroup.DoesNotExist: + cg = CrashGroup(crash_group_id=cg_id) + cg.save() + + try: + app = Application.objects.get(version=ap["version"]) + except Application.DoesNotExist: + app = Application(version=ap["version"]) + app.save() + + try: + system = SystemInfo.objects.get(version=sysinfo["version"]) + except SystemInfo.DoesNotExist: + system = SystemInfo(version=sysinfo["version"]) + system.save() + + new_crash_report = CrashReport( + crash_report_id=cr_id, crash_group_id=cg, + stderr_output=std_err, exit_code=ex_code, application=app, + systeminfo=system + ) + + new_crash_report.save() + return cr_id + return -1 + + +# Adds the solution to the database. Returns solution's id on success. +# Otherwise returns -1 +def add_solution(request): + if request.method == 'POST': + + # assume the data is passed correctly ... + sol_id = int(request.data['solution_id']) + cg_id = int(request.data['crash_group_id']) + shellscript = request.data['shell_script'] + + try: + Solution.objects.get(pk=sol_id) + return -1 + except Solution.DoesNotExist: + pass + + try: + cg = CrashGroup.objects.get(pk=cg_id) + new_sol = Solution( + solution_id=sol_id, crash_group_id=cg, + shell_script=shellscript + ) + new_sol.save() + return sol_id + except CrashGroup.DoesNotExist: + return -1 + + return -1 + + +# list all crash groups or create a new one +@api_view(['GET', 'POST']) +def crash_group_list(request): + if request.method == 'GET': + crash_grp_list = CrashGroup.objects.all() + serializer = CrashGroupSerializer(crash_grp_list, many=True) + return Response(serializer.data) + + elif request.method == 'POST': + serializer = CrashGroupSerializer(data=request.data) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + +# view responsible for listing or deleting a crash group +@api_view(['GET', 'PUT', 'DELETE']) +def crash_group_detail(request, pk): + try: + group = CrashGroup.objects.get(pk=pk) + except CrashGroup.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + if request.method == 'GET': + serializer = CrashGroupSerializer(group) + return Response(serializer.data) + + elif request.method == 'DELETE': + group.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + +# view, responsible for listing all the crash reports and dealing +# with HTTP requests to create a crash report +@api_view(['GET', 'POST']) +def crash_report_list(request): + if request.method == 'GET': + reports = CrashReport.objects.all() + serializer = CrashReportSerializer(reports, many=True) + return Response(serializer.data) + + elif request.method == 'POST': + added = add_crash_report(request) + if added == -1: + return Response( + "Could not add the crash report ..." + "Make sure you passed correct arguments ...", + status=status.HTTP_400_BAD_REQUEST + ) + report = CrashReport.objects.get(pk=added) + serializer = CrashReportSerializer(report) + return Response(serializer.data, status=status.HTTP_201_CREATED) + + +# view responsible for either: listing information on a specific crash report, +# changing a specific crash report or deleting it +@api_view(['GET', 'PUT', 'DELETE']) +def crash_report_detail(request, pk): + try: + report = CrashReport.objects.get(pk=pk) + except CrashReport.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + if request.method == 'GET': + serializer = CrashReportSerializer(report) + return Response(serializer.data) + elif request.method == 'PUT': + serializer = CrashReportSerializer(report, request.data, partial=True) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data) + else: + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + elif request.method == 'DELETE': + report.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + +# OPTIONAL! (quite ugly GUI) +# view for listing or adding a crash group to the database, using basic +# GUI(inputs+html+submit button) +def crash_group_add(request): + groups = CrashGroup.objects.all() + if request.method == 'GET': + return render_to_response( + 'alpha/crash_group_add.html', + {'groups': groups}, + context_instance=RequestContext(request) + ) + if request.method == 'POST': + crash_grp_id = request.POST.get('crash_group_id') + try: + CrashGroup.objects.get(crash_group_id=crash_grp_id) + return render_to_response( + 'alpha/crash_group_add.html', + {'groups': groups, + 'msg': "This crash_group is already in database."}, + context_instance=RequestContext(request) + ) + except CrashGroup.DoesNotExist: + new_crash_grp = CrashGroup(crash_group_id=crash_grp_id) + new_crash_grp.save() + return render_to_response( + 'alpha/crash_group_add.html', + {'groups': groups, 'msg': "Saved new crash_grp"}, + context_instance=RequestContext(request) + ) + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + +# OPTIONAL! (quite ugly GUI) +# view for listing or adding a crash report to the database, using basic +# GUI(inputs+html+submit button) +def crash_report_add(request): + reports = CrashReport.objects.all() + if request.method == 'GET': + return render_to_response( + 'alpha/crash_report_add.html', + {'reports': reports}, + context_instance=RequestContext(request) + ) + elif request.method == 'POST': + cr_id = int(request.POST.get('crash_report_id')) + cg_id = int(request.POST.get('crash_group_id')) + std_err = request.POST.get('stderr_output') + ex_code = int(request.POST.get('exit_code')) + app_name = request.POST.get('application_name') + app_v = request.POST.get('application_version') + sys_ver = request.POST.get('system_version') + + try: + cg = CrashGroup.objects.get(crash_group_id=cg_id) + except CrashGroup.DoesNotExist: + cg = CrashGroup(crash_group_id=cg_id) + cg.save() + try: + app = Application.objects.get(name=app_name, version=app_v) + except Application.DoesNotExist: + app = Application(name=app_name, version=app_v) + app.save() + + try: + system = SystemInfo.objects.get(version=sys_ver) + except SystemInfo.DoesNotExist: + system = SystemInfo(version=sys_ver) + system.save() + + new_crash_report = CrashReport( + crash_report_id=cr_id, crash_group_id=cg, + stderr_output=std_err, exit_code=ex_code, application=app, + systeminfo=system + ) + new_crash_report.save() + + return render_to_response( + 'alpha/crash_report_add.html', + {'reports': reports}, + context_instance=RequestContext(request) + ) + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + +# view for displaying info about a solution, adding a solution or deleting it +@api_view(['GET', 'PUT', 'DELETE']) +def solution_detail(request, pk): + try: + sol = Solution.objects.get(pk=pk) + except Solution.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + if request.method == 'GET': + serializer = SolutionSerializer(sol) + return Response(serializer.data) + elif request.method == 'PUT': + serializer = SolutionSerializer(sol, request.data, partial=True) + if serializer.is_valid(): + serializer.save() + return Response(serializer.data) + else: + return Response( + serializer.errors, + status=status.HTTP_400_BAD_REQUEST + ) + elif request.method == 'DELETE': + sol.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + +# view used for listing all the solutions or adding a new one +@api_view(['POST', 'GET']) +def solution_list(request): + if request.method == 'POST': + new_sol_id = add_solution(request) + if new_sol_id == -1: + return Response( + "Could not add the solution ..." + "Make sure you passed correct arguments ...", + status=status.HTTP_400_BAD_REQUEST + ) + sol = Solution.objects.get(pk=new_sol_id) + serializer = SolutionSerializer(sol) + return Response(serializer.data, status=status.HTTP_201_CREATED) + elif request.method == 'GET': + solutions = Solution.objects.all() + serializer = SolutionSerializer(solutions, many=True) + return Response(serializer.data) diff --git a/server/alpha/docker-compose.yml b/server/alpha/docker-compose.yml new file mode 100644 index 0000000..f6de5af --- /dev/null +++ b/server/alpha/docker-compose.yml @@ -0,0 +1,11 @@ +django: + build: . + ports: + - "8000:8000" + - "80:80" + - "8123:8123" + links: + - postgres + +postgres: + image: postgres:9.4 diff --git a/server/alpha/example_requests b/server/alpha/example_requests new file mode 100644 index 0000000..6bedd4e --- /dev/null +++ b/server/alpha/example_requests @@ -0,0 +1,59 @@ +#### Displaying list with either: crash groups, crash reports or solutions #### +localhost:8000/vd1/crash-groups/ +localhost:8000/vd1/crash-reports/ +localhost:8000/vd1/solutions/ + + +#### Displaying info about a specific: crash groups, crash reports or solutions #### +localhost:8000/vd1/crash-groups/ +localhost:8000/vd1/crash-reports/ +localhost:8000/vd1/solutions/ + + +### Create a new crash group #### +curl -X POST http://localhost:8000/vd1/crash-groups/ -d "crash_group_id=1000" +curl -X POST http://localhost:8000/vd1/crash-groups/ -d "crash_group_id=1001" + + +#### display the created crash groups #### +curl -X GET http://localhost:8000/vd1/crash-groups/ + + +### Create a new crash report using POST method. Take into account that we can't + create a new crash report with an already taken ID #### +curl -v -H "Accept: application/json" -H "Content-type: application/json" -X POST -d ' {"crash_report_id":10001, "crash_group_id":1000, "stderr_output":"Cos sie posulo", "exit_code":-1, "systeminfo":{"name":"Ubuntu","version":"10.10"}, "application":{"name":"Chrome", "version":"10.10"}}' http://localhost:8000/vd1/crash-reports/ + + + +#### Change crash report's information using PUT method - a few examples #### +curl -v -H "Accept: application/json" -H "Content-type: application/json" -X PUT -d '{"exit_code":25, "stderr_output":"Znowu sie cos popsulo. Echhhhh"}' http://localhost:8000/vd1/crash-reports/10001/ + +curl -X PUT -d "stderr_output=Znowu sie cos popsulo. Echhhhh&exit_code=34" http://localhost:8000/vd1/crash-reports/10001/ + +curl -X PUT -d "exit_code=34&stderr_output=Nie dziala" http://localhost:8000/vd1/crash-reports/10001/ + + + +#### Create a new solution that is good for solving a problem with the given crash group id #### +curl -X POST -d "solution_id=1&crash_group_id=1000&shell_script=Better call Saul" http://localhost:8000/vd1/solutions/ + +### Now if we try to add a solution with the same id +curl -X POST -d "solution_id=1&crash_group_id=1000&shell_script=Better call Saul" http://localhost:8000/vd1/solutions/ +#### we get an error: +#### "Could not add the solution ...Make sure you passed correct arguments ..." + + + +#### Get info about a solution with given id #### +curl -X GET http://localhost:8000/vd1/solutions/1/ + + + +#### Assign a crash group to a given solution #### +curl -X PUT -d "crash_group_id=1000" http://localhost:8000/vd1/solutions/1/ + + + +#### Delete a solution with a certain ID #### +curl -X DELETE http://localhost:8000/vd1/solutions/1/ + diff --git a/server/alpha/gunicorn_conf.py b/server/alpha/gunicorn_conf.py new file mode 100644 index 0000000..a95ed88 --- /dev/null +++ b/server/alpha/gunicorn_conf.py @@ -0,0 +1,7 @@ +# Gunicorn configuration file + +bind = '0.0.0.0:8000' + +loglevel = 'info' +errorlog = '-' +accesslog = '-' diff --git a/server/alpha/manage.py b/server/alpha/manage.py new file mode 100755 index 0000000..007b166 --- /dev/null +++ b/server/alpha/manage.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server_django.dev_settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/server/alpha/nginx.conf b/server/alpha/nginx.conf new file mode 100644 index 0000000..a45a960 --- /dev/null +++ b/server/alpha/nginx.conf @@ -0,0 +1,25 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +error_log /var/log/nginx/error.log info; + +http { + server { + listen 80; + access_log /var/log/nginx/access.log; + location /static/ { + root /dpcs/; + autoindex off; + } + location / { + proxy_pass http://localhost:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + } + } +} diff --git a/server/alpha/rebuild_docker.sh b/server/alpha/rebuild_docker.sh new file mode 100755 index 0000000..5e26dd8 --- /dev/null +++ b/server/alpha/rebuild_docker.sh @@ -0,0 +1,3 @@ +docker-compose build +docker-compose up -d +docker-compose ps diff --git a/server/alpha/requirements/requirements.common b/server/alpha/requirements/requirements.common new file mode 100644 index 0000000..0a0eaf6 --- /dev/null +++ b/server/alpha/requirements/requirements.common @@ -0,0 +1,4 @@ +django==1.9.4 +djangorestframework +markdown +django-filter \ No newline at end of file diff --git a/server/alpha/requirements/requirements.dev b/server/alpha/requirements/requirements.dev new file mode 100644 index 0000000..e69de29 diff --git a/server/alpha/requirements/requirements.prod b/server/alpha/requirements/requirements.prod new file mode 100644 index 0000000..283e130 --- /dev/null +++ b/server/alpha/requirements/requirements.prod @@ -0,0 +1 @@ +gunicorn \ No newline at end of file diff --git a/server/alpha/server_django/__init__.py b/server/alpha/server_django/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/alpha/server_django/common_settings.py b/server/alpha/server_django/common_settings.py new file mode 100644 index 0000000..12f6f6c --- /dev/null +++ b/server/alpha/server_django/common_settings.py @@ -0,0 +1,127 @@ +""" +Django settings for server_django project. + +Generated by 'django-admin startproject' using Django 1.9.3. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.9/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '#66f=^4km!&1*3b4j87zjy9kl*yb8%$((72d0tm$8st(6(1njr' + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'alpha', +] + +MIDDLEWARE_CLASSES = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'server_django.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(BASE_DIR, 'templates')] + , + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'server_django.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.9/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.9/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.9/howto/static-files/ + +STATIC_URL = '/static/' + +#REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. +# 'DEFAULT_PERMISSION_CLASSES': [ +# 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly' +# ] +#} diff --git a/server/alpha/server_django/dev_settings.py b/server/alpha/server_django/dev_settings.py new file mode 100644 index 0000000..c67e28e --- /dev/null +++ b/server/alpha/server_django/dev_settings.py @@ -0,0 +1,4 @@ +from .common_settings import * + +DEBUG = True +ALLOWED_HOSTS = ['*'] \ No newline at end of file diff --git a/server/alpha/server_django/prod_settings.py b/server/alpha/server_django/prod_settings.py new file mode 100644 index 0000000..f0e9f79 --- /dev/null +++ b/server/alpha/server_django/prod_settings.py @@ -0,0 +1,11 @@ +from .common_settings import * + +STATIC_ROOT = '/dpcs/static' +DEBUG = False +ALLOWED_HOSTS = ['*'] +SECURE_CONTENT_TYPE_NOSNIFF = True +SECURE_BROWSER_XSS_FILTER = True +SESSION_COOKIE_SECURE = True +CSRF_COOKIE_SECURE = True +CSRF_COOKIE_HTTPONLY = True +X_FRAME_OPTIONS = 'DENY' \ No newline at end of file diff --git a/server/alpha/server_django/urls.py b/server/alpha/server_django/urls.py new file mode 100644 index 0000000..7a871ec --- /dev/null +++ b/server/alpha/server_django/urls.py @@ -0,0 +1,22 @@ +"""server_django URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.9/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import url, include +from django.contrib import admin + +urlpatterns = [ + url(r'^admin/', admin.site.urls), + url(r'^vd1/', include('alpha.urls')) +] diff --git a/server/alpha/server_django/wsgi.py b/server/alpha/server_django/wsgi.py new file mode 100644 index 0000000..830fe95 --- /dev/null +++ b/server/alpha/server_django/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for server_django project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server_django.settings") + +application = get_wsgi_application() diff --git a/server/backup/server_flask/README.md b/server/backup/server_flask/README.md new file mode 100644 index 0000000..3d28124 --- /dev/null +++ b/server/backup/server_flask/README.md @@ -0,0 +1,8 @@ +# Server API implementation +API docs: http://docs.dpcs.apiary.io/#reference/0/api-paths + +Custom python modules: +* Flask +* Flask_RESTful +* jsonschema +* sqlalchemy diff --git a/server/backup/server_flask/__init__.py b/server/backup/server_flask/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/backup/server_flask/api.py b/server/backup/server_flask/api.py new file mode 100755 index 0000000..6943eb0 --- /dev/null +++ b/server/backup/server_flask/api.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Listen to the server requests.""" + +from flask import Flask +from flask_restful import Api + +from resources.helloWorld import HelloWorld +from resources.crashReport import CrashReportById, CrashReportNoId + +app = Flask(__name__) +api = Api(app) + +# api config +app.config['ERROR_404_HELP'] = False + +# api routes +api.add_resource(HelloWorld, '/') +api.add_resource(CrashReportById, '/vd1/crash-reports/') +api.add_resource(CrashReportNoId, '/vd1/crash-reports/') + +if __name__ == '__main__': + app.run(debug=True) diff --git a/server/backup/server_flask/resources/__init__.py b/server/backup/server_flask/resources/__init__.py new file mode 100644 index 0000000..d854322 --- /dev/null +++ b/server/backup/server_flask/resources/__init__.py @@ -0,0 +1 @@ +"""Initialize resources package of dpcs_api.""" diff --git a/server/backup/server_flask/resources/classification.py b/server/backup/server_flask/resources/classification.py new file mode 100644 index 0000000..c02b1c3 --- /dev/null +++ b/server/backup/server_flask/resources/classification.py @@ -0,0 +1,6 @@ +"""Placeholder for classification algorithm.""" + + +def classify(crash_report): + """Return crash_group_id for given crash report.""" + return 1 diff --git a/server/backup/server_flask/resources/crashReport.py b/server/backup/server_flask/resources/crashReport.py new file mode 100755 index 0000000..bc63929 --- /dev/null +++ b/server/backup/server_flask/resources/crashReport.py @@ -0,0 +1,55 @@ +"""Retrieve and modify information about crash report(s).""" +from flask import abort +from flask_restful import Resource, reqparse +# Place holder for postgres communication +from database import database_instance as db +import types +import json + + +class CrashReportById(Resource): + """Retrieve and modify information about given crash report.""" + + def get(self, crash_report_id): + """Retrieve information about given crash report.""" + report = db.getCrashReport(crash_report_id) + if report: + return report + else: + abort(404) + + def put(self, crash_report_id): + """Update information about given crash report.""" + pass + + def delete(self, crash_report_id): + """Delete information about given crash report.""" + pass + + +class CrashReportNoId(Resource): + """Search for and send new crash report.""" + + # get_parser parses incoming search requests + get_parser = reqparse.RequestParser() + get_parser.add_argument('browser', location='args') + get_parser.add_argument('version', location='args') + + # post_parser parses incoming new crash reports + crashReport = types.useSchema('inputCrashReportSchema') + post_parser = reqparse.RequestParser() + post_parser.add_argument('crash_report', location='json', type=crashReport) + + def get(self): + """Search for crash report.""" + args = self.get_parser.parse_args() + print args + print args['version'] + print args['browser'] + pass + + def post(self): + """Post new crash report and return its reference with solution.""" + read_data = self.post_parser.parse_args() + print json.dumps(read_data, indent=5) + return db.insertCrashReport(read_data, commit=True), 201 diff --git a/server/backup/server_flask/resources/database.py b/server/backup/server_flask/resources/database.py new file mode 100644 index 0000000..1b27227 --- /dev/null +++ b/server/backup/server_flask/resources/database.py @@ -0,0 +1,157 @@ +"""Connect to the stored data.""" +from sqlalchemy import create_engine +from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import sessionmaker +from models import Applications, CrashReports, CrashGroups, SystemInfo,\ + Solutions +from models import Base +from classification import classify + + +class database(): + """Database crash_fixer.""" + + def __init__(self): + """Initialize and return on stdout if success.""" + self.engine = create_engine( + # database adress + "postgresql://postgres:postgres@localhost:5433/crash_fixer", + echo=True + ) + Base.metadata.create_all(self.engine) + self.Session = sessionmaker(bind=self.engine) + self.session = self.Session() + print " * DATABASE: Conncetion established" + + def __del__(self): + """Delete object and connection.""" + self.session.commit() + self.session.close() + print " * DATABASE: Connection closed" + + def instertApplication(self, app, commit=False): + """Send new application info to database.""" + app_sql = Applications(name=app['name'], version=app['version']) + self.session.add(app_sql) + self.session.commit() + return app + + def insertSystemInfo(self, info, commit=False): + """Send new system info to database.""" + info_sql = SystemInfo(version=info['version']) + self.session.add(info_sql) + self.session.commit() + info['id'] = info_sql.id + return info_sql + + def insertSolution(self, solution, commit=False): + """Send new solution to the database.""" + solution_sql = Solutions(details=solution['details']) + self.session.add(solution_sql) + if commit: + self.session.commit() + return solution_sql + + def insertCrashGroup(self, crash_group, commit=False): + """Send new crash group to database.""" + self.insertSolution(crash_group['solution']) + crash_group_sql = CrashGroups( + solution_id=crash_group['solution']['id'] + ) + self.session.add(crash_group_sql) + if commit: + self.session.commit() + crash_group['id'] = crash_group_sql.id + return crash_group_sql + + def insertCrashReport(self, crash_report, commit=False, todict=True): + """"Classify crash_report, insert to the database, return updated.""" + crash_report = crash_report['crash_report'] + + # Find crash group + crash_group_id = classify(crash_report) + try: + crash_group = self.session.query(CrashGroups).\ + filter(CrashGroups.id == crash_group_id).one() + except: + print "Clusterization error" + exit(1) + + # Find application data + app_dict = crash_report['application'] + try: + application = self.session.query(Applications).\ + filter(Applications.name == app_dict['name']).\ + filter(Applications.version == app_dict['version']).\ + one() + except NoResultFound: + application = self.instertApplication(app_dict) + + # Find system_info data + sys_info_dict = crash_report['system_info'] + try: + system_info = self.session.query(SystemInfo).\ + filter(SystemInfo.version == sys_info_dict['version']).\ + one() + except NoResultFound: + system_info = self.insertSystemInfo(sys_info_dict) + + result = CrashReports( + exit_code=crash_report['exit_code'], + stderr_output=crash_report['stderr_output'], + crash_group=crash_group, + application=application, + system_info=system_info + ) + self.session.add(result) + if commit: + self.session.commit() + if todict: + return { + 'crash_report_ack': { + 'crash_report_id': result.id, + 'crash_report_url': 'vd1/crash_reports/' + str(result.id), + 'crash_group_id': result.crash_group.id, + 'crash_group_url': ('vd1/crash_groups/' + + str(result.crash_group.id)), + 'solution': { + "solution_id": result.crash_group.solution.id, + "solution_url": ('vd1/solutions/' + + str(result.crash_group.solution.id)), + "shell_script": result.crash_group.solution.details + } + } + } + else: + return result + + def getCrashReport(self, crash_report_id, todict=True): + """Return object representing interesting crash report.""" + result = self.session.query(CrashReports).\ + filter(CrashReports.id == crash_report_id).\ + one() + + if todict: + return { + 'crash_report': { + 'crash_report_id': result.id, + 'crash_report_url': 'vd1/crash_reports/' + str(result.id), + 'crash_group_id': result.crash_group.id, + 'crash_group_url': ('vd1/crash_groups/' + + str(result.crash_group.id)), + 'exit_code': result.exit_code, + 'stderr_output': result.stderr_output, + 'application': { + 'name': result.application.name, + 'version': result.application.version, + }, + 'system_info': { + 'version': result.system_info.version + } + + } + } + else: + return result + +database_instance = database() diff --git a/server/backup/server_flask/resources/helloWorld.py b/server/backup/server_flask/resources/helloWorld.py new file mode 100755 index 0000000..2e52572 --- /dev/null +++ b/server/backup/server_flask/resources/helloWorld.py @@ -0,0 +1,10 @@ +"""Test wether server is working.""" +from flask_restful import Resource + + +class HelloWorld(Resource): + """Test wether server is working by return of 'hello world'.""" + + def get(self): + """Return hello world in JSON.""" + return {'hello': 'world'} diff --git a/server/backup/server_flask/resources/models.py b/server/backup/server_flask/resources/models.py new file mode 100644 index 0000000..f5dabaf --- /dev/null +++ b/server/backup/server_flask/resources/models.py @@ -0,0 +1,62 @@ +"""Models representation for SQLAlchemy.""" +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, Text, ForeignKey +from sqlalchemy.orm import relationship +Base = declarative_base() + + +class Applications(Base): + """Representation of applications table.""" + + __tablename__ = 'applications' + id = Column(Integer, primary_key=True) + name = Column(String(250)) + version = Column(String(250)) + + +class SystemInfo(Base): + """Representation of system_info table.""" + + __tablename__ = 'system_info' + id = Column(Integer, primary_key=True) + version = Column(String(250)) + + +class Solutions(Base): + """Representation of solution table.""" + + __tablename__ = 'solutions' + id = Column(Integer, primary_key=True) + details = Column(Text()) + connected_groups = relationship("CrashGroups", back_populates="solution") + + +class CrashGroups(Base): + """Representation of crash group table.""" + + __tablename__ = 'crash_groups' + id = Column(Integer, primary_key=True) + solution_id = Column(Integer, ForeignKey('solutions.id')) + solution = relationship("Solutions", back_populates="connected_groups") + connected_reports = relationship("CrashReports", + back_populates="crash_group") + + +class CrashReports(Base): + """Representation of crash reports table.""" + + __tablename__ = 'crash_reports' + id = Column(Integer, primary_key=True) + exit_code = Column(Integer) + stderr_output = Column(String) + + # Foreign keys + crash_group_id = Column(Integer, ForeignKey('crash_groups.id')) + crash_group = relationship("CrashGroups", + back_populates="connected_reports") + + application_id = Column(Integer, ForeignKey('applications.id')) + application = relationship("Applications") + + system_info_id = Column(Integer, ForeignKey('system_info.id')) + system_info = relationship("SystemInfo") diff --git a/server/backup/server_flask/resources/types.py b/server/backup/server_flask/resources/types.py new file mode 100644 index 0000000..798f70c --- /dev/null +++ b/server/backup/server_flask/resources/types.py @@ -0,0 +1,47 @@ +"""Types for type validation during flask_restful argument parsing.""" +from jsonschema import validate + +schemas = { + 'applicationSchema': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'} + } + }, + + 'systemInfoSchema': { + 'type': 'object', + 'properties': { + 'version': {'type': 'string'} + } + }, + + 'inputCrashReportSchema': { + "type": "object", + 'properties': { + 'application': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'} + } + }, + "system_info": { + 'type': 'object', + 'properties': { + 'version': {'type': 'string'} + } + }, + } + } +} + + +def useSchema(schema): + """Universal type checker generator.""" + def temp(value, name): + """Function returned as type checker.""" + validate(value, schemas[schema]) + return value + return temp diff --git a/server/clustering/clustering.py b/server/clustering/clustering.py index 3e3c8d3..512b1b1 100644 --- a/server/clustering/clustering.py +++ b/server/clustering/clustering.py @@ -1,14 +1,15 @@ import json import sys import difflib -import re -def similarity(a,b): + +def similarity(a, b): astr = a['crash_report']['stderr_output'] bstr = b['crash_report']['stderr_output'] - seq=difflib.SequenceMatcher(a=astr.lower(), b=bstr.lower()) + seq = difflib.SequenceMatcher(a=astr.lower(), b=bstr.lower()) return seq.ratio() + def cluster(json_obj): groups = [0] * (len(json_obj) + 1) group_counter = 0 @@ -22,10 +23,10 @@ def cluster(json_obj): for obj2 in json_obj: id2 = obj2['crash_report']['crash_report_id'] - + if (groups[id2] != 0): continue - + if (similarity(obj, obj2) > 0.8): groups[id2] = group_counter @@ -38,6 +39,6 @@ def cluster(json_obj): if __name__ == '__main__': json_string = unicode(sys.stdin.read(), errors='ignore') - json_obj = json.loads(json_string) + json_obj = json.loads(json_string) json_obj = cluster(json_obj) print(json_obj) diff --git a/server/main/readme.md b/server/main/readme.md deleted file mode 100644 index 9c558e3..0000000 --- a/server/main/readme.md +++ /dev/null @@ -1 +0,0 @@ -. diff --git a/systemcheck/readme.md b/systemcheck/readme.md deleted file mode 100644 index 72d8079..0000000 --- a/systemcheck/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -Small Python2 script that returns system data (kernel, architecture, -installed packages) in JSON.