From 7d50e371afa95eaaacc0a2897da6bee8a0b8fc4a Mon Sep 17 00:00:00 2001 From: Kostas Georgiou Date: Thu, 7 May 2020 19:44:47 +0300 Subject: [PATCH] Initial commit --- .circleci/config.yml | 15 + .gitignore | 133 ++++ LICENSE | 674 ++++++++++++++++++ Makefile | 94 +++ Procfile | 4 + README.md | 307 ++++++++ TODO.md | 21 + cloudstore/__init__.py | 0 cloudstore/abstract_cloudstore.py | 72 ++ cloudstore/dropbox_cloudstore.py | 105 +++ configuration/__init__.py | 0 configuration/configuration.py | 162 +++++ configuration/yml_schema.json | 139 ++++ confs/template_conf.yml | 18 + data/sample_data.txt | 1 + datastore/__init__.py | 0 datastore/abstract_datastore.py | 58 ++ datastore/mysql_datastore.py | 192 +++++ email_app/__init__.py | 0 email_app/abstract_email_app.py | 39 + email_app/gmail_email_app.py | 85 +++ logs/out.log | 1 + main.py | 148 ++++ requirements.txt | 6 + setup.py | 44 ++ tests/test_configuration.py | 109 +++ .../actual_output_to_yaml.yml | 13 + .../minimal_conf_correct.yml | 7 + .../test_configuration/minimal_conf_wrong.yml | 7 + .../minimal_yml_schema.json | 44 ++ .../test_configuration/template_conf.yml | 13 + .../test_dropbox_cloudstore/template_conf.yml | 13 + .../test_gmail_email_app/sample_data.txt | 1 + .../test_gmail_email_app/template_conf.yml | 18 + .../test_mysql_datastore/template_conf.yml | 13 + tests/test_dropbox_cloudstore.py | 111 +++ tests/test_gmail_email_app.py | 143 ++++ tests/test_mysql_datastore.py | 120 ++++ 38 files changed, 2930 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 Procfile create mode 100644 README.md create mode 100644 TODO.md create mode 100644 cloudstore/__init__.py create mode 100644 cloudstore/abstract_cloudstore.py create mode 100644 cloudstore/dropbox_cloudstore.py create mode 100644 configuration/__init__.py create mode 100644 configuration/configuration.py create mode 100644 configuration/yml_schema.json create mode 100644 confs/template_conf.yml create mode 100644 data/sample_data.txt create mode 100644 datastore/__init__.py create mode 100644 datastore/abstract_datastore.py create mode 100644 datastore/mysql_datastore.py create mode 100644 email_app/__init__.py create mode 100644 email_app/abstract_email_app.py create mode 100644 email_app/gmail_email_app.py create mode 100644 logs/out.log create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/test_configuration.py create mode 100644 tests/test_data/test_configuration/actual_output_to_yaml.yml create mode 100644 tests/test_data/test_configuration/minimal_conf_correct.yml create mode 100644 tests/test_data/test_configuration/minimal_conf_wrong.yml create mode 100644 tests/test_data/test_configuration/minimal_yml_schema.json create mode 100644 tests/test_data/test_configuration/template_conf.yml create mode 100644 tests/test_data/test_dropbox_cloudstore/template_conf.yml create mode 100644 tests/test_data/test_gmail_email_app/sample_data.txt create mode 100644 tests/test_data/test_gmail_email_app/template_conf.yml create mode 100644 tests/test_data/test_mysql_datastore/template_conf.yml create mode 100644 tests/test_dropbox_cloudstore.py create mode 100644 tests/test_gmail_email_app.py create mode 100644 tests/test_mysql_datastore.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..4d13639 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,15 @@ +version: 2 # use CircleCI 2.0 +jobs: # A basic unit of work in a run + build: # runs not using Workflows must have a `build` job as entry point + # directory where steps are run + working_directory: ~/template_python_project + docker: # run the steps with Docker + # CircleCI Python images available at: https://hub.docker.com/r/circleci/python/ + - image: circleci/python:3.6.9 + steps: # steps that comprise the `build` job + - checkout # check out source code to working directory + - run: make clean + - run: make create_venv + - run: make requirements + - run: make run_tests + - run: make setup diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb99f0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,133 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# PyCharm +/.idea +/tests/test_data/test_dropbox_cloudstore/*.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..16eb22c --- /dev/null +++ b/Makefile @@ -0,0 +1,94 @@ +# Makefile for the template_python_project + +SHELL=/bin/bash +PYTHON_VERSION=3.6 +PYTHON_BIN=venv/bin/ +TESTS_FOLDER=tests +#-------------------------------------------- +ifeq ($(server),prod) + AN_ENVIRONMENT_SPECIFIC_VARIABLE='production' + SETUP_FLAG='' + DEBUG=False +else ifeq ($(server),dev) + AN_ENVIRONMENT_SPECIFIC_VARIABLE='development' + SETUP_FLAG='' + DEBUG=True +else ifeq ($(server),local) + AN_ENVIRONMENT_SPECIFIC_VARIABLE='local' + SETUP_FLAG='--local' + DEBUG=True +else + AN_ENVIRONMENT_SPECIFIC_VARIABLE='production' + SETUP_FLAG= + DEBUG=True +endif +#-------------------------------------------- + + +all: + $(MAKE) help +help: + @echo + @echo "-----------------------------------------------------------------------------------------------------------" + @echo " DISPLAYING HELP " + @echo "-----------------------------------------------------------------------------------------------------------" + @echo "make delete_venv" + @echo " Delete the current venv" + @echo "make create_venv" + @echo " Create a new venv for the specified python version" + @echo "make requirements" + @echo " Upgrade pip and install the requirements" + @echo "make run_tests" + @echo " Run all the tests from the specified folder" + @echo "make setup" + @echo " Call setup.py install" + @echo "make clean_pyc" + @echo " Clean all the pyc files" + @echo "make clean_build" + @echo " Clean all the build folders" + @echo "make clean" + @echo " Call delete_venv clean_pyc clean_build" + @echo "make install" + @echo " Call clean create_venv requirements run_tests setup" + @echo "make help" + @echo " Display this message" + @echo "-----------------------------------------------------------------------------------------------------------" +install: + $(MAKE) clean + $(MAKE) create_venv + $(MAKE) requirements + $(MAKE) run_tests + $(MAKE) setup +clean: + $(MAKE) delete_venv + $(MAKE) clean_pyc + $(MAKE) clean_build +delete_venv: + @echo "Deleting venv.." + rm -rf venv +create_venv: + @echo "Creating venv.." + python$(PYTHON_VERSION) -m venv ./venv +requirements: + @echo "Upgrading pip.." + $(PYTHON_BIN)pip install --upgrade pip wheel setuptools + @echo "Installing requirements.." + $(PYTHON_BIN)pip install -r requirements.txt +run_tests: + source $(PYTHON_BIN)activate && \ + export PYTHONPATH=$(PWD) && \ + cd tests && python -m unittest +setup: + $(PYTHON_BIN)python setup.py install $(SETUP_FLAG) +clean_pyc: + @echo "Cleaning pyc files.." + find . -name '*.pyc' -delete + find . -name '*.pyo' -delete + find . -name '*~' -delete +clean_build: + @echo "Cleaning build directories.." + rm --force --recursive build/ + rm --force --recursive dist/ + rm --force --recursive *.egg-info + +.PHONY: delete_venv create_venv requirements run_tests setup clean_pyc clean_build clean help \ No newline at end of file diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..c889269 --- /dev/null +++ b/Procfile @@ -0,0 +1,4 @@ +make_help: make help +make_tests: make run_tests +main_help: python main.py --help +main: python main.py -m run_mode_1 -c confs/template_conf.yml -l logs/output.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..6074d99 --- /dev/null +++ b/README.md @@ -0,0 +1,307 @@ +# Template Python Project +[![CircleCI](https://circleci.com/gh/drkostas/template_python_project/tree/master.svg?style=svg)](https://circleci.com/gh/drkostas/template_python_project/tree/master) +[![GitHub license](https://img.shields.io/badge/license-GNU-blue.svg)](https://raw.githubusercontent.com/drkostas/template_python_project/master/LICENSE) + +## Table of Contents ++ [About](#about) ++ [Getting Started](#getting_started) + + [Prerequisites](#prerequisites) + + [Environment Variables](#env_variables) ++ [Installing, Testing, Building](#installing) + + [Available Make Commands](#check_make_commamnds) + + [Clean Previous Builds](#clean_previous) + + [Venv and Requirements](#venv_requirements) + + [Run the tests](#tests) + + [Build Locally](#build_locally) ++ [Running locally](#run_locally) + + [Configuration](#configuration) + + [Execution Options](#execution_options) ++ [Deployment](#deployment) ++ [Continuous Ιntegration](#ci) ++ [Todo](#todo) ++ [Built With](#built_with) ++ [License](#license) ++ [Acknowledgments](#acknowledgments) + +## About +This is a template repository for python projects. + +This README serves as a template too. Feel free to modify it until it describes your project. + +## Getting Started + + +These instructions will get you a copy of the project up and running on your local machine for development +and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites + +You need to have a machine with Python > 3.6 and any Bash based shell (e.g. zsh) installed. + + +``` +$ python3.6 -V +Python 3.6.9 + +echo $SHELL +/usr/bin/zsh +``` + +You will also need to setup the following: +- Gmail: An application-specific password for your Google account. +[Reference 1](https://support.google.com/mail/?p=InvalidSecondFactor), +[Reference 2](https://security.google.com/settings/security/apppasswords) +- Dropbox: An Api key for your Dropbox account. +[Reference 1](http://99rabbits.com/get-dropbox-access-token/), +[Reference 2](https://dropbox.tech/developers/generate-an-access-token-for-your-own-account) +- MySql: If you haven't any, you can create a free one on Amazon RDS. +[Reference 1](https://aws.amazon.com/rds/free/), +[Reference 2](https://bigdataenthusiast.wordpress.com/2016/03/05/aws-rds-instance-setup-oracle-db-on-cloud-free-tier/) + + +### Set the required environment variables + +In order to run the [main.py](main.py) or the tests you will need to set the following +environmental variables in your system: + +```bash +$ export DROPBOX_API_KEY= +$ export MYSQL_HOST= +$ export MYSQL_USERNAME= +$ export MYSQL_PASSWORD= +$ export MYSQL_DB_NAME= +$ export EMAIL_ADDRESS= +$ export GMAIL_API_KEY= +``` + +## Installing, Testing, Building + +All the installation steps are being handled by the [Makefile](Makefile). + +If you don't want to go through the setup steps and finish the installation and run the tests, +execute the following command: + +```bash +$ make install server=local +``` + +If you executed the previous command, you can skip through to the [Running locally](#run_locally) section. + +### Check the available make commands + +```bash +$ make help + +----------------------------------------------------------------------------------------------------------- + DISPLAYING HELP +----------------------------------------------------------------------------------------------------------- +make delete_venv + Delete the current venv +make create_venv + Create a new venv for the specified python version +make requirements + Upgrade pip and install the requirements +make run_tests + Run all the tests from the specified folder +make setup + Call setup.py install +make clean_pyc + Clean all the pyc files +make clean_build + Clean all the build folders +make clean + Call delete_venv clean_pyc clean_build +make install + Call clean create_venv requirements run_tests setup +make help + Display this message +----------------------------------------------------------------------------------------------------------- +``` + +### Clean any previous builds + +```bash +$ make clean server=local +make delete_venv +make[1]: Entering directory '/home/drkostas/Projects/template_python_project' +Deleting venv.. +rm -rf venv +make[1]: Leaving directory '/home/drkostas/Projects/template_python_project' +make clean_pyc +make[1]: Entering directory '/home/drkostas/Projects/template_python_project' +Cleaning pyc files.. +find . -name '*.pyc' -delete +find . -name '*.pyo' -delete +find . -name '*~' -delete +make[1]: Leaving directory '/home/drkostas/Projects/template_python_project' +make clean_build +make[1]: Entering directory '/home/drkostas/Projects/template_python_project' +Cleaning build directories.. +rm --force --recursive build/ +rm --force --recursive dist/ +rm --force --recursive *.egg-info +make[1]: Leaving directory '/home/drkostas/Projects/template_python_project' + +``` + +### Create a new venv and install the requirements + +```bash +$ make create_venv server=local +Creating venv.. +python3.6 -m venv ./venv + +$ make requirements server=local +Upgrading pip.. +venv/bin/pip install --upgrade pip wheel setuptools +Collecting pip +................. +``` + + + +### Run the tests + +The tests are located in the `tests` folder. To run all of them, execute the following command: + +```bash +$ make run_tests server=local +source venv/bin/activate && \ +................. +``` + +### Build the project locally + +To build the project locally using the setup.py command, execute the following command: + +```bash +$ make setup server=local +venv/bin/python setup.py install '--local' +running install +................. +``` + +## Running the code locally + +In order to run the code now, you will only need to change the yml file if you need to +and run either the main or the created console script. + +### Modifying the Configuration + +There is an already configured yml file under [confs/template_conf.yml](confs/template_conf.yml) with the following structure: + +```yaml +tag: production +cloudstore: + config: + api_key: !ENV ${DROPBOX_API_KEY} + type: dropbox +datastore: + config: + hostname: !ENV ${MYSQL_HOST} + username: !ENV ${MYSQL_USERNAME} + password: !ENV ${MYSQL_PASSWORD} + db_name: !ENV ${MYSQL_DB_NAME} + port: 3306 + type: mysql +email_app: + config: + email_address: !ENV ${EMAIL_ADDRESS} + api_key: !ENV ${GMAIL_API_KEY} + type: gmail +``` + +The `!ENV` flag indicates that a environmental value follows. +You can change the values/environmental var names as you wish. +If a yaml variable name is changed/added/deleted, the corresponding changes should be reflected +on the [Configuration class](configuration/configuration.py) and the [yml_schema.json](configuration/yml_schema.json) too. + +### Execution Options + +First, make sure you are in the created virtual environment: + +```bash +$ source venv/bin/activate +(venv) +OneDrive/Projects/template_python_project dev + +$ which python +/home/drkostas/Projects/template_python_project/venv/bin/python +(venv) +``` + +Now, in order to run the code you can either call the `main.py` directly, or the `template_python_project` console script. + +```bash +$ python main.py --help +usage: main.py -m {run_mode_1,run_mode_2,run_mode_3} -c CONFIG_FILE [-l LOG] + [-d] [-h] + +A template for python projects. + +required arguments: + -m {run_mode_1,run_mode_2,run_mode_3}, --run-mode {run_mode_1,run_mode_2,run_mode_3} + Description of the run modes + -c CONFIG_FILE, --config-file CONFIG_FILE + The configuration yml file + -l LOG, --log LOG Name of the output log file + +optional arguments: + -d, --debug enables the debug log messages + +# Or + +$ template_python_project --help +usage: template_python_project -m {run_mode_1,run_mode_2,run_mode_3} -c + CONFIG_FILE [-l LOG] [-d] [-h] + +A template for python projects. + +required arguments: + -m {run_mode_1,run_mode_2,run_mode_3}, --run-mode {run_mode_1,run_mode_2,run_mode_3} + Description of the run modes + -c CONFIG_FILE, --config-file CONFIG_FILE + The configuration yml file + -l LOG, --log LOG Name of the output log file + +optional arguments: + -d, --debug enables the debug log messages + -h, --help Show this help message and exit +``` + +## Deployment + +The deployment is being done to Heroku. For more information +you can check the [setup guide](https://devcenter.heroku.com/articles/getting-started-with-python). + +Make sure you check the defined [Procfile](Procfile) ([reference](https://devcenter.heroku.com/articles/getting-started-with-python#define-a-procfile)) +and that you set the [above-mentioned environmental variables](#env_variables) ([reference](https://devcenter.heroku.com/articles/config-vars)). + +## Continuous Integration + +For the continuous integration, the CircleCI service is being used. +For more information you can check the [setup guide](https://circleci.com/docs/2.0/language-python/). + +Again, you should set the [above-mentioned environmental variables](#env_variables) ([reference](https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-context)) +and for any modifications, edit the [circleci config](/.circleci/config.yml). + +## TODO + +Read the [TODO](TODO.md) to see the current task list. + +## Built With + +* [Dropbox Python API](https://www.dropbox.com/developers/documentation/python) - Used for the Cloudstore Class +* [Gmail Sender](https://github.com/paulc/gmail-sender) - Used for the EmailApp Class +* [Heroku](https://www.heroku.com) - The deployment environment +* [CircleCI](https://www.circleci.com/) - Continuous Integration service + + +## License + +This project is licensed under the GNU License - see the [LICENSE](LICENSE) file for details. + +## Acknowledgments + +* Thanks το PurpleBooth for the [README template](https://gist.github.com/PurpleBooth/109311bb0361f32d87a2) + diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..193a262 --- /dev/null +++ b/TODO.md @@ -0,0 +1,21 @@ +# TODO +- [X] Argument Parser +- [X] Logging +- [X] Proper Configuration Class that handles env variables inside the yml +- [X] Create Json schema for validating the yml configuration +- [X] Dropbox/Cloudstore Class +- [X] MySQL/Datastore Class +- [X] Create README template +- [X] Generate requirements.txt +- [X] UnitTests for the current classes +- [X] Sample setup file +- [X] Makefile for installation and build +- [X] Continuous integration +- [X] Heroku Procfile +- [X] Modify Readme to match the instruction for this project +- [X] Gmail Class +- [X] Support multiple occurrences in config +- [ ] Kafka Class +- [ ] MongoDB/Datastore Class +- [ ] Amazon S3/Cloudstore CLass +- [ ] Frontend \ No newline at end of file diff --git a/cloudstore/__init__.py b/cloudstore/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudstore/abstract_cloudstore.py b/cloudstore/abstract_cloudstore.py new file mode 100644 index 0000000..d626230 --- /dev/null +++ b/cloudstore/abstract_cloudstore.py @@ -0,0 +1,72 @@ +from abc import ABC, abstractmethod + + +class AbstractCloudstore(ABC): + __slots__ = ('_handler',) + + @abstractmethod + def __init__(self, *args, **kwargs) -> None: + """ + Tha basic constructor. Creates a new instance of Cloudstore using the specified credentials + """ + + pass + + @staticmethod + @abstractmethod + def get_handler(*args, **kwargs): + """ + Returns a Cloudstore handler. + + :param args: + :param kwargs: + :return: + """ + + pass + + @abstractmethod + def upload_file(self, *args, **kwargs): + """ + Uploads a file to the Cloudstore + + :param args: + :param kwargs: + :return: + """ + + pass + + @abstractmethod + def download_file(self, *args, **kwargs): + """ + Downloads a file from the Cloudstore + + :param args: + :param kwargs: + :return: + """ + + pass + + @abstractmethod + def delete_file(self, *args, **kwargs): + """ + Deletes a file from the Cloudstore + + :param args: + :param kwargs: + :return: + """ + + pass + + @abstractmethod + def ls(self, *args, **kwargs): + """ + List the files and folders in the Cloudstore + :param args: + :param kwargs: + :return: + """ + pass diff --git a/cloudstore/dropbox_cloudstore.py b/cloudstore/dropbox_cloudstore.py new file mode 100644 index 0000000..0453e8a --- /dev/null +++ b/cloudstore/dropbox_cloudstore.py @@ -0,0 +1,105 @@ +from typing import Dict, Union +import logging +from dropbox import Dropbox, files, exceptions + +from .abstract_cloudstore import AbstractCloudstore + +logger = logging.getLogger('DropboxCloudstore') + + +class DropboxCloudstore(AbstractCloudstore): + __slots__ = '_handler' + + _handler: Dropbox + + def __init__(self, config: Dict) -> None: + """ + The basic constructor. Creates a new instance of Cloudstore using the specified credentials + + :param config: + """ + + self._handler = self.get_handler(api_key=config['api_key']) + super().__init__() + + @staticmethod + def get_handler(api_key: str) -> Dropbox: + """ + Returns a Cloudstore handler. + + :param api_key: + :return: + """ + + dbx = Dropbox(api_key) + return dbx + + def upload_file(self, file_bytes: bytes, upload_path: str, write_mode: str = 'overwrite') -> None: + """ + Uploads a file to the Cloudstore + + :param file_bytes: + :param upload_path: + :param write_mode: + :return: + """ + + # TODO: Add option to support FileStream, StringIO and FilePath + try: + logger.debug("Uploading file to path: %s" % upload_path) + self._handler.files_upload(f=file_bytes, path=upload_path, mode=files.WriteMode(write_mode)) + except exceptions.ApiError as err: + logger.error('API error: %s' % err) + + def download_file(self, frompath: str, tofile: str = None) -> Union[bytes, None]: + """ + Downloads a file from the Cloudstore + + :param frompath: + :param tofile: + :return: + """ + + try: + if tofile is not None: + logger.debug("Downloading file from path: %s to path %s" % (frompath, tofile)) + self._handler.files_download_to_file(download_path=tofile, path=frompath) + else: + logger.debug("Downloading file from path: %s to variable" % frompath) + md, res = self._handler.files_download(path=frompath) + data = res.content # The bytes of the file + return data + except exceptions.HttpError as err: + logger.error('HTTP error %s' % err) + return None + + def delete_file(self, file_path: str) -> None: + """ + Deletes a file from the Cloudstore + + :param file_path: + :return: + """ + + try: + logger.debug("Deleting file from path: %s" % file_path) + self._handler.files_delete_v2(path=file_path) + except exceptions.ApiError as err: + logger.error('API error %s' % err) + + def ls(self, path: str = '') -> Dict: + """ + List the files and folders in the Cloudstore + + :param path: + :return: + """ + try: + files_list = self._handler.files_list_folder(path=path) + files_dict = {} + for entry in files_list.entries: + files_dict[entry.name] = entry + return files_dict + except exceptions.ApiError as err: + logger.error('Folder listing failed for %s -- assumed empty: %s' % (path, err)) + return {} diff --git a/configuration/__init__.py b/configuration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/configuration/configuration.py b/configuration/configuration.py new file mode 100644 index 0000000..1f057b4 --- /dev/null +++ b/configuration/configuration.py @@ -0,0 +1,162 @@ +import os +import logging +from typing import Dict, List, Tuple, Union +import json +import _io +from io import StringIO, TextIOWrapper +import re +import yaml +from jsonschema import validate as validate_json_schema + +logger = logging.getLogger('Configuration') + + +class Configuration: + __slots__ = ('config', 'config_path', 'datastore', 'cloudstore', 'email_app', 'tag') + + config: Dict + config_path: str + datastore: Dict + cloudstore: Dict + email_app: Dict + tag: str + config_attributes: List = [] + env_variable_tag: str = '!ENV' + env_variable_pattern: str = r'.*?\${(\w+)}.*?' # ${var} + + def __init__(self, config_src: Union[TextIOWrapper, StringIO, str], config_schema_path: str = 'yml_schema.json'): + """ + The basic constructor. Creates a new instance of the Configuration class. + + :param config_src: + :param config_schema_path: + """ + + # Load the predefined schema of the configuration + configuration_schema = self.load_configuration_schema(config_schema_path=config_schema_path) + # Load the configuration + self.config, self.config_path = self.load_yml(config_src=config_src, + env_tag=self.env_variable_tag, + env_pattern=self.env_variable_pattern) + logger.debug("Loaded config: %s" % self.config) + # Validate the config + validate_json_schema(self.config, configuration_schema) + # Set the config properties as instance attributes + self.tag = self.config['tag'] + all_config_attributes = ('datastore', 'cloudstore', 'email_app') + for config_attribute in all_config_attributes: + if config_attribute in self.config.keys(): + setattr(self, config_attribute, self.config[config_attribute]) + self.config_attributes.append(config_attribute) + else: + setattr(self, config_attribute, None) + + @staticmethod + def load_configuration_schema(config_schema_path: str) -> Dict: + with open('/'.join([os.path.dirname(os.path.realpath(__file__)), config_schema_path])) as f: + configuration_schema = json.load(f) + return configuration_schema + + @staticmethod + def load_yml(config_src: Union[TextIOWrapper, StringIO, str], env_tag: str, env_pattern: str) -> Tuple[Dict, str]: + pattern = re.compile(env_pattern) + loader = yaml.SafeLoader + loader.add_implicit_resolver(env_tag, pattern, None) + + def constructor_env_variables(loader, node): + """ + Extracts the environment variable from the node's value + :param yaml.Loader loader: the yaml loader + :param node: the current node in the yaml + :return: the parsed string that contains the value of the environment + variable + """ + value = loader.construct_scalar(node) + match = pattern.findall(value) # to find all env variables in line + if match: + full_value = value + for g in match: + full_value = full_value.replace( + f'${{{g}}}', os.environ.get(g, g) + ) + return full_value + return value + + loader.add_constructor(env_tag, constructor_env_variables) + + if isinstance(config_src, TextIOWrapper): + logging.debug("Loading yaml from TextIOWrapper") + config = yaml.load(config_src, Loader=loader) + config_path = config_src.name + elif isinstance(config_src, StringIO): + logging.debug("Loading yaml from StringIO") + config = yaml.load(config_src, Loader=loader) + config_path = "StringIO" + elif isinstance(config_src, str): + logging.debug("Loading yaml from path") + with open(config_src) as f: + config = yaml.load(f, Loader=loader) + config_path = config_src + else: + raise TypeError('Config file must be TextIOWrapper or path to a file') + return config, config_path + + def get_datastores(self) -> List: + if 'datastore' in self.config_attributes: + return [sub_config['config'] for sub_config in self.datastore] + else: + raise ConfigurationError('Config property datastore not set!') + + def get_cloudstores(self) -> List: + if 'cloudstore' in self.config_attributes: + return [sub_config['config'] for sub_config in self.cloudstore] + else: + raise ConfigurationError('Config property cloudstore not set!') + + def get_email_apps(self) -> List: + if 'email_app' in self.config_attributes: + return [sub_config['config'] for sub_config in self.email_app] + else: + raise ConfigurationError('Config property email_app not set!') + + def to_yml(self, fn: Union[str, _io.TextIOWrapper], include_tag=False) -> None: + """ + Writes the configuration to a stream. For example a file. + + :param fn: + :param include_tag: + :return: None + """ + + dict_conf = dict() + for config_attribute in self.config_attributes: + dict_conf[config_attribute] = getattr(self, config_attribute) + + if include_tag: + dict_conf['tag'] = self.tag + + if isinstance(fn, str): + with open(fn, 'w') as f: + yaml.dump(dict_conf, f, default_flow_style=False) + elif isinstance(fn, _io.TextIOWrapper): + yaml.dump(dict_conf, fn, default_flow_style=False) + else: + raise TypeError('Expected str or _io.TextIOWrapper not %s' % (type(fn))) + + to_yaml = to_yml + + def to_json(self) -> Dict: + dict_conf = dict() + for config_attribute in self.config_attributes: + dict_conf[config_attribute] = getattr(self, config_attribute) + dict_conf['tag'] = self.tag + return dict_conf + + def __getitem__(self, item): + return self.__getattribute__(item) + + +class ConfigurationError(Exception): + def __init__(self, message): + # Call the base class constructor with the parameters it needs + super().__init__(message) diff --git a/configuration/yml_schema.json b/configuration/yml_schema.json new file mode 100644 index 0000000..17ceb93 --- /dev/null +++ b/configuration/yml_schema.json @@ -0,0 +1,139 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "tag": { + "type": "string" + }, + "datastore": { + "$ref": "#/definitions/datastore" + }, + "cloudstore": { + "$ref": "#/definitions/cloudstore" + }, + "email_app": { + "$ref": "#/definitions/email_app" + } + }, + "required": [ + "tag" + ], + "definitions": { + "datastore": { + "type": "array", + "items": { + "type": "object" + }, + "additionalProperties": false, + "required": [ + "type", + "config" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "mysql", + "mongodb" + ] + }, + "config": { + "type": "object", + "additionalProperties": false, + "required": [ + "hostname", + "username", + "password", + "db_name" + ], + "properties": { + "hostname": { + "type": "string" + }, + "username": { + "type": "string" + }, + "password": { + "type": "string" + }, + "db_name": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + }, + "cloudstore": { + "type": "array", + "items": { + "type": "object" + }, + "additionalProperties": false, + "required": [ + "config", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "dropbox", + "s3" + ] + }, + "config": { + "type": "object", + "required": [ + "api_key" + ], + "properties": { + "api_key": { + "type": "string" + } + }, + "additionalProperties": true + } + } + }, + "email_app": { + "type": "array", + "items": { + "type": "object" + }, + "additionalProperties": false, + "required": [ + "config", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "gmail", + "hotmail" + ] + }, + "config": { + "type": "object", + "required": [ + "email_address", + "api_key" + ], + "properties": { + "email_address": { + "type": "string" + }, + "api_key": { + "type": "string" + } + }, + "additionalProperties": true + } + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/confs/template_conf.yml b/confs/template_conf.yml new file mode 100644 index 0000000..5deff53 --- /dev/null +++ b/confs/template_conf.yml @@ -0,0 +1,18 @@ +tag: production +cloudstore: + - config: + api_key: !ENV ${DROPBOX_API_KEY} + type: dropbox +datastore: + - config: + hostname: !ENV ${MYSQL_HOST} + username: !ENV ${MYSQL_USERNAME} + password: !ENV ${MYSQL_PASSWORD} + db_name: !ENV ${MYSQL_DB_NAME} + port: 3306 + type: mysql +email_app: + - config: + email_address: !ENV ${EMAIL_ADDRESS} + api_key: !ENV ${GMAIL_API_KEY} + type: gmail \ No newline at end of file diff --git a/data/sample_data.txt b/data/sample_data.txt new file mode 100644 index 0000000..5c611d5 --- /dev/null +++ b/data/sample_data.txt @@ -0,0 +1 @@ +This is a sample data file \ No newline at end of file diff --git a/datastore/__init__.py b/datastore/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datastore/abstract_datastore.py b/datastore/abstract_datastore.py new file mode 100644 index 0000000..34bf22e --- /dev/null +++ b/datastore/abstract_datastore.py @@ -0,0 +1,58 @@ +from abc import ABC, abstractmethod +from typing import List, Dict + + +class AbstractDatastore(ABC): + __slots__ = ('_connection', '_cursor') + + @abstractmethod + def __init__(self, config: Dict) -> None: + """ + Tha basic constructor. Creates a new instance of Datastore using the specified credentials + + :param config: + """ + + self._connection, self._cursor = self.get_connection(username=config['username'], + password=config['password'], + hostname=config['hostname'], + db_name=config['db_name'], + port=config['port']) + + @staticmethod + @abstractmethod + def get_connection(username: str, password: str, hostname: str, db_name: str, port: int): + pass + + @abstractmethod + def create_table(self, table: str, schema: str): + pass + + @abstractmethod + def drop_table(self, table: str) -> None: + pass + + @abstractmethod + def truncate_table(self, table: str) -> None: + pass + + @abstractmethod + def insert_into_table(self, table: str, data: dict) -> None: + pass + + @abstractmethod + def update_table(self, table: str, set_data: dict, where: str) -> None: + pass + + @abstractmethod + def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', order_by: str = 'NULL', + asc_or_desc: str = 'ASC', limit: int = 1000) -> List: + pass + + @abstractmethod + def delete_from_table(self, table: str, where: str) -> None: + pass + + @abstractmethod + def show_tables(self, *args, **kwargs) -> List: + pass diff --git a/datastore/mysql_datastore.py b/datastore/mysql_datastore.py new file mode 100644 index 0000000..96278f0 --- /dev/null +++ b/datastore/mysql_datastore.py @@ -0,0 +1,192 @@ +import logging +from typing import List, Tuple, Dict + +from mysql import connector as mysql_connector + +from .abstract_datastore import AbstractDatastore + +logger = logging.getLogger('MySqlDataStore') + + +class MySqlDatastore(AbstractDatastore): + __slots__ = ('_connection', '_cursor') + + _connection: mysql_connector.connection_cext.CMySQLConnection + _cursor: mysql_connector.connection_cext.CMySQLCursor + + def __init__(self, config: Dict) -> None: + """ + The basic constructor. Creates a new instance of Datastore using the specified credentials + + :param config: + """ + + super().__init__(config) + + @staticmethod + def get_connection(username: str, password: str, hostname: str, db_name: str, port: int = 3306) \ + -> Tuple[mysql_connector.connection_cext.CMySQLConnection, mysql_connector.connection_cext.CMySQLCursor]: + """ + Creates and returns a connection and a cursor/session to the MySQL DB + + :param username: + :param password: + :param hostname: + :param db_name: + :param port: + :return: + """ + + connection = mysql_connector.connect( + host=hostname, + user=username, + passwd=password, + database=db_name, + use_pure=True + ) + + cursor = connection.cursor() + + return connection, cursor + + def create_table(self, table: str, schema: str) -> None: + """ + Creates a table using the specified schema + + :param self: + :param table: + :param schema: + :return: + """ + + query = "CREATE TABLE IF NOT EXISTS {table} ({schema})".format(table=table, schema=schema) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def drop_table(self, table: str) -> None: + """ + Drops the specified table if it exists + + :param self: + :param table: + :return: + """ + + query = "DROP TABLE IF EXISTS {table}".format(table=table) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def truncate_table(self, table: str) -> None: + """ + Truncates the specified table + + :param self: + :param table: + :return: + """ + + query = "TRUNCATE TABLE {table}".format(table=table) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def insert_into_table(self, table: str, data: dict) -> None: + """ + Inserts into the specified table a row based on a column_name: value dictionary + + :param self: + :param table: + :param data: + :return: + """ + + data_str = ", ".join( + list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), data.keys(), data.values()))) + + query = "INSERT INTO {table} SET {data}".format(table=table, data=data_str) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def update_table(self, table: str, set_data: dict, where: str) -> None: + """ + Updates the specified table using a column_name: value dictionary and a where statement + + :param self: + :param table: + :param set_data: + :param where: + :return: + """ + + set_data_str = ", ".join( + list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), set_data.keys(), + set_data.values()))) + + query = "UPDATE {table} SET {data} WHERE {where}".format(table=table, data=set_data_str, where=where) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', order_by: str = 'NULL', + asc_or_desc: str = 'ASC', limit: int = 1000) -> List: + """ + Selects from a specified table based on the given columns, where, ordering and limit + + :param self: + :param table: + :param columns: + :param where: + :param order_by: + :param asc_or_desc: + :param limit: + :return results: + """ + + query = "SELECT {columns} FROM {table} WHERE {where} ORDER BY {order_by} {asc_or_desc} LIMIT {limit}".format( + columns=columns, table=table, where=where, order_by=order_by, asc_or_desc=asc_or_desc, limit=limit) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + results = self._cursor.fetchall() + + return results + + def delete_from_table(self, table: str, where: str) -> None: + """ + Deletes data from the specified table based on a where statement + + :param self: + :param table: + :param where: + :return: + """ + + query = "DELETE FROM {table} WHERE {where}".format(table=table, where=where) + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + self._connection.commit() + + def show_tables(self) -> List: + """ + Show a list of the tables present in the db + :return: + """ + + query = 'SHOW TABLES' + logger.debug("Executing: %s" % query) + self._cursor.execute(query) + results = self._cursor.fetchall() + + return [result[0] for result in results] + + def __exit__(self) -> None: + """ + Flushes and closes the connection + + :return: + """ + + self._connection.commit() + self._cursor.close() diff --git a/email_app/__init__.py b/email_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/email_app/abstract_email_app.py b/email_app/abstract_email_app.py new file mode 100644 index 0000000..3b180e2 --- /dev/null +++ b/email_app/abstract_email_app.py @@ -0,0 +1,39 @@ +from abc import ABC, abstractmethod + + +class AbstractEmailApp(ABC): + __slots__ = ('_handler',) + + @abstractmethod + def __init__(self, *args, **kwargs) -> None: + """ + Tha basic constructor. Creates a new instance of EmailApp using the specified credentials + + """ + + pass + + @staticmethod + @abstractmethod + def get_handler(*args, **kwargs): + """ + Returns an EmailApp handler. + + :param args: + :param kwargs: + :return: + """ + + pass + + @abstractmethod + def send_email(self, *args, **kwargs): + """ + Sends an email with the specified arguments. + + :param args: + :param kwargs: + :return: + """ + + pass diff --git a/email_app/gmail_email_app.py b/email_app/gmail_email_app.py new file mode 100644 index 0000000..9126252 --- /dev/null +++ b/email_app/gmail_email_app.py @@ -0,0 +1,85 @@ +from typing import List, Dict +import logging +from gmail import GMail, Message + +from .abstract_email_app import AbstractEmailApp + +logger = logging.getLogger('GmailEmailApp') + + +class GmailEmailApp(AbstractEmailApp): + __slots__ = ('_handler', 'email_address', 'test_mode') + + _handler: GMail + test_mode: bool + + def __init__(self, config: Dict, test_mode: bool = False) -> None: + """ + The basic constructor. Creates a new instance of EmailApp using the specified credentials + + :param config: + :param test_mode: + """ + + self.email_address = config['email_address'] + self._handler = self.get_handler(email_address=self.email_address, + api_key=config['api_key']) + self.test_mode = test_mode + super().__init__() + + @staticmethod + def get_handler(email_address: str, api_key: str) -> GMail: + """ + Returns an EmailApp handler. + + :param email_address: + :param api_key: + :return: + """ + + gmail_handler = GMail(username=email_address, password=api_key) + gmail_handler.connect() + return gmail_handler + + def is_connected(self) -> bool: + return self._handler.is_connected() + + def get_self_email(self): + return self.email_address + + def send_email(self, subject: str, to: List, cc: List = None, bcc: List = None, text: str = None, html: str = None, + attachments: List = None, sender: str = None, reply_to: str = None) -> None: + """ + Sends an email with the specified arguments. + + :param subject: + :param to: + :param cc: + :param bcc: + :param text: + :param html: + :param attachments: + :param sender: + :param reply_to: + :return: + """ + + if self.test_mode: + to = self.email_address + cc = self.email_address if cc is not None else None + bcc = self.email_address if bcc is not None else None + + msg = Message(subject=subject, + to=",".join(to), + cc=",".join(cc) if cc is not None else None, + bcc=",".join(bcc) if cc is not None else None, + text=text, + html=html, + attachments=attachments, + sender=sender, + reply_to=reply_to) + logger.debug("Sending email with Message: %s" % msg) + self._handler.send(msg) + + def __exit__(self): + self._handler.close() diff --git a/logs/out.log b/logs/out.log new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/logs/out.log @@ -0,0 +1 @@ + diff --git a/main.py b/main.py new file mode 100644 index 0000000..94f2275 --- /dev/null +++ b/main.py @@ -0,0 +1,148 @@ +import traceback +import logging +import argparse +import os + +from configuration.configuration import Configuration +from datastore.mysql_datastore import MySqlDatastore +from cloudstore.dropbox_cloudstore import DropboxCloudstore +from email_app.gmail_email_app import GmailEmailApp + +logger = logging.getLogger('Main') + + +def _setup_log(log_path: str = 'logs/output.log', debug: bool = False) -> None: + log_path = log_path.split(os.sep) + if len(log_path) > 1: + + try: + os.makedirs((os.sep.join(log_path[:-1]))) + except FileExistsError: + pass + log_filename = os.sep.join(log_path) + # noinspection PyArgumentList + logging.basicConfig(level=logging.INFO if debug is not True else logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_filename), + # logging.handlers.TimedRotatingFileHandler(log_filename, when='midnight', interval=1), + logging.StreamHandler() + ] + ) + + +def _argparser() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description='A template for python projects.', + add_help=False) + # Required Args + required_arguments = parser.add_argument_group('Required Arguments') + config_file_params = { + 'type': argparse.FileType('r'), + 'required': True, + 'help': "The configuration yml file" + } + required_arguments.add_argument('-m', '--run-mode', choices=['run_mode_1', 'run_mode_2', 'run_mode_3'], + required=True, + default='run_mode_1', + help='Description of the run modes') + required_arguments.add_argument('-c', '--config-file', **config_file_params) + required_arguments.add_argument('-l', '--log', help="Name of the output log file") + # Optional args + optional = parser.add_argument_group('Optional Arguments') + optional.add_argument('-d', '--debug', action='store_true', help='Enables the debug log messages') + optional.add_argument("-h", "--help", action="help", help="Show this help message and exit") + + return parser.parse_args() + + +def main(): + """ + :Example: + python main.py -m run_mode_1 + -c confs/template_conf.yml + -l logs/output.log + """ + + # Initializing + args = _argparser() + _setup_log(args.log, args.debug) + logger.info("Starting in run mode: {0}".format(args.run_mode)) + # Load the configuration + configuration = Configuration(config_src=args.config_file) + # Init the Cloudstore + cloud_store = DropboxCloudstore(config=configuration.get_cloudstores()[0]) + # Init the Datastore + data_store = MySqlDatastore(**configuration.get_datastores()[0]) + # Init the Email App + gmail_configuration = configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=configuration.get_email_apps()[0]) + + # Mysql examples + logger.info("\n\nMYSQL EXAMPLE\n-------------------------") + logger.info("\n\nTables in current DB: {0}".format(list(data_store.show_tables()))) + logger.info("Creating Table: orders") + table_schema = """ order_id INT(6) PRIMARY KEY, + order_type VARCHAR(30) NOT NULL, + location VARCHAR(30) NOT NULL """ + data_store.create_table(table='orders', schema=table_schema) + logger.info("Tables in current DB:\n{0}".format(list(data_store.show_tables()))) + logger.info("Inserting into orders the values:\n(1 simple newyork)..") + insert_data = {"order_id": 1, + "order_type": "plain", + "location": "new_york"} + data_store.insert_into_table(table='orders', data=insert_data) + logger.info("SELECT * FROM orders;\n{0}".format(data_store.select_from_table(table='orders'))) + logger.info("Deleting the inserted row from table orders..") + data_store.delete_from_table(table='orders', where='order_id=1') + logger.info("SELECT * FROM orders;\n{0}".format(data_store.select_from_table(table='orders'))) + logger.info("Dropping Table: orders") + data_store.drop_table(table='orders') + logger.info("Tables in current DB:\n{0}".format(list(data_store.show_tables()))) + + # Dropbox examples + logger.info("\n\nDROPBOX EXAMPLE\n-------------------------") + logger.info( + "List of files in Dropbox /python_template:\n{0}".format(list(cloud_store.ls(path='/python_template').keys()))) + upload_path = "/python_template/file1.txt" + file_content = "test file content" + logger.info("Uploading file {file} with content:\n{content}".format(file=upload_path, content=file_content)) + cloud_store.upload_file(file_bytes=file_content.encode(), upload_path=upload_path) + logger.info( + "List of files in Dropbox /python_template:\n{0}".format(list(cloud_store.ls(path='/python_template').keys()))) + downloaded_file = cloud_store.download_file(frompath=upload_path) + logger.info("Downloaded file and its content is:\n{0}".format(downloaded_file)) + cloud_store.delete_file(file_path=upload_path) + logger.info("Deleting file {file}..".format(file=upload_path)) + logger.info( + "List of files in Dropbox /python_template:\n{0}".format(list(cloud_store.ls(path='/python_template').keys()))) + + # Gmail examples + logger.info("\n\nGMAIL EXAMPLE\n-------------------------") + subject = "Email example" + body = "

This is an html body example


This goes to the html argument. " \ + "You can use the text argument for plain text." + emails_list = [gmail_configuration['email_address']] + attachments_paths = [os.path.join('data', 'sample_data.txt')] + logger.info( + "Sending email with `subject` = `{subject}`, `from,to,cc,bcc,reply_to` = `{email_addr}`, " + "`html` = `{body}` and `attachments` = `{attachments}`".format( + subject=subject, email_addr=emails_list[0], body=body, attachments=attachments_paths)) + gmail_app.send_email(subject=subject, + to=emails_list, + cc=emails_list, + bcc=emails_list, + html=body, + attachments=attachments_paths, + sender=emails_list[0], + reply_to=emails_list[0] + ) + + +if __name__ == '__main__': + try: + main() + except Exception as e: + logging.error(str(e) + '\n' + str(traceback.format_exc())) + raise e diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d62e24d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +mysql-connector-python==8.0.19 +mysql-connector==2.2.9 +dropbox==10.1.1 +PyYAML==5.3.1 +jsonschema==3.2.0 +gmail==0.6.3 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..51b5d91 --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +from setuptools import setup +import sys + +# import subprocess + +LOCAL_ARG = '--local' + +# Required Version: Python3.6 +if sys.version_info < (3, 6): + print('Python >= 3.6 required') + +# Configure Requirements +with open('requirements.txt') as f: + requirements = f.readlines() + +# For the cases you want a different package to be installed on local and prod environments +if LOCAL_ARG in sys.argv: + index = sys.argv.index(LOCAL_ARG) # Index of the local argument + sys.argv.pop(index) # Removes the local argument in order to prevent the setup() error + # subprocess.check_call([sys.executable, "-m", "pip", "install", 'A package that works locally']) +else: + # subprocess.check_call([sys.executable, "-m", "pip", "install", 'A package that works on production']) + pass + +# Run the Setup +setup( + name='template_python_project', + version='0.1', + # package_dir={'': '.'}, + packages=['datastore', 'cloudstore', 'configuration', 'email_app'], + py_modules=['main'], + data_files=[('', ['configuration/yml_schema.json'])], + entry_points={ + 'console_scripts': [ + 'template_python_project=main:main', + ] + }, + url='https://github.com/drkostas/template_python_project', + license='GNU General Public License v3.0', + author='drkostas', + author_email='georgiou.kostas94@gmail.com', + description='A template for python projects.' + +) diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..422d5d7 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,109 @@ +import unittest +from jsonschema.exceptions import ValidationError +from typing import Dict +import logging +import os + +from configuration.configuration import Configuration + +logger = logging.getLogger('TestConfiguration') + + +class TestConfiguration(unittest.TestCase): + test_data_path: str = os.path.join('test_data', 'test_configuration') + + def test_schema_validation(self): + try: + logger.info('Loading the correct Configuration..') + Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), + config_schema_path=os.path.join('..', 'tests', self.test_data_path, + 'minimal_yml_schema.json')) + except ValidationError as e: + logger.error('Error validating the correct yml: %s', e) + self.fail('Error validating the correct yml') + else: + logger.info('First yml validated successfully.') + + with self.assertRaises(ValidationError): + logger.info('Loading the wrong Configuration..') + Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_wrong.yml')) + logger.info('Second yml failed to validate successfully.') + + def test_to_json(self): + logger.info('Loading Configuration..') + configuration = Configuration(config_src=os.path.join(self.test_data_path, 'template_conf.yml')) + expected_json = {'tag': 'production', + 'datastore': [{'config': + {'hostname': 'host123', + 'username': 'user1', + 'password': 'pass2', + 'db_name': 'db3', + 'port': 3306}, + 'type': 'mysql'}], + 'cloudstore': [{'config': + {'api_key': 'apiqwerty'}, + 'type': 'dropbox'}]} + # Compare + logger.info('Comparing the results..') + self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(configuration.to_json())) + + def test_to_yaml(self): + logger.info('Loading Configuration..') + configuration = Configuration(config_src=os.path.join(self.test_data_path, 'template_conf.yml')) + # Modify and export yml + logger.info('Changed the host and the api_key..') + configuration.datastore[0]['config']['hostname'] = 'changedhost' + configuration.cloudstore[0]['config']['api_key'] = 'changed_api' + logger.info('Exporting to yaml..') + configuration.to_yaml('test_data/test_configuration/actual_output_to_yaml.yml', include_tag=True) + # Load the modified yml + logger.info('Loading the exported yaml..') + modified_configuration = Configuration( + config_src=os.path.join(self.test_data_path, 'actual_output_to_yaml.yml')) + # Compare + logger.info('Comparing the results..') + expected_json = {'tag': 'production', + 'datastore': [{'config': + {'hostname': 'changedhost', + 'username': 'user1', + 'password': 'pass2', + 'db_name': 'db3', + 'port': 3306}, + 'type': 'mysql'}], + 'cloudstore': [{'config': + {'api_key': 'changed_api'}, + 'type': 'dropbox'}]} + self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(modified_configuration.to_json())) + + @classmethod + def _sort_dict(cls, dictionary: Dict) -> Dict: + return {k: cls._sort_dict(v) if isinstance(v, dict) else v + for k, v in sorted(dictionary.items())} + + @staticmethod + def _setup_log() -> None: + # noinspection PyArgumentList + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler() + ] + ) + + def setUp(self) -> None: + pass + + def tearDown(self) -> None: + pass + + @classmethod + def setUpClass(cls): + cls._setup_log() + + @classmethod + def tearDownClass(cls): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_data/test_configuration/actual_output_to_yaml.yml b/tests/test_data/test_configuration/actual_output_to_yaml.yml new file mode 100644 index 0000000..d2f5c32 --- /dev/null +++ b/tests/test_data/test_configuration/actual_output_to_yaml.yml @@ -0,0 +1,13 @@ +cloudstore: +- config: + api_key: changed_api + type: dropbox +datastore: +- config: + db_name: db3 + hostname: changedhost + password: pass2 + port: 3306 + username: user1 + type: mysql +tag: production diff --git a/tests/test_data/test_configuration/minimal_conf_correct.yml b/tests/test_data/test_configuration/minimal_conf_correct.yml new file mode 100644 index 0000000..bdde89f --- /dev/null +++ b/tests/test_data/test_configuration/minimal_conf_correct.yml @@ -0,0 +1,7 @@ +datastore: test +cloudstore: + - subproperty1: 1 + subproperty2: + - 123 + - 234 +tag: test_tag \ No newline at end of file diff --git a/tests/test_data/test_configuration/minimal_conf_wrong.yml b/tests/test_data/test_configuration/minimal_conf_wrong.yml new file mode 100644 index 0000000..60089e5 --- /dev/null +++ b/tests/test_data/test_configuration/minimal_conf_wrong.yml @@ -0,0 +1,7 @@ +datastore: test +cloudstore: + - subproperty1: 10 + subproperty2: + - 123 + - 234 +tag: test_tag \ No newline at end of file diff --git a/tests/test_data/test_configuration/minimal_yml_schema.json b/tests/test_data/test_configuration/minimal_yml_schema.json new file mode 100644 index 0000000..c391b67 --- /dev/null +++ b/tests/test_data/test_configuration/minimal_yml_schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "datastore": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "cloudstore": { + "$ref": "#/definitions/cloudstore" + } + }, + "required": [ + "tag" + ], + "definitions": { + "cloudstore": { + "type": "array", + "items": { + "type": "object" + }, + "additionalProperties": false, + "required": [ + "subproperty1", + "subproperty2" + ], + "properties": { + "subproperty1": { + "type": "number", + "enum": [ + 1, + 2 + ] + }, + "subproperty2": { + "type": "array" + } + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/tests/test_data/test_configuration/template_conf.yml b/tests/test_data/test_configuration/template_conf.yml new file mode 100644 index 0000000..27ef9a9 --- /dev/null +++ b/tests/test_data/test_configuration/template_conf.yml @@ -0,0 +1,13 @@ +tag: production +cloudstore: + - config: + api_key: apiqwerty + type: dropbox +datastore: + - config: + hostname: host123 + username: user1 + password: pass2 + db_name: db3 + port: 3306 + type: mysql \ No newline at end of file diff --git a/tests/test_data/test_dropbox_cloudstore/template_conf.yml b/tests/test_data/test_dropbox_cloudstore/template_conf.yml new file mode 100644 index 0000000..3b1ff28 --- /dev/null +++ b/tests/test_data/test_dropbox_cloudstore/template_conf.yml @@ -0,0 +1,13 @@ +tag: production +cloudstore: + - config: + api_key: !ENV ${DROPBOX_API_KEY} + type: dropbox +datastore: + - config: + hostname: host123 + username: user1 + password: pass2 + db_name: db3 + port: 3306 + type: mysql \ No newline at end of file diff --git a/tests/test_data/test_gmail_email_app/sample_data.txt b/tests/test_data/test_gmail_email_app/sample_data.txt new file mode 100644 index 0000000..5c611d5 --- /dev/null +++ b/tests/test_data/test_gmail_email_app/sample_data.txt @@ -0,0 +1 @@ +This is a sample data file \ No newline at end of file diff --git a/tests/test_data/test_gmail_email_app/template_conf.yml b/tests/test_data/test_gmail_email_app/template_conf.yml new file mode 100644 index 0000000..5deff53 --- /dev/null +++ b/tests/test_data/test_gmail_email_app/template_conf.yml @@ -0,0 +1,18 @@ +tag: production +cloudstore: + - config: + api_key: !ENV ${DROPBOX_API_KEY} + type: dropbox +datastore: + - config: + hostname: !ENV ${MYSQL_HOST} + username: !ENV ${MYSQL_USERNAME} + password: !ENV ${MYSQL_PASSWORD} + db_name: !ENV ${MYSQL_DB_NAME} + port: 3306 + type: mysql +email_app: + - config: + email_address: !ENV ${EMAIL_ADDRESS} + api_key: !ENV ${GMAIL_API_KEY} + type: gmail \ No newline at end of file diff --git a/tests/test_data/test_mysql_datastore/template_conf.yml b/tests/test_data/test_mysql_datastore/template_conf.yml new file mode 100644 index 0000000..fd96f52 --- /dev/null +++ b/tests/test_data/test_mysql_datastore/template_conf.yml @@ -0,0 +1,13 @@ +tag: production +cloudstore: + - config: + api_key: sample_api_key + type: dropbox +datastore: + - config: + hostname: !ENV ${MYSQL_HOST} + username: !ENV ${MYSQL_USERNAME} + password: !ENV ${MYSQL_PASSWORD} + db_name: !ENV ${MYSQL_DB_NAME} + port: 3306 + type: mysql \ No newline at end of file diff --git a/tests/test_dropbox_cloudstore.py b/tests/test_dropbox_cloudstore.py new file mode 100644 index 0000000..b09f238 --- /dev/null +++ b/tests/test_dropbox_cloudstore.py @@ -0,0 +1,111 @@ +import unittest +import os +import random +import string +import logging +import copy +from typing import Tuple +from dropbox.exceptions import BadInputError + +from configuration.configuration import Configuration +from cloudstore.dropbox_cloudstore import DropboxCloudstore + +logger = logging.getLogger('TestDropboxCloudstore') + + +class TestDropboxCloudstore(unittest.TestCase): + __slots__ = ('configuration', 'file_name') + + configuration: Configuration + file_name: str + test_data_path: str = os.path.join('test_data', 'test_dropbox_cloudstore') + + def test_connect(self): + # Test the connection with the correct api key + try: + cloud_store_correct_key = DropboxCloudstore(config=self.configuration.get_cloudstores()[0]) + cloud_store_correct_key.ls() + except BadInputError as e: + logger.error('Error connecting with the correct credentials: %s', e) + self.fail('Error connecting with the correct credentials') + else: + logger.info('Connected with the correct credentials successfully.') + # Test that the connection is failed with the wrong credentials + with self.assertRaises(BadInputError): + cloud_store_wrong_configuration = copy.deepcopy(self.configuration.get_cloudstores()[0]) + cloud_store_wrong_configuration['api_key'] = 'wrong_key' + cloud_store_wrong_key = DropboxCloudstore(config=cloud_store_wrong_configuration) + cloud_store_wrong_key.ls() + logger.info("Loading Dropbox with wrong credentials failed successfully.") + + def test_upload_download(self): + cloud_store = DropboxCloudstore(config=self.configuration.get_cloudstores()[0]) + # Upload file + logger.info('Uploading file..') + file_to_upload = open(os.path.join(self.test_data_path, self.file_name), 'rb').read() + cloud_store.upload_file(file_to_upload, '/tests/' + self.file_name) + # Check if it was uploaded + self.assertIn(self.file_name, cloud_store.ls('/tests/').keys()) + # Download it + logger.info('Downloading file..') + cloud_store.download_file(frompath='/tests/' + self.file_name, + tofile=os.path.join(self.test_data_path, 'actual_downloaded.txt')) + # Compare contents of downloaded file with the original + self.assertEqual(open(os.path.join(self.test_data_path, self.file_name), 'rb').read(), + open(os.path.join(self.test_data_path, 'actual_downloaded.txt'), 'rb').read()) + + def test_upload_delete(self): + cloud_store = DropboxCloudstore(config=self.configuration.get_cloudstores()[0]) + # Upload file + logger.info('Uploading file..') + file_to_upload = open(os.path.join(self.test_data_path, self.file_name), 'rb').read() + cloud_store.upload_file(file_to_upload, '/tests/' + self.file_name) + # Check if it was uploaded + self.assertIn(self.file_name, cloud_store.ls('/tests/').keys()) + # Delete it + cloud_store.delete_file('/tests/' + self.file_name) + # Check if it was deleted + self.assertNotIn(self.file_name, cloud_store.ls('/tests/').keys()) + + @staticmethod + def _generate_random_filename_and_contents() -> Tuple[str, str]: + letters = string.ascii_lowercase + file_name = ''.join(random.choice(letters) for _ in range(10)) + '.txt' + contents = ''.join(random.choice(letters) for _ in range(20)) + return file_name, contents + + @staticmethod + def _setup_log() -> None: + # noinspection PyArgumentList + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler() + ] + ) + + def setUp(self) -> None: + self.file_name, contents = self._generate_random_filename_and_contents() + with open(os.path.join(self.test_data_path, self.file_name), 'a') as f: + f.write(contents) + + def tearDown(self) -> None: + os.remove(os.path.join(self.test_data_path, self.file_name)) + + @classmethod + def setUpClass(cls): + cls._setup_log() + if "DROPBOX_API_KEY" not in os.environ: + logger.error('DROPBOX_API_KEY env variable is not set!') + raise Exception('DROPBOX_API_KEY env variable is not set!') + logger.info('Loading Configuration..') + cls.configuration = Configuration(config_src=os.path.join(cls.test_data_path, 'template_conf.yml')) + + @classmethod + def tearDownClass(cls): + cloud_store = DropboxCloudstore(config=cls.configuration.get_cloudstores()[0]) + cloud_store.delete_file('/tests') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_gmail_email_app.py b/tests/test_gmail_email_app.py new file mode 100644 index 0000000..205429d --- /dev/null +++ b/tests/test_gmail_email_app.py @@ -0,0 +1,143 @@ +import unittest +import os +import random +import string +import logging +import copy +from typing import Tuple +from smtplib import SMTPAuthenticationError + +from configuration.configuration import Configuration +from email_app.gmail_email_app import GmailEmailApp + +logger = logging.getLogger('TestGmailEmailApp') + + +class TestGmailEmailApp(unittest.TestCase): + __slots__ = ('configuration', 'file_name') + + configuration: Configuration + file_name: str + test_data_path: str = os.path.join('test_data', 'test_gmail_email_app') + + def test_connect(self): + # Test the connection with the correct api key + try: + gmail_configuration = self.configuration.get_email_apps()[0] + GmailEmailApp(config=gmail_configuration) + except SMTPAuthenticationError as e: + logger.error('Error connecting with the correct credentials: %s', e) + self.fail('Error connecting with the correct credentials') + else: + logger.info('Connected with the correct credentials successfully.') + # Test that the connection is failed with the wrong credentials + with self.assertRaises(SMTPAuthenticationError): + gmail_wrong_configuration = copy.deepcopy(gmail_configuration) + gmail_wrong_configuration['api_key'] = 'wrong_key' + GmailEmailApp(config=gmail_wrong_configuration) + logger.info("Loading Dropbox with wrong credentials failed successfully.") + + def test_is_connected_and_exit(self): + gmail_configuration = self.configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=gmail_configuration) + self.assertEqual(True, gmail_app.is_connected()) + gmail_app.__exit__() + self.assertEqual(False, gmail_app.is_connected()) + + def test_send_email_with_all_args(self): + try: + gmail_configuration = self.configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=gmail_configuration) + + gmail_app.send_email(subject='test_send_email_with_all_args', + to=[gmail_configuration['email_address']], + cc=[gmail_configuration['email_address']], + bcc=[gmail_configuration['email_address']], + text='Test plain/text body', + html='

Test html body

', + attachments=[os.path.join(self.test_data_path, 'sample_data.txt')], + sender=gmail_configuration['email_address'], + reply_to=gmail_configuration['email_address'] + ) + except Exception as e: + logger.error("Test failed with exception: %s" % e) + self.fail("Test failed with exception: %s" % e) + + def test_send_email_with_required_args(self): + try: + gmail_configuration = self.configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=gmail_configuration) + + gmail_app.send_email(subject='test_send_email_with_required_args', + to=[gmail_configuration['email_address']] + ) + except Exception as e: + logger.error("Test failed with exception: %s" % e) + self.fail("Test failed with exception: %s" % e) + + def test_send_email_with_html(self): + try: + gmail_configuration = self.configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=gmail_configuration) + + gmail_app.send_email(subject='test_send_email_with_html', + to=[gmail_configuration['email_address']], + html='

Html only

' + ) + except Exception as e: + logger.error("Test failed with exception: %s" % e) + self.fail("Test failed with exception: %s" % e) + + def test_send_email_with_text(self): + try: + gmail_configuration = self.configuration.get_email_apps()[0] + gmail_app = GmailEmailApp(config=gmail_configuration) + + gmail_app.send_email(subject='test_send_email_with_text', + to=[gmail_configuration['email_address']], + text='Text only' + ) + except Exception as e: + logger.error("Test failed with exception: %s" % e) + self.fail("Test failed with exception: %s" % e) + + @staticmethod + def _generate_random_filename_and_contents() -> Tuple[str, str]: + letters = string.ascii_lowercase + file_name = ''.join(random.choice(letters) for _ in range(10)) + '.txt' + contents = ''.join(random.choice(letters) for _ in range(20)) + return file_name, contents + + @staticmethod + def _setup_log() -> None: + # noinspection PyArgumentList + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler() + ] + ) + + def setUp(self) -> None: + pass + + def tearDown(self) -> None: + pass + + @classmethod + def setUpClass(cls): + cls._setup_log() + gmail_os_vars = ['EMAIL_ADDRESS', 'GMAIL_API_KEY'] + if not all(gmail_os_var in os.environ for gmail_os_var in gmail_os_vars): + logger.error('Gmail env variables are not set!') + raise Exception('Gmail env variables are not set!') + logger.info('Loading Configuration..') + cls.configuration = Configuration(config_src=os.path.join(cls.test_data_path, 'template_conf.yml')) + + @classmethod + def tearDownClass(cls): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_mysql_datastore.py b/tests/test_mysql_datastore.py new file mode 100644 index 0000000..e86c40e --- /dev/null +++ b/tests/test_mysql_datastore.py @@ -0,0 +1,120 @@ +import unittest +import os +import copy +import random +import string +import logging +from typing import List +from mysql.connector.errors import ProgrammingError as MsqlProgrammingError + +from configuration.configuration import Configuration +from datastore.mysql_datastore import MySqlDatastore + +logger = logging.getLogger('TestMysqlDatastore') + + +class TestMysqlDatastore(unittest.TestCase): + __slots__ = ('configuration', 'test_table_schema') + + configuration: Configuration + test_table_schema: str + generated_table_names: List[str] = list() + test_data_path: str = os.path.join('test_data', 'test_mysql_datastore') + + def test_connect(self): + # Test the connection with the correct api key + try: + MySqlDatastore(config=self.configuration.get_datastores()[0]) + except MsqlProgrammingError as e: + logger.error('Error connecting with the correct credentials: %s', e) + self.fail('Error connecting with the correct credentials') + else: + logger.info('Connected with the correct credentials successfully.') + # Test that the connection is failed with the wrong credentials + with self.assertRaises(MsqlProgrammingError): + datastore_conf_copy = copy.deepcopy(self.configuration.get_datastores()[0]) + datastore_conf_copy['password'] = 'wrong_password' + MySqlDatastore(config=datastore_conf_copy) + logger.info("Loading Mysql with wrong credentials failed successfully.") + + def test_create_drop(self): + data_store = MySqlDatastore(config=self.configuration.get_datastores()[0]) + # Create table + logger.info('Creating table..') + data_store.create_table(self.table_name, self.test_table_schema) + # Check if it was created + self.assertIn(self.table_name, data_store.show_tables()) + # Drop table + logger.info('Dropping table..') + data_store.drop_table(table=self.table_name) + self.assertNotIn(self.table_name, data_store.show_tables()) + + def test_insert_update_delete(self): + data_store = MySqlDatastore(config=self.configuration.get_datastores()[0]) + # Create table + logger.info('Creating table..') + data_store.create_table(self.table_name, self.test_table_schema) + # Ensure it is empty + results = data_store.select_from_table(table=self.table_name) + self.assertEqual([], results) + # Insert into table + insert_data = {"order_id": 1, + "order_type": "plain", + "is_delivered": False} + logger.info("Inserting into table..") + data_store.insert_into_table(table=self.table_name, data=insert_data) + # Check if the data was inserted + results = data_store.select_from_table(table=self.table_name) + self.assertEqual([(1, "plain", False)], results) + logger.info("Deleting from table..") + data_store.delete_from_table(table=self.table_name, where='order_id =1 ') + # Check if the data was inserted + results = data_store.select_from_table(table=self.table_name) + self.assertEqual([], results) + + @staticmethod + def _generate_random_filename() -> str: + letters = string.ascii_lowercase + file_name = 'test_table_' + ''.join(random.choice(letters) for _ in range(10)) + return file_name + + @staticmethod + def _setup_log() -> None: + # noinspection PyArgumentList + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler() + ] + ) + + def setUp(self) -> None: + self.table_name = self._generate_random_filename() + self.generated_table_names.append(self.table_name) + + def tearDown(self) -> None: + pass + + @classmethod + def setUpClass(cls): + cls._setup_log() + mysql_os_vars = ['MYSQL_HOST', 'MYSQL_USERNAME', 'MYSQL_PASSWORD', 'MYSQL_DB_NAME'] + if not all(mysql_os_var in os.environ for mysql_os_var in mysql_os_vars): + logger.error('Mysql env variables are not set!') + raise Exception('Mysql env variables are not set!') + logger.info('Loading Configuration..') + cls.configuration = Configuration(config_src=os.path.join(cls.test_data_path, 'template_conf.yml')) + cls.test_table_schema = """ order_id INT(6) PRIMARY KEY, + order_type VARCHAR(30) NOT NULL, + is_delivered BOOLEAN NOT NULL """ + + @classmethod + def tearDownClass(cls): + data_store = MySqlDatastore(config=cls.configuration.get_datastores()[0]) + for table in cls.generated_table_names: + logger.info('Dropping table {0}'.format(table)) + data_store.drop_table(table=table) + + +if __name__ == '__main__': + unittest.main()