diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..be3f7b28 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md index 196f9ccd..2452a651 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## DeepPhysX +![logo](docs/source/_static/image/logo.png) + ### Interfacing AI with simulation The **DeepPhysX** project provides Python packages allowing users to easily interface their **numerical simulations** @@ -38,34 +40,26 @@ $ pip install DeepPhysX.Sofa # Install simulation package $ pip install DeepPhysX.Torch # Install AI package ``` -If cloning sources, create a `DeepPhysX` repository to store every package. -Packages must be cloned in a directory with the corresponding name as shown below: - -``` bash -$ mkdir DeepPhysX -$ cd DeepPhysX -$ git clone https://github.com/mimesis-inria/DeepPhysX.git Core # Clone default package -$ git clone https://github.com/mimesis-inria/DeepPhysX.Sofa.git Sofa # Clone simulation package -$ git clone https://github.com/mimesis-inria/DeepPhysX.Torch.git Torch # Clone AI package -$ ls -Core Sofa Torch -``` - ### Demos **DeepPhysX** includes a set of detailed tutorials, examples and demos. -Following this installation process to directly try the **interactive demos**: +As these scripts are producing data, they cannot be run in the python site-packages, thus they should be run locally. +Use the *command line interface* to get the examples or to run **interactive demos**: ``` bash -$ mkdir DeepPhysX -$ cd DeepPhysX -$ git clone https://github.com/mimesis/deepphysx.git Core # Make shure to clone this repository in 'DeepPhysX/Core' -$ cd Core -$ python3 config.py # Answer 'yes' to install Torch package to launch examples -$ pip install . +$ DPX --get # Get the full example repository locally +$ DPX --run # Run one of the demo scripts ``` -| **Armadillo**
`python3 demo.py armadillo` | **Beam**
`python3 demo.py beam` | **Liver**
`python3 demo.py liver` | +| **Armadillo**
`DPX -r armadillo` | **Beam**
`DPX -r beam` | **Liver**
`DPX -r liver` | |:-----------------------------------------------------:|:-------------------------------------------:|:---------------------------------------------:| | ![armadillo](docs/source/_static/image/armadillo.png) | ![beam](docs/source/_static/image/beam.png) | ![liver](docs/source/_static/image/liver.png) | + + +### References + +Did this project help you for your research ? Please cite us as: + +R. Enjalbert, A. Odot and S. Cotin, *DeepPhysX, a python framework to interface AI with numerical simulation*, +Zenodo, 2022, [**DOI**](https://doi.org/10.5281/zenodo.7389505) diff --git a/src/Utils/Visualizer/__init__.py b/__init__.py similarity index 100% rename from src/Utils/Visualizer/__init__.py rename to __init__.py diff --git a/config.py b/config.py deleted file mode 100644 index 294405f7..00000000 --- a/config.py +++ /dev/null @@ -1,86 +0,0 @@ -from os import chdir, pardir, system, sep, rename, listdir, getcwd, mkdir -from os.path import exists, abspath, join -from json import dump -from shutil import move - -PROJECT = 'DeepPhysX' -PACKAGES = {'Torch': False, - 'Sofa': False} -AVAILABLE = {'AI': ['Torch'], - 'Simulation': ['Sofa']} -GIT = {'Torch': 'https://github.com/mimesis-inria/DeepPhysX.Torch.git', - 'Sofa': 'https://github.com/mimesis-inria/DeepPhysX.Sofa.git'} -ANSWERS = ['y', 'yes', 'n', 'no'] - - -def check_repositories(): - - # Check current repository - path = abspath(join(__file__, pardir, pardir)) - repository = abspath(join(__file__, pardir)).split(sep)[-1] - chdir(join(path, repository)) - size = 2 - if repository != 'Core': - print(f"WARNING: Wrong repository, moving '{repository}' --> '{join(PROJECT, 'Core')}'") - chdir(pardir) - rename(repository, 'Core') - mkdir(PROJECT) - move(src=join(path, f'Core{sep}'), - dst=join(path, PROJECT)) - size += 1 - path = join(path, PROJECT) - chdir(join(path, 'Core')) - - # Check other repositories - for i in range(size): - for repository in listdir(getcwd()): - if 'DeepPhysX.' in repository: - if repository[10:] in PACKAGES.keys(): - print(f"WARNING: Wrong repository, moving '{repository}' --> '{join(PROJECT, repository[10:])}'") - rename(repository, repository[10:]) - if not exists(join(path, f'{repository[10:]}{sep}')): - move(src=join(getcwd(), f'{repository[10:]}{sep}'), - dst=path) - chdir(pardir) - chdir(join(path, 'Core')) - - -if __name__ == '__main__': - - # Check repositories names - check_repositories() - - # Get user entry for each package - for package_type, package_names in AVAILABLE.items(): - print(f"\nAvailable {package_type} packages : {package_names}") - for package_name in package_names: - while (do_install := input(f" >> Installing package {package_name} (y/n): ").lower()) not in ANSWERS: - pass - PACKAGES[package_name] = do_install in ANSWERS[:2] - - # Ask user confirmation - print("\nApplying following configuration: \n * DeepPhysX.Core: True (default)") - for package_name, do_install in PACKAGES.items(): - print(f" * DeepPhysX.{package_name}: {do_install}") - while (do_validate := input("Confirm (y/n): ")) not in ANSWERS: - pass - if do_validate in ANSWERS[2:]: - print("Aborting") - quit() - - # Save config - with open('config.json', 'w') as file: - dump(PACKAGES, file) - print("Configuration saved in 'config.json'") - - # Clone missing packages - chdir(pardir) - for package_name, do_install in PACKAGES.items(): - if do_install and not exists(package_name): - print(f"\nPackage {package_name} not found, cloning from {GIT[package_name]}") - system(f'git clone {GIT[package_name]} {package_name}') - - # End config - print("\nConfiguration done, install DeepPhysX with the following commands:" - "\n - 'pip install .' for user mode" - "\n - 'python3 dev.py set' for developer mode") diff --git a/demo.py b/demo.py deleted file mode 100644 index aec206f2..00000000 --- a/demo.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import sys -import argparse - - -parser = argparse.ArgumentParser(prog='demo', description='Specify the name of the demo.') -parser.add_argument('Demo', metavar='demo', type=str, help='Name of the demo to run') -args = parser.parse_args() - -demo = args.Demo -demos = ['armadillo', 'beam', 'liver'] - -if demo not in demos: - raise ValueError(f"Unknown demo '{demo}', available are: {demos}") - -repo = os.path.join(os.path.dirname(__file__), 'examples', 'demos', demo[0].upper() + demo[1:].lower(), 'FC') -os.chdir(repo) -os.system(f'{sys.executable} interactive.py') diff --git a/dev.py b/dev.py deleted file mode 100644 index 4b59b5f3..00000000 --- a/dev.py +++ /dev/null @@ -1,52 +0,0 @@ -from os import listdir, symlink, unlink, mkdir -from os.path import join, islink, abspath, pardir, isdir -from pathlib import Path -from sys import argv -from site import USER_SITE -from shutil import rmtree - -from config import check_repositories - - -# Check user entry -if len(argv) != 2 or argv[1] not in ['set', 'del']: - print("\nInvalid script option." - "\nRun 'python3 dev.py set' to link DPX to your site package." - "\nRun 'python3 dev.py del' to remove DPX links in your site package.") - quit() - -# Check repositories names -check_repositories() - -# Init DeepPhysX packages and dependencies to install -PROJECT = 'DeepPhysX' -packages = ['Core'] -available = ['Torch', 'Sofa'] -root = abspath(join(Path(__file__).parent.absolute(), pardir)) - -# Option 1: create the symbolic links -if argv[1] == 'set': - - # Create main repository in site-packages - if not isdir(join(USER_SITE, PROJECT)): - mkdir(join(USER_SITE, PROJECT)) - - # Link to every existing packages - for package_name in listdir(root): - if package_name in available: - packages.append(package_name) - - # Create symbolic links in site-packages - for package_name in packages: - if not islink(join(USER_SITE, PROJECT, package_name)): - symlink(src=join(root, package_name, 'src'), dst=join(USER_SITE, PROJECT, package_name)) - print(f"Linked {join(USER_SITE, PROJECT, package_name)} -> {join(root, package_name, 'src')}") - -# Option 2: remove the symbolic links -else: - - if isdir(join(USER_SITE, PROJECT)): - for package_name in listdir(join(USER_SITE, PROJECT)): - unlink(join(USER_SITE, PROJECT, package_name)) - print(f"Unlinked {join(USER_SITE, PROJECT, package_name)} -> {join(root, package_name, 'src')}") - rmtree(join(USER_SITE, PROJECT)) diff --git a/docs/source/_static/image/about_tree.png b/docs/source/_static/image/about_tree.png new file mode 100644 index 00000000..47caba3e Binary files /dev/null and b/docs/source/_static/image/about_tree.png differ diff --git a/docs/source/_static/image/about_working_session.png b/docs/source/_static/image/about_working_session.png deleted file mode 100644 index ec46dffa..00000000 Binary files a/docs/source/_static/image/about_working_session.png and /dev/null differ diff --git a/docs/source/_static/image/database.png b/docs/source/_static/image/database.png new file mode 100644 index 00000000..bcf95044 Binary files /dev/null and b/docs/source/_static/image/database.png differ diff --git a/docs/source/_static/image/dataset.png b/docs/source/_static/image/dataset.png deleted file mode 100644 index a0a5c669..00000000 Binary files a/docs/source/_static/image/dataset.png and /dev/null differ diff --git a/docs/source/_static/image/environment_tcp_ip.png b/docs/source/_static/image/environment_tcp_ip.png new file mode 100644 index 00000000..9fe2d2f5 Binary files /dev/null and b/docs/source/_static/image/environment_tcp_ip.png differ diff --git a/docs/source/_static/image/environment_tcpip.png b/docs/source/_static/image/environment_tcpip.png deleted file mode 100644 index d170ed79..00000000 Binary files a/docs/source/_static/image/environment_tcpip.png and /dev/null differ diff --git a/docs/source/_static/image/logo.png b/docs/source/_static/image/logo.png new file mode 100644 index 00000000..51944a83 Binary files /dev/null and b/docs/source/_static/image/logo.png differ diff --git a/docs/source/_static/image/overview_architecture.png b/docs/source/_static/image/overview_architecture.png deleted file mode 100644 index d30490ae..00000000 Binary files a/docs/source/_static/image/overview_architecture.png and /dev/null differ diff --git a/docs/source/_static/image/overview_components.png b/docs/source/_static/image/overview_components.png new file mode 100644 index 00000000..92c1654f Binary files /dev/null and b/docs/source/_static/image/overview_components.png differ diff --git a/docs/source/api/core.rst b/docs/source/api/core.rst index 51945bd6..c7dec455 100644 --- a/docs/source/api/core.rst +++ b/docs/source/api/core.rst @@ -11,25 +11,22 @@ CORE - :doc:`core/environment` - :doc:`core/manager` - * - | :ref:`asyncsocket.abstractenvironment` - | :ref:`asyncsocket.bytesconverter` + * - | :ref:`asyncsocket.bytesconverter` | :ref:`asyncsocket.tcpipclient` | :ref:`asyncsocket.tcpipobject` | :ref:`asyncsocket.tcpipserver` - - | :ref:`dataset.basedataset` - | :ref:`dataset.basedatasetconfig` + - | :ref:`database.databasehandler` + | :ref:`database.basedatabaseconfig` - | :ref:`environment.baseenvironment` | :ref:`environment.baseenvironmentconfig` - | :ref:`manager.datamanager` - | :ref:`manager.datasetmanager` + | :ref:`manager.databasemanager` | :ref:`manager.environmentmanager` - | :ref:`manager.manager` | :ref:`manager.networkmanager` | :ref:`manager.statsmanager` - | :ref:`manager.visualizermanager` .. list-table:: @@ -46,15 +43,13 @@ CORE | :ref:`network.baseoptimization` | :ref:`network.datatransformation` - - | :ref:`pipelines.basedatagenerator` + - | :ref:`pipelines.basedatageneration` | :ref:`pipelines.basepipeline` - | :ref:`pipelines.baserunner` - | :ref:`pipelines.basetrainer` + | :ref:`pipelines.baseprediction` + | :ref:`pipelines.basetraining` - - | :ref:`visualizer.vedoobjects` + - | :ref:`visualizer.vedofactory` | :ref:`visualizer.vedovisualizer` - | - | :ref:`Visualizer.Factories ` .. toctree:: diff --git a/docs/source/api/core/asyncsocket.rst b/docs/source/api/core/asyncsocket.rst index bf4d54e8..b097d540 100644 --- a/docs/source/api/core/asyncsocket.rst +++ b/docs/source/api/core/asyncsocket.rst @@ -1,14 +1,6 @@ AsyncSocket =========== -.. _asyncsocket.abstractenvironment: - -AbstractEnvironment -___________________ - -.. autoclass:: AbstractEnvironment.AbstractEnvironment - :members: - .. _asyncsocket.bytesconverter: BytesConverter @@ -23,7 +15,6 @@ TcpIpClient ___________ Bases: -:py:class:`AbstractEnvironment.AbstractEnvironment` :py:class:`TcpIpObject.TcpIpObject` .. autoclass:: TcpIpClient.TcpIpClient diff --git a/docs/source/api/core/dataset.rst b/docs/source/api/core/dataset.rst index 69d9eb87..469d2f7e 100644 --- a/docs/source/api/core/dataset.rst +++ b/docs/source/api/core/dataset.rst @@ -1,18 +1,18 @@ Dataset ======= -.. _dataset.basedataset: +.. _database.databasehandler: -BaseDataset ------------ +DatabaseHandler +--------------- -.. autoclass:: BaseDataset.BaseDataset +.. autoclass:: DatabaseHandler.DatabaseHandler :members: -.. _dataset.basedatasetconfig: +.. _database.basedatabaseconfig: -BaseDatasetConfig ------------------ +BaseDatabaseConfig +------------------ -.. autoclass:: BaseDatasetConfig.BaseDatasetConfig +.. autoclass:: BaseDatabaseConfig.BaseDatabaseConfig :members: diff --git a/docs/source/api/core/environment.rst b/docs/source/api/core/environment.rst index f14829bc..53d1fdc4 100644 --- a/docs/source/api/core/environment.rst +++ b/docs/source/api/core/environment.rst @@ -6,9 +6,6 @@ Environment BaseEnvironment --------------- -Bases: -:py:class:`TcpIpClient.TcpIpClient` - .. autoclass:: BaseEnvironment.BaseEnvironment :members: diff --git a/docs/source/api/core/factories.rst b/docs/source/api/core/factories.rst deleted file mode 100644 index 9c7c8681..00000000 --- a/docs/source/api/core/factories.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _factories: - -Factories -========= - -ArrowsFactory -------------- - -.. autoclass:: VedoObjectFactories.ArrowsFactory.ArrowsFactory - :members: - - -BaseObjectFactory ------------------ - -.. autoclass:: VedoObjectFactories.BaseObjectFactory.BaseObjectFactory - :members: - - -GlyphFactory ------------- - -.. autoclass:: VedoObjectFactories.GlyphFactory.GlyphFactory - :members: - - -MarkerFactory -------------- - -.. autoclass:: VedoObjectFactories.MarkerFactory.MarkerFactory - :members: - - -MeshFactory ------------ - -.. autoclass:: VedoObjectFactories.MeshFactory.MeshFactory - :members: - - -PointsFactory -------------- - -.. autoclass:: VedoObjectFactories.PointsFactory.PointsFactory - :members: - - -VedoObjectFactory ------------------ - -.. autoclass:: VedoObjectFactories.VedoObjectFactory.VedoObjectFactory - :members: - - -WindowFactory -------------- - -.. autoclass:: VedoObjectFactories.WindowFactory.WindowFactory - :members: diff --git a/docs/source/api/core/manager.rst b/docs/source/api/core/manager.rst index 379febbc..ce177ab3 100644 --- a/docs/source/api/core/manager.rst +++ b/docs/source/api/core/manager.rst @@ -9,12 +9,12 @@ DataManager .. autoclass:: DataManager.DataManager :members: -.. _manager.datasetmanager: +.. _manager.databasemanager: -DatasetManager --------------- +DatabaseManager +--------------- -.. autoclass:: DatasetManager.DatasetManager +.. autoclass:: DatabaseManager.DatabaseManager :members: .. _manager.environmentmanager: @@ -25,14 +25,6 @@ EnvironmentManager .. autoclass:: EnvironmentManager.EnvironmentManager :members: -.. _manager.manager: - -Manager -------- - -.. autoclass:: Manager.Manager - :members: - .. _manager.networkmanager: NetworkManager @@ -48,11 +40,3 @@ StatsManager .. autoclass:: StatsManager.StatsManager :members: - -.. _manager.visualizermanager: - -VisualizerManager ------------------ - -.. autoclass:: VisualizerManager.VisualizerManager - :members: diff --git a/docs/source/api/core/network.rst b/docs/source/api/core/network.rst index a25b0b0f..7fb22845 100644 --- a/docs/source/api/core/network.rst +++ b/docs/source/api/core/network.rst @@ -30,5 +30,5 @@ BaseOptimization DataTransformation ------------------ -.. autoclass:: DataTransformation.DataTransformation +.. autoclass:: BaseTransformation.BaseTransformation :members: diff --git a/docs/source/api/core/pipeline.rst b/docs/source/api/core/pipeline.rst index 39f44301..05acebc3 100644 --- a/docs/source/api/core/pipeline.rst +++ b/docs/source/api/core/pipeline.rst @@ -1,15 +1,15 @@ Pipelines ========= -.. _pipelines.basedatagenerator: +.. _pipelines.basedatageneration: -BaseDataGenerator ------------------ +BaseDataGeneration +------------------ Bases: :py:class:`BasePipeline.BasePipeline` -.. autoclass:: BaseDataGenerator.BaseDataGenerator +.. autoclass:: BaseDataGeneration.BaseDataGeneration :members: .. _pipelines.basepipeline: @@ -20,18 +20,18 @@ BasePipeline .. autoclass:: BasePipeline.BasePipeline :members: -.. _pipelines.baserunner: +.. _pipelines.baseprediction: -BaseRunner ----------- +BasePrediction +-------------- Bases: :py:class:`BasePipeline.BasePipeline` -.. autoclass:: BaseRunner.BaseRunner +.. autoclass:: BasePrediction.BasePrediction :members: -.. _pipelines.basetrainer: +.. _pipelines.basetraining: BaseTrainer ----------- @@ -39,5 +39,5 @@ BaseTrainer Bases: :py:class:`BasePipeline.BasePipeline` -.. autoclass:: BaseTrainer.BaseTrainer +.. autoclass:: BaseTraining.BaseTraining :members: diff --git a/docs/source/api/core/visualizer.rst b/docs/source/api/core/visualizer.rst index 14e87172..df95549a 100644 --- a/docs/source/api/core/visualizer.rst +++ b/docs/source/api/core/visualizer.rst @@ -1,17 +1,12 @@ Visualizer ========== -.. toctree:: - :hidden: +.. _visualizer.vedofactory: - factories.rst - -.. _visualizer.vedoobjects: - -VedoObjects +VedoFactory ----------- -.. autoclass:: VedoObjects.VedoObjects +.. autoclass:: VedoFactory.VedoFactory :members: .. _visualizer.vedovisualizer: diff --git a/docs/source/api/sofa.rst b/docs/source/api/sofa.rst index 71a4752d..3d2c866c 100644 --- a/docs/source/api/sofa.rst +++ b/docs/source/api/sofa.rst @@ -1,4 +1,4 @@ SOFA ==== -Find **DeepPhysX_Sofa** API `here `_ +Find **DeepPhysX.Sofa** API `here `_ diff --git a/docs/source/api/torch.rst b/docs/source/api/torch.rst index 62141b27..ba810e24 100644 --- a/docs/source/api/torch.rst +++ b/docs/source/api/torch.rst @@ -1,4 +1,4 @@ TORCH ===== -Find **DeepPhysX_Torch** API `here `_. +Find **DeepPhysX.Torch** API `here `_. diff --git a/docs/source/component/dataset.rst b/docs/source/component/dataset.rst index ea122a3e..a751ef41 100644 --- a/docs/source/component/dataset.rst +++ b/docs/source/component/dataset.rst @@ -6,16 +6,16 @@ Dedicated code in :guilabel:`Core/Dataset` module. Behavior -------- -**DeepPhysX** comes with its own *Dataset* management system. -The synthetic data produced in *Environments* is stored as Numpy arrays in a dedicated repository named ``dataset`` in -the training session. +**DeepPhysX** comes with its own *Dataset* management system, using the features from the :SSD:`SSD <>` library. +The synthetic data produced in *Environments* is stored on a *Database* that is in a dedicated repository named +``dataset`` in the training session. Data is stored as partitions: these partitions correspond to the different *Dataset* modes (training data, test data, prediction data) and can be multiple for each data field in order not to exceed the maximum size of the current amount of loaded data. Each partition will have a unique name: ``___.npy``. -.. figure:: ../_static/image/dataset.png - :alt: dataset.png +.. figure:: ../_static/image/database.png + :alt: database.png :align: center :width: 80% @@ -24,16 +24,14 @@ Each partition will have a unique name: ``___ When adding a batch to the *Dataset*, a new partition is created for each data field if the current *Dataset* size exceeds the threshold. The batch is then appended to the *Dataset* for each data field. -Default *Dataset* fields are inputs and outputs, but users can add any data to the *Dataset* from *Environment* using -``additional_in_dataset`` or ``additional_out_dataset`` (see :ref:`dedicated section `). +Default *Dataset* fields are inputs and outputs, but users can define any data field from *Environment* +(see :ref:`dedicated section `). Each field must always be filled at each batch. -A ``dataset.json`` file gathers information about the produced dataset. +A ``dataset.json`` file gathers information about the produced dataset and normalization coefficients if the +normalization is applied. -When loading data from an existing *Dataset*, the repository is loaded first. -If there is a single partition for each field, only those partitions are loaded into the *Dataset*. -Otherwise, a proportional part of each partition will be loaded each time. -Batches of data are accessed in read order (random or not) until the read cursor reaches the end, triggering either the -reloading of a single partition or the loading of the subsequent slices of partitions. +When loading data from an existing *Dataset*, the partitions are loaded and can be accessed randomly or not among the +whole set of partitions. Configuration @@ -49,33 +47,30 @@ Here is a description of attributes related to *Dataset* configuration. :width: 100% :widths: 15 85 - * - ``dataset_class`` - - *Dataset* class from which an instance will be created (*BaseDataset* by default). - - * - ``dataset_dir`` + * - ``existing_dir`` - Path to an existing *Dataset* repository if this repository needs to be loaded or completed. - * - ``partition_size`` + * - ``max_file_size`` - Maximum size (in Gb) of the total *Dataset* object. - * - ``shuffle_dataset`` - - Specify if the loading order is random or not (True by default). - - * - ``use_mode`` + * - ``mode`` - Specify the *Dataset* mode between "Training", "Validation" and "Running". - * - ``normalize_data`` + * - ``normalize`` - If True, normalization parameters are computed from training data and applied to any loaded data. + * - ``shuffle`` + - Specify if the loading order is random or not (True by default). + .. highlight:: python See following example:: # Import DatasetConfig - from DeepPhysX_Core.Dataset.BaseDatasetConfig import BaseDatasetConfig + from DeepPhysX_Core.Database.BaseDatabaseConfig import BaseDatabaseConfig # Create the config - dataset_config = BaseDatasetConfig(partition_size=1, - shuffle_dataset=True, - use_mode='Training', - normalize_data=True) + database_config = BaseDatabaseConfig(max_file_size=1, + shuffle=True, + mode='Training', + normalize=True) diff --git a/docs/source/component/environment.rst b/docs/source/component/environment.rst index 3fd00c30..e7912fb1 100644 --- a/docs/source/component/environment.rst +++ b/docs/source/component/environment.rst @@ -26,6 +26,11 @@ implement its *Environment* regardless of this dependence. This method is automatically called when the *Environment* component is created. + * - ``init_database`` + - Define the fields of the training dataset. + + This method is automatically called when the *Environment* component is created. + * - ``step`` - Describe the transitions between simulation states. @@ -47,17 +52,12 @@ implement its *Environment* regardless of this dependence. This method is automatically called when the *Environment* component is created, right after the create method. - * - ``recv_parameters`` - - In the case of using multiple *Environments* connected to one *TcpIpServer*, each of them can be parameterize - differently. - - These parameters can be set in the *EnvironmentConfig* and are then automatically sent to *Environments* right - after their initialization. + * - ``init_visualization`` + - Define the visualization objects to send to the *Visualizer*. - * - ``send_parameters`` - - On the other side, one might want to send back data from *Environments* to the *EnvironmentConfig*. + A *Factory* is available in the *Environment* to easily design visualization objects. - If the method is implemented, parameters will be sent right after the previous receipt. + This method is automatically called when the *Environment* component is created, right after the init. * - ``check_sample`` - Each step will be followed by a sample checking. @@ -83,32 +83,22 @@ implement its *Environment* regardless of this dependence. :width: 100% :widths: 15 85 + * - ``define_training_data`` + - This method must be used in ``init_database`` to specify the training data fields names and types. + + * - ``define_additional_data`` + - This method must be used in ``init_database`` to specify the additional data fields names and types. + * - ``set_training_data`` - This method defines which data will be considered as training data (data sent to the *NetworkManager* to feed the *Network*). - * - ``set_loss_data`` - - This method allows adding specific data to compute the loss function with numeric values from the simulation. - * - ``set_additional_dataset`` - In addition to the training data, some additional data can be sent directly to the dataset to replay some simulation states. This method adds a new field to the *Dataset* and must be then called at each step. - * - ``set_dataset_sample`` - - This method is already implemented and must not be overwritten by the user. - - When loading existing data, the *DataManager* might need to send *Dataset* samples to the *Environment* to - produce training data. - - These samples are set in the variables ``sample_in`` & ``sample_out``. - - If *Dataset* has additional data fields, these additional fields are set in ``additional_inputs`` & - ``additional_outputs``. - - Each sample can then be processed in the step function. - | **Requests** | The *Environment* is also able to perform some requests. These requests are sent either directly to the *EnvironmentManager* or through a *TcpIpServer*. @@ -117,6 +107,12 @@ implement its *Environment* regardless of this dependence. :width: 100% :widths: 15 85 + * - ``save_parameters`` + - Save a set of parameters in the Database. + + * - ``load_parameters`` + - Load a set of parameters from the Database. + * - ``get_prediction`` - Depending on the application, a prediction of the *Network* can be useful before the end of a step. @@ -125,23 +121,11 @@ implement its *Environment* regardless of this dependence. The training data must obviously be set before triggering this request. - * - ``send_visualization`` - - The framework comes with an integrated *Visualizer* tool to render some components of the simulation. - - Parts of the simulation to render must be defined when creating or when initializing the *Environment*. - - An *Environment* has a visualization factory to easily create visualization data from templates with - ``addObject`` method. - - User only has to set the object type and to fill the required fields to init this type of object in the - *Visualizer* (see :ref:`dedicated section `). - * - ``update_visualization`` - If a *Visualizer* was created, it must be manually updated at each step by sending the updated state of the simulation. - The factory allows to easily create updated visualization data from objects ids with ``updateObject_dict`` - method. + The factory allows to easily create updated visualization data from objects ids with ``update_obj`` methods. User only has to fill the object id (same as initialization order) and the required fields (detailed for each object in :ref:`dedicated section `). @@ -151,8 +135,8 @@ Configuration ------------- Using an *Environment* in one of the **DeepPhysX** *Pipeline* always requires an *EnvironmentConfig*. -This component’s role is both to bring together all the options for configuring an *Environment* and to either create -an instance of a single *Environment* or launch a *TcpIpServer* with several *TcpIpClients*. +The role of this component is both to bring together all the options for configuring an *Environment* and to create an +instance of a single *Environment* or launch a *TcpIpServer* with several *TcpIpClients*. In the first case, the single *Environment* will simply be created within the ``create_environment`` method, while in the other case, the ``create_server`` method will simply create and launch a *TcpIpServer* and then start several subprocesses, each using the ``launcherBaseEnvironment.py`` script to create and launch an *Environment* as a *Client*. @@ -169,19 +153,19 @@ subprocesses, each using the ``launcherBaseEnvironment.py`` script to create and The attribute requires the class and not an instance, as it will be automatically created as explained above. - * - ``visualizer`` - - A visualization tool is provided, which renders the specified parts of each *Environment*. - - If no *Visualizer* is provided, the pipeline will run without any render window. - * - ``simulations_per_step`` - The number of iterations to compute in the *Environment* at each time step. An *Environment* will compute one iteration by default. - * - ``use_prediction_in_environment`` - - Each *Network* prediction will be automatically applied in the *Environment* if this flag is set to True - (set to False by default). + * - ``visualizer`` + - A visualization tool is provided, which renders the specified parts of each *Environment*. + + If no *Visualizer* is provided, the pipeline will run without any render window. + + * - ``env_kwargs`` + - *Environments* can receive additional parameters within this dictionary if they need to be parameterized + differently. | **Data parameters** | Here is a description of attributes related to sample generation. @@ -190,7 +174,7 @@ subprocesses, each using the ``launcherBaseEnvironment.py`` script to create and :width: 100% :widths: 15 85 - * - ``always_create_data`` + * - ``always_produce`` - This flag is useful for the training *Pipeline*. If False (by default), the *DataManager* requests batches to the *Environment* during the first epoch and @@ -198,26 +182,13 @@ subprocesses, each using the ``launcherBaseEnvironment.py`` script to create and If set to True, the *DataManager* requests new batches to the *Environment* during the whole training session. - * - ``screenshot_sample_rate`` - - This option is only available if a *Visualizer* is defined. - - In addition to *Dataset* partitions, samples can also be saved as screenshots so representative ones can be - easily found. - - A screenshot of the viewer will be taken every x samples (set to 0 by default). + * - ``load_samples`` + - If True, the dataset will always be used in the environment. - * - ``record_wrong_samples`` - - By default, only the good samples are stored in the *Dataset* (sorted by check_sample, see the section above). + * - ``only_first_epoch`` + - If True, data will always be created from *Environment*. - If this flag is set to True, the wrong samples will also be saved to dedicated partitions. - - * - ``max_wrong_samples_per_step`` - - If an *Environment* produces too many wrong samples, it may be configured incorrectly. - - To avoid an unnecessary extended data generation, a threshold can be set so that the session can be stopped - early. - - This happens when too many wrong samples are produced to fill a single batch. + If False, data will be created from the *Environment* during the first epoch and then re-used from the *Dataset*. | **TcpIP parameters** | Here is a description of attributes related to the *Client* configuration. @@ -238,27 +209,11 @@ subprocesses, each using the ``launcherBaseEnvironment.py`` script to create and The default value is “localhost” to host the *Server* and *Clients* locally. - * - ``port`` - - TCP port’s number through which *TcpIpObjects* will communicate (10000 by default). - - * - ``environment_file`` - - When launching an *Environment* as a *Client*, the *EnvironmentConfig* starts a subprocess involving that - *Environment*. - - To do this, the launcher will need the script path in which the *Environment* is defined. - - This script is in most cases automatically detected, but users may need to enter the path to their python file. - * - ``number_of_thread`` - The number of *Environments* to launch simultaneously if the flag ``as_tcp_ip_client`` is True. - * - ``max_client_connection`` - - The maximum number of *Client* connections allowed by a *Server*. - - * - ``param_dict`` - - *Environments* can receive additional parameters if they need to be parameterized differently. - - These parameters are sent in the form of dictionaries by the *Server* when creating the *Environment*. + * - ``port`` + - TCP port’s number through which *TcpIpObjects* will communicate (10000 by default). .. highlight:: python @@ -285,7 +240,7 @@ Client-Server Architecture The :guilabel:`Core/AsyncSocket` module defines a **Client-Server architecture** where a *TcpIpServer* communicates with several *TcpIpClients* using a **TcpIp protocol**. -.. figure:: ../_static/image/environment_tcpip.png +.. figure:: ../_static/image/environment_tcp_ip.png :alt: environment_tcpip.png :width: 80% :align: center @@ -309,39 +264,3 @@ The data is sent as a custom bytes message converted with a *BytesConverter*, wh NumPy arrays. On top of these low level data exchange methods are built higher level protocols to send labeled data, labeled dictionaries and commands. - -A list of available commands is defined. *TcpIpServer* and *TcpIpClients* have then their own action implementations -to perform when receiving a command: - -* A *TcpIpClient* defines the following actions to perform on commands: - - .. list-table:: - :width: 90% - :widths: 15 85 - - * - ``exit`` - - Set the closing flag to True to terminate the communication loop. - - * - ``prediction`` - - Receive the prediction sent by the *TcpIpServer* and apply it in the *Environment*. - - * - ``sample`` - - When using data from *Dataset*, the sample is received and defined in the *Environment* on this command - - * - ``step`` - - Trigger a simulation step to produce data. - - Data should be sent to the *TcpIpServer* when the produced sample is identified as usable by sample - checking. - -* A *TcpIpServer* defines the following actions to perform on commands: - - .. list-table:: - :width: 90% - :widths: 15 85 - - * - ``prediction`` - - Receive data to feed the *Network*, then send back the prediction to the same *TcpIpClient*. - - * - ``visualization`` - - Receive initial or updated visualization data, then call the *Visualizer* update. diff --git a/docs/source/component/network.rst b/docs/source/component/network.rst index 114e32f5..11ef7c95 100644 --- a/docs/source/component/network.rst +++ b/docs/source/component/network.rst @@ -23,6 +23,9 @@ Even if a *BaseNetwork* is not usable, any **DeepPhysX** AI package provides the * - ``__init__`` - Define the *Network* architecture. + * - ``predict`` + - Define the data fields used to compute a prediction. + * - ``forward`` - Define the forward pass of the *Network*. @@ -57,16 +60,23 @@ Even if a *BaseNetwork* is not usable, any **DeepPhysX** AI package provides the * - ``nb_parameters`` - Return the number of parameters in the architecture. - * - ``transform_from_numpy`` + * - ``numpy_to_tensor`` - Convert a Numpy array to a tensor with the compatible type. Received data from Core will always be Numpy arrays. - * - ``transform_to_numpy`` + * - ``tensor_to_numpy`` - Convert a tensor to a Numpy array. Data provided to Core must be converted to Numpy arrays. +| **Data fields** +| The training data is given to the *Network* as a dictionary of tensors. + By default, the data used for inference has a single "input" field, the data produced by the *Network* has a single + "prediction" field, the data used for optimization has a single "ground_truth" field. + If another field should be defined, the *Network* class should specify them in the corresponding ``self.net_fields``, + ``self.pred_fields`` & ``self.opt_fields`` variables. + .. _network-optimization: Optimization Implementation @@ -126,14 +136,14 @@ Users are then free to define their own tensor transformations with the followin :widths: 15 85 * - ``transform_before_prediction`` - - Apply a tensor transformation to the input data before *Network* forward pass. + - Apply a tensor transformation to the *Network* data (before a prediction). * - ``transform_before_loss`` - - Apply a tensor transformation to the ground truth data and / or the *Network* output before the loss - computation. + - Apply a tensor transformation to the ground truth data and / or the prediction data (after the prediction, + before the loss computation). * - ``transform_before_apply`` - - Apply a tensor transformation to the *Network* prediction before sending it to the *Environment*. + - Apply a tensor transformation to the prediction data (before sending it to the *Environment*). Configurations diff --git a/docs/source/component/pipelines.rst b/docs/source/component/pipelines.rst index f61e3099..c84f34ea 100644 --- a/docs/source/component/pipelines.rst +++ b/docs/source/component/pipelines.rst @@ -8,9 +8,9 @@ General Policy Several *Pipelines* are available with **DeepPhysX**, allowing the user to: - * **Generate** synthetic data from simulations → ``DataGenerator`` - * **Train** artificial neural networks with synthetic data → ``Trainer`` - * Use the **predictions** of trained networks inside a simulation → ``Runner`` + * **Generate** synthetic data from simulations → ``DataGeneration`` + * **Train** artificial neural networks with synthetic data → ``Training`` + * Use the **predictions** of trained networks inside a simulation → ``Prediction`` A *Pipeline* is always associated with a :ref:`working session `, whether it already exists or whether it is automatically created when the *Pipeline* is launched. @@ -27,8 +27,8 @@ Once these *Configurations* are defined, the *Pipeline* can be created and launc Pipeline - Data generation -------------------------- -The *DataGenerator* will only involve an *Environment* and a *Dataset*, so this *Pipeline* requires the corresponding -*Configurations* +The *DataGeneration* will only involve an *Environment* and a *Dataset*, so this *Pipeline* requires the corresponding +*Configurations*. As the purpose of this *Pipeline* is only to create synthetic data, the working session will always be created at the same time. @@ -48,8 +48,8 @@ Furthermore, users have to define which data to save and how much : See following example:: - # Import BaseDataGenerator and Config objects - from DeepPhysX_Core.Pipelines.BaseDataGenerator import BaseDataGenerator + # Import BaseDataGeneration and Config objects + from DeepPhysX.Core.Pipelines.BaseDataGeneration import BaseDataGeneration ... # Define configs @@ -57,11 +57,12 @@ See following example:: environment_config = ... # Create the pipeline - data_generator = BaseDataGenerator(session_name='sessions/my_data_generation', - dataset_config=dataset_config, - environment_config=environment_config, - nb_batches=500, - batch_size=16) + data_generator = BaseDataGeneration(session_dir='sessions', + session_name='my_data_generation', + dataset_config=dataset_config, + environment_config=environment_config, + batch_nb=500, + batch_size=16) # Launch the pipeline data_generator.execute() @@ -70,12 +71,12 @@ See following example:: Pipeline - Training ------------------- -The *Trainer* can involve an *Environment*, a *Dataset* and a *Network*, so this *Pipeline* might require the +The *Training* can involve an *Environment*, a *Dataset* and a *Network*, so this *Pipeline* might require the corresponding *Configurations*. There are several ways to use this pipeline: **Training a Network from scratch** - To train a *Network* from scratch, the *Trainer* requires the whole set of *Configurations*. + To train a *Network* from scratch, the *Training* requires the whole set of *Configurations*. A new working session will be created, whose name can be set as a parameter. **Training a Network with an existing Dataset** @@ -88,7 +89,7 @@ There are several ways to use this pipeline: **Training a Network from an existing Network state** Training from an existing *Network* state can be done both in an existing session or in a new session. - If you want to work in the same session, you have to configure the *Trainer* to do so, otherwise a new working + If you want to work in the same session, you have to configure the *Training* to do so, otherwise a new working session will be automatically created. In the same session, a new set of trained parameters will be added in the ``network`` repository, either trained with data from an external *Dataset* (whose path must be provided) or with data from the *Environment* (whose @@ -109,8 +110,8 @@ The last parameters to set in the *Trainer* are: See following example:: - # Import BaseTrainer and Config objects - from DeepPhysX_Core.Pipelines.BaseTrainer import BaseTrainer + # Import BaseTraining and Config objects + from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining ... # Define configs @@ -119,13 +120,14 @@ See following example:: network_config = ... # Create the pipeline - trainer = BaseTrainer(session_name='sessions/my_training', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_epochs=100, - nb_batches=500, - batch_size=16) + trainer = BaseTraining(session_dir='sessions', + session_name='my_training', + dataset_config=dataset_config, + environment_config=env_config, + network_config=net_config, + epoch_nb=100, + batch_nb=500, + batch_size=16) # Launch the pipeline trainer.execute() @@ -134,25 +136,25 @@ See following example:: Pipeline - Prediction --------------------- -The *Runner* always requires a *Network* to compute predictions and an *Environment* to apply them, so this *Pipeline* -will always require the corresponding *Configurations*. +The *Prediction* always requires a *Network* to compute predictions and an *Environment* to apply them, so this +*Pipeline* will always require the corresponding *Configurations*. -This *Pipeline* always works with an existing working session, no new sessions can be created within a *Runner*. +This *Pipeline* always works with an existing working session, no new sessions can be created within a *Prediction*. The path to the session is therefore required, assuming that it contains a trained *Network*. -The *Runner* can either run a specified **number of steps** or run an **infinite loop**. +The *Prediction* can either run a specified **number of steps** or run an **infinite loop**. A *Dataset* configuration can be provided. -In this case, the *Runner* can record input or / and output data. +In this case, the *Prediction* can record prediction data. Each sample computed during the prediction phase will then be added to the *Dataset* in dedicated partitions. -With a *Dataset*, the *Runner* can also load its data to **replay** stored samples. +With a *Dataset*, the *Prediction* can also load its data to **replay** stored samples. .. highlight:: python See following example:: - # Import BaseRunner and Config objects - from DeepPhysX_Core.Pipelines.BaseRunner import BaseTrainer + # Import BasePrediction and Config objects + from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction ... # Define configs @@ -161,11 +163,12 @@ See following example:: network_config = ... # Create the pipeline - runner = BaseRunner(session_dir='sessions/my_training', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=-1) + runner = BasePrediction(session_dir='sessions', + session_name='my_training', + dataset_config=dataset_config, + environment_config=env_config, + network_config=net_config, + step_nb=-1) # Launch the pipeline runner.execute() diff --git a/docs/source/component/visualizer.rst b/docs/source/component/visualizer.rst index b30d7234..577ac3c9 100644 --- a/docs/source/component/visualizer.rst +++ b/docs/source/component/visualizer.rst @@ -9,7 +9,7 @@ How to use ---------- **DeepPhysX** provides a visualization tool written with :Vedo:`Vedo <>` (a Python library based on Numpy and VTK) -called *VedoVisualizer*. +called *VedoVisualizer*, using the implementation form the :SSD:`SSD <>` library. This *Visualizer* brings several advantages: * Users can add any component of the simulation in the *Visualizer*; @@ -17,272 +17,20 @@ This *Visualizer* brings several advantages: * Parallel running *Environments* are rendered in the same window with sub-windows; * A *Factory* is created with each *Environment* so that users can access templates to define visualization data. -Objects are created using the ``add_object`` method of the *Factory* in the *Environment*. -This method requires the object type name and a dictionary containing the required fields detailed above (common fields -and object-specific fields). -These objects must be defined in the ``send_visualization`` method of the *Environment*, which must return the objects -dictionary of the *Factory*. +Objects are created using the ``add_`` methods of the *Factory* in the *Environment*. +These methods require the fields detailed above (common fields and object-specific fields). +These objects must be added in the ``init_visualization`` method of the *Environment*, which is automatically called to +create the objects within the *Factory*. -Objects are updated using the ``update_object_dict`` method of the *Factory* in the *Environment*. -This method requires the object index (indices follow the order of creation) and a dictionary containing the updated -data fields. +Objects are updated using the ``update_`` methods of the *Factory* in the *Environment*. +These methods require the object index (indices follow the order of creation) and a the updated data fields. The *Factory* will only use templates to create an updated objects dictionary which must be sent with the request ``update_visualization`` to update the view. -| **General parameters** -| Visual objects share default data fields that could also be filled at init and are all optional: - -.. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``c`` - - Optional - - Unnecessary - - Opacity of the object between 0 and 1. - - * - ``alpha`` - - Optional - - Unnecessary - - Marker object. - - * - ``at`` - - Optional - - Unnecessary - - Sub-window in which the object will be rendered. - - Set to -1 by default, meaning a new window is created for the object. - - Advise: set to ``self.instance_id`` to gather objects from an *Environment* in the same sub-window. - - * - ``colormap`` - - Optional - - Unnecessary - - Name of color palette that samples a continuous function between two end colors. - - * - ``scalar_field`` - - Optional - - Unnecessary - - List of scalar values to set individual points or cell color. - - * - ``scalar_field_name`` - - Optional - - Unnecessary - - Name of the scalar field. | **Visual Objects Parameters** -| A list of templates are available in the *Factory* to initialize and update a list of objects. - Here is a description of available objects and the required data fields: - -* Create a :VedoObject:`Mesh `: - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``positions`` - - **Required** - - **Required** - - List of vertices. - Updated position vector must always have the same size. - - * - ``cells`` - - **Required** - - Unnecessary - - List of connections between vertices. - - * - ``computeNormals`` - - Optional - - Unnecessary - - Compute cells and points normals at creation. - -* Create a :VedoObject:`Point Cloud `: - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``positions`` - - **Required** - - **Required** - - List of vertices. - Updated position vector must always have the same size. - - * - ``r`` - - Optional - - Optional - - Radius of points. - -* Create a :VedoObject:`Marker ` (single point with associated symbol): - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``positions`` - - **Required** - - **Required** - - Position of the Marker. - - * - ``symbol`` - - **Required** - - Unnecessary - - Associated symbol. - - * - ``s`` - - Optional - - Unnecessary - - Radius of symbol. - - * - ``filled`` - - Optional - - Unnecessary - - Fill the shape or only draw outline. - -* Create a :VedoObject:`Glyph ` (point cloud with oriented markers): - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``positions`` - - **Required** - - **Required** - - Position of the Markers. - - * - ``glyphObj`` - - **Required** - - Unnecessary - - Marker object. - - * - ``orientationArray`` - - **Required** - - Unnecessary - - List of orientation vectors. - - * - ``scaleByScalar`` - - Optional - - Unnecessary - - Glyph is scaled by the scalar field. - - * - ``scaleByVectorSize`` - - Optional - - Unnecessary - - Glyph is scaled by the size of the orientation vectors. - - * - ``scaleByVectorComponents`` - - Optional - - Unnecessary - - Glyph is scaled by the components of the orientation vectors. - - * - ``colorByScalar`` - - Optional - - Unnecessary - - Glyph is colored based on the colormap and the scalar field. - - * - ``colorByVectorSize`` - - Optional - - Unnecessary - - Glyph is colored based on the size of the orientation vectors. - - * - ``tol`` - - Optional - - Unnecessary - - Minimum distance between two Glyphs. - -* Create :VedoObject:`3D Arrows `: - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``positions`` - - **Required** - - **Required** - - Start points of the arrows. - - * - ``vectors`` - - **Required** - - **Required** - - Vector that must represent the arrows. - - * - ``res`` - - Optional - - Unnecessary - - Arrows visual resolution. - -* Change window parameters - - .. list-table:: - :width: 95% - :widths: 21 11 11 57 - :header-rows: 1 - - * - Field - - Init - - Update - - Description - - * - ``objects_id`` - - **Required** - - Unnecessary - - Indices of objects to set in this particular window. - - * - ``title`` - - Optional - - Unnecessary - - Title of the window. - - * - ``axes`` - - Optional - - Unnecessary - - Type of axes to show. - - * - ``sharecam`` - - Optional - - Unnecessary - - If True (default), all subwindows will share the same camera parameters. - - * - ``interactive`` - - Optional - - Unnecessary - - If True (default), the window will be interactive. +| All the available objects and their parameters are listed on the + :SSDd:`SSD documentation `. Configuration @@ -300,7 +48,7 @@ See following example:: # Import EnvironmentConfig and Visualizer from DeepPhysX_Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig - from DeepPhysX_Core.Visualizer.VedoVisualizer import VedoVisualizer + from DeepPhysX_Core.Visualization.VedoVisualizer import VedoVisualizer # Create the config env_config = BaseEnvironmentConfig(environment_class=MyEnvironment, diff --git a/docs/source/conf.py b/docs/source/conf.py index 1de1c421..ae492141 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,8 +14,8 @@ import sys # DeepPhysX root -root = abspath(join(abspath(__file__), pardir, pardir, pardir, 'src')) -all_modules = ['AsyncSocket', 'Dataset', 'Environment', 'Manager', 'Network', 'Pipelines', 'Visualizer'] +root = abspath(join(abspath(__file__), pardir, pardir, pardir, 'src', 'Core')) +all_modules = ['AsyncSocket', 'Database', 'Environment', 'Manager', 'Network', 'Pipelines', 'Visualization'] # Import all modules sys.path.append(root) @@ -73,14 +73,16 @@ html_static_path = ['_static'] html_css_files = ['theme.css'] -extlinks = {'Caribou': ('https://caribou.readthedocs.io/', None), - 'CaribouI': ('https://caribou.readthedocs.io/en/latest/Building.html#', None), - 'Numpy': ('https://numpy.org/', None), - 'PyTorch': ('https://pytorch.org/', None), - 'SOFA': ('https://www.sofa-framework.org/%s', None), - 'SOFAI': ('https://www.sofa-framework.org/community/doc/getting-started/build/linux/', None), - 'SP3': ('https://sofapython3.readthedocs.io/en/latest/', None), - 'SP3I': ('https://sofapython3.readthedocs.io/en/latest/menu/Compilation.html', None), - 'Tensorboard': ('https://www.tensorflow.org/tensorboard/', None), - 'Vedo': ('https://vedo.embl.es/', None), - 'VedoObject': ('https://vedo.embl.es/autodocs/content/vedo/%s', '%s')} +extlinks = {'Caribou': ('https://caribou.readthedocs.io/%s', '%s'), + 'CaribouI': ('https://caribou.readthedocs.io/en/latest/Building.html#/%s', '%s'), + 'Numpy': ('https://numpy.org/%s', '%s'), + 'PyTorch': ('https://pytorch.org/%s', '%s'), + 'SOFA': ('https://www.sofa-framework.org/%s', '%s'), + 'SOFAI': ('https://www.sofa-framework.org/community/doc/getting-started/build/linux/%s', '%s'), + 'SP3': ('https://sofapython3.readthedocs.io/en/latest/%s', '%s'), + 'SP3I': ('https://sofapython3.readthedocs.io/en/latest/menu/Compilation.html/%s', '%s'), + 'Tensorboard': ('https://www.tensorflow.org/tensorboard/%s', '%s'), + 'Vedo': ('https://vedo.embl.es/%s', '%s'), + 'VedoObject': ('https://vedo.embl.es/autodocs/content/vedo/%s', '%s'), + 'SSD': ('https://github.com/RobinEnjalbert/SimulationSimpleDatabase/%s', '%s'), + 'SSDd': ('https://simulationsimpledatabase.readthedocs.io/en/latest/%s', '%s')} diff --git a/docs/source/index.rst b/docs/source/index.rst index 2329b202..8429c2b8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,6 +10,8 @@ with **learning algorithms**. **DeepPhysX** is mainly designed for :SOFA:`SOFA <>` and :PyTorch:`PyTorch <>` frameworks, but other simulation and AI frameworks can also be used. +The project is closely linked to the :SSD:`SSD <>` external Python library. + Let's get started ----------------- diff --git a/docs/source/presentation/about.rst b/docs/source/presentation/about.rst index 1aca8eb1..105d1bb4 100644 --- a/docs/source/presentation/about.rst +++ b/docs/source/presentation/about.rst @@ -70,5 +70,5 @@ typically contains the following **tree structure**: :width: 80% :align: center - * - .. image:: ../_static/image/about_working_session.png + * - .. image:: ../_static/image/about_tree.png :alt: about_working_session.png diff --git a/docs/source/presentation/install.rst b/docs/source/presentation/install.rst index 6f9a757a..516f7183 100644 --- a/docs/source/presentation/install.rst +++ b/docs/source/presentation/install.rst @@ -20,23 +20,25 @@ Prerequisites .. table:: :widths: 20 20 10 30 - +-----------------------------+-------------------------------+--------------+-------------------------------------+ - | **Package** | **Dependency** | **Type** | **Install** | - +=============================+===============================+==============+=====================================+ - | :guilabel:`DeepPhysX_Core` | :Numpy:`Numpy <>` | **Required** | ``pip install numpy`` | - | +-------------------------------+--------------+-------------------------------------+ - | | :Vedo:`Vedo <>` | **Required** | ``pip install vedo`` | - | +-------------------------------+--------------+-------------------------------------+ - | | :Tensorboard:`Tensorboard <>` | **Required** | ``pip install tensorboard`` | - +-----------------------------+-------------------------------+--------------+-------------------------------------+ - | :guilabel:`DeepPhysX_Sofa` | :SOFA:`SOFA Framework <>` | **Required** | :SOFAI:`Follow instructions <>` | - | +-------------------------------+--------------+-------------------------------------+ - | | :SP3:`SofaPython3 <>` | **Required** | :SP3I:`Follow instructions <>` | - | +-------------------------------+--------------+-------------------------------------+ - | | :Caribou:`Caribou <>` | Optional | :CaribouI:`Follow instructions <>` | - +-----------------------------+-------------------------------+--------------+-------------------------------------+ - | :guilabel:`DeepPhysX_Torch` | :PyTorch:`PyTorch <>` | **Required** | ``pip install torch`` | - +-----------------------------+-------------------------------+--------------+-------------------------------------+ + +-----------------------------+-------------------------------+--------------+------------------------------------------+ + | **Package** | **Dependency** | **Type** | **Install** | + +=============================+===============================+==============+==========================================+ + | :guilabel:`DeepPhysX_Core` | :Numpy:`Numpy <>` | **Required** | ``pip install numpy`` | + | +-------------------------------+--------------+------------------------------------------+ + | | :Vedo:`Vedo <>` | **Required** | ``pip install vedo`` | + | +-------------------------------+--------------+------------------------------------------+ + | | :Tensorboard:`Tensorboard <>` | **Required** | ``pip install tensorboard`` | + | +-------------------------------+--------------+------------------------------------------+ + | | :SSD:`SSD <>` | **Required** | ``pip install SimulationSimpleDatabase`` | + +-----------------------------+-------------------------------+--------------+------------------------------------------+ + | :guilabel:`DeepPhysX_Sofa` | :SOFA:`SOFA Framework <>` | **Required** | :SOFAI:`Follow instructions <>` | + | +-------------------------------+--------------+------------------------------------------+ + | | :SP3:`SofaPython3 <>` | **Required** | :SP3I:`Follow instructions <>` | + | +-------------------------------+--------------+------------------------------------------+ + | | :Caribou:`Caribou <>` | Optional | :CaribouI:`Follow instructions <>` | + +-----------------------------+-------------------------------+--------------+------------------------------------------+ + | :guilabel:`DeepPhysX_Torch` | :PyTorch:`PyTorch <>` | **Required** | ``pip install torch`` | + +-----------------------------+-------------------------------+--------------+------------------------------------------+ .. note:: :guilabel:`DeepPhysX.Sofa` has a dependency to :Caribou:`Caribou <>` to run the demo scripts from @@ -53,17 +55,17 @@ They can easily be installed with ``pip``: .. code-block:: bash - pip3 install DeepPhysX - pip3 install DeepPhysX.Sofa - pip3 install DeepPhysX.Torch + $ pip3 install DeepPhysX + $ pip3 install DeepPhysX.Sofa + $ pip3 install DeepPhysX.Torch Then, you should be able to run: .. code-block:: bash - pip3 show DeepPhysX - pip3 show DeepPhysX.Sofa - pip3 show DeepPhysX.Torch + $ pip3 show DeepPhysX + $ pip3 show DeepPhysX.Sofa + $ pip3 show DeepPhysX.Torch .. code-block:: python @@ -92,51 +94,48 @@ Start by cloning the **DeepPhysX** source code from its Github repository in a d $ mkdir DeepPhysX $ cd DeepPhysX $ git clone https://github.com/mimesis-inria/DeepPhysX.git Core - $ cd Core -Specify which packages to install by running the configuration script. -This way, all the packages are gathered in a single installation. +Then, you can add compatibility layers to your **DeepPhysX** environment and install packages: -.. code-block:: bash +* **Option 1 (recommended):** run one of the ``setup_.py`` scripts that handle the installation of all packages - $ python3 config.py - > Available AI packages : ['Torch'] - > >> Installing package Torch (y/n): yes - > - > Available Simulation packages : ['Sofa'] - > >> Installing package Sofa (y/n): yes - > - > Applying following configuration: - > * DeepPhysX.Core: True (default) - > * DeepPhysX.Torch: True - > * DeepPhysX.Sofa: True - > Confirm (y/n): yes - > Configuration saved in 'config.json' + * Use ``setup_user.py`` to install and manage packages with ``pip`` as non-editable. + .. code-block:: bash -.. note:: - Configuration script will **automatically clone** missing packages. + $ python3 setup_user.py -Finally, install the defined packages: + * Use ``setup_dev.py`` to link packages in the site-packages. -* by using ``pip`` to install and manage them as non-editable + .. code-block:: bash - .. code-block:: bash + $ python3 setup_dev.py set - $ pip3 install . + .. note:: + Both scripts will asks the packages to install and will **automatically clone** missing packages. -* by running ``dev.py`` to link them as editable in the site-packages +* **Option 2:** clone the corresponding Github repositories in the created ``DeepPhysX`` directory, then install + packages manually. .. code-block:: bash - $ python3 dev.py set + # Clone compatibility layers + $ git clone https://github.com/mimesis-inria/DeepPhysX.Sofa.git Sofa + $ git clone https://github.com/mimesis-inria/DeepPhysX.Torch.git Torch -Then, you should be able to run: + # Install packages manually + $ cd Core ; pip3 install . + $ cd ../Sofa ; pip3 install . + $ cd ../Torch ; pip3 install . + +Finally, you should be able to run: .. code-block:: bash # If installed with pip - $ pip show DeepPhysX + $ pip3 show DeepPhysX + $ pip3 show DeepPhysX.Sofa + $ pip3 show DeepPhysX.Torch .. code-block:: python diff --git a/docs/source/presentation/overview.rst b/docs/source/presentation/overview.rst index 87a920fe..8d1ac8de 100644 --- a/docs/source/presentation/overview.rst +++ b/docs/source/presentation/overview.rst @@ -31,7 +31,7 @@ This package is named :guilabel:`DeepPhysX.Core`. .. admonition:: Dependencies - NumPy, Tensorboard, Vedo + NumPy, Tensorboard, Vedo, SimulationSimpleDatabase Simulation """""""""" @@ -70,7 +70,7 @@ Users might use one of the provided *Pipelines* for their **data generation**, t These *Pipelines* trigger a **loop** which defines the number of samples to produce, the number of epochs to perform during a training session or the number of steps of prediction. -.. figure:: ../_static/image/overview_architecture.png +.. figure:: ../_static/image/overview_components.png :alt: overview_architecture.png :width: 80% :align: center @@ -81,12 +81,12 @@ The *Pipeline* will involve several components (data producers and data consumer communicate with their *Manager* first. A main *Manager* will provide the *Pipeline* an intermediary with all the existing *Managers*: -:``DatasetManager``: It will manage the *Dataset* component to create **storage** partitions, to fill these partitions - with the synthetic training data produced by the *Environment* and to **reload** an existing *Dataset* for training or +:``DatabaseManager``: It will manage the *Database* component to create **storage** partitions, to fill these partitions + with the synthetic training data produced by the *Environment* and to **reload** an existing *Database* for training or prediction sessions. .. note:: - If training and data generation are done simultaneously (by default for the training *Pipeline*), the *Dataset* + If training and data generation are done simultaneously (by default for the training *Pipeline*), the *Database* can be built only during the first epoch and then reloaded for the remaining epochs. :``EnvironmentManager``: It will manage the *Environment* (the numerical simulation) component to **create** it, to @@ -99,10 +99,10 @@ A main *Manager* will provide the *Pipeline* an intermediary with all the existi :ref:`dedicated section `). .. note:: - The two above *Managers* are managed by the ``DataManager`` since both the *Environment* and the *Dataset* + The two above *Managers* are managed by the ``DataManager`` since both the *Environment* and the *Database* components provide training data to the *Network*. This ``DataManager`` is the one who decides if data should be requested from the *Environment* or from the - *Dataset* depending on the current state of the *Pipeline* and on the components configurations. + *Database* depending on the current state of the *Pipeline* and on the components configurations. :``NetworkManager``: It will manage several objects to **train** or **exploit** your *Network*: @@ -126,16 +126,6 @@ A main *Manager* will provide the *Pipeline* an intermediary with all the existi mean and the variance of this loss value per batch and per epoch), but other custom fields can be added and filled as well. -:``VisualizerManager``: It will manage the *Visualizer* which **initialize** and **update** visualization data. - Then, it updates the **render** of the simulated objects defined in the visualization data. - *Factories* are provided to easily **template** visualization data for a wide variety of objects (meshes, point clouds, - etc.). - - .. note:: - It must be specified in an *Environment* which objects to add in the *Visualizer* and when they must be updated. - In the case where several *Environments* are running in parallel, the rendering windows will be split in several - sub-windows to gather all the renderings. - .. warning:: It is not possible to use the default *Network* and *Environment* provided in the :core:`CORE` package, since they are not implemented at all. diff --git a/examples/demos/Armadillo/FC/Environment/Armadillo.py b/examples/demos/Armadillo/FC/Environment/Armadillo.py index 0140e923..e5fd74ca 100644 --- a/examples/demos/Armadillo/FC/Environment/Armadillo.py +++ b/examples/demos/Armadillo/FC/Environment/Armadillo.py @@ -10,10 +10,10 @@ import os import sys -import numpy as np +from numpy import ndarray, array, concatenate, arange, zeros, reshape +from numpy.random import randint, uniform from vedo import Mesh from math import pow -from time import sleep # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -28,20 +28,14 @@ class Armadillo(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None @@ -53,12 +47,12 @@ def __init__(self, # Force fields self.forces = [] self.areas = [] - self.compute_sample = True # Force pattern step = 0.05 - self.amplitudes = np.concatenate((np.arange(0, 1, step), - np.arange(1, 0, -step))) + self.amplitudes = concatenate((arange(0, 1, step), + arange(1, 0, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.idx_zone = 0 @@ -73,11 +67,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): + def init_database(self): - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + # Define the fields of the Training database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) def create(self): @@ -97,34 +90,26 @@ def create(self): if sphere(pts, p_forces.centers[zone]) <= pow(p_forces.radius[zone], 2): self.areas[-1].append(i) # Init force value - self.forces.append(np.zeros(3, )) + self.forces.append(zeros(3, )) - def send_visualization(self): + def init_visualization(self): # Mesh representing detailed Armadillo (object will have id = 0) - self.factory.add_object(object_type="Mesh", - data_dict={"positions": self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - "c": "orange", - "at": self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': [0, 0, 0], - 'vectors': [0, 0, 0], - 'c': 'green', - 'at': self.instance_id}) - + self.factory.add_arrows(positions=array([0., 0., 0.]), + vectors=array([0., 0., 0.]), + c='green', + at=self.instance_id) # Points representing the grid (object will have id = 2) - self.factory.add_object(object_type='Points', - data_dict={'positions': self.sparse_grid.points(), - 'r': 1., - 'c': 'black', - 'at': self.instance_id}) - - # Return the visualization data - return self.factory.objects_dict + self.factory.add_points(positions=self.sparse_grid.points(), + point_size=1, + c='black', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -134,38 +119,31 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - # Generate a new force - if self.idx_amplitude == 0: - self.idx_zone = np.random.randint(0, len(self.forces)) - zone = p_forces.zones[self.idx_zone] - self.force_value = np.random.uniform(low=-1, high=1, size=(3,)) * p_forces.amplitude[zone] - - # Update current force amplitude - self.forces[self.idx_zone] = self.force_value * self.amplitudes[self.idx_amplitude] + # Generate a new force + if self.idx_amplitude == 0: + self.idx_zone = randint(0, len(self.forces)) + zone = p_forces.zones[self.idx_zone] + self.force_value = uniform(low=-1, high=1, size=(3,)) * p_forces.amplitude[zone] - # Update force amplitude index - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) + # Update current force amplitude + self.forces[self.idx_zone] = self.force_value * self.amplitudes[self.idx_amplitude] - # Create input array - F = np.zeros(self.input_size) - F[self.areas[self.idx_zone]] = self.forces[self.idx_zone] + # Update force amplitude index + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.input_size) if self.sample_in is None else self.sample_in + # Create input array + F = zeros(self.input_size) + F[self.areas[self.idx_zone]] = self.forces[self.idx_zone] # Set training data self.F = F - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.output_size)) + self.set_training_data(input=F.copy(), + ground_truth=zeros(self.output_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.output_size) + U = reshape(prediction['prediction'], self.output_size) self.update_visual(U) def update_visual(self, U): @@ -176,20 +154,20 @@ def update_visual(self, U): mesh_coarse_position = self.mapping_coarse.apply(updated_position) # Update surface mesh - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': mesh_position}) + self.factory.update_mesh(object_id=0, + positions=mesh_position) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': mesh_coarse_position, - 'vectors': 0.25 * self.F / p_model.scale}) + self.factory.update_arrows(object_id=1, + positions=mesh_coarse_position, + vectors=0.25 * self.F / p_model.scale) # Update sparse grid positions - self.factory.update_object_dict(object_id=2, - new_data_dict={'positions': updated_position}) + self.factory.update_points(object_id=2, + positions=updated_position) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Armadillo/FC/Environment/ArmadilloInteractive.py b/examples/demos/Armadillo/FC/Environment/ArmadilloInteractive.py index 2baad605..ba9e071c 100644 --- a/examples/demos/Armadillo/FC/Environment/ArmadilloInteractive.py +++ b/examples/demos/Armadillo/FC/Environment/ArmadilloInteractive.py @@ -28,20 +28,14 @@ class Armadillo(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topologies & mappings self.mesh = None @@ -68,6 +62,11 @@ def __init__(self, self.input_size = (p_model.nb_nodes_mesh, 3) self.output_size = (p_model.nb_nodes_grid, 3) + def init_database(self): + + # Define the fields of the Training database + self.define_training_fields(fields=[('input', np.ndarray), ('ground_truth', np.ndarray)]) + def create(self): # Load the meshes and the sparse grid @@ -101,6 +100,7 @@ def create(self): # Create plotter self.plotter = Plotter(title='Interactive Armadillo', N=1, interactive=True, offscreen=False, bg2='lightgray') + self.plotter.render() self.plotter.add(*self.spheres) self.plotter.add(self.mesh) self.plotter.add(Plane(pos=plane_origin, normal=[0, 1, 0], s=(10 * p_model.scale, 10 * p_model.scale), @@ -121,8 +121,8 @@ async def step(self): # Launch Vedo window self.plotter.show().close() # Smooth close - self.set_training_data(input_array=np.zeros(self.input_size), - output_array=np.zeros(self.output_size)) + self.set_training_data(input=np.zeros(self.input_size), + ground_truth=np.zeros(self.output_size)) def key_press(self, evt): @@ -170,7 +170,7 @@ def mouse_move(self, evt): if not self.interactive_window and self.selected is not None: # Compute input force vector - mouse_3D = self.plotter.computeWorldPosition(evt.picked2d) + mouse_3D = self.plotter.compute_world_position(evt.picked2d) move_3D = (mouse_3D - self.spheres_init[self.selected]) / self.mouse_factor if np.linalg.norm(move_3D) > 2: move_3D = 2 * move_3D / np.linalg.norm(move_3D) @@ -179,7 +179,7 @@ def mouse_move(self, evt): F[self.areas[self.selected]] = move_3D * amp # Apply output displacement - U = self.get_prediction(F).reshape(self.output_size) + U = self.get_prediction(input=F)['prediction'].reshape(self.output_size) updated_grid = self.sparse_grid.points().copy() + U updated_coarse = self.mapping_coarse.apply(updated_grid) diff --git a/examples/demos/Armadillo/FC/download.py b/examples/demos/Armadillo/FC/download.py index 983b2f07..ad285792 100644 --- a/examples/demos/Armadillo/FC/download.py +++ b/examples/demos/Armadillo/FC/download.py @@ -20,11 +20,11 @@ def __init__(self): 'session': [240], 'network': [193], 'stats': [192], - 'dataset_info': [242], - 'dataset_valid': [244, 239], - 'dataset_train': [241, 243]} + 'dataset_info': [317], + 'dataset_valid': [319], + 'dataset_train': [318]} if __name__ == '__main__': - ArmadilloDownloader().get_session('valid_data') + ArmadilloDownloader().get_session('all') diff --git a/examples/demos/Armadillo/FC/interactive.py b/examples/demos/Armadillo/FC/interactive.py index 15ea305b..440f89af 100644 --- a/examples/demos/Armadillo/FC/interactive.py +++ b/examples/demos/Armadillo/FC/interactive.py @@ -8,15 +8,15 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import ArmadilloDownloader -ArmadilloDownloader().get_session('predict') +ArmadilloDownloader().get_session('all') from Environment.ArmadilloInteractive import Armadillo from Environment.parameters import p_model @@ -24,31 +24,28 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Armadillo, - as_tcp_ip_client=False) + environment_config = BaseEnvironmentConfig(environment_class=Armadillo) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes_mesh * 3 nb_final_neurons = p_model.nb_nodes_grid * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_final_neurons] - net_config = FCConfig(network_name='armadillo_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig() + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='armadillo_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=1) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='armadillo_dpx', + step_nb=1) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Armadillo/FC/prediction.py b/examples/demos/Armadillo/FC/prediction.py index dbc339e2..65d40446 100644 --- a/examples/demos/Armadillo/FC/prediction.py +++ b/examples/demos/Armadillo/FC/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import ArmadilloDownloader -ArmadilloDownloader().get_session('valid_data') +ArmadilloDownloader().get_session('all') from Environment.Armadillo import Armadillo from Environment.parameters import p_model @@ -25,35 +25,29 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Armadillo, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True}) + environment_config = BaseEnvironmentConfig(environment_class=Armadillo, + visualizer=VedoVisualizer) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes_mesh * 3 nb_final_neurons = p_model.nb_nodes_grid * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_final_neurons] - net_config = FCConfig(network_name='armadillo_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig(dataset_dir='sessions/armadillo_dpx', - normalize=True, - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='armadillo_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=500) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='armadillo_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Armadillo/UNet/Environment/Armadillo.py b/examples/demos/Armadillo/UNet/Environment/Armadillo.py index 41691080..7d321de6 100644 --- a/examples/demos/Armadillo/UNet/Environment/Armadillo.py +++ b/examples/demos/Armadillo/UNet/Environment/Armadillo.py @@ -9,11 +9,10 @@ # Python related imports import os import sys - -import numpy as np +from numpy import ndarray, array, concatenate, arange, zeros, unique, reshape +from numpy.random import randint, uniform from vedo import Mesh from math import pow -from time import sleep, time # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -28,20 +27,14 @@ class Armadillo(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None @@ -56,12 +49,12 @@ def __init__(self, self.forces = [] self.areas = [] self.g_areas = [] - self.compute_sample = None # Force pattern step = 0.05 - self.amplitudes = np.concatenate((np.arange(step, 1, step), - np.arange(1, step, -step))) + self.amplitudes = concatenate((arange(step, 1, step), + arange(1, step, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.idx_zone = 0 @@ -75,11 +68,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): + def init_database(self): - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + # Define the fields of the Training database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) def create(self): @@ -92,19 +84,19 @@ def create(self): x_reg = [p_grid.origin[0] + i * p_grid.size[0] / p_grid.nb_cells[0] for i in range(p_grid.nb_cells[0] + 1)] y_reg = [p_grid.origin[1] + i * p_grid.size[1] / p_grid.nb_cells[1] for i in range(p_grid.nb_cells[1] + 1)] z_reg = [p_grid.origin[2] + i * p_grid.size[2] / p_grid.nb_cells[2] for i in range(p_grid.nb_cells[2] + 1)] - grid_positions = np.array([[[[x, y, z] for x in x_reg] for y in y_reg] for z in z_reg]).reshape(-1, 3) + grid_positions = array([[[[x, y, z] for x in x_reg] for y in y_reg] for z in z_reg]).reshape(-1, 3) cell_corner = lambda x, y, z: len(x_reg) * (len(y_reg) * z + y) + x - grid_cells = np.array([[[[[cell_corner(ix, iy, iz), - cell_corner(ix + 1, iy, iz), - cell_corner(ix + 1, iy + 1, iz), - cell_corner(ix, iy + 1, iz), - cell_corner(ix, iy, iz + 1), - cell_corner(ix + 1, iy, iz + 1), - cell_corner(ix + 1, iy + 1, iz + 1), - cell_corner(ix, iy + 1, iz + 1)] - for ix in range(p_grid.nb_cells[0])] - for iy in range(p_grid.nb_cells[1])] - for iz in range(p_grid.nb_cells[2])]]).reshape(-1, 8) + grid_cells = array([[[[[cell_corner(ix, iy, iz), + cell_corner(ix + 1, iy, iz), + cell_corner(ix + 1, iy + 1, iz), + cell_corner(ix, iy + 1, iz), + cell_corner(ix, iy, iz + 1), + cell_corner(ix + 1, iy, iz + 1), + cell_corner(ix + 1, iy + 1, iz + 1), + cell_corner(ix, iy + 1, iz + 1)] + for ix in range(p_grid.nb_cells[0])] + for iy in range(p_grid.nb_cells[1])] + for iz in range(p_grid.nb_cells[2])]]).reshape(-1, 8) self.grid = Mesh([grid_positions, grid_cells]) # Init mappings between meshes and sparse grid @@ -112,16 +104,16 @@ def create(self): self.mapping_coarse = GridMapping(self.sparse_grid, self.mesh_coarse) # Init correspondences between sparse and regular grid - self.grid_correspondences = np.zeros(self.sparse_grid.N(), dtype=int) - x_sparse = np.unique(self.sparse_grid.points()[:, 0]) - y_sparse = np.unique(self.sparse_grid.points()[:, 1]) - z_sparse = np.unique(self.sparse_grid.points()[:, 2]) + self.grid_correspondences = zeros(self.sparse_grid.N(), dtype=int) + x_sparse = unique(self.sparse_grid.points()[:, 0]) + y_sparse = unique(self.sparse_grid.points()[:, 1]) + z_sparse = unique(self.sparse_grid.points()[:, 2]) if len(x_reg) != len(x_sparse) or len(y_reg) != len(y_sparse) or len(z_reg) != len(z_sparse): raise ValueError('Grids should have the same dimension') d = [x_sparse[1] - x_sparse[0], y_sparse[1] - y_sparse[0], z_sparse[1] - z_sparse[0]] origin = [x_sparse[0], y_sparse[0], z_sparse[0]] for i, node in enumerate(self.sparse_grid.points()): - p = np.array(node) - np.array(origin) + p = array(node) - array(origin) ix = int(round(p[0] / d[0])) iy = int(round(p[1] / d[1])) iz = int(round(p[2] / d[2])) @@ -129,7 +121,8 @@ def create(self): self.grid_correspondences[i] = idx # Define force fields - sphere = lambda x, z: sum([pow(x_i - c_i, 2) for x_i, c_i in zip(x, p_forces.centers[z])]) <= pow(p_forces.radius[z], 2) + sphere = lambda x, z: sum([pow(x_i - c_i, 2) for x_i, c_i in zip(x, p_forces.centers[z])]) <= pow( + p_forces.radius[z], 2) for zone in p_forces.zones: self.areas.append([]) self.g_areas.append([]) @@ -137,36 +130,28 @@ def create(self): if sphere(node, zone): self.areas[-1].append(i) self.g_areas[-1] += self.mapping_coarse.cells[i].tolist() - self.areas[-1] = np.array(self.areas[-1]) - self.g_areas[-1] = np.unique(self.g_areas[-1]) - self.forces.append(np.zeros((len(self.areas[-1]), 3))) + self.areas[-1] = array(self.areas[-1]) + self.g_areas[-1] = unique(self.g_areas[-1]) + self.forces.append(zeros((len(self.areas[-1]), 3))) - def send_visualization(self): + def init_visualization(self): # Mesh representing detailed Armadillo (object will have id = 0) - self.factory.add_object(object_type='Mesh', - data_dict={'positions': self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - 'c': 'orange', - 'at': self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': [0, 0, 0], - 'vectors': [0., 0., 0.], - 'c': 'green', - 'at': self.instance_id}) - + self.factory.add_arrows(positions=array([0., 0., 0.]), + vectors=array([0., 0., 0.]), + c='green', + at=self.instance_id) # Sparse grid (object will have id = 2) - self.factory.add_object(object_type='Points', - data_dict={'positions': self.sparse_grid.points(), - 'r': 2, - 'c': 'grey', - 'at': self.instance_id}) - - # Return the visualization data - return self.factory.objects_dict + self.factory.add_points(positions=self.sparse_grid.points(), + point_size=2, + c='grey', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -176,38 +161,30 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - # Generate a new force - if self.idx_amplitude == 0: - self.idx_zone = np.random.randint(0, len(self.forces)) - zone = p_forces.zones[self.idx_zone] - self.force_value = np.random.uniform(low=-1, high=1, size=(3,)) * p_forces.amplitude[zone] - - # Update current force amplitude - self.forces[self.idx_zone] = self.force_value * self.amplitudes[self.idx_amplitude] - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - - # Create input array - F = np.zeros(self.data_size) - F[self.grid_correspondences[self.g_areas[self.idx_zone]]] = self.forces[self.idx_zone] - self.F = np.zeros((self.mesh_coarse.N(), 3)) - self.F[self.areas[self.idx_zone]] = self.forces[self.idx_zone] - - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.data_size) if self.sample_in is None else self.sample_in - self.F = F[self.grid_correspondences] + # Generate a new force + if self.idx_amplitude == 0: + self.idx_zone = randint(0, len(self.forces)) + zone = p_forces.zones[self.idx_zone] + self.force_value = uniform(low=-1, high=1, size=(3,)) * p_forces.amplitude[zone] + + # Update current force amplitude + self.forces[self.idx_zone] = self.force_value * self.amplitudes[self.idx_amplitude] + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) + + # Create input array + F = zeros(self.data_size) + F[self.grid_correspondences[self.g_areas[self.idx_zone]]] = self.forces[self.idx_zone] + self.F = zeros((self.mesh_coarse.N(), 3)) + self.F[self.areas[self.idx_zone]] = self.forces[self.idx_zone] # Set training data - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.data_size)) + self.set_training_data(input=F.copy(), + ground_truth=zeros(self.data_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.data_size) + U = reshape(prediction['prediction'], self.data_size) U_sparse = U[self.grid_correspondences] self.update_visual(U_sparse) @@ -219,20 +196,20 @@ def update_visual(self, U): mesh_coarse_position = self.mapping_coarse.apply(updated_position) # Update surface mesh - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': mesh_position}) + self.factory.update_mesh(object_id=0, + positions=mesh_position) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': mesh_coarse_position if self.compute_sample else updated_position, - 'vectors': 0.25 * self.F / p_model.scale}) + self.factory.update_arrows(object_id=1, + positions=mesh_coarse_position, + vectors=0.25 * self.F / p_model.scale) # Update sparse grid positions - self.factory.update_object_dict(object_id=2, - new_data_dict={'positions': updated_position}) + self.factory.update_points(object_id=2, + positions=updated_position) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Armadillo/UNet/download.py b/examples/demos/Armadillo/UNet/download.py index e34eae10..8172d9f6 100644 --- a/examples/demos/Armadillo/UNet/download.py +++ b/examples/demos/Armadillo/UNet/download.py @@ -20,10 +20,9 @@ def __init__(self): 'session': [247], 'network': [230], 'stats': [224], - 'dataset_info': [253], - 'dataset_valid': [245, 257], - 'dataset_train': [252, 255, 246, 256, 258, 251, - 250, 248, 254, 249, 259, 260]} + 'dataset_info': [321], + 'dataset_valid': [320], + 'dataset_train': [323, 322]} if __name__ == '__main__': diff --git a/examples/demos/Armadillo/UNet/prediction.py b/examples/demos/Armadillo/UNet/prediction.py index 2f4019e5..9752f206 100644 --- a/examples/demos/Armadillo/UNet/prediction.py +++ b/examples/demos/Armadillo/UNet/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.UNet.UNetConfig import UNetConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import ArmadilloDownloader -ArmadilloDownloader().get_session('valid_data') +ArmadilloDownloader().get_session('all') from Environment.Armadillo import Armadillo from Environment.parameters import grid_resolution @@ -25,38 +25,32 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Armadillo, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True}) + environment_config = BaseEnvironmentConfig(environment_class=Armadillo, + visualizer=VedoVisualizer) # UNet config - net_config = UNetConfig(network_name='armadillo_UNet', - input_size=grid_resolution, - nb_dims=3, - nb_input_channels=3, - nb_first_layer_channels=128, - nb_output_channels=3, - nb_steps=3, - two_sublayers=True, - border_mode='same', - skip_merge=False) + network_config = UNetConfig(input_size=grid_resolution, + nb_dims=3, + nb_input_channels=3, + nb_first_layer_channels=128, + nb_output_channels=3, + nb_steps=3, + two_sublayers=True, + border_mode='same', + skip_merge=False) # Dataset config - dataset_config = BaseDatasetConfig(dataset_dir='sessions/armadillo_dpx', - normalize=True, - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) + # Runner - runner = BaseRunner(session_dir='sessions', - session_name='armadillo_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=0) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='armadillo_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': - launch_runner() diff --git a/src/Visualization/__init__.py b/examples/demos/Armadillo/__init__.py similarity index 100% rename from src/Visualization/__init__.py rename to examples/demos/Armadillo/__init__.py diff --git a/examples/demos/Beam/FC/Environment/Beam.py b/examples/demos/Beam/FC/Environment/Beam.py index ad59fd24..e99644be 100644 --- a/examples/demos/Beam/FC/Environment/Beam.py +++ b/examples/demos/Beam/FC/Environment/Beam.py @@ -12,7 +12,6 @@ import numpy as np from vedo import Mesh -from time import sleep # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -26,30 +25,24 @@ class Beam(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None self.surface = None # Force - self.compute_sample = True step = 0.1 self.amplitudes = np.concatenate((np.arange(0, 1, step), np.arange(1, 0, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.zone = None @@ -63,11 +56,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): + def init_database(self): - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + # Define the fields of the Training database + self.define_training_fields(fields=[('input', np.ndarray), ('ground_truth', np.ndarray)]) def create(self): @@ -83,24 +75,19 @@ def create(self): np.argwhere(pts[:, 2] == np.max(pts[:, 2])))).reshape(-1) self.surface = np.unique(self.surface) - def send_visualization(self): + def init_visualization(self): # Mesh representing the grid (object will have id = 0) - self.factory.add_object(object_type='Mesh', - data_dict={'positions': self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - 'c': 'orange', - 'at': self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': [0, 0, 0], - 'vectors': [0, 0, 0], - 'c': 'green', - 'at': self.instance_id}) - - return self.factory.objects_dict + self.factory.add_arrows(positions=np.array([0., 0., 0.]), + vectors=np.array([0., 0., 0.]), + c='green', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -110,65 +97,58 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - # Generate a new force - if self.idx_amplitude == 0: - # Define zone - pts = self.mesh.points().copy() - side = np.random.randint(0, 6) - x_min = np.min(pts[:, 0]) if side == 0 else np.random.randint(np.min(pts[:, 0]), np.max(pts[:, 0]) - 10) - y_min = np.min(pts[:, 1]) if side == 1 else np.random.randint(np.min(pts[:, 1]), np.max(pts[:, 1]) - 10) - z_min = np.min(pts[:, 2]) if side == 2 else np.random.randint(np.min(pts[:, 2]), np.max(pts[:, 2]) - 10) - x_max = np.max(pts[:, 0]) if side == 3 else np.random.randint(x_min + 10, np.max(pts[:, 0]) + 1) - y_max = np.max(pts[:, 1]) if side == 4 else np.random.randint(y_min + 10, np.max(pts[:, 1]) + 1) - z_max = np.max(pts[:, 2]) if side == 5 else np.random.randint(z_min + 10, np.max(pts[:, 2]) + 1) - # Find points on surface and in box - self.zone = np.where((pts[:, 0] >= x_min) & (pts[:, 0] <= x_max) & (pts[:, 1] >= y_min) & - (pts[:, 1] <= y_max) & (pts[:, 2] >= z_min) & (pts[:, 2] <= z_max)) - self.zone = np.array(list(set(self.zone[0].tolist()).intersection(set(list(self.surface))))) - # Force value - F = np.random.uniform(low=-1, high=1, size=(3,)) * np.random.randint(20, 30) - self.force_value = F - - # Update current amplitude - self.force = self.force_value * self.amplitudes[self.idx_amplitude] - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - - # Create input array - F = np.zeros(self.data_size) - F[self.zone] = self.force - - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.data_size) if self.sample_in is None else self.sample_in + # Generate a new force + if self.idx_amplitude == 0: + # Define zone + pts = self.mesh.points().copy() + side = np.random.randint(0, 6) + x_min = np.min(pts[:, 0]) if side == 0 else np.random.randint(np.min(pts[:, 0]), np.max(pts[:, 0]) - 10) + y_min = np.min(pts[:, 1]) if side == 1 else np.random.randint(np.min(pts[:, 1]), np.max(pts[:, 1]) - 10) + z_min = np.min(pts[:, 2]) if side == 2 else np.random.randint(np.min(pts[:, 2]), np.max(pts[:, 2]) - 10) + x_max = np.max(pts[:, 0]) if side == 3 else np.random.randint(x_min + 10, np.max(pts[:, 0]) + 1) + y_max = np.max(pts[:, 1]) if side == 4 else np.random.randint(y_min + 10, np.max(pts[:, 1]) + 1) + z_max = np.max(pts[:, 2]) if side == 5 else np.random.randint(z_min + 10, np.max(pts[:, 2]) + 1) + # Find points on surface and in box + self.zone = np.where((pts[:, 0] >= x_min) & (pts[:, 0] <= x_max) & (pts[:, 1] >= y_min) & + (pts[:, 1] <= y_max) & (pts[:, 2] >= z_min) & (pts[:, 2] <= z_max)) + self.zone = np.array(list(set(self.zone[0].tolist()).intersection(set(list(self.surface))))) + # Force value + F = np.random.uniform(low=-1, high=1, size=(3,)) * np.random.randint(20, 30) + self.force_value = F + + # Update current amplitude + force = self.force_value * self.amplitudes[self.idx_amplitude] + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) + + # Create input array + F = np.zeros(self.data_size) + F[self.zone] = force # Set training data self.F = F - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.data_size)) + self.set_training_data(input=F.copy(), + ground_truth=np.zeros(self.data_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.data_size) + U = np.reshape(prediction['prediction'], self.data_size) self.update_visual(U) def update_visual(self, U): # Update surface mesh updated_mesh = self.mesh.clone().points(self.mesh.points().copy() + U) - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': updated_mesh.points().copy()}) + self.factory.update_mesh(object_id=0, + positions=updated_mesh.points().copy()) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': updated_mesh.points().copy(), - 'vectors': 0.25 * self.F}) + self.factory.update_arrows(object_id=1, + positions=updated_mesh.points().copy(), + vectors=0.25 * self.F) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Beam/FC/Environment/BeamInteractive.py b/examples/demos/Beam/FC/Environment/BeamInteractive.py index c60830d4..14f9a069 100644 --- a/examples/demos/Beam/FC/Environment/BeamInteractive.py +++ b/examples/demos/Beam/FC/Environment/BeamInteractive.py @@ -26,20 +26,14 @@ class Beam(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topologies & mappings self.mesh = None @@ -62,10 +56,16 @@ def __init__(self, # Data sizes self.data_size = (p_model.nb_nodes, 3) + def init_database(self): + + # Define the fields of the Training database + self.define_training_fields(fields=[('input', np.ndarray), ('ground_truth', np.ndarray)]) + def create(self): # Load the meshes and the sparse grid self.mesh = Mesh(p_model.grid, c='o').lineWidth(0.1).lighting('ambient') + self.mesh.shift(0, -7.5, -7.5) self.mesh_init = self.mesh.clone().points() # Get the surface points @@ -110,18 +110,12 @@ def create(self): (pts[:, other[1]] >= o1_min) & (pts[:, other[1]] <= o1_max)) self.areas.append(np.array(list(set(zone[0].tolist()).intersection(set(list(self.surface)))))) - # Define fixed plane - mesh_x = self.mesh.points()[:, 0] - fixed = np.where(mesh_x <= np.min(mesh_x) + 0.05 * (np.max(mesh_x) - np.min(mesh_x))) - plane_origin = [np.min(mesh_x), - np.mean(self.mesh.points()[:, 1][fixed]), - np.mean(self.mesh.points()[:, 2][fixed])] - # Create plotter self.plotter = Plotter(title='Interactive Beam', N=1, interactive=True, offscreen=False, bg2='lightgray') + self.plotter.render() self.plotter.add(*self.spheres) self.plotter.add(self.mesh) - self.plotter.add(Plane(pos=plane_origin, normal=[1, 0, 0], s=(20, 20), c='darkred', alpha=0.2)) + self.plotter.add(Plane(pos=[0., 0., 0.], normal=[1, 0, 0], s=(20, 20), c='darkred', alpha=0.2)) self.plotter.add(Text2D("Press 'Alt' to interact with the object.\n" "Left click to select a sphere.\n" "Right click to unselect a sphere.", s=0.75)) @@ -138,8 +132,8 @@ async def step(self): # Launch Vedo window self.plotter.show().close() # Smooth close - self.set_training_data(input_array=np.zeros(self.data_size), - output_array=np.zeros(self.data_size)) + self.set_training_data(input=np.zeros(self.data_size), + ground_truth=np.zeros(self.data_size)) def key_press(self, evt): @@ -187,7 +181,7 @@ def mouse_move(self, evt): if not self.interactive_window and self.selected is not None: # Compute input force vector - mouse_3D = self.plotter.computeWorldPosition(evt.picked2d) + mouse_3D = self.plotter.compute_world_position(evt.picked2d) move_3D = (mouse_3D - self.mesh_init[self.spheres_init[self.selected]]) / self.mouse_factor if np.linalg.norm(move_3D) > 10: move_3D = 10 * move_3D / np.linalg.norm(move_3D) @@ -195,7 +189,7 @@ def mouse_move(self, evt): F[self.areas[self.selected]] = move_3D * 2 # Apply output displacement - U = self.get_prediction(F).reshape(self.data_size) + U = self.get_prediction(input=F)['prediction'].reshape(self.data_size) updated_grid = self.mesh_init + U # Update view diff --git a/examples/demos/Beam/FC/download.py b/examples/demos/Beam/FC/download.py index 530d0457..4397eb7a 100644 --- a/examples/demos/Beam/FC/download.py +++ b/examples/demos/Beam/FC/download.py @@ -20,11 +20,11 @@ def __init__(self,): 'session': [266], 'network': [219], 'stats': [216], - 'dataset_info': [262], - 'dataset_valid': [263, 264], - 'dataset_train': [261, 265]} + 'dataset_info': [324], + 'dataset_valid': [326], + 'dataset_train': [325]} if __name__ == '__main__': - BeamDownloader().get_session('valid_data') + BeamDownloader().get_session('all') diff --git a/examples/demos/Beam/FC/interactive.py b/examples/demos/Beam/FC/interactive.py index 463dbe40..1b399d86 100644 --- a/examples/demos/Beam/FC/interactive.py +++ b/examples/demos/Beam/FC/interactive.py @@ -8,15 +8,15 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import BeamDownloader -BeamDownloader().get_session('predict') +BeamDownloader().get_session('all') from Environment.BeamInteractive import Beam from Environment.parameters import p_model @@ -24,30 +24,27 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Beam, - as_tcp_ip_client=False) + environment_config = BaseEnvironmentConfig(environment_class=Beam) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_neurons] - net_config = FCConfig(network_name='beam_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig() + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='beam_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=1) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='beam_dpx', + step_nb=1) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Beam/FC/prediction.py b/examples/demos/Beam/FC/prediction.py index 17af8802..c0fc1a70 100644 --- a/examples/demos/Beam/FC/prediction.py +++ b/examples/demos/Beam/FC/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import BeamDownloader -BeamDownloader().get_session('valid_data') +BeamDownloader().get_session('all') from Environment.Beam import Beam from Environment.parameters import p_model @@ -25,34 +25,28 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Beam, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True}) + environment_config = BaseEnvironmentConfig(environment_class=Beam, + visualizer=VedoVisualizer) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_neurons] - net_config = FCConfig(network_name='beam_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig(normalize=True, - dataset_dir='sessions/beam_dpx', - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='beam_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=500) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='beam_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Beam/UNet/Environment/Beam.py b/examples/demos/Beam/UNet/Environment/Beam.py index e255ef39..4e827773 100644 --- a/examples/demos/Beam/UNet/Environment/Beam.py +++ b/examples/demos/Beam/UNet/Environment/Beam.py @@ -12,7 +12,6 @@ import numpy as np from vedo import Mesh -from time import sleep # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -26,31 +25,24 @@ class Beam(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None self.surface = None # Force - self.compute_sample = True - step = 0.1 - self.amplitudes = np.concatenate((np.arange(0, 1, step), - np.arange(1, -1, -step), - np.arange(-1, 0, step))) + step = 0.05 + self.amplitudes = np.concatenate((np.arange(step, 1, step), + np.arange(1, step, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.zone = None @@ -64,11 +56,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): + def init_database(self): - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + # Define the fields of the Training database + self.define_training_fields(fields=[('input', np.ndarray), ('ground_truth', np.ndarray)]) def create(self): @@ -84,24 +75,19 @@ def create(self): np.argwhere(pts[:, 2] == np.max(pts[:, 2])))).reshape(-1) self.surface = np.unique(self.surface) - def send_visualization(self): + def init_visualization(self): # Mesh representing the grid (object will have id = 0) - self.factory.add_object(object_type='Mesh', - data_dict={'positions': self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - 'c': 'orange', - 'at': self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': [0, 0, 0], - 'vectors': [0, 0, 0], - 'c': 'green', - 'at': self.instance_id}) - - return self.factory.objects_dict + self.factory.add_arrows(positions=np.array([0., 0., 0.]), + vectors=np.array([0., 0., 0.]), + c='green', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -111,65 +97,58 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - # Generate a new force - if self.idx_amplitude == 0: - # Define zone - pts = self.mesh.points().copy() - side = np.random.randint(0, 11) - x_min = np.min(pts[:, 0]) if side in (3, 4) else np.random.randint(np.min(pts[:, 0]), np.max(pts[:, 0]) - 10) - y_min = np.min(pts[:, 1]) if side in (7, 8) else np.random.randint(np.min(pts[:, 1]), np.max(pts[:, 1]) - 10) - z_min = np.min(pts[:, 2]) if side == 0 else np.random.randint(np.min(pts[:, 2]), np.max(pts[:, 2]) - 10) - x_max = np.max(pts[:, 0]) if side in (5, 6) else np.random.randint(x_min + 10, np.max(pts[:, 0]) + 1) - y_max = np.max(pts[:, 1]) if side in (9, 10) else np.random.randint(y_min + 10, np.max(pts[:, 1]) + 1) - z_max = np.max(pts[:, 2]) if side in (1, 2, 3) else np.random.randint(z_min + 10, np.max(pts[:, 2]) + 1) - # Find points on surface and in box - self.zone = np.where((pts[:, 0] >= x_min) & (pts[:, 0] <= x_max) & (pts[:, 1] >= y_min) & - (pts[:, 1] <= y_max) & (pts[:, 2] >= z_min) & (pts[:, 2] <= z_max)) - self.zone = np.array(list(set(self.zone[0].tolist()).intersection(set(list(self.surface))))) - # Force value - F = np.random.uniform(low=-1, high=1, size=(3,)) - self.force_value = (F / np.linalg.norm(F)) * np.random.randint(10, 50) - - # Update current amplitude - self.force = self.force_value * self.amplitudes[self.idx_amplitude] - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - - # Create input array - F = np.zeros(self.data_size) - F[self.zone] = self.force - - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.data_size) if self.sample_in is None else self.sample_in + # Generate a new force + if self.idx_amplitude == 0: + # Define zone + pts = self.mesh.points().copy() + side = np.random.randint(0, 11) + x_min = np.min(pts[:, 0]) if side in (3, 4) else np.random.randint(np.min(pts[:, 0]), np.max(pts[:, 0]) - 10) + y_min = np.min(pts[:, 1]) if side in (7, 8) else np.random.randint(np.min(pts[:, 1]), np.max(pts[:, 1]) - 10) + z_min = np.min(pts[:, 2]) if side == 0 else np.random.randint(np.min(pts[:, 2]), np.max(pts[:, 2]) - 10) + x_max = np.max(pts[:, 0]) if side in (5, 6) else np.random.randint(x_min + 10, np.max(pts[:, 0]) + 1) + y_max = np.max(pts[:, 1]) if side in (9, 10) else np.random.randint(y_min + 10, np.max(pts[:, 1]) + 1) + z_max = np.max(pts[:, 2]) if side in (1, 2, 3) else np.random.randint(z_min + 10, np.max(pts[:, 2]) + 1) + # Find points on surface and in box + self.zone = np.where((pts[:, 0] >= x_min) & (pts[:, 0] <= x_max) & (pts[:, 1] >= y_min) & + (pts[:, 1] <= y_max) & (pts[:, 2] >= z_min) & (pts[:, 2] <= z_max)) + self.zone = np.array(list(set(self.zone[0].tolist()).intersection(set(list(self.surface))))) + # Force value + F = np.random.uniform(low=-1, high=1, size=(3,)) + self.force_value = (F / np.linalg.norm(F)) * np.random.randint(10, 50) + + # Update current amplitude + force = self.force_value * self.amplitudes[self.idx_amplitude] + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) + + # Create input array + F = np.zeros(self.data_size) + F[self.zone] = force # Set training data self.F = F - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.data_size)) + self.set_training_data(input=F.copy(), + ground_truth=np.zeros(self.data_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.data_size) + U = np.reshape(prediction['prediction'], self.data_size) self.update_visual(U) def update_visual(self, U): # Update surface mesh updated_position = self.mesh.points().copy() + U - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': updated_position}) + self.factory.update_mesh(object_id=0, + positions=updated_position) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': updated_position, - 'vectors': 0.25 * self.F}) + self.factory.update_arrows(object_id=1, + positions=updated_position, + vectors=0.25 * self.F) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Beam/UNet/Environment/parameters.py b/examples/demos/Beam/UNet/Environment/parameters.py index 60485aa0..aa1a9da3 100644 --- a/examples/demos/Beam/UNet/Environment/parameters.py +++ b/examples/demos/Beam/UNet/Environment/parameters.py @@ -5,14 +5,13 @@ """ import os -from numpy import array from vedo import Mesh from collections import namedtuple # Model parameters grid = os.path.dirname(os.path.abspath(__file__)) + '/models/beam.obj' -grid_resolution = array([5, 5, 25]) +grid_resolution = [5, 5, 25] model = {'grid': grid, 'nb_nodes': Mesh(grid).N()} p_model = namedtuple('p_model', model)(**model) diff --git a/examples/demos/Beam/UNet/download.py b/examples/demos/Beam/UNet/download.py index f78be871..8ddcdab8 100644 --- a/examples/demos/Beam/UNet/download.py +++ b/examples/demos/Beam/UNet/download.py @@ -20,9 +20,9 @@ def __init__(self): 'session': [312], 'network': [315], 'stats': [314], - 'dataset_info': [309], - 'dataset_valid': [313, 316], - 'dataset_train': [310, 311]} + 'dataset_info': [329], + 'dataset_valid': [328], + 'dataset_train': [327]} if __name__ == '__main__': diff --git a/examples/demos/Beam/UNet/prediction.py b/examples/demos/Beam/UNet/prediction.py index 067ff21f..8f194c29 100644 --- a/examples/demos/Beam/UNet/prediction.py +++ b/examples/demos/Beam/UNet/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.UNet.UNetConfig import UNetConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import BeamDownloader -BeamDownloader().get_session('valid_data') +BeamDownloader().get_session('all') from Environment.Beam import Beam from Environment.parameters import grid_resolution @@ -25,37 +25,31 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Beam, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True}) + environment_config = BaseEnvironmentConfig(environment_class=Beam, + visualizer=VedoVisualizer) # UNet config - net_config = UNetConfig(network_name='beam_UNet', - input_size=grid_resolution.tolist(), - nb_dims=3, - nb_input_channels=3, - nb_first_layer_channels=128, - nb_output_channels=3, - nb_steps=3, - two_sublayers=True, - border_mode='same', - skip_merge=False) + network_config = UNetConfig(input_size=grid_resolution, + nb_dims=3, + nb_input_channels=3, + nb_first_layer_channels=128, + nb_output_channels=3, + nb_steps=3, + two_sublayers=True, + border_mode='same', + skip_merge=False) # Dataset config - dataset_config = BaseDatasetConfig(normalize=True, - dataset_dir='sessions/beam_dpx', - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='beam_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=0) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='beam_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Beam/__init__.py b/examples/demos/Beam/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/demos/Liver/FC/Environment/Liver.py b/examples/demos/Liver/FC/Environment/Liver.py index c34f741d..d8cb9963 100644 --- a/examples/demos/Liver/FC/Environment/Liver.py +++ b/examples/demos/Liver/FC/Environment/Liver.py @@ -9,10 +9,11 @@ # Python related imports import os import sys -import numpy as np +from numpy import ndarray, array, concatenate, arange, zeros, reshape +from numpy.random import randint, uniform +from numpy.linalg import norm from vedo import Mesh from math import pow -from time import sleep # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -27,20 +28,15 @@ class Liver(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1, + nb_forces=3): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None @@ -50,15 +46,15 @@ def __init__(self, self.mapping_coarse = None # Force fields - self.forces = None - self.areas = None - self.compute_sample = True + self.nb_forces = nb_forces + self.forces = [[]] * self.nb_forces + self.areas = [[]] * self.nb_forces # Force pattern step = 0.05 - self.amplitudes = np.concatenate((np.arange(0, 1, step), - np.arange(1, 0, -step))) - self.nb_forces = p_forces.nb_simultaneous_forces + self.amplitudes = concatenate((arange(0, 1, step), + arange(1, 0, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.F = None @@ -72,16 +68,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): - - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + def init_database(self): - # Receive the number of forces - self.nb_forces = min(param_dict['nb_forces'], self.nb_forces) if 'nb_forces' in param_dict else self.nb_forces - self.forces = [None] * self.nb_forces - self.areas = [None] * self.nb_forces + # Define the fields of the Training database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) def create(self): @@ -92,32 +82,24 @@ def create(self): self.mapping = GridMapping(self.sparse_grid, self.mesh) self.mapping_coarse = GridMapping(self.sparse_grid, self.mesh_coarse) - def send_visualization(self): + def init_visualization(self): # Mesh representing detailed Armadillo (object will have id = 0) - self.factory.add_object(object_type="Mesh", - data_dict={"positions": self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - "c": "orange", - "at": self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': p_model.fixed_point, - 'vectors': [0, 0, 0], - 'c': 'green', - 'at': self.instance_id}) - + self.factory.add_arrows(positions=p_model.fixed_point, + vectors=array([0., 0., 0.]), + c='green', + at=self.instance_id) # Points representing the grid (object will have id = 2) - self.factory.add_object(object_type='Points', - data_dict={'positions': self.sparse_grid.points(), - 'r': 1., - 'c': 'black', - 'at': self.instance_id}) - - # Return the visualization data - return self.factory.objects_dict + self.factory.add_points(positions=self.sparse_grid.points(), + point_size=1, + c='black', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -127,63 +109,55 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - - # Generate a new force - if self.idx_amplitude == 0: - - # Define zones - selected_centers = [] - pts = self.mesh_coarse.points().copy() - for i in range(self.nb_forces): - # Pick a random sphere center, check distance with other spheres - current_point = pts[np.random.randint(0, self.mesh_coarse.N())] - distance_check = True - for p in selected_centers: - distance = np.linalg.norm(current_point - p) - if distance < p_forces.inter_distance_thresh: - distance_check = False - break - # Reset force field value and indices - self.areas[i] = [] - self.forces[i] = np.array([0, 0, 0]) - # Fill the force field - if distance_check: - # Add center - selected_centers.append(current_point) - # Find node in the sphere - sphere = lambda x, y: sum([pow(x_i - y_i, 2) for x_i, y_i in zip(x, y)]) - for j, p in enumerate(pts): - if sphere(p, current_point) <= pow(p_forces.inter_distance_thresh / 2, 2): - self.areas[i].append(j) - # If the sphere is non-empty, create a force vector - if len(self.areas[i]) > 0: - f = np.random.uniform(low=-1, high=1, size=(3,)) - self.forces[i] = (f / np.linalg.norm(f)) * p_forces.amplitude - - # Create input array - F = np.zeros(self.input_size) - for i, force in enumerate(self.forces): - F[self.areas[i]] = self.forces[i] * self.amplitudes[self.idx_amplitude] - - # Update current force amplitude - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.input_size) if self.sample_in is None else self.sample_in + # Generate a new force + if self.idx_amplitude == 0: + + # Define zones + selected_centers = [] + pts = self.mesh_coarse.points().copy() + for i in range(self.nb_forces): + # Pick a random sphere center, check distance with other spheres + current_point = pts[randint(0, self.mesh_coarse.N())] + distance_check = True + for p in selected_centers: + distance = norm(current_point - p) + if distance < p_forces.inter_distance_thresh: + distance_check = False + break + # Reset force field value and indices + self.areas[i] = [] + self.forces[i] = array([0, 0, 0]) + # Fill the force field + if distance_check: + # Add center + selected_centers.append(current_point) + # Find node in the sphere + sphere = lambda x, y: sum([pow(x_i - y_i, 2) for x_i, y_i in zip(x, y)]) + for j, p in enumerate(pts): + if sphere(p, current_point) <= pow(p_forces.inter_distance_thresh / 2, 2): + self.areas[i].append(j) + # If the sphere is non-empty, create a force vector + if len(self.areas[i]) > 0: + f = uniform(low=-1, high=1, size=(3,)) + self.forces[i] = (f / norm(f)) * p_forces.amplitude + + # Create input array + F = zeros(self.input_size) + for i, force in enumerate(self.forces): + F[self.areas[i]] = self.forces[i] * self.amplitudes[self.idx_amplitude] + + # Update current force amplitude + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) # Set training data self.F = F - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.output_size)) + self.set_training_data(input=F.copy(), + ground_truth=zeros(self.output_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.output_size) + U = reshape(prediction['prediction'], self.output_size) self.update_visual(U) def update_visual(self, U): @@ -194,20 +168,20 @@ def update_visual(self, U): mesh_coarse_position = self.mapping_coarse.apply(updated_position) # Update surface mesh - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': mesh_position}) + self.factory.update_mesh(object_id=0, + positions=mesh_position) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': mesh_coarse_position, - 'vectors': self.F}) + self.factory.update_arrows(object_id=1, + positions=mesh_coarse_position, + vectors=self.F) # Update sparse grid positions - self.factory.update_object_dict(object_id=2, - new_data_dict={'positions': updated_position}) + self.factory.update_points(object_id=2, + positions=updated_position) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Liver/FC/Environment/LiverInteractive.py b/examples/demos/Liver/FC/Environment/LiverInteractive.py index f7e432c9..81fed433 100644 --- a/examples/demos/Liver/FC/Environment/LiverInteractive.py +++ b/examples/demos/Liver/FC/Environment/LiverInteractive.py @@ -12,7 +12,7 @@ import vtk import numpy as np -from vedo import Mesh, Sphere, Arrows, Plotter, Text2D, Box, fitSphere +from vedo import Mesh, Sphere, Arrows, Plotter, Text2D, Box # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -28,20 +28,14 @@ class Liver(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topologies & mappings self.mesh = None @@ -68,6 +62,11 @@ def __init__(self, self.input_size = (p_model.nb_nodes_mesh, 3) self.output_size = (p_model.nb_nodes_grid, 3) + def init_database(self): + + # Define the fields of the Training database + self.define_training_fields(fields=[('input', np.ndarray), ('ground_truth', np.ndarray)]) + def create(self): # Load the meshes and the sparse grid @@ -95,6 +94,7 @@ def create(self): # Create plotter self.plotter = Plotter(title='Interactive Armadillo', N=1, interactive=True, offscreen=False, bg2='lightgray') + self.plotter.render() self.plotter.add(*self.spheres) self.plotter.add(box) self.plotter.add(self.mesh) @@ -114,8 +114,8 @@ async def step(self): # Launch Vedo window self.plotter.show().close() # Smooth close - self.set_training_data(input_array=np.zeros(self.input_size), - output_array=np.zeros(self.output_size)) + self.set_training_data(input=np.zeros(self.input_size), + ground_truth=np.zeros(self.output_size)) def key_press(self, evt): @@ -163,7 +163,7 @@ def mouse_move(self, evt): if not self.interactive_window and self.selected is not None: # Compute input force vector - mouse_3D = self.plotter.computeWorldPosition(evt.picked2d) + mouse_3D = self.plotter.compute_world_position(evt.picked2d) move_3D = (mouse_3D - self.mesh_coarse.points()[self.spheres_init[self.selected]]) / self.mouse_factor if np.linalg.norm(move_3D) > 1.5: move_3D = 1.5 * move_3D / np.linalg.norm(move_3D) @@ -171,7 +171,7 @@ def mouse_move(self, evt): F[self.areas[self.selected]] = move_3D * p_forces.amplitude # Apply output displacement - U = self.get_prediction(F).reshape(self.output_size) + U = self.get_prediction(input=F)['prediction'].reshape(self.output_size) updated_grid = self.sparse_grid.points().copy() + U updated_coarse = self.mapping_coarse.apply(updated_grid) diff --git a/examples/demos/Liver/FC/download.py b/examples/demos/Liver/FC/download.py index f17fa317..1e7d616f 100644 --- a/examples/demos/Liver/FC/download.py +++ b/examples/demos/Liver/FC/download.py @@ -20,11 +20,11 @@ def __init__(self): 'session': [281], 'network': [195], 'stats': [198], - 'dataset_info': [276], - 'dataset_valid': [279, 278], - 'dataset_train': [277, 282, 280, 283]} + 'dataset_info': [331], + 'dataset_valid': [330], + 'dataset_train': [332]} if __name__ == '__main__': - LiverDownloader().get_session('valid_data') + LiverDownloader().get_session('all') diff --git a/examples/demos/Liver/FC/interactive.py b/examples/demos/Liver/FC/interactive.py index e16fc4ba..1e2eb974 100644 --- a/examples/demos/Liver/FC/interactive.py +++ b/examples/demos/Liver/FC/interactive.py @@ -8,15 +8,15 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import LiverDownloader -LiverDownloader().get_session('predict') +LiverDownloader().get_session('all') from Environment.LiverInteractive import Liver from Environment.parameters import p_model @@ -24,31 +24,28 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Liver, - as_tcp_ip_client=False) + environment_config = BaseEnvironmentConfig(environment_class=Liver) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes_mesh * 3 nb_final_neurons = p_model.nb_nodes_grid * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_final_neurons] - net_config = FCConfig(network_name='liver_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig() + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='liver_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=1) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='liver_dpx', + step_nb=1) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Liver/FC/prediction.py b/examples/demos/Liver/FC/prediction.py index 9833fa58..83b3a02e 100644 --- a/examples/demos/Liver/FC/prediction.py +++ b/examples/demos/Liver/FC/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import LiverDownloader -LiverDownloader().get_session('valid_data') +LiverDownloader().get_session('all') from Environment.Liver import Liver from Environment.parameters import p_model @@ -25,36 +25,30 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Liver, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True, - 'nb_forces': 3}) + environment_config = BaseEnvironmentConfig(environment_class=Liver, + visualizer=VedoVisualizer, + env_kwargs={'nb_forces': 3}) # FC config nb_hidden_layers = 3 nb_neurons = p_model.nb_nodes_mesh * 3 nb_final_neurons = p_model.nb_nodes_grid * 3 layers_dim = [nb_neurons] + [nb_neurons for _ in range(nb_hidden_layers)] + [nb_final_neurons] - net_config = FCConfig(network_name='liver_FC', - dim_output=3, - dim_layers=layers_dim, - biases=True) + network_config = FCConfig(dim_layers=layers_dim, + dim_output=3, + biases=True) # Dataset config - dataset_config = BaseDatasetConfig(dataset_dir='sessions/liver_dpx', - normalize=True, - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='liver_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=500) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='liver_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Liver/UNet/Environment/Liver.py b/examples/demos/Liver/UNet/Environment/Liver.py index ee2ef3e1..6e9c10f4 100644 --- a/examples/demos/Liver/UNet/Environment/Liver.py +++ b/examples/demos/Liver/UNet/Environment/Liver.py @@ -9,10 +9,11 @@ # Python related imports import os import sys -import numpy as np +from numpy import ndarray, array, concatenate, arange, zeros, unique, reshape +from numpy.random import randint, uniform +from numpy.linalg import norm from vedo import Mesh from math import pow -from time import sleep # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -27,20 +28,15 @@ class Liver(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None): + instance_id=1, + instance_nb=1, + nb_forces=3): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_id=instance_id, + instance_nb=instance_nb) # Topology self.mesh = None @@ -52,9 +48,10 @@ def __init__(self, self.grid_correspondences = None # Force fields - self.forces = None - self.areas = None - self.g_areas = None + self.nb_forces = nb_forces + self.forces = [[]] * self.nb_forces + self.areas = [[]] * self.nb_forces + self.g_areas = [[]] * self.nb_forces self.compute_sample = True self.cell_corner = None self.sparse_grid_position = None @@ -64,9 +61,9 @@ def __init__(self, # Force pattern step = 0.05 - self.amplitudes = np.concatenate((np.arange(0, 1, step), - np.arange(1, 0, -step))) - self.nb_forces = p_forces.nb_simultaneous_forces + self.amplitudes = concatenate((arange(step, 1, step), + arange(1, step, -step))) + self.amplitudes[0] = 0 self.idx_amplitude = 0 self.force_value = None self.F = None @@ -80,17 +77,10 @@ def __init__(self, Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): - - # Get the model definition parameters - self.compute_sample = param_dict['compute_sample'] if 'compute_sample' in param_dict else True - self.amplitudes[0] = 0 if self.compute_sample else 1 + def init_database(self): - # Receive the number of forces - self.nb_forces = min(param_dict['nb_forces'], self.nb_forces) if 'nb_forces' in param_dict else self.nb_forces - self.forces = [None] * self.nb_forces - self.areas = [None] * self.nb_forces - self.g_areas = [None] * self.nb_forces + # Define the fields of the Training database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) def create(self): @@ -103,19 +93,19 @@ def create(self): x_reg = [p_grid.origin[0] + i * p_grid.size[0] / p_grid.nb_cells[0] for i in range(p_grid.nb_cells[0] + 1)] y_reg = [p_grid.origin[1] + i * p_grid.size[1] / p_grid.nb_cells[1] for i in range(p_grid.nb_cells[1] + 1)] z_reg = [p_grid.origin[2] + i * p_grid.size[2] / p_grid.nb_cells[2] for i in range(p_grid.nb_cells[2] + 1)] - grid_positions = np.array([[[[x, y, z] for x in x_reg] for y in y_reg] for z in z_reg]).reshape(-1, 3) + grid_positions = array([[[[x, y, z] for x in x_reg] for y in y_reg] for z in z_reg]).reshape(-1, 3) cell_corner = lambda x, y, z: len(x_reg) * (len(y_reg) * z + y) + x - grid_cells = np.array([[[[[cell_corner(ix, iy, iz), - cell_corner(ix + 1, iy, iz), - cell_corner(ix + 1, iy + 1, iz), - cell_corner(ix, iy + 1, iz), - cell_corner(ix, iy, iz + 1), - cell_corner(ix + 1, iy, iz + 1), - cell_corner(ix + 1, iy + 1, iz + 1), - cell_corner(ix, iy + 1, iz + 1)] - for ix in range(p_grid.nb_cells[0])] - for iy in range(p_grid.nb_cells[1])] - for iz in range(p_grid.nb_cells[2])]]).reshape(-1, 8) + grid_cells = array([[[[[cell_corner(ix, iy, iz), + cell_corner(ix + 1, iy, iz), + cell_corner(ix + 1, iy + 1, iz), + cell_corner(ix, iy + 1, iz), + cell_corner(ix, iy, iz + 1), + cell_corner(ix + 1, iy, iz + 1), + cell_corner(ix + 1, iy + 1, iz + 1), + cell_corner(ix, iy + 1, iz + 1)] + for ix in range(p_grid.nb_cells[0])] + for iy in range(p_grid.nb_cells[1])] + for iz in range(p_grid.nb_cells[2])]]).reshape(-1, 8) self.regular_grid = Mesh([grid_positions, grid_cells]) # Init mappings between meshes and sparse grid @@ -123,16 +113,16 @@ def create(self): self.mapping_coarse = GridMapping(self.sparse_grid, self.mesh_coarse) # Init correspondences between sparse and regular grid - self.grid_correspondences = np.zeros(self.sparse_grid.N(), dtype=int) - x_sparse = np.unique(self.sparse_grid.points()[:, 0]) - y_sparse = np.unique(self.sparse_grid.points()[:, 1]) - z_sparse = np.unique(self.sparse_grid.points()[:, 2]) + self.grid_correspondences = zeros(self.sparse_grid.N(), dtype=int) + x_sparse = unique(self.sparse_grid.points()[:, 0]) + y_sparse = unique(self.sparse_grid.points()[:, 1]) + z_sparse = unique(self.sparse_grid.points()[:, 2]) if len(x_reg) != len(x_sparse) or len(y_reg) != len(y_sparse) or len(z_reg) != len(z_sparse): raise ValueError('Grids should have the same dimension') d = [x_sparse[1] - x_sparse[0], y_sparse[1] - y_sparse[0], z_sparse[1] - z_sparse[0]] origin = [x_sparse[0], y_sparse[0], z_sparse[0]] for i, node in enumerate(self.sparse_grid.points()): - p = np.array(node) - np.array(origin) + p = array(node) - array(origin) ix = int(round(p[0] / d[0])) iy = int(round(p[1] / d[1])) iz = int(round(p[2] / d[2])) @@ -140,40 +130,32 @@ def create(self): self.grid_correspondences[i] = idx # Init grid force field stuff - self.x_sparse = np.unique(self.sparse_grid.points()[:, 0]) - self.y_sparse = np.unique(self.sparse_grid.points()[:, 1]) - self.z_sparse = np.unique(self.sparse_grid.points()[:, 2]) + self.x_sparse = unique(self.sparse_grid.points()[:, 0]) + self.y_sparse = unique(self.sparse_grid.points()[:, 1]) + self.z_sparse = unique(self.sparse_grid.points()[:, 2]) self.sparse_grid_position = self.sparse_grid.points().tolist() self.cell_corner = lambda x, y, z: self.sparse_grid_position.index([self.x_sparse[x], self.y_sparse[y], self.z_sparse[z]]) - def send_visualization(self): + def init_visualization(self): # Mesh representing detailed Armadillo (object will have id = 0) - self.factory.add_object(object_type="Mesh", - data_dict={"positions": self.mesh.points(), - 'cells': self.mesh.cells(), - 'wireframe': True, - "c": "orange", - "at": self.instance_id}) - + self.factory.add_mesh(positions=self.mesh.points(), + cells=self.mesh.cells(), + wireframe=True, + c='orange', + at=self.instance_id) # Arrows representing the force fields (object will have id = 1) - self.factory.add_object(object_type='Arrows', - data_dict={'positions': p_model.fixed_point, - 'vectors': [0, 0, 0], - 'c': 'green', - 'at': self.instance_id}) - + self.factory.add_arrows(positions=p_model.fixed_point, + vectors=array([0., 0., 0.]), + c='green', + at=self.instance_id) # Sparse grid (object will have id = 2) - self.factory.add_object(object_type='Points', - data_dict={'positions': self.sparse_grid.points(), - 'r': 2, - 'c': 'grey', - 'at': self.instance_id}) - - # Return the visualization data - return self.factory.objects_dict + self.factory.add_points(positions=self.sparse_grid.points(), + point_size=2, + c='grey', + at=self.instance_id) """ ENVIRONMENT BEHAVIOR @@ -183,70 +165,61 @@ def send_visualization(self): async def step(self): - # Compute a force sample - if self.compute_sample: - - # Generate a new force - if self.idx_amplitude == 0: - - # Define zones - selected_centers = [] - pts = self.mesh_coarse.points().copy() - for i in range(self.nb_forces): - # Pick a random sphere center, check distance with other spheres - current_point = pts[np.random.randint(0, self.mesh_coarse.N())] - distance_check = True - for p in selected_centers: - distance = np.linalg.norm(current_point - p) - if distance < p_forces.inter_distance_thresh: - distance_check = False - break - # Reset force field value and indices - self.areas[i] = [] - self.g_areas[i] = [] - self.forces[i] = np.array([0, 0, 0]) - # Fill the force field - if distance_check: - # Add center - selected_centers.append(current_point) - # Find node in the sphere - sphere = lambda x, c: sum([pow(x_i - c_i, 2) for x_i, c_i in zip(x, c)]) <= \ - pow(p_forces.inter_distance_thresh / 2, 2) - for j, p in enumerate(pts): - if sphere(p, current_point): - self.areas[i].append(j) - self.g_areas[i] += self.mapping_coarse.cells[j].tolist() - # If the sphere is non-empty, create a force vector - if len(self.areas[i]) > 0: - f = np.random.uniform(low=-1, high=1, size=(3,)) - self.forces[i] = (f / np.linalg.norm(f)) * p_forces.amplitude - self.areas[i] = np.array(self.areas[i]) - self.g_areas[i] = np.unique(self.g_areas[i]) - - # Create input array - F = np.zeros(self.data_size) - self.F = np.zeros((self.mesh_coarse.N(), 3)) - for i, force in enumerate(self.forces): - F[self.grid_correspondences[self.g_areas[i]]] = force * self.amplitudes[self.idx_amplitude] - self.F[self.areas[i]] = force * self.amplitudes[self.idx_amplitude] - - # Update current force amplitude - self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) - - # Load a force sample from Dataset - else: - sleep(0.5) - F = np.zeros(self.data_size) if self.sample_in is None else self.sample_in - self.F = F[self.grid_correspondences] + # Generate a new force + if self.idx_amplitude == 0: + + # Define zones + selected_centers = [] + pts = self.mesh_coarse.points().copy() + for i in range(self.nb_forces): + # Pick a random sphere center, check distance with other spheres + current_point = pts[randint(0, self.mesh_coarse.N())] + distance_check = True + for p in selected_centers: + distance = norm(current_point - p) + if distance < p_forces.inter_distance_thresh: + distance_check = False + break + # Reset force field value and indices + self.areas[i] = [] + self.g_areas[i] = [] + self.forces[i] = array([0, 0, 0]) + # Fill the force field + if distance_check: + # Add center + selected_centers.append(current_point) + # Find node in the sphere + sphere = lambda x, c: sum([pow(x_i - c_i, 2) for x_i, c_i in zip(x, c)]) <= \ + pow(p_forces.inter_distance_thresh / 2, 2) + for j, p in enumerate(pts): + if sphere(p, current_point): + self.areas[i].append(j) + self.g_areas[i] += self.mapping_coarse.cells[j].tolist() + # If the sphere is non-empty, create a force vector + if len(self.areas[i]) > 0: + f = uniform(low=-1, high=1, size=(3,)) + self.forces[i] = (f / norm(f)) * p_forces.amplitude + self.areas[i] = array(self.areas[i]) + self.g_areas[i] = unique(self.g_areas[i]) + + # Create input array + F = zeros(self.data_size) + self.F = zeros((self.mesh_coarse.N(), 3)) + for i, force in enumerate(self.forces): + F[self.grid_correspondences[self.g_areas[i]]] = force * self.amplitudes[self.idx_amplitude] + self.F[self.areas[i]] = force * self.amplitudes[self.idx_amplitude] + + # Update current force amplitude + self.idx_amplitude = (self.idx_amplitude + 1) % len(self.amplitudes) # Set training data - self.set_training_data(input_array=F.copy(), - output_array=np.zeros(self.data_size)) + self.set_training_data(input=F.copy(), + ground_truth=zeros(self.data_size)) def apply_prediction(self, prediction): # Reshape to correspond to sparse grid - U = np.reshape(prediction, self.data_size) + U = reshape(prediction['prediction'], self.data_size) U_sparse = U[self.grid_correspondences] self.update_visual(U_sparse) @@ -258,20 +231,20 @@ def update_visual(self, U): mesh_coarse_position = self.mapping_coarse.apply(updated_position) # Update surface mesh - self.factory.update_object_dict(object_id=0, - new_data_dict={'positions': mesh_position}) + self.factory.update_mesh(object_id=0, + positions=mesh_position) # Update arrows representing force fields - self.factory.update_object_dict(object_id=1, - new_data_dict={'positions': mesh_coarse_position if self.compute_sample else updated_position, - 'vectors': self.F}) + self.factory.update_arrows(object_id=1, + positions=mesh_coarse_position, + vectors=self.F) # Update sparse grid positions - self.factory.update_object_dict(object_id=2, - new_data_dict={'positions': updated_position}) + self.factory.update_points(object_id=2, + positions=updated_position) # Send visualization data to update - self.update_visualisation(visu_dict=self.factory.updated_object_dict) + self.update_visualisation() def close(self): # Shutdown message diff --git a/examples/demos/Liver/UNet/download.py b/examples/demos/Liver/UNet/download.py index 490d6aed..5c354326 100644 --- a/examples/demos/Liver/UNet/download.py +++ b/examples/demos/Liver/UNet/download.py @@ -20,12 +20,11 @@ def __init__(self): 'session': [290], 'network': [302], 'stats': [285], - 'dataset_info': [291], - 'dataset_valid': [292, 296], - 'dataset_train': [295, 286, 297, 299, 294, 287, - 289, 300, 288, 301, 298, 293]} + 'dataset_info': [333], + 'dataset_valid': [336], + 'dataset_train': [335, 334]} if __name__ == '__main__': - LiverDownloader().get_session('valid_data') + LiverDownloader().get_session('all') diff --git a/examples/demos/Liver/UNet/prediction.py b/examples/demos/Liver/UNet/prediction.py index 47770dd2..9d937001 100644 --- a/examples/demos/Liver/UNet/prediction.py +++ b/examples/demos/Liver/UNet/prediction.py @@ -8,16 +8,16 @@ import sys # DeepPhysX related imports -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.UNet.UNetConfig import UNetConfig # Session related imports sys.path.append(os.path.dirname(os.path.abspath(__file__))) from download import LiverDownloader -LiverDownloader().get_session('valid_data') +LiverDownloader().get_session('all') from Environment.Liver import Liver from Environment.parameters import grid_resolution @@ -25,39 +25,32 @@ def launch_runner(): # Environment config - env_config = BaseEnvironmentConfig(environment_class=Liver, - visualizer=VedoVisualizer, - as_tcp_ip_client=False, - param_dict={'compute_sample': True, - 'nb_forces': 3}) + environment_config = BaseEnvironmentConfig(environment_class=Liver, + visualizer=VedoVisualizer, + env_kwargs={'nb_forces': 3}) # UNet config - net_config = UNetConfig(network_name='liver_UNet', - save_each_epoch=True, - input_size=grid_resolution, - nb_dims=3, - nb_input_channels=3, - nb_first_layer_channels=128, - nb_output_channels=3, - nb_steps=3, - two_sublayers=True, - border_mode='same', - skip_merge=False) + network_config = UNetConfig(input_size=grid_resolution, + nb_dims=3, + nb_input_channels=3, + nb_first_layer_channels=128, + nb_output_channels=3, + nb_steps=3, + two_sublayers=True, + border_mode='same', + skip_merge=False) # Dataset config - dataset_config = BaseDatasetConfig(dataset_dir='sessions/liver_dpx', - normalize=True, - use_mode='Validation') + database_config = BaseDatabaseConfig(normalize=True) # Runner - runner = BaseRunner(session_dir='sessions', - session_name='liver_dpx', - dataset_config=dataset_config, - environment_config=env_config, - network_config=net_config, - nb_steps=500) + runner = BasePrediction(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='liver_dpx', + step_nb=500) runner.execute() - runner.close() if __name__ == '__main__': diff --git a/examples/demos/Liver/__init__.py b/examples/demos/Liver/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/demos/__init__.py b/examples/demos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/features/Environment.py b/examples/features/Environment.py index e8034717..9093e6b7 100644 --- a/examples/features/Environment.py +++ b/examples/features/Environment.py @@ -5,7 +5,7 @@ """ # Python related imports -from numpy import mean, pi, array +from numpy import mean, pi, array, ndarray from numpy.random import random, randint from time import sleep @@ -17,66 +17,58 @@ class MeanEnvironment(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, - instance_id=0, - number_of_instances=1, as_tcp_ip_client=True, - environment_manager=None, - visu_db=None): + instance_id=1, + instance_nb=1, + constant=False, + data_size=(30, 2), + delay=False, + allow_request=True): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, - instance_id=instance_id, - number_of_instances=number_of_instances, as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager, - visu_db=visu_db) + instance_id=instance_id, + instance_nb=instance_nb) # Define training data values - self.input_value = array([]) - self.output_value = array([]) + self.pcd = array([]) + self.mean = array([]) # Environment parameters - self.constant = False - self.data_size = [30, 2] - self.sleep = False - self.allow_requests = True + self.constant = constant + self.data_size = data_size + self.delay = delay + self.allow_requests = allow_request """ ENVIRONMENT INITIALIZATION Methods will be automatically called in this order to create and initialize Environment. """ - def recv_parameters(self, param_dict): - # If True, the same data is always sent so one can observe how the prediction crawl toward the ground truth. - self.constant = param_dict['constant'] if 'constant' in param_dict else self.constant - # Define the data size - self.data_size = param_dict['data_size'] if 'data_size' in param_dict else self.data_size - # If True, step will sleep a random time to simulate longer processes - self.sleep = param_dict['sleep'] if 'sleep' in param_dict else self.sleep - # If True, requests can be performed - self.allow_requests = param_dict['allow_requests'] if 'allow_requests' in param_dict else self.allow_requests - def create(self): - # The vector is the input of the network and the ground truth is the mean. - self.input_value = pi * random(self.data_size) - self.output_value = mean(self.input_value, axis=0) + + self.pcd = pi * random(self.data_size) + self.mean = mean(self.pcd, axis=0) + + def init_database(self): + + # Define the fields of the training Database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) def init_visualization(self): + # Point cloud (object will have id = 0) - self.factory.add_points(positions=self.input_value, + self.factory.add_points(positions=self.pcd, at=self.instance_id, c='blue', point_size=8) # Ground truth value (object will have id = 1) - self.factory.add_points(positions=self.output_value, + self.factory.add_points(positions=self.mean, at=self.instance_id, c='green', point_size=10) # Prediction value (object will have id = 2) - self.factory.add_points(positions=self.output_value, + self.factory.add_points(positions=self.mean, at=self.instance_id, c='orange', point_size=12) @@ -88,34 +80,42 @@ def init_visualization(self): """ async def step(self): - # Compute new data - if not self.constant: - self.input_value = pi * random(self.data_size) - self.output_value = mean(self.input_value, axis=0) + # Simulate longer process - if self.sleep: + if self.delay: sleep(0.01 * randint(0, 10)) + + # Compute new data + if not self.constant: + self.pcd = pi * random(self.data_size) + self.mean = mean(self.pcd, axis=0) + + # Update visualization data + if not self.constant: + # Point cloud + self.factory.update_points(object_id=0, + positions=self.pcd) + # Ground truth value + self.factory.update_points(object_id=1, + positions=self.mean) + # Send the training data - self.set_training_data(input_array=self.input_value, - output_array=self.output_value) + self.set_training_data(input=self.pcd, + ground_truth=self.mean) + if self.allow_requests: # Request a prediction for the given input - prediction = self.get_prediction(input_array=self.input_value) + prediction = self.get_prediction(input=self.pcd) # Apply this prediction in Environment self.apply_prediction(prediction) + else: + self.update_visualisation() def apply_prediction(self, prediction): - # Update visualization with new input and ground truth - if not self.constant: - # Point cloud - self.factory.update_points(object_id=0, - positions=self.input_value) - # Ground truth value - self.factory.update_points(object_id=1, - positions=self.output_value) + # Update visualization with prediction self.factory.update_points(object_id=2, - positions=prediction) + positions=prediction['prediction']) # Send visualization data to update self.update_visualisation() diff --git a/examples/features/dataGeneration_multi.py b/examples/features/dataGeneration_multi.py index 74aa3533..e9e98994 100644 --- a/examples/features/dataGeneration_multi.py +++ b/examples/features/dataGeneration_multi.py @@ -9,34 +9,41 @@ from time import time # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseDataGenerator import BaseDataGenerator +from DeepPhysX.Core.Pipelines.BaseDataGeneration import BaseDataGeneration from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer # Session related imports from Environment import MeanEnvironment def launch_data_generation(use_tcp_ip): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Environment configuration environment_config = BaseEnvironmentConfig(environment_class=MeanEnvironment, - param_dict={'constant': False, - 'data_size': [nb_points, dimension], - 'sleep': True, - 'allow_requests': False}, + visualizer=VedoVisualizer, as_tcp_ip_client=use_tcp_ip, - number_of_thread=10) + number_of_thread=5, + env_kwargs={'constant': False, + 'data_size': [nb_points, dimension], + 'delay': True, + 'allow_request': False}) # Dataset configuration - dataset_config = BaseDatasetConfig(normalize=False) + database_config = BaseDatabaseConfig(max_file_size=1, + normalize=False) # Create DataGenerator - data_generator = BaseDataGenerator(session_name='sessions/data_generation_compare', - environment_config=environment_config, - dataset_config=dataset_config, - nb_batches=20, - batch_size=10) + data_generator = BaseDataGeneration(environment_config=environment_config, + database_config=database_config, + session_dir='sessions', + session_name='data_generation_compare', + batch_nb=20, + batch_size=10) + # Launch the training session start_time = time() data_generator.execute() diff --git a/examples/features/dataGeneration_single.py b/examples/features/dataGeneration_single.py index 23c5de46..c9c0295b 100644 --- a/examples/features/dataGeneration_single.py +++ b/examples/features/dataGeneration_single.py @@ -4,33 +4,41 @@ """ # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseDataGenerator import BaseDataGenerator +from DeepPhysX.Core.Pipelines.BaseDataGeneration import BaseDataGeneration from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer # Session related imports from Environment import MeanEnvironment def launch_data_generation(): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Environment configuration environment_config = BaseEnvironmentConfig(environment_class=MeanEnvironment, - param_dict={'constant': False, - 'data_size': [nb_points, dimension], - 'sleep': False, - 'allow_requests': False}, - as_tcp_ip_client=False) - # Dataset configuration - dataset_config = BaseDatasetConfig(normalize=False) + visualizer=VedoVisualizer, + as_tcp_ip_client=False, + env_kwargs={'constant': False, + 'data_size': (nb_points, dimension), + 'delay': False, + 'allow_request': False}) + # Database configuration + database_config = BaseDatabaseConfig(max_file_size=1, + normalize=False) + # Create DataGenerator - data_generator = BaseDataGenerator(session_name='sessions/data_generation', - environment_config=environment_config, - dataset_config=dataset_config, - nb_batches=500, - batch_size=10) + data_generator = BaseDataGeneration(environment_config=environment_config, + database_config=database_config, + session_dir='sessions', + session_name='data_generation', + batch_nb=100, + batch_size=10) + # Launch the training session data_generator.execute() diff --git a/examples/features/gradientDescent.py b/examples/features/gradientDescent.py index dcb5de75..23e8d613 100644 --- a/examples/features/gradientDescent.py +++ b/examples/features/gradientDescent.py @@ -9,9 +9,9 @@ from torch.optim import Adam # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseTrainer import BaseTrainer +from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig @@ -20,37 +20,42 @@ def launch_training(): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Environment configuration environment_config = BaseEnvironmentConfig(environment_class=MeanEnvironment, visualizer=VedoVisualizer, - param_dict={'constant': True, + as_tcp_ip_client=False, + env_kwargs={'constant': True, 'data_size': [nb_points, dimension], - 'sleep': False, - 'allow_requests': True}, - as_tcp_ip_client=True, - number_of_thread=2) + 'delay': False, + 'allow_request': True}) + # Fully Connected configuration (the number of neurones on the first and last layer is defined by the total amount # of parameters in the input and the output vectors respectively) - network_config = FCConfig(loss=MSELoss, - lr=1e-3, + network_config = FCConfig(lr=1e-3, + loss=MSELoss, optimizer=Adam, dim_layers=[nb_points * dimension, nb_points * dimension, dimension], dim_output=dimension) - # Dataset configuration - dataset_config = BaseDatasetConfig(normalize=False) + + # Database configuration + database_config = BaseDatabaseConfig(normalize=False) + # Create Trainer - trainer = BaseTrainer(session_dir='sessions', - session_name='gradient_descent', - environment_config=environment_config, - dataset_config=dataset_config, - network_config=network_config, - nb_epochs=1, - nb_batches=200, - batch_size=1, - debug=True) + trainer = BaseTraining(network_config=network_config, + environment_config=environment_config, + database_config=database_config, + session_dir='sessions', + session_name='gradient_descent', + epoch_nb=1, + batch_nb=200, + batch_size=1, + debug=True) + # Launch the training session trainer.execute() diff --git a/examples/features/offlineTraining.py b/examples/features/offlineTraining.py index b03e986e..0f5b8646 100644 --- a/examples/features/offlineTraining.py +++ b/examples/features/offlineTraining.py @@ -9,42 +9,49 @@ from torch.optim import Adam # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseTrainer import BaseTrainer -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig from DeepPhysX.Torch.FC.FCConfig import FCConfig def launch_training(): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Fully Connected configuration (the number of neurones on the first and last layer is defined by the total amount # of parameters in the input and the output vectors respectively) - network_config = FCConfig(loss=MSELoss, - lr=1e-3, + network_config = FCConfig(lr=1e-3, + loss=MSELoss, optimizer=Adam, dim_layers=[nb_points * dimension, nb_points * dimension, dimension], dim_output=dimension) + # Dataset configuration with the path to the existing Dataset - dataset_config = BaseDatasetConfig(dataset_dir=os.path.join(os.getcwd(), 'sessions/data_generation'), - shuffle_dataset=True, - normalize=False) + database_config = BaseDatabaseConfig(existing_dir='sessions/data_generation', + shuffle=True, + normalize=False) + # Create DataGenerator - trainer = BaseTrainer(session_dir='sessions', - session_name='offline_training', - dataset_config=dataset_config, - network_config=network_config, - nb_epochs=1, - nb_batches=500, - batch_size=10) + trainer = BaseTraining(network_config=network_config, + database_config=database_config, + session_dir='sessions', + session_name='offline_training', + epoch_nb=1, + batch_nb=100, + batch_size=10) + # Launch the training session trainer.execute() if __name__ == '__main__': + if not os.path.exists(os.path.join(os.getcwd(), 'sessions/data_generation')): print("Existing Dataset required, 'sessions/data_generation' not found. " "Run dataGeneration_single.py script first.") from dataGeneration_single import launch_data_generation launch_data_generation() + launch_training() diff --git a/examples/features/onlineTraining.py b/examples/features/onlineTraining.py index 172b5a78..780849ec 100644 --- a/examples/features/onlineTraining.py +++ b/examples/features/onlineTraining.py @@ -8,48 +8,54 @@ from torch.optim import Adam # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseTrainer import BaseTrainer +from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer # Session imports from Environment import MeanEnvironment def launch_training(): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Environment configuration environment_config = BaseEnvironmentConfig(environment_class=MeanEnvironment, visualizer=VedoVisualizer, - param_dict={'constant': False, - 'data_size': [nb_points, dimension], - 'sleep': False, - 'allow_requests': True}, as_tcp_ip_client=True, - number_of_thread=10) + number_of_thread=5, + env_kwargs={'constant': False, + 'data_size': [nb_points, dimension], + 'delay': False, + 'allow_request': True}) + # Fully Connected configuration (the number of neurones on the first and last layer is defined by the total amount # of parameters in the input and the output vectors respectively) - network_config = FCConfig(loss=MSELoss, - lr=1e-3, + network_config = FCConfig(lr=1e-3, + loss=MSELoss, optimizer=Adam, dim_layers=[nb_points * dimension, nb_points * dimension, dimension], dim_output=dimension) + # Dataset configuration with the path to the existing Dataset - dataset_config = BaseDatasetConfig(shuffle_dataset=True, - normalize=False) + database_config = BaseDatabaseConfig(max_file_size=1, + shuffle=True, + normalize=False) # Create DataGenerator - trainer = BaseTrainer(session_dir='sessions', - session_name='online_training', - environment_config=environment_config, - dataset_config=dataset_config, - network_config=network_config, - nb_epochs=5, - nb_batches=100, - batch_size=10) + trainer = BaseTraining(network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir='sessions', + session_name='online_training', + epoch_nb=5, + batch_nb=100, + batch_size=10) + # Launch the training session trainer.execute() diff --git a/examples/features/prediction.py b/examples/features/prediction.py index 67a8e34e..7fd8adf3 100644 --- a/examples/features/prediction.py +++ b/examples/features/prediction.py @@ -7,41 +7,42 @@ import os # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer from DeepPhysX.Torch.FC.FCConfig import FCConfig -from DeepPhysX.Core.Visualizer.VedoVisualizer import VedoVisualizer -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig + # Session imports from Environment import MeanEnvironment def launch_prediction(session): + # Define the number of points and the dimension nb_points = 30 dimension = 3 + # Environment configuration environment_config = BaseEnvironmentConfig(environment_class=MeanEnvironment, visualizer=VedoVisualizer, - param_dict={'constant': False, + env_kwargs={'constant': False, 'data_size': [nb_points, dimension], - 'sleep': True, - 'allow_requests': False}, - as_tcp_ip_client=False) + 'delay': True, + 'allow_request': False}) + # Fully Connected configuration (the number of neurones on the first and last layer is defined by the total amount # of parameters in the input and the output vectors respectively) network_config = FCConfig(dim_layers=[nb_points * dimension, nb_points * dimension, dimension], dim_output=dimension) - # Dataset configuration - dataset_config = BaseDatasetConfig(normalize=False) + # Create DataGenerator - trainer = BaseRunner(session_dir='sessions', - session_name=session, - environment_config=environment_config, - network_config=network_config, - dataset_config=dataset_config, - nb_steps=100) + trainer = BasePrediction(environment_config=environment_config, + network_config=network_config, + session_dir='sessions', + session_name=session, + step_nb=100) + # Launch the training session trainer.execute() diff --git a/examples/tutorial/T1_environment.py b/examples/tutorial/T1_environment.py index 81390f4b..4220745f 100644 --- a/examples/tutorial/T1_environment.py +++ b/examples/tutorial/T1_environment.py @@ -4,7 +4,7 @@ """ # Python related imports -from numpy import array +from numpy import array, ndarray # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment @@ -14,57 +14,48 @@ class DummyEnvironment(BaseEnvironment): def __init__(self, - ip_address='localhost', - port=10000, + as_tcp_ip_client=True, instance_id=1, - number_of_instances=1, - as_tcp_ip_client=True, - environment_manager=None): + instance_nb=1): BaseEnvironment.__init__(self, - ip_address=ip_address, - port=port, + as_tcp_ip_client=as_tcp_ip_client, instance_id=instance_id, - number_of_instances=number_of_instances, - as_tcp_ip_client=as_tcp_ip_client, - environment_manager=environment_manager) + instance_nb=instance_nb) - self.nb_step = 0 - self.increment = 0 + self.step_nb: int = 0 """ INITIALIZING ENVIRONMENT - Methods will be automatically called it this order: - - recv_parameters: Receive a dictionary of parameters that can be set in EnvironmentConfig - create: Create the Environment - init: Initialize the Environment if required - - send_parameters: Same as recv_parameters, Environment can send back a set of parameters if required - - send_visualization: Send initial visualization data (see Example/CORE/Features to add visualization data) + - init_database: Define the training data fields + - init_visualization: Define and send initial visualization data """ - # Optional - def recv_parameters(self, param_dict): - # Set data size - self.increment = param_dict['increment'] if 'increment' in param_dict else 1 - # MANDATORY def create(self): + # Nothing to create in our DummyEnvironment pass # Optional def init(self): + # Nothing to init in our DummyEnvironment pass - # Optional - def send_parameters(self): - # Nothing to send back - return {} + # MANDATORY + def init_database(self): + + # Define the fields of the training Database + self.define_training_fields(fields=[('input', ndarray), ('ground_truth', ndarray)]) # Optional - def send_visualization(self): - # Nothing to visualize (see Example/CORE/Features to add visualization data) - return {} + def init_visualization(self): + + # Nothing to visualize + pass """ ENVIRONMENT BEHAVIOR - Methods will be automatically called at each simulation step in this order: @@ -79,25 +70,26 @@ def send_visualization(self): # MANDATORY async def step(self): + # Setting (and sending) training data - self.set_training_data(input_array=array([self.nb_step]), - output_array=array([self.nb_step])) - self.nb_step += self.increment - # Other data fields can be filled: - # - set_loss_data: Define an additional data to compute loss value (see Optimization.transform_loss) - # - set_additional_dataset: Add a field to the dataset + self.step_nb += 1 + self.set_training_data(input=array([self.step_nb]), + ground_truth=array([self.step_nb])) # Optional - def check_sample(self, check_input=True, check_output=True): + def check_sample(self): + # Nothing to check in our DummyEnvironment return True # Optional def apply_prediction(self, prediction): + # Nothing to apply in our DummyEnvironment - print(f"Prediction at step {self.nb_step - 1} = {prediction}") + print(f"Prediction at step {self.step_nb} = {prediction}") # Optional def close(self): + # Shutdown procedure print("Bye!") diff --git a/examples/tutorial/T2_network.py b/examples/tutorial/T2_network.py index c59ad8bb..df8b5aa8 100644 --- a/examples/tutorial/T2_network.py +++ b/examples/tutorial/T2_network.py @@ -5,24 +5,27 @@ """ # Python related imports -from numpy import save, array +from numpy import save, array, ndarray # DeepPhysX related imports from DeepPhysX.Core.Network.BaseNetwork import BaseNetwork from DeepPhysX.Core.Network.BaseOptimization import BaseOptimization +from DeepPhysX.Core.Network.BaseTransformation import BaseTransformation # Create a Network as a BaseNetwork child class class DummyNetwork(BaseNetwork): def __init__(self, config): + BaseNetwork.__init__(self, config) # There is no Network architecture to define in our DummyNetwork # MANDATORY - def forward(self, x): + def forward(self, input_data): + # Return the input - return x + return input_data """ The following methods should be already defined in a DeepPhysX AI package. @@ -57,19 +60,12 @@ def save_parameters(self, path): def nb_parameters(self): return 0 - # MANDATORY - def transform_from_numpy(self, x, grad=True): - return x - - # MANDATORY - def transform_to_numpy(self, x): - return x - # Create an Optimization as a BaseOptimization child class class DummyOptimization(BaseOptimization): def __init__(self, config): + BaseOptimization.__init__(self, config) """ @@ -82,11 +78,11 @@ def set_loss(self): pass # MANDATORY - def compute_loss(self, prediction, ground_truth, data): + def compute_loss(self, data_pred, data_opt): return {'loss': 0.} # Optional - def transform_loss(self, data): + def transform_loss(self, data_opt): pass # MANDATORY @@ -96,3 +92,27 @@ def set_optimizer(self, net): # MANDATORY def optimize(self): pass + + +# Create a BaseTransformation as a BaseTransformation child class +class DummyTransformation(BaseTransformation): + + def __init__(self, config): + + BaseTransformation.__init__(self, config) + self.data_type = ndarray + + def transform_before_prediction(self, data_net): + + # Do not transform the Network data + return data_net + + def transform_before_loss(self, data_pred, data_opt=None): + + # Do not transform the Prediction data and the Optimizer data + return data_pred, data_opt + + def transform_before_apply(self, data_pred): + + # Do not transform Prediction data + return data_pred diff --git a/examples/tutorial/T3_configuration.py b/examples/tutorial/T3_configuration.py index 964831d0..69d965f2 100644 --- a/examples/tutorial/T3_configuration.py +++ b/examples/tutorial/T3_configuration.py @@ -6,35 +6,42 @@ # DeepPhysX related imports from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig # Tutorial related imports from T1_environment import DummyEnvironment -from T2_network import DummyNetwork, DummyOptimization - +from T2_network import DummyNetwork, DummyOptimization, DummyTransformation # Create the Environment config -env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment, # The Environment class to create - visualizer=None, # The Visualizer to use - simulations_per_step=1, # The number of bus-steps to run - use_dataset_in_environment=False, # Dataset will not be sent to Environment - param_dict={'increment': 1}, # Parameters to send at init - as_tcp_ip_client=True, # Create a Client / Server architecture - number_of_thread=3, # Number of Clients connected to Server - ip_address='localhost', # IP address to use for communication - port=10001) # Port number to use for communication +env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment, # The Environment class to create + as_tcp_ip_client=True, # Create a Client / Server architecture + number_of_thread=3, # Number of Clients connected to Server + ip_address='localhost', # IP address to use for communication + port=10001, # Port number to use for communication + simulations_per_step=1, # The number of bus-steps to run + load_samples=False, # Load samples from Database to Environment + only_first_epoch=True, # Use the Environment on the first epoch only + always_produce=False) # Environment is always producing data # Create the Network config -net_config = BaseNetworkConfig(network_class=DummyNetwork, # The Network class to create - optimization_class=DummyOptimization, # The Optimization class to create - network_name='DummyNetwork', # Nickname of the Network - network_type='Dummy', # Type of the Network - save_each_epoch=False, # Do not save the network at each epoch - require_training_stuff=False, # loss and optimizer can remain at None - lr=None, # Learning rate - loss=None, # Loss class - optimizer=None) # Optimizer class +net_config = BaseNetworkConfig(network_class=DummyNetwork, # The Network class to create + optimization_class=DummyOptimization, # The Optimization class to create + data_transformation_class=DummyTransformation, # The DataTransformation class to create + network_dir=None, # Path to an existing Network repository + network_name='DummyNetwork', # Nickname of the Network + network_type='Dummy', # Type of the Network + which_network=-1, # The index of Network to load + save_each_epoch=False, # Do not save the network at each epoch + data_type='float32', # Training data type + require_training_stuff=False, # loss and optimizer can remain at None + lr=None, # Learning rate + loss=None, # Loss class + optimizer=None) # Optimizer class # Create the Dataset config -dataset_config = BaseDatasetConfig(partition_size=1, # Max size of the Dataset - shuffle_dataset=False) # Dataset should be shuffled +database_config = BaseDatabaseConfig(existing_dir=None, # Path to an existing Database + mode='training', # Database mode + max_file_size=1, # Max size of the Dataset (Gb) + shuffle=False, # Dataset should be shuffled + normalize=False, # Database should be normalized + recompute_normalization=False) # Normalization should be recomputed at loading diff --git a/examples/tutorial/T4_dataGeneration.py b/examples/tutorial/T4_dataGeneration.py index e5039e11..729ea9d5 100644 --- a/examples/tutorial/T4_dataGeneration.py +++ b/examples/tutorial/T4_dataGeneration.py @@ -4,19 +4,34 @@ """ # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseDataGenerator import BaseDataGenerator +from DeepPhysX.Core.Pipelines.BaseDataGeneration import BaseDataGeneration +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig # Tutorial related imports -from T3_configuration import env_config, dataset_config +from T1_environment import DummyEnvironment def launch_data_generation(): + + # Create the Environment config + env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment, + as_tcp_ip_client=True, + number_of_thread=3) + + # Create the Database config + database_config = BaseDatabaseConfig(max_file_size=1, + normalize=False) + # Create the Pipeline - pipeline = BaseDataGenerator(session_name='sessions/tutorial_data_generation', - dataset_config=dataset_config, - environment_config=env_config, - nb_batches=100, - batch_size=10) + pipeline = BaseDataGeneration(environment_config=env_config, + database_config=database_config, + session_dir='sessions', + session_name='tutorial_data_generation', + new_session=True, + batch_nb=20, + batch_size=10) + # Launch the Pipeline pipeline.execute() diff --git a/examples/tutorial/T5_offlineTraining.py b/examples/tutorial/T5_offlineTraining.py index d8761412..63fece14 100644 --- a/examples/tutorial/T5_offlineTraining.py +++ b/examples/tutorial/T5_offlineTraining.py @@ -3,31 +3,41 @@ Launch a training session with an existing Dataset. """ -# Python related imports -import os - # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseTrainer import BaseTrainer -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig +from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig # Tutorial related imports -from T3_configuration import env_config, net_config +from T2_network import DummyNetwork, DummyOptimization, DummyTransformation def launch_training(): - # Adapt the Dataset config with the existing dataset directory - dataset_config = BaseDatasetConfig(dataset_dir=os.path.join(os.getcwd(), 'sessions/tutorial_data_generation'), - partition_size=1, - shuffle_dataset=False) + + # Create the Network config + net_config = BaseNetworkConfig(network_class=DummyNetwork, + optimization_class=DummyOptimization, + data_transformation_class=DummyTransformation, + network_name='DummyNetwork', + network_type='Dummy', + save_each_epoch=False, + require_training_stuff=False) + + # Create the Dataset config + database_config = BaseDatabaseConfig(existing_dir='sessions/tutorial_data_generation', + shuffle=False) + # Create the Pipeline - pipeline = BaseTrainer(session_dir='sessions', - session_name='tutorial_offline_training', - environment_config=env_config, - dataset_config=dataset_config, - network_config=net_config, - nb_epochs=2, - nb_batches=100, - batch_size=10) + pipeline = BaseTraining(network_config=net_config, + database_config=database_config, + session_dir='sessions', + session_name='tutorial_offline_training', + new_session=True, + epoch_nb=2, + batch_nb=20, + batch_size=10, + debug=True) + # Launch the Pipeline pipeline.execute() diff --git a/examples/tutorial/T6_onlineTraining.py b/examples/tutorial/T6_onlineTraining.py index 3d77eb4b..3d73587a 100644 --- a/examples/tutorial/T6_onlineTraining.py +++ b/examples/tutorial/T6_onlineTraining.py @@ -4,22 +4,48 @@ """ # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseTrainer import BaseTrainer +from DeepPhysX.Core.Pipelines.BaseTraining import BaseTraining +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig # Tutorial related imports -from T3_configuration import env_config, net_config, dataset_config +from T1_environment import DummyEnvironment +from T2_network import DummyNetwork, DummyOptimization, DummyTransformation def launch_training(): + + # Create the Environment config + env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment, + as_tcp_ip_client=True, + number_of_thread=3) + + # Create the Network config + net_config = BaseNetworkConfig(network_class=DummyNetwork, + optimization_class=DummyOptimization, + data_transformation_class=DummyTransformation, + network_name='DummyNetwork', + network_type='Dummy', + save_each_epoch=False, + require_training_stuff=False) + + # Create the Dataset config + database_config = BaseDatabaseConfig(max_file_size=1, + normalize=False) + # Create the Pipeline - pipeline = BaseTrainer(session_dir='sessions', - session_name='tutorial_online_training', - environment_config=env_config, - dataset_config=dataset_config, - network_config=net_config, - nb_epochs=2, - nb_batches=100, - batch_size=10) + pipeline = BaseTraining(network_config=net_config, + database_config=database_config, + environment_config=env_config, + session_dir='sessions', + session_name='tutorial_online_training', + new_session=True, + epoch_nb=2, + batch_nb=20, + batch_size=10, + debug=True) + # Launch the Pipeline pipeline.execute() diff --git a/examples/tutorial/T7_prediction.py b/examples/tutorial/T7_prediction.py index 526eb26a..4752f349 100644 --- a/examples/tutorial/T7_prediction.py +++ b/examples/tutorial/T7_prediction.py @@ -4,28 +4,30 @@ """ # DeepPhysX related imports -from DeepPhysX.Core.Pipelines.BaseRunner import BaseRunner +from DeepPhysX.Core.Pipelines.BasePrediction import BasePrediction from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig # Tutorial related imports -from T3_configuration import DummyEnvironment, net_config, dataset_config +from T1_environment import DummyEnvironment +from T2_network import DummyNetwork def launch_prediction(): - # Adapt the Environment config to avoid using Client / Server Architecture - env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment, - visualizer=None, - simulations_per_step=1, - use_dataset_in_environment=False, - param_dict={'increment': 1}, - as_tcp_ip_client=False) + + # Create the Environment config + env_config = BaseEnvironmentConfig(environment_class=DummyEnvironment) + + # Create the Network config + net_config = BaseNetworkConfig(network_class=DummyNetwork) + # Create the Pipeline - pipeline = BaseRunner(session_dir='sessions', - session_name='tutorial_online_training', - environment_config=env_config, - dataset_config=dataset_config, - network_config=net_config, - nb_steps=20) + pipeline = BasePrediction(network_config=net_config, + environment_config=env_config, + session_dir='sessions', + session_name='tutorial_online_training', + step_nb=20) + # Launch the Pipeline pipeline.execute() diff --git a/requirements.txt b/requirements.txt index 859b901f..8b68c135 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ numpy SimulationSimpleDatabase tensorboard -tensorboardX pyDataverse torch \ No newline at end of file diff --git a/setup.py b/setup.py index 7bc7443a..72acda40 100644 --- a/setup.py +++ b/setup.py @@ -1,69 +1,40 @@ -# SHARED: packages are installed within the shared namespace 'DeepPhysX'. Local install with pip. -# SINGLE: only Core package is installed with name 'DeepPhysX'. Distant install with pip. - from setuptools import setup, find_packages -from os.path import join, pardir, exists -from json import load, dump +from os.path import join PROJECT = 'DeepPhysX' PACKAGE = 'Core' -PACKAGES = {'Torch': False, - 'Sofa': False} -DEPENDENCIES = {'Core': ['numpy', 'SimulationSimpleDatabase', 'tensorboard', 'tensorboardX', 'pyDataverse'], - 'Sofa': [], - 'Torch': ['torch']} - -# (SHARED) Loading existing configuration file -if exists('config.json'): - with open('config.json') as file: - PACKAGES = load(file) - # Check config validity - correct_config = True - for package_name, do_install in PACKAGES.items(): - if do_install and not exists(join(pardir, package_name)): - PACKAGES[package_name] = False - correct_config = False - # Write correction - if not correct_config: - with open('config.json', 'w') as file: - dump(PACKAGES, file) -# (SINGLE / SHARED) Getting the packages to be installed -roots = [PACKAGE] -for package_name, do_install in PACKAGES.items(): - if do_install: - roots.append(package_name) -packages = [] -packages_dir = {} -requires = [] +packages = [f'{PROJECT}'] +packages_dir = {f'{PROJECT}': 'src'} -# (SINGLE) Specifying package list and corresponding directories -if len(roots) == 1: - packages.append(f'{PROJECT}.{PACKAGE}') - packages_dir[f'{PROJECT}.{PACKAGE}'] = 'src' - requires += DEPENDENCIES[PACKAGE] - for subpackage in find_packages(where='src'): - packages.append(f'{PROJECT}.{PACKAGE}.{subpackage}') - packages_dir[f'{PROJECT}.{PACKAGE}.{subpackage}'] = join('src', *subpackage.split('.')) +# Configure packages list and directories +for subpackage in find_packages(where='src'): + packages.append(f'{PROJECT}.{subpackage}') + packages_dir[f'{PROJECT}.{subpackage}'] = join('src', *subpackage.split('.')) -# (SHARED) Specifying package list and corresponding directories -else: - for package_name in roots: - packages.append(f'{PROJECT}.{package_name}') - packages_dir[f'{PROJECT}.{package_name}'] = join(pardir, package_name, 'src') - requires += DEPENDENCIES[package_name] - for sub_package in find_packages(where=join(pardir, package_name, 'src')): - packages.append(f'{PROJECT}.{package_name}.{sub_package}') - packages_dir[f'{PROJECT}.{package_name}.{sub_package}'] = join(pardir, package_name, 'src', - *sub_package.split('.')) +# Add examples as subpackages +packages.append(f'{PROJECT}.examples.{PACKAGE}') +packages_dir[f'{PROJECT}.examples.{PACKAGE}'] = 'examples' +for example_dir in find_packages(where='examples'): + packages.append(f'{PROJECT}.examples.{PACKAGE}.{example_dir}') -# (SINGLE / SHARED) Extract README.md content +# Extract README.md content with open('README.md') as f: long_description = f.read() -# (SINGLE / SHARED) Installation -setup(name='DeepPhysX', - version='22.06', + +def get_SSD(): + # If SSD was installed in dev mode, pip will re-install it + try: + import SSD + except ModuleNotFoundError: + return ['SimulationSimpleDatabase >= 22.12'] + return [] + + +# Installation +setup(name=f'{PROJECT}', + version='22.12', description='A Python framework interfacing AI with numerical simulation.', long_description=long_description, long_description_content_type='text/markdown', @@ -72,4 +43,10 @@ url='https://github.com/mimesis-inria/DeepPhysX', packages=packages, package_dir=packages_dir, - install_requires=requires) + namespace_packages=[PROJECT], + install_requires=get_SSD() + ['numpy >= 1.23.5', + 'tensorboard >= 2.10.0', + 'tensorboardX >= 2.5.1', + 'pyDataverse >= 0.3.1', + 'torch >= 1.13.0'], + entry_points={'console_scripts': ['DPX=DeepPhysX.cli:execute_cli']}) diff --git a/setup_dev.py b/setup_dev.py new file mode 100644 index 00000000..948f487c --- /dev/null +++ b/setup_dev.py @@ -0,0 +1,152 @@ +from os import sep, chdir, mkdir, listdir, getcwd, rename, symlink, unlink, remove +from os.path import dirname, join, isfile, exists, isdir, islink +from pathlib import Path +from shutil import move, rmtree, which +from site import USER_SITE +from subprocess import run +from sys import argv + +from pip._internal.operations.install.wheel import PipScriptMaker + +PROJECT = 'DeepPhysX' +AI_PACKAGES = ['Torch'] +SIMU_PACKAGES = ['Sofa'] +PACKAGES = ['Core'] + AI_PACKAGES + SIMU_PACKAGES +GIT = {'Torch': 'https://github.com/mimesis-inria/DeepPhysX.Torch.git', + 'Sofa': 'https://github.com/mimesis-inria/DeepPhysX.Sofa.git'} + + +def check_repositories(): + + dpx_path = dirname(dirname(__file__)) + + # Check the DeepPhysX root + if dpx_path.split(sep)[-1] != PROJECT: + # Create the right root + chdir(dpx_path) + if dirname(__file__).split(sep)[-1] == PROJECT: + rename(PROJECT, f'{PROJECT}.Core') + mkdir(PROJECT) + # Move DeepPhysX packages in the root + for repository in listdir(getcwd()): + if is_dpx_package(repository): + move(src=join(dpx_path, repository), + dst=join(dpx_path, PROJECT)) + dpx_path = join(dpx_path, PROJECT) + + # Check the packages repositories + for repository in listdir(dpx_path): + # Core package + if repository == PROJECT: + rename(repository, 'Core') + # Layers + if 'DeepPhysX.' in repository and repository[10:] in PACKAGES: + rename(src=join(dpx_path, repository), + dst=join(dpx_path, repository[10:])) + + return dpx_path + + +def is_dpx_package(repository): + + for key in ['DeepPhysX', 'DeepPhysX.'] + PACKAGES: + if key in repository and isfile(join(repository, 'README.md')): + with open(join(repository, 'README.md')) as f: + if PROJECT in f.readline(): + return True + return False + + +def define_config(root_dir): + + # Get the user configuration + config = ['Core'] + answers = ['y', 'yes', 'n', 'no'] + for package_list, package_type in zip([SIMU_PACKAGES, AI_PACKAGES], ['SIMULATION', 'AI']): + print(f"\nAvailable {package_type} packages: {[f'{PROJECT}.{pkg}' for pkg in package_list]}") + for pkg in package_list: + while (user := input(f" >> Install package {f'{PROJECT}.{pkg}'} (y/n): ").lower()) not in answers: + pass + if user in answers[:2]: + config.append(pkg) + print(f"\nThe following packages will be installed : {[f'{PROJECT}.{package}' for package in config]}") + while (user := input("Confirm (y/n): ").lower()) not in answers: + pass + if user in answers[2:]: + quit(print("Aborting.")) + + # Clone the missing packages + chdir(root_dir) + if len(config) > 1: + for pkg in config[1:]: + if not exists(pkg): + print(f"\nPackage {f'{PROJECT}.{pkg}'} not found, cloning from {GIT[pkg]}...") + run(['git', 'clone', f'{GIT[pkg]}', f'{pkg}'], cwd=root_dir) + + return config + + +if __name__ == '__main__': + + # Check the project tree + root = check_repositories() + + # Check user entry + if len(argv) == 2 and argv[1] not in ['set', 'del']: + quit(print(f"\nInvalid script option." + f"\nRun 'python3 setup_dev.py set to link {PROJECT} to your site-packages folder." + f"\nRun 'python3 setup_dev.py del to remove {PROJECT} link from your site-packages folder.")) + + # Option 1: create the symbolic links + if len(argv) == 1 or argv[1] == 'set': + + # Get the user configuration + packages = define_config(root) + + # Create the main repository in site-packages + if not isdir(join(USER_SITE, PROJECT)): + mkdir(join(USER_SITE, PROJECT)) + + # Create symbolic links in site-packages + for package in packages: + if not islink(join(USER_SITE, PROJECT, package)): + symlink(src=join(root, package, 'src', package), + dst=join(USER_SITE, PROJECT, package)) + print(f"\nLinked {join(USER_SITE, PROJECT, package)} -> {join(root, package)}") + + # Add examples and the CLI script + if not isdir(join(USER_SITE, PROJECT, 'examples')): + symlink(src=join(Path(__file__).parent.absolute(), 'examples'), + dst=join(USER_SITE, PROJECT, 'examples')) + print(f"\nLinked {join(USER_SITE, PROJECT, 'examples')} -> {join(Path(__file__).parent.absolute(), 'examples')}") + if not isfile(join(USER_SITE, PROJECT, 'cli.py')): + symlink(src=join(Path(__file__).parent.absolute(), 'src', 'cli.py'), + dst=join(USER_SITE, PROJECT, 'cli.py')) + + # Create the CLI + if which('DPX') is None: + # Generate the scripts + maker = PipScriptMaker(None, dirname(which('vedo'))) + generated_scripts = maker.make_multiple(['DPX = DeepPhysX.cli:execute_cli']) + for script in generated_scripts: + if script.split(sep)[-1].split('.')[0] != 'DPX': + remove(script) + + # Option 2: remove the symbolic links + else: + + # Remove everything from site-packages + if isdir(join(USER_SITE, PROJECT)): + for package in listdir(join(USER_SITE, PROJECT)): + if islink(join(USER_SITE, PROJECT, package)): + unlink(join(USER_SITE, PROJECT, package)) + print(f"Unlinked {join(USER_SITE, PROJECT, package)} -> {join(root, package)}") + elif isdir(join(USER_SITE, PROJECT, package)): + rmtree(join(USER_SITE, PROJECT, package)) + elif isfile(join(USER_SITE, PROJECT, package)): + remove(join(USER_SITE, PROJECT, package)) + rmtree(join(USER_SITE, PROJECT)) + + # Remove the CLI + if isfile(which('DPX')): + remove(which('DPX')) diff --git a/setup_user.py b/setup_user.py new file mode 100644 index 00000000..bc7cd348 --- /dev/null +++ b/setup_user.py @@ -0,0 +1,25 @@ +from os.path import join +from subprocess import run + +from setup_dev import check_repositories, define_config + +PROJECT = 'DeepPhysX' +AI_PACKAGES = ['Torch'] +SIMU_PACKAGES = ['Sofa'] +PACKAGES = ['Core'] + AI_PACKAGES + SIMU_PACKAGES +GIT = {'Torch': 'https://github.com/mimesis-inria/DeepPhysX.Torch.git', + 'Sofa': 'https://github.com/mimesis-inria/DeepPhysX.Sofa.git'} + + +if __name__ == '__main__': + + # Check the project tree + root = check_repositories() + + # Get the user configuration + config = define_config(root) + + # Pip install packages + for package in config: + print(f"\nInstalling {f'{PROJECT}.{package}'} package...") + run(['pip', 'install', '.'], cwd=join(root, package)) diff --git a/src/AsyncSocket/AbstractEnvironment.py b/src/AsyncSocket/AbstractEnvironment.py deleted file mode 100644 index c8519105..00000000 --- a/src/AsyncSocket/AbstractEnvironment.py +++ /dev/null @@ -1,121 +0,0 @@ -from typing import Optional, Dict, Any -from numpy import array, ndarray - - -class AbstractEnvironment: - - def __init__(self, - as_tcp_ip_client: bool = True, - instance_id: int = 0, - number_of_instances: int = 1): - """ - AbstractEnvironment sets the Environment API for TcpIpClient. - Do not use AbstractEnvironment to implement a personal Environment, use BaseEnvironment instead. - - :param instance_id: ID of the instance. - :param number_of_instances: Number of simultaneously launched instances. - :param as_tcp_ip_client: Environment is a TcpIpObject if True, is owned by an EnvironmentManager if False. - """ - - self.name: str = self.__class__.__name__ + f" n°{instance_id}" - - # TcpIpClient variables - if instance_id > number_of_instances: - raise ValueError(f"[{self.name}] Instance ID ({instance_id}) is bigger than max instances " - f"({number_of_instances}).") - self.as_tcp_ip_client: bool = as_tcp_ip_client - self.instance_id: int = instance_id - self.number_of_instances: int = number_of_instances - - # Training data variables - self.input: ndarray = array([]) - self.output: ndarray = array([]) - self.loss_data: Any = None - self.compute_essential_data: bool = True - - # Dataset data variables - self.sample_in: Optional[ndarray] = None - self.sample_out: Optional[ndarray] = None - self.additional_fields: Dict[str, Any] = {} - - def create(self) -> None: - """ - Create the Environment. Automatically called when Environment is launched. - Must be implemented by user. - """ - - raise NotImplementedError - - def init(self) -> None: - """ - Initialize the Environment. Automatically called when Environment is launched. - Not mandatory. - """ - - pass - - def close(self) -> None: - """ - Close the Environment. Automatically called when Environment is shut down. - Not mandatory. - """ - - pass - - async def step(self) -> None: - """ - Compute the number of steps in the Environment specified by simulations_per_step in EnvironmentConfig. - Must be implemented by user. - """ - - raise NotImplementedError - - def check_sample(self) -> bool: - """ - Check if the current produced sample is usable. - Not mandatory. - - :return: Current sample can be used or not - """ - - return True - - def recv_parameters(self, - param_dict: Dict[str, Any]) -> None: - """ - Receive simulation parameters set in Config from the Server. Automatically called before create and init. - - :param param_dict: Dictionary of parameters. - """ - - pass - - def init_visualization(self) -> None: - """ - Define the visualization objects to send to the Visualizer. Automatically called after create and init. - Not mandatory. - - :return: Initial visualization data dictionary. - """ - - pass - - def send_parameters(self) -> dict: - """ - Create a dictionary of parameters to send to the manager. Automatically called after create and init. - Not mandatory. - - :return: Dictionary of parameters. - """ - - return {} - - def apply_prediction(self, prediction: ndarray) -> None: - """ - Apply network prediction in environment. - Not mandatory. - - :param prediction: Prediction data. - """ - - pass diff --git a/src/Core/AsyncSocket/AbstractEnvironment.py b/src/Core/AsyncSocket/AbstractEnvironment.py new file mode 100644 index 00000000..229f2de7 --- /dev/null +++ b/src/Core/AsyncSocket/AbstractEnvironment.py @@ -0,0 +1,104 @@ +from typing import Optional, Dict, Any, Union, Tuple +from numpy import ndarray + +from SSD.Core.Storage.Database import Database + +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler + + +class AbstractEnvironment: + + def __init__(self, + as_tcp_ip_client: bool = True, + instance_id: int = 1, + instance_nb: int = 1): + """ + AbstractEnvironment sets the Environment API for TcpIpClient. + Do not use AbstractEnvironment to implement an Environment, use BaseEnvironment instead. + + :param as_tcp_ip_client: If True, the Environment is a TcpIpObject, else it is owned by an EnvironmentManager. + :param instance_id: Index of this instance. + :param instance_nb: Number of simultaneously launched instances. + """ + + self.name: str = self.__class__.__name__ + f" n°{instance_id}" + + # TcpIpClient variables + if instance_id > instance_nb: + raise ValueError(f"[{self.name}] Instance ID ({instance_id}) is bigger than max instances " + f"({instance_nb}).") + self.as_tcp_ip_client: bool = as_tcp_ip_client + self.instance_id: int = max(1, instance_id) + self.instance_nb: int = instance_nb + + # Manager of the Environment + self.environment_manager: Any = None + self.tcp_ip_client: Any = None + + # Training data variables + self.compute_training_data: bool = True + + # Dataset data variables + self.update_line: Optional[int] = None + self.sample_training: Optional[Dict[str, Any]] = None + self.sample_additional: Optional[Dict[str, Any]] = None + + ########################################################################################## + ########################################################################################## + # Environment initialization # + ########################################################################################## + ########################################################################################## + + def create(self) -> None: + raise NotImplementedError + + def init(self) -> None: + raise NotImplementedError + + def init_database(self) -> None: + raise NotImplementedError + + def init_visualization(self) -> None: + raise NotImplementedError + + ########################################################################################## + ########################################################################################## + # Environment behavior # + ########################################################################################## + ########################################################################################## + + async def step(self) -> None: + raise NotImplementedError + + def check_sample(self) -> bool: + raise NotImplementedError + + def apply_prediction(self, prediction: Dict[str, ndarray]) -> None: + raise NotImplementedError + + def close(self) -> None: + raise NotImplementedError + + ########################################################################################## + ########################################################################################## + # Defining data samples # + ########################################################################################## + ########################################################################################## + + def get_database_handler(self) -> DatabaseHandler: + raise NotImplementedError + + def _create_visualization(self, visualization_db: Union[Database, Tuple[str, str]]) -> None: + raise NotImplementedError + + def _send_training_data(self) -> None: + raise NotImplementedError + + def _reset_training_data(self) -> None: + raise NotImplementedError + + def _update_training_data(self, line_id: int) -> None: + raise NotImplementedError + + def _get_training_data(self, line_id: int) -> None: + raise NotImplementedError diff --git a/src/AsyncSocket/BytesConverter.py b/src/Core/AsyncSocket/BytesConverter.py similarity index 80% rename from src/AsyncSocket/BytesConverter.py rename to src/Core/AsyncSocket/BytesConverter.py index 4992472c..37f63361 100644 --- a/src/AsyncSocket/BytesConverter.py +++ b/src/Core/AsyncSocket/BytesConverter.py @@ -6,12 +6,12 @@ class BytesConverter: - """ - | Convert usual types to bytes and vice versa. - | Available types: None, bytes, str, bool, int, float, list, ndarray. - """ def __init__(self): + """ + Convert usual types to bytes and vice versa. + Available types: None, bytes, str, bool, int, float, list, ndarray. + """ # Data to bytes conversions self.__data_to_bytes_conversion: Dict[type, Callable[[Convertible], bytes]] = { @@ -42,17 +42,17 @@ def __init__(self): self.size_from_bytes: Callable[[bytes], int] = lambda b: self.__bytes_to_data_conversion[int.__name__](b) self.int_size: int = calcsize("i") - def data_to_bytes(self, data: Convertible, as_list: bool = False) -> Union[bytes, List[bytes]]: + def data_to_bytes(self, + data: Convertible, + as_list: bool = False) -> Union[bytes, List[bytes]]: """ - | Convert data to bytes. - | Available types: None, bytes, str, bool, signed int, float, list, ndarray. + Convert data to bytes. + Available types: None, bytes, str, bool, signed int, float, list, ndarray. :param data: Data to convert. - :type data: Union[type(None), bytes, str, bool, int, float, List, ndarray] - :param bool as_list: (For tests only, False by default) - If False, the whole bytes message is returned. - If True, the return will be a list of bytes fields. - :return: Concatenated bytes fields (Number of fields, Size of fields, Type, Data, Args) + :param as_list: (For tests only, False by default) If False, the whole bytes message is returned. If True, the + return will be a list of bytes fields. + :return: Concatenated bytes fields (Number of fields, Size of fields, Type, Data, Args). """ # Convert the type of 'data' from str to bytes @@ -91,13 +91,14 @@ def data_to_bytes(self, data: Convertible, as_list: bool = False) -> Union[bytes bytes_message += f return bytes_message - def bytes_to_data(self, bytes_fields: List[bytes]) -> Convertible: + def bytes_to_data(self, + bytes_fields: List[bytes]) -> Convertible: """ - | Recover data from bytes fields. - | Available types: None, bytes, str, bool, signed int, float, list, ndarray. + Recover data from bytes fields. + Available types: None, bytes, str, bool, signed int, float, list, ndarray. - :param List[bytes] bytes_fields: Bytes fields (Type, Data, Args) - :return: Converted data + :param bytes_fields: Bytes fields (Type, Data, Args). + :return: Converted data. """ # Recover the data type diff --git a/src/AsyncSocket/TcpIpClient.py b/src/Core/AsyncSocket/TcpIpClient.py similarity index 51% rename from src/AsyncSocket/TcpIpClient.py rename to src/Core/AsyncSocket/TcpIpClient.py index 6ac2e00c..c46a9fa2 100644 --- a/src/AsyncSocket/TcpIpClient.py +++ b/src/Core/AsyncSocket/TcpIpClient.py @@ -1,48 +1,45 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, Type from socket import socket from asyncio import get_event_loop from asyncio import AbstractEventLoop as EventLoop from asyncio import run as async_run -from numpy import ndarray, array +from numpy import ndarray from DeepPhysX.Core.AsyncSocket.TcpIpObject import TcpIpObject from DeepPhysX.Core.AsyncSocket.AbstractEnvironment import AbstractEnvironment -class TcpIpClient(TcpIpObject, AbstractEnvironment): +class TcpIpClient(TcpIpObject): def __init__(self, + environment: Type[AbstractEnvironment], ip_address: str = 'localhost', port: int = 10000, - as_tcp_ip_client: bool = True, instance_id: int = 0, - number_of_instances: int = 1,): + instance_nb: int = 1): """ - TcpIpClient is both a TcpIpObject which communicate with a TcpIpServer and an AbstractEnvironment to compute - simulated data. + TcpIpClient is a TcpIpObject which communicate with a TcpIpServer and manages an Environment to compute data. + :param environment: Environment class. :param ip_address: IP address of the TcpIpObject. :param port: Port number of the TcpIpObject. - :param as_tcp_ip_client: Environment is a TcpIpObject if True, is owned by an EnvironmentManager if False. - :param instance_id: ID of the instance. - :param number_of_instances: Number of simultaneously launched instances. + :param instance_id: Index of this instance. + :param instance_nb: Number of simultaneously launched instances. """ - AbstractEnvironment.__init__(self, - as_tcp_ip_client=as_tcp_ip_client, - instance_id=instance_id, - number_of_instances=number_of_instances) - - # Bind to client address - if self.as_tcp_ip_client: - TcpIpObject.__init__(self, - ip_address=ip_address, - port=port) - self.sock.connect((ip_address, port)) - # Send ID - self.sync_send_labeled_data(data_to_send=instance_id, label="instance_ID", receiver=self.sock, - send_read_command=False) - # Flag to trigger client's shutdown + TcpIpObject.__init__(self, + ip_address=ip_address, + port=port) + + # Environment instance + self.environment: AbstractEnvironment + self.environment_class = environment + self.environment_instance = (instance_id, instance_nb) + + # Bind to client address and send ID + self.sock.connect((ip_address, port)) + self.sync_send_labeled_data(data_to_send=instance_id, label="instance_ID", receiver=self.sock, + send_read_command=False) self.close_client: bool = False ########################################################################################## @@ -53,37 +50,62 @@ def __init__(self, def initialize(self) -> None: """ - Run __initialize method with asyncio. + Receive parameters from the server to create environment. """ async_run(self.__initialize()) async def __initialize(self) -> None: """ - Receive parameters from the server to create environment, send parameters to the server in exchange. + Receive parameters from the server to create environment. """ loop = get_event_loop() + + # Receive additional arguments + env_kwargs = {} + await self.receive_dict(recv_to=env_kwargs, loop=loop, sender=self.sock) + env_kwargs = env_kwargs['env_kwargs'] if 'env_kwargs' in env_kwargs else {} + + self.environment = self.environment_class(as_tcp_ip_client=True, + instance_id=self.environment_instance[0], + instance_nb=self.environment_instance[1], + **env_kwargs) + self.environment.tcp_ip_client = self + + # Receive prediction requests authorization + self.allow_prediction_requests = await self.receive_data(loop=loop, sender=self.sock) + # Receive number of sub-steps self.simulations_per_step = await self.receive_data(loop=loop, sender=self.sock) - # Receive parameters - recv_param_dict = {} - await self.receive_dict(recv_to=recv_param_dict, sender=self.sock, loop=loop) - # Use received parameters - if 'parameters' in recv_param_dict: - self.recv_parameters(recv_param_dict['parameters']) - - # Create the environment - self.create() - self.init() - self.init_visualization() - - # Send parameters - param_dict = self.send_parameters() - await self.send_dict(name="parameters", dict_to_send=param_dict, loop=loop, receiver=self.sock) + + # Receive partitions + partitions_list = await self.receive_data(loop=loop, sender=self.sock) + partitions_list, exchange = partitions_list.split('%%%') + partitions = [[partitions_list.split('///')[0], partition_name] + for partition_name in partitions_list.split('///')[1:]] + exchange = [exchange.split('///')[0], exchange.split('///')[1]] + self.environment.get_database_handler().init_remote(storing_partitions=partitions, + exchange_db=exchange) + + # Receive visualization database + visualization_db = await self.receive_data(loop=loop, sender=self.sock) + visualization_db = None if visualization_db == 'None' else visualization_db.split('///') + + # Initialize the environment + self.environment.create() + self.environment.init() + self.environment.init_database() + if visualization_db is not None: + self.environment._create_visualization(visualization_db=visualization_db) + self.environment.init_visualization() # Initialization done - await self.send_command_done(loop=loop, receiver=self.sock) + await self.send_data(data_to_send='done', loop=loop, receiver=self.sock) + + # Synchronize Database + _ = await self.receive_data(loop=loop, sender=self.sock) + self.environment.get_database_handler().load() ########################################################################################## ########################################################################################## @@ -93,7 +115,7 @@ async def __initialize(self) -> None: def launch(self) -> None: """ - Run __launch method with asyncio. + Trigger the main communication protocol with the server. """ async_run(self.__launch()) @@ -132,7 +154,7 @@ async def __close(self) -> None: # Close environment try: - self.close() + self.environment.close() except NotImplementedError: pass # Confirm exit command to the server @@ -141,79 +163,29 @@ async def __close(self) -> None: # Close socket self.sock.close() - ########################################################################################## - ########################################################################################## - # Data sending to Server # - ########################################################################################## - ########################################################################################## - - async def __send_training_data(self, - loop: Optional[EventLoop] = None, - receiver: Optional[socket] = None) -> None: - """ - Send the training data to the TcpIpServer. - - :param loop: get_event_loop() return. - :param receiver: TcpIpObject receiver. - """ - - loop = get_event_loop() if loop is None else loop - receiver = self.sock if receiver is None else receiver - - # Send and reset network input - if self.input.tolist(): - await self.send_labeled_data(data_to_send=self.input, label="input", loop=loop, receiver=receiver) - self.input = array([]) - # Send network output - if self.output.tolist(): - await self.send_labeled_data(data_to_send=self.output, label="output", loop=loop, receiver=receiver) - self.output = array([]) - # Send loss data - if self.loss_data: - await self.send_labeled_data(data_to_send=self.loss_data, label='loss', loop=loop, receiver=receiver) - self.loss_data = None - # Send additional dataset fields - for key in self.additional_fields.keys(): - await self.send_labeled_data(data_to_send=self.additional_fields[key], label='dataset_' + key, - loop=loop, receiver=receiver) - self.additional_fields = {} - - def send_prediction_data(self, - network_input: ndarray, - receiver: socket = None) -> ndarray: - """ - Request a prediction from the Environment. - - :param network_input: Data to send under the label 'input'. - :param receiver: TcpIpObject receiver. - :return: Prediction of the Network. - """ - - receiver = self.sock if receiver is None else receiver - # Send prediction command - self.sync_send_command_prediction() - # Send the network input - self.sync_send_labeled_data(data_to_send=network_input, label='input', receiver=receiver) - # Receive the network prediction - _, pred = self.sync_receive_labeled_data() - return pred - ########################################################################################## ########################################################################################## # Available requests to Server # ########################################################################################## ########################################################################################## - def request_get_prediction(self, - input_array: ndarray) -> ndarray: + def get_prediction(self, **kwargs) -> Dict[str, ndarray]: """ Request a prediction from Network. - :param input_array: Network input. :return: Prediction of the Network. """ - return self.send_prediction_data(network_input=input_array) + # Get a prediction + self.environment.get_database_handler().update(table_name='Exchange', + data=kwargs, + line_id=self.environment.instance_id) + self.sync_send_command_prediction() + _ = self.sync_receive_data() + data_pred = self.environment.get_database_handler().get_line(table_name='Exchange', + line_id=self.environment.instance_id) + del data_pred['id'] + return data_pred def request_update_visualization(self) -> None: """ @@ -221,7 +193,7 @@ def request_update_visualization(self) -> None: """ self.sync_send_command_visualisation() - self.sync_send_labeled_data(data_to_send=self.instance_id, + self.sync_send_labeled_data(data_to_send=self.environment.instance_id, label='instance') ########################################################################################## @@ -240,7 +212,7 @@ async def action_on_exit(self, :param data: Dict storing data. :param client_id: ID of the TcpIpClient. - :param loop: asyncio.get_event_loop() return. + :param loop: Asyncio event loop. :param sender: TcpIpObject sender. """ @@ -257,14 +229,14 @@ async def action_on_prediction(self, :param data: Dict storing data. :param client_id: ID of the TcpIpClient. - :param loop: asyncio.get_event_loop() return. + :param loop: Asyncio event loop. :param sender: TcpIpObject sender. """ # Receive prediction prediction = await self.receive_data(loop=loop, sender=sender) # Apply the prediction in Environment - self.apply_prediction(prediction) + self.environment.apply_prediction(prediction) async def action_on_sample(self, data: ndarray, @@ -276,24 +248,12 @@ async def action_on_sample(self, :param data: Dict storing data. :param client_id: ID of the TcpIpClient. - :param loop: asyncio.get_event_loop() return. + :param loop: Asyncio event loop. :param sender: TcpIpObject sender. """ - # Receive input sample - if await self.receive_data(loop=loop, sender=sender): - self.sample_in = await self.receive_data(loop=loop, sender=sender) - # Receive output sample - if await self.receive_data(loop=loop, sender=sender): - self.sample_out = await self.receive_data(loop=loop, sender=sender) - - additional_fields = {} - # Receive additional input sample if there are any - if await self.receive_data(loop=loop, sender=sender): - await self.receive_dict(recv_to=additional_fields, loop=loop, sender=sender) - - # Set the samples from Dataset - self.additional_fields = additional_fields.get('additional_fields', {}) + dataset_batch = await self.receive_data(loop=loop, sender=sender) + self.environment._get_training_data(dataset_batch) async def action_on_step(self, data: ndarray, @@ -305,25 +265,46 @@ async def action_on_step(self, :param data: Dict storing data. :param client_id: ID of the TcpIpClient. - :param loop: asyncio.get_event_loop() return. + :param loop: Asyncio event loop. :param sender: TcpIpObject sender. """ # Execute the required number of steps for step in range(self.simulations_per_step): # Compute data only on final step - self.compute_essential_data = step == self.simulations_per_step - 1 - await self.step() + self.compute_training_data = step == self.simulations_per_step - 1 + await self.environment.step() # If produced sample is not usable, run again - # Todo: add the max_rate here - if self.sample_in is None and self.sample_out is None: - while not self.check_sample(): - for step in range(self.simulations_per_step): - # Compute data only on final step - self.compute_essential_data = step == self.simulations_per_step - 1 - await self.step() + while not self.environment.check_sample(): + for step in range(self.simulations_per_step): + # Compute data only on final step + self.compute_training_data = step == self.simulations_per_step - 1 + await self.environment.step() # Sent training data to Server - await self.__send_training_data() - await self.send_command_done() + if self.environment.update_line is None: + line = self.environment._send_training_data() + else: + self.environment._update_training_data(self.environment.update_line) + line = self.environment.update_line + self.environment._reset_training_data() + await self.send_command_done(loop=loop, receiver=sender) + await self.send_data(data_to_send=line, loop=loop, receiver=sender) + + async def action_on_change_db(self, + data: Dict[Any, Any], + client_id: int, sender: socket, + loop: EventLoop) -> None: + """ + Action to run when receiving the 'step' command. + + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. + """ + + # Update the partition list in the DatabaseHandler + new_database = await self.receive_data(loop=loop, sender=sender) + self.environment.get_database_handler().update_list_partitions_remote(new_database.split('///')) diff --git a/src/AsyncSocket/TcpIpObject.py b/src/Core/AsyncSocket/TcpIpObject.py similarity index 53% rename from src/AsyncSocket/TcpIpObject.py rename to src/Core/AsyncSocket/TcpIpObject.py index 7d2f247d..32bf1b7a 100644 --- a/src/AsyncSocket/TcpIpObject.py +++ b/src/Core/AsyncSocket/TcpIpObject.py @@ -15,16 +15,16 @@ class TcpIpObject: - """ - | TcpIpObject defines communication protocols to send and receive data and commands. - - :param str ip_address: IP address of the TcpIpObject - :param int port: Port number of the TcpIpObject - """ def __init__(self, ip_address: str = 'localhost', port: int = 10000): + """ + TcpIpObject defines communication protocols to send and receive data and commands. + + :param ip_address: IP address of the TcpIpObject. + :param port: Port number of the TcpIpObject. + """ self.name: str = self.__class__.__name__ @@ -39,7 +39,7 @@ def __init__(self, # Available commands self.command_dict: Dict[str, bytes] = {'exit': b'exit', 'step': b'step', 'done': b'done', 'finished': b'fini', 'prediction': b'pred', 'read': b'read', 'sample': b'samp', - 'visualisation': b'visu'} + 'visualisation': b'visu', 'db': b'chdb'} self.action_on_command: Dict[bytes, Any] = { self.command_dict["exit"]: self.action_on_exit, self.command_dict["step"]: self.action_on_step, @@ -49,10 +49,8 @@ def __init__(self, self.command_dict["read"]: self.action_on_read, self.command_dict["sample"]: self.action_on_sample, self.command_dict["visualisation"]: self.action_on_visualisation, + self.command_dict['db']: self.action_on_change_db } - # Synchronous variables - # self.send_lock = Lock() - # self.receive_lock = Lock() ########################################################################################## ########################################################################################## @@ -60,15 +58,16 @@ def __init__(self, ########################################################################################## ########################################################################################## - async def send_data(self, data_to_send: Convertible, loop: Optional[EventLoop] = None, + async def send_data(self, + data_to_send: Convertible, + loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: """ - | Send data through the given socket. + Send data through the given socket. - :param data_to_send: Data that will be sent on socket - :type data_to_send: Union[type(None), bytes, str, bool, int, float, List, ndarray] - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: socket receiver + :param data_to_send: Data that will be sent on socket. + :param loop: Asyncio event loop. + :param receiver: Socket receiver. """ loop = get_event_loop() if loop is None else loop @@ -79,14 +78,15 @@ async def send_data(self, data_to_send: Convertible, loop: Optional[EventLoop] = if await loop.sock_sendall(sock=receiver, data=data_as_bytes) is not None: ValueError(f"[{self.name}] Could not send all of the data for an unknown reason") - def sync_send_data(self, data_to_send: Convertible, receiver: Optional[socket] = None) -> None: + def sync_send_data(self, + data_to_send: Convertible, + receiver: Optional[socket] = None) -> None: """ - | Send data through the given socket. - | Synchronous version of 'TcpIpObject.send_data'. + Send data through the given socket. + Synchronous version of 'TcpIpObject.send_data'. - :param data_to_send: Data that will be sent on socket* - :type data_to_send: Union[type(None), bytes, str, bool, int, float, List, ndarray] - :param Optional[socket] receiver: socket receiver + :param data_to_send: Data that will be sent on socket. + :param receiver: Socket receiver. """ receiver = self.sock if receiver is None else receiver @@ -95,13 +95,15 @@ def sync_send_data(self, data_to_send: Convertible, receiver: Optional[socket] = # Send the whole message receiver.sendall(data_as_bytes) - async def receive_data(self, loop: EventLoop, sender: socket) -> Convertible: + async def receive_data(self, + loop: EventLoop, + sender: socket) -> Convertible: """ - | Receive data from a socket. + Receive data from a socket. - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: socket sender - :return: Converted data + :param loop: Asyncio event loop. + :param sender: Socket sender. + :return: Converted data. """ # Receive the number of fields to receive @@ -117,10 +119,10 @@ async def receive_data(self, loop: EventLoop, sender: socket) -> Convertible: def sync_receive_data(self) -> Convertible: """ - | Receive data from a socket. - | Synchronous version of 'TcpIpObject.receive_data'. + Receive data from a socket. + Synchronous version of 'TcpIpObject.receive_data'. - :return: Converted data + :return: Converted data. """ self.sock.setblocking(True) @@ -135,14 +137,17 @@ def sync_receive_data(self) -> Convertible: # Return the data in the expected format return self.data_converter.bytes_to_data(bytes_fields) - async def read_data(self, loop: EventLoop, sender: socket, read_size: int) -> bytes: + async def read_data(self, + loop: EventLoop, + sender: socket, + read_size: int) -> bytes: """ - | Read the data on the socket with value of buffer size as relatively small powers of 2. + Read the data on the socket with value of buffer size as relatively small powers of 2. - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: socket sender - :param int read_size: Amount of data to read on the socket - :return: Bytes field with 'read_size' length + :param loop: Asyncio event loop. + :param sender: Socket sender. + :param read_size: Amount of data to read on the socket. + :return: Bytes field with 'read_size' length. """ # Maximum read size @@ -165,13 +170,14 @@ async def read_data(self, loop: EventLoop, sender: socket, read_size: int) -> by return bytes_field - def sync_read_data(self, read_size: int) -> bytes: + def sync_read_data(self, + read_size: int) -> bytes: """ - | Read the data on the socket with value of buffer size as relatively small powers of 2. - | Synchronous version of 'TcpIpObject.read_data'. + Read the data on the socket with value of buffer size as relatively small powers of 2. + Synchronous version of 'TcpIpObject.read_data'. - :param int read_size: Amount of data to read on the socket - :return: Bytes field with 'read_size' length + :param read_size: Amount of data to read on the socket. + :return: Bytes field with 'read_size' length. """ # Maximum read sizes array @@ -199,17 +205,20 @@ def sync_read_data(self, read_size: int) -> bytes: ########################################################################################## ########################################################################################## - async def send_labeled_data(self, data_to_send: Convertible, label: str, loop: Optional[EventLoop] = None, - receiver: Optional[socket] = None, send_read_command: bool = True) -> None: + async def send_labeled_data(self, + data_to_send: Convertible, + label: str, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None, + send_read_command: bool = True) -> None: """ - | Send data with an associated label. + Send data with an associated label. - :param data_to_send: Data that will be sent on socket - :type data_to_send: Union[type(None), bytes, str, bool, int, float, List, ndarray] - :param str label: Associated label - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver - :param bool send_read_command: If True, the command 'read' is sent before sending data + :param data_to_send: Data that will be sent on socket. + :param label: Associated label. + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. + :param send_read_command: If True, the command 'read' is sent before sending data. """ loop = get_event_loop() if loop is None else loop @@ -222,18 +231,19 @@ async def send_labeled_data(self, data_to_send: Convertible, label: str, loop: O # Send data await self.send_data(data_to_send=data_to_send, loop=loop, receiver=receiver) - def sync_send_labeled_data(self, data_to_send: Convertible, label: str, receiver: Optional[socket] = None, + def sync_send_labeled_data(self, + data_to_send: Convertible, + label: str, + receiver: Optional[socket] = None, send_read_command: bool = True) -> None: """ - | Send data with an associated label. - | Synchronous version of 'TcpIpObject.send_labeled_data'. + Send data with an associated label. + Synchronous version of 'TcpIpObject.send_labeled_data'. - :param data_to_send: Data that will be sent on socket - :type data_to_send: Union[type(None), bytes, str, bool, int, float, List, ndarray] - :param str label: Associated label - :param Optional[socket] receiver: TcpIpObject receiver - :param bool send_read_command: If True, the command 'read' is sent before sending data - :return: + :param data_to_send: Data that will be sent on socket. + :param label: Associated label. + :param receiver: TcpIpObject receiver. + :param send_read_command: If True, the command 'read' is sent before sending data. """ receiver = self.sock if receiver is None else receiver @@ -245,13 +255,15 @@ def sync_send_labeled_data(self, data_to_send: Convertible, label: str, receiver # Send data self.sync_send_data(data_to_send=data_to_send, receiver=receiver) - async def receive_labeled_data(self, loop: EventLoop, sender: socket) -> Tuple[str, Convertible]: + async def receive_labeled_data(self, + loop: EventLoop, + sender: socket) -> Tuple[str, Convertible]: """ - | Receive data and an associated label. + Receive data and an associated label. - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender - :return: Label, Data + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. + :return: Label, Data. """ # Listen to sender @@ -267,10 +279,10 @@ async def receive_labeled_data(self, loop: EventLoop, sender: socket) -> Tuple[s def sync_receive_labeled_data(self) -> Tuple[str, Convertible]: """ - | Receive data and an associated label. - | Synchronous version of 'TcpIpObject.receive_labeled_data'. + Receive data and an associated label. + Synchronous version of 'TcpIpObject.receive_labeled_data'. - :return: Label, Data + :return: Label, Data. """ # Listen to sender @@ -284,15 +296,18 @@ def sync_receive_labeled_data(self) -> Tuple[str, Convertible]: data = self.sync_receive_data() return label, data - async def send_dict(self, name: str, dict_to_send: Dict[Any, Any], loop: Optional[EventLoop] = None, + async def send_dict(self, + name: str, + dict_to_send: Dict[Any, Any], + loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: """ - | Send a whole dictionary field by field as labeled data. + Send a whole dictionary field by field as labeled data. - :param str name: Name of the dictionary - :param Dict[Any, Any] dict_to_send: Dictionary to send - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param name: Name of the dictionary. + :param dict_to_send: Dictionary to send. + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ loop = get_event_loop() if loop is None else loop @@ -324,14 +339,17 @@ async def send_dict(self, name: str, dict_to_send: Dict[Any, Any], loop: Optiona await self.send_command_finished(loop=loop, receiver=receiver) await self.send_command_finished(loop=loop, receiver=receiver) - def sync_send_dict(self, name: str, dict_to_send: Dict[Any, Any], receiver: Optional[socket] = None) -> None: + def sync_send_dict(self, + name: str, + dict_to_send: Dict[Any, Any], + receiver: Optional[socket] = None) -> None: """ - | Send a whole dictionary field by field as labeled data. - | Synchronous version of 'TcpIpObject.receive_labeled_data'. + Send a whole dictionary field by field as labeled data. + Synchronous version of 'TcpIpObject.receive_labeled_data'. - :param str name: Name of the dictionary - :param Dict[Any, Any] dict_to_send: Dictionary to send - :param Optional[socket] receiver: TcpIpObject receiver + :param name: Name of the dictionary. + :param dict_to_send: Dictionary to send. + :param receiver: TcpIpObject receiver. """ receiver = self.sock if receiver is None else receiver @@ -362,14 +380,16 @@ def sync_send_dict(self, name: str, dict_to_send: Dict[Any, Any], receiver: Opti self.sync_send_command_finished(receiver=receiver) self.sync_send_command_finished(receiver=receiver) - async def __send_unnamed_dict(self, dict_to_send: Dict[Any, Any], loop: Optional[EventLoop] = None, + async def __send_unnamed_dict(self, + dict_to_send: Dict[Any, Any], + loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: """ - | Send a whole dictionary field by field as labeled data. Dictionary will be unnamed. + Send a whole dictionary field by field as labeled data. Dictionary will be unnamed. - :param dict dict_to_send: Dictionary to send - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param dict_to_send: Dictionary to send + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ loop = get_event_loop() if loop is None else loop @@ -391,13 +411,15 @@ async def __send_unnamed_dict(self, dict_to_send: Dict[Any, Any], loop: Optional # The sending is finished await self.send_command_finished(loop=loop, receiver=receiver) - def __sync_send_unnamed_dict(self, dict_to_send: Dict[Any, Any], receiver: Optional[socket] = None) -> None: + def __sync_send_unnamed_dict(self, + dict_to_send: Dict[Any, Any], + receiver: Optional[socket] = None) -> None: """ - | Send a whole dictionary field by field as labeled data. Dictionary will be unnamed. - | Synchronous version of 'TcpIpObject.receive_labeled_data'. + Send a whole dictionary field by field as labeled data. Dictionary will be unnamed. + Synchronous version of 'TcpIpObject.receive_labeled_data'. - :param Dict[Any, Any] dict_to_send: Dictionary to send - :param Optional[socket] receiver: TcpIpObject receiver + :param dict_to_send: Dictionary to send. + :param receiver: TcpIpObject receiver. """ receiver = self.sock if receiver is None else receiver @@ -420,14 +442,16 @@ def __sync_send_unnamed_dict(self, dict_to_send: Dict[Any, Any], receiver: Optio # The sending is finished self.sync_send_command_finished(receiver=receiver) - async def receive_dict(self, recv_to: Dict[Any, Any], loop: Optional[EventLoop] = None, + async def receive_dict(self, + recv_to: Dict[Any, Any], + loop: Optional[EventLoop] = None, sender: Optional[socket] = None) -> None: """ - | Receive a whole dictionary field by field as labeled data. + Receive a whole dictionary field by field as labeled data. - :param Dict[Any, Any] recv_to: Dictionary to fill with received fields - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] sender: TcpIpObject sender + :param recv_to: Dictionary to fill with received fields. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ loop = get_event_loop() if loop is None else loop @@ -445,13 +469,15 @@ async def receive_dict(self, recv_to: Dict[Any, Any], loop: Optional[EventLoop] else: recv_to[label] = param - def sync_receive_dict(self, recv_to: Dict[Any, Any], sender: Optional[socket] = None) -> None: + def sync_receive_dict(self, + recv_to: Dict[Any, Any], + sender: Optional[socket] = None) -> None: """ - | Receive a whole dictionary field by field as labeled data. - | Synchronous version of 'TcpIpObject.receive_labeled_data'. + Receive a whole dictionary field by field as labeled data. + Synchronous version of 'TcpIpObject.receive_labeled_data'. - :param Dict[Any, Any] recv_to: Dictionary to fill with received fields - :param Optional[socket] sender: TcpIpObject sender + :param recv_to: Dictionary to fill with received fields. + :param sender: TcpIpObject sender. """ sender = self.sock if sender is None else sender @@ -474,14 +500,17 @@ def sync_receive_dict(self, recv_to: Dict[Any, Any], sender: Optional[socket] = ########################################################################################## ########################################################################################## - async def __send_command(self, loop: EventLoop, receiver: socket, command: str = '') -> None: + async def __send_command(self, + loop: EventLoop, + receiver: socket, + command: str = '') -> None: """ - | Send a bytes command among the available commands. - | Do not use this one. Use the dedicated function 'send_command_{cmd}(...)'. + Send a bytes command among the available commands. + Do not use this one. Use the dedicated function 'send_command_{cmd}(...)'. - :param EventLoop loop: asyncio.get_event_loop() return - :param socket receiver: TcpIpObject receiver - :param str command: Name of the command, must be in 'self.command_dict' + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. + :param command: Name of the command, must be in 'self.command_dict'. """ # Check if the command exists @@ -492,14 +521,16 @@ async def __send_command(self, loop: EventLoop, receiver: socket, command: str = # Send command as a byte data await self.send_data(data_to_send=cmd, loop=loop, receiver=receiver) - def __sync_send_command(self, receiver: socket, command: str = '') -> None: + def __sync_send_command(self, + receiver: socket, + command: str = '') -> None: """ - | Send a bytes command among the available commands. - | Do not use this one. Use the dedicated function 'sync_send_command_{cmd}(...)'. - | Synchronous version of 'TcpIpObject.send_command'. + Send a bytes command among the available commands. + Do not use this one. Use the dedicated function 'sync_send_command_{cmd}(...)'. + Synchronous version of 'TcpIpObject.send_command'. - :param str command: name of the command to send - :param socket receiver: TcpIpObject receiver + :param command: Name of the command to send. + :param receiver: TcpIpObject receiver. """ # Check if the command exists @@ -510,203 +541,254 @@ def __sync_send_command(self, receiver: socket, command: str = '') -> None: # Send command as a byte data self.sync_send_data(data_to_send=cmd, receiver=receiver) - async def send_command_compute(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_compute(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'compute' command. + Send the 'compute' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='compute') - def sync_send_command_compute(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_compute(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'compute' command. - | Synchronous version of 'TcpIpObject.send_command_compute'. + Send the 'compute' command. + Synchronous version of 'TcpIpObject.send_command_compute'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='compute') - async def send_command_done(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_done(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'done' command. + Send the 'done' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='done') - def sync_send_command_done(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_done(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'done' command. - | Synchronous version of 'TcpIpObject.send_command_done'. + Send the 'done' command. + Synchronous version of 'TcpIpObject.send_command_done'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='done') - async def send_command_exit(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_exit(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'exit' command. + Send the 'exit' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='exit') - def sync_send_command_exit(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_exit(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'exit' command. - | Synchronous version of 'TcpIpObject.send_command_exit'. + Send the 'exit' command. + Synchronous version of 'TcpIpObject.send_command_exit'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='exit') - async def send_command_finished(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_finished(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'finished' command. + Send the 'finished' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='finished') - def sync_send_command_finished(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_finished(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'finished' command. - | Synchronous version of 'TcpIpObject.send_command_finished'. + Send the 'finished' command. + Synchronous version of 'TcpIpObject.send_command_finished'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='finished') - async def send_command_prediction(self, loop: Optional[EventLoop] = None, + async def send_command_prediction(self, + loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: """ - | Send the 'prediction' command. + Send the 'prediction' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='prediction') - def sync_send_command_prediction(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_prediction(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'prediction' command. - | Synchronous version of 'TcpIpObject.send_command_prediction'. + Send the 'prediction' command. + Synchronous version of 'TcpIpObject.send_command_prediction'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='prediction') - async def send_command_read(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_read(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'read' command. + Send the 'read' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='read') - def sync_send_command_read(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_read(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'read' command. - | Synchronous version of 'TcpIpObject.send_command_read'. + Send the 'read' command. + Synchronous version of 'TcpIpObject.send_command_read'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='read') - async def send_command_sample(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_sample(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'sample' command. + Send the 'sample' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='sample') - def sync_send_command_sample(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_sample(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'sample' command. - | Synchronous version of 'TcpIpObject.send_command_sample'. + Send the 'sample' command. + Synchronous version of 'TcpIpObject.send_command_sample'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='sample') - async def send_command_step(self, loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: + async def send_command_step(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: """ - | Send the 'step' command. + Send the 'step' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='step') - def sync_send_command_step(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_step(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'step' command. - | Synchronous version of 'TcpIpObject.send_command_step'. + Send the 'step' command. + Synchronous version of 'TcpIpObject.send_command_step'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='step') - async def send_command_visualisation(self, loop: Optional[EventLoop] = None, + async def send_command_visualisation(self, + loop: Optional[EventLoop] = None, receiver: Optional[socket] = None) -> None: """ - | Send the 'visualisation' command. + Send the 'visualisation' command. - :param Optional[EventLoop] loop: asyncio.get_event_loop() return - :param Optional[socket] receiver: TcpIpObject receiver + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. """ await self.__send_command(loop=loop, receiver=receiver, command='visualisation') - def sync_send_command_visualisation(self, receiver: Optional[socket] = None) -> None: + def sync_send_command_visualisation(self, + receiver: Optional[socket] = None) -> None: """ - | Send the 'visualisation' command. - | Synchronous version of 'TcpIpObject.send_command_visualisation'. + Send the 'visualisation' command. + Synchronous version of 'TcpIpObject.send_command_visualisation'. - :param Optional[socket] receiver: TcpIpObject receiver + :param receiver: TcpIpObject receiver. """ self.__sync_send_command(receiver=receiver, command='visualisation') + async def send_command_change_db(self, + loop: Optional[EventLoop] = None, + receiver: Optional[socket] = None) -> None: + """ + Send the 'change_database' command. + + :param loop: Asyncio event loop. + :param receiver: TcpIpObject receiver. + """ + + await self.__send_command(loop=loop, receiver=receiver, command='db') + + def sync_send_command_change_db(self, + receiver: Optional[socket] = None) -> None: + """ + Send the 'change_database' command. + Synchronous version of 'TcpIpObject.send_command_change_database'. + + :param receiver: TcpIpObject receiver. + """ + + self.__sync_send_command(receiver=receiver, command='db') + ########################################################################################## ########################################################################################## # Actions to perform on commands # ########################################################################################## ########################################################################################## - async def listen_while_not_done(self, loop: EventLoop, sender: socket, data_dict: Dict[Any, Any], + async def listen_while_not_done(self, + loop: EventLoop, + sender: socket, + data_dict: Dict[Any, Any], client_id: int = None) -> Dict[Any, Any]: """ - | Compute actions until 'done' command is received. + Compute actions until 'done' command is received. - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender - :param Dict[Any, Any] data_dict: Dictionary to collect data - :param int client_id: ID of a Client + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. + :param data_dict: Dictionary to collect data. + :param client_id: ID of a Client. """ # Compute actions until 'done' command is received @@ -717,74 +799,98 @@ async def listen_while_not_done(self, loop: EventLoop, sender: socket, data_dict # Return collected data return data_dict - async def action_on_compute(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_compute(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'compute' command. + Action to run when receiving the 'compute' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_done(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_done(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'done' command. + Action to run when receiving the 'done' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_exit(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_exit(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'exit' command. + Action to run when receiving the 'exit' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_finished(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_finished(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'finished' command. + Action to run when receiving the 'finished' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_prediction(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_prediction(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'prediction' command. + Action to run when receiving the 'prediction' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_read(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_read(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'read' command. + Action to run when receiving the 'read' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ # Receive labeled data @@ -797,39 +903,66 @@ async def action_on_read(self, data: Dict[Any, Any], client_id: int, sender: soc else: data[client_id][label] = param - async def action_on_sample(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_sample(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'sample' command. + Action to run when receiving the 'sample' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_step(self, data: Dict[Any, Any], client_id: int, sender: socket, loop: EventLoop) -> None: + async def action_on_step(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: """ - | Action to run when receiving the 'step' command. + Action to run when receiving the 'step' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass - async def action_on_visualisation(self, data: Dict[Any, Any], client_id: int, sender: socket, + async def action_on_visualisation(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, loop: EventLoop) -> None: """ - | Action to run when receiving the 'visualisation' command. + Action to run when receiving the 'visualisation' command. + + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. + """ + + pass + + async def action_on_change_db(self, + data: Dict[Any, Any], + client_id: int, + sender: socket, + loop: EventLoop) -> None: + """ + Action to run when receiving the 'change_database' command. - :param Dict[Any, Any] data: Dict storing data - :param int client_id: ID of the TcpIpClient - :param EventLoop loop: asyncio.get_event_loop() return - :param socket sender: TcpIpObject sender + :param data: Dict storing data. + :param client_id: ID of the TcpIpClient. + :param loop: Asyncio event loop. + :param sender: TcpIpObject sender. """ pass diff --git a/src/AsyncSocket/TcpIpServer.py b/src/Core/AsyncSocket/TcpIpServer.py similarity index 53% rename from src/AsyncSocket/TcpIpServer.py rename to src/Core/AsyncSocket/TcpIpServer.py index fa5a8761..e8b3f350 100644 --- a/src/AsyncSocket/TcpIpServer.py +++ b/src/Core/AsyncSocket/TcpIpServer.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple from asyncio import get_event_loop, gather from asyncio import AbstractEventLoop as EventLoop from asyncio import run as async_run from socket import socket -from numpy import ndarray from queue import SimpleQueue from DeepPhysX.Core.AsyncSocket.TcpIpObject import TcpIpObject +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler class TcpIpServer(TcpIpObject): @@ -49,12 +49,42 @@ def __init__(self, self.data_fifo: SimpleQueue = SimpleQueue() self.data_dict: Dict[Any, Any] = {} self.sample_to_client_id: List[int] = [] - self.batch_from_dataset: Optional[Dict[str, ndarray]] = None + self.batch_from_dataset: Optional[List[int]] = None self.first_time: bool = True + self.data_lines: List[List[int]] = [] # Reference to EnvironmentManager self.environment_manager: Optional[Any] = manager + # Connect the Server to the Database + self.database_handler = DatabaseHandler(on_partitions_handler=self.__database_handler_partitions) + self.environment_manager.data_manager.connect_handler(self.database_handler) + + ########################################################################################## + ########################################################################################## + # DatabaseHandler management # + ########################################################################################## + ########################################################################################## + + def get_database_handler(self) -> DatabaseHandler: + """ + Get the DatabaseHandler of the TcpIpServer. + """ + + return self.database_handler + + def __database_handler_partitions(self) -> None: + """ + Partition update event of the DatabaseHandler. + """ + + # Send the new partition to every Client + for _, client in self.clients: + self.sync_send_command_change_db(receiver=client) + new_partition = self.database_handler.get_partitions()[-1] + self.sync_send_data(data_to_send=f'{new_partition.get_path()[0]}///{new_partition.get_path()[1]}', + receiver=client) + ########################################################################################## ########################################################################################## # Connect Clients # @@ -63,7 +93,7 @@ def __init__(self, def connect(self) -> None: """ - Run __connect method with asyncio. + Accept connections from clients. """ print(f"[{self.name}] Waiting for clients...") @@ -91,44 +121,72 @@ async def __connect(self) -> None: ########################################################################################## def initialize(self, - param_dict: Dict[Any, Any]) -> Dict[Any, Any]: + env_kwargs: Dict[str, Any], + visualization_db: Optional[Tuple[str, str]] = None) -> None: """ - Run __initialize method with asyncio. Manage parameters exchange. + Send parameters to the clients to create their environments. - :param param_dict: Dictionary of parameters to send to the client's environment. - :return: Dictionary of parameters for each environment to send the manager. + :param env_kwargs: Additional arguments to pass to the Environment. + :param visualization_db: Path to the visualization Database to connect to. """ print(f"[{self.name}] Initializing clients...") - async_run(self.__initialize(param_dict)) - # Return param dict - param_dict = {} - for client_id in self.data_dict: - if 'parameters' in self.data_dict[client_id]: - param_dict[client_id] = self.data_dict[client_id]['parameters'] - return param_dict + async_run(self.__initialize(env_kwargs, visualization_db)) - async def __initialize(self, param_dict: Dict[Any, Any]) -> None: + async def __initialize(self, + env_kwargs: Dict[str, Any], + visualization_db: Optional[Tuple[str, str]] = None) -> None: """ - Send parameters to the clients to create their environments, receive parameters from clients in exchange. + Send parameters to the clients to create their environments. - :param param_dict: Dictionary of parameters to send to the client's environment. + :param env_kwargs: Additional arguments to pass to the Environment. + :param visualization_db: Path to the visualization Database to connect to. """ loop = get_event_loop() - # Empty dictionaries for received parameters from clients - self.data_dict = {client_ID: {} for client_ID in range(len(self.clients))} + # Initialisation process for each client for client_id, client in self.clients: + + # Send additional arguments + await self.send_dict(name='env_kwargs', dict_to_send=env_kwargs, loop=loop, receiver=client) + + # Send prediction request authorization + await self.send_data(data_to_send=self.environment_manager.allow_prediction_requests, + loop=loop, receiver=client) + # Send number of sub-steps nb_steps = self.environment_manager.simulations_per_step if self.environment_manager else 1 await self.send_data(data_to_send=nb_steps, loop=loop, receiver=client) - # Send parameters to client - await self.send_dict(name="parameters", dict_to_send=param_dict, loop=loop, receiver=client) - # Receive visualization data and parameters - await self.listen_while_not_done(loop=loop, sender=client, data_dict=self.data_dict, client_id=client_id) + + # Send partitions + partitions = self.database_handler.get_partitions() + if len(partitions) == 0: + partitions_list = 'None' + else: + partitions_list = partitions[0].get_path()[0] + for partition in partitions: + partitions_list += f'///{partition.get_path()[1]}' + partitions_list += '%%%' + exchange = self.database_handler.get_exchange() + if exchange is None: + partitions += 'None' + else: + partitions_list += f'{exchange.get_path()[0]}///{exchange.get_path()[1]}' + await self.send_data(data_to_send=partitions_list, loop=loop, receiver=client) + + # Send visualization Database + visualization = 'None' if visualization_db is None else f'{visualization_db[0]}///{visualization_db[1]}' + await self.send_data(data_to_send=visualization, loop=loop, receiver=client) + + # Wait Client init + await self.receive_data(loop=loop, sender=client) print(f"[{self.name}] Client n°{client_id} initialisation done") + # Synchronize Clients + for client_id, client in self.clients: + await self.send_data(data_to_send='sync', loop=loop, receiver=client) + ########################################################################################## ########################################################################################## # Data: produce batch & dispatch batch # @@ -136,165 +194,80 @@ async def __initialize(self, param_dict: Dict[Any, Any]) -> None: ########################################################################################## def get_batch(self, - get_inputs: bool = True, - get_outputs: bool = True, - animate: bool = True) -> Dict[str, Union[ndarray, dict]]: + animate: bool = True) -> List[List[int]]: """ Build a batch from clients samples. - :param get_inputs: If True, compute and return input. - :param get_outputs: If True, compute and return output. :param animate: If True, triggers an environment step. - :return: Batch (list of samples) & additional data in a dictionary. """ # Trigger communication protocol - async_run(self.__request_data_to_clients(get_inputs=get_inputs, get_outputs=get_outputs, animate=animate)) - - # Sort stored data between following fields - data_sorter = {'input': [], 'output': [], 'additional_fields': {}, 'loss': []} - list_fields = [key for key in data_sorter.keys() if type(data_sorter[key]) == list] - # Map produced samples with clients ID - self.sample_to_client_id = [] - - # Process while queue is empty or batch is full - while max([len(data_sorter[key]) for key in list_fields]) < self.batch_size and not self.data_fifo.empty(): - # Get data dict from queue - data = self.data_fifo.get() - # Network in / out / loss - for field in ['input', 'output', 'loss']: - if field in data: - data_sorter[field].append(data[field]) - # Additional fields - field = 'additional_fields' - if field in data: - for key in data[field]: - if key not in data_sorter[field].keys(): - data_sorter[field][key] = [] - data_sorter[field][key].append(data[field][key]) - # ID of client - if 'ID' in data: - self.sample_to_client_id.append(data['ID']) - return data_sorter + async_run(self.__request_data_to_clients(animate=animate)) + return self.data_lines async def __request_data_to_clients(self, - get_inputs: bool = True, - get_outputs: bool = True, animate: bool = True) -> None: """ Trigger a communication protocol for each client. Wait for all clients before to launch another communication protocol while the batch is not full. - :param get_inputs: If True, compute and return input - :param get_outputs: If True, compute and return output :param animate: If True, triggers an environment step """ - client_launched = 0 + nb_sample = 0 + self.data_lines = [] # Launch the communication protocol while the batch needs to be filled - while client_launched < self.batch_size: + while nb_sample < self.batch_size: + clients = self.clients[:min(len(self.clients), self.batch_size - nb_sample)] # Run communicate protocol for each client and wait for the last one to finish - await gather(*[self.__communicate(client=client, client_id=client_id, get_inputs=get_inputs, - get_outputs=get_outputs, animate=animate) - for client_id, client in self.clients]) - client_launched += len(self.clients) + await gather(*[self.__communicate(client=client, + client_id=client_id, + animate=animate) for client_id, client in clients]) + nb_sample += len(clients) async def __communicate(self, client: Optional[socket] = None, client_id: Optional[int] = None, - get_inputs: bool = True, - get_outputs: bool = True, animate: bool = True) -> None: """ - | Communication protocol with a client. It goes through different steps: - | 1) Eventually send samples to Client - | 2) Running steps & Receiving training data - | 3) Add data to the Queue + Communication protocol with a client. :param client: TcpIpObject client to communicate with. :param client_id: Index of the client. - :param get_inputs: If True, compute and return input. - :param get_outputs: If True, compute and return output. :param animate: If True, triggers an environment step. """ loop = get_event_loop() - # 1) If a sample from Dataset is given, sent it to the TcpIpClient + # 1. Send a sample to the Client if a batch from the Dataset is given if self.batch_from_dataset is not None: - # Check if there is remaining samples, otherwise client is not used - if len(self.batch_from_dataset['input']) == 0 or len(self.batch_from_dataset['output']) == 0: + # Check if there is remaining samples, otherwise the Client is not used + if len(self.batch_from_dataset) == 0: return - # Send the sample to the TcpIpClient + # Send the sample to the Client await self.send_command_sample(loop=loop, receiver=client) - # Pop the first sample of the numpy batch for network in / out - for field in ['input', 'output']: - # Tell if there is something to read - await self.send_data(data_to_send=field in self.batch_from_dataset, loop=loop, receiver=client) - if field in self.batch_from_dataset: - # Pop sample from array if there are some - sample = self.batch_from_dataset[field][0] - self.batch_from_dataset[field] = self.batch_from_dataset[field][1:] - # Keep the sample in memory - self.data_dict[client_id][field] = sample - # Send network in / out sample - await self.send_data(data_to_send=sample, loop=loop, receiver=client) - # Pop the first sample of the numpy batch for each additional dataset field - field = 'additional_fields' - # Tell TcpClient if there is additional data for this field - await self.send_data(data_to_send=field in self.batch_from_dataset, loop=loop, receiver=client) - if field in self.batch_from_dataset: - sample = {} - # Get each additional data field - for key in self.batch_from_dataset[field]: - # Pop sample from array - sample[key] = self.batch_from_dataset[field][key][0] - self.batch_from_dataset[field][key] = self.batch_from_dataset[field][key][1:] - # Keep the sample in memory - self.data_dict[client_id][field + '_' + key] = sample[key] - # Send additional in / out sample - await self.send_dict(name="additional_fields", dict_to_send=sample, loop=loop, receiver=client) - - # 2) Execute n steps, the last one send data computation signal + line = self.batch_from_dataset.pop(0) + await self.send_data(data_to_send=line, loop=loop, receiver=client) + + # 2. Execute n steps, the last one send data computation signal if animate: await self.send_command_step(loop=loop, receiver=client) # Receive data await self.listen_while_not_done(loop=loop, sender=client, data_dict=self.data_dict, client_id=client_id) - - # 3.1) Add all received in / out data to queue - data = {} - for get_data, net_field in zip([get_inputs, get_outputs], ['input', 'output']): - if get_data: - # Add network field - data[net_field] = self.data_dict[client_id][net_field] - # 3.2) Add loss data if provided - if 'loss' in self.data_dict[client_id]: - data['loss'] = self.data_dict[client_id]['loss'] - # 3.3) Add additional fields (transform key from 'dataset_{FIELD}' to '{FIELD}') - additional_fields = [key for key in self.data_dict[client_id].keys() if key.__contains__('dataset_')] - data['additional_fields'] = {} - for field in additional_fields: - data['additional_fields'][field[len('dataset_'):]] = self.data_dict[client_id][field] - # 3.4) Identify sample - data['ID'] = client_id - # 3.5) Add data to the Queue - self.data_fifo.put(data) + line = await self.receive_data(loop=loop, sender=client) + self.data_lines.append(line) def set_dataset_batch(self, - batch: Dict[str, Union[ndarray, Dict]]) -> None: + data_lines: List[int]) -> None: """ Receive a batch of data from the Dataset. Samples will be dispatched between clients. - :param batch: Batch of data. + :param data_lines: Batch of indices of samples. """ - # Check batch size - if len(batch['input']) != self.batch_size: - raise ValueError(f"[{self.name}] The size of batch from Dataset is {len(batch['input'])} while the batch " - f"size was set to {self.batch_size}.") # Define batch from dataset - self.batch_from_dataset = batch.copy() + self.batch_from_dataset = data_lines.copy() ########################################################################################## ########################################################################################## @@ -359,25 +332,10 @@ async def action_on_prediction(self, :param sender: TcpIpObject sender. """ - # Receive network input - label, network_input = await self.receive_labeled_data(loop=loop, sender=sender) - - # Check that manager hierarchy is well-defined if self.environment_manager.data_manager is None: raise ValueError("Cannot request prediction if DataManager does not exist") - elif self.environment_manager.data_manager.manager is None: - raise ValueError("Cannot request prediction if Manager does not exist") - elif not hasattr(self.environment_manager.data_manager.manager, 'network_manager'): - raise AttributeError("Cannot request prediction if NetworkManager does not exist. If using a data " - "generation pipeline, please disable get_prediction requests.") - elif self.environment_manager.data_manager.manager.network_manager is None: - raise ValueError("Cannot request prediction if NetworkManager does not exist") - - # Get the prediction from NetworkPrediction - prediction = self.environment_manager.data_manager.get_prediction(network_input=network_input[None, ]) - # Send back the prediction to the Client - await self.send_labeled_data(data_to_send=prediction, label="prediction", receiver=sender, - send_read_command=False) + self.environment_manager.data_manager.get_prediction(client_id) + await self.send_data(data_to_send=True, receiver=sender) async def action_on_visualisation(self, data: Dict[Any, Any], diff --git a/src/AsyncSocket/__init__.py b/src/Core/AsyncSocket/__init__.py similarity index 100% rename from src/AsyncSocket/__init__.py rename to src/Core/AsyncSocket/__init__.py diff --git a/src/Core/Database/BaseDatabaseConfig.py b/src/Core/Database/BaseDatabaseConfig.py new file mode 100644 index 00000000..0ffe5934 --- /dev/null +++ b/src/Core/Database/BaseDatabaseConfig.py @@ -0,0 +1,72 @@ +from typing import Optional +from os.path import isdir, sep, join + + +class BaseDatabaseConfig: + + def __init__(self, + existing_dir: Optional[str] = None, + mode: Optional[str] = None, + max_file_size: Optional[float] = None, + shuffle: bool = False, + normalize: bool = False, + recompute_normalization: bool = False): + """ + BaseDatabaseConfig is a configuration class to parameterize the Database and the DatabaseManager. + + :param existing_dir: Path to an existing Dataset repository. + :param mode: Specify the Dataset mode that should be used between 'training', 'validation' and 'running'. + :param max_file_size: Maximum size (in Gb) of a single dataset file. + :param shuffle: Specify if the Dataset should be shuffled when a batch is taken. + :param normalize: If True, the data will be normalized using standard score. + :param recompute_normalization: If True, compute the normalization coefficients. + """ + + self.name: str = self.__class__.__name__ + + # Check directory variable + if existing_dir is not None: + if type(existing_dir) != str: + raise TypeError(f"[{self.name}] The given 'existing_dir'={existing_dir} must be a str.") + if not isdir(existing_dir): + raise ValueError(f"[{self.name}] The given 'existing_dir'={existing_dir} does not exist.") + if len(existing_dir.split(sep)) > 1 and existing_dir.split(sep)[-1] == 'dataset': + existing_dir = join(*existing_dir.split(sep)[:-1]) + + # Check storage variables + if mode is not None: + if type(mode) != str: + raise TypeError(f"[{self.name}] The given 'mode'={mode} must be a str.") + if mode.lower() not in (available_modes := ['training', 'validation', 'prediction']): + raise ValueError(f"[{self.name}] The given 'mode'={mode} must be in {available_modes}.") + if max_file_size is not None: + if type(max_file_size) not in [int, float]: + raise TypeError(f"[{self.name}] The given 'max_file_size'={max_file_size} must be a float.") + max_file_size = int(max_file_size * 1e9) if max_file_size > 0 else None + if type(shuffle) != bool: + raise TypeError(f"[{self.name}] The given 'shuffle'={shuffle} must be a bool.") + if type(normalize) != bool: + raise TypeError(f"[{self.name}] The given 'normalize'={normalize} must be a bool.") + if type(recompute_normalization) != bool: + raise TypeError(f"[{self.name}] The given 'recompute_normalization'={recompute_normalization} must be a " + f"bool.") + + # DatabaseManager parameterization + self.existing_dir: Optional[str] = existing_dir + self.mode: Optional[str] = mode + self.max_file_size: int = max_file_size + self.shuffle: bool = shuffle + self.normalize: bool = normalize + self.recompute_normalization: bool = recompute_normalization + + def __str__(self): + + description = "\n" + description += f"{self.name}\n" + description += f" Existing directory: {False if self.existing_dir is None else self.existing_dir}\n" + description += f" Mode: {self.mode}\n" + description += f" Max size: {self.max_file_size}\n" + description += f" Shuffle: {self.shuffle}\n" + description += f" Normalize: {self.normalize}\n" + description += f" Recompute normalization: {self.recompute_normalization}\n" + return description diff --git a/src/Core/Database/DatabaseHandler.py b/src/Core/Database/DatabaseHandler.py new file mode 100644 index 00000000..ca3d16a8 --- /dev/null +++ b/src/Core/Database/DatabaseHandler.py @@ -0,0 +1,279 @@ +from typing import Union, List, Dict, Any, Callable, Optional, Type, Tuple +from numpy import array, where +from itertools import chain + +from SSD.Core.Storage.Database import Database + + +class DatabaseHandler: + + def __init__(self, + on_init_handler: Optional[Callable] = None, + on_partitions_handler: Optional[Callable] = None): + """ + DatabaseHandler allows components to be synchronized with the Database partitions and to read / write data. + + :param on_init_handler: Event to trigger when the DatabaseHandler is initialized. + :param on_partitions_handler: Event to trigger when the list of partitions is updated. + """ + + # Databases variables + self.__storing_partitions: List[Database] = [] + self.__exchange_db: Optional[Database] = None + + # Event handlers + self.__on_init_handler = self.default_handler if on_init_handler is None else on_init_handler + self.__on_partitions_handler = self.default_handler if on_partitions_handler is None else on_partitions_handler + + def default_handler(self): + pass + + ########################################################################################## + ########################################################################################## + # Partitions management # + ########################################################################################## + ########################################################################################## + + def init(self, + storing_partitions: List[Database], + exchange_db: Database) -> None: + """ + Initialize the list of the partitions. + + :param storing_partitions: List of the storing Database partitions. + :param exchange_db: Exchange Database. + """ + + self.__storing_partitions = storing_partitions.copy() + self.__exchange_db = exchange_db + self.__on_init_handler() + + def init_remote(self, + storing_partitions: List[List[str]], + exchange_db: List[str]) -> None: + """ + Initialize the list of partitions in remote DatabaseHandlers. + + :param storing_partitions: List of paths to the storing Database partitions. + :param exchange_db: Path to the exchange Database. + """ + + self.__storing_partitions = [Database(database_dir=partition[0], + database_name=partition[1]).load() for partition in storing_partitions] + self.__exchange_db = Database(database_dir=exchange_db[0], + database_name=exchange_db[1]).load() + self.__on_init_handler() + + def update_list_partitions(self, + partition: Database) -> None: + """ + Add a new storing partition to the list. + + :param partition: New storing partition to add. + """ + + self.__storing_partitions.append(partition) + self.__on_partitions_handler() + + def update_list_partitions_remote(self, + partition: List[str]) -> None: + """ + Add a new storing partition to the list in remote DatabaseHandler. + + :param partition: Path to the new storing partition. + """ + + self.__storing_partitions.append(Database(database_dir=partition[0], + database_name=partition[1]).load()) + self.__on_partitions_handler() + + def load(self) -> None: + """ + Load the Database partitions stored by the component. + """ + + for db in self.__storing_partitions: + db.load() + if self.__exchange_db is not None: + self.__exchange_db.load() + + def get_database_dir(self) -> str: + """ + Get the database repository of the session. + """ + + return self.__storing_partitions[0].get_path()[0] + + def get_partitions(self) -> List[Database]: + """ + Get the storing Database partitions. + """ + + return self.__storing_partitions + + def get_exchange(self) -> Database: + """ + Get the exchange Database. + """ + + return self.__exchange_db + + ########################################################################################## + ########################################################################################## + # Databases architecture # + ########################################################################################## + ########################################################################################## + + def create_fields(self, + table_name: str, + fields: Union[List[Tuple[str, Type]], Tuple[str, Type]]) -> None: + """ + Create new Fields in a Table from one of the Databases. + + :param table_name: Name of the Table. + :param fields: Field or list of Fields names and types. + """ + + # Create the Field(s) in the exchange Database + if table_name == 'Exchange': + self.__exchange_db.load() + if len(self.__exchange_db.get_fields(table_name=table_name)) == 1: + self.__exchange_db.create_fields(table_name=table_name, + fields=fields) + + # Create the Field(s) in the storing Database + else: + if len(self.__storing_partitions) == 1: + self.__storing_partitions[0].load() + if len(self.__storing_partitions[0].get_fields(table_name=table_name)) <= 2: + self.__storing_partitions[0].create_fields(table_name=table_name, + fields=fields) + + def get_fields(self, + table_name: str) -> List[str]: + """ + Get the list of Fields in a Table. + + :param table_name: Name of the Table. + """ + + if self.__exchange_db is None and len(self.__storing_partitions) == 0: + return [] + database = self.__exchange_db if table_name == 'Exchange' else self.__storing_partitions[0] + return database.get_fields(table_name=table_name) + + ########################################################################################## + ########################################################################################## + # Databases editing # + ########################################################################################## + ########################################################################################## + + def add_data(self, + table_name: str, + data: Dict[str, Any]) -> Union[int, List[int]]: + """ + Add a new line of data in a Database. + + :param table_name: Name of the Table. + :param data: New line of the Table. + """ + + # Add data in the exchange Database + if table_name == 'Exchange': + return self.__exchange_db.add_data(table_name=table_name, data=data) + + # Add data in the storing Database + else: + return [len(self.__storing_partitions) - 1, + self.__storing_partitions[-1].add_data(table_name=table_name, data=data)] + + def add_batch(self, + table_name: str, + batch: Dict[str, List[Any]]) -> None: + """ + Add a batch of data in a Database. + + :param table_name: Name of the Table. + :param batch: New lines of the Table. + """ + + # Only available in the storing Database + if table_name == 'Exchange': + raise ValueError(f"Cannot add a batch in the Exchange Database.") + self.__storing_partitions[-1].add_batch(table_name=table_name, + batch=batch) + + def update(self, + table_name: str, + data: Dict[str, Any], + line_id: Union[int, List[int]]) -> None: + """ + Update a line in a Database. + + :param table_name: Name of the Table. + :param data: Updated line of the Table. + :param line_id: Index of the line to edit. + """ + + database = self.__exchange_db if table_name == 'Exchange' else self.__storing_partitions[line_id[0]] + line_id = line_id[1] if type(line_id) == list else line_id + database.update(table_name=table_name, data=data, line_id=line_id) + + def get_line(self, + table_name: str, + line_id: Union[int, List[int]], + fields: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]: + """ + Get a line of data from a Database. + + :param table_name: Name of the Table. + :param line_id: Index of the line to get. + :param fields: Data fields to extract. + """ + + database = self.__exchange_db if table_name == 'Exchange' else self.__storing_partitions[line_id[0]] + line_id = line_id[1] if type(line_id) == list else line_id + if database.nb_lines(table_name=table_name) == 0: + return {} + return database.get_line(table_name=table_name, + line_id=line_id, + fields=fields) + + def get_lines(self, + table_name: str, + lines_id: List[List[int]], + fields: Optional[Union[str, List[str]]] = None) -> Dict[str, Any]: + """ + Get lines of data from a Database. + + :param table_name: Name of the Table. + :param lines_id: Indices of the lines to get. + :param fields: Data fields to extract. + """ + + # Transform list of lines to batch of lines per partition + batch_indices = array(lines_id) + partition_batch_indices = [] + for i in range(len(self.__storing_partitions)): + partition_indices = where(batch_indices[:, 0] == i)[0] + if len(partition_indices) > 0: + partition_batch_indices.append([i, batch_indices[partition_indices, 1].tolist()]) + + # Get lines of data + partition_batches = [] + for partition_indices in partition_batch_indices: + data = self.__storing_partitions[partition_indices[0]].get_lines(table_name=table_name, + lines_id=partition_indices[1], + fields=fields, + batched=True) + del data['id'] + partition_batches.append(data) + + # Merge batches + if len(partition_batches) == 1: + return partition_batches[0] + else: + return dict(zip(partition_batches[0].keys(), + [list(chain.from_iterable([partition_batches[i][key] + for i in range(len(partition_batches))])) + for key in partition_batches[0].keys()])) diff --git a/src/Core/Database/__init__.py b/src/Core/Database/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Core/Environment/BaseEnvironment.py b/src/Core/Environment/BaseEnvironment.py new file mode 100644 index 00000000..41bc90b6 --- /dev/null +++ b/src/Core/Environment/BaseEnvironment.py @@ -0,0 +1,413 @@ +from typing import Any, Optional, Dict, Union, Tuple, List, Type +from numpy import ndarray +from os.path import isfile, join + +from SSD.Core.Storage.Database import Database + +from DeepPhysX.Core.Visualization.VedoFactory import VedoFactory +from DeepPhysX.Core.AsyncSocket.AbstractEnvironment import AbstractEnvironment +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler + + +class BaseEnvironment(AbstractEnvironment): + + def __init__(self, + as_tcp_ip_client: bool = True, + instance_id: int = 1, + instance_nb: int = 1, + **kwargs): + """ + BaseEnvironment computes simulated data for the Network and its training process. + + :param as_tcp_ip_client: Environment is a TcpIpObject if True, is owned by an EnvironmentManager if False. + :param instance_id: ID of the instance. + :param instance_nb: Number of simultaneously launched instances. + """ + + AbstractEnvironment.__init__(self, + as_tcp_ip_client=as_tcp_ip_client, + instance_id=instance_id, + instance_nb=instance_nb) + + # Training data variables + self.__data_training: Dict[str, ndarray] = {} + self.__data_additional: Dict[str, ndarray] = {} + self.compute_training_data: bool = True + + # Dataset data variables + self.update_line: Optional[int] = None + self.sample_training: Optional[Dict[str, Any]] = None + self.sample_additional: Optional[Dict[str, Any]] = None + self.__first_add: List[bool] = [True, True] + + # Connect the Environment to the data Database + self.__database_handler = DatabaseHandler(on_init_handler=self.__database_handler_init) + + # Connect the Factory to the visualization Database + self.factory: Optional[VedoFactory] = None + + ########################################################################################## + ########################################################################################## + # Environment initialization # + ########################################################################################## + ########################################################################################## + + def create(self) -> None: + """ + Create the Environment. Automatically called when Environment is launched. + Must be implemented by user. + """ + + raise NotImplementedError + + def init(self) -> None: + """ + Initialize the Environment. Automatically called when Environment is launched. + Not mandatory. + """ + + pass + + def init_database(self) -> None: + """ + Define the fields of the training dataset. Automatically called when Environment is launched. + Must be implemented by user. + """ + + raise NotImplementedError + + def init_visualization(self) -> None: + """ + Define the visualization objects to send to the Visualizer. Automatically called when Environment is launched. + Not mandatory. + """ + + pass + + def save_parameters(self, **kwargs) -> None: + """ + Save a set of parameters in the Database. + """ + + # Create a dedicated Database + database_dir = self.__database_handler.get_database_dir() + if isfile(join(database_dir, 'environment_parameters.db')): + database = Database(database_dir=database_dir, + database_name='environment_parameters').load() + else: + database = Database(database_dir=database_dir, + database_name='environment_parameters').new() + + # Create fields and add data + fields = [(field, type(value)) for field, value in kwargs.items()] + database.create_table(table_name=f'Environment_{self.instance_id}', fields=fields) + database.add_data(table_name=f'Environment_{self.instance_id}', data=kwargs) + database.close() + + def load_parameters(self) -> Dict[str, Any]: + """ + Load a set of parameters from the Database. + """ + + # Load the dedicated Database and the parameters + database_dir = self.__database_handler.get_database_dir() + if isfile(join(database_dir, 'environment_parameters.db')): + database = Database(database_dir=database_dir, + database_name='environment_parameters').load() + parameters = database.get_line(table_name=f'Environment_{self.instance_id}') + del parameters['id'] + return parameters + return {} + + ########################################################################################## + ########################################################################################## + # Environment behavior # + ########################################################################################## + ########################################################################################## + + async def step(self) -> None: + """ + Compute the number of steps in the Environment specified by simulations_per_step in EnvironmentConfig. + Must be implemented by user. + """ + + raise NotImplementedError + + def check_sample(self) -> bool: + """ + Check if the current produced sample is usable for training. + Not mandatory. + + :return: Current data can be used or not + """ + + return True + + def apply_prediction(self, + prediction: Dict[str, ndarray]) -> None: + """ + Apply network prediction in environment. + Not mandatory. + + :param prediction: Prediction data. + """ + + pass + + def close(self) -> None: + """ + Close the Environment. Automatically called when Environment is shut down. + Not mandatory. + """ + + pass + + ########################################################################################## + ########################################################################################## + # Defining data samples # + ########################################################################################## + ########################################################################################## + + def define_training_fields(self, + fields: Union[List[Tuple[str, Type]], Tuple[str, Type]]) -> None: + """ + Specify the training data fields names and types. + + :param fields: Field or list of fields to tag as training data. + """ + + self.__database_handler.create_fields(table_name='Training', + fields=fields) + + def define_additional_fields(self, + fields: Union[List[Tuple[str, Type]], Tuple[str, Type]]) -> None: + """ + Specify the additional data fields names and types. + + :param fields: Field or list of Fields to tag as additional data. + """ + + self.__database_handler.create_fields(table_name='Additional', + fields=fields) + + def set_training_data(self, **kwargs) -> None: + """ + Set the training data to send to the TcpIpServer or the EnvironmentManager. + """ + + # Check kwargs + if self.__first_add[0]: + self.__database_handler.load() + self.__first_add[0] = False + required_fields = list(set(self.__database_handler.get_fields(table_name='Training')) - {'id', 'env_id'}) + if len(required_fields) > 0: + for field in kwargs.keys(): + if field not in required_fields: + raise ValueError(f"[{self.name}] The field '{field}' is not in the training Database." + f"Required fields are {required_fields}.") + for field in required_fields: + if field not in kwargs.keys(): + raise ValueError(f"[{self.name}] The field '{field}' was not defined in training data." + f"Required fields are {required_fields}.") + + # Training data is set if the Environment can compute data + if self.compute_training_data: + self.__data_training = kwargs + self.__data_training['env_id'] = self.instance_id + + def set_additional_data(self, + **kwargs) -> None: + """ + Set the additional data to send to the TcpIpServer or the EnvironmentManager. + """ + + # Additional data is also set if the Environment can compute data + if self.compute_training_data: + self.__data_additional = kwargs + self.__data_additional['env_id'] = self.instance_id + + ########################################################################################## + ########################################################################################## + # Available requests # + ########################################################################################## + ########################################################################################## + + def get_prediction(self, **kwargs) -> Dict[str, ndarray]: + """ + Request a prediction from Network. + + :return: Network prediction. + """ + + # Check kwargs + if self.__first_add[1]: + + if (self.environment_manager is not None and not self.environment_manager.allow_prediction_requests) or \ + (self.tcp_ip_client is not None and not self.tcp_ip_client.allow_prediction_requests): + raise ValueError(f"[{self.name}] Prediction request is not available in data generation Pipeline.") + + if len(kwargs) == 0 and len(self.__data_training) == 0: + raise ValueError(f"[{self.name}] The prediction request requires the network fields.") + self.__database_handler.load() + self.__first_add[1] = False + required_fields = list(set(self.__database_handler.get_fields(table_name='Exchange')) - {'id'}) + for field in kwargs.keys(): + if field not in required_fields: + raise ValueError(f"[{self.name}] The field '{field}' is not in the training Database." + f"Required fields are {required_fields}.") + + # Avoid empty sample + if len(kwargs) == 0: + required_fields = set(self.__database_handler.get_fields(table_name='Exchange')) - {'id'} + necessary_fields = list(required_fields.intersection(self.__data_training.keys())) + kwargs = {field: self.__data_training[field] for field in necessary_fields} + + # If Environment is a TcpIpClient, send request to the Server + if self.tcp_ip_client is not None: + return self.tcp_ip_client.get_prediction(**kwargs) + + # Otherwise, check the hierarchy of managers + elif self.environment_manager is not None: + if self.environment_manager.data_manager is None: + raise ValueError("Cannot request prediction if DataManager does not exist") + # Get a prediction + self.__database_handler.update(table_name='Exchange', + data=kwargs, + line_id=self.instance_id) + self.environment_manager.data_manager.get_prediction(self.instance_id) + data_pred = self.__database_handler.get_line(table_name='Exchange', + line_id=self.instance_id) + del data_pred['id'] + return data_pred + + else: + raise ValueError(f"[{self.name}] This Environment has not Manager.") + + def update_visualisation(self) -> None: + """ + Triggers the Visualizer update. + """ + + if self.factory is not None: + # If Environment is a TcpIpClient, request to the Server + if self.as_tcp_ip_client: + self.tcp_ip_client.request_update_visualization() + self.factory.render() + + def _get_prediction(self): + """ + Request a prediction from Network and apply it to the Environment. + Should not be used by users. + """ + + training_data = self.__data_training.copy() + required_fields = self.__database_handler.get_fields(table_name='Exchange') + for field in self.__data_training.keys(): + if field not in required_fields: + del training_data[field] + self.apply_prediction(self.get_prediction(**training_data)) + + ########################################################################################## + ########################################################################################## + # Database communication # + ########################################################################################## + ########################################################################################## + + def get_database_handler(self) -> DatabaseHandler: + """ + Get the DatabaseHandler of the Environment. + """ + + return self.__database_handler + + def __database_handler_init(self): + """ + Init event of the DatabaseHandler. + """ + + # Load Database and create basic fields + self.__database_handler.load() + + def _create_visualization(self, + visualization_db: Union[Database, Tuple[str, str]]) -> None: + """ + Create a Factory for the Environment. + """ + + if type(visualization_db) == list: + self.factory = VedoFactory(database_path=visualization_db, + idx_instance=self.instance_id, + remote=True) + else: + self.factory = VedoFactory(database=visualization_db, + idx_instance=self.instance_id) + + def _send_training_data(self) -> List[int]: + """ + Add the training data and the additional data in their respective Databases. + Should not be used by users. + + :return: Index of the samples in the Database. + """ + + line_id = self.__database_handler.add_data(table_name='Training', + data=self.__data_training) + self.__database_handler.add_data(table_name='Additional', + data=self.__data_additional) + return line_id + + def _update_training_data(self, + line_id: List[int]) -> None: + """ + Update the training data and the additional data in their respective Databases. + Should not be used by users. + + :param line_id: Index of the samples to update. + """ + + if self.__data_training != {}: + self.__database_handler.update(table_name='Training', + data=self.__data_training, + line_id=line_id) + if self.__data_additional != {}: + self.__database_handler.update(table_name='Additional', + data=self.__data_additional, + line_id=line_id) + + def _get_training_data(self, + line_id: List[int]) -> None: + """ + Get the training data and the additional data from their respective Databases. + Should not be used by users. + + :param line_id: Index of the sample to get. + """ + + self.update_line = line_id + self.sample_training = self.__database_handler.get_line(table_name='Training', + line_id=line_id) + self.sample_additional = self.__database_handler.get_line(table_name='Additional', + line_id=line_id) + self.sample_additional = None if len(self.sample_additional) == 1 else self.sample_additional + + def _reset_training_data(self) -> None: + """ + Reset the training data and the additional data variables. + Should not be used by users. + """ + + self.__data_training = {} + self.__data_additional = {} + self.sample_training = None + self.sample_additional = None + self.update_line = None + + def __str__(self): + + description = "\n" + description += f" {self.name}\n" + description += f" Name: {self.name} n°{self.instance_id}\n" + description += f" Comments:\n" + description += f" Input size:\n" + description += f" Output size:\n" + return description diff --git a/src/Environment/BaseEnvironmentConfig.py b/src/Core/Environment/BaseEnvironmentConfig.py similarity index 54% rename from src/Environment/BaseEnvironmentConfig.py rename to src/Core/Environment/BaseEnvironmentConfig.py index c9da9ea5..94d840a9 100644 --- a/src/Environment/BaseEnvironmentConfig.py +++ b/src/Core/Environment/BaseEnvironmentConfig.py @@ -1,8 +1,8 @@ -from typing import Any, Dict, Optional, Type +from typing import Any, Optional, Type, Dict, Tuple from os import cpu_count from os.path import join, dirname from threading import Thread -from subprocess import run as subprocessRun +from subprocess import run from sys import modules, executable from DeepPhysX.Core.AsyncSocket.TcpIpServer import TcpIpServer @@ -14,45 +14,38 @@ class BaseEnvironmentConfig: def __init__(self, environment_class: Type[BaseEnvironment], - visualizer: Optional[Type[VedoVisualizer]] = None, - simulations_per_step: int = 1, - max_wrong_samples_per_step: int = 10, - always_create_data: bool = False, - record_wrong_samples: bool = False, - screenshot_sample_rate: int = 0, - use_dataset_in_environment: bool = False, - param_dict: Optional[Dict[Any, Any]] = None, as_tcp_ip_client: bool = True, number_of_thread: int = 1, - max_client_connection: int = 1000, - environment_file: Optional[str] = None, ip_address: str = 'localhost', - port: int = 10000): + port: int = 10000, + simulations_per_step: int = 1, + max_wrong_samples_per_step: int = 10, + load_samples: bool = False, + only_first_epoch: bool = True, + always_produce: bool = False, + visualizer: Optional[Type[VedoVisualizer]] = None, + record_wrong_samples: bool = False, + env_kwargs: Optional[Dict[str, Any]] = None): """ BaseEnvironmentConfig is a configuration class to parameterize and create a BaseEnvironment for the EnvironmentManager. :param environment_class: Class from which an instance will be created. - :param visualizer: Class of the Visualizer to use. - :param simulations_per_step: Number of iterations to compute in the Environment at each time step. - :param max_wrong_samples_per_step: Maximum number of wrong samples to produce in a step. - :param always_create_data: If True, data will always be created from environment. If False, data will be - created from the environment during the first epoch and then re-used from the - Dataset. - :param record_wrong_samples: If True, wrong samples are recorded through Visualizer. - :param screenshot_sample_rate: A screenshot of the viewer will be done every x sample. - :param use_dataset_in_environment: If True, the dataset will always be used in the environment. - :param param_dict: Dictionary containing specific environment parameters. :param as_tcp_ip_client: Environment is owned by a TcpIpClient if True, by an EnvironmentManager if False. :param number_of_thread: Number of thread to run. - :param max_client_connection: Maximum number of handled instances. - :param environment_file: Path of the file containing the Environment class. :param ip_address: IP address of the TcpIpObject. :param port: Port number of the TcpIpObject. + :param simulations_per_step: Number of iterations to compute in the Environment at each time step. + :param max_wrong_samples_per_step: Maximum number of wrong samples to produce in a step. + :param load_samples: If True, the dataset will always be used in the environment. + :param only_first_epoch: If True, data will always be created from environment. If False, data will be created + from the environment during the first epoch and then re-used from the Dataset. + :param always_produce: If True, data will always be produced in Environment(s). + :param visualizer: Class of the Visualizer to use. + :param record_wrong_samples: If True, wrong samples are recorded through Visualizer. + :param env_kwargs: Additional arguments to pass to the Environment. """ - if param_dict is None: - param_dict = {} self.name: str = self.__class__.__name__ # Check simulations_per_step type and value @@ -67,141 +60,123 @@ def __init__(self, f"{type(max_wrong_samples_per_step)}") if max_wrong_samples_per_step < 1: raise ValueError(f"[{self.name}] Given max_wrong_simulations_per_step value is negative or null") - # Check always_create_data type - if type(always_create_data) != bool: + # Check only_first_epoch type + if type(only_first_epoch) != bool: raise TypeError(f"[{self.name}] Wrong always_create_data type: bool required, get " - f"{type(always_create_data)}") + f"{type(only_first_epoch)}") if type(number_of_thread) != int: raise TypeError(f"[{self.name}] The number_of_thread number must be a positive integer.") if number_of_thread < 0: raise ValueError(f"[{self.name}] The number_of_thread number must be a positive integer.") - # TcpIpClients parameterization + # TcpIpClients variables self.environment_class: Type[BaseEnvironment] = environment_class - self.environment_file: str = environment_file if environment_file is not None else modules[ - self.environment_class.__module__].__file__ - self.param_dict: Optional[Dict[Any, Any]] = param_dict + self.environment_file: str = modules[self.environment_class.__module__].__file__ self.as_tcp_ip_client: bool = as_tcp_ip_client - # EnvironmentManager parameterization - self.received_parameters: Dict[Any, Any] = {} - self.always_create_data: bool = always_create_data - self.record_wrong_samples: bool = record_wrong_samples - self.screenshot_sample_rate: int = screenshot_sample_rate - self.use_dataset_in_environment: bool = use_dataset_in_environment - self.simulations_per_step: int = simulations_per_step - self.max_wrong_samples_per_step: int = max_wrong_samples_per_step - self.visualizer: Optional[Type[VedoVisualizer]] = visualizer - - # TcpIpServer parameterization + # TcpIpServer variables + self.number_of_thread: int = min(max(number_of_thread, 1), cpu_count()) # Assert nb is between 1 and cpu_count self.ip_address: str = ip_address self.port: int = port self.server_is_ready: bool = False - self.number_of_thread: int = min(max(number_of_thread, 1), cpu_count()) # Assert nb is between 1 and cpu_count - self.max_client_connections = max_client_connection + self.max_client_connections: int = 100 + + # EnvironmentManager variables + self.simulations_per_step: int = simulations_per_step + self.max_wrong_samples_per_step: int = max_wrong_samples_per_step + self.load_samples: bool = load_samples + self.only_first_epoch: bool = only_first_epoch + self.always_produce: bool = always_produce + self.env_kwargs: Dict[str, Any] = {} if env_kwargs is None else env_kwargs + + # Visualizer variables + self.visualizer: Optional[Type[VedoVisualizer]] = visualizer + self.record_wrong_samples: bool = record_wrong_samples def create_server(self, environment_manager: Optional[Any] = None, batch_size: int = 1, - visu_db: Optional[int] = None) -> TcpIpServer: + visualization_db: Optional[Tuple[str, str]] = None) -> TcpIpServer: """ Create a TcpIpServer and launch TcpIpClients in subprocesses. :param environment_manager: EnvironmentManager. :param batch_size: Number of sample in a batch. - :param visu_db: The path to the visualization Database to connect to. + :param visualization_db: Path to the visualization Database to connect to. :return: TcpIpServer object. """ # Create server - server = TcpIpServer(max_client_count=self.max_client_connections, batch_size=batch_size, - nb_client=self.number_of_thread, manager=environment_manager, - ip_address=self.ip_address, port=self.port) - server_thread = Thread(target=self.start_server, args=(server,)) + server = TcpIpServer(ip_address=self.ip_address, + port=self.port, + nb_client=self.number_of_thread, + max_client_count=self.max_client_connections, + batch_size=batch_size, + manager=environment_manager) + server_thread = Thread(target=self.start_server, args=(server, visualization_db)) server_thread.start() # Create clients client_threads = [] for i in range(self.number_of_thread): - client_thread = Thread(target=self.start_client, args=(i, visu_db)) + client_thread = Thread(target=self.start_client, args=(i + 1,)) client_threads.append(client_thread) for client in client_threads: client.start() - # Return server to manager when ready + # Return server to manager when it is ready while not self.server_is_ready: pass return server def start_server(self, - server: TcpIpServer) -> None: + server: TcpIpServer, + visualization_db: Optional[Tuple[str, str]] = None) -> None: """ Start TcpIpServer. :param server: TcpIpServer. + :param visualization_db: Path to the visualization Database to connect to. """ - # Allow clients connections server.connect() - # Send and receive parameters with clients - self.received_parameters = server.initialize(self.param_dict) - # Server is ready - self.server_is_ready: bool = True + server.initialize(visualization_db=visualization_db, + env_kwargs=self.env_kwargs) + self.server_is_ready = True def start_client(self, - idx: int = 1, - visu_db: Optional[int] = None) -> None: + idx: int = 1) -> None: """ Run a subprocess to start a TcpIpClient. :param idx: Index of client. - :param visu_db: The path to the visualization Database to connect to. """ script = join(dirname(modules[BaseEnvironment.__module__].__file__), 'launcherBaseEnvironment.py') - # Usage: python3 script.py " - subprocessRun([executable, - script, - self.environment_file, - self.environment_class.__name__, - self.ip_address, - str(self.port), - str(idx), - str(self.number_of_thread), - str(visu_db)]) - - def create_environment(self, - environment_manager: Any, - visu_db: Optional[Any] = None) -> BaseEnvironment: + run([executable, script, self.environment_file, self.environment_class.__name__, + self.ip_address, str(self.port), str(idx), str(self.number_of_thread)]) + + def create_environment(self) -> BaseEnvironment: """ Create an Environment that will not be a TcpIpObject. - :param environment_manager: EnvironmentManager that handles the Environment. - :param visu_db: The visualisation Database to connect to. :return: Environment object. """ # Create instance - environment = self.environment_class(environment_manager=environment_manager, - as_tcp_ip_client=False, - visu_db=visu_db) - # Create & Init Environment - environment.recv_parameters(self.param_dict) - environment.create() - environment.init() - environment.init_visualization() + environment = self.environment_class(as_tcp_ip_client=False, + **self.env_kwargs) + if not isinstance(environment, BaseEnvironment): + raise TypeError(f"[{self.name}] The given 'environment_class'={self.environment_class} must be a " + f"BaseEnvironment.") return environment - def __str__(self) -> str: - """ - :return: String containing information about the BaseEnvironmentConfig object - """ + def __str__(self): - # Todo: fields in Configs are the set in Managers or objects, remove __str__ method description = "\n" description += f"{self.name}\n" description += f" Environment class: {self.environment_class.__name__}\n" description += f" Simulations per step: {self.simulations_per_step}\n" description += f" Max wrong samples per step: {self.max_wrong_samples_per_step}\n" - description += f" Always create data: {self.always_create_data}\n" + description += f" Always create data: {self.only_first_epoch}\n" return description diff --git a/src/Environment/__init__.py b/src/Core/Environment/__init__.py similarity index 100% rename from src/Environment/__init__.py rename to src/Core/Environment/__init__.py diff --git a/src/Environment/launcherBaseEnvironment.py b/src/Core/Environment/launcherBaseEnvironment.py similarity index 66% rename from src/Environment/launcherBaseEnvironment.py rename to src/Core/Environment/launcherBaseEnvironment.py index 15c68b28..2d5b63e6 100644 --- a/src/Environment/launcherBaseEnvironment.py +++ b/src/Core/Environment/launcherBaseEnvironment.py @@ -3,13 +3,14 @@ from sys import argv, path from DeepPhysX.Core.Environment.BaseEnvironment import BaseEnvironment as Environment +from DeepPhysX.Core.AsyncSocket.TcpIpClient import TcpIpClient if __name__ == '__main__': # Check script call - if len(argv) != 8: + if len(argv) != 7: print(f"Usage: python3 {argv[0]} " - f" ") + f"") exit(1) # Import environment_class @@ -18,9 +19,11 @@ exec(f"from {module_name} import {argv[2]} as Environment") # Create, init and run Tcp-Ip environment - visu_db = None if argv[7] == 'None' else [s[1:-1] for s in argv[7][1:-1].split(', ')] - client = Environment(ip_address=argv[3], port=int(argv[4]), instance_id=int(argv[5]), - number_of_instances=int(argv[6]), visu_db=visu_db) + client = TcpIpClient(environment=Environment, + ip_address=argv[3], + port=int(argv[4]), + instance_id=int(argv[5]), + instance_nb=int(argv[6])) client.initialize() client.launch() diff --git a/src/Core/Manager/DataManager.py b/src/Core/Manager/DataManager.py new file mode 100644 index 00000000..c4241984 --- /dev/null +++ b/src/Core/Manager/DataManager.py @@ -0,0 +1,188 @@ +from typing import Any, Optional, Dict, List + +from DeepPhysX.Core.Manager.DatabaseManager import DatabaseManager +from DeepPhysX.Core.Manager.EnvironmentManager import EnvironmentManager +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler + + +class DataManager: + + def __init__(self, + pipeline: Any, + database_config: Optional[BaseDatabaseConfig] = None, + environment_config: Optional[BaseEnvironmentConfig] = None, + session: str = 'sessions/default', + new_session: bool = True, + produce_data: bool = True, + batch_size: int = 1): + + """ + DataManager deals with the generation, storage and loading of training data. + + :param pipeline: Pipeline that handle the DataManager. + :param database_config: Configuration object with the parameters of the Database. + :param environment_config: Configuration object with the parameters of the Environment. + :param session: Path to the session repository. + :param new_session: If True, the session is done in a new repository. + :param produce_data: If True, this session will store data in the Database. + :param batch_size: Number of samples in a single batch. + """ + + self.name: str = self.__class__.__name__ + + # Session variables + self.pipeline: Optional[Any] = pipeline + self.database_manager: Optional[DatabaseManager] = None + self.environment_manager: Optional[EnvironmentManager] = None + + # Create a DatabaseManager + self.database_manager = DatabaseManager(database_config=database_config, + data_manager=self, + pipeline=pipeline.type, + session=session, + new_session=new_session, + produce_data=produce_data) + + # Create an EnvironmentManager if required + if environment_config is not None: + self.environment_manager = EnvironmentManager(environment_config=environment_config, + data_manager=self, + pipeline=pipeline.type, + session=session, + produce_data=produce_data, + batch_size=batch_size) + + # DataManager variables + self.produce_data = produce_data + self.batch_size = batch_size + self.data_lines: List[List[int]] = [] + + @property + def nb_environment(self) -> Optional[int]: + """ + Get the number of Environments managed by the EnvironmentManager. + """ + + if self.environment_manager is None: + return None + return 1 if self.environment_manager.server is None else self.environment_manager.number_of_thread + + @property + def normalization(self) -> Dict[str, List[float]]: + """ + Get the normalization coefficients computed by the DatabaseManager. + """ + + return self.database_manager.normalization + + def connect_handler(self, + handler: DatabaseHandler) -> None: + """ + Add a new DatabaseHandler to the list of handlers of the DatabaseManager. + + :param handler: New handler to register. + """ + + self.database_manager.connect_handler(handler) + + def get_data(self, + epoch: int = 0, + animate: bool = True, + load_samples: bool = True) -> None: + """ + Fetch data from the EnvironmentManager or the DatabaseManager according to the context. + + :param epoch: Current epoch number. + :param animate: Allow EnvironmentManager to trigger a step itself in order to generate a new sample. + :param load_samples: If True, trigger a sample loading from the Database. + """ + + # Data generation case + if self.pipeline.type == 'data_generation': + self.environment_manager.get_data(animate=animate) + self.database_manager.add_data() + + # Training case + elif self.pipeline.type == 'training': + + # Get data from Environment(s) if used and if the data should be created at this epoch + if self.environment_manager is not None and self.produce_data and \ + (epoch == 0 or self.environment_manager.always_produce): + self.data_lines = self.environment_manager.get_data(animate=animate) + self.database_manager.add_data(self.data_lines) + + # Get data from Dataset + else: + self.data_lines = self.database_manager.get_data(batch_size=self.batch_size) + # Dispatch a batch to clients + if self.environment_manager is not None: + if self.environment_manager.load_samples and \ + (epoch == 0 or not self.environment_manager.only_first_epoch): + self.environment_manager.dispatch_batch(data_lines=self.data_lines, + animate=animate) + # Environment is no longer used + else: + self.environment_manager.close() + self.environment_manager = None + + # Prediction pipeline + else: + + # Get data from Dataset + if self.environment_manager.load_samples: + if load_samples: + self.data_lines = self.database_manager.get_data(batch_size=1) + self.environment_manager.dispatch_batch(data_lines=self.data_lines, + animate=animate, + request_prediction=True, + save_data=self.produce_data) + # Get data from Environment + else: + self.data_lines = self.environment_manager.get_data(animate=animate, + request_prediction=True, + save_data=self.produce_data) + if self.produce_data: + self.database_manager.add_data(self.data_lines) + + def load_sample(self) -> List[int]: + """ + Load a sample from the Database. + + :return: Index of the loaded line. + """ + + self.data_lines = self.database_manager.get_data(batch_size=1) + return self.data_lines[0] + + def get_prediction(self, + instance_id: int) -> None: + """ + Get a Network prediction for the specified Environment instance. + """ + + # Get a prediction + if self.pipeline is None: + raise ValueError("Cannot request prediction if Manager (and then NetworkManager) does not exist.") + self.pipeline.network_manager.compute_online_prediction(instance_id=instance_id, + normalization=self.normalization) + + def close(self) -> None: + """ + Launch the closing procedure of the DataManager. + """ + + if self.environment_manager is not None: + self.environment_manager.close() + if self.database_manager is not None: + self.database_manager.close() + + def __str__(self): + + data_manager_str = "" + if self.environment_manager: + data_manager_str += str(self.environment_manager) + if self.database_manager: + data_manager_str += str(self.database_manager) + return data_manager_str diff --git a/src/Core/Manager/DatabaseManager.py b/src/Core/Manager/DatabaseManager.py new file mode 100644 index 00000000..3efc35b1 --- /dev/null +++ b/src/Core/Manager/DatabaseManager.py @@ -0,0 +1,638 @@ +from typing import Any, Dict, List, Optional +from os.path import isfile, isdir, join +from os import listdir, symlink, sep, remove, rename +from json import dump as json_dump +from json import load as json_load +from numpy import arange, ndarray, array, abs, mean, sqrt, empty, concatenate +from numpy.random import shuffle + +from SSD.Core.Storage.Database import Database + +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler +from DeepPhysX.Core.Utils.path import create_dir, copy_dir, get_first_caller +from DeepPhysX.Core.Utils.jsonUtils import CustomJSONEncoder + + +class DatabaseManager: + + def __init__(self, + database_config: Optional[BaseDatabaseConfig] = None, + data_manager: Optional[Any] = None, + pipeline: str = '', + session: str = 'sessions/default', + new_session: bool = True, + produce_data: bool = True): + """ + DatabaseManager handle all operations with input / output files. Allows saving and read tensors from files. + + :param database_config: Configuration object with the parameters of the Database. + :param data_manager: DataManager that handles the DatabaseManager. + :param pipeline: Type of the Pipeline. + :param session: Path to the session repository. + :param new_session: If True, the session is done in a new repository. + :param produce_data: If True, this session will store data in the Database. + """ + + self.name: str = self.__class__.__name__ + + # Session variables + self.pipeline: str = pipeline + self.data_manager: Optional[Any] = data_manager + self.database_dir: str = join(session, 'dataset') + self.database_handlers: List[DatabaseHandler] = [] + root = get_first_caller() + database_config = BaseDatabaseConfig() if database_config is None else database_config + + # Dataset parameters + self.max_file_size: int = database_config.max_file_size + self.shuffle: bool = database_config.shuffle + self.produce_data = produce_data + self.normalize: bool = database_config.normalize + self.total_nb_sample: int = 0 + self.recompute_normalization: bool = database_config.recompute_normalization + + # Dataset modes + self.modes: List[str] = ['training', 'validation', 'prediction'] + if pipeline == 'data_generation': + self.mode: str = 'training' if database_config.mode is None else database_config.mode + elif pipeline == 'training': + self.mode: str = 'training' + else: + self.mode: str = 'prediction' if database_config.mode is None else database_config.mode + self.mode = 'prediction' if produce_data else self.mode + + # Dataset partitions + session_name = session.split(sep)[-1] + self.partition_template: Dict[str, str] = {mode: f'{session_name}_{mode}_' + '{}' for mode in self.modes} + self.partition_index: Dict[str, int] = {mode: 0 for mode in self.modes} + self.partition_names: Dict[str, List[str]] = {mode: [] for mode in self.modes} + self.partitions: Dict[str, List[Database]] = {mode: [] for mode in self.modes} + + # Dataset indexing + self.sample_indices: ndarray = empty((0, 2), dtype=int) + self.sample_id: int = 0 + self.first_add = True + + # Dataset json file + self.json_default: Dict[str, Dict[str, Any]] = {'partitions': {mode: [] for mode in self.modes}, + 'nb_samples': {mode: [] for mode in self.modes}, + 'architecture': {}, + 'data_shape': {}, + 'normalization': {}} + self.json_content: Dict[str, Dict[str, Any]] = self.json_default.copy() + + # DataGeneration case + if self.pipeline == 'data_generation': + + # Generate data in a new session + if new_session: + # Generate data from scratch --> create a new directory + if database_config.existing_dir is None: + create_dir(session_dir=session, session_name='dataset') + self.create_partition() + # Complete a Database in a new session --> copy and load the existing directory + else: + copy_dir(src_dir=join(root, database_config.existing_dir), dest_dir=session, + sub_folders='dataset') + self.load_directory(rename_partitions=True) + # Complete a Database in the same session --> load the directory + else: + self.load_directory() + + # Training case + elif self.pipeline == 'training': + + # Generate data + if produce_data: + # Generate data in a new session + if new_session: + # Generate data from scratch --> create a new directory + if database_config.existing_dir is None: + create_dir(session_dir=session, session_name='dataset') + self.create_partition() + # Complete a Database in a new session --> copy and load the existing directory + else: + copy_dir(src_dir=join(root, database_config.existing_dir), dest_dir=session, + sub_folders='dataset') + self.load_directory() + # Complete a Database in the same directory --> load the directory + else: + self.load_directory() + + # Load data + else: + # Load data in a new session --> link and load the existing directory + if new_session: + symlink(src=join(root, database_config.existing_dir, 'dataset'), + dst=join(session, 'dataset')) + self.load_directory() + # Load data in the same session --> load the directory + else: + self.load_directory() + + # Prediction case + else: + self.load_directory() + + # Finally create an exchange database + self.exchange = Database(database_dir=self.database_dir, + database_name='Exchange').new(remove_existing=True) + self.exchange.create_table(table_name='Exchange') + + ########################################################################################## + ########################################################################################## + # Partitions Management # + ########################################################################################## + ########################################################################################## + + def load_directory(self, + rename_partitions: bool = False) -> None: + """ + Get the Database information from the json file (partitions, samples, etc). + Load all the partitions or create one if necessary. + + :param rename_partitions: If True, the existing partitions should be renamed to match the session name. + """ + + # 1. Check the directory existence to prevent bugs + if not isdir(self.database_dir): + raise Warning(f"[{self.name}] Impossible to load Dataset from {self.database_dir}.") + + # 2. Get the .json description file + json_found = False + if isfile(join(self.database_dir, 'dataset.json')): + json_found = True + with open(join(self.database_dir, 'dataset.json')) as json_file: + self.json_content = json_load(json_file) + + # 3. Update json file if not found + if not json_found or self.json_content == self.json_default: + self.search_partitions_info() + self.update_json() + if self.recompute_normalization or ( + self.normalize and self.json_content['normalization'] == self.json_default['normalization']): + self.json_content['normalization'] = self.compute_normalization() + self.update_json() + + # 4. Load partitions for each mode + self.partition_names = self.json_content['partitions'] + self.partition_index = {mode: len(self.partition_names[mode]) for mode in self.modes} + if rename_partitions: + for mode in self.modes: + current_name = self.partition_template[mode].split(f'_{mode}_')[0] + for i, name in enumerate(self.partition_names[mode]): + if name.split(f'_{mode}_')[0] != current_name: + self.partition_names[mode][i] = current_name + f'_{mode}_{i}' + rename(src=join(self.database_dir, f'{name}.db'), + dst=join(self.database_dir, f'{self.partition_names[mode][i]}.db')) + + # 5. Load the partitions + for mode in self.modes: + for name in self.partition_names[mode]: + db = Database(database_dir=self.database_dir, + database_name=name).load() + self.partitions[mode].append(db) + if len(self.partitions[self.mode]) == 0: + self.create_partition() + elif self.max_file_size is not None and self.partitions[self.mode][-1].memory_size > self.max_file_size \ + and self.produce_data: + self.create_partition() + + # 6. Index partitions + self.index_samples() + + def create_partition(self) -> None: + """ + Create a new partition of the Database. + """ + + # 1. Define the partition name + partition_name = self.partition_template[self.mode].format(self.partition_index[self.mode]) + self.partition_names[self.mode].append(partition_name) + + # 2. Create the Database partition + db = Database(database_dir=self.database_dir, + database_name=partition_name).new() + db.create_table(table_name='Training') + db.create_table(table_name='Additional') + self.partitions[self.mode].append(db) + + # 3. If the partition is an additional one, create all fields + if self.partition_index[self.mode] > 0: + # Get fields + fields = {} + types = {'INT': int, 'FLOAT': float, 'STR': str, 'BOOL': bool, 'NUMPY': ndarray} + for table_name in self.partitions[self.mode][0].get_tables(): + fields[table_name] = [] + F = self.partitions[self.mode][0].get_fields(table_name=table_name, + only_names=False) + for field in [f for f in F if f not in ['id', '_dt_']]: + fields[table_name].append((field, types[F[field].field_type])) + # Re-create them + for table_name in fields.keys(): + self.partitions[self.mode][-1].create_fields(table_name=table_name, + fields=fields[table_name]) + else: + self.partitions[self.mode][-1].create_fields(table_name='Training', + fields=('env_id', int)) + self.partitions[self.mode][-1].create_fields(table_name='Additional', + fields=('env_id', int)) + + # 4. Update the partitions in handlers + for handler in self.database_handlers: + handler.update_list_partitions(self.partitions[self.mode][-1]) + self.partition_index[self.mode] += 1 + self.json_content['partitions'] = self.partition_names + self.get_nb_samples() + self.update_json() + + def get_partition_objects(self) -> List[Database]: + """ + Get the list of partitions of the Database for the current mode. + """ + + return self.partitions[self.mode] + + def get_partition_names(self) -> List[List[str]]: + """ + Get the list of partition paths of the Database for the current mode. + """ + + return [db.get_path() for db in self.partitions[self.mode]] + + def remove_empty_partitions(self): + """ + Remove every empty partitions of the Database. + """ + + for mode in self.modes: + # Check if the last partition for the mode is empty + if len(self.partitions[mode]) > 0 and self.partitions[mode][-1].nb_lines(table_name='Training') == 0: + # Erase partition file + path = self.partitions[mode].pop(-1).get_path() + remove(join(path[0], f'{path[1]}.db')) + # Remove from information + self.partition_names[mode].pop(-1) + self.partition_index[mode] -= 1 + self.json_content['nb_samples'][mode].pop(-1) + self.json_content['partitions'] = self.partition_names + self.update_json() + + def change_mode(self, + mode: str) -> None: + """ + Change the current Database mode. + + :param mode: Name of the Database mode. + """ + + pass + + ########################################################################################## + ########################################################################################## + # JSON Information file # + ########################################################################################## + ########################################################################################## + + def search_partitions_info(self) -> None: + """ + Get the information about the Database manually if the json file is not found. + """ + + # 1. Get all the partitions + raw_partitions = {mode: [f for f in listdir(self.database_dir) if isfile(join(self.database_dir, f)) + and f.endswith('.db') and f.__contains__(mode)] for mode in self.modes} + raw_partitions = {mode: [f.split('.')[0] for f in raw_partitions[mode]] for mode in self.modes} + self.json_content['partitions'] = {mode: sorted(raw_partitions[mode]) for mode in self.modes} + + # 2. Get the number of samples + for mode in self.modes: + for name in self.json_content['partitions'][mode]: + db = Database(database_dir=self.database_dir, + database_name=name).load() + self.json_content['nb_samples'][mode].append(db.nb_lines(table_name='Training')) + db.close() + + # 3. Get the Database architecture + self.json_content['architecture'] = self.get_database_architecture() + self.first_add = False + + # 4. Get the data shapes + self.json_content['data_shape'] = self.get_data_shapes() + + def get_database_architecture(self) -> Dict[str, List[str]]: + """ + Get the Tables and Fields structure of the Database. + """ + + # Get a training or validation partition + if len(self.json_content['partitions']['training']) != 0: + db = Database(database_dir=self.database_dir, + database_name=self.json_content['partitions']['training'][0]).load() + elif len(self.json_content['partitions']['validation']) != 0: + db = Database(database_dir=self.database_dir, + database_name=self.json_content['partitions']['validation'][0]).load() + else: + return {} + + # Get the architecture, keep relevant fields only + architecture = db.get_architecture() + for fields in architecture.values(): + for field in fields.copy(): + if field.split(' ')[0] in ['id', '_dt_']: + fields.remove(field) + db.close() + + return architecture + + def get_data_shapes(self) -> Dict[str, List[int]]: + """ + Get the shape of data Fields. + """ + + # Get a training or validation partition + if len(self.json_content['partitions']['training']) != 0: + db = Database(database_dir=self.database_dir, + database_name=self.json_content['partitions']['training'][0]).load() + elif len(self.json_content['partitions']['validation']) != 0: + db = Database(database_dir=self.database_dir, + database_name=self.json_content['partitions']['validation'][0]).load() + else: + return {} + + # Get the data shape for each numpy Field + shapes = {} + for table_name, fields in self.json_content['architecture'].items(): + if db.nb_lines(table_name=table_name) > 0: + data = db.get_line(table_name=table_name) + for field in fields: + if 'NUMPY' in field: + field_name = field.split(' ')[0] + shapes[f'{table_name}.{field_name}'] = data[field_name].shape + db.close() + + return shapes + + def get_nb_samples(self) -> None: + """ + Get the number of sample in each partition. + """ + + nb_samples = self.partitions[self.mode][-1].nb_lines(table_name='Training') + if len(self.json_content['nb_samples'][self.mode]) == self.partition_index[self.mode]: + self.json_content['nb_samples'][self.mode][-1] = nb_samples + else: + self.json_content['nb_samples'][self.mode].append(nb_samples) + + def update_json(self) -> None: + """ + Update the JSON info file with the current Database information. + """ + + # Overwrite json file + with open(join(self.database_dir, 'dataset.json'), 'w') as json_file: + json_dump(self.json_content, json_file, indent=3, cls=CustomJSONEncoder) + + ########################################################################################## + ########################################################################################## + # Database access and edition # + ########################################################################################## + ########################################################################################## + + def connect_handler(self, + handler: DatabaseHandler) -> None: + """ + Add and init a new DatabaseHandler to the list. + + :param handler: New DatabaseHandler. + """ + + handler.init(storing_partitions=self.get_partition_objects(), + exchange_db=self.exchange) + self.database_handlers.append(handler) + + def index_samples(self) -> None: + """ + Create a new indexing list of samples. Samples are identified by [partition_id, line_id]. + """ + + # Create the indices for each sample such as [partition_id, line_id] + for i, nb_sample in enumerate(self.json_content['nb_samples'][self.mode]): + partition_indices = empty((nb_sample, 2), dtype=int) + partition_indices[:, 0] = i + partition_indices[:, 1] = arange(1, nb_sample + 1) + self.sample_indices = concatenate((self.sample_indices, partition_indices)) + # Init current sample position + self.sample_id = 0 + # Shuffle the indices if required + if self.shuffle: + shuffle(self.sample_indices) + + def add_data(self, + data_lines: Optional[List[int]] = None) -> None: + """ + Manage new lines adding in the Database. + + :param data_lines: Indices of the newly added lines. + """ + + # 1. Update the json file + self.get_nb_samples() + self.update_json() + # 1.1. Init partitions information on the first sample + if self.first_add: + for handler in self.database_handlers: + handler.load() + self.json_content['partitions'] = self.partition_names + self.json_content['architecture'] = self.get_database_architecture() + self.json_content['data_shape'] = self.get_data_shapes() + self.update_json() + self.first_add = False + # 1.2. Update the normalization coefficients if required + if self.normalize and self.mode == 'training' and self.pipeline == 'training' and data_lines is not None: + self.json_content['normalization'] = self.update_normalization(data_lines=data_lines) + self.update_json() + + # 2. Check the size of the current partition + if self.max_file_size is not None: + if self.partitions[self.mode][-1].memory_size > self.max_file_size: + self.create_partition() + + def get_data(self, + batch_size: int) -> List[List[int]]: + """ + Select a batch of indices to read in the Database. + + :param batch_size: Number of sample in a single batch. + """ + + # 1. Check if dataset is loaded and if the current sample is not the last + if self.sample_id >= len(self.sample_indices): + self.index_samples() + + # 2. Update dataset index and get a batch of data + idx = self.sample_id + self.sample_id += batch_size + lines = self.sample_indices[idx:self.sample_id].tolist() + + # 3. Ensure the batch has the good size + if len(lines) < batch_size: + lines += self.get_data(batch_size=batch_size - len(lines)) + + return lines + + ########################################################################################## + ########################################################################################## + # Data normalization computation # + ########################################################################################## + ########################################################################################## + + @property + def normalization(self) -> Optional[Dict[str, List[float]]]: + """ + Get the normalization coefficients. + """ + + if self.json_content['normalization'] == {} or not self.normalize: + return None + return self.json_content['normalization'] + + def compute_normalization(self) -> Dict[str, List[float]]: + """ + Compute the mean and the standard deviation of all the training samples for each data field. + """ + + # 1. Get the fields to normalize + fields = [] + for field in self.json_content['data_shape']: + table_name, field_name = field.split('.') + fields += [field_name] if table_name == 'Training' else [] + normalization = {field: [0., 0.] for field in fields} + + # 2. Compute the mean of samples for each field + means = {field: [] for field in fields} + nb_samples = [] + # 2.1. Compute the mean for each partition + for partition in self.partitions['training']: + data_to_normalize = self.load_partitions_fields(partition=partition, fields=fields) + nb_samples.append(data_to_normalize['id'][-1]) + for field in fields: + data = array(data_to_normalize[field]) + means[field].append(data.mean()) + # 2.2. Compute the global mean + for field in fields: + normalization[field][0] = sum([(n / sum(nb_samples)) * m + for n, m in zip(nb_samples, means[field])]) + + # 3. Compute the standard deviation of samples for each field + stds = {field: [] for field in fields} + # 3.1. Compute the standard deviation for each partition + for partition in self.partitions['training']: + data_to_normalize = self.load_partitions_fields(partition=partition, + fields=fields) + for field in fields: + data = array(data_to_normalize[field]) + stds[field].append(mean(abs(data - normalization[field][0]) ** 2)) + # 3.2. Compute the global standard deviation + for field in fields: + normalization[field][1] = sqrt(sum([(n / sum(nb_samples)) * std + for n, std in zip(nb_samples, stds[field])])) + + return normalization + + def update_normalization(self, + data_lines: List[int]) -> Dict[str, List[float]]: + """ + Update the mean and the standard deviation of all the training samples with newly added samples for each data + field. + + :param data_lines: Indices of the newly added lines. + """ + + # 1. Get the previous normalization coefficients and number of samples + previous_normalization = self.normalization + if previous_normalization is None: + return self.compute_normalization() + new_normalization = previous_normalization.copy() + previous_nb_samples = self.total_nb_sample + self.total_nb_sample += len(data_lines) + + # 2. Compute the global mean of samples for each field + fields = list(previous_normalization.keys()) + data_to_normalize = self.partitions[self.mode][-1].get_lines(table_name='Training', + fields=fields, + lines_id=data_lines, + batched=True) + for field in fields: + data = array(data_to_normalize[field]) + m = (previous_nb_samples / self.total_nb_sample) * previous_normalization[field][0] + \ + (len(data_lines) / self.total_nb_sample) * data.mean() + new_normalization[field][0] = m + + # 3. Compute standard deviation of samples for each field + stds = {field: [] for field in fields} + nb_samples = [] + # 3.1. Recompute the standard deviation for each partition with the new mean value + for partition in self.partitions['training']: + data_to_normalize = self.load_partitions_fields(partition=partition, + fields=fields) + nb_samples.append(data_to_normalize['id'][-1]) + for field in fields: + data = array(data_to_normalize[field]) + stds[field].append(mean(abs(data - new_normalization[field][0]) ** 2)) + # 3.2. Compute the global standard deviation + for field in fields: + new_normalization[field][1] = sqrt(sum([(n / sum(nb_samples)) * std + for n, std in zip(nb_samples, stds[field])])) + + return new_normalization + + @staticmethod + def load_partitions_fields(partition: Database, + fields: List[str]) -> Dict[str, ndarray]: + """ + Load all the samples from a Field of a Table in the Database. + + :param partition: Database partition to load. + :param fields: Data Fields to get. + """ + + partition.load() + return partition.get_lines(table_name='Training', + fields=fields, + batched=True) + + ########################################################################################## + ########################################################################################## + # Manager behavior # + ########################################################################################## + ########################################################################################## + + def close(self): + """ + Launch the closing procedure of the DatabaseManager. + """ + + # Check non-empty last partition + self.remove_empty_partitions() + + # Compute final normalization if required + if self.normalize and self.pipeline == 'data_generation': + self.json_content['normalization'] = self.compute_normalization() + self.update_json() + + # Close Database partitions + for mode in self.modes: + for database in self.partitions[mode]: + database.close() + self.exchange.close(erase_file=True) + + def __str__(self): + + description = "\n" + description += f"# {self.name}\n" + description += f" Dataset Repository: {self.database_dir}\n" + size = f"No limits" if self.max_file_size is None else f"{self.max_file_size * 1e-9} Gb" + description += f" Partitions size: {size}\n" + return description diff --git a/src/Core/Manager/EnvironmentManager.py b/src/Core/Manager/EnvironmentManager.py new file mode 100644 index 00000000..1e85c39f --- /dev/null +++ b/src/Core/Manager/EnvironmentManager.py @@ -0,0 +1,256 @@ +from typing import Any, Optional, List +from asyncio import run as async_run +from os.path import join + +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig, TcpIpServer, BaseEnvironment +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler +from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer + + +class EnvironmentManager: + + def __init__(self, + environment_config: BaseEnvironmentConfig, + data_manager: Optional[Any] = None, + pipeline: str = '', + session: str = 'sessions/default', + produce_data: bool = True, + batch_size: int = 1): + """ + EnvironmentManager handle the communication with Environment(s). + + :param environment_config: Configuration object with the parameters of the Environment. + :param data_manager: DataManager that handles the EnvironmentManager. + :param pipeline: Type of the pipeline. + :param session: Path to the session repository. + :param produce_data: If True, this session will store data in the Database. + :param batch_size: Number of samples in a single batch. + """ + + self.name: str = self.__class__.__name__ + + # Session variables + self.data_manager: Any = data_manager + + # Data production variables + self.batch_size: int = batch_size + self.only_first_epoch: bool = environment_config.only_first_epoch + self.load_samples: bool = environment_config.load_samples + self.always_produce: bool = environment_config.always_produce + self.simulations_per_step: int = environment_config.simulations_per_step + self.max_wrong_samples_per_step: int = environment_config.max_wrong_samples_per_step + self.allow_prediction_requests: bool = pipeline != 'data_generation' + self.dataset_batch: Optional[List[List[int]]] = None + + # Create a Visualizer to provide the visualization Database + force_local = pipeline == 'prediction' + self.visualizer: Optional[VedoVisualizer] = None + if environment_config.visualizer is not None: + self.visualizer = environment_config.visualizer(database_dir=join(session, 'dataset'), + database_name='Visualization', + remote=environment_config.as_tcp_ip_client and not force_local, + record=produce_data) + + # Create a single Environment or a TcpIpServer + self.number_of_thread: int = 1 if force_local else environment_config.number_of_thread + self.server: Optional[TcpIpServer] = None + self.environment: Optional[BaseEnvironment] = None + # Create Server + if environment_config.as_tcp_ip_client and not force_local: + self.server = environment_config.create_server(environment_manager=self, + batch_size=batch_size, + visualization_db=None if self.visualizer is None else + self.visualizer.get_path()) + # Create Environment + else: + self.environment = environment_config.create_environment() + self.environment.environment_manager = self + self.data_manager.connect_handler(self.environment.get_database_handler()) + self.environment.create() + self.environment.init() + self.environment.init_database() + if self.visualizer is not None: + self.environment._create_visualization(visualization_db=self.visualizer.get_database()) + self.environment.init_visualization() + + # Define whether methods are used for environment or server + self.get_database_handler = self.__get_server_db_handler if self.server else self.__get_environment_db_handler + self.get_data = self.__get_data_from_server if self.server else self.__get_data_from_environment + self.dispatch_batch = self.__dispatch_batch_to_server if self.server else self.__dispatch_batch_to_environment + + # Init the Visualizer once Environments are initialized + if self.visualizer is not None: + if len(self.visualizer.get_database().get_tables()) == 1: + self.visualizer.get_database().load() + self.visualizer.init_visualizer() + + ########################################################################################## + ########################################################################################## + # DatabaseHandler management # + ########################################################################################## + ########################################################################################## + + def __get_server_db_handler(self) -> DatabaseHandler: + """ + Get the DatabaseHandler of the TcpIpServer. + """ + + return self.server.get_database_handler() + + def __get_environment_db_handler(self) -> DatabaseHandler: + """ + Get the DatabaseHandler of the Environment. + """ + + return self.environment.get_database_handler() + + ########################################################################################## + ########################################################################################## + # Data creation management # + ########################################################################################## + ########################################################################################## + + def __get_data_from_server(self, + animate: bool = True) -> List[List[int]]: + """ + Compute a batch of data from Environments requested through TcpIpServer. + + :param animate: If True, triggers an environment step. + """ + + return self.server.get_batch(animate) + + def __get_data_from_environment(self, + animate: bool = True, + save_data: bool = True, + request_prediction: bool = False) -> List[List[int]]: + """ + Compute a batch of data directly from Environment. + + :param animate: If True, triggers an environment step. + :param save_data: If True, data must be stored in the Database. + :param request_prediction: If True, a prediction request will be triggered. + """ + + # Produce batch while batch size is not complete + nb_sample = 0 + dataset_lines = [] + while nb_sample < self.batch_size: + + # 1. Send a sample from the Database if one is given + update_line = None + if self.dataset_batch is not None: + update_line = self.dataset_batch.pop(0) + self.environment._get_training_data(update_line) + + # 2. Run the defined number of steps + if animate: + for current_step in range(self.simulations_per_step): + # Sub-steps do not produce data + self.environment.compute_training_data = current_step == self.simulations_per_step - 1 + async_run(self.environment.step()) + + # 3. Add the produced sample index to the batch if the sample is validated + if self.environment.check_sample(): + nb_sample += 1 + # 3.1. The prediction Pipeline triggers a prediction request + if request_prediction: + self.environment._get_prediction() + # 3.2. Add the data to the Database + if save_data: + # Update the line if the sample was given by the database + if update_line is None: + new_line = self.environment._send_training_data() + dataset_lines.append(new_line) + # Create a new line otherwise + else: + self.environment._update_training_data(update_line) + dataset_lines.append(update_line) + # 3.3. Rest the data variables + self.environment._reset_training_data() + + return dataset_lines + + def __dispatch_batch_to_server(self, + data_lines: List[int], + animate: bool = True) -> None: + """ + Send samples from the Database to the Environments and get back the produced data. + + :param data_lines: Batch of indices of samples. + :param animate: If True, triggers an environment step. + """ + + # Define the batch to dispatch + self.server.set_dataset_batch(data_lines) + # Get data + self.__get_data_from_server(animate=animate) + + def __dispatch_batch_to_environment(self, + data_lines: List[int], + animate: bool = True, + save_data: bool = True, + request_prediction: bool = False) -> None: + """ + Send samples from the Database to the Environment and get back the produced data. + + :param data_lines: Batch of indices of samples. + :param animate: If True, triggers an environment step. + :param save_data: If True, data must be stored in the Database. + :param request_prediction: If True, a prediction request will be triggered. + """ + + # Define the batch to dispatch + self.dataset_batch = data_lines.copy() + # Get data + self.__get_data_from_environment(animate=animate, + save_data=save_data, + request_prediction=request_prediction) + + ########################################################################################## + ########################################################################################## + # Requests management # + ########################################################################################## + ########################################################################################## + + def update_visualizer(self, + instance: int) -> None: + """ + Update the Visualizer. + + :param instance: Index of the Environment render to update. + """ + + if self.visualizer is not None: + self.visualizer.render_instance(instance) + + ########################################################################################## + ########################################################################################## + # Manager behavior # + ########################################################################################## + ########################################################################################## + + def close(self) -> None: + """ + Launch the closing procedure of the EnvironmentManager. + """ + + # Server case + if self.server: + self.server.close() + + # Environment case + if self.environment: + self.environment.close() + + # Visualizer + if self.visualizer: + self.visualizer.close() + + def __str__(self) -> str: + + description = "\n" + description += f"# {self.name}\n" + description += f" Always create data: {self.only_first_epoch}\n" + description += f" Number of threads: {self.number_of_thread}\n" + return description diff --git a/src/Core/Manager/NetworkManager.py b/src/Core/Manager/NetworkManager.py new file mode 100644 index 00000000..d00d995a --- /dev/null +++ b/src/Core/Manager/NetworkManager.py @@ -0,0 +1,303 @@ +from typing import Any, Dict, Optional, List +from os import listdir +from os.path import join, isdir, isfile, sep +from numpy import ndarray, array + +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Utils.path import copy_dir, create_dir + + +class NetworkManager: + + def __init__(self, + network_config: BaseNetworkConfig, + pipeline: str = '', + session: str = 'sessions/default', + new_session: bool = True): + """ + NetworkManager deals with all the interactions with a neural network: predictions, saves, initialisation, + loading, optimization. + + :param network_config: Configuration object with the parameters of the Network. + :param pipeline: Type of the Pipeline. + :param session: Path to the session repository. + :param new_session: If True, the session is done in a new repository. + """ + + self.name: str = self.__class__.__name__ + + # Storage variables + self.database_handler: DatabaseHandler = DatabaseHandler() + self.batch: Optional[Any] = None + self.session: str = session + self.new_session: bool = new_session + self.network_dir: Optional[str] = None + self.network_template_name: str = session.split(sep)[-1] + '_network_{}' + self.saved_counter: int = 0 + self.save_each_epoch: bool = network_config.save_each_epoch + + # Init Network + self.network = network_config.create_network() + self.network.set_device() + if pipeline == 'training' and not network_config.training_stuff: + raise ValueError(f"[{self.name}] Training requires a loss and an optimizer in your NetworkConfig") + self.is_training: bool = pipeline == 'training' + + # Init Optimization + self.optimization = network_config.create_optimization() + if self.optimization.loss_class is not None: + self.optimization.set_loss() + + # Init DataTransformation + self.data_transformation = network_config.create_data_transformation() + + # Training configuration + if self.is_training: + self.network.set_train() + self.optimization.set_optimizer(self.network) + # Setting network directory + if new_session and network_config.network_dir is not None and isdir(network_config.network_dir): + self.network_dir = copy_dir(src_dir=network_config.network_dir, + dest_dir=session, + sub_folders='network') + self.load_network(which_network=network_config.which_network) + else: + self.network_dir = create_dir(session_dir=session, session_name='network') + + # Prediction configuration + else: + self.network.set_eval() + self.network_dir = join(session, 'network/') + self.load_network(which_network=network_config.which_network) + + ########################################################################################## + ########################################################################################## + # DatabaseHandler management # + ########################################################################################## + ########################################################################################## + + def get_database_handler(self) -> DatabaseHandler: + """ + Get the DatabaseHandler of the NetworkManager. + """ + + return self.database_handler + + def link_clients(self, + nb_clients: Optional[int] = None) -> None: + """ + Update the data Exchange Database with a new line for each TcpIpClient. + + :param nb_clients: Number of Clients to connect. + """ + + if nb_clients is not None: + # Create the network fields in the Exchange Database + fields = [(field_name, ndarray) for field_name in self.network.net_fields + self.network.pred_fields] + self.database_handler.create_fields(table_name='Exchange', fields=fields) + # Add an empty line for each Client + for _ in range(nb_clients): + self.database_handler.add_data(table_name='Exchange', data={}) + + ########################################################################################## + ########################################################################################## + # Network parameters management # + ########################################################################################## + ########################################################################################## + + def load_network(self, + which_network: int = -1) -> None: + """ + Load an existing set of parameters of the Network. + + :param which_network: If several sets of parameters were saved, specify which one to load. + """ + + # 1. Get the list of all sets of saved parameters + networks_list = [join(self.network_dir, f) for f in listdir(self.network_dir) if + isfile(join(self.network_dir, f)) and f.__contains__('network_')] + networks_list = sorted(networks_list) + last_saved_network = [join(self.network_dir, f) for f in listdir(self.network_dir) if + isfile(join(self.network_dir, f)) and f.__contains__('network.')] + networks_list = networks_list + last_saved_network + + # 2. Check the Network to access + if len(networks_list) == 0: + raise FileNotFoundError(f"[{self.name}]: There is no network in {self.network_dir}.") + elif len(networks_list) == 1: + which_network = 0 + elif which_network > len(networks_list): + print(f"[{self.name}] The network 'network_{self.saved_counter} doesn't exist, loading the most trained " + f"by default.") + which_network = -1 + + # 3. Load the set of parameters + print(f"[{self.name}]: Loading network from {networks_list[which_network]}.") + self.network.load_parameters(networks_list[which_network]) + + def save_network(self, + last_save: bool = False) -> None: + """ + Save the current set of parameters of the Network. + + :param last_save: If True, the Network training is done then give a special denomination. + """ + + # Final session saving + if last_save: + path = join(self.network_dir, 'network') + print(f"[{self.name}] Saving final network at {self.network_dir}.") + self.network.save_parameters(path) + + # Intermediate states saving + elif self.save_each_epoch: + path = self.network_dir + self.network_template_name.format(self.saved_counter) + self.saved_counter += 1 + print(f"[{self.name}] Saving intermediate network at {path}.") + self.network.save_parameters(path) + + ########################################################################################## + ########################################################################################## + # Network optimization and prediction # + ########################################################################################## + ########################################################################################## + + def compute_prediction_and_loss(self, + optimize: bool, + data_lines: List[List[int]], + normalization: Optional[Dict[str, List[float]]] = None) -> Dict[str, float]: + """ + Make a prediction with the data passed as argument, optimize or not the network + + :param optimize: If true, run a backward propagation. + :param data_lines: Batch of indices of samples in the Database. + :param normalization: Normalization coefficients. + :return: The prediction and the associated loss value + """ + + # 1. Define Network and Optimization batches + batches = {} + normalization = {} if normalization is None else normalization + for side, fields in zip(['net', 'opt'], [self.network.net_fields, self.network.opt_fields]): + # Get the batch from the Database + batch = self.database_handler.get_lines(table_name='Training', + fields=fields, + lines_id=data_lines) + # Apply normalization and convert to tensor + for field in batch.keys(): + batch[field] = array(batch[field]) + if field in normalization: + batch[field] = self.normalize_data(data=batch[field], + normalization=normalization[field]) + batch[field] = self.network.numpy_to_tensor(data=batch[field], + grad=optimize) + batches[side] = batch + data_net, data_opt = batches.values() + + # 2. Compute prediction + data_net = self.data_transformation.transform_before_prediction(data_net) + data_pred = self.network.predict(data_net) + + # 3. Compute loss + data_pred, data_opt = self.data_transformation.transform_before_loss(data_pred, data_opt) + data_loss = self.optimization.compute_loss(data_pred, data_opt) + + # 4. Optimize network if training + if optimize: + self.optimization.optimize() + + return data_loss + + def compute_online_prediction(self, + instance_id: int, + normalization: Optional[Dict[str, List[float]]] = None) -> None: + """ + Make a prediction with the data passed as argument. + + :param instance_id: Index of the Environment instance to provide a prediction. + :param normalization: Normalization coefficients. + """ + + # Get Network data + normalization = {} if normalization is None else normalization + sample = self.database_handler.get_line(table_name='Exchange', + fields=self.network.net_fields, + line_id=instance_id) + del sample['id'] + + # Apply normalization and convert to tensor + for field in sample.keys(): + sample[field] = array([sample[field]]) + if field in normalization.keys(): + sample[field] = self.normalize_data(data=sample[field], + normalization=normalization[field]) + sample[field] = self.network.numpy_to_tensor(data=sample[field]) + + # Compute prediction + data_net = self.data_transformation.transform_before_prediction(sample) + data_pred = self.network.predict(data_net) + data_pred, _ = self.data_transformation.transform_before_loss(data_pred) + data_pred = self.data_transformation.transform_before_apply(data_pred) + + # Return the prediction + for field in data_pred.keys(): + data_pred[field] = self.network.tensor_to_numpy(data=data_pred[field][0]) + if self.network.pred_norm_fields[field] in normalization.keys(): + data_pred[field] = self.normalize_data(data=data_pred[field], + normalization=normalization[self.network.pred_norm_fields[field]], + reverse=True) + data_pred[field].reshape(-1) + self.database_handler.update(table_name='Exchange', + data=data_pred, + line_id=instance_id) + + @classmethod + def normalize_data(cls, + data: ndarray, + normalization: List[float], + reverse: bool = False) -> ndarray: + """ + Apply or unapply normalization following current standard score. + + :param data: Data to normalize. + :param normalization: Normalization coefficients. + :param reverse: If True, apply normalization; if False, unapply normalization. + :return: Data with applied or misapplied normalization. + """ + + # Unapply normalization + if reverse: + return (data * normalization[1]) + normalization[0] + + # Apply normalization + return (data - normalization[0]) / normalization[1] + + ########################################################################################## + ########################################################################################## + # Manager behavior # + ########################################################################################## + ########################################################################################## + + def close(self) -> None: + """ + Launch the closing procedure of the NetworkManager. + """ + + if self.is_training: + self.save_network(last_save=True) + del self.network + + def __str__(self) -> str: + + description = "\n" + description += f"# {self.__class__.__name__}\n" + description += f" Network Directory: {self.network_dir}\n" + description += f" Save each Epoch: {self.save_each_epoch}\n" + description += f" Managed objects: Network: {self.network.__class__.__name__}\n" + description += f" Optimization: {self.optimization.__class__.__name__}\n" + description += f" Data Transformation: {self.data_transformation.__class__.__name__}\n" + description += str(self.network) + description += str(self.optimization) + description += str(self.data_transformation) + return description diff --git a/src/Manager/StatsManager.py b/src/Core/Manager/StatsManager.py similarity index 83% rename from src/Manager/StatsManager.py rename to src/Core/Manager/StatsManager.py index c79e7bdc..060c986f 100644 --- a/src/Manager/StatsManager.py +++ b/src/Core/Manager/StatsManager.py @@ -1,5 +1,5 @@ -from typing import Dict, Union, Any, Iterable, Optional -from tensorboardX import SummaryWriter +from typing import Dict, Any, Iterable, Optional +from torch.utils.tensorboard import SummaryWriter from tensorboard import program from webbrowser import open as w_open from numpy import full, inf, array, ndarray, append, concatenate @@ -17,33 +17,29 @@ def generate_default_material(): class StatsManager: - """ - | Record all given values using the tensorboard framework. Open a tab in the navigator to inspect these values - during the training. - - :param str log_dir: Path of the created directory - :param Manager manager: Manager that handles the StatsManager - :param bool keep_losses: If True Allow saving loss to .csv file - """ def __init__(self, session: str, - manager: Any = None, keep_losses: bool = False): + """ + StatsManager records all the given values using the Tensorboard framework. + Open a tab in the navigator to inspect these values during the training. + + :param session: Path to the session repository. + :param keep_losses: If True, allow saving loss to .csv file. + """ self.name: str = self.__class__.__name__ # Init writer - self.manager = manager self.log_dir: str = join(session, 'stats/') self.writer: SummaryWriter = SummaryWriter(self.log_dir) # Open Tensorboard - if not self.manager.pipeline.debug: - tb = program.TensorBoard() - tb.configure(argv=[None, '--logdir', self.log_dir]) - url = tb.launch() - w_open(url) + tb = program.TensorBoard() + tb.configure(argv=[None, '--logdir', self.log_dir]) + url = tb.launch() + w_open(url) # Values self.mean: ndarray = full(4, inf) # Contains in the 1st dimension the mean, and 2nd the variance of the mean @@ -51,18 +47,9 @@ def __init__(self, self.keep_losses: bool = keep_losses self.tag_dict: Dict[str, int] = {} - def get_manager(self) -> Any: - """ - | Return the Manager of the StatsManager. - - :return: Manager that handles the StatsManager - """ - - return self.manager - def add_train_batch_loss(self, value: float, count: int) -> None: """ - | Add batch loss to tensorboard framework. Also compute mean and variance. + Add batch loss to tensorboard framework. Also compute mean and variance. :param float value: Value to store :param int count: ID of the value @@ -78,7 +65,7 @@ def add_train_batch_loss(self, value: float, count: int) -> None: def add_train_epoch_loss(self, value: float, count: int) -> None: """ - | Add epoch loss to tensorboard framework. Also compute mean and variance. + Add epoch loss to tensorboard framework. Also compute mean and variance. :param float value: Value to store :param int count: ID of the value @@ -92,7 +79,7 @@ def add_train_epoch_loss(self, value: float, count: int) -> None: def add_train_test_batch_loss(self, train_value: float, test_value: float, count: int) -> None: """ - | Add train and test batch loss to tensorboard framework. + Add train and test batch loss to tensorboard framework. :param float train_value: Value of the training batch :param float test_value: Value of the testing batch @@ -106,7 +93,7 @@ def add_train_test_batch_loss(self, train_value: float, test_value: float, count def add_values_multi_plot(self, graph_name: str, tags: Iterable, values: Iterable, counts: Iterable) -> None: """ - | Plot multiples value on the same graph + Plot multiples value on the same graph :param str graph_name: Name of the graph :param Iterable tags: Iterable containing the names of the values @@ -119,7 +106,7 @@ def add_values_multi_plot(self, graph_name: str, tags: Iterable, values: Iterabl def add_test_loss(self, value: float, count: int) -> None: """ - | Add test loss to tensorboard framework. Also compute mean and variance. + Add test loss to tensorboard framework. Also compute mean and variance. :param float value: Value to store :param int count: ID of the value @@ -133,7 +120,7 @@ def add_test_loss(self, value: float, count: int) -> None: def add_test_loss_OOB(self, value: float, count: int) -> None: """ - | Add out of bound test loss to tensorboard framework. Also compute mean and variance. + Add out of bound test loss to tensorboard framework. Also compute mean and variance. :param float value: Value to store :param int count: ID of the value @@ -147,7 +134,7 @@ def add_test_loss_OOB(self, value: float, count: int) -> None: def add_custom_scalar(self, tag: str, value: float, count: int) -> None: """ - | Add a custom scalar to tensorboard framework. + Add a custom scalar to tensorboard framework. :param str tag: Graph name :param float value: Value to store @@ -158,7 +145,7 @@ def add_custom_scalar(self, tag: str, value: float, count: int) -> None: def add_custom_scalar_full(self, tag: str, value: float, count: int) -> None: """ - | Add a custom scalar to tensorboard framework. Also compute mean and variance. + Add a custom scalar to tensorboard framework. Also compute mean and variance. :param str tag: Graph name :param float value: Value to store @@ -177,7 +164,7 @@ def add_custom_scalar_full(self, tag: str, value: float, count: int) -> None: def update_mean_get_var(self, index: int, value: float, count: int) -> Optional[ndarray]: """ - | Update mean and return the variance of the selected value + Update mean and return the variance of the selected value :param float value: Value to add in the computation of the mean :param int index: Target that is updated by the value @@ -227,7 +214,7 @@ def add_3D_mesh(self, tag: str, vertices: ndarray, colors: Optional[ndarray] = N faces: Optional[ndarray] = None, b_n_3: bool = False, config_dict: Optional[Dict[Any, Any]] = None) -> None: """ - | Add 3D Mesh cloud to tensorboard framework. + Add 3D Mesh cloud to tensorboard framework. :param str tag: Data identifier :param ndarray vertices: List of the 3D coordinates of vertices. @@ -255,7 +242,7 @@ def add_3D_mesh(self, tag: str, vertices: ndarray, colors: Optional[ndarray] = N def add_network_weight_grad(self, network: Any, count: int, save_weights: bool = False, save_gradients: bool = True) -> None: """ - | Add network weights and gradiant if specified to tensorboard framework. + Add network weights and gradiant if specified to tensorboard framework. :param BaseNetwork network: Network you want to display :param int count: ID of the sample @@ -272,18 +259,13 @@ def add_network_weight_grad(self, network: Any, count: int, save_weights: bool = def close(self) -> None: """ - | Closing procedure - - :return: + Launch the closing procedure of the StatsManager. """ self.writer.close() del self.train_loss - def __str__(self) -> str: - """ - :return: A string containing valuable information about the StatsManager - """ + def __str__(self): description = "\n" description += f"# {self.name}\n" diff --git a/src/Dataset/__init__.py b/src/Core/Manager/__init__.py similarity index 62% rename from src/Dataset/__init__.py rename to src/Core/Manager/__init__.py index 6501d21a..d99a2553 100644 --- a/src/Dataset/__init__.py +++ b/src/Core/Manager/__init__.py @@ -5,6 +5,6 @@ exceptions = ['__init__.py', '__pycache__'] modules = [module for module in listdir(package) if module.endswith('.py') and module not in exceptions] __all__ = [] -for module in sorted(modules): - exec(f"from DeepPhysX.Core.Dataset.{module[:-3]} import {module[:-3]}") - __all__.append(module[:-3]) +# for module in sorted(modules): +# exec(f"from DeepPhysX.Core.Manager.{module[:-3]} import {module[:-3]}") +# __all__.append(module[:-3]) diff --git a/src/Core/Network/BaseNetwork.py b/src/Core/Network/BaseNetwork.py new file mode 100644 index 00000000..32830ded --- /dev/null +++ b/src/Core/Network/BaseNetwork.py @@ -0,0 +1,139 @@ +from typing import Any, Dict +from numpy import ndarray +from collections import namedtuple + + +class BaseNetwork: + + def __init__(self, + config: namedtuple): + """ + BaseNetwork computes predictions from input data according to actual set of weights. + + :param config: Set of BaseNetwork parameters. + """ + + # Config + self.device = None + self.config = config + + # Data fields + self.net_fields = ['input'] + self.opt_fields = ['ground_truth'] + self.pred_fields = ['prediction'] + self.pred_norm_fields = {'prediction': 'ground_truth'} + + def predict(self, + data_net: Dict[str, Any]) -> Dict[str, Any]: + """ + Compute a forward pass of the Network. + + :param data_net: Data used by the Network. + :return: Data produced by the Network. + """ + + return {'prediction': self.forward(data_net['input'])} + + def forward(self, + input_data: Any) -> Any: + """ + Compute a forward pass of the Network. + + :param input_data: Input tensor. + :return: Network prediction. + """ + + raise NotImplementedError + + def set_train(self) -> None: + """ + Set the Network in training mode (compute gradient). + """ + + raise NotImplementedError + + def set_eval(self) -> None: + """ + Set the Network in prediction mode (does not compute gradient). + """ + + raise NotImplementedError + + def set_device(self) -> None: + """ + Set computer device on which Network's parameters will be stored and tensors will be computed. + """ + + raise NotImplementedError + + def load_parameters(self, + path: str) -> None: + """ + Load network parameter from path. + + :param path: Path to Network parameters to load. + """ + + raise NotImplementedError + + def get_parameters(self) -> Dict[str, Any]: + """ + Return the current state of Network parameters. + + :return: Network parameters. + """ + + raise NotImplementedError + + def save_parameters(self, + path: str) -> None: + """ + Saves the network parameters to the path location. + + :param path: Path where to save the parameters. + """ + + raise NotImplementedError + + def nb_parameters(self) -> int: + """ + Return the number of parameters of the network. + + :return: Number of parameters. + """ + + raise NotImplementedError + + def numpy_to_tensor(self, + data: ndarray, + grad: bool = True) -> Any: + """ + Transform and cast data from numpy to the desired tensor type. + + :param data: Array data to convert. + :param grad: If True, gradient will record operations on this tensor. + :return: Converted tensor. + """ + + return data.astype(self.config.data_type) + + def tensor_to_numpy(self, + data: Any) -> ndarray: + """ + Transform and cast data from tensor type to numpy. + + :param data: Tensor to convert. + :return: Converted array. + """ + + return data.astype(self.config.data_type) + + def __str__(self) -> str: + + description = "\n" + description += f" {self.__class__.__name__}\n" + description += f" Name: {self.config.network_name}\n" + description += f" Type: {self.config.network_type}\n" + description += f" Number of parameters: {self.nb_parameters()}\n" + description += f" Estimated size: {self.nb_parameters() * 32 * 1.25e-10} Go\n" + return description diff --git a/src/Core/Network/BaseNetworkConfig.py b/src/Core/Network/BaseNetworkConfig.py new file mode 100644 index 00000000..efde937a --- /dev/null +++ b/src/Core/Network/BaseNetworkConfig.py @@ -0,0 +1,155 @@ +from typing import Any, Optional, Type +from os.path import isdir +from numpy import typeDict + +from DeepPhysX.Core.Network.BaseNetwork import BaseNetwork +from DeepPhysX.Core.Network.BaseOptimization import BaseOptimization +from DeepPhysX.Core.Network.BaseTransformation import BaseTransformation +from DeepPhysX.Core.Utils.configs import make_config, namedtuple + + +class BaseNetworkConfig: + + def __init__(self, + network_class: Type[BaseNetwork] = BaseNetwork, + optimization_class: Type[BaseOptimization] = BaseOptimization, + data_transformation_class: Type[BaseTransformation] = BaseTransformation, + network_dir: Optional[str] = None, + network_name: str = 'Network', + network_type: str = 'BaseNetwork', + which_network: int = -1, + save_each_epoch: bool = False, + data_type: str = 'float32', + lr: Optional[float] = None, + require_training_stuff: bool = True, + loss: Optional[Any] = None, + optimizer: Optional[Any] = None): + """ + BaseNetworkConfig is a configuration class to parameterize and create BaseNetwork, BaseOptimization and + BaseTransformation for the NetworkManager. + + :param network_class: BaseNetwork class from which an instance will be created. + :param optimization_class: BaseOptimization class from which an instance will be created. + :param data_transformation_class: BaseTransformation class from which an instance will be created. + :param network_dir: Name of an existing network repository. + :param network_name: Name of the network. + :param network_type: Type of the network. + :param which_network: If several networks in network_dir, load the specified one. + :param save_each_epoch: If True, network state will be saved at each epoch end; if False, network state + will be saved at the end of the training. + :param data_type: Type of the training data. + :param lr: Learning rate. + :param require_training_stuff: If specified, loss and optimizer class can be not necessary for training. + :param loss: Loss class. + :param optimizer: Network's parameters optimizer class. + """ + + self.name = self.__class__.__name__ + + # Check network_dir type and existence + if network_dir is not None: + if type(network_dir) != str: + raise TypeError( + f"[{self.__class__.__name__}] Wrong 'network_dir' type: str required, get {type(network_dir)}") + if not isdir(network_dir): + raise ValueError(f"[{self.__class__.__name__}] Given 'network_dir' does not exists: {network_dir}") + # Check network_name type + if type(network_name) != str: + raise TypeError( + f"[{self.__class__.__name__}] Wrong 'network_name' type: str required, get {type(network_name)}") + # Check network_tpe type + if type(network_type) != str: + raise TypeError( + f"[{self.__class__.__name__}] Wrong 'network_type' type: str required, get {type(network_type)}") + # Check which_network type and value + if type(which_network) != int: + raise TypeError( + f"[{self.__class__.__name__}] Wrong 'which_network' type: int required, get {type(which_network)}") + # Check save_each_epoch type + if type(save_each_epoch) != bool: + raise TypeError( + f"[{self.__class__.__name__}] Wrong 'save each epoch' type: bool required, get {type(save_each_epoch)}") + # Check data type + if data_type not in typeDict: + raise ValueError( + f"[{self.__class__.__name__}] The following data type is not a numpy type: {data_type}") + + # BaseNetwork parameterization + self.network_class: Type[BaseNetwork] = network_class + self.network_config: namedtuple = make_config(configuration_object=self, + configuration_name='network_config', + network_name=network_name, + network_type=network_type, + data_type=data_type) + + # BaseOptimization parameterization + self.optimization_class: Type[BaseOptimization] = optimization_class + self.optimization_config: namedtuple = make_config(configuration_object=self, + configuration_name='optimization_config', + loss=loss, + lr=lr, + optimizer=optimizer) + self.training_stuff: bool = (loss is not None) and (optimizer is not None) or (not require_training_stuff) + + # NetworkManager parameterization + self.data_transformation_class: Type[BaseTransformation] = data_transformation_class + self.data_transformation_config: namedtuple = make_config(configuration_object=self, + configuration_name='data_transformation_config') + + # NetworkManager parameterization + self.network_dir: str = network_dir + self.which_network: int = which_network + self.save_each_epoch: bool = save_each_epoch and self.training_stuff + + def create_network(self) -> BaseNetwork: + """ + Create an instance of network_class with given parameters. + + :return: BaseNetwork object from network_class and its parameters. + """ + + # Create instance + network = self.network_class(config=self.network_config) + if not isinstance(network, BaseNetwork): + raise TypeError(f"[{self.name}] The given 'network_class'={self.network_class} must be a BaseNetwork.") + return network + + def create_optimization(self) -> BaseOptimization: + """ + Create an instance of optimization_class with given parameters. + + :return: BaseOptimization object from optimization_class and its parameters. + """ + + # Create instance + optimization = self.optimization_class(config=self.optimization_config) + if not isinstance(optimization, BaseOptimization): + raise TypeError(f"[{self.name}] The given 'optimization_class'={self.optimization_class} must be a " + f"BaseOptimization.") + return optimization + + def create_data_transformation(self) -> BaseTransformation: + """ + Create an instance of data_transformation_class with given parameters. + + :return: BaseTransformation object from data_transformation_class and its parameters. + """ + + # Create instance + data_transformation = self.data_transformation_class(config=self.data_transformation_config) + if not isinstance(data_transformation, BaseTransformation): + raise TypeError(f"[{self.name}] The given 'data_transformation_class'={self.data_transformation_class} " + f"must be a BaseTransformation.") + return data_transformation + + def __str__(self): + + description = "\n" + description += f"{self.__class__.__name__}\n" + description += f" Network class: {self.network_class.__name__}\n" + description += f" Optimization class: {self.optimization_class.__name__}\n" + description += f" Training materials: {self.training_stuff}\n" + description += f" Network directory: {self.network_dir}\n" + description += f" Which network: {self.which_network}\n" + description += f" Save each epoch: {self.save_each_epoch}\n" + return description diff --git a/src/Core/Network/BaseOptimization.py b/src/Core/Network/BaseOptimization.py new file mode 100644 index 00000000..03116fac --- /dev/null +++ b/src/Core/Network/BaseOptimization.py @@ -0,0 +1,84 @@ +from typing import Dict, Any +from collections import namedtuple + +from DeepPhysX.Core.Network.BaseNetwork import BaseNetwork + + +class BaseOptimization: + + def __init__(self, config: namedtuple): + """ + BaseOptimization computes loss between prediction and target and optimizes the Network parameters. + + :param config: Set of BaseOptimization parameters. + """ + + self.manager: Any = None + + # Loss + self.loss_class = config.loss + self.loss = None + self.loss_value = 0. + + # Optimizer + self.optimizer_class = config.optimizer + self.optimizer = None + self.lr = config.lr + + def set_loss(self) -> None: + """ + Initialize the loss function. + """ + + raise NotImplementedError + + def compute_loss(self, + data_pred: Dict[str, Any], + data_opt: Dict[str, Any]) -> Dict[str, Any]: + """ + Compute loss from prediction / ground truth. + + :param data_pred: Tensor produced by the forward pass of the Network. + :param data_opt: Ground truth tensor to be compared with prediction. + :return: Loss value. + """ + + raise NotImplementedError + + def transform_loss(self, + data_opt: Dict[str, Any]) -> Dict[str, float]: + """ + Apply a transformation on the loss value using the potential additional data. + + :param data_opt: Additional data sent as dict to compute loss value + :return: Transformed loss value. + """ + + raise NotImplementedError + + def set_optimizer(self, + net: BaseNetwork) -> None: + """ + Define an optimization process. + + :param net: Network whose parameters will be optimized. + """ + + raise NotImplementedError + + def optimize(self) -> None: + """ + Run an optimization step. + """ + + raise NotImplementedError + + def __str__(self): + + description = "\n" + description += f" {self.__class__.__name__}\n" + description += f" Loss class: {self.loss_class.__name__}\n" if self.loss_class else f" Loss class: None\n" + description += f" Optimizer class: {self.optimizer_class.__name__}\n" if self.optimizer_class else \ + f" Optimizer class: None\n" + description += f" Learning rate: {self.lr}\n" + return description diff --git a/src/Core/Network/BaseTransformation.py b/src/Core/Network/BaseTransformation.py new file mode 100644 index 00000000..f2956770 --- /dev/null +++ b/src/Core/Network/BaseTransformation.py @@ -0,0 +1,74 @@ +from typing import Callable, Any, Optional, Tuple, Dict +from collections import namedtuple + + +class BaseTransformation: + + def __init__(self, config: namedtuple): + """ + BaseTransformation manages data operations before and after network predictions. + + :param config: Set of BaseTransformation parameters. + """ + + self.name = self.__class__.__name__ + + self.config: Any = config + self.data_type = any + + @staticmethod + def check_type(func: Callable[[Any, Any], Any]): + + def inner(self, *args): + for data in [a for a in args if a is not None]: + for value in data.values(): + if value is not None and type(value) != self.data_type: + raise TypeError(f"[{self.name}] Wrong data type: {self.data_type} required, get {type(value)}") + return func(self, *args) + + return inner + + def transform_before_prediction(self, + data_net: Dict[str, Any]) -> Dict[str, Any]: + """ + Apply data operations before network's prediction. + + :param data_net: Data used by the Network. + :return: Transformed data_net. + """ + + return data_net + + def transform_before_loss(self, + data_pred: Dict[str, Any], + data_opt: Optional[Dict[str, Any]] = None) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]: + """ + Apply data operations between network's prediction and loss computation. + + :param data_pred: Data produced by the Network. + :param data_opt: Data used by the Optimizer. + :return: Transformed data_pred, data_opt. + """ + + return data_pred, data_opt + + def transform_before_apply(self, + data_pred: Dict[str, Any]) -> Dict[str, Any]: + """ + Apply data operations between loss computation and prediction apply in environment. + + :param data_pred: Data produced by the Network. + :return: Transformed data_pred. + """ + + return data_pred + + def __str__(self): + + description = "\n" + description += f" {self.__class__.__name__}\n" + description += f" Data type: {self.data_type}\n" + description += f" Transformation before prediction: Identity\n" + description += f" Transformation before loss: Identity\n" + description += f" Transformation before apply: Identity\n" + return description diff --git a/src/Network/__init__.py b/src/Core/Network/__init__.py similarity index 100% rename from src/Network/__init__.py rename to src/Core/Network/__init__.py diff --git a/src/Core/Pipelines/BaseDataGeneration.py b/src/Core/Pipelines/BaseDataGeneration.py new file mode 100644 index 00000000..fb2751b1 --- /dev/null +++ b/src/Core/Pipelines/BaseDataGeneration.py @@ -0,0 +1,138 @@ +from typing import Optional +from os.path import join, sep, exists +from vedo import ProgressBar + +from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline +from DeepPhysX.Core.Manager.DataManager import DataManager +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Utils.path import get_first_caller, create_dir + + +class BaseDataGeneration(BasePipeline): + + def __init__(self, + environment_config: BaseEnvironmentConfig, + database_config: Optional[BaseDatabaseConfig] = None, + session_dir: str = 'sessions', + session_name: str = 'data_generation', + new_session: bool = True, + batch_nb: int = 0, + batch_size: int = 0): + """ + BaseDataGeneration implements the main loop that only produces and stores data (no Network training). + + :param database_config: Configuration object with the parameters of the Database. + :param environment_config: Configuration object with the parameters of the Environment. + :param session_dir: Relative path to the directory which contains sessions repositories. + :param session_name: Name of the new the session repository. + :param new_session: If True, a new repository will be created for this session. + :param batch_nb: Number of batches to produce. + :param batch_size: Number of samples in a single batch. + """ + + BasePipeline.__init__(self, + database_config=database_config, + environment_config=environment_config, + session_dir=session_dir, + session_name=session_name, + new_session=new_session, + pipeline='data_generation') + + # Define the session repository + root = get_first_caller() + session_dir = join(root, session_dir) + + # Create a new session if required + if not new_session: + new_session = not exists(join(session_dir, session_name)) + if new_session: + session_name = create_dir(session_dir=session_dir, + session_name=session_name).split(sep)[-1] + self.session = join(session_dir, session_name) + + # Create a DataManager + self.data_manager = DataManager(pipeline=self, + database_config=database_config, + environment_config=environment_config, + session=join(session_dir, session_name), + new_session=new_session, + produce_data=True, + batch_size=batch_size) + + # Data generation variables + self.batch_nb: int = batch_nb + self.batch_id: int = 0 + self.batch_size = batch_size + self.progress_bar = ProgressBar(start=0, stop=self.batch_nb, c='orange', title="Data Generation") + + def execute(self) -> None: + """ + Launch the data generation Pipeline. + Each event is already implemented for a basic Pipeline but can also be rewritten via inheritance to describe a + more complex Pipeline. + """ + + self.data_generation_begin() + while self.batch_condition(): + self.batch_begin() + self.batch_produce() + self.batch_count() + self.batch_end() + self.data_generation_end() + + def data_generation_begin(self) -> None: + """ + Called once at the beginning of the data generation Pipeline. + """ + + pass + + def batch_condition(self) -> bool: + """ + Check the batch number condition. + """ + + return self.batch_id < self.batch_nb + + def batch_begin(self) -> None: + """ + Called once at the beginning of a batch production. + """ + + pass + + def batch_produce(self) -> None: + """ + Trigger the data production. + """ + + self.data_manager.get_data() + + def batch_count(self) -> None: + """ + Increment the batch counter. + """ + + self.batch_id += 1 + + def batch_end(self) -> None: + """ + Called once at the end of a batch production. + """ + + self.progress_bar.print(counts=self.batch_id) + + def data_generation_end(self) -> None: + """ + Called once at the beginning of the data generation Pipeline. + """ + + self.data_manager.close() + + def __str__(self): + + description = BasePipeline.__str__(self) + description += f" Number of batches: {self.batch_nb}\n" + description += f" Number of sample per batch: {self.batch_size}\n" + return description diff --git a/src/Core/Pipelines/BasePipeline.py b/src/Core/Pipelines/BasePipeline.py new file mode 100644 index 00000000..31103f01 --- /dev/null +++ b/src/Core/Pipelines/BasePipeline.py @@ -0,0 +1,146 @@ +from typing import Optional, Any, List, Union +from os.path import join + +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Manager.DataManager import DataManager +from DeepPhysX.Core.Manager.StatsManager import StatsManager +from DeepPhysX.Core.Manager.NetworkManager import NetworkManager +from DeepPhysX.Core.Manager.DatabaseManager import DatabaseManager +from DeepPhysX.Core.Manager.EnvironmentManager import EnvironmentManager + + +class BasePipeline: + + def __init__(self, + network_config: Optional[BaseNetworkConfig] = None, + database_config: Optional[BaseDatabaseConfig] = None, + environment_config: Optional[BaseEnvironmentConfig] = None, + session_dir: str = 'sessions', + session_name: str = 'default', + new_session: bool = True, + pipeline: str = ''): + """ + Pipelines implement the main loop that defines data flow through components (Environment, Dataset, Network...). + + :param network_config: Configuration object with the parameters of the Network. + :param database_config: Configuration object with the parameters of the Database. + :param environment_config: Configuration object with the parameters of the Environment. + :param session_dir: Relative path to the directory which contains sessions repositories. + :param session_name: Name of the new the session repository. + :param new_session: If True, a new repository will be created for this session. + :param pipeline: Name of the Pipeline. + """ + + self.name: str = self.__class__.__name__ + + # Check the configurations + if network_config is not None and not isinstance(network_config, BaseNetworkConfig): + raise TypeError(f"[{self.name}] The Network configuration must be a BaseNetworkConfig object.") + if database_config is not None and not isinstance(database_config, BaseDatabaseConfig): + raise TypeError(f"[{self.name}] The Dataset configuration must be a BaseDatabaseConfig object.") + if environment_config is not None and not isinstance(environment_config, BaseEnvironmentConfig): + raise TypeError(f"[{self.name}] The Environment configuration must be a BaseEnvironmentConfig object.") + + # Check the session path variables + if type(session_dir) != str: + raise TypeError(f"[{self.name}] The given 'session_dir'={session_dir} must be a str.") + elif len(session_dir) == 0: + session_dir = 'sessions' + if type(session_name) != str: + raise TypeError(f"[{self.name}] The given 'session=name'={session_name} must be a str.") + elif len(session_name) == 0: + session_name = pipeline + + # Configuration variables + self.database_config: BaseDatabaseConfig = database_config + self.network_config: BaseNetworkConfig = network_config + self.environment_config: BaseEnvironmentConfig = environment_config + + # Session variables + self.session = join(session_dir, session_name) + self.new_session = new_session + self.type = pipeline + + def execute(self): + """ + Launch the Pipeline. + """ + + raise NotImplemented + + def __get_any_manager(self, + manager_names: Union[str, List[str]]) -> Optional[Any]: + """ + Return the desired Manager associated with the Pipeline if it exists. + + :param manager_names: Name of the desired Manager or order of access to this desired Manager. + :return: The desired Manager associated with the Pipeline. + """ + + # Direct access to manager + if type(manager_names) == str: + return getattr(self, manager_names) if hasattr(self, manager_names) else None + + # Intermediates to access manager + accessed_manager = self + for next_manager in manager_names: + if hasattr(accessed_manager, next_manager): + accessed_manager = getattr(accessed_manager, next_manager) + else: + return None + return accessed_manager + + def get_network_manager(self) -> Optional[NetworkManager]: + """ + Return the NetworkManager associated with the Pipeline if it exists. + + :return: The NetworkManager associated with the Pipeline. + """ + + return self.__get_any_manager(manager_names='network_manager') + + def get_data_manager(self) -> Optional[DataManager]: + """ + Return the DataManager associated with the Pipeline if it exists. + + :return: The DataManager associated with the Pipeline. + """ + + return self.__get_any_manager(manager_names='data_manager') + + def get_stats_manager(self) -> Optional[StatsManager]: + """ + Return the StatsManager associated with the Pipeline if it exists. + + :return: The StatsManager associated with the Pipeline. + """ + + return self.__get_any_manager(manager_names='stats_manager') + + def get_database_manager(self) -> Optional[DatabaseManager]: + """ + Return the DatabaseManager associated with the Pipeline if it exists. + + :return: The DatabaseManager associated with the Pipeline. + """ + + return self.__get_any_manager(manager_names=['data_manager', 'database_manager']) + + def get_environment_manager(self) -> Optional[EnvironmentManager]: + """ + Return the EnvironmentManager associated with the Pipeline if it exists. + + :return: The EnvironmentManager associated with the Pipeline. + """ + + return self.__get_any_manager(manager_names=['data_manager', 'environment_manager']) + + def __str__(self): + + description = "\n" + description += f"# {self.name}\n" + description += f" Pipeline type: {self.type}\n" + description += f" Session repository: {self.session}\n" + return description diff --git a/src/Core/Pipelines/BasePrediction.py b/src/Core/Pipelines/BasePrediction.py new file mode 100644 index 00000000..34ca2b52 --- /dev/null +++ b/src/Core/Pipelines/BasePrediction.py @@ -0,0 +1,137 @@ +from typing import Optional +from os.path import join, exists + +from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline +from DeepPhysX.Core.Manager.DataManager import DataManager +from DeepPhysX.Core.Manager.NetworkManager import NetworkManager +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Utils.path import get_first_caller + + +class BasePrediction(BasePipeline): + + def __init__(self, + network_config: BaseNetworkConfig, + environment_config: BaseEnvironmentConfig, + database_config: Optional[BaseDatabaseConfig] = None, + session_dir: str = 'session', + session_name: str = 'training', + step_nb: int = -1, + record: bool = False): + """ + BasePrediction is a pipeline defining the running process of an artificial neural network. + It provides a highly tunable learning process that can be used with any machine learning library. + + :param network_config: Configuration object with the parameters of the Network. + :param environment_config: Configuration object with the parameters of the Environment. + :param database_config: Configuration object with the parameters of the Database. + :param session_dir: Relative path to the directory which contains sessions repositories. + :param session_name: Name of the new the session repository. + :param step_nb: Number of simulation step to play. + :param record: If True, prediction data will be saved in a dedicated Database. + """ + + BasePipeline.__init__(self, + network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir=session_dir, + session_name=session_name, + new_session=False, + pipeline='prediction') + + # Define the session repository + root = get_first_caller() + session_dir = join(root, session_dir) + if not exists(join(session_dir, session_name)): + raise ValueError(f"[{self.name}] The following directory does not exist: {join(session_dir, session_name)}") + self.session = join(session_dir, session_name) + + # Create a NetworkManager + self.network_manager = NetworkManager(network_config=network_config, + pipeline=self.type, + session=self.session, + new_session=False) + + # Create a DataManager + self.data_manager = DataManager(pipeline=self, + database_config=database_config, + environment_config=environment_config, + session=self.session, + new_session=False, + produce_data=record, + batch_size=1) + self.data_manager.connect_handler(self.network_manager.get_database_handler()) + self.network_manager.link_clients(self.data_manager.nb_environment) + + # Prediction variables + self.step_nb = step_nb + self.step_id = 0 + + def execute(self) -> None: + """ + Launch the prediction Pipeline. + Each event is already implemented for a basic pipeline but can also be rewritten via inheritance to describe a + more complex Pipeline. + """ + + self.prediction_begin() + while self.prediction_condition(): + self.sample_begin() + self.predict() + self.sample_end() + self.prediction_end() + + def prediction_begin(self) -> None: + """ + Called once at the beginning of the prediction Pipeline. + """ + + pass + + def prediction_condition(self) -> bool: + """ + Condition that characterize the end of the prediction Pipeline. + """ + + running = self.step_id < self.step_nb if self.step_nb > 0 else True + self.step_id += 1 + return running + + def sample_begin(self) -> None: + """ + Called one at the beginning of each sample. + """ + + pass + + def predict(self) -> None: + """ + Pull the data from the manager and return the prediction. + """ + + self.data_manager.get_data(epoch=0, + animate=True) + + def sample_end(self) -> None: + """ + Called one at the end of each sample. + """ + + pass + + def prediction_end(self) -> None: + """ + Called once at the end of the prediction Pipeline. + """ + + self.data_manager.close() + self.network_manager.close() + + def __str__(self): + + description = BasePipeline.__str__(self) + description += f" Number of step: {self.step_nb}\n" + return description diff --git a/src/Core/Pipelines/BaseTraining.py b/src/Core/Pipelines/BaseTraining.py new file mode 100644 index 00000000..f326d515 --- /dev/null +++ b/src/Core/Pipelines/BaseTraining.py @@ -0,0 +1,262 @@ +from typing import Optional +from os.path import join, isfile, exists, sep +from datetime import datetime +from vedo import ProgressBar + +from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline +from DeepPhysX.Core.Manager.DataManager import DataManager +from DeepPhysX.Core.Manager.NetworkManager import NetworkManager +from DeepPhysX.Core.Manager.StatsManager import StatsManager +from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig +from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig +from DeepPhysX.Core.Utils.path import get_first_caller, create_dir + + +class BaseTraining(BasePipeline): + + def __init__(self, + network_config: BaseNetworkConfig, + database_config: BaseDatabaseConfig, + environment_config: Optional[BaseEnvironmentConfig] = None, + session_dir: str = 'sessions', + session_name: str = 'training', + new_session: bool = True, + epoch_nb: int = 0, + batch_nb: int = 0, + batch_size: int = 0, + debug: bool = False): + """ + BaseTraining implements the main loop that defines the training process of an artificial neural network. + Training can be launched with several data sources (from a Dataset, from an Environment, from combined sources). + It provides a highly tunable learning process that can be used with any machine learning library. + + :param network_config: Configuration object with the parameters of the Network. + :param database_config: Configuration object with the parameters of the Database. + :param environment_config: Configuration object with the parameters of the Environment. + :param session_dir: Relative path to the directory which contains sessions repositories. + :param session_name: Name of the new the session repository. + :param new_session: If True, a new repository will be created for this session. + :param epoch_nb: Number of epochs to perform. + :param batch_nb: Number of batches to use. + :param batch_size: Number of samples in a single batch. + :param debug: If True, main training features will not be launched. + """ + + BasePipeline.__init__(self, + network_config=network_config, + database_config=database_config, + environment_config=environment_config, + session_dir=session_dir, + session_name=session_name, + new_session=new_session, + pipeline='training') + + # Define the session repository + root = get_first_caller() + session_dir = join(root, session_dir) + + # Create a new session if required + if not new_session: + new_session = not exists(join(session_dir, session_name)) + if new_session: + session_name = create_dir(session_dir=session_dir, + session_name=session_name).split(sep)[-1] + self.session = join(session_dir, session_name) + + # Configure 'produce_data' flag + if environment_config is None and database_config.existing_dir is None: + raise ValueError(f"[{self.name}] No data source provided.") + produce_data = database_config.existing_dir is None + + # Create a DataManager + self.data_manager = DataManager(pipeline=self, + database_config=database_config, + environment_config=environment_config, + session=self.session, + new_session=new_session, + produce_data=produce_data, + batch_size=batch_size) + self.batch_size = batch_size + + # Create a NetworkManager + self.network_manager = NetworkManager(network_config=network_config, + pipeline=self.type, + session=self.session, + new_session=new_session) + self.data_manager.connect_handler(self.network_manager.get_database_handler()) + self.network_manager.link_clients(self.data_manager.nb_environment) + + # Create a StatsManager + self.stats_manager = StatsManager(session=self.session) if not debug else None + + # Training variables + self.epoch_nb = epoch_nb + self.epoch_id = 0 + self.batch_nb = batch_nb + self.batch_size = batch_size + self.batch_id = 0 + self.nb_samples = batch_nb * batch_size * epoch_nb + self.loss_dict = None + self.debug = debug + + # Progressbar + self.progress_counter = 0 + self.digits = ['{' + f':0{len(str(self.epoch_nb))}d' + '}', + '{' + f':0{len(str(self.batch_nb))}d' + '}'] + epoch_id, epoch_nb = self.digits[0].format(0), self.digits[0].format(self.epoch_nb) + batch_id, batch_nb = self.digits[1].format(0), self.digits[1].format(self.batch_nb) + self.progress_bar = ProgressBar(start=0, stop=self.batch_nb * self.epoch_nb, c='orange', + title=f'Epoch n°{epoch_id}/{epoch_nb} - Batch n°{batch_id}/{batch_nb}') + + self.save_info_file() + + def execute(self) -> None: + """ + Launch the training Pipeline. + Each event is already implemented for a basic pipeline but can also be rewritten via inheritance to describe a + more complex Pipeline. + """ + + self.train_begin() + while self.epoch_condition(): + self.epoch_begin() + while self.batch_condition(): + self.batch_begin() + self.optimize() + self.batch_count() + self.batch_end() + self.epoch_count() + self.epoch_end() + self.train_end() + + def train_begin(self) -> None: + """ + Called once at the beginning of the training Pipeline. + """ + + pass + + def epoch_condition(self) -> bool: + """ + Check the epoch number condition. + """ + + return self.epoch_id < self.epoch_nb + + def epoch_begin(self) -> None: + """ + Called one at the beginning of each epoch. + """ + + self.batch_id = 0 + + def batch_condition(self) -> bool: + """ + Check the batch number condition. + """ + + return self.batch_id < self.batch_nb + + def batch_begin(self) -> None: + """ + Called one at the beginning of a batch production. + """ + + self.progress_counter += 1 + id_epoch, nb_epoch = self.digits[0].format(self.epoch_id + 1), self.digits[0].format(self.epoch_nb) + id_batch, nb_batch = self.digits[1].format(self.batch_id + 1), self.digits[1].format(self.batch_nb) + self.progress_bar.title = f'Epoch n°{id_epoch}/{nb_epoch} - Batch n°{id_batch}/{nb_batch} ' + self.progress_bar.print(counts=self.progress_counter) + + def optimize(self) -> None: + """ + Pulls data, run a prediction and an optimizer step. + """ + + self.data_manager.get_data(epoch=self.epoch_id, + animate=True) + self.loss_dict = self.network_manager.compute_prediction_and_loss( + data_lines=self.data_manager.data_lines, + normalization=self.data_manager.normalization, + optimize=True) + + def batch_count(self) -> None: + """ + Increment the batch counter. + """ + + self.batch_id += 1 + + def batch_end(self) -> None: + """ + Called one at the end of a batch production. + """ + + if self.stats_manager is not None: + self.stats_manager.add_train_batch_loss(self.loss_dict['loss'], + self.epoch_id * self.batch_nb + self.batch_id) + for key in self.loss_dict.keys(): + if key != 'loss': + self.stats_manager.add_custom_scalar(tag=key, + value=self.loss_dict[key], + count=self.epoch_id * self.batch_nb + self.batch_id) + + def epoch_count(self) -> None: + """ + Increment the epoch counter. + """ + + self.epoch_id += 1 + + def epoch_end(self) -> None: + """ + Called one at the end of each epoch. + """ + + if self.stats_manager is not None: + self.stats_manager.add_train_epoch_loss(self.loss_dict['loss'], self.epoch_id) + self.network_manager.save_network() + + def train_end(self) -> None: + """ + Called once at the end of the training Pipeline. + """ + + self.data_manager.close() + self.network_manager.close() + if self.stats_manager is not None: + self.stats_manager.close() + + def save_info_file(self) -> None: + """ + Save a .txt file that provides a template for user notes and the description of all the components. + """ + + filename = join(self.session, 'info.txt') + date_time = datetime.now().strftime('%d/%m/%Y %H:%M:%S') + if not isfile(filename): + f = open(filename, "w+") + # Session description template for user + f.write("## DeepPhysX Training Session ##\n") + f.write(date_time + "\n\n") + f.write("Personal notes on the training session:\nNetwork Input:\nNetwork Output:\nComments:\n\n") + # Listing every component descriptions + f.write("## List of Components Parameters ##\n") + f.write(str(self)) + f.write(str(self.network_manager)) + f.write(str(self.data_manager)) + if self.stats_manager is not None: + f.write(str(self.stats_manager)) + f.close() + + def __str__(self): + + description = BasePipeline.__str__(self) + description += f" Number of epochs: {self.epoch_nb}\n" + description += f" Number of batches per epoch: {self.batch_nb}\n" + description += f" Number of samples per batch: {self.batch_size}\n" + description += f" Number of samples per epoch: {self.batch_nb * self.batch_size}\n" + description += f" Total: Number of batches : {self.batch_nb * self.epoch_nb}\n" + description += f" Number of samples : {self.nb_samples}\n" + return description diff --git a/src/Pipelines/__init__.py b/src/Core/Pipelines/__init__.py similarity index 100% rename from src/Pipelines/__init__.py rename to src/Core/Pipelines/__init__.py diff --git a/src/Utils/Visualizer/GridMapping.py b/src/Core/Utils/Visualizer/GridMapping.py similarity index 100% rename from src/Utils/Visualizer/GridMapping.py rename to src/Core/Utils/Visualizer/GridMapping.py diff --git a/src/Utils/Visualizer/SampleVisualizer.py b/src/Core/Utils/Visualizer/SampleVisualizer.py similarity index 100% rename from src/Utils/Visualizer/SampleVisualizer.py rename to src/Core/Utils/Visualizer/SampleVisualizer.py diff --git a/src/Core/Utils/Visualizer/__init__.py b/src/Core/Utils/Visualizer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Utils/Visualizer/barycentric_mapping.py b/src/Core/Utils/Visualizer/barycentric_mapping.py similarity index 100% rename from src/Utils/Visualizer/barycentric_mapping.py rename to src/Core/Utils/Visualizer/barycentric_mapping.py diff --git a/src/Utils/__init__.py b/src/Core/Utils/__init__.py similarity index 100% rename from src/Utils/__init__.py rename to src/Core/Utils/__init__.py diff --git a/src/Core/Utils/configs.py b/src/Core/Utils/configs.py new file mode 100644 index 00000000..8ac16c6f --- /dev/null +++ b/src/Core/Utils/configs.py @@ -0,0 +1,33 @@ +from typing import Any +from collections import namedtuple + + +def make_config(configuration_object: Any, + configuration_name: str, + **kwargs) -> namedtuple: + """ + Create a namedtuple which gathers all the parameters for any configuration object. + For a child config class, only new items are required since parent's items will be added by default. + + :param configuration_object: Instance of any Config class. + :param configuration_name: Name of the variable containing the namedtuple. + :param kwargs: Parameters to add to the namedtuple. + :return: Namedtuple which contains newly added parameters. + """ + + # Get items set as keyword arguments + fields = tuple(kwargs.keys()) + args = tuple(kwargs.values()) + + # Check if a dataset_config already exists (child class will have the parent's config by default) + if configuration_name in configuration_object.__dict__: + configuration: namedtuple = configuration_object.__getattribute__(configuration_name) + # Get items set in the existing config + for key, value in configuration._asdict().items(): + # Only new items are required for children, check if the parent's items are set again anyway + if key not in fields: + fields += (key,) + args += (value,) + + # Create namedtuple with collected items + return namedtuple(configuration_name, fields)._make(args) diff --git a/src/Core/Utils/converter.py b/src/Core/Utils/converter.py new file mode 100644 index 00000000..4302df12 --- /dev/null +++ b/src/Core/Utils/converter.py @@ -0,0 +1,147 @@ +from typing import Dict, List, Optional +from os import listdir +from os.path import join, exists, isfile, dirname, sep +from numpy import ndarray, load +from vedo import ProgressBar + +from DeepPhysX.Core.Utils.path import get_first_caller, create_dir +from DeepPhysX.Core.Manager.DatabaseManager import DatabaseManager +from DeepPhysX.Core.Database.DatabaseHandler import DatabaseHandler +from DeepPhysX.Core.Database.BaseDatabaseConfig import BaseDatabaseConfig + + +class DatasetConverter: + + def __init__(self, + session_path: str): + """ + Convert a Dataset between the previous Numpy partitions and the new SQL Database. + + :param session_path: Relative path to the session to convert. + """ + + root = get_first_caller() + self.dataset_dir = join(root, session_path, 'dataset') + if not exists(self.dataset_dir): + raise ValueError(f"The given path does not exist: {self.dataset_dir}") + + def numpy_to_database(self, + batch_size: int, + max_file_size: Optional[float] = None, + normalize: bool = False): + """ + Convert a Database from the previous Numpy partitions to the new SQL Database. + """ + + # 1. Get the partitions files + partitions = self.load_numpy_partitions() + + # 2. Create a new repository + session_name = f'{self.dataset_dir.split(sep)[-2]}_converted' + session = create_dir(session_dir=dirname(dirname(self.dataset_dir)), + session_name=session_name) + database_manager = DatabaseManager(pipeline='data_generation', + session=session) + database_manager.close() + + # 3. Create the Database for each mode + for mode in partitions.keys(): + print(f"\nConverting {mode} partitions...") + + # 3.1. Create a DatabaseManager with a DatabaseHandler + database_config = BaseDatabaseConfig(mode=mode, + max_file_size=max_file_size, + normalize=normalize) + database_manager = DatabaseManager(database_config=database_config, + pipeline='data_generation', + session=session, + new_session=False) + database_handler = DatabaseHandler() + database_manager.connect_handler(database_handler) + database_manager.first_add = True + + # 3.2. Create Fields in Tables + training_fields = [] + additional_fields = [] + nb_partition = 0 + for field in partitions[mode].keys(): + if field in ['input', 'ground_truth']: + training_fields.append((field, ndarray)) + + else: + additional_fields.append((field.split('_')[-1], ndarray)) + if nb_partition == 0: + nb_partition = len(partitions[mode][field]) + elif len(partitions[mode][field]) != nb_partition: + raise ValueError(f"The number of partition is not consistent in {mode} mode.") + database_handler.create_fields(table_name='Training', fields=training_fields) + database_handler.create_fields(table_name='Additional', fields=additional_fields) + + # 3.3. Add each partition to the Database + if nb_partition == 0: + print(" No partition.") + database_manager.normalize = False + for i in range(nb_partition): + data_training = {field: load(join(self.dataset_dir, f'{partitions[mode][field][i]}.npy')) + for field, _ in training_fields} + data_additional = {field: load(join(self.dataset_dir, f'{partitions[mode][field][i]}.npy')) + for field, _ in additional_fields} + nb_sample = 0 + for data in [data_training, data_additional]: + for field in data.keys(): + if nb_sample == 0: + nb_sample = data[field].shape[0] + elif data[field].shape[0] != nb_sample: + raise ValueError(f"The number of sample is not consistent in {mode} mode.") + id_sample = 0 + pb = ProgressBar(0, nb_sample, c='orange', title=f" Loading partition {i + 1}/{nb_partition}") + while id_sample < nb_sample: + last_sample = min(nb_sample, id_sample + batch_size) + if len(data_training.keys()) > 0: + sample_training = {field: data_training[field][id_sample:last_sample] + for field in data_training.keys()} + database_handler.add_batch(table_name='Training', batch=sample_training) + if len(data_additional.keys()) > 0: + sample_additional = {field: data_additional[field][id_sample:last_sample] + for field in data_additional.keys()} + database_handler.add_batch(table_name='Additional', batch=sample_additional) + database_manager.add_data() + id_sample += batch_size + pb.print(counts=id_sample) + + # 3. Close DatabaseManager + database_manager.close() + + print("\nConversion done.") + + def load_numpy_partitions(self) -> Dict[str, Dict[str, List[str]]]: + """ + Get all the partition files in the repository. Do not use the JSON file to prevent bugs. + """ + + # 1. Get the partition files for each mode + modes = ['training', 'validation', 'prediction'] + partitions = {mode: [f.split('.')[0] for f in listdir(self.dataset_dir) if isfile(join(self.dataset_dir, f)) + and f.endswith('.npy') and f.__contains__(mode)] for mode in modes} + + # 2. Sort partitions by field (IN, OUT, ADD) and by name + sorted_partitions = {mode: {field[1:-1]: sorted([f for f in partitions[mode] if f.__contains__(field)]) + for field in ['_IN_', '_OUT_', '_ADD_']} + for mode in modes} + all_partitions = {} + for mode in modes: + all_partitions[mode] = {} + for field, name in zip(['IN', 'OUT', 'ADD'], ['input', 'ground_truth', '']): + for partition in sorted_partitions[mode][field]: + # Extract information from the filename + partition_name = partition.split('_') + partition_name = partition_name[partition_name.index(field):] + # Additional data: ___ + if len(partition_name) == 3: + name = partition_name[-2] + # Add partition + if name not in all_partitions[mode]: + all_partitions[mode][name] = [] + all_partitions[mode][name].append(partition) + + return all_partitions diff --git a/src/Utils/data_downloader.py b/src/Core/Utils/data_downloader.py similarity index 100% rename from src/Utils/data_downloader.py rename to src/Core/Utils/data_downloader.py diff --git a/src/Utils/jsonUtils.py b/src/Core/Utils/jsonUtils.py similarity index 96% rename from src/Utils/jsonUtils.py rename to src/Core/Utils/jsonUtils.py index abc49337..bce413b1 100644 --- a/src/Utils/jsonUtils.py +++ b/src/Core/Utils/jsonUtils.py @@ -64,7 +64,7 @@ def encode(self, o: Iterable) -> str: elif isinstance(o, dict): self.indentation_level += 1 output = [f"{self.indent_str}{json.dumps(key)}: {self.encode(value)}" for key, value in o.items()] - join_output = ",\n".join(output) + join_output = ",\n".join(output) if self.indentation_level != 1 else ",\n\n".join(output) self.indentation_level -= 1 return f"\n{self.indent_str}{'{'}\n{join_output}\n{self.indent_str}{'}'}" diff --git a/src/Utils/mathUtils.py b/src/Core/Utils/mathUtils.py similarity index 100% rename from src/Utils/mathUtils.py rename to src/Core/Utils/mathUtils.py diff --git a/src/Core/Utils/path.py b/src/Core/Utils/path.py new file mode 100644 index 00000000..14ede966 --- /dev/null +++ b/src/Core/Utils/path.py @@ -0,0 +1,79 @@ +from typing import Optional +from os.path import join, isdir, abspath, normpath, dirname, basename +from os import listdir, pardir, makedirs +from inspect import getmodule, stack +from shutil import copytree + + +def get_first_caller() -> str: + """ + Return the repertory in which the main script is stored. + """ + + # Get the stack of calls + scripts_list = stack()[-1] + # Get the first one (the one launched by the user) + module = getmodule(scripts_list[0]) + # Return the path of this script + return dirname(abspath(module.__file__)) + + +def create_dir(session_dir: str, session_name: str) -> str: + """ + Create a new directory of the given name in the given directory. + If it already exists, add a unique identifier at the end of the directory name. + + :param session_dir: Path where to create the directory. + :param session_name: Name of the directory to create. + :return: Path to the newly created directory. + """ + + if isdir(join(session_dir, session_name)): + print(f"Directory conflict: you are going to overwrite {join(session_dir, session_name)}.") + # Find all the duplicated folders + session_name += '_' + copies = [folder for folder in listdir(session_dir) if isdir(join(session_dir, folder)) and + folder.find(session_name) == 0] + # Get the indices of copies + indices = [int(folder[len(session_name):]) for folder in copies] + # The new copy is the max index + 1 + max_idx = max(indices) if len(indices) > 0 else 0 + session_name += f'{max_idx + 1}' + + session = join(session_dir, session_name) + print(f"Create a new directory {session} for this session.") + makedirs(session) + return session + + +def copy_dir(src_dir: str, dest_dir: str, dest_name: Optional[str] = None, sub_folders: Optional[str] = None) -> str: + """ + Copy the source directory to the destination directory. + + :param src_dir: Source directory to copy. + :param dest_dir: Parent of the destination directory to copy. + :param dest_name: Destination directory to copy to. + :param sub_folders: If sub folders are specified, the latest is actually copied. + + :return: Path to the newly copied directory. + """ + + if dest_name is not None and isdir(join(dest_dir, dest_name)): + print(f"Directory conflict: you are going to overwrite {join(dest_dir, dest_name)}.") + # Find all the duplicated folders + dest_name += '_' + copies = [folder for folder in listdir(dest_dir) if isdir(join(dest_dir, folder)) and + folder.find(dest_name) == 0] + # Get the indices of the copies + indices = [int(folder[len(dest_name):]) for folder in copies] + # The new copy is the max index + 1 + max_id = max(indices) if len(indices) > 0 else 0 + dest_name += f'{max_id + 1}' + + dest = join(dest_dir, dest_name) if dest_name is not None else dest_dir + print(f"Copying the source directory {src_dir} to {dest} for this session.") + if sub_folders is None: + copytree(src_dir, dest) + else: + copytree(join(src_dir, sub_folders), join(dest, sub_folders)) + return dest diff --git a/src/Utils/tensor_transform_utils.py b/src/Core/Utils/tensor_transform_utils.py similarity index 100% rename from src/Utils/tensor_transform_utils.py rename to src/Core/Utils/tensor_transform_utils.py diff --git a/src/Visualization/VedoFactory.py b/src/Core/Visualization/VedoFactory.py similarity index 100% rename from src/Visualization/VedoFactory.py rename to src/Core/Visualization/VedoFactory.py diff --git a/src/Visualization/VedoVisualizer.py b/src/Core/Visualization/VedoVisualizer.py similarity index 90% rename from src/Visualization/VedoVisualizer.py rename to src/Core/Visualization/VedoVisualizer.py index d84e367c..d4187f1d 100644 --- a/src/Visualization/VedoVisualizer.py +++ b/src/Core/Visualization/VedoVisualizer.py @@ -12,7 +12,8 @@ def __init__(self, database_name: Optional[str] = None, remove_existing: bool = False, offscreen: bool = False, - remote: bool = False): + remote: bool = False, + record: bool = True): """ Manage the creation, update and rendering of Vedo Actors. @@ -22,6 +23,7 @@ def __init__(self, :param remove_existing: If True, overwrite a Database with the same path. :param offscreen: If True, visual data will be saved but not rendered. :param remote: If True, the Visualizer will treat the Factories as remote. + :param record: If True, the visualization Database is saved in memory. """ # Define Database @@ -47,6 +49,7 @@ def __init__(self, if not remote: self.__database.register_post_save_signal(table_name='Sync', handler=self.__sync_visualizer) + self.record = record def render_instance(self, instance: int): """ @@ -80,3 +83,11 @@ def render_instance(self, instance: int): # 3. Render Plotter if offscreen is False if not self.__offscreen: self.__plotter.render() + + def close(self): + """ + Launch the closing procedure of the Visualizer. + """ + + if not self.record: + self.__database.close(erase_file=True) diff --git a/src/Core/Visualization/__init__.py b/src/Core/Visualization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/Core/__init__.py b/src/Core/__init__.py new file mode 100644 index 00000000..d431a0bb --- /dev/null +++ b/src/Core/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname +from os import listdir + +package = dirname(__file__) +exceptions = ['__init__.py', '__pycache__'] +modules = [module for module in listdir(package) if module not in exceptions] +__all__ = [] +for module in sorted(modules): + __all__.append(module) diff --git a/src/Dataset/BaseDataset.py b/src/Dataset/BaseDataset.py deleted file mode 100644 index decd434c..00000000 --- a/src/Dataset/BaseDataset.py +++ /dev/null @@ -1,252 +0,0 @@ -from typing import Dict, List, Optional, Tuple -from numpy import array, ndarray, concatenate, save, arange -from numpy.random import shuffle -from collections import namedtuple - - -class BaseDataset: - """ - | BaseDataset is a dataset class to store any data from a BaseEnvironment or from files. - | Given data is split into input data and output data. - | Saving data results in multiple partitions of input and output data. - - :param namedtuple config: Namedtuple which contains BaseDataset parameters - """ - - def __init__(self, config: namedtuple): - - self.name: str = self.__class__.__name__ - - # Data fields containers - self.data_type = ndarray - self.fields: List[str] = ['input', 'output'] - self.data: Dict[str, ndarray] = {'input': array([]), 'output': array([])} - self.shape: Dict[str, Optional[List[int]]] = {'input': None, 'output': None} - - # Indexing - self.shuffle_pattern: Optional[List[int]] = None - self.current_sample: int = 0 - - # Dataset memory - self.max_size: int = config.max_size - self.batch_per_field: Dict[str, int] = {field: 0 for field in ['input', 'output']} - self.__empty: bool = True - - @property - def nb_samples(self) -> int: - """ - | Property returning the current number of samples - - :return: The current number of samples in all partitions - """ - - return max([len(self.data[field]) for field in self.fields]) - - def is_empty(self) -> bool: - """ - | Check if the fields of the dataset are empty. A field is considered as non-empty if it is filled with another - sample. - - :return: The Dataset is empty or not - """ - - # The empty flag is set to False once the Dataset is considered as non-empty - if not self.__empty: - return False - # Check each registered data field - for field in self.fields: - # Dataset is considered as non-empty if a field is filled with another sample - if self.batch_per_field[field] > 1: - self.__empty = False - return False - # If all field are considered as non-empty then the Dataset is empty - return True - - def init_data_size(self, field: str, shape: List[int]) -> None: - """ - | Store the original shape of data. Reshape data containers. - - :param str field: Data field name - :param List[int] shape: Shape of the corresponding tensor - """ - - # Store the original data shape - self.shape[field] = shape - # Reshape the data container - self.data[field] = array([]).reshape((0, *shape)) - - def get_data_shape(self, field: str) -> List[int]: - """ - | Returns the data shape of field. - - :param str field: Data field name - :return: Data shape for field - """ - - return self.shape[field] - - def init_additional_field(self, field: str, shape: List[int]) -> None: - """ - | Register a new data field. - - :param str field: Name of the data field - :param List[int] shape: Data shape - """ - - # Register the data field - self.fields.append(field) - # Init the number of adds in the field - self.batch_per_field[field] = 0 - # Init the field shape - self.init_data_size(field, shape) - - def empty(self) -> None: - """ - | Empty the dataset. - """ - - # Reinit each data container - for field in self.fields: - self.data[field] = array([]) if self.shape[field] is None else array([]).reshape((0, *self.shape[field])) - # Reinit indexing variables - self.shuffle_pattern = None - self.current_sample = 0 - self.batch_per_field = {field: 0 for field in self.fields} - self.__empty = True - - def memory_size(self, field: Optional[str] = None) -> int: - """ - | Return the actual memory size of the dataset if field is None. Otherwise, return the actual memory size of the - field. - - :param Optional[str] field: Name of the data field - :return: Size in bytes of the current dataset. - """ - - # Return the total memory size - if field is None: - return sum([self.data[field].nbytes for field in self.fields]) - # Return the memory size for the specified field - return self.data[field].nbytes - - def check_data(self, field: str, data: ndarray) -> None: - """ - | Check if the data is a numpy array. - - :param str field: Values at 'input' or anything else. Define if the associated shape is correspond to input - shape or output one. - :param ndarray data: New data - """ - - if type(data) != self.data_type: - raise TypeError(f"[{self.name}] Wrong data type in field '{field}': numpy array required, got {type(data)}") - - def add(self, field: str, data: ndarray, partition_file: Optional[str] = None) -> None: - """ - | Add data to the dataset. - - :param str field: Name of the data field - :param ndarray data: New data as batch of samples - :param Optional[str] partition_file: Path to the file in which to write the data - """ - - # Check data type - self.check_data(field, data) - - # Check if field is registered - if field not in self.fields: - # Fields can be register only if Dataset is empty - if not self.is_empty(): - raise ValueError(f"[{self.name}] A new field {field} tries to be created as Dataset is non empty. This " - f"will lead to a different number of sample for each field of the dataset.") - # Add new field if not registered - self.init_additional_field(field, data[0].shape) - - # Check data size initialization - if self.shape[field] is None: - self.init_data_size(field, data[0].shape) - - # Add batched samples - self.data[field] = concatenate((self.data[field], data)) - # Save in partition - if partition_file is not None: - self.save(field, partition_file) - - # Update sample indexing in dataset - self.batch_per_field[field] += 1 - self.current_sample = max([len(self.data[f]) for f in self.fields]) - - def save(self, field: str, file: str) -> None: - """ - | Save the corresponding field of the Dataset. - - :param str field: Name of the data field - :param str file: Path to the file in which to write the data - """ - - save(file, self.data[field]) - - def set(self, field: str, data: ndarray) -> None: - """ - | Set a full field of the dataset. - - :param str field: Name of the data field - :param ndarray data: New data as batch of samples - """ - - # Check data type - self.check_data(field, data) - - # Check if field is registered - if field not in self.fields: - # Add new field if not registered - self.init_additional_field(field, data[0].shape) - - # Check data size initialization - if self.shape[field] is None: - self.init_data_size(field, data[0].shape) - - # Set the full field - self.data[field] = data - - # Update sample indexing in dataset - self.__empty = False - self.current_sample = 0 - - def get(self, field: str, idx_begin: int, idx_end: int) -> ndarray: - """ - | Get a batch of data in 'field' container. - - :param str field: Name of the data field - :param int idx_begin: Index of the first sample - :param int idx_end: Index of the last sample - :return: Batch of data from 'field' - """ - - indices = slice(idx_begin, idx_end) if self.shuffle_pattern is None else self.shuffle_pattern[idx_begin:idx_end] - return self.data[field][indices] - - def shuffle(self) -> None: - """ - | Define a random shuffle pattern. - """ - - # Nothing to shuffle if Dataset is empty - if self.is_empty(): - return - # Generate a shuffle pattern - self.shuffle_pattern = arange(self.nb_samples) - shuffle(self.shuffle_pattern) - - def __str__(self) -> str: - """ - :return: String containing information about the BaseDatasetConfig object - """ - - description = "\n" - description += f" {self.name}\n" - description += f" Max size: {self.max_size}\n" - description += f" Data fields: {self.fields}" - for field in self.fields: - description += f" {field} shape: {self.shape[field]}" - return description diff --git a/src/Dataset/BaseDatasetConfig.py b/src/Dataset/BaseDatasetConfig.py deleted file mode 100644 index acec6302..00000000 --- a/src/Dataset/BaseDatasetConfig.py +++ /dev/null @@ -1,116 +0,0 @@ -from typing import Type, Optional -from os.path import isdir -from collections import namedtuple - -from DeepPhysX.Core.Dataset.BaseDataset import BaseDataset - - -class BaseDatasetConfig: - """ - | BaseDatasetConfig is a configuration class to parameterize and create a BaseDataset for the DatasetManager. - - :param Type[BaseDataset] dataset_class: BaseDataset class from which an instance will be created - :param Optional[str] dataset_dir: Name of an existing dataset repository - :param float partition_size: Maximum size in Gb of a single dataset partition - :param bool shuffle_dataset: Specify if existing dataset should be shuffled - :param Optional[str] use_mode: Specify the Dataset mode that should be used between 'Training', 'Validation' and - 'Running' - :param bool normalize: If True, normalizing dataset using standard score - :param bool recompute_normalization: If True, triggers a normalization coefficients computation - """ - - def __init__(self, - dataset_class: Type[BaseDataset] = BaseDataset, - dataset_dir: Optional[str] = None, - partition_size: float = 1., - shuffle_dataset: bool = True, - use_mode: Optional[str] = None, - normalize: bool = True, - recompute_normalization: bool = False): - - self.name: str = self.__class__.__name__ - - # Check dataset_dir type and existence - if dataset_dir is not None: - if type(dataset_dir) != str: - raise TypeError(f"[{self.name}] Wrong dataset_dir type: str required, get {type(dataset_dir)}") - if not isdir(dataset_dir): - raise ValueError(f"[{self.name}] Given dataset_dir doesn't exists: {dataset_dir}") - # Check partition_size type and value - if type(partition_size) != int and type(partition_size) != float: - raise TypeError(f"[{self.name}] Wrong partition_size type: float required, get {type(partition_size)}") - if partition_size <= 0: - raise ValueError(f"[{self.name}] Given partition_size is negative or null") - # Check shuffle_dataset type - if type(shuffle_dataset) != bool: - raise TypeError(f"[{self.name}] Wrong shuffle_dataset type: bool required, get {type(shuffle_dataset)}") - # Check use_mode type and value - if use_mode is not None: - if type(use_mode) != str: - raise TypeError(f"[{self.name}] Wrong use_mode type: str required, get {type(dataset_dir)}") - if use_mode not in ['Training', 'Validation', 'Running']: - raise ValueError(f"[{self.name}] Wrong use_mode value, must be in " - f"{['Training', 'Validation', 'Running']}") - - # BaseDataset parameterization - self.dataset_class: Type[BaseDataset] = dataset_class - self.dataset_config: namedtuple = self.make_config(max_size=int(partition_size * 1e9)) - - # DatasetManager parameterization - self.dataset_dir: str = dataset_dir - self.shuffle_dataset: bool = shuffle_dataset - self.use_mode: Optional[str] = use_mode - self.normalize: bool = normalize - self.recompute_normalization = recompute_normalization - - def make_config(self, **kwargs) -> namedtuple: - """ - | Create a namedtuple which gathers all the parameters for the Dataset configuration. - | For a child config class, only new items are required since parent's items will be added by default. - - :param kwargs: Items to add to the Dataset configuration. - :return: Namedtuple which contains Dataset parameters - """ - - # Get items set as keyword arguments - fields = tuple(kwargs.keys()) - args = tuple(kwargs.values()) - # Check if a dataset_config already exists (child class will have the parent's config by default) - if 'dataset_config' in self.__dict__: - # Get items set in the existing config - for key, value in self.dataset_config._asdict().items(): - # Only new items are required for children, check if the parent's items are set again anyway - if key not in fields: - fields += (key,) - args += (value,) - # Create namedtuple with collected items - return namedtuple('dataset_config', fields)._make(args) - - def create_dataset(self) -> BaseDataset: - """ - | Create an instance of dataset_class with given parameters. - - :return: Dataset object - """ - - try: - dataset = self.dataset_class(config=self.dataset_config) - except: - raise ValueError(f"[{self.name}] Given dataset_class got an unexpected keyword argument 'config'") - if not isinstance(dataset, BaseDataset): - raise TypeError(f"[{self.name}] Wrong dataset_class type: BaseDataset required, get {self.dataset_class}") - return dataset - - def __str__(self) -> str: - """ - :return: String containing information about the BaseDatasetConfig object - """ - - # Todo: fields in Configs are the set in Managers or objects, the remove __str__ method - description = "\n" - description += f"{self.name}\n" - description += f" Dataset class: {self.dataset_class.__name__}\n" - description += f" Max size: {self.dataset_config.max_size}\n" - description += f" Dataset dir: {self.dataset_dir}\n" - description += f" Shuffle dataset: {self.shuffle_dataset}\n" - return description diff --git a/src/Environment/BaseEnvironment.py b/src/Environment/BaseEnvironment.py deleted file mode 100644 index ff3bf23f..00000000 --- a/src/Environment/BaseEnvironment.py +++ /dev/null @@ -1,263 +0,0 @@ -from typing import Any, Optional, Dict, Union, Tuple -from numpy import array, ndarray - -from SSD.Core.Storage.Database import Database - -from DeepPhysX.Core.Visualization.VedoFactory import VedoFactory -from DeepPhysX.Core.AsyncSocket.TcpIpClient import TcpIpClient - - -class BaseEnvironment(TcpIpClient): - - def __init__(self, - ip_address: str = 'localhost', - port: int = 10000, - instance_id: int = 0, - number_of_instances: int = 1, - as_tcp_ip_client: bool = True, - environment_manager: Optional[Any] = None, - visu_db: Optional[Union[Database, Tuple[str, str]]] = None): - """ - BaseEnvironment is an environment class to compute simulated data for the network and its optimization process. - - :param ip_address: IP address of the TcpIpObject. - :param port: Port number of the TcpIpObject. - :param instance_id: ID of the instance. - :param number_of_instances: Number of simultaneously launched instances. - :param as_tcp_ip_client: Environment is owned by a TcpIpClient if True, by an EnvironmentManager if False. - :param environment_manager: EnvironmentManager that handles the Environment if 'as_tcp_ip_client' is False. - :param visu_db: The path to the visualization Database or the visualization Database object to connect to. - """ - - TcpIpClient.__init__(self, - instance_id=instance_id, - number_of_instances=number_of_instances, - as_tcp_ip_client=as_tcp_ip_client, - ip_address=ip_address, - port=port) - - # Input and output to give to the network - self.input: ndarray = array([]) - self.output: ndarray = array([]) - # Variables to store samples from Dataset - self.sample_in: Optional[ndarray] = None - self.sample_out: Optional[ndarray] = None - # Loss data - self.loss_data: Any = None - # Manager if the Environment is not a TcpIpClient - self.environment_manager: Any = environment_manager - - self.factory: Optional[VedoFactory] = None - if visu_db is not None: - if type(visu_db) == list: - self.factory = VedoFactory(database_path=visu_db, - idx_instance=instance_id, - remote=True) - else: - self.factory = VedoFactory(database=visu_db, - idx_instance=instance_id) - - ########################################################################################## - ########################################################################################## - # Initializing Environment # - ########################################################################################## - ########################################################################################## - - def recv_parameters(self, - param_dict: Dict[Any, Any]) -> None: - """ - Exploit received parameters before scene creation. - Not mandatory. - - :param param_dict: Dictionary of parameters. - """ - - pass - - def create(self) -> None: - """ - Create the Environment. - Must be implemented by user. - """ - - raise NotImplementedError - - def init(self) -> None: - """ - Initialize the Environment. - Not mandatory. - """ - - pass - - def send_parameters(self) -> Dict[Any, Any]: - """ - Create a dictionary of parameters to send to the manager. - Not mandatory. - - :return: Dictionary of parameters - """ - - return {} - - def init_visualization(self) -> None: - """ - Define the visualization objects to send to he Visualizer. - Not mandatory. - - :return: Dictionary of visualization data - """ - - pass - - ########################################################################################## - ########################################################################################## - # Environment behavior # - ########################################################################################## - ########################################################################################## - - async def step(self) -> None: - """ - Compute the number of steps in the Environment specified by simulations_per_step in EnvironmentConfig. - Must be implemented by user. - """ - - raise NotImplementedError - - def check_sample(self) -> bool: - """ - Check if the current produced sample is usable for training. - Not mandatory. - - :return: Current data can be used or not - """ - - return True - - def apply_prediction(self, - prediction: ndarray) -> None: - """ - Apply network prediction in environment. - Not mandatory. - - :param prediction: Prediction data. - """ - - pass - - def close(self) -> None: - """ - Close the Environment. - Not mandatory. - """ - - pass - - ########################################################################################## - ########################################################################################## - # Defining a sample # - ########################################################################################## - ########################################################################################## - - def set_training_data(self, - input_array: ndarray, - output_array: ndarray) -> None: - """ - Set the training data to send to the TcpIpServer or the EnvironmentManager. - - :param input_array: Network input - :param output_array: Network expected output - """ - - # Training data is set if the Environment can compute data - if self.compute_essential_data: - self.input = input_array - self.output = output_array - - def set_loss_data(self, - loss_data: Any) -> None: - """ - Set the loss data to send to the TcpIpServer or the EnvironmentManager. - - :param loss_data: Optional data to compute loss. - """ - - # Training data is set if the Environment can compute data - if self.compute_essential_data: - self.loss_data = loss_data if type(loss_data) in [list, ndarray] else array([loss_data]) - - def set_additional_dataset(self, - label: str, - data: ndarray) -> None: - """ - Set additional data fields to store in the dataset. - - :param label: Name of the data field. - :param data: Data to store. - """ - - # Training data is set if the Environment can compute data - if self.compute_essential_data: - self.additional_fields[label] = data if type(data) in [list, ndarray] else array([data]) - - def reset_additional_datasets(self) -> None: - """ - Reset the additional dataset dictionaries. - """ - - self.additional_fields = {} - - ########################################################################################## - ########################################################################################## - # Available requests # - ########################################################################################## - ########################################################################################## - - def get_prediction(self, - input_array: ndarray) -> ndarray: - """ - Request a prediction from Network. - - :param input_array: Network input. - :return: Network prediction. - """ - - # If Environment is a TcpIpClient, send request to the Server - if self.as_tcp_ip_client: - return TcpIpClient.request_get_prediction(self, input_array=input_array) - - # Otherwise, check the hierarchy of managers - if self.environment_manager.data_manager is None: - raise ValueError("Cannot request prediction if DataManager does not exist") - elif self.environment_manager.data_manager.manager is None: - raise ValueError("Cannot request prediction if Manager does not exist") - elif not hasattr(self.environment_manager.data_manager.manager, 'network_manager'): - raise AttributeError("Cannot request prediction if NetworkManager does not exist. If using a data " - "generation pipeline, please disable get_prediction requests.") - elif self.environment_manager.data_manager.manager.network_manager is None: - raise ValueError("Cannot request prediction if NetworkManager does not exist") - # Get a prediction - return self.environment_manager.data_manager.get_prediction(network_input=input_array[None, ]) - - def update_visualisation(self) -> None: - """ - Triggers the Visualizer update. - """ - - # If Environment is a TcpIpClient, request to the Server - if self.as_tcp_ip_client: - self.request_update_visualization() - self.factory.render() - - def __str__(self) -> str: - """ - :return: String containing information about the BaseEnvironment object - """ - - description = "\n" - description += f" {self.name}\n" - description += f" Name: {self.name} n°{self.instance_id}\n" - description += f" Comments:\n" - description += f" Input size:\n" - description += f" Output size:\n" - return description diff --git a/src/Manager/DataManager.py b/src/Manager/DataManager.py deleted file mode 100644 index 2e2708a2..00000000 --- a/src/Manager/DataManager.py +++ /dev/null @@ -1,249 +0,0 @@ -from typing import Any, Optional, Dict, List -import os.path -from numpy import ndarray -from json import load as json_load - -from DeepPhysX.Core.Manager.DatasetManager import DatasetManager -from DeepPhysX.Core.Manager.EnvironmentManager import EnvironmentManager -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig - - -class DataManager: - - def __init__(self, - dataset_config: BaseDatasetConfig, - environment_config: BaseEnvironmentConfig, - session: str, - manager: Optional[Any] = None, - new_session: bool = True, - training: bool = True, - offline: bool = False, - store_data: bool = True, - batch_size: int = 1): - - """ - DataManager deals with the generation of input / output tensors. His job is to call get_data on either the - DatasetManager or the EnvironmentManager according to the context. - - :param dataset_config: Specialisation containing the parameters of the dataset manager - :param environment_config: Specialisation containing the parameters of the environment manager - :param session: Path to the session directory. - :param manager: Manager that handle The DataManager - :param new_session: Define the creation of new directories to store data - :param training: True if this session is a network training - :param offline: True if the session is done offline - :param store_data: Format {\'in\': bool, \'out\': bool} save the tensor when bool is True - :param batch_size: Number of samples in a batch - """ - - self.name: str = self.__class__.__name__ - - # Manager references - self.manager: Optional[Any] = manager - self.dataset_manager: Optional[DatasetManager] = None - self.environment_manager: Optional[EnvironmentManager] = None - - # DataManager parameters - self.is_training: bool = training - self.allow_dataset_fetch: bool = True - self.data: Optional[Dict[str, ndarray]] = None - - # Data normalization coefficients (default: mean = 0, standard deviation = 1) - self.normalization: Dict[str, List[float]] = {'input': [0., 1.], 'output': [0., 1.]} - # If normalization flag is set to True, try to load existing coefficients - if dataset_config is not None and dataset_config.normalize: - json_file_path = None - # Existing Dataset in the current session - if os.path.exists(os.path.join(session, 'dataset')): - json_file_path = os.path.join(session, 'dataset', 'dataset.json') - # Dataset provided by config - elif dataset_config.dataset_dir is not None: - dataset_dir = dataset_config.dataset_dir - if dataset_dir[-1] != "/": - dataset_dir += "/" - if dataset_dir[-8:] != "dataset/": - dataset_dir += "dataset/" - if os.path.exists(dataset_dir): - json_file_path = os.path.join(dataset_dir, 'dataset.json') - # If Dataset exists then a json file is associated - if json_file_path is not None: - with open(json_file_path) as json_file: - json_dict = json_load(json_file) - # Get the normalization coefficients - for field in self.normalization.keys(): - if field in json_dict['normalization']: - self.normalization[field] = json_dict['normalization'][field] - - # Training - if self.is_training: - # Always create a dataset_manager for training - create_dataset = True - # Create an environment if prediction must be applied else ask DatasetManager - create_environment = False - if environment_config is not None: - create_environment = None if not environment_config.use_dataset_in_environment else True - # Prediction - else: - # Create an environment for prediction if an environment config is provided - create_environment = environment_config is not None - # Create a dataset if data will be stored from environment during prediction - create_dataset = store_data - # Create a dataset also if data should be loaded from any partition - create_dataset = create_dataset or (dataset_config is not None and dataset_config.dataset_dir is not None) - - # Create dataset if required - if create_dataset: - self.dataset_manager = DatasetManager(data_manager=self, dataset_config=dataset_config, - session=session, new_session=new_session, training=self.is_training, - offline=offline, store_data=store_data) - # Create environment if required - if create_environment is None: # If None then the dataset_manager exists - create_environment = self.dataset_manager.new_dataset() - if create_environment: - self.environment_manager = EnvironmentManager(data_manager=self, environment_config=environment_config, - session=session, batch_size=batch_size, - training=self.is_training) - - def get_manager(self) -> Any: - """ - Return the Manager of this DataManager. - - :return: The Manager of this DataManager. - """ - - return self.manager - - def get_data(self, - epoch: int = 0, - batch_size: int = 1, - animate: bool = True) -> Dict[str, ndarray]: - """ - Fetch data from the EnvironmentManager or the DatasetManager according to the context. - - :param epoch: Current epoch number. - :param batch_size: Size of the desired batch. - :param animate: Allow EnvironmentManager to generate a new sample. - :return: Dict containing the newly computed data. - """ - - # Training - if self.is_training: - data = None - # Get data from environment if used and if the data should be created at this epoch - if data is None and self.environment_manager is not None and \ - (epoch == 0 or self.environment_manager.always_create_data) and self.dataset_manager.new_dataset(): - self.allow_dataset_fetch = False - data = self.environment_manager.get_data(animate=animate, get_inputs=True, get_outputs=True) - self.dataset_manager.add_data(data) - # Force data from the dataset - else: - data = self.dataset_manager.get_data(batch_size=batch_size, get_inputs=True, get_outputs=True) - if self.environment_manager is not None and self.environment_manager.use_dataset_in_environment: - new_data = self.environment_manager.dispatch_batch(batch=data) - if len(new_data['input']) != 0: - data['input'] = new_data['input'] - if len(new_data['output']) != 0: - data['output'] = new_data['output'] - if 'loss' in new_data: - data['loss'] = new_data['loss'] - elif self.environment_manager is not None: - # EnvironmentManager is no longer used - self.environment_manager.close() - self.environment_manager = None - - # Prediction - else: - if self.dataset_manager is not None and not self.dataset_manager.new_dataset(): - # Get data from dataset - data = self.dataset_manager.get_data(batch_size=1, get_inputs=True, get_outputs=True) - if self.environment_manager is not None: - new_data = self.environment_manager.dispatch_batch(batch=data, animate=animate) - else: - new_data = data - if len(new_data['input']) != 0: - data['input'] = new_data['input'] - if len(new_data['output']) != 0: - data['output'] = new_data['output'] - if 'loss' in new_data: - data['loss'] = new_data['loss'] - else: - # Get data from environment - data = self.environment_manager.get_data(animate=animate, get_inputs=True, get_outputs=True) - # Record data - if self.dataset_manager is not None: - self.dataset_manager.add_data(data) - - self.data = data - return data - - def get_prediction(self, - network_input: ndarray) -> ndarray: - """ - Get a Network prediction from an input array. Normalization is applied on input and prediction. - - :param network_input: Input array of the Network. - :return: Network prediction. - """ - - # Apply normalization - network_input = self.normalize_data(network_input, 'input') - # Get a prediction - prediction = self.manager.network_manager.compute_online_prediction(network_input=network_input) - # Unapply normalization on prediction - return self.normalize_data(prediction, 'output', reverse=True) - - def apply_prediction(self, - prediction: ndarray) -> None: - """ - Apply the Network prediction in the Environment. - - :param prediction: Prediction of the Network to apply. - """ - - if self.environment_manager is not None: - # Unapply normalization on prediction - prediction = self.normalize_data(prediction, 'output', reverse=True) - # Apply prediction - self.environment_manager.environment.apply_prediction(prediction) - - def normalize_data(self, - data: ndarray, - field: str, - reverse: bool = False) -> ndarray: - """ - Apply or unapply normalization following current standard score. - - :param data: Data to normalize. - :param field: Specify if data is an 'input' or an 'output'. - :param reverse: If False, apply normalization; if False, unapply normalization. - :return: Data with applied or misapplied normalization. - """ - - if not reverse: - # Apply normalization - return (data - self.normalization[field][0]) / self.normalization[field][1] - # Unapply normalization - return (data * self.normalization[field][1]) + self.normalization[field][0] - - def close(self) -> None: - """ - Launch the closing procedure of Managers. - """ - - if self.environment_manager is not None: - self.environment_manager.close() - if self.dataset_manager is not None: - self.dataset_manager.close() - - def __str__(self) -> str: - """ - :return: A string containing valuable information about the DataManager - """ - - data_manager_str = "" - if self.environment_manager: - data_manager_str += str(self.environment_manager) - if self.dataset_manager: - data_manager_str += str(self.dataset_manager) - return data_manager_str diff --git a/src/Manager/DatasetManager.py b/src/Manager/DatasetManager.py deleted file mode 100644 index 048512a9..00000000 --- a/src/Manager/DatasetManager.py +++ /dev/null @@ -1,749 +0,0 @@ -from typing import Any, Dict, Tuple, List, Optional, Union -from os.path import join as osPathJoin -from os.path import isfile, isdir, abspath -from os import listdir, symlink, sep -from json import dump as json_dump -from json import load as json_load -from numpy import load, squeeze, ndarray, concatenate, float64 - -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Utils.pathUtils import create_dir -from DeepPhysX.Core.Utils.jsonUtils import CustomJSONEncoder - - -class DatasetManager: - - def __init__(self, - dataset_config: BaseDatasetConfig, - session: str, - data_manager: Optional[Any] = None, - new_session: bool = True, - training: bool = True, - offline: bool = False, - store_data: bool = True): - """ - | DatasetManager handle all operations with input / output files. Allows saving and read tensors from files. - - :param dataset_config: Specialisation containing the parameters of the dataset manager - :param data_manager: DataManager that handles the DatasetManager - :param new_session: Define the creation of new directories to store data - :param training: True if this session is a network training - :param offline: True if the session is done offline - :param store_data: Format {\'in\': bool, \'out\': bool} save the tensor when bool is True - """ - - self.name: str = self.__class__.__name__ - self.data_manager: Optional[Any] = data_manager - - # Checking arguments - if dataset_config is not None and not isinstance(dataset_config, BaseDatasetConfig): - raise TypeError(f"[{self.name}] The dataset config must be a BaseDatasetConfig object.") - if type(session) != str: - raise TypeError(f"[{self.name}] The session name must be a str.") - elif not isdir(session): - raise ValueError(f"[{self.name}] Given 'session' does not exists: {session}") - if type(new_session) != bool: - raise TypeError(f"[{self.name}] The 'new_network' argument must be a boolean.") - if type(training) != bool: - raise TypeError(f"[{self.name}] The 'train' argument must be a boolean.") - if type(store_data) != bool: - raise TypeError(f"[{self.name}] The 'store_data' argument must be a boolean.") - - # Create the Dataset object (default if no config specified) - dataset_config = BaseDatasetConfig() if dataset_config is None else dataset_config - self.dataset = dataset_config.create_dataset() - - # Dataset parameters - self.max_size: int = self.dataset.max_size - self.shuffle_dataset: bool = dataset_config.shuffle_dataset - self.record_data: bool = store_data - self.first_add: bool = True - self.__writing: bool = False - self.normalize: bool = dataset_config.normalize - self.normalization_security = dataset_config.recompute_normalization - self.offline = offline - - # Dataset modes - self.modes: Dict[str, int] = {'Training': 0, 'Validation': 1, 'Running': 2} - self.mode: int = self.modes['Training'] if training else self.modes['Running'] - self.mode = self.mode if dataset_config.use_mode is None else self.modes[dataset_config.use_mode] - self.last_loaded_dataset_mode: int = self.mode - - # Dataset partitions - session_name = session.split(sep)[-1] - self.partitions_templates: Tuple[str, str, str] = (session_name + '_training_{}_{}.npy', - session_name + '_validation_{}_{}.npy', - session_name + '_running_{}_{}.npy') - self.fields: List[str] = ['input', 'output'] - self.list_partitions: Dict[str, Optional[List[List[ndarray]]]] = { - 'input': [[], [], []] if self.record_data else None, - 'output': [[], [], []] if self.record_data else None} - self.idx_partitions: List[int] = [0, 0, 0] - self.current_partition_path: Dict[str, Optional[str]] = {'input': None, 'output': None} - - # Dataset loading with multiple partitions variables - self.mul_part_list_path: Optional[List[Dict[str, str]]] = None - self.mul_part_slices: Optional[List[List[int]]] = None - self.mul_part_idx: Optional[int] = None - - # Dataset Json file - self.json_filename: str = 'dataset.json' - self.json_empty: Dict[str, Dict[str, Union[List[int], Dict[Any, Any]]]] = {'data_shape': {}, - 'nb_samples': {mode: [] for mode in - self.modes}, - 'partitions': {mode: {} for mode in - self.modes}, - 'normalization': {}} - self.json_dict: Dict[str, Dict[str, Union[List[int], Dict[Any, Any]]]] = self.json_empty.copy() - self.json_found: bool = False - - # Dataset repository - self.session: str = session - dataset_dir: str = dataset_config.dataset_dir - self.new_session: bool = new_session - self.__new_dataset: bool = False - - # Training - if training: - # New training session - if new_session: - # New training session with new dataset - if dataset_dir is None: - self.dataset_dir: str = create_dir(dir_path=osPathJoin(self.session, 'dataset/'), - dir_name='dataset') - self.__new_dataset = True - # New training session with existing dataset - else: - if dataset_dir[-1] != "/": - dataset_dir += "/" - if dataset_dir[-8:] != "dataset/": - dataset_dir += "dataset/" - self.dataset_dir = dataset_dir - if abspath(self.dataset_dir) != osPathJoin(self.session, 'dataset'): - symlink(abspath(self.dataset_dir), osPathJoin(self.session, 'dataset')) - self.load_directory() - # Special case: adding data in existing Dataset with DataGeneration - else: - self.load_directory(load_data=False) - self.__new_dataset = True - # Existing training session - else: - self.dataset_dir = osPathJoin(self.session, 'dataset/') - self.load_directory() - # Prediction - else: - # Saving running data - if dataset_dir is None: - self.dataset_dir = osPathJoin(self.session, 'dataset/') - self.__new_dataset = True - # self.create_running_partitions() - self.load_directory(load_data=False) - # Loading partitions - else: - if dataset_dir[-1] != "/": - dataset_dir += "/" - if dataset_dir[-8:] != "dataset/": - dataset_dir += "dataset/" - self.dataset_dir = dataset_dir - self.load_directory() - - def get_data_manager(self) -> Any: - """ - | Return the Manager of the DataManager. - - :return: DataManager that handle The DatasetManager - """ - - return self.data_manager - - def add_data(self, data: Dict[str, Union[ndarray, Dict[str, ndarray]]]) -> None: - """ - | Push the data in the dataset. If max size is reached generate a new partition and write into it. - - :param data: Format {'input':numpy.ndarray, 'output':numpy.ndarray} contain in 'input' input tensors and - in 'output' output tensors - :type data: Dict[str, Union[ndarray, Dict[str, ndarray]]] - """ - - # 1. If first add, create first partitions - if self.first_add: - self.__writing = True - if 'additional_fields' in data: - self.register_new_fields(list(data['additional_fields'].keys())) - self.create_partitions() - - # 2. Add network data - for field in ['input', 'output']: - if self.record_data: - self.dataset.add(field, data[field], self.current_partition_path[field]) - - # 3. Add additional data - # 3.1 If there is additional data, convert field names then add each field - if 'additional_fields' in data.keys(): - # Check all registered are in additional data - for field in self.fields[2:]: - if field not in data['additional_fields']: - raise ValueError(f"[{self.name}] No data received for the additional field {field}.") - # Add each field to the dataset - for field in data['additional_fields']: - self.dataset.add(field, data['additional_fields'][field], self.current_partition_path[field]) - # 3.2 If there is no additional data but registered additional data - elif 'additional_fields' not in data.keys() and len(self.fields) > 2: - raise ValueError(f"[{self.name}] No data received for the additional fields {self.fields[:2]}") - - # 4. Update json file - self.update_json(update_nb_samples=True) - if self.first_add: - self.update_json(update_partitions_lists=True, update_shapes=True) - self.first_add = False - if self.normalize and not self.offline and self.mode == 0: - self.update_json(update_normalization=True) - - # 5. Check the size of the dataset - if self.dataset.memory_size() > self.max_size: - self.save_data() - self.create_partitions() - self.update_json(update_partitions_lists=True, update_nb_samples=True) - self.dataset.empty() - - def get_data(self, get_inputs: bool, get_outputs: bool, batch_size: int = 1, - batched: bool = True) -> Dict[str, ndarray]: - """ - | Fetch tensors from the dataset or reload partitions if dataset is empty or specified. - - :param bool get_inputs: If True fill the data['input'] field - :param bool get_outputs: If True fill the data['output'] field - :param int batch_size: Size of a batch - :param bool batched: Add an empty dimension before [4,100] -> [0,4,100] - - :return: Dict of format {'input':numpy.ndarray, 'output':numpy.ndarray} filled with desired data - """ - - # Do not allow overwriting partitions - self.__writing = False - - # 1. Check if a dataset is loaded and if the current sample is not the last - if self.current_partition_path['input'] is None or self.dataset.current_sample >= self.dataset.nb_samples: - # if not force_partition_reload: - # return None - self.load_partitions() - if self.shuffle_dataset: - self.dataset.shuffle() - self.dataset.current_sample = 0 - - # 2. Update dataset indices with batch size - idx = self.dataset.current_sample - self.dataset.current_sample += batch_size - - # 3. Get a batch of each data field - data = {} - fields = self.fields[2:] - fields += ['input'] if get_inputs else [] - fields += ['output'] if get_outputs else [] - for field in fields: - # Network input and output fields - if field in ['input', 'output']: - data[field] = self.dataset.get(field, idx, idx + batch_size) - if not batched: - data[field] = squeeze(data[field], axis=0) - # Additional data fields - else: - if 'additional_fields' not in data.keys(): - data['additional_fields'] = {} - data['additional_fields'][field] = self.dataset.get(field, idx, idx + batch_size) - if not batched: - data['additional_fields'][field] = squeeze(data['additional_fields'][field], axis=0) - - # 4. Ensure each field received the same batch size - if data['input'].shape[0] != data['output'].shape[0]: - raise ValueError(f"[{self.name}] Size of loaded batch mismatch for input and output " - f"(in: {data['input'].shape} / out: {data['output'].shape}") - if 'additional_data' in data.keys(): - for field in data['additional_fields'].keys(): - if data['additional_fields'][field].shape[0] != data['input'].shape[0]: - raise ValueError(f"[{self.name}] Size of loaded batch mismatch for additional field {field} " - f"(net: {data['input'].shape} / {field}: {data['additional_fields'][field].shape}") - - # 5. Ensure the batch has the good size, otherwise load new data to complete it - if data['input'].shape[0] < batch_size: - # Load next samples from the dataset - self.load_partitions() - if self.shuffle_dataset: - self.dataset.shuffle() - self.dataset.current_sample = 0 - # Get the remaining samples - missing_samples = batch_size - data['input'].shape[0] - missing_data = self.get_data(get_inputs=get_inputs, get_outputs=get_outputs, batch_size=missing_samples) - # Merge fields - for field in ['input', 'output']: - data[field] = concatenate((data[field], missing_data[field])) - if 'additional_fields' in data.keys(): - for field in data['additional_fields'].keys(): - data['additional_fields'][field] = concatenate((data['additional_fields'][field], - missing_data['additional_fields'][field])) - return data - - def register_new_fields(self, new_fields: List[str]) -> None: - """ - | Add new data fields in the dataset. - - :param List[str] new_fields: Name of the new fields split in either 'IN' side or 'OUT' side of the dataset - """ - - for field in new_fields: - self.register_new_field(field) - - def register_new_field(self, new_field: str) -> None: - """ - | Add a new data field in the dataset. - - :param str new_field: Name of the new field. - """ - - if new_field not in self.fields or self.list_partitions[new_field] is None: - self.fields.append(new_field) - self.list_partitions[new_field] = [[], [], []] - - def create_partitions(self) -> None: - """ - | Create a new partition for current mode and for each registered fields. - """ - - print(f"[{self.name}] New partitions added for each field with max size ~{float(self.max_size) / 1e9}Gb.") - for field in self.fields: - if self.record_data: - # Fill partition name template - if field in ['input', 'output']: - name = 'IN' if field == 'input' else 'OUT' - else: - name = 'ADD_' + field - partition_path = self.partitions_templates[self.mode].format(name, self.idx_partitions[self.mode]) - # Make it current partition - self.list_partitions[field][self.mode].append(partition_path) - self.current_partition_path[field] = self.dataset_dir + partition_path - # Index partitions - self.idx_partitions[self.mode] += 1 - - def create_running_partitions(self) -> None: - """ - | Run specific function. Handle partitions creation when not training. - """ - - # 1. Load the directory without loading data - self.load_directory(load_data=False) - - # 2. Find out how many partitions exists for running mode - mode = list(self.modes.keys())[self.mode] - partitions_dict = self.json_dict['partitions'][mode] - nb_running_partitions = max([len(partitions_dict[field]) for field in partitions_dict.keys()]) - - # 3. Create a new partition partitions - self.idx_partitions[self.mode] = nb_running_partitions - self.create_partitions() - - def load_directory(self, load_data: bool = True) -> None: - """ - | Load the desired directory. Try to find partition list and upload it. - | No data loading here. - """ - - # 1. Check the directory exists - if not isdir(self.dataset_dir): - raise Warning(f"[{self.name}] Loading directory: The given path is not an existing directory") - if load_data: - print(f"[{self.name}] Loading directory: Read dataset from {self.dataset_dir}") - - # 2. Look for the json info file - if isfile(osPathJoin(self.dataset_dir, self.json_filename)): - self.json_found = True - with open(osPathJoin(self.dataset_dir, self.json_filename)) as json_file: - self.json_dict = json_load(json_file) - - # 3. Load partitions for each mode - for mode in self.modes: - # 3.1. Get sorted partitions - partitions_dict = self.json_dict['partitions'][mode] if self.json_found else self.search_partitions(mode) - # 3.2. Register additional fields - for field in partitions_dict: - self.register_new_field(field) - # 3.3. Register each partition - for field in partitions_dict: - self.list_partitions[field][self.modes[mode]] = partitions_dict[field] - # 3.4. Check that the number of partitions is the same for each field - number_of_partitions = len(self.list_partitions[self.fields[0]][self.modes[mode]]) - for field in self.fields: - if len(self.list_partitions[field][self.modes[mode]]) != number_of_partitions: - raise ValueError(f"[{self.name}] The number of partitions is different for {field} with " - f"{len(self.list_partitions[field][self.modes[mode]])} partitions found.") - - # 4. Update Json file if not found - if not self.json_found or self.empty_json_fields(): - self.search_partitions_info() - self.update_json(update_partitions_lists=True) - if self.normalization_security or (self.normalize and self.json_dict['normalization'] == self.json_empty['normalization']): - self.update_json(update_normalization=True) - - # 4. Load data from partitions - self.idx_partitions = [len(partitions_list) for partitions_list in self.list_partitions['input']] - if load_data: - self.load_partitions(force_reload=True) - - def search_partitions(self, mode: str) -> Dict[str, List[str]]: - """ - | If loading a directory without JSON info file, search for existing partitions manually. - - :param str mode: Mode of the partitions to find. - :return: Dictionary with found partitions for specified mode - """ - - # 1. Get all the partitions for the mode - partitions_dict = {} - partitions_list = [f for f in listdir(self.dataset_dir) if isfile(osPathJoin(self.dataset_dir, f)) - and f.endswith('.npy') and f.__contains__(mode.lower())] - - # 2. Sort partitions by side (IN, OUT, ADD) and by name - in_partitions = sorted([file for file in partitions_list if file.__contains__('_IN_')]) - out_partitions = sorted([file for file in partitions_list if file.__contains__('_OUT_')]) - add_partitions = sorted([file for file in partitions_list if file.__contains__('_ADD_')]) - - # 3. Sort partitions by data field - for side, partitions, field_name in zip(['IN', 'OUT', 'ADD'], [in_partitions, out_partitions, add_partitions], - ['input', 'output', '']): - for partition in partitions: - # Extract information from the filenames - split_name = partition.split('_') - clues = split_name[split_name.index(side):] - # Partition name for network data: {NAME_OF_SESSION}_SIDE_IDX.npy - if len(clues) == 2: - if field_name not in partitions_dict.keys(): - partitions_dict[field_name] = [] - partitions_dict[field_name].append(partition) - # Partition name for additional data: {NAME_OF_SESSION}_SIDE_{NAME_OF_FIELD}_IDX.npy - else: - field_name = '_'.join(clues[:-1]) - if field_name not in partitions_dict.keys(): - partitions_dict[field_name] = [] - partitions_dict[field_name].append(partition) - - return partitions_dict - - def search_partitions_info(self) -> None: - """ - | If loading a directory without JSON info file. - """ - - # 1. Get the shape of each partition - partition_shapes = [{field: [] for field in self.fields} for _ in self.modes] - for mode in self.modes: - for field in self.fields: - for partition in [self.dataset_dir + path for path in self.list_partitions[field][self.modes[mode]]]: - partition_data = load(partition) - partition_shapes[self.modes[mode]][field].append(partition_data.shape) - del partition_data - - # 2. Get the number of samples per partition for each mode - for mode in self.modes: - number_of_samples = {} - for field in self.fields: - number_of_samples[field] = [shape[0] for shape in partition_shapes[self.modes[mode]][field]] - # Number of samples through partitions must be the same along fields - if number_of_samples[field] != list(number_of_samples.values())[0]: - raise ValueError(f"[{self.name}] The number of sample in each partition is not consistent:\n" - f"{number_of_samples}") - # Store the number of samples per partition for the mode - self.json_dict['nb_samples'][mode] = list(number_of_samples.values())[0] - - # 3. Get the data shape for each field - data_shape = {field: [] for field in self.fields} - for mode in self.modes: - for field in self.fields: - for i, shape in enumerate(partition_shapes[self.modes[mode]][field]): - if len(data_shape[field]) == 0: - data_shape[field] = shape[1:] - # Data shape must be the same along partitions and mode - if shape[1:] != data_shape[field]: - raise ValueError(f"[{self.name}] Two different data sizes found for mode {mode}, field {field}," - f" partition n°{i}: {data_shape[field]} vs {shape[1:]}") - # Store the data shapes - self.json_dict['data_shape'] = data_shape - - def load_partitions(self, force_reload: bool = False) -> None: - """ - | Load data from partitions. - - :param bool force_reload: If True, force partitions reload - """ - - # 1. If there is only one partition for the current mode for input field at least, don't need to reload it - if self.last_loaded_dataset_mode == self.mode and self.idx_partitions[self.mode] == 1 and not force_reload: - if self.shuffle_dataset: - self.dataset.shuffle() - return - - # 2. Check partitions existence for the current mode - if self.idx_partitions[self.mode] == 0: - raise ValueError(f"[{self.name}] No partitions to read for {list(self.modes.keys())[self.mode]} mode.") - - # 3. Load new data in dataset - self.dataset.empty() - # Training mode with mixed dataset: read multiple partitions per field - if self.mode == self.modes['Training'] and self.idx_partitions[self.modes['Running']] > 0: - if self.mul_part_idx is None: - self.load_multiple_partitions([self.modes['Training'], self.modes['Running']]) - self.read_multiple_partitions() - return - # Training mode without mixed dataset or other modes: check the number of partitions per field to read - if self.idx_partitions[self.mode] == 1: - self.read_last_partitions() - else: - if self.mul_part_idx is None: - self.load_multiple_partitions([self.mode]) - self.read_multiple_partitions() - - def read_last_partitions(self) -> None: - """ - | Load the last loaded partitions for each data field. - """ - - for field in self.fields: - self.current_partition_path[field] = self.dataset_dir + self.list_partitions[field][self.mode][-1] - data = load(self.current_partition_path[field]) - self.dataset.set(field, data) - - def load_multiple_partitions(self, modes: List[int]) -> None: - """ - | Specialisation of the load_partitions() function. It can load a list of partitions. - - :param List[int] modes: Recommended to use modes['name_of_desired_mode'] in order to correctly load the dataset - """ - - # 1. Initialize multiple partition loading variables - self.mul_part_list_path = {field: [] for field in self.fields} - self.mul_part_slices = [] - self.mul_part_idx = 0 - nb_sample_per_partition = {field: [] for field in self.fields} - - # 2. For each field, load all partitions - for field in self.fields: - for mode in modes: - # 2.1. Add partitions to the list of partitions to read - self.mul_part_list_path[field] += [self.dataset_dir + partition - for partition in self.list_partitions[field][mode]] - # 2.2. Find the number of samples in each partition - nb_sample_per_partition[field] += self.json_dict['nb_samples'][list(self.modes.keys())[mode]] - - # 3. Invert the partitions list structure - nb_partition = len(nb_sample_per_partition[self.fields[0]]) - inverted_list = [{} for _ in range(nb_partition)] - for i in range(nb_partition): - for field in self.fields: - inverted_list[i][field] = self.mul_part_list_path[field][i] - self.mul_part_list_path = inverted_list - - # 4. Define the slicing pattern of reading for partitions - for idx in nb_sample_per_partition[self.fields[0]]: - idx_slicing = [0] - for _ in range(nb_partition - 1): - idx_slicing.append(idx_slicing[-1] + idx // nb_partition + 1) - idx_slicing.append(idx) - self.mul_part_slices.append(idx_slicing) - - def read_multiple_partitions(self) -> None: - """ - | Read data in a list of partitions. - """ - - for i, partitions in enumerate(self.mul_part_list_path): - for field in partitions.keys(): - dataset = load(partitions[field]) - samples = slice(self.mul_part_slices[i][self.mul_part_idx], - self.mul_part_slices[i][self.mul_part_idx + 1]) - self.dataset.add(field, dataset[samples]) - del dataset - self.mul_part_idx = (self.mul_part_idx + 1) % (len(self.mul_part_slices[0]) - 1) - self.current_partition_path['input'] = self.mul_part_list_path[0][self.fields[0]] - self.dataset.current_sample = 0 - - def update_json(self, update_shapes: bool = False, update_nb_samples: bool = False, - update_partitions_lists: bool = False, update_normalization: bool = False) -> None: - """ - | Update the json info file with the current Dataset repository information. - - :param bool update_shapes: If True, data shapes per field are overwritten - :param bool update_nb_samples: If True, number of samples per partition are overwritten - :param bool update_partitions_lists: If True, list of partitions is overwritten - :param bool update_normalization: If True, compute and save current normalization coefficients - """ - - # Update data shapes - if update_shapes: - for field in self.fields: - self.json_dict['data_shape'][field] = self.dataset.get_data_shape(field) - - # Update number of samples - if update_nb_samples: - idx_mode = list(self.modes.keys())[self.mode] - if len(self.json_dict['nb_samples'][idx_mode]) == self.idx_partitions[self.mode]: - self.json_dict['nb_samples'][idx_mode][-1] = self.dataset.current_sample - else: - self.json_dict['nb_samples'][idx_mode].append(self.dataset.current_sample) - - # Update partitions lists - if update_partitions_lists: - for mode in self.modes: - for field in self.fields: - self.json_dict['partitions'][mode][field] = self.list_partitions[field][self.modes[mode]] - - # Update normalization coefficients - if update_normalization: - for field in ['input', 'output']: - # Normalization is only done on training data (mode = 0) - if len(self.list_partitions[field][0]) > 0: - self.json_dict['normalization'][field] = self.compute_normalization(field) - - # Overwrite json file - with open(self.dataset_dir + self.json_filename, 'w') as json_file: - json_dump(self.json_dict, json_file, indent=3, cls=CustomJSONEncoder) - - def empty_json_fields(self) -> bool: - """ - | Check if the json info file contains empty fields. - - :return: The json file contains empty fields or not - """ - - for key in self.json_empty: - if self.json_dict[key] == self.json_empty[key]: - return True - return False - - def save_data(self) -> None: - """ - | Close all open files - """ - - if self.__new_dataset: - for field in self.current_partition_path.keys(): - self.dataset.save(field, self.current_partition_path[field]) - - def set_mode(self, mode: int) -> None: - """ - | Set the DatasetManager working mode. - - :param int mode: Recommended to use modes['name_of_desired_mode'] in order to correctly set up the - DatasetManager - """ - - # Nothing has to be done if you do not change mode - if mode == self.mode: - return - if self.mode == self.modes['Running']: - print(f"[{self.name}] It's not possible to switch dataset mode while running.") - else: - # Save dataset before changing mode - self.save_data() - self.mode = mode - self.dataset.empty() - # Create or load partition for the new mode - if self.idx_partitions[self.mode] == 0: - print(f"[{self.name}] Change to {self.mode} mode, create a new partition") - self.create_partitions() - else: - print(f"[{self.name}] Change to {self.mode} mode, load last partition") - self.read_last_partitions() - - def compute_normalization(self, field: str) -> List[float]: - """ - | Compute the normalization coefficients (mean & standard deviation) of input and output data fields. - | Compute the normalization on the whole set of partitions for a given data field. - - :param str field: Field for which normalization coefficients should be computed - :return: List containing normalization coefficients (mean, standard deviation) - """ - - partitions_content = [0., 0.] - for i, partition in enumerate(self.list_partitions[field][0]): - loaded = load(self.dataset_dir + partition).astype(float64) - partitions_content[0] += loaded.mean() #Computing the mean - partitions_content[1] += (loaded**2).mean() #Computing the variance - if len(self.list_partitions[field][0]) > 0: - partitions_content[0] /= len(self.list_partitions[field][0]) #Computing the global mean - # Computing the global std - partitions_content[1] = (partitions_content[1] / len(self.list_partitions[field][0]) - partitions_content[0]**2)**(0.5) - if self.data_manager is not None: - self.data_manager.normalization[field] = partitions_content - return partitions_content - - def new_dataset(self) -> bool: - """ - | Check if the Dataset is a new entity or not. - - :return: The Dataset is new or not - """ - - return self.__new_dataset - - def get_next_batch(self, batch_size: int) -> Dict[str, ndarray]: - """ - | Specialization of get_data to get a batch. - - :param int batch_size: Size of the batch - :return: Batch with format {'input': numpy.ndarray, 'output': numpy.ndarray} - """ - - return self.get_data(get_inputs=True, get_outputs=True, batch_size=batch_size, batched=True) - - def get_next_sample(self, batched: bool = True) -> Dict[str, ndarray]: - """ - | Specialization of get_data to get a sample. - - :param batched: If True, return the sample as a batch with batch_size = 1 - :return: Sample with format {'input': numpy.ndarray, 'output': numpy.ndarray} - """ - - return self.get_data(get_inputs=True, get_outputs=True, batched=batched) - - def get_next_input(self, batched: bool = False) -> Dict[str, ndarray]: - """ - | Specialization of get_data to get an input sample. - - :param batched: If True, return the sample as a batch with batch_size = 1 - :return: Sample with format {'input': numpy.ndarray, 'output': numpy.ndarray} where only the input field is - filled - """ - - return self.get_data(get_inputs=True, get_outputs=False, batched=batched) - - def getNextOutput(self, batched: bool = False) -> Dict[str, ndarray]: - """ - | Specialization of get_data to get an output sample. - - :param batched: If True, return the sample as a batch with batch_size = 1 - :return: Sample with format {'input': numpy.ndarray, 'output': numpy.ndarray} where only the output field is - filled - """ - - return self.get_data(get_inputs=False, get_outputs=True, batched=batched) - - def close(self) -> None: - """ - | Launch the close procedure of the dataset manager - """ - - if self.__writing: - self.save_data() - if self.offline and self.normalize: - self.update_json(update_normalization=True) - - def __str__(self) -> str: - """ - :return: A string containing valuable information about the DatasetManager - """ - - description = "\n" - description += f"# {self.name}\n" - description += f" Dataset Repository: {self.dataset_dir}\n" - description += f" Partitions size: {self.max_size * 1e-9} Go\n" - description += f" Managed objects: Dataset: {self.dataset.name}\n" - description += str(self.dataset) - return description diff --git a/src/Manager/EnvironmentManager.py b/src/Manager/EnvironmentManager.py deleted file mode 100644 index f8c93c5e..00000000 --- a/src/Manager/EnvironmentManager.py +++ /dev/null @@ -1,266 +0,0 @@ -from typing import Any, Dict, Optional, Union -from numpy import array, ndarray -from asyncio import run as async_run -from copy import copy -from os.path import join - -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig, TcpIpServer, BaseEnvironment -from DeepPhysX.Core.Visualization.VedoVisualizer import VedoVisualizer - - -class EnvironmentManager: - - def __init__(self, - environment_config: BaseEnvironmentConfig, - session: str, - data_manager: Any = None, - batch_size: int = 1, - training: bool = True): - """ - Deals with the online generation of data for both training and running of the neural networks. - - :param environment_config: Specialisation containing the parameters of the environment manager. - :param session: Path to the session directory. - :param data_manager: DataManager that handles the EnvironmentManager. - :param batch_size: Number of samples in a batch of data. - :param training: True if this session is a network training. - """ - - self.name: str = self.__class__.__name__ - - # Managers architecture - self.data_manager: Any = data_manager - - # Data producing parameters - self.batch_size: int = batch_size - self.always_create_data: bool = environment_config.always_create_data - self.use_dataset_in_environment: bool = environment_config.use_dataset_in_environment - self.simulations_per_step: int = environment_config.simulations_per_step - self.max_wrong_samples_per_step: int = environment_config.max_wrong_samples_per_step - self.train: bool = training - self.dataset_batch: Optional[Dict[str, Dict[int, Any]]] = None - - # Create the Visualizer - self.visualizer: Optional[VedoVisualizer] = None - visu_db = None - if environment_config.visualizer is not None: - self.visualizer = environment_config.visualizer(database_dir=join(session, 'dataset'), - database_name='Visualization', - remote=environment_config.as_tcp_ip_client) - visu_db = self.visualizer.get_database() - - # Create a single Environment or a TcpIpServer - self.number_of_thread: int = environment_config.number_of_thread - self.server: Optional[TcpIpServer] = None - self.environment: Optional[BaseEnvironment] = None - if environment_config.as_tcp_ip_client: - self.server = environment_config.create_server(environment_manager=self, - batch_size=batch_size, - visu_db=visu_db.get_path()) - else: - self.environment = environment_config.create_environment(environment_manager=self, - visu_db=visu_db) - - # Define get_data and dispatch methods - self.get_data = self.get_data_from_server if self.server else self.get_data_from_environment - self.dispatch_batch = self.dispatch_batch_to_server if self.server else self.dispatch_batch_to_environment - - # Init visualizer - if self.visualizer is not None: - if len(self.visualizer.get_database().get_tables()) == 1: - self.visualizer.get_database().load() - self.visualizer.init_visualizer() - - def get_data_manager(self) -> Any: - """ - Get the DataManager of this EnvironmentManager. - - :return: The DataManager of this EnvironmentManager. - """ - - return self.data_manager - - def get_data_from_server(self, - get_inputs: bool = True, - get_outputs: bool = True, - animate: bool = True) -> Dict[str, Union[ndarray, dict]]: - """ - Compute a batch of data from Environments requested through TcpIpServer. - - :param get_inputs: If True, compute and return input. - :param get_outputs: If True, compute and return output. - :param animate: If True, triggers an environment step. - :return: Dictionary containing all labeled data sent by the clients in their own dictionary + in and out key - corresponding to the batch. - """ - - # Get data from server - batch = self.server.get_batch(get_inputs, get_outputs, animate) - # Filter input and output - training_data = {'input': array(batch['input']) if get_inputs else array([]), - 'output': array(batch['output']) if get_outputs else array([])} - # Convert each additional field - for field in batch['additional_fields']: - batch['additional_fields'][field] = array(batch['additional_fields'][field]) - training_data['additional_fields'] = batch['additional_fields'] - # Convert loss data - if 'loss' in batch and len(batch['loss']) != 0: - training_data['loss'] = array(batch['loss']) - # Return batch - return training_data - - def get_data_from_environment(self, - get_inputs: bool = True, - get_outputs: bool = True, - animate: bool = True) -> Dict[str, Union[ndarray, dict]]: - """ - Compute a batch of data directly from Environment. - - :param get_inputs: If True, compute and return input. - :param get_outputs: If True, compute and return output. - :param animate: If True, triggers an environment step. - :return: Dictionary containing all labeled data sent by the clients in their own dictionary + in and out key - corresponding to the batch. - """ - - # Init training data container, define production conditions - input_condition = lambda x: len(x) < self.batch_size if get_inputs else lambda _: False - output_condition = lambda x: len(x) < self.batch_size if get_outputs else lambda _: False - training_data = {'input': [], 'output': []} - - # 1. Produce batch while batch size is not complete - while input_condition(training_data['input']) and output_condition(training_data['output']): - - # 1.1 Send a sample if a batch from dataset is given - if self.dataset_batch is not None: - # Extract a sample from dataset batch: input - self.environment.sample_in = self.dataset_batch['input'][0] - self.dataset_batch['input'] = self.dataset_batch['input'][1:] - # Extract a sample from dataset batch: output - self.environment.sample_out = self.dataset_batch['output'][0] - self.dataset_batch['output'] = self.dataset_batch['output'][1:] - # Extract a sample from dataset batch: additional fields - additional_fields = {} - if 'additional_fields' in self.dataset_batch: - for field in self.dataset_batch['additional_fields']: - additional_fields[field] = self.dataset_batch['additional_fields'][field][0] - self.dataset_batch['additional_fields'][field] = self.dataset_batch['additional_fields'][field][1:] - self.environment.additional_fields = additional_fields - - # 1.2 Run the defined number of step - if animate: - for current_step in range(self.simulations_per_step): - # Sub-steps do not produce data - self.environment.compute_essential_data = current_step == self.simulations_per_step - 1 - async_run(self.environment.step()) - - # 1.3 Add the produced sample to the batch if the sample is validated - if self.environment.check_sample(): - # Network's input - if get_inputs: - training_data['input'].append(self.environment.input) - self.environment.input = array([]) - # Network's output - if get_outputs: - training_data['output'].append(self.environment.output) - self.environment.output = array([]) - # Check if there is loss data - if self.environment.loss_data: - if 'loss' not in training_data: - training_data['loss'] = [] - training_data['loss'].append(self.environment.loss_data) - self.environment.loss_data = None - # Check if there is additional dataset fields - if self.environment.additional_fields != {}: - if 'additional_fields' not in training_data: - training_data['additional_fields'] = {} - for field in self.environment.additional_fields: - if field not in training_data['additional_fields']: - training_data['additional_fields'][field] = [] - training_data['additional_fields'][field].append(self.environment.additional_fields[field]) - self.environment.additional_fields = {} - - # 2. Convert data in ndarray - for key in training_data: - # If key does not contain a dict, convert value directly - if key != 'additional_fields': - training_data[key] = array(training_data[key]) - # If key contains a dict, convert item by item - else: - for field in training_data[key]: - training_data[key][field] = array(training_data[key][field]) - - return training_data - - def dispatch_batch_to_server(self, - batch: Dict[str, Union[ndarray, dict]], - animate: bool = True) -> Dict[str, Union[ndarray, dict]]: - """ - Send samples from dataset to the Environments. Get back the training data. - - :param batch: Batch of samples. - :param animate: If True, triggers an environment step. - :return: Batch of training data. - """ - - # Define the batch to dispatch - self.server.set_dataset_batch(batch) - # Empty the server queue - while not self.server.data_fifo.empty(): - self.server.data_fifo.get() - # Get data - return self.get_data(animate=animate) - - def dispatch_batch_to_environment(self, - batch: Dict[str, Union[ndarray, dict]], - animate: bool = True) -> Dict[str, Union[ndarray, dict]]: - """ - Send samples from dataset to the Environments. Get back the training data. - - :param batch: Batch of samples. - :param animate: If True, triggers an environment step. - :return: Batch of training data. - """ - - # Define the batch to dispatch - self.dataset_batch = copy(batch) - # Get data - return self.get_data(animate=animate) - - def update_visualizer(self, - instance: int) -> None: - """ - Update the Visualizer. - - :param instance: Index of the Environment render to update. - """ - - if self.visualizer is not None: - self.visualizer.render_instance(instance) - - def close(self) -> None: - """ - Close the environment - """ - - # Server case - if self.server: - self.server.close() - # No server case - if self.environment: - self.environment.close() - - def __str__(self) -> str: - """ - :return: A string containing valuable information about the EnvironmentManager - """ - - description = "\n" - description += f"# {self.name}\n" - description += f" Always create data: {self.always_create_data}\n" - # description += f" Record wrong samples: {self.record_wrong_samples}\n" - description += f" Number of threads: {self.number_of_thread}\n" - # description += f" Managed objects: Environment: {self.environment.env_name}\n" - # Todo: manage the print log of each Environment since they can have different parameters - # description += str(self.environment) - return description diff --git a/src/Manager/Manager.py b/src/Manager/Manager.py deleted file mode 100644 index fbbfb1a3..00000000 --- a/src/Manager/Manager.py +++ /dev/null @@ -1,160 +0,0 @@ -from typing import Any, Dict, Tuple, Optional -from os.path import join as osPathJoin -from os.path import isfile, basename, exists -from datetime import datetime -from numpy import ndarray - -from DeepPhysX.Core.Manager.DataManager import DataManager -from DeepPhysX.Core.Manager.NetworkManager import NetworkManager -from DeepPhysX.Core.Manager.StatsManager import StatsManager -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Utils.pathUtils import get_first_caller, create_dir - - -class Manager: - - def __init__(self, - network_config: BaseNetworkConfig, - dataset_config: BaseDatasetConfig, - environment_config: BaseEnvironmentConfig, - pipeline: Optional[Any] = None, - session_dir: Optional[str] = None, - session_name: str = 'DPX_default', - new_session: bool = True, - training: bool = True, - store_data: bool = True, - batch_size: int = 1): - """ - Collection of all the specialized managers. Allows for some basic functions call. - More specific behaviour have to be directly call from the corresponding manager. - - :param network_config: Specialisation containing the parameters of the network manager. - :param dataset_config: Specialisation containing the parameters of the dataset manager. - :param environment_config: Specialisation containing the parameters of the environment manager. - :param session_name: Name of the newly created directory if session is not defined. - :param session_dir: Name of the directory in which to write all the necessary data - :param bool new_session: Define the creation of new directories to store data - :param int batch_size: Number of samples in a batch - """ - - self.pipeline: Optional[Any] = pipeline - - # Constructing the session with the provided arguments - if session_name is None: - raise ValueError("[Manager] The session name cannot be set to None (will raise error).") - if session_dir is None: - # Create manager directory from the session name - self.session: str = osPathJoin(get_first_caller(), session_name) - else: - self.session: str = osPathJoin(session_dir, session_name) - - # Trainer: must create a new session to avoid overwriting - if training: - # Avoid unwanted overwritten data - if new_session: - self.session: str = create_dir(self.session, dir_name=session_name) - # Prediction: work in an existing session - else: - if not exists(self.session): - raise ValueError("[Manager] The session directory {} does not exists.".format(self.session)) - - # Always create the NetworkMmanager - self.network_manager = NetworkManager(manager=self, - network_config=network_config, - session=self.session, - new_session=new_session, - training=training) - # Always create the DataManager for same reason - self.data_manager = DataManager(manager=self, - dataset_config=dataset_config, - environment_config=environment_config, - session=self.session, - new_session=new_session, - training=training, - store_data=store_data, - batch_size=batch_size) - # Create the StatsManager for training - self.stats_manager = StatsManager(manager=self, - session=self.session) if training else None - - def get_data(self, epoch: int = 0, batch_size: int = 1, animate: bool = True) -> None: - """ - | Fetch data from the DataManager. - - :param int epoch: Epoch ID - :param int batch_size: Size of a batch - :param bool animate: If True allows running environment step - """ - - self.data_manager.get_data(epoch=epoch, batch_size=batch_size, animate=animate) - - def optimize_network(self) -> Tuple[ndarray, Dict[str, float]]: - """ - | Compute a prediction and run a back propagation with the current batch. - - :return: The network prediction and the associated loss value - """ - - # Normalize input and output data - data = self.data_manager.data - for field in ['input', 'output']: - if field in data: - data[field] = self.data_manager.normalize_data(data[field], field) - # Forward pass and optimization step - prediction, loss_dict = self.network_manager.compute_prediction_and_loss(data, optimize=True) - return prediction, loss_dict - - def save_network(self) -> None: - """ - | Save network weights as a pth file - """ - - self.network_manager.save_network() - - def close(self) -> None: - """ - | Call all managers close procedure - """ - - if self.network_manager is not None: - self.network_manager.close() - if self.stats_manager is not None: - self.stats_manager.close() - if self.data_manager is not None: - self.data_manager.close() - - def save_info_file(self) -> None: - """ - | Called by the Trainer to save a .txt file which provides a quick description template to the user and lists - the description of all the components. - """ - - filename = osPathJoin(self.session, 'infos.txt') - date_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - if not isfile(filename): - f = open(filename, "w+") - # Session description template for user - f.write("## DeepPhysX Training Session ##\n") - f.write(date_time + "\n\n") - f.write("Purpose of the training session:\nNetwork Input:\nNetwork Output:\nComments:\n\n") - # Listing every component descriptions - f.write("## List of Components Parameters ##\n") - f.write(str(self.pipeline)) - f.write(str(self)) - f.close() - - def __str__(self) -> str: - """ - :return: A string containing valuable information about the Managers - """ - - manager_description = "" - if self.network_manager is not None: - manager_description += str(self.network_manager) - if self.data_manager is not None: - manager_description += str(self.data_manager) - if self.stats_manager is not None: - manager_description += str(self.stats_manager) - return manager_description diff --git a/src/Manager/NetworkManager.py b/src/Manager/NetworkManager.py deleted file mode 100644 index 15006278..00000000 --- a/src/Manager/NetworkManager.py +++ /dev/null @@ -1,243 +0,0 @@ -from typing import Any, Dict, Tuple, Optional -from os import listdir, sep -from os.path import join as osPathJoin -from os.path import isdir, isfile -from numpy import copy, array, ndarray - -from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Utils.pathUtils import copy_dir, create_dir - - -class NetworkManager: - - def __init__(self, - network_config: BaseNetworkConfig, - session: str, - manager: Optional[Any] = None, - new_session: bool = True, - training: bool = True): - """ - Deals with all the interactions with the neural network. Predictions, saves, initialisation, loading, - back-propagation, etc... - - :param network_config: Specialisation containing the parameters of the network manager - :param manager: Manager that handle the network manager - :param new_session: Define the creation of new directories to store data - :param training: If True prediction will cause tensors gradient creation - """ - - self.name: str = self.__class__.__name__ - - # Check network_config type - if not isinstance(network_config, BaseNetworkConfig): - raise TypeError(f"[{self.name}] Wrong 'network_config' type: BaseNetworkConfig required, " - f"get {type(network_config)}") - # Check session type and existence - if type(session) != str: - raise TypeError(f"[{self.name}] Wrong 'session' type: str required, get {type(session)}") - if not isdir(session): - raise ValueError(f"[{self.name}] Given 'session' does not exists: {session}") - # Check new_session type - if type(new_session) != bool: - raise TypeError(f"[{self.name}] Wrong 'new_session' type: bool required, get {type(new_session)}") - # Check train type - if type(training) != bool: - raise TypeError(f"[{self.name}] Wrong 'train' type: bool required, get {type(training)}") - - # Storage management - self.session: str = session - self.new_session: bool = new_session - self.network_dir: Optional[str] = None - self.network_template_name: str = session.split(sep)[-1] + '_network_{}' - - # Network management - self.manager: Any = manager - if training and not network_config.training_stuff: - raise ValueError(f"[{self.name}] Training requires a loss and an optimizer in your NetworkConfig") - self.training: bool = training - self.save_each_epoch: bool = network_config.save_each_epoch - self.saved_counter: int = 0 - - # Init network objects: Network, Optimization, DataTransformation - self.network: Any = None - self.optimization: Any = None - self.data_transformation: Any = None - self.network_config: BaseNetworkConfig = network_config - self.set_network() - - def get_manager(self) -> Any: - """ - | Return the Manager of the NetworkManager. - - :return: Manager that handles the NetworkManager - """ - - return self.manager - - def set_network(self) -> None: - """ - | Set the network to the corresponding weight from a given file. - """ - - # Init network - self.network = self.network_config.create_network() - self.network.set_device() - # Init optimization - self.optimization = self.network_config.create_optimization() - self.optimization.manager = self - if self.optimization.loss_class: - self.optimization.set_loss() - - # Init DataTransformation - self.data_transformation = self.network_config.create_data_transformation() - - # Training - if self.training: - # Configure as training - self.network.set_train() - self.optimization.set_optimizer(self.network) - # Setting network directory - if self.new_session and self.network_config.network_dir and isdir(self.network_config.network_dir): - self.network_dir = self.network_config.network_dir - self.network_dir = copy_dir(self.network_dir, self.session, dest_dir='network') - self.load_network() - else: - self.network_dir = osPathJoin(self.session, 'network/') - self.network_dir = create_dir(self.network_dir, dir_name='network') - - # Prediction - else: - # Configure as prediction - self.network.set_eval() - # Need an existing network - self.network_dir = osPathJoin(self.session, 'network/') - # Load parameters - self.load_network() - - def load_network(self) -> None: - """ - | Load an existing set of parameters to the network. - """ - - # Get eventual epoch saved networks - networks_list = [osPathJoin(self.network_dir, f) for f in listdir(self.network_dir) if - isfile(osPathJoin(self.network_dir, f)) and f.__contains__('_network_.')] - networks_list = sorted(networks_list) - # Add the final saved network - last_saved_network = [osPathJoin(self.network_dir, f) for f in listdir(self.network_dir) if - isfile(osPathJoin(self.network_dir, f)) and f.__contains__('network.')] - networks_list = networks_list + last_saved_network - which_network = self.network_config.which_network - if len(networks_list) == 0: - print(f"[{self.name}]: There is no network in {self.network_dir}. Shutting down.") - quit(0) - elif len(networks_list) == 1: - which_network = 0 - elif len(networks_list) > 1 and which_network is None: - print(f"[{self.name}] There is more than one network in this directory, loading the most trained by " - f"default. If you want to load another network please use the 'which_network' variable.") - which_network = -1 - elif which_network > len(networks_list) > 1: - print(f"[{self.name}] The selected network doesn't exist (index is too big), loading the most trained " - f"by default.") - which_network = -1 - print(f"[{self.name}]: Loading network from {networks_list[which_network]}.") - self.network.load_parameters(networks_list[which_network]) - - def compute_prediction_and_loss(self, batch: Dict[str, ndarray], - optimize: bool) -> Tuple[ndarray, Dict[str, float]]: - """ - | Make a prediction with the data passed as argument, optimize or not the network - - :param Dict[str, ndarray] batch: Format {'input': numpy.ndarray, 'output': numpy.ndarray}. - Contains the input value and ground truth to compare against - :param bool optimize: If true run a back propagation - - :return: The prediction and the associated loss value - """ - - # Getting data from the data manager - data_in = self.network.transform_from_numpy(batch['input'], grad=optimize) - data_gt = self.network.transform_from_numpy(batch['output'], grad=optimize) - loss_data = self.network.transform_from_numpy(batch['loss'], grad=False) if 'loss' in batch.keys() else None - - # Compute prediction - data_in = self.data_transformation.transform_before_prediction(data_in) - data_out = self.network.predict(data_in) - - # Compute loss - data_out, data_gt = self.data_transformation.transform_before_loss(data_out, data_gt) - loss_dict = self.optimization.compute_loss(data_out.reshape(data_gt.shape), data_gt, loss_data) - # Optimizing network if training - if optimize: - self.optimization.optimize() - # Transform prediction to be compatible with environment - data_out = self.data_transformation.transform_before_apply(data_out) - prediction = self.network.transform_to_numpy(data_out) - return prediction, loss_dict - - def compute_online_prediction(self, network_input: ndarray) -> ndarray: - """ - | Make a prediction with the data passed as argument. - - :param ndarray network_input: Input of the network= - :return: The prediction - """ - - # Getting data from the data manager - data_in = self.network.transform_from_numpy(copy(network_input), grad=False) - - # Compute prediction - data_in = self.data_transformation.transform_before_prediction(data_in) - pred = self.network.predict(data_in) - pred, _ = self.data_transformation.transform_before_loss(pred) - pred = self.data_transformation.transform_before_apply(pred) - pred = self.network.transform_to_numpy(pred) - return pred.reshape(-1) - - def save_network(self, last_save: bool = False) -> None: - """ - | Save the network with the corresponding suffix, so they do not erase the last save. - - :param bool last_save: Do not add suffix if it's the last save - """ - - # Final session saving - if last_save: - path = self.network_dir + "network" - print(f"[{self.name}] Saving final network at {path}.") - self.network.save_parameters(path) - - # Intermediate states saving - elif self.save_each_epoch: - path = self.network_dir + self.network_template_name.format(self.saved_counter) - self.saved_counter += 1 - print(f"[{self.name}] Saving intermediate network at {path}.") - self.network.save_parameters(path) - - def close(self) -> None: - """ - | Closing procedure. - """ - - if self.training: - self.save_network(last_save=True) - del self.network - del self.network_config - - def __str__(self) -> str: - """ - :return: String containing information about the BaseNetwork object - """ - - description = "\n" - description += f"# {self.__class__.__name__}\n" - description += f" Network Directory: {self.network_dir}\n" - description += f" Save each Epoch: {self.save_each_epoch}\n" - description += f" Managed objects: Network: {self.network.__class__.__name__}\n" - description += f" Optimization: {self.optimization.__class__.__name__}\n" - description += f" Data Transformation: {self.data_transformation.__class__.__name__}\n" - description += str(self.network) - description += str(self.optimization) - description += str(self.data_transformation) - return description diff --git a/src/Manager/__init__.py b/src/Manager/__init__.py deleted file mode 100644 index e7481a52..00000000 --- a/src/Manager/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from os.path import dirname -from os import listdir - -package = dirname(__file__) -exceptions = ['__init__.py', '__pycache__'] -modules = [module for module in listdir(package) if module.endswith('.py') and module not in exceptions] -__all__ = [] -for module in sorted(modules): - exec(f"from DeepPhysX.Core.Manager.{module[:-3]} import {module[:-3]}") - __all__.append(module[:-3]) diff --git a/src/Network/BaseNetwork.py b/src/Network/BaseNetwork.py deleted file mode 100644 index e61139fb..00000000 --- a/src/Network/BaseNetwork.py +++ /dev/null @@ -1,128 +0,0 @@ -from typing import Any, Dict -from numpy import ndarray -from collections import namedtuple - - -class BaseNetwork: - """ - | BaseNetwork is a network class to compute predictions from input data according to actual state. - - :param namedtuple config: namedtuple containing BaseNetwork parameters - """ - - def __init__(self, config: namedtuple): - - # Config - self.device = None - self.config = config - - def predict(self, input_data: Any) -> Any: - """ - | Same as forward - - :param Any input_data: Input tensor - :return: Network prediction - """ - - return self.forward(input_data) - - def forward(self, input_data: Any) -> Any: - """ - | Gives input_data as raw input to the neural network. - - :param Any input_data: Input tensor - :return: Network prediction - """ - - raise NotImplementedError - - def set_train(self) -> None: - """ - | Set the Network in train mode (compute gradient). - """ - - raise NotImplementedError - - def set_eval(self) -> None: - """ - | Set the Network in eval mode (does not compute gradient). - """ - - raise NotImplementedError - - def set_device(self) -> None: - """ - | Set computer device on which Network's parameters will be stored and tensors will be computed. - """ - - raise NotImplementedError - - def load_parameters(self, path: str) -> None: - """ - | Load network parameter from path. - - :param str path: Path to Network parameters to load - """ - - raise NotImplementedError - - def get_parameters(self) -> Dict[str, Any]: - """ - | Return the current state of Network parameters. - - :return: Network parameters - """ - - raise NotImplementedError - - def save_parameters(self, path) -> None: - """ - | Saves the network parameters to the path location. - - :param str path: Path where to save the parameters. - """ - - raise NotImplementedError - - def nb_parameters(self) -> int: - """ - | Return the number of parameters of the network. - - :return: Number of parameters - """ - - raise NotImplementedError - - def transform_from_numpy(self, data: ndarray, grad: bool = True) -> Any: - """ - | Transform and cast data from numpy to the desired tensor type. - - :param ndarray data: Array data to convert - :param bool grad: If True, gradient will record operations on this tensor - :return: Converted tensor - """ - - return data.astype(self.config.data_type) - - def transform_to_numpy(self, data: Any) -> ndarray: - """ - | Transform and cast data from tensor type to numpy. - - :param Any data: Any to convert - :return: Converted array - """ - - return data.astype(self.config.data_type) - - def __str__(self) -> str: - """ - :return: String containing information about the BaseNetwork object - """ - - description = "\n" - description += f" {self.__class__.__name__}\n" - description += f" Name: {self.config.network_name}\n" - description += f" Type: {self.config.network_type}\n" - description += f" Number of parameters: {self.nb_parameters()}\n" - description += f" Estimated size: {self.nb_parameters() * 32 * 1.25e-10} Go\n" - return description diff --git a/src/Network/BaseNetworkConfig.py b/src/Network/BaseNetworkConfig.py deleted file mode 100644 index 24a943b2..00000000 --- a/src/Network/BaseNetworkConfig.py +++ /dev/null @@ -1,196 +0,0 @@ -from typing import Any, Optional, Type -from collections import namedtuple -from os.path import isdir -from numpy import typeDict - -from DeepPhysX.Core.Network.BaseNetwork import BaseNetwork -from DeepPhysX.Core.Network.BaseOptimization import BaseOptimization -from DeepPhysX.Core.Network.DataTransformation import DataTransformation - -NetworkType = BaseNetwork -OptimizationType = BaseOptimization -DataTransformationType = DataTransformation - - -class BaseNetworkConfig: - """ - | BaseNetworkConfig is a configuration class to parameterize and create BaseNetwork, BaseOptimization and - DataTransformation for the NetworkManager. - - :param Type[BaseNetwork] network_class: BaseNetwork class from which an instance will be created - :param Type[BaseOptimization] optimization_class: BaseOptimization class from which an instance will be created - :param Type[DataTransformation] data_transformation_class: DataTransformation class from which an instance will - be created - :param Optional[str] network_dir: Name of an existing network repository - :param str network_name: Name of the network - :param str network_type: Type of the network - :param int which_network: If several networks in network_dir, load the specified one - :param bool save_each_epoch: If True, network state will be saved at each epoch end; if False, network state - will be saved at the end of the training - :param str data_type: Type of the training data - :param Optional[float] lr: Learning rate - :param bool require_training_stuff: If specified, loss and optimizer class can be not necessary for training - :param Optional[Any] loss: Loss class - :param Optional[Any] optimizer: Network's parameters optimizer class - """ - - def __init__(self, - network_class: Type[BaseNetwork] = BaseNetwork, - optimization_class: Type[BaseOptimization] = BaseOptimization, - data_transformation_class: Type[DataTransformation] = DataTransformation, - network_dir: Optional[str] = None, - network_name: str = 'Network', - network_type: str = 'BaseNetwork', - which_network: int = 0, - save_each_epoch: bool = False, - data_type: str = 'float32', - lr: Optional[float] = None, - require_training_stuff: bool = True, - loss: Optional[Any] = None, - optimizer: Optional[Any] = None): - - # Check network_dir type and existence - if network_dir is not None: - if type(network_dir) != str: - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'network_dir' type: str required, get {type(network_dir)}") - if not isdir(network_dir): - raise ValueError(f"[{self.__class__.__name__}] Given 'network_dir' does not exists: {network_dir}") - # Check network_name type - if type(network_name) != str: - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'network_name' type: str required, get {type(network_name)}") - # Check network_tpe type - if type(network_type) != str: - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'network_type' type: str required, get {type(network_type)}") - # Check which_network type and value - if type(which_network) != int: - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'which_network' type: int required, get {type(which_network)}") - if which_network < 0: - raise ValueError(f"[{self.__class__.__name__}] Given 'which_network' value is negative") - # Check save_each_epoch type - if type(save_each_epoch) != bool: - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'save each epoch' type: bool required, get {type(save_each_epoch)}") - # Check data type - if data_type not in typeDict: - raise ValueError( - f"[{self.__class__.__name__}] The following data type is not a numpy type: {data_type}") - - # BaseNetwork parameterization - self.network_class: Type[BaseNetwork] = network_class - self.network_config: namedtuple = self.make_config(config_name='network_config', - network_name=network_name, - network_type=network_type, - data_type=data_type) - - # BaseOptimization parameterization - self.optimization_class: Type[BaseOptimization] = optimization_class - self.optimization_config: namedtuple = self.make_config(config_name='optimization_config', - loss=loss, - lr=lr, - optimizer=optimizer) - self.training_stuff: bool = (loss is not None) and (optimizer is not None) or (not require_training_stuff) - - # NetworkManager parameterization - self.data_transformation_class: Type[DataTransformation] = data_transformation_class - self.data_transformation_config: namedtuple = self.make_config(config_name='data_transformation_config') - self.network_dir: str = network_dir - self.which_network: int = which_network - self.save_each_epoch: bool = save_each_epoch and self.training_stuff - - def make_config(self, config_name: str, **kwargs) -> namedtuple: - """ - | Create a namedtuple which gathers all the parameters for an Object configuration (Network or Optimization). - | For a child config class, only new items are required since parent's items will be added by default. - - :param str config_name: Name of the configuration to fill - :param kwargs: Items to add to the Object configuration - :return: Namedtuple which contains Object parameters - """ - - # Get items set as keyword arguments - fields = tuple(kwargs.keys()) - args = tuple(kwargs.values()) - # Check if a config already exists with the same name (child class will have the parent's config by default) - if config_name in self.__dict__: - config = self.__getattribute__(config_name) - for key, value in config._asdict().items(): - # Only new items are required for children, check if the parent's items are set again anyway - if key not in fields: - fields += (key,) - args += (value,) - # Create namedtuple with collected items - return namedtuple(config_name, fields)._make(args) - - def create_network(self) -> NetworkType: - """ - | Create an instance of network_class with given parameters. - - :return: BaseNetwork object from network_class and its parameters. - """ - - try: - network = self.network_class(config=self.network_config) - except: - raise ValueError( - f"[{self.__class__.__name__}] Given 'network_class' cannot be created in {self.__class__.__name__}") - if not isinstance(network, BaseNetwork): - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'network_class' type: BaseNetwork required, get " - f"{self.network_class}") - return network - - def create_optimization(self) -> OptimizationType: - """ - | Create an instance of optimization_class with given parameters. - - :return: BaseOptimization object from optimization_class and its parameters. - """ - - try: - optimization = self.optimization_class(config=self.optimization_config) - except: - raise ValueError( - f"[{self.__class__.__name__}] Given 'optimization_class' got an unexpected keyword argument 'config'") - if not isinstance(optimization, BaseOptimization): - raise TypeError(f"[{self.__class__.__name__}] Wrong 'optimization_class' type: BaseOptimization required, " - f"get {self.optimization_class}") - return optimization - - def create_data_transformation(self) -> DataTransformationType: - """ - | Create an instance of data_transformation_class with given parameters. - - :return: DataTransformation object from data_transformation_class and its parameters. - """ - - try: - data_transformation = self.data_transformation_class(config=self.data_transformation_config) - except: - raise ValueError( - f"[{self.__class__.__name__}] Given 'data_transformation_class' got an unexpected keyword argument " - f"'config'") - if not isinstance(data_transformation, DataTransformation): - raise TypeError( - f"[{self.__class__.__name__}] Wrong 'data_transformation_class' type: DataTransformation required, " - f"get {self.data_transformation_class}") - return data_transformation - - def __str__(self) -> str: - """ - :return: String containing information about the BaseDatasetConfig object - """ - - # Todo: fields in Configs are the set in Managers or objects, then remove __str__ method - description = "\n" - description += f"{self.__class__.__name__}\n" - description += f" Network class: {self.network_class.__name__}\n" - description += f" Optimization class: {self.optimization_class.__name__}\n" - description += f" Training materials: {self.training_stuff}\n" - description += f" Network directory: {self.network_dir}\n" - description += f" Which network: {self.which_network}\n" - description += f" Save each epoch: {self.save_each_epoch}\n" - return description diff --git a/src/Network/BaseOptimization.py b/src/Network/BaseOptimization.py deleted file mode 100644 index 1c75d66c..00000000 --- a/src/Network/BaseOptimization.py +++ /dev/null @@ -1,85 +0,0 @@ -from typing import Dict, Any -from collections import namedtuple - -from DeepPhysX.Core.Network.BaseNetwork import BaseNetwork - - -class BaseOptimization: - """ - | BaseOptimization is dedicated to network optimization: compute loss between prediction and target, update - network parameters. - - :param namedtuple config: Namedtuple containing BaseOptimization parameters - """ - - def __init__(self, config: namedtuple): - - self.manager: Any = None - - # Loss - self.loss_class = config.loss - self.loss = None - self.loss_value = 0. - - # Optimizer - self.optimizer_class = config.optimizer - self.optimizer = None - self.lr = config.lr - - def set_loss(self) -> None: - """ - | Initialize the loss function. - """ - - raise NotImplementedError - - def compute_loss(self, prediction: Any, ground_truth: Any, data: Dict[str, Any]) -> Dict[str, float]: - """ - | Compute loss from prediction / ground truth. - - :param Any prediction: Tensor produced by the forward pass of the Network - :param Any ground_truth: Ground truth tensor to be compared with prediction - :param Dict[str, Any] data: Additional data sent as dict to compute loss value - :return: Loss value - """ - - raise NotImplementedError - - def transform_loss(self, data: Dict[str, Any]) -> Dict[str, float]: - """ - | Apply a transformation on the loss value using the potential additional data. - - :param Dict[str, Any] data: Additional data sent as dict to compute loss value - :return: Transformed loss value - """ - - raise NotImplementedError - - def set_optimizer(self, net: BaseNetwork) -> None: - """ - | Define an optimization process. - - :param BaseNetwork net: Network whose parameters will be optimized. - """ - - raise NotImplementedError - - def optimize(self) -> None: - """ - | Run an optimization step. - """ - - raise NotImplementedError - - def __str__(self) -> str: - """ - :return: String containing information about the BaseOptimization object - """ - - description = "\n" - description += f" {self.__class__.__name__}\n" - description += f" Loss class: {self.loss_class.__name__}\n" if self.loss_class else f" Loss class: None\n" - description += f" Optimizer class: {self.optimizer_class.__name__}\n" if self.optimizer_class else \ - f" Optimizer class: None\n" - description += f" Learning rate: {self.lr}\n" - return description diff --git a/src/Network/DataTransformation.py b/src/Network/DataTransformation.py deleted file mode 100644 index 8e3d7fbb..00000000 --- a/src/Network/DataTransformation.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import Callable, Any, Optional, Tuple -from collections import namedtuple - - -class DataTransformation: - """ - | DataTransformation is dedicated to data operations before and after network predictions. - - :param namedtuple config: Namedtuple containing the parameters of the network manager - """ - - def __init__(self, config: namedtuple): - - self.name = self.__class__.__name__ - - self.config: Any = config - self.data_type = any - - @staticmethod - def check_type(func: Callable[[Any, Any], Any]): - - def inner(self, *args): - for data in args: - if data is not None and type(data) != self.data_type: - raise TypeError(f"[{self.name}] Wrong data type: {self.data_type} required, get {type(data)}") - return func(self, *args) - - return inner - - def transform_before_prediction(self, data_in: Any) -> Any: - """ - | Apply data operations before network's prediction. - - :param Any data_in: Input data - :return: Transformed input data - """ - - return data_in - - def transform_before_loss(self, data_out: Any, data_gt: Optional[Any] = None) -> Tuple[Any, Optional[Any]]: - """ - | Apply data operations between network's prediction and loss computation. - - :param Any data_out: Prediction data - :param Optional[Any] data_gt: Ground truth data - :return: Transformed prediction data, transformed ground truth data - """ - - return data_out, data_gt - - def transform_before_apply(self, data_out: Any) -> Any: - """ - | Apply data operations between loss computation and prediction apply in environment. - - :param Any data_out: Prediction data - :return: Transformed prediction data - """ - - return data_out - - def __str__(self) -> str: - """ - :return: String containing information about the DataTransformation object - """ - - description = "\n" - description += f" {self.__class__.__name__}\n" - description += f" Data type: {self.data_type}\n" - description += f" Transformation before prediction: Identity\n" - description += f" Transformation before loss: Identity\n" - description += f" Transformation before apply: Identity\n" - return description diff --git a/src/Pipelines/BaseDataGenerator.py b/src/Pipelines/BaseDataGenerator.py deleted file mode 100644 index 949aeba4..00000000 --- a/src/Pipelines/BaseDataGenerator.py +++ /dev/null @@ -1,86 +0,0 @@ -import os.path -from os.path import join as osPathJoin -from os.path import basename -from sys import stdout - -from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline -from DeepPhysX.Core.Manager.DataManager import DataManager -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Utils.progressbar import Progressbar -from DeepPhysX.Core.Utils.pathUtils import create_dir, get_first_caller - - -class BaseDataGenerator(BasePipeline): - """ - | BaseDataGenerator implement a minimalist execute function that simply produce and store data without - training a neural network. - - :param BaseDatasetConfig dataset_config: Specialisation containing the parameters of the dataset manager - :param BaseEnvironmentConfig environment_config: Specialisation containing the parameters of the environment manager - :param str session_name: Name of the newly created directory if session is not defined - :param int nb_batches: Number of batches - :param int batch_size: Size of a batch - :param bool record_input: True if the input must be stored - :param bool record_output: True if the output must be stored - """ - - def __init__(self, - dataset_config: BaseDatasetConfig, - environment_config: BaseEnvironmentConfig, - session_name: str = 'default', - nb_batches: int = 0, - batch_size: int = 0, - record_input: bool = True, - record_output: bool = True): - - BasePipeline.__init__(self, - dataset_config=dataset_config, - environment_config=environment_config, - session_name=session_name, - pipeline='dataset') - - # Init session repository - dataset_dir = dataset_config.dataset_dir - if dataset_dir is not None: - if dataset_dir[-1] == "/": - dataset_dir = dataset_dir[:-1] - if dataset_dir[-8:] == "/dataset": - dataset_dir = dataset_dir[:-8] - if osPathJoin(get_first_caller(), session_name) != osPathJoin(get_first_caller(), dataset_dir): - dataset_dir = None - elif not os.path.exists(osPathJoin(get_first_caller(), dataset_dir)): - dataset_dir = None - if dataset_dir is None: - session_dir = create_dir(osPathJoin(get_first_caller(), session_name), dir_name=session_name) - session_name = (session_name if session_name is not None else basename(session_dir)).split("/")[-1] - else: - session_dir = osPathJoin(get_first_caller(), dataset_dir) - session_name = (session_name if session_name is not None else basename(session_dir)).split("/")[-1] - - # Create a DataManager directly - self.data_manager = DataManager(manager=self, - dataset_config=dataset_config, - environment_config=environment_config, - session_name=session_name, - session_dir=session_dir, - new_session=True, - offline=True, - record_data={'input': record_input, 'output': record_output}, - batch_size=batch_size) - self.nb_batch: int = nb_batches - self.progress_bar = Progressbar(start=0, stop=self.nb_batch, c='orange', title="Data Generation") - - def execute(self) -> None: - """ - | Run the data generation and recording process. - """ - - for i in range(self.nb_batch): - # Produce a batch - self.data_manager.get_data() - # Update progress bar - stdout.write("\033[K") - self.progress_bar.print(counts=i + 1) - # Close manager - self.data_manager.close() diff --git a/src/Pipelines/BasePipeline.py b/src/Pipelines/BasePipeline.py deleted file mode 100644 index a792bc3d..00000000 --- a/src/Pipelines/BasePipeline.py +++ /dev/null @@ -1,131 +0,0 @@ -from typing import Dict, Optional, Any, List, Union - -from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Manager.Manager import Manager -from DeepPhysX.Core.Manager.NetworkManager import NetworkManager -from DeepPhysX.Core.Manager.DataManager import DataManager -from DeepPhysX.Core.Manager.StatsManager import StatsManager -from DeepPhysX.Core.Manager.DatasetManager import DatasetManager -from DeepPhysX.Core.Manager.EnvironmentManager import EnvironmentManager - - -class BasePipeline: - """ - | Base class defining Pipelines common variables. - - :param BaseNetworkConfig network_config: Specialisation containing the parameters of the network manager - :param BaseDatasetConfig dataset_config: Specialisation containing the parameters of the dataset manager - :param BaseEnvironmentConfig environment_config: Specialisation containing the parameters of the environment manager - :param str session_name: Name of the newly created directory if session is not defined - :param Optional[str] session_dir: Name of the directory in which to write all the necessary data - :param Optional[str] pipeline: Values at either 'training' or 'prediction' - """ - - def __init__(self, - network_config: Optional[BaseNetworkConfig] = None, - dataset_config: Optional[BaseDatasetConfig] = None, - environment_config: Optional[BaseEnvironmentConfig] = None, - session_name: str = 'default', - session_dir: Optional[str] = None, - pipeline: Optional[str] = None): - - self.name: str = self.__class__.__name__ - - # Check the arguments - if network_config is not None and not isinstance(network_config, BaseNetworkConfig): - raise TypeError(f"[{self.name}] The network configuration must be a BaseNetworkConfig") - if environment_config is not None and not isinstance(environment_config, BaseEnvironmentConfig): - raise TypeError(f"[{self.name}] The environment configuration must be a BaseEnvironmentConfig") - if dataset_config is not None and not isinstance(dataset_config, BaseDatasetConfig): - raise TypeError(f"[{self.name}] The dataset configuration must be a BaseDatasetConfig") - if type(session_name) != str: - raise TypeError(f"[{self.name}] The network config must be a BaseNetworkConfig object.") - if session_dir is not None and type(session_dir) != str: - raise TypeError(f"[{self.name}] The session directory must be a str.") - - self.type: str = pipeline # Either training or prediction - self.debug: bool = False - self.new_session: bool = True - self.record_data: Optional[Dict[str, bool]] = None # Can be of type {'in': bool, 'out': bool} - - # Dataset variables - self.dataset_config: BaseDatasetConfig = dataset_config - # Network variables - self.network_config: BaseNetworkConfig = network_config - # Simulation variables - self.environment_config: BaseEnvironmentConfig = environment_config - # Main manager - self.manager: Optional[Manager] = None - - def get_any_manager(self, manager_names: Union[str, List[str]]) -> Optional[Any]: - """ - | Return the desired Manager associated with the pipeline if it exists. - - :param Union[str, List[str]] manager_names: Name of the desired Manager or order of access to the desired - Manager - :return: Manager associated with the Pipeline - """ - - # If manager variable is not defined, cannot access other manager - if self.manager is None: - return None - - # Direct access to manager - if type(manager_names) == str: - return getattr(self.manager, manager_names) if hasattr(self.manager, manager_names) else None - - # Intermediates to access manager - accessed_manager = self.manager - for next_manager in manager_names: - if hasattr(accessed_manager, next_manager): - accessed_manager = getattr(accessed_manager, next_manager) - else: - return None - return accessed_manager - - def get_network_manager(self) -> NetworkManager: - """ - | Return the NetworkManager associated with the pipeline. - - :return: NetworkManager associated with the pipeline - """ - - return self.get_any_manager('network_manager') - - def get_data_manager(self) -> DataManager: - """ - | Return the DataManager associated with the pipeline. - - :return: DataManager associated with the pipeline - """ - - return self.get_any_manager('data_manager') - - def get_stats_manager(self) -> StatsManager: - """ - | Return the StatsManager associated with the pipeline. - - :return: StatsManager associated with the pipeline - """ - - return self.get_any_manager('stats_manager') - - def get_dataset_manager(self) -> DatasetManager: - """ - | Return the DatasetManager associated with the pipeline. - - :return: DatasetManager associated with the pipeline - """ - - return self.get_any_manager(['data_manager', 'dataset_manager']) - - def get_environment_manager(self) -> EnvironmentManager: - """ - | Return the EnvironmentManager associated with the pipeline. - - :return: EnvironmentManager associated with the pipeline - """ - - return self.get_any_manager(['data_manager', 'environment_manager']) diff --git a/src/Pipelines/BaseRunner.py b/src/Pipelines/BaseRunner.py deleted file mode 100644 index 12771724..00000000 --- a/src/Pipelines/BaseRunner.py +++ /dev/null @@ -1,157 +0,0 @@ -from typing import Optional -from numpy import ndarray - -from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline -from DeepPhysX.Core.Manager.Manager import Manager - - -class BaseRunner(BasePipeline): - """ - | BaseRunner is a pipeline defining the running process of an artificial neural network. - | It provides a highly tunable learning process that can be used with any machine learning library. - - :param BaseNetworkConfig network_config: Specialisation containing the parameters of the network manager - :param BaseEnvironmentConfig environment_config: Specialisation containing the parameters of the environment manager - :param Optional[BaseDatasetConfig] dataset_config: Specialisation containing the parameters of the dataset manager - :param str session_name: Name of the newly created directory if session is not defined - :param Optional[str] session_dir: Name of the directory in which to write all the necessary data - :param int nb_steps: Number of simulation step to play - :param bool record_inputs: Save or not the input in a numpy file - :param bool record_outputs: Save or not the output in a numpy file - """ - - def __init__(self, - network_config: BaseNetworkConfig, - environment_config: Optional[BaseEnvironmentConfig] = None, - dataset_config: Optional[BaseDatasetConfig] = None, - session_name: str = 'default', - session_dir: Optional[str] = None, - nb_steps: int = 0, - record_inputs: bool = False, - record_outputs: bool = False): - - BasePipeline.__init__(self, - network_config=network_config, - dataset_config=dataset_config, - environment_config=environment_config, - session_name=session_name, - session_dir=session_dir, - pipeline='prediction') - - self.name = self.__class__.__name__ - - if type(nb_steps) != int or nb_steps < 0: - raise TypeError("[BaseRunner] The number of steps must be a positive int") - - self.nb_samples = nb_steps - self.idx_step = 0 - - # Tell if data is recording while predicting (output is recorded only if input too) - self.record_data = {'input': False, 'output': False} - if dataset_config is not None: - self.record_data = {'input': record_inputs, 'output': record_outputs and record_inputs} - self.is_environment = environment_config is not None - - self.manager = Manager(pipeline=self, - network_config=self.network_config, - dataset_config=dataset_config, - environment_config=self.environment_config, - session_name=session_name, - session_dir=session_dir, - new_session=True) - - def execute(self) -> None: - """ - | Main function of the running process "execute" call the functions associated with the learning process. - | Each of the called functions are already implemented so one can start a basic run session. - | Each of the called function can also be rewritten via inheritance to provide more specific / complex running - process. - """ - - self.run_begin() - while self.running_condition(): - self.sample_begin() - self.sample_end(self.predict()) - self.run_end() - - def predict(self, animate: bool = True) -> ndarray: - """ - | Pull the data from the manager and return the prediction - - :param bool animate: True if getData fetch from the environment - :return: Prediction from the Network - """ - - self.manager.get_data(animate=animate) - data = self.manager.data_manager.data['input'] - data = self.manager.data_manager.normalize_data(data, 'input') - return self.manager.network_manager.compute_online_prediction(data) - - def run_begin(self) -> None: - """ - | Called once at the very beginning of the Run process. - | Allows the user to run some pre-computations. - """ - - pass - - def run_end(self) -> None: - """ - | Called once at the very end of the Run process. - | Allows the user to run some post-computations. - """ - - pass - - def running_condition(self) -> bool: - """ - | Condition that characterize the end of the running process. - - :return: False if the training needs to stop. - """ - - running = self.idx_step < self.nb_samples if self.nb_samples > 0 else True - self.idx_step += 1 - return running - - def sample_begin(self) -> None: - """ - | Called one at the start of each step. - | Allows the user to run some pre-step computations. - """ - - pass - - def sample_end(self, prediction: ndarray) -> None: - """ - | Called one at the end of each step. - | Allows the user to run some post-step computations. - - :param ndarray prediction: Prediction of the Network. - """ - - if self.is_environment: - self.manager.data_manager.apply_prediction(prediction) - - def close(self) -> None: - """ - | End the running process and close all the managers - """ - - self.manager.close() - - def __str__(self) -> str: - """ - :return: str Contains running information about the running process - """ - - description = "" - description += f"Running statistics :\n" - description += f"Number of simulation step: {self.nb_samples}\n" - description += f"Record inputs : {self.record_data[0]}\n" - description += f"Record outputs : {self.record_data[1]}\n" - - return description diff --git a/src/Pipelines/BaseTrainer.py b/src/Pipelines/BaseTrainer.py deleted file mode 100644 index 2c98addd..00000000 --- a/src/Pipelines/BaseTrainer.py +++ /dev/null @@ -1,233 +0,0 @@ -from typing import Optional -from sys import stdout - -from DeepPhysX.Core.Pipelines.BasePipeline import BasePipeline -from DeepPhysX.Core.Manager.Manager import Manager -from DeepPhysX.Core.Network.BaseNetworkConfig import BaseNetworkConfig -from DeepPhysX.Core.Dataset.BaseDatasetConfig import BaseDatasetConfig -from DeepPhysX.Core.Environment.BaseEnvironmentConfig import BaseEnvironmentConfig -from DeepPhysX.Core.Utils.progressbar import Progressbar - - -class BaseTrainer(BasePipeline): - """ - | BaseTrainer is a pipeline defining the training process of an artificial neural network. - | It provides a highly tunable learning process that can be used with any machine learning library. - - :param BaseNetworkConfig network_config: Specialisation containing the parameters of the network manager - :param BaseDatasetConfig dataset_config: Specialisation containing the parameters of the dataset manager - :param Optional[BaseEnvironmentConfig] environment_config: Specialisation containing the parameters of the - environment manager - :param str session_name: Name of the newly created directory if session is not defined - :param Optional[str] session_dir: Name of the directory in which to write all the necessary data - :param bool new_session: Define the creation of new directories to store data - :param int nb_epochs: Number of epochs - :param int nb_batches: Number of batches - :param int batch_size: Size of a batch - :param bool debug: If True, main training features will not be launched - """ - - def __init__(self, - network_config: BaseNetworkConfig, - dataset_config: BaseDatasetConfig, - environment_config: Optional[BaseEnvironmentConfig] = None, - session_name: str = 'default', - session_dir: Optional[str] = None, - new_session: bool = True, - nb_epochs: int = 0, - nb_batches: int = 0, - batch_size: int = 0, - debug: bool = False): - - if environment_config is None and dataset_config.dataset_dir is None: - print("BaseTrainer: You have to give me a dataset source (existing dataset directory or simulation to " - "create data on the fly") - quit(0) - - BasePipeline.__init__(self, - network_config=network_config, - dataset_config=dataset_config, - environment_config=environment_config, - session_name=session_name, - session_dir=session_dir, - pipeline='training') - - # Training variables - self.nb_epochs = nb_epochs - self.id_epoch = 0 - self.nb_batches = nb_batches - self.batch_size = batch_size - self.id_batch = 0 - self.nb_samples = nb_batches * batch_size * nb_epochs - self.loss_dict = None - - # Tell if data is recording while predicting (output is recorded only if input too) - self.record_data = {'input': True, 'output': True} - - self.debug = debug - if not self.debug: - self.progress_counter = 0 - self.digits = ['{' + f':0{len(str(self.nb_epochs))}d' + '}', - '{' + f':0{len(str(self.nb_batches))}d' + '}'] - id_epoch, nb_epoch = self.digits[0].format(0), self.digits[0].format(self.nb_epochs) - id_batch, nb_batch = self.digits[1].format(0), self.digits[1].format(self.nb_batches) - self.progress_bar = Progressbar(start=0, stop=self.nb_batches * self.nb_epochs, c='orange', - title=f'Epoch n°{id_epoch}/{nb_epoch} - Batch n°{id_batch}/{nb_batch} ') - - self.manager = Manager(pipeline=self, - network_config=self.network_config, - dataset_config=dataset_config, - environment_config=self.environment_config, - session_name=session_name, - session_dir=session_dir, - new_session=new_session, - batch_size=batch_size) - - self.manager.save_info_file() - - def execute(self) -> None: - """ - | Main function of the training process \"execute\" call the functions associated with the learning process. - | Each of the called functions are already implemented so one can start a basic training. - | Each of the called function can also be rewritten via inheritance to provide more specific / complex training - process. - """ - - self.train_begin() - while self.epoch_condition(): - self.epoch_begin() - while self.batch_condition(): - self.batch_begin() - self.optimize() - self.batch_count() - self.batch_end() - self.epoch_count() - self.epoch_end() - self.save_network() - self.train_end() - - def optimize(self) -> None: - """ - | Pulls data from the manager and run a prediction and optimizer step. - """ - - self.manager.get_data(self.id_epoch, self.batch_size) - _, self.loss_dict = self.manager.optimize_network() - - def save_network(self) -> None: - """ - | Registers the network weights and biases in the corresponding directory (session_name/network or - session/network) - """ - - self.manager.save_network() - - def train_begin(self) -> None: - """ - | Called once at the very beginning of the training process. - | Allows the user to run some pre-computations. - """ - - pass - - def train_end(self) -> None: - """ - | Called once at the very end of the training process. - | Allows the user to run some post-computations. - """ - - self.manager.close() - - def epoch_begin(self) -> None: - """ - | Called one at the start of each epoch. - | Allows the user to run some pre-epoch computations. - """ - - self.id_batch = 0 - - def epoch_end(self) -> None: - """ - | Called one at the end of each epoch. - | Allows the user to run some post-epoch computations. - """ - - self.manager.stats_manager.add_train_epoch_loss(self.loss_dict['loss'], self.id_epoch) - - def epoch_condition(self) -> bool: - """ - | Condition that characterize the end of the training process. - - :return: False if the training needs to stop. - """ - - return self.id_epoch < self.nb_epochs - - def epoch_count(self) -> None: - """ - | Allows user for custom update of epochs count. - """ - - self.id_epoch += 1 - - def batch_begin(self) -> None: - """ - | Called one at the start of each batch. - | Allows the user to run some pre-batch computations. - """ - - if not self.debug: - stdout.write("\033[K") - self.progress_counter += 1 - id_epoch, nb_epoch = self.digits[0].format(self.id_epoch + 1), self.digits[0].format(self.nb_epochs) - id_batch, nb_batch = self.digits[1].format(self.id_batch + 1), self.digits[1].format(self.nb_batches) - self.progress_bar.title = f'Epoch n°{id_epoch}/{nb_epoch} - Batch n°{id_batch}/{nb_batch} ' - self.progress_bar.print(counts=self.progress_counter) - - def batch_end(self) -> None: - """ - | Called one at the start of each batch. - | Allows the user to run some post-batch computations. - """ - - self.manager.stats_manager.add_train_batch_loss(self.loss_dict['loss'], - self.id_epoch * self.nb_batches + self.id_batch) - for key in self.loss_dict.keys(): - if key != 'loss': - self.manager.stats_manager.add_custom_scalar(tag=key, - value=self.loss_dict[key], - count=self.id_epoch * self.nb_batches + self.id_batch) - - def batch_condition(self) -> bool: - """ - | Condition that characterize the end of the epoch. - - :return: False if the epoch needs to stop. - """ - - return self.id_batch < self.nb_batches - - def batch_count(self): - """ - | Allows user for custom update of batches count. - - :return: - """ - - self.id_batch += 1 - - def __str__(self) -> str: - """ - :return: str Contains training information about the training process - """ - - description = "\n" - description += f"# {self.__class__.__name__}\n" - description += f" Session directory: {self.manager.session}\n" - description += f" Number of epochs: {self.nb_epochs}\n" - description += f" Number of batches per epoch: {self.nb_batches}\n" - description += f" Number of samples per batch: {self.batch_size}\n" - description += f" Number of samples per epoch: {self.nb_batches * self.batch_size}\n" - description += f" Total: Number of batches : {self.nb_batches * self.nb_epochs}\n" - description += f" Number of samples : {self.nb_samples}\n" - return description diff --git a/src/Utils/pathUtils.py b/src/Utils/pathUtils.py deleted file mode 100644 index 4f25ceb3..00000000 --- a/src/Utils/pathUtils.py +++ /dev/null @@ -1,75 +0,0 @@ -from os.path import join as osPathJoin -from os.path import isdir, abspath, normpath, dirname, basename -from os import listdir, pardir, makedirs -from inspect import getmodule, stack -from shutil import copytree - - -def create_dir(dir_path: str, dir_name: str) -> str: - """ - Create a directory of the given name. If it already exist and specified, add a unique identifier at the end. - - :param str dir_path: Absolute directory to create - :param str dir_name: Name of the directory to check for existence of a similar directories - - :return: Name of the created directory as string - """ - if isdir(dir_path): - print(f"Directory conflict: you are going to overwrite {dir_path}.") - # Get the parent dir of training sessions - parent = abspath(osPathJoin(dir_path, pardir)) - # Find all the duplicated folder - deepest_repertory = dir_name.split('/')[-1] + '_' - copies_list = [folder for folder in listdir(parent) if - isdir(osPathJoin(parent, folder)) and - folder.__contains__(deepest_repertory) and - folder.find(deepest_repertory) == 0 and - len(folder) in [len(deepest_repertory) + i for i in range(1, 4)]] - # Get the indices of copies - indices = [int(folder[len(deepest_repertory):]) for folder in copies_list] - # The new copy is the max int + 1 - max_ind = max(indices) if len(indices) > 0 else 0 - new_name = basename(normpath(dir_path)) + f'_{max_ind + 1}/' - dir_path = osPathJoin(parent, new_name) - print(f"Create a new directory {dir_path} for this session.") - makedirs(dir_path) - return dir_path - - -def copy_dir(src_dir: str, dest_parent_dir: str, dest_dir: str) -> str: - """ - Copy source directory to destination directory at the end of destination parent directory - - :param str src_dir: Source directory to copy - :param str dest_parent_dir: Parent of the destination directory to copy - :param str dest_dir: Destination directory to copy to - - :return: destination directory that source has been copied to - """ - dest_dir = osPathJoin(dest_parent_dir, dest_dir) - if isdir(dest_dir): - print("Directory conflict: you are going to overwrite by copying in {}.".format(dest_dir)) - copies_list = [folder for folder in listdir(dest_parent_dir) if - isdir(osPathJoin(dest_parent_dir, folder)) and - folder.__contains__(dest_dir)] - new_name = dest_dir + '({})/'.format(len(copies_list)) - dest_dir = osPathJoin(dest_parent_dir, new_name) - print("Copying {} into the new directory {} for this session.".format(src_dir, dest_dir)) - else: - new_name = dest_dir + '/' - dest_dir = osPathJoin(dest_parent_dir, new_name) - copytree(src_dir, dest_dir) - return dest_dir - - -def get_first_caller() -> str: - """ - Return the repertory in which the main script is - """ - # Get the stack of called scripts - scripts_list = stack()[-1] - # Get the first one (the one launched by the user) - module = getmodule(scripts_list[0]) - # Return the path of this script - return dirname(abspath(module.__file__)) - diff --git a/src/Utils/progressbar.py b/src/Utils/progressbar.py deleted file mode 100644 index 836223f5..00000000 --- a/src/Utils/progressbar.py +++ /dev/null @@ -1,34 +0,0 @@ -from vedo import ProgressBar - - -class Progressbar(ProgressBar): - - def __init__(self, start, stop, step=1, c=None, title=''): - - ProgressBar.__init__(self, start=start, stop=stop, step=step, c=c, title=title) - - self.percent_int = 0 - - def _update(self, counts): - if counts < self.start: - counts = self.start - elif counts > self.stop: - counts = self.stop - self._counts = counts - self.percent = (self._counts - self.start) * 100 - dd = self.stop - self.start - if dd: - self.percent /= self.stop - self.start - else: - self.percent = 0 - self.percent_int = int(round(self.percent)) - af = self.width - 2 - nh = int(round(self.percent_int / 100 * af)) - br_bk = "\x1b[2m" + self.char_back * (af - nh) - br = "%s%s%s" % (self.char * (nh - 1), self.char_arrow, br_bk) - self.bar = self.title + self.char0 + br + self.char1 - if self.percent < 100: - ps = " " + str(self.percent_int) + "%" - else: - ps = "" - self.bar += ps diff --git a/src/__init__.py b/src/__init__.py index d431a0bb..6745185f 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,9 +1 @@ -from os.path import dirname -from os import listdir - -package = dirname(__file__) -exceptions = ['__init__.py', '__pycache__'] -modules = [module for module in listdir(package) if module not in exceptions] -__all__ = [] -for module in sorted(modules): - __all__.append(module) +__import__('pkg_resources').declare_namespace('DeepPhysX') diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 00000000..44c21431 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,163 @@ +from argparse import ArgumentParser +from json import load +from os import listdir, getcwd, readlink, chdir +from os.path import islink, dirname, join, abspath, isdir +from platform import system +from shutil import copytree, rmtree +from subprocess import run +from sys import executable + + +def is_pip_installed(): + + import DeepPhysX.Core + return not islink(DeepPhysX.Core.__path__[0]) + + +def get_sources(): + + import DeepPhysX + site_packages = dirname(DeepPhysX.__path__[0]) + metadata_repo = [f for f in listdir(site_packages) if f.split('-')[0] == 'DeepPhysX' and + ('.dist-info' in f or '.egg-info' in f)] + if len(metadata_repo) == 0: + quit(print("The project does not seem to be properly installed. Try to re-install using 'pip'.")) + elif len(metadata_repo) > 1: + quit(print("There might be several version of the project, try to clean your site-packages.")) + metadata_repo = metadata_repo.pop(0) + if 'direct_url.json' not in listdir(join(site_packages, metadata_repo)): + return None + with open(join(site_packages, metadata_repo, 'direct_url.json'), 'r') as file: + direct_url = load(file) + if system() == 'Linux': + return abspath(direct_url['url'].split('//')[1]) + elif system() == 'Windows': + return abspath(direct_url['url'].split('///')[1]) + else: + return abspath(direct_url['url'].split('///')[1]) + + +def copy_examples_dir(): + + user = input(f"WARNING: The project was installed with pip, examples must be run in a new repository to avoid " + f"writing data in your installation of SSD. Allow the creation of this new repository " + f"'{join(getcwd(), 'DPX_examples')}' to run examples (use 'DPX --clean' to cleanly" + f"remove it afterward) (y/n):") + if user.lower() not in ['y', 'yes']: + quit(print("Aborting.")) + + import DeepPhysX.examples + copytree(src=DeepPhysX.examples.__path__[0], + dst=join(getcwd(), 'DPX_examples')) + + +def clean_examples_dir(): + + if not isdir(examples_dir := join(getcwd(), 'DPX_examples')): + quit(print(f"The directory '{examples_dir}' does not exists.")) + user = input(f"Do you want to remove the repository '{examples_dir}' (y/n):") + if user.lower() not in ['y', 'yes']: + quit(print("Aborting.")) + rmtree(examples_dir) + + +def print_available_examples(examples): + + example_names = sorted(list(examples.keys())) + example_per_repo = {} + for example_name in example_names: + if type(examples[example_name]) == str: + root, repo = examples[example_name].split('.')[0], examples[example_name].split('.')[1] + else: + root, repo = examples[example_name][0].split('.')[0], examples[example_name][0].split('.')[1] + repo = 'rendering' if repo == 'rendering-offscreen' else repo + if root not in example_per_repo: + example_per_repo[root] = {} + if repo not in example_per_repo[root]: + example_per_repo[root][repo] = [] + example_per_repo[root][repo].append(example_name) + + description = '\navailable examples:' + for repo, sub_repos in example_per_repo.items(): + for sub_repo, names in sub_repos.items(): + description += f'\n {repo}.{sub_repo}: {names}' + print(description) + + +def execute_cli(): + + description = "Command Line Interface dedicated to DPX examples." + parser = ArgumentParser(prog='SSD', description=description) + parser.add_argument('-c', '--clean', help='clean the example repository.', action='store_true') + parser.add_argument('-g', '--get', help='get the full example repository locally.', action='store_true') + parser.add_argument('-r', '--run', type=str, help='run one of the demo sessions.', metavar='') + args = parser.parse_args() + + # Get a copy of the example repository if pip installed from PyPi.org + if args.get: + # Installed with setup_dev.py + if not is_pip_installed(): + quit(print("The project was installed from sources in dev mode, examples will then be run in " + "'DeepPhysX..examples'.")) + # Installed with pip from sources + if (source_dir := get_sources()) is not None: + quit(print(f"The project was installed with pip from sources, examples will then be run in " + f"'{join(source_dir, 'examples')}'.")) + # Installed with pip from PyPi + copy_examples_dir() + return + + # Clean the examples repository if pip installed from PyPi.org + elif args.clean: + # Installed with setup_dev.py + if not is_pip_installed(): + quit(print("The project was installed from sources in dev mode, you cannot clean " + "'DPX..examples'.")) + # Installed with pip from sources + if (source_dir := get_sources()) is not None: + quit(print(f"The project was installed with pip from sources, you cannot clean " + f"'{join(source_dir, 'examples')}'.")) + # Installed with pip from PyPi + clean_examples_dir() + return + + examples = {'armadillo': 'Core/demos/Armadillo/FC/interactive.py', + 'beam': 'Core/demos/Beam/FC/interactive.py', + 'liver': 'Core/demos/Liver/FC/interactive.py'} + + # Run a demo script + if (example := args.run) is not None: + # Check the example name + if example.lower() not in examples.keys(): + print(f"Unknown demo '{example}'.") + quit(print_available_examples(examples)) + # Get the example directory + if not is_pip_installed(): + import DeepPhysX.Core + source_dir = readlink(DeepPhysX.Core.__path__[0]) + examples_dir = join(dirname(dirname(source_dir)), 'examples') + repo = join(*examples[example].split('/')[1:-1]) + elif (source_dir := get_sources()) is not None: + examples_dir = join(source_dir, 'examples') + repo = join(*examples[example].split('/')[1:-1]) + else: + if not isdir(join(getcwd(), 'DPX_examples')): + print(f"The directory '{join(getcwd(), 'DPX_examples')}' does not exists.") + copy_examples_dir() + examples_dir = join(getcwd(), 'DPX_examples') + repo = join(*examples[example].split('/')[:-1]) + # Run the example + script = examples[example].split('/')[-1] + chdir(join(examples_dir, repo)) + run([f'{executable}', f'{script}'], cwd=join(examples_dir, repo)) + + return + + # No command + else: + parser.print_help() + print_available_examples(examples) + + +if __name__ == '__main__': + execute_cli()