diff --git a/.goreleaser.yml b/.goreleaser.yml index b322d0b..e84fa8b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,11 +1,11 @@ -project_name: cloudcat +project_name: ski env: - GO111MODULE=on builds: - - id: cloudcat - main: ./cmd + - id: ski + main: ./ski env: - CGO_ENABLED=0 ldflags: @@ -14,7 +14,7 @@ builds: - -X main.CommitSHA={{ .ShortCommit }} flags: - -trimpath - binary: cloudcat + binary: ski goos: - darwin - linux diff --git a/LICENSE b/LICENSE index 0ad25db..a2bd002 100644 --- a/LICENSE +++ b/LICENSE @@ -1,661 +1,19 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. +Copyright (c) 2024 shiroyk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile index 2105b47..abff014 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ LDFLAGS += -X "main.Version=$(VERSION)" -X "main.CommitSHA=$(VERSION_HASH)" all: build build: - cd cmd && go build -ldflags '$(LDFLAGS)' -o ../dist/cloudcat && cd .. + go build -ldflags '$(LDFLAGS)' -o ./dist/ski ./ski format: find . -name '*.go' -exec gofmt -s -w {} + diff --git a/README.md b/README.md index 06f8957..b1b2a31 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ -# Cloudcat -![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/shiroyk/cloudcat) -[![Go Report Card](https://goreportcard.com/badge/github.com/shiroyk/cloudcat)](https://goreportcard.com/report/github.com/shiroyk/cloudcat) -![GitHub](https://img.shields.io/github/license/shiroyk/cloudcat)
-**Cloudcat** is a tool for extracting structured data from websites using extensible YAML syntax rules.
-Before v1.0.0 is for initial development. Anything MAY change at any time. The public API SHOULD NOT be considered stable. +# ski +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/shiroyk/ski) +[![Go Report Card](https://goreportcard.com/badge/github.com/shiroyk/ski)](https://goreportcard.com/report/github.com/shiroyk/ski) +![GitHub](https://img.shields.io/github/license/shiroyk/ski)
+**ski** is a tool written in Golang for extracting structured data. ## Documentation -See [Wiki](https://github.com/shiroyk/cloudcat/wiki) +See [Wiki](https://github.com/shiroyk/ski/wiki) ## License -cloudcat is distributed under the [AGPL-3.0 license](https://github.com/shiroyk/cloudcat/blob/master/LICENSE.md). \ No newline at end of file +ski is distributed under the [AGPL-3.0 license](https://github.com/shiroyk/ski/blob/master/LICENSE.md). \ No newline at end of file diff --git a/analyzer.go b/analyzer.go deleted file mode 100644 index 2d47980..0000000 --- a/analyzer.go +++ /dev/null @@ -1,329 +0,0 @@ -package cloudcat - -import ( - "encoding/json" - "fmt" - "log/slog" - "runtime/debug" - "strings" - "sync/atomic" - - "github.com/shiroyk/cloudcat/plugin" - "github.com/spf13/cast" -) - -var attr = slog.String("source", "analyze") - -var defaultAnalyzer atomic.Value - -func init() { - defaultAnalyzer.Store(NewAnalyzer(new(defaultFormatHandler), true)) -} - -// SetAnalyzer sets the default Analyzer -func SetAnalyzer(analyzer Analyzer) { - defaultAnalyzer.Store(analyzer) -} - -// Analyze a Schema with default Analyzer, returns the result. -func Analyze(ctx *plugin.Context, schema *Schema, content string) any { - return defaultAnalyzer.Load().(Analyzer).Analyze(ctx, schema, content) -} - -// Analyzer the schema with content. -type Analyzer interface { - // Analyze a Schema, returns the result. - Analyze(ctx *plugin.Context, schema *Schema, content string) any -} - -// NewAnalyzer creates a new analyzer -func NewAnalyzer(formatter FormatHandler, debug bool) Analyzer { - return &analyzer{formatter, debug} -} - -type analyzer struct { - formatter FormatHandler - debug bool -} - -// Analyze a Schema, returns the result -func (a *analyzer) Analyze(ctx *plugin.Context, schema *Schema, content string) any { - defer func() { - if r := recover(); r != nil { - ctx.Logger().Error(fmt.Sprintf("analyze error %s", r), - "stack", string(debug.Stack()), attr) - } - }() - return a.analyze(ctx, schema, content, "$") -} - -// analyze execute the corresponding to analyze by schema.Type -func (a *analyzer) analyze( - ctx *plugin.Context, - schema *Schema, - content any, - path string, // the path of properties -) any { - switch schema.Type { - default: - return nil - case StringType, IntegerType, NumberType, BooleanType: - return a.string(ctx, schema, content, path) - case ObjectType: - return a.object(ctx, schema, content, path) - case ArrayType: - return a.array(ctx, schema, content, path) - } -} - -// string get string or slice string and convert it to the specified type. -// If the type is not schema.StringType then convert to the specified type. -// -//nolint:nakedret -func (a *analyzer) string( - ctx *plugin.Context, - schema *Schema, - content any, - path string, // the path of properties -) (ret any) { - var err error - if schema.Type == ArrayType { //nolint:nestif - ret, err = GetStrings(ctx, schema.Rule, content) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed analyze %s", path), "error", err, attr) - return nil - } - if a.debug { - ctx.Logger().Debug("parse", "path", path, "result", ret, attr) - } - } else { - ret, err = GetString(ctx, schema.Rule, content) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed analyze %s", path), "error", err, attr) - return nil - } - if a.debug { - ctx.Logger().Debug("parse", "path", path, "result", ret, attr) - } - - if schema.Type != StringType { - ret, err = a.formatter.Format(ret, schema.Type) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed format %s %v to %v", - path, ret, schema.Type), "error", err, attr) - return - } - if a.debug { - ctx.Logger().Debug("format", "path", path, "result", ret, attr) - } - } - } - - if schema.Format != "" { - ret, err = a.formatter.Format(ret, schema.Format) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed format %s %v to %v", - path, ret, schema.Format), "error", err, attr) - return - } - if a.debug { - ctx.Logger().Debug("format", "path", path, "result", ret, attr) - } - } - - return -} - -// object get object. -// If properties is not empty, execute init to get the object element then analyze properties. -// If rule is not empty, execute string to get object. -func (a *analyzer) object( - ctx *plugin.Context, - schema *Schema, - content any, - path string, // the path of properties -) (ret any) { - if schema.Properties == nil { - return a.string(ctx, &Schema{ - Type: ObjectType, - Format: schema.Format, - Rule: schema.Rule, - }, content, path) - } - - var object map[string]any - ks, k := schema.Properties["$key"] - vs, v := schema.Properties["$value"] - if after, ae := schema.Properties["$after"]; ae { - defer func() { - _, err := GetString(ctx, after.Rule, object) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed to analyze after %v", path), "error", err, attr) - } - }() - } - if k && v { - elements := a.init(ctx, schema.Init, ArrayType, content, path) - if len(elements) == 0 { - return - } - object = make(map[string]any, len(elements)) - for i, element := range elements { - key, err := GetString(ctx, ks.Rule, element) - keyPath := fmt.Sprintf("%s.[%v].value", path, i) - if a.debug { - ctx.Logger().Debug("parse", "path", keyPath, "result", key, attr) - } - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed to analyze key %v", keyPath), "error", err, attr) - return - } - object[key] = a.analyze(ctx, &vs, element, keyPath) - } - return object - } - - element := a.init(ctx, schema.Init, schema.Type, content, path) - if len(element) == 0 { - return - } - object = make(map[string]any, len(schema.Properties)) - - for field, fieldSchema := range schema.Properties { - if strings.HasPrefix(field, "$") { - continue - } - object[field] = a.analyze(ctx, &fieldSchema, element[0], path+"."+field) //nolint:gosec - } - - return object -} - -// array get array. -// If properties is not empty, execute init to get the slice of elements then analyze properties. -// If rule is not empty, execute string to get array -func (a *analyzer) array( - ctx *plugin.Context, - s *Schema, - content any, - path string, // the path of properties -) any { - if s.Properties != nil { - elements := a.init(ctx, s.Init, s.Type, content, path) - array := make([]any, len(elements)) - - for i, item := range elements { - newSchema := NewSchema(ObjectType).SetProperty(s.Properties) - array[i] = a.object(ctx, newSchema, item, fmt.Sprintf("%s.[%v]", path, i)) - } - - return array - } - - return a.string(ctx, &Schema{ - Type: ArrayType, - Format: s.Format, - Rule: s.Rule, - }, content, path) -} - -// init get elements -func (a *analyzer) init( - ctx *plugin.Context, - init Action, - typ Type, - content any, - path string, // the path of properties -) (ret []string) { - if init == nil { - switch data := content.(type) { - case []string: - return data - case string: - return []string{data} - default: - ctx.Logger().Error(fmt.Sprintf("failed analyze init %s", path), - "error", fmt.Errorf("unexpected content type %T", content), attr) - return - } - } - - if typ == ArrayType { - elements, err := GetElements(ctx, init, content) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed analyze init %s", path), "error", err, attr) - return - } - if a.debug { - ctx.Logger().Debug("init", "path", path, "result", strings.Join(elements, "\n"), attr) - } - return elements - } - - element, err := GetElement(ctx, init, content) - if err != nil { - ctx.Logger().Error(fmt.Sprintf("failed analyze init %s", path), "error", err, attr) - return - } - if a.debug { - ctx.Logger().Debug("init", "path", path, "result", element, attr) - } - return []string{element} -} - -// FormatHandler schema property formatter -type FormatHandler interface { - // Format the data to the given Type - Format(data any, format Type) (any, error) -} - -type defaultFormatHandler struct{} - -// Format the data to the given Type -func (f defaultFormatHandler) Format(data any, format Type) (ret any, err error) { - switch ori := data.(type) { - case string: - if ori == "" && format != StringType { - return - } - switch format { - case StringType: - return ori, nil - case IntegerType: - ret, err = cast.ToIntE(ori) - case NumberType: - ret, err = cast.ToFloat64E(ori) - case BooleanType: - ret, err = cast.ToBoolE(ori) - case ArrayType: - ret = make([]any, 0) - err = json.Unmarshal([]byte(ori), &ret) - case ObjectType: - ret = make(map[string]any) - err = json.Unmarshal([]byte(ori), &ret) - } - if err != nil { - return nil, err - } - return - case []string: - slice := make([]any, len(ori)) - for i, o := range ori { - slice[i], err = f.Format(o, format) - if err != nil { - return nil, err - } - } - return slice, nil - case map[string]any: - maps := make(map[string]any, len(ori)) - for k, v := range ori { - maps[k], err = f.Format(v, format) - if err != nil { - return nil, err - } - } - return maps, nil - } - return nil, fmt.Errorf("unable format type %T to %s", data, format) -} diff --git a/analyzer_test.go b/analyzer_test.go deleted file mode 100644 index fd9fafd..0000000 --- a/analyzer_test.go +++ /dev/null @@ -1,304 +0,0 @@ -package cloudcat - -import ( - "encoding/json" - "fmt" - "strconv" - "testing" - - "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" -) - -func eval() func(any, string) (any, error) { - rt := goja.New() - program := goja.MustCompile("", "(c, code)=>eval(code)", false) - callable, err := rt.RunProgram(program) - if err != nil { - panic(err) - } - call, ok := goja.AssertFunction(callable) - if !ok { - panic("err init executor") - } - return func(c any, a string) (any, error) { - value, err := call(goja.Undefined(), rt.ToValue(c), rt.ToValue(a)) - if err != nil { - return nil, err - } - if value == nil { - return nil, nil - } - return value.Export(), nil - } -} - -type ap struct { - eval func(any, string) (any, error) -} - -func (p *ap) GetString(_ *plugin.Context, c any, a string) (string, error) { - v, err := p.eval(c, a) - if err != nil { - return "", err - } - if v == nil { - return "", nil - } - if s, ok := v.(string); ok { - return s, nil - } - bytes, err := json.Marshal(v) - if err != nil { - return "", err - } - return string(bytes), nil -} - -func (p *ap) GetStrings(_ *plugin.Context, c any, a string) ([]string, error) { - v, err := p.eval(c, a) - if err != nil { - return nil, err - } - slice, ok := v.([]any) - if !ok { - slice = []any{v} - } - ret := make([]string, len(slice)) - for i, v := range slice { - if s, ok := v.(string); ok { - ret[i] = s - } else { - bytes, _ := json.Marshal(v) - ret[i] = string(bytes) - } - } - return ret, nil -} - -func (p *ap) GetElement(_ *plugin.Context, c any, a string) (string, error) { - return p.GetString(nil, c, a) -} - -func (p *ap) GetElements(_ *plugin.Context, c any, a string) ([]string, error) { - return p.GetStrings(nil, c, a) -} - -func TestAnalyzer(t *testing.T) { - ctx := plugin.NewContext(plugin.ContextOptions{}) - parser.Register("ap", &ap{eval()}) - testCases := []struct { - schema string - want any - }{ - { - ` -{ ap: '"foo"' } -`, `"foo"`, - }, - { - ` -- ap: "null" -- or -- ap: '"foo"' -`, `"foo"`, - }, - { - ` -- ap: "null" -- or -- ap: "null" -- or -- ap: '"foo"' -`, `"foo"`, - }, - { - ` -- ap: '"foo"' -- and -- ap: '"bar"' -`, `"foobar"`, - }, - { - ` -- ap: '"foo"' -- and -- ap: '"bar"' -- and -- ap: '"aaa"' -`, `"foobaraaa"`, - }, - { - ` -type: integer -rule: { ap: '1' } -`, 1, - }, - { - ` -type: boolean -rule: { ap: '1' } -`, true, - }, - { - ` -type: number -rule: { ap: '2.1' } -`, 2.1, - }, - { - ` -type: array -rule: - - ap: '1' - - and - - ap: '2' -`, `["1","2"]`, - }, - { - ` -type: object -properties: - string: { ap: '"str"' } - integer: !integer { ap: '1' } - number: !number { ap: '1.1' } - boolean: !boolean { ap: '1' } - array: !array { ap: "[\"i1\", \"i2\"]" } - object: !object { ap: "({\"foo\":\"bar\"})" } -`, `{"array":["i1","i2"],"boolean":true,"integer":1,"number":1.1,"object":{"foo":"bar"},"string":"str"}`, - }, - { - ` -type: object -format: number -rule: { ap: '({"foo":"1.1"})' } -`, `{"foo":1.1}`, - }, - { - ` -type: array -properties: - n: !number { ap: '12' } -`, `[{"n":12}]`, - }, - { - ` -type: array -format: number -rule: { ap: "1" } -`, `[1]`, - }, - { - ` -type: object -properties: - ? ap: '"k"' - : ap: '"v"' -`, `{"k":"v"}`, - }, - { - ` -type: object -properties: - $key: { ap: '"k"' } - $value: { ap: '"v"' } -`, `{"k":"v"}`, - }, - { - ` -type: object -init: { ap: "[1,2,3]" } -properties: - ? ap: c - : ap: c + 1 -`, `{"1":"11", "2":"21", "3":"31"}`, - }, - { - ` -type: object -init: { ap: '["a","b","c",1,2,3]' } -properties: - $key: { ap: c } - $value: { ap: c } -`, `{"1":"1", "2":"2", "3":"3", "a":"a", "b":"b", "c":"c"}`, - }, - { - ` -type: object -properties: - num: !integer { ap: '2' } - msg: { ap: '"foooo"' } - $after: { ap: c.num = c.num + 1; c.msg = "hello" } -`, `{"num":3,"msg":"hello"}`, - }, - } - for i, testCase := range testCases { - t.Run(strconv.Itoa(i), func(t *testing.T) { - s := new(Schema) - err := yaml.Unmarshal([]byte(testCase.schema), s) - if err != nil { - t.Fatal(err) - } - result := Analyze(ctx, s, "") - if want, ok := testCase.want.(string); ok { - bytes, err := json.Marshal(result) - assert.NoError(t, err) - assert.JSONEq(t, want, string(bytes)) - return - } - assert.Equal(t, testCase.want, result) - }) - } -} - -func TestFormat(t *testing.T) { - t.Parallel() - formatter := new(defaultFormatHandler) - testCases := []struct { - data any - typ Type - want any - }{ - {"", StringType, ""}, - {"1", StringType, "1"}, - {"2.1", NumberType, 2.1}, - {"", NumberType, nil}, - {"3", IntegerType, 3}, - {"1", BooleanType, true}, - {"", BooleanType, nil}, - {`{"k":"v"}`, ObjectType, map[string]any{"k": "v"}}, - {`[1,2]`, ArrayType, []any{1.0, 2.0}}, - {[]string{"1", "2"}, IntegerType, []any{1, 2}}, - {map[string]any{"k": "1"}, IntegerType, map[string]any{"k": 1}}, - } - - for i, testCase := range testCases { - t.Run(fmt.Sprintf("Cases %v", i), func(t *testing.T) { - got, err := formatter.Format(testCase.data, testCase.typ) - assert.NoError(t, err) - assert.Equal(t, testCase.want, got) - }) - } - - errCases := []struct { - data any - typ Type - want any - }{ - {"9-", IntegerType, nil}, - {"114", BooleanType, nil}, - {[]string{"1", "?"}, IntegerType, nil}, - {map[string]any{"k": "!"}, NumberType, nil}, - } - - for i, testCase := range errCases { - t.Run(fmt.Sprintf("Err cases %v", i), func(t *testing.T) { - got, err := formatter.Format(testCase.data, testCase.typ) - assert.Error(t, err) - assert.Equal(t, testCase.want, got) - }) - } -} diff --git a/cache.go b/cache.go index 4717ecd..f739ed7 100644 --- a/cache.go +++ b/cache.go @@ -1,35 +1,30 @@ -package cloudcat +package ski import ( "context" "sync" "time" - "github.com/shiroyk/cloudcat/plugin" "github.com/spf13/cast" ) // A Cache interface is used to store bytes. type Cache interface { - Get(ctx context.Context, key string) ([]byte, bool) - Set(ctx context.Context, key string, value []byte) - Del(ctx context.Context, key string) + Get(ctx context.Context, key string) ([]byte, error) + Set(ctx context.Context, key string, value []byte) error + Del(ctx context.Context, key string) error } -type cacheTimeoutKey struct{} +var cacheTimeoutKey byte // WithCacheTimeout returns the context with the cache timeout. func WithCacheTimeout(ctx context.Context, timeout time.Duration) context.Context { - if c, ok := ctx.(*plugin.Context); ok { - c.SetValue(cacheTimeoutKey{}, timeout) - return ctx - } - return context.WithValue(ctx, cacheTimeoutKey{}, timeout) + return WithValue(ctx, &cacheTimeoutKey, timeout) } -// CacheTimeout returns the context cache timeout value. +// CacheTimeout returns the context cache timeout values. func CacheTimeout(ctx context.Context) time.Duration { - return cast.ToDuration(ctx.Value(cacheTimeoutKey{})) + return cast.ToDuration(ctx.Value(&cacheTimeoutKey)) } // memoryCache is an implementation of Cache that stores bytes in in-memory. @@ -40,38 +35,40 @@ type memoryCache struct { } // Get returns the []byte and true, if not existing returns false. -func (c *memoryCache) Get(_ context.Context, key string) ([]byte, bool) { +func (c *memoryCache) Get(_ context.Context, key string) ([]byte, error) { c.Lock() defer c.Unlock() if ddl, exist := c.timeout[key]; exist { if time.Now().Unix() > ddl { delete(c.items, key) delete(c.timeout, key) - return []byte{}, false + return []byte{}, nil } } if b, ok := c.items[key]; ok { - return b, true + return b, nil } - return []byte{}, false + return nil, nil } // Set saves []byte to the cache with key -func (c *memoryCache) Set(ctx context.Context, key string, value []byte) { +func (c *memoryCache) Set(ctx context.Context, key string, value []byte) error { c.Lock() + defer c.Unlock() c.items[key] = value if timeout := CacheTimeout(ctx); timeout > 0 { c.timeout[key] = time.Now().Add(timeout).Unix() } - c.Unlock() + return nil } // Del removes key from the cache -func (c *memoryCache) Del(_ context.Context, key string) { +func (c *memoryCache) Del(_ context.Context, key string) error { c.Lock() + defer c.Unlock() delete(c.items, key) delete(c.timeout, key) - c.Unlock() + return nil } // NewCache returns a new Cache that will store items in in-memory. diff --git a/cache_test.go b/cache_test.go index 893a1d4..b95687c 100644 --- a/cache_test.go +++ b/cache_test.go @@ -1,4 +1,4 @@ -package cloudcat +package ski import ( "context" @@ -14,26 +14,24 @@ func TestCache(t *testing.T) { ctx := context.Background() key, value := "testCacheKey", "testCacheValue" - if _, ok := c.Get(ctx, key); ok { - t.Fatal("retrieved value before adding it") + if v, _ := c.Get(ctx, key); len(v) != 0 { + t.Fatal("retrieved values before adding it") } - c.Set(ctx, key, []byte(value)) + _ = c.Set(ctx, key, []byte(value)) v, _ := c.Get(ctx, key) assert.Equal(t, value, string(v)) - c.Del(ctx, key) - if _, ok := c.Get(ctx, key); ok { - t.Fatal("delete failed") - } + _ = c.Del(ctx, key) + v, _ = c.Get(ctx, key) + assert.Empty(t, v) - c.Set(WithCacheTimeout(ctx, time.Millisecond), key, []byte(value)) + _ = c.Set(WithCacheTimeout(ctx, time.Millisecond), key, []byte(value)) v1, _ := c.Get(ctx, key) assert.Equal(t, value, string(v1)) time.Sleep(1 * time.Second) - if _, ok := c.Get(ctx, key); ok { - t.Fatalf("not expired: %v", key) - } + v, _ = c.Get(ctx, key) + assert.Empty(t, v, "not expired: %v", key) } diff --git a/cmd/README.md b/cmd/README.md deleted file mode 100644 index aa8e45b..0000000 --- a/cmd/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Cloudcat - -## Usage -run the **Model** -```shell -cat << EOF | ./cloudcat -d -m - -source: - name: HackerNews - http: https://news.ycombinator.com/best -schema: - type: array - init: - - gq: "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')" - js: | - content?.reduce((acc, v, i, arr) => { - if (i % 2 === 0) { - acc.push(arr.slice(i, i + 2).join('')); - } - return acc; - }, []) - properties: - index: !integer - - gq: .rank - regex: /[^\d]/ - title: { gq: .titleline>:first-child } - by: { gq: .hnuser } - age: { gq: .age } - comments: !integer - - gq: .subline>:last-child - regex: /[^\d]/ -EOF -``` -run the **Script** -```shell -cat << EOF | ./cloudcat -d -s - -const http = require('cloudcat/http'); -let res = http.get('https://news.ycombinator.com/best'); -let stories = cat.getElements('gq', "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')", res.string()); -stories?.reduce((acc, v, i, arr) => { - if (i % 2 === 0) { - let item = arr.slice(i, i + 2).join(''); - let index = cat.getString('gq', '.rank', item); - let title = cat.getString('gq', '.titleline>:first-child', item); - let by = cat.getString('gq', '.hnuser', item); - let age = cat.getString('gq', '.age', item); - let comments = cat.getString('gq', '.subline>:last-child', item); - acc.push({ - index: parseInt(index?.replace(/[^\d]+/g, ''), 10), - title: title, - by: by, - age: age, - comments: parseInt(comments?.replace(/[^\d]+/g, ''), 10) - }); - } - return acc; -}, []); -EOF -``` \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index e2dc641..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,239 +0,0 @@ -package main - -import ( - "bufio" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "net/http" - urlpkg "net/url" - "os" - "path/filepath" - "strings" - - "log/slog" - - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - _ "github.com/shiroyk/cloudcat/js/modules" - _ "github.com/shiroyk/cloudcat/parsers" - "github.com/shiroyk/cloudcat/plugin" - "gopkg.in/yaml.v3" -) - -// Model the model -type Model struct { - Source struct { - Name string `yaml:"name"` - HTTP string `yaml:"http"` - Proxy string `yaml:"proxy"` - } `yaml:"source"` - Schema *cloudcat.Schema `yaml:"schema"` -} - -var ( - scriptFlag = flag.String("s", "", "run script") - modelFlag = flag.String("m", "", "run model") - timeoutFlag = flag.Duration("t", plugin.DefaultTimeout, "run timeout") - debugFlag = flag.Bool("d", false, "output the debug log") - outputFlag = flag.String("o", "", "write to file instead of stdout") - pluginFlag = flag.String("p", "", "plugin directory path") - versionFlag = flag.Bool("v", false, "output version") -) - -func runModel() (err error) { - var bytes []byte - if *modelFlag == "-" { - bytes, err = io.ReadAll(os.Stdin) - } else { - bytes, err = os.ReadFile(*modelFlag) //nolint:gosec - } - if err != nil { - return - } - - var model Model - err = yaml.Unmarshal(bytes, &model) - if err != nil { - return - } - - if model.Source.HTTP == "" || model.Schema == nil { - return errors.New("model is invalid") - } - - timeout := plugin.DefaultTimeout - if timeoutFlag != nil { - timeout = *timeoutFlag - } - - requestURI := model.Source.HTTP - if _, urlErr := urlpkg.Parse(requestURI); urlErr == nil { - requestURI = fmt.Sprintf("GET %s HTTP/1.1\n\n", requestURI) - } - - req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(requestURI))) - if err != nil { - return err - } - req.RequestURI = "" - - ctx := plugin.NewContext(plugin.ContextOptions{ - Timeout: timeout, - Logger: slog.New(loggerHandler()), - URL: req.URL.String(), - }) - defer ctx.Cancel() - - fetch := cloudcat.MustResolve[cloudcat.Fetch]() - - if model.Source.Proxy != "" { - url, err := urlpkg.Parse(model.Source.Proxy) - if err != nil { - return err - } - req = req.WithContext(cloudcat.WithProxyURL(ctx, url)) - } else { - req = req.WithContext(ctx) - } - - res, err := fetch.Do(req) - if err != nil { - return err - } - - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - return err - } - - return outputJSON(cloudcat.Analyze(ctx, model.Schema, string(body))) -} - -func runScript() (err error) { - var bytes []byte - if *scriptFlag == "-" { - bytes, err = io.ReadAll(os.Stdin) - } else { - bytes, err = os.ReadFile(*scriptFlag) //nolint:gosec - } - if err != nil { - return - } - - timeout := plugin.DefaultTimeout - if timeoutFlag != nil { - timeout = *timeoutFlag - } - - ctx := plugin.NewContext(plugin.ContextOptions{ - Timeout: timeout, - Logger: slog.New(loggerHandler()), - }) - defer ctx.Cancel() - - value, err := js.RunString(ctx, string(bytes)) - if err != nil { - return err - } - - return outputJSON(value) -} - -func loggerHandler() slog.Handler { - opt := new(slog.HandlerOptions) - if *debugFlag { - opt.Level = slog.LevelDebug - } - return slog.NewTextHandler(os.Stdout, opt) -} - -func outputJSON(data any) (err error) { - bytes, err := json.MarshalIndent(data, "", "\t") - if err != nil { - return err - } - - if *outputFlag == "" { - fmt.Println(string(bytes)) //nolint:forbidigo - return - } - - ext := filepath.Ext(*outputFlag) - if ext == "" { - *outputFlag += ".json" - } - return os.WriteFile(*outputFlag, bytes, 0o600) -} - -// expandPath expands path "." or "~" -func expandPath(path string) (string, error) { - // expand local directory - if strings.HasPrefix(path, ".") { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - return filepath.Join(cwd, path[1:]), nil - } - // expand ~ as shortcut for home directory - if strings.HasPrefix(path, "~") { - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, path[1:]), nil - } - return path, nil -} - -func main() { - flag.Parse() - - if *versionFlag { - fmt.Println(fmt.Sprintf("cloudcat %v/%v", Version, CommitSHA)) - os.Exit(0) - return - } - - cloudcat.Provide(cloudcat.NewCache()) - cloudcat.Provide(cloudcat.NewCookie()) - cloudcat.ProvideLazy[cloudcat.Fetch](func() (cloudcat.Fetch, error) { - transport := http.DefaultTransport.(*http.Transport) - transport.Proxy = cloudcat.ProxyFromRequest - client := &http.Client{Transport: transport} - return client, nil - }) - - if pluginFlag != nil && *pluginFlag != "" { - pluginPath, err := expandPath(*pluginFlag) - if err != nil { - return - } - size, err := plugin.LoadPlugin(pluginPath) - if err != nil { - if size == 0 { - panic(fmt.Sprintf("failed to load external modules: %v", err)) - } else { - slog.Warn(fmt.Sprintf("failed to load some external modules :%v", err)) - } - } - } - - if scriptFlag != nil && *scriptFlag != "" { - if err := runScript(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - } else if modelFlag != nil && *modelFlag != "" { - if err := runModel(); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - } else { - flag.Usage() - } -} diff --git a/context.go b/context.go new file mode 100644 index 0000000..e22a0f9 --- /dev/null +++ b/context.go @@ -0,0 +1,64 @@ +package ski + +import ( + "context" + "maps" + "sync" +) + +// Context multiple values context +type Context interface { + context.Context + // SetValue store key with value + SetValue(key, value any) +} + +type valuesCtx struct { + context.Context + mu sync.RWMutex + values map[any]any +} + +func (c *valuesCtx) Value(key any) any { + c.mu.RLock() + v, ok := c.values[key] + c.mu.RUnlock() + if ok { + return v + } + return c.Context.Value(key) +} + +func (c *valuesCtx) SetValue(key, value any) { + c.mu.Lock() + c.values[key] = value + c.mu.Unlock() +} + +var _ctxKey byte + +// NewContext returns a new can store multiple values context with values +func NewContext(parent context.Context, values map[any]any) Context { + if parent == nil { + panic("cannot create context from nil parent") + } + var clone map[any]any + if values == nil { + clone = make(map[any]any) + } else { + clone = maps.Clone(values) + } + ctx := &valuesCtx{Context: parent, values: clone} + clone[&_ctxKey] = ctx + return ctx +} + +// WithValue if parent exists multiple values Context then set the key/value. +// or returns a copy of parent in which the value associated with key is val. +func WithValue(ctx context.Context, key, value any) context.Context { + if v, ok := ctx.Value(&_ctxKey).(*valuesCtx); ok { + v.SetValue(key, value) + return ctx + } + return context.WithValue(ctx, key, value) +} diff --git a/context_test.go b/context_test.go new file mode 100644 index 0000000..83beeff --- /dev/null +++ b/context_test.go @@ -0,0 +1,28 @@ +package ski + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewContext(t *testing.T) { + timeout, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + ctx := NewContext(timeout, nil) + assert.Nil(t, ctx.Value("key1")) + + ctx.SetValue("key1", "value1") + assert.Equal(t, "value1", ctx.Value("key1")) + + var key2 byte + ctx.SetValue(&key2, "value2") + assert.Equal(t, "value2", ctx.Value(&key2)) + + WithValue(context.WithValue(ctx, "key3", "value3"), "key4", "value4") + + assert.Equal(t, "value4", ctx.Value("key4")) +} diff --git a/cookie.go b/cookie.go index 0873f2f..ae8b7ce 100644 --- a/cookie.go +++ b/cookie.go @@ -1,4 +1,4 @@ -package cloudcat +package ski import ( "net/http" @@ -6,30 +6,23 @@ import ( "net/url" ) -// Cookie manages storage and use of cookies in HTTP requests. -// Implementations of Cookie must be safe for concurrent use by multiple +// CookieJar manages storage and use of cookies in HTTP requests. +// Implementations of CookieJar must be safe for concurrent use by multiple // goroutines. -type Cookie interface { +type CookieJar interface { http.CookieJar - // CookieString returns the cookies string for the given URL. - CookieString(u *url.URL) []string - // DeleteCookie delete the cookies for the given URL. - DeleteCookie(u *url.URL) + // RemoveCookie delete the cookies for the given URL. + RemoveCookie(u *url.URL) } -// memoryCookie is an implementation of Cookie that stores http.Cookie in in-memory. +// memoryCookie is an implementation of CookieJar that stores http.Cookie in in-memory. type memoryCookie struct { *cookiejar.Jar } -// CookieString returns the cookies string for the given URL. -func (c *memoryCookie) CookieString(u *url.URL) []string { - return CookieToString(c.Cookies(u)) -} - -// DeleteCookie delete the cookies for the given URL. -func (c *memoryCookie) DeleteCookie(u *url.URL) { +// RemoveCookie remove the cookies for the given URL. +func (c *memoryCookie) RemoveCookie(u *url.URL) { exists := c.Cookies(u) cookie := make([]*http.Cookie, 0, len(exists)) for _, e := range exists { @@ -39,11 +32,8 @@ func (c *memoryCookie) DeleteCookie(u *url.URL) { c.SetCookies(u, cookie) } -// NewCookie returns a new Cookie that will store cookies in in-memory. -func NewCookie() Cookie { - jar, err := cookiejar.New(nil) - if err != nil { - panic(err) - } +// NewCookieJar returns a new CookieJar that will store cookies in in-memory. +func NewCookieJar() CookieJar { + jar, _ := cookiejar.New(nil) return &memoryCookie{jar} } diff --git a/cookie_test.go b/cookie_test.go index fd74989..024deca 100644 --- a/cookie_test.go +++ b/cookie_test.go @@ -1,6 +1,7 @@ -package cloudcat +package ski import ( + "net/http" "net/url" "testing" @@ -9,17 +10,13 @@ import ( func TestCookie(t *testing.T) { t.Parallel() - c := NewCookie() + c := NewCookieJar() u, _ := url.Parse("https://github.com") - if len(c.Cookies(u)) > 0 { - t.Fatal("retrieved cookie before adding it") - } - - raw := "has_recent_activity=1; path=/; secure; HttpOnly; SameSite=Lax" - c.SetCookies(u, ParseSetCookie(raw)) - assert.Equal(t, []string{"has_recent_activity=1"}, c.CookieString(u)) - c.DeleteCookie(u) + cookies := []*http.Cookie{{Name: "has_recent_activity", Value: "1", Path: "/", Secure: true}} + c.SetCookies(u, cookies) + assert.NotNil(t, c.Cookies(u)) + c.RemoveCookie(u) assert.Nil(t, c.Cookies(u)) } diff --git a/di.go b/di.go deleted file mode 100644 index fa53e1a..0000000 --- a/di.go +++ /dev/null @@ -1,143 +0,0 @@ -package cloudcat - -import ( - "fmt" - "reflect" - "sync" - "sync/atomic" -) - -// di a simple dependencies injection -// Inspired by https://github.com/samber/do - -var diServices = new(sync.Map) - -type lazyService[T any] struct { - load atomic.Bool - instance T - initFunc func() (T, error) -} - -func (s *lazyService[T]) initOrGet() (instance T, err error) { - if s.load.Load() { - return s.instance, nil - } - if !s.load.Swap(true) { - s.instance, err = s.initFunc() - if err != nil { - s.load.Store(false) - } - } - return s.instance, err -} - -// Provide save the value and return is it saved -func Provide[T any](value T) bool { - return ProvideNamed(getName[T](), value) -} - -// ProvideLazy save the lazy init value and return is it saved -func ProvideLazy[T any](initFunc func() (T, error)) bool { - return ProvideNamed(getName[T](), &lazyService[T]{initFunc: initFunc}) -} - -// ProvideNamed save the value for the name and return is it saved -func ProvideNamed(name string, value any) (ok bool) { - _, ok = diServices.LoadOrStore(name, value) - return !ok -} - -// Override save the value and return is it override -func Override[T any](value T) bool { - return OverrideNamed(getName[T](), value) -} - -// OverrideLazy save the value for the name and return is it override -func OverrideLazy[T any](initFunc func() (T, error)) bool { - return OverrideNamed(getName[T](), &lazyService[T]{initFunc: initFunc}) -} - -// OverrideNamed save the value for the name and return is it override -func OverrideNamed(name string, value any) (ok bool) { - _, ok = diServices.Swap(name, value) - return ok -} - -// Resolve get the value, if not exist returns error -func Resolve[T any]() (T, error) { - return ResolveNamed[T](getName[T]()) -} - -// ResolveLazy get the value lazy once, if not exist returns error -func ResolveLazy[T any]() func() (T, error) { - var ( - once sync.Once - value T - err error - ) - g := func() { - value, err = Resolve[T]() - } - return func() (T, error) { - once.Do(g) - return value, err - } -} - -// ResolveNamed get the value for the name if not exist returns error -func ResolveNamed[T any](name string) (value T, err error) { - if v, exists := diServices.Load(name); exists { - switch target := v.(type) { - case *lazyService[T]: - return target.initOrGet() - case T: - return target, nil - } - return value, fmt.Errorf("%T type assertion to %T error", v, value) - } - - return value, fmt.Errorf("%s not declared", name) -} - -// MustResolve get the value, if not exist create panic -func MustResolve[T any]() T { - value, err := Resolve[T]() - if err != nil { - panic(err) - } - return value -} - -// MustResolveLazy get the value lazy once, if not exist create panic -func MustResolveLazy[T any]() func() T { - g := ResolveLazy[T]() - return func() T { - value, err := g() - if err != nil { - panic(err) - } - return value - } -} - -// MustResolveNamed get the value for the name, if not exist create panic -func MustResolveNamed[T any](name string) T { - value, err := ResolveNamed[T](name) - if err != nil { - panic(err) - } - return value -} - -// getName returns the type name -func getName[T any]() string { - var v T - - // struct - if t := reflect.TypeOf(v); t != nil { - return t.String() - } - - // interface - return reflect.TypeOf(new(T)).String() -} diff --git a/di_test.go b/di_test.go deleted file mode 100644 index 83cb0a0..0000000 --- a/di_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package cloudcat - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestProvide(t *testing.T) { - t.Parallel() - - type test1 interface{} - Provide(new(test1)) - _, err := Resolve[test1]() - assert.NoError(t, err) -} - -func TestProvideLazy(t *testing.T) { - t.Parallel() - - times := 0 - type test2 interface{} - ProvideLazy(func() (test2, error) { - if times == 0 { - times++ - return nil, errors.New("something") - } - return new(test2), nil //nolint:nilnil - }) - v, _ := Resolve[test2]() - assert.Nil(t, v) - v, _ = Resolve[test2]() - assert.NotNil(t, v) -} - -func TestResolve(t *testing.T) { - t.Parallel() - - type test3 struct{} - Provide(test3{}) - _, err := Resolve[test3]() - assert.NoError(t, err) -} - -func TestResolveLazy(t *testing.T) { - t.Parallel() - - type test7 struct{} - Provide(test7{}) - f := ResolveLazy[test7]() - _, err := f() - assert.NoError(t, err) - - times := 0 - type test8 interface{} - ProvideLazy(func() (test8, error) { - if times == 0 { - times++ - return nil, errors.New("something") - } - return new(test8), nil //nolint:nilnil - }) - f2 := ResolveLazy[test8]() - v, _ := f2() - assert.Nil(t, v) - v, _ = f2() - assert.Nil(t, v) -} - -func TestMustResolve(t *testing.T) { - t.Parallel() - - type test4 interface{} - Provide(new(test4)) - MustResolve[test4]() -} - -func TestMustResolveLazy(t *testing.T) { - t.Parallel() - defer func() { - r := recover() - assert.NotNil(t, r) - assert.ErrorContains(t, r.(error), "test10 not declared") - }() - - type test9 interface{} - Provide(new(test9)) - assert.NotNil(t, MustResolveLazy[test9]()()) - type test10 interface{} - assert.NotNil(t, MustResolveLazy[test10]()()) -} - -func TestMustResolveNamed(t *testing.T) { - t.Parallel() - - type test5 struct{} - assert.True(t, ProvideNamed("named1", test5{})) - assert.False(t, ProvideNamed("named1", test5{})) - MustResolveNamed[test5]("named1") -} - -func TestOverride(t *testing.T) { - t.Parallel() - - type test6 struct{} - assert.False(t, Override(test6{})) - assert.True(t, Override(test6{})) - MustResolve[test6]() -} diff --git a/fetch.go b/fetch.go index 5fe0396..8ffac7a 100644 --- a/fetch.go +++ b/fetch.go @@ -1,11 +1,11 @@ -package cloudcat +package ski import ( "context" + "net" "net/http" "net/url" - - "github.com/shiroyk/cloudcat/plugin" + "time" ) // Fetch http client interface @@ -16,23 +16,42 @@ type Fetch interface { Do(*http.Request) (*http.Response, error) } -type requestProxyKey struct{} +// NewFetch return the http.Client implementation +func NewFetch() Fetch { + return &http.Client{ + Transport: &http.Transport{ + Proxy: ProxyFromRequest, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + Jar: NewCookieJar(), + } +} + +var requestProxyKey byte // WithProxyURL returns a copy of parent context in which the proxy associated with context. func WithProxyURL(ctx context.Context, proxy *url.URL) context.Context { if proxy == nil { return ctx } - if c, ok := ctx.(*plugin.Context); ok { - c.SetValue(requestProxyKey{}, proxy) - return ctx + if c, ok := ctx.(Context); ok { + c.SetValue(&requestProxyKey, proxy) + return c } - return context.WithValue(ctx, requestProxyKey{}, proxy) + return context.WithValue(ctx, &requestProxyKey, proxy) } // ProxyFromContext returns a proxy URL on context. func ProxyFromContext(ctx context.Context) *url.URL { - if proxy := ctx.Value(requestProxyKey{}); proxy != nil { + if proxy := ctx.Value(&requestProxyKey); proxy != nil { return proxy.(*url.URL) } return nil diff --git a/go.mod b/go.mod index 5999a50..829ad4f 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,29 @@ -module github.com/shiroyk/cloudcat +module github.com/shiroyk/ski go 1.21 require ( - github.com/PuerkitoBio/goquery v1.8.1 + github.com/PuerkitoBio/goquery v1.9.1 + github.com/andybalholm/cascadia v1.3.2 github.com/antchfx/htmlquery v1.3.0 - github.com/dlclark/regexp2 v1.10.0 - github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d - github.com/ohler55/ojg v1.21.0 - github.com/shiroyk/cloudcat/plugin v0.4.0 + github.com/antchfx/xpath v1.2.5 + github.com/dlclark/regexp2 v1.11.0 + github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 + github.com/ohler55/ojg v1.21.4 github.com/spf13/cast v1.6.0 - github.com/stretchr/testify v1.8.4 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.19.0 + github.com/stretchr/testify v1.9.0 + golang.org/x/crypto v0.21.0 + golang.org/x/net v0.22.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/antchfx/xpath v1.2.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect + github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/text v0.14.0 // indirect ) -replace ( - github.com/dop251/goja => github.com/mstoykov/goja v0.0.0-20231212144616-08f562ee86d0 - github.com/shiroyk/cloudcat/plugin => ./plugin -) +replace github.com/dop251/goja => github.com/mstoykov/goja v0.0.0-20231212144616-08f562ee86d0 diff --git a/go.sum b/go.sum index b75bc2d..e4c6b5b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= +github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= @@ -15,11 +14,11 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible h1:bopx7t9jyUNX1ebhr0G4gtQWmUOgwQRI0QsYhdYLgkU= github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -28,8 +27,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk= -github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -39,44 +38,39 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mstoykov/goja v0.0.0-20231115172654-7aaf816c3720 h1:jrBzw98yL3+cwfcqlyyKzquVTUNEHRFcxFwNpd7Bd9U= -github.com/mstoykov/goja v0.0.0-20231115172654-7aaf816c3720/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/mstoykov/goja v0.0.0-20231212144616-08f562ee86d0 h1:AcJZgDvroNJdSX/Ip5hN0P5xhatMwmJBbLHqn3jqjME= github.com/mstoykov/goja v0.0.0-20231212144616-08f562ee86d0/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= -github.com/ohler55/ojg v1.20.3 h1:Z+fnElsA/GbI5oiT726qJaG4Ca9q5l7UO68Qd0PtkD4= -github.com/ohler55/ojg v1.20.3/go.mod h1:uHcD1ErbErC27Zhb5Df2jUjbseLLcmOCo6oxSr3jZxo= +github.com/ohler55/ojg v1.21.4 h1:2iWyz/xExx0XySVIxR9kWFxIdsLNrpWLrKuAcs5aOZU= +github.com/ohler55/ojg v1.21.4/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -91,7 +85,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -105,9 +98,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/js/console.go b/js/console.go index fc5e273..675b9e9 100644 --- a/js/console.go +++ b/js/console.go @@ -1,41 +1,121 @@ package js import ( - "context" + "bytes" + "log/slog" "github.com/dop251/goja" - "log/slog" + "github.com/shiroyk/ski" ) // console implements the js console type console struct{} -// EnableConsole enables the console -func EnableConsole(vm *goja.Runtime) { - _ = vm.Set("console", new(console)) +// EnableConsole enables the console with the slog.Logger +func EnableConsole(rt *goja.Runtime) { + _ = rt.Set("console", new(console)) } -func (c *console) log(level slog.Level, call goja.FunctionCall, vm *goja.Runtime) goja.Value { - slog.Log(context.Background(), level, Format(call, vm).String()) +func (c *console) log(level slog.Level, call goja.FunctionCall, rt *goja.Runtime) goja.Value { + ski.Logger(Context(rt)).Log(Context(rt), level, Format(call, rt).String()) return goja.Undefined() } -// Log calls Logger.Log. -func (c *console) Log(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return c.log(slog.LevelInfo, call, vm) +// Log calls slog.Log. +func (c *console) Log(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + return c.log(slog.LevelInfo, call, rt) +} + +// Info calls slog.Info. +func (c *console) Info(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + return c.log(slog.LevelInfo, call, rt) +} + +// Warn calls slog.Warn. +func (c *console) Warn(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + return c.log(slog.LevelWarn, call, rt) +} + +// Warn calls slog.Error. +func (c *console) Error(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + return c.log(slog.LevelError, call, rt) +} + +// Debug calls slog.Debug. +func (c *console) Debug(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + return c.log(slog.LevelDebug, call, rt) } -// Info calls Logger.Info. -func (c *console) Info(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return c.log(slog.LevelInfo, call, vm) +func runeFormat(rt *goja.Runtime, f rune, val goja.Value, w *bytes.Buffer) bool { + switch f { + case 's': + w.WriteString(val.String()) + case 'd': + w.WriteString(val.ToNumber().String()) + case 'j': + if json, ok := rt.Get("JSON").(*goja.Object); ok { + if stringify, ok := goja.AssertFunction(json.Get("stringify")); ok { + res, err := stringify(json, val) + if err != nil { + panic(err) + } + w.WriteString(res.String()) + } + } + case '%': + w.WriteByte('%') + return false + default: + w.WriteByte('%') + w.WriteRune(f) + return false + } + return true } -// Warn calls Logger.Warn. -func (c *console) Warn(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return c.log(slog.LevelWarn, call, vm) +func bufferFormat(vm *goja.Runtime, b *bytes.Buffer, f string, args ...goja.Value) { + pct := false + argNum := 0 + for _, chr := range f { + if pct { //nolint:nestif + if argNum < len(args) { + if runeFormat(vm, chr, args[argNum], b) { + argNum++ + } + } else { + b.WriteByte('%') + b.WriteRune(chr) + } + pct = false + } else { + if chr == '%' { + pct = true + } else { + b.WriteRune(chr) + } + } + } + + for _, arg := range args[argNum:] { + b.WriteByte(' ') + b.WriteString(arg.String()) + } } -// Warn calls Logger.Error. -func (c *console) Error(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return c.log(slog.LevelError, call, vm) +// Format implements js format +func Format(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + var b bytes.Buffer + var f string + + if arg := call.Argument(0); !goja.IsUndefined(arg) { + f = arg.String() + } + + var args []goja.Value + if len(call.Arguments) > 0 { + args = call.Arguments[1:] + } + bufferFormat(rt, &b, f, args...) + + return rt.ToValue(b.String()) } diff --git a/js/console_test.go b/js/console_test.go index 45f7990..1f525e4 100644 --- a/js/console_test.go +++ b/js/console_test.go @@ -1,21 +1,36 @@ package js import ( + "bytes" + "context" + "log/slog" + "strconv" "testing" - "github.com/dop251/goja" + "github.com/shiroyk/ski" "github.com/stretchr/testify/assert" ) func TestConsole(t *testing.T) { t.Parallel() - vm := goja.New() - vm.SetFieldNameMapper(FieldNameMapper{}) - EnableConsole(vm) + data := new(bytes.Buffer) + vm := NewVM() + ctx := ski.WithLogger(context.Background(), slog.New(slog.NewTextHandler(data, nil))) - _, err := vm.RunString(` - console.log("hello %s", "cloudcat"); - console.log("json %j", {'foo': 'bar'}); - `) - assert.NoError(t, err) + for i, c := range []struct { + str, want string + }{ + {`console.log("hello %s", "ski");`, "hello ski"}, + {`console.log("json %j", {'foo': 'bar'});`, `json {\"foo\":\"bar\"}`}, + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + data.Reset() + vm.Run(ctx, func() { + _, err := vm.Runtime().RunString(c.str) + if assert.NoError(t, err) { + assert.Contains(t, data.String(), c.want) + } + }) + }) + } } diff --git a/js/ctx.go b/js/ctx.go deleted file mode 100644 index 8741ec4..0000000 --- a/js/ctx.go +++ /dev/null @@ -1,113 +0,0 @@ -package js - -import ( - "fmt" - "log/slog" - - "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" -) - -var attr = slog.String("source", "js") - -// ctxWrapper an analyzer context -type ctxWrapper struct { - ctx *plugin.Context - BaseURL string - URL string `js:"url"` -} - -// NewCtxWrapper returns a new ctxWrapper instance -func NewCtxWrapper(vm VM, ctx *plugin.Context) goja.Value { - return vm.Runtime().ToValue(&ctxWrapper{ctx, ctx.BaseURL(), ctx.URL()}) -} - -// Log print the msg to logger -func (c *ctxWrapper) Log(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - c.ctx.Logger().Info(Format(call, vm).String(), attr) - return goja.Undefined() -} - -// Get returns the value associated with this context for key, or nil -// if no value is associated with key. -func (c *ctxWrapper) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return vm.ToValue(c.ctx.Value(call.Argument(0).String())) -} - -// Set value associated with key is val. -func (c *ctxWrapper) Set(key string, value goja.Value) error { - v, err := Unwrap(value) - if err != nil { - return err - } - c.ctx.SetValue(key, v) - return nil -} - -// ClearVar clean all values -func (c *ctxWrapper) ClearVar() { - c.ctx.ClearValue() -} - -// Cancel this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func (c *ctxWrapper) Cancel() { - c.ctx.Cancel() -} - -// GetString gets the string of the content with the given arguments -func (c *ctxWrapper) GetString(key string, rule string, content any) (ret string, err error) { - str, err := ToStrings(content) - if err != nil { - return - } - - if p, ok := parser.GetParser(key); ok { - return p.GetString(c.ctx, str, rule) - } - - return ret, fmt.Errorf("parser %s not found", key) -} - -// GetStrings gets the string of the content with the given arguments -func (c *ctxWrapper) GetStrings(key string, rule string, content any) (ret []string, err error) { - str, err := ToStrings(content) - if err != nil { - return - } - - if p, ok := parser.GetParser(key); ok { - return p.GetStrings(c.ctx, str, rule) - } - - return ret, fmt.Errorf("parser %s not found", key) -} - -// GetElement gets the string of the content with the given arguments -func (c *ctxWrapper) GetElement(key string, rule string, content any) (ret string, err error) { - str, err := ToStrings(content) - if err != nil { - return - } - - if p, ok := parser.GetParser(key); ok { - return p.GetElement(c.ctx, str, rule) - } - - return ret, fmt.Errorf("parser %s not found", key) -} - -// GetElements gets the string of the content with the given arguments -func (c *ctxWrapper) GetElements(key string, rule string, content any) (ret []string, err error) { - str, err := ToStrings(content) - if err != nil { - return - } - - if p, ok := parser.GetParser(key); ok { - return p.GetElements(c.ctx, str, rule) - } - - return ret, fmt.Errorf("parser %s not found", key) -} diff --git a/js/ctx_test.go b/js/ctx_test.go deleted file mode 100644 index 48a1aea..0000000 --- a/js/ctx_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package js - -import ( - "fmt" - "log/slog" - "testing" - - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" -) - -type testParser struct{} - -func (t *testParser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - if str, ok := content.(string); ok { - return str + arg, nil - } - return "", fmt.Errorf("type not supported") -} - -func (t *testParser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - if str, ok := content.([]string); ok { - return append(str, arg), nil - } - return nil, fmt.Errorf("type not supported") -} - -func (t *testParser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return t.GetString(ctx, content, arg) -} - -func (t *testParser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return t.GetStrings(ctx, content, arg) -} - -func TestCtxWrapper(t *testing.T) { - t.Parallel() - parser.Register("test", new(testParser)) - ctx := plugin.NewContext(plugin.ContextOptions{ - URL: "http://localhost/home", - Logger: slog.Default(), - }) - vm := NewTestVM(t) - - testCase := []string{ - `ctx.log('start test');`, - `assert.equal(ctx.baseURL, "http://localhost");`, - `assert.equal(ctx.url,"http://localhost/home");`, - `ctx.set('v1', 114514);`, - `assert.equal(ctx.get('v1'), 114514);`, - `ctx.clearVar(); - assert.equal(ctx.get('v1'), null);`, - `assert.equal(ctx.getString('test', '1', 'foo'), 'foo1');`, - `assert.equal(ctx.getStrings('test', '2', ['foo'])[1], '2');`, - `assert.equal(ctx.getElement('test', '3', 'foo'), 'foo3');`, - `assert.equal(ctx.getElements('test', '4', ['foo'])[1], '4');`, - } - for i, s := range testCase { - t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) - if err != nil { - t.Fatal(err) - } - }) - } -} diff --git a/js/eventloop.go b/js/eventloop.go index 253d14c..8cc5597 100644 --- a/js/eventloop.go +++ b/js/eventloop.go @@ -1,226 +1,159 @@ package js import ( - "fmt" "sync" - - "github.com/dop251/goja" ) -// Copyright grafana/k6, licensed under the AGPL License. - -// EventLoop implements an event with -// handling of unhandled rejected promises. -// -// A specific thing about this event loop is that it will wait to return -// not only until the queue is empty but until nothing is registered that it will run in the future. -// This is in contrast with more common behaviours where it only returns on -// a specific event/action or when the loop is empty. -// This is required as in k6 iterations (for which event loop will be primary used) -// are supposed to be independent and any work started in them needs to finish, -// but also they need to end when all the instructions are done. -// Additionally because of this on any error while the event loop will exit it's -// required to wait on the event loop to be empty before the execution can continue. +// EventLoop implements an eventloop. type EventLoop struct { - lock sync.Mutex - queue []func() error - wakeupCh chan struct{} // TODO: maybe use sync.Cond ? - registeredCallbacks int - runtime *goja.Runtime - - // pendingPromiseRejections are rejected promises with no handler, - // if there is something in this map at an end of an event loop then it will exit with an error. - // It's similar to what Deno and Node do. - pendingPromiseRejections map[*goja.Promise]struct{} + queue []func() // queue to store the job to be executed + doneJobs []func() // job of Done + enqueue uint // Count of job in the event loop + cond *sync.Cond // Condition variable for synchronization } -// NewEventLoop returns a new event loop with a few helpers attached to it: -// - adding setTimeout javascript implementation -// - reporting (and aborting on) unhandled promise rejections -func NewEventLoop(runtime *goja.Runtime) *EventLoop { - e := &EventLoop{ - wakeupCh: make(chan struct{}, 1), - pendingPromiseRejections: make(map[*goja.Promise]struct{}), - runtime: runtime, +// NewEventLoop create a new EventLoop instance +func NewEventLoop() *EventLoop { + return &EventLoop{ + cond: sync.NewCond(new(sync.Mutex)), + doneJobs: make([]func(), 0), } - runtime.SetPromiseRejectionTracker(e.promiseRejectionTracker) - _ = runtime.GlobalObject().SetSymbol(enqueueCallbackSymbol, e.RegisterCallback) - return e } -func (e *EventLoop) wakeup() { - select { - case e.wakeupCh <- struct{}{}: - default: +// Start the event loop and execute the provided function +func (e *EventLoop) Start(f func()) { + e.cond.L.Lock() + e.queue = []func(){f} + e.cond.L.Unlock() + for { + e.cond.L.Lock() + + if len(e.queue) > 0 { + queue := e.queue + e.queue = make([]func(), 0, len(queue)) + e.cond.L.Unlock() + + for _, job := range queue { + job() + } + continue + } + + if e.enqueue > 0 { + e.cond.Wait() + e.cond.L.Unlock() + continue + } + + if len(e.doneJobs) > 0 { + for _, job := range e.doneJobs { + job() + } + e.doneJobs = e.doneJobs[:0] + } + + e.cond.L.Unlock() + return } } -var enqueueCallbackSymbol = goja.NewSymbol("__enqueueCallback__") - -type EnqueueCallback func(func() error) +type Enqueue func(func()) -// RegisterCallback signals to the event loop that you are going to do some -// asynchronous work off the main thread and that you may need to execute some -// code back on the main thread when you are done. So, once you call this -// method, the event loop will wait for you to finish and give it the callback -// it needs to run back on the main thread before it can end the whole current -// script iteration. +// EnqueueJob return a function Enqueue to add a job to the job queue. +// Usage: // -// RegisterCallback() *must* be called from the main runtime thread, but its -// result enqueueCallback() is thread-safe and can be called from any goroutine. -// enqueueCallback() ensures that its callback parameter is added to the VM -// runtime's tasks queue, to be executed on the main runtime thread eventually, -// when the VM is done with the other tasks before it. Unless the whole event -// loop has been stopped, invoking enqueueCallback() will queue its argument and -// "wake up" the loop (if it was idle, but not stopped). +// func main() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// w.WriteHeader(http.StatusOK) +// _, _ = w.Write([]byte(`{"foo":"bar"}`)) +// })) +// defer server.Close() // -// Keep in mind that once you call RegisterCallback(), you *must* also call -// enqueueCallback() exactly once, even if don't actually need to run any code -// on the main thread. If that's the case, you can pass an empty no-op callback -// to it, but you must call it! The event loop will wait for the -// enqueueCallback() invocation and the k6 iteration won't finish and will be -// stuck until the VM itself has been stopped (e.g. because the whole test or -// scenario has ended). Any error returned by any callback on the main thread -// will abort the current iteration and no further event loop callbacks will be -// executed in the same iteration. +// loop := NewEventLoop() +// runtime := goja.New() // -// A common pattern for async work is something like this: +// _ = runtime.Set("fetch", func(call goja.FunctionCall) goja.Value { +// promise, resolve, reject := runtime.NewPromise() +// enqueue := loop.EnqueueJob() // -// func doAsyncWork(vm js.VM) *goja.Promise { -// enqueueCallback := vm.Runtime().GlobalObject().GetSymbol(enqueueCallbackSymbol).Export().(func() EnqueueCallback)() -// p, resolve, reject := vm.Runtime().NewPromise() +// go func() { +// res, err := http.Get(call.Argument(0).String()) +// if err != nil { +// callback(func() { reject(err) }) +// return +// } +// loop.OnDone(func() { res.Body.Close() }) // -// // Do the actual async work in a new independent goroutine, but make -// // sure that the Promise resolution is done on the main thread: -// go func() { -// // Also make sure to abort early if the context is cancelled, so -// // the VM is not stuck when the scenario ends or Ctrl+C is used: -// result, err := doTheActualAsyncWork() -// enqueueCallback(func() error { -// if err != nil { -// reject(err) -// } else { -// resolve(result) -// } -// return nil // do not abort the iteration -// }) -// }() +// data, err := io.ReadAll(res.Body) +// if err != nil { +// callback(func() { reject(err) }) +// return +// } // -// return p -// } +// enqueue(func() { resolve(string(data)) }) +// }() // -// This ensures that the actual work happens asynchronously, while the Promise -// is immediately returned and the main thread resumes execution. It also -// ensures that the Promise resolution happens safely back on the main thread -// once the async work is done, as required by goja and all other JS runtimes. +// return runtime.ToValue(promise) +// }) // -// TODO: rename to ReservePendingCallback or something more appropriate? -func (e *EventLoop) RegisterCallback() EnqueueCallback { - e.lock.Lock() - var callbackCalled bool - e.registeredCallbacks++ - e.lock.Unlock() - - return func(f func() error) { - e.lock.Lock() - if callbackCalled { // this is protected by the lock on the event loop - e.lock.Unlock() // let not lock up the whole event loop, somebody could recover from the panic - panic("RegisterCallback called twice") +// var ( +// ret goja.Value +// err error +// ) +// +// loop.Start(func() { ret, err = runtime.RunString(fmt.Sprintf(`fetch("%s")`, server.URL)) }) +// +// if err != nil { +// fmt.Println(err) +// } +// promise, ok := ret.Export().(*goja.Promise) +// if !ok { +// panic("expect promise") +// return +// } +// +// switch promise.State() { +// case goja.PromiseStatePending: +// panic("unexpect pending state") +// case goja.PromiseStateRejected: +// fmt.Println(promise.Result().String()) +// case goja.PromiseStateFulfilled: +// fmt.Println(promise.Result().Export()) +// } +// } +func (e *EventLoop) EnqueueJob() Enqueue { + e.cond.L.Lock() + called := false + e.enqueue++ + e.cond.L.Unlock() + return func(job func()) { + e.cond.L.Lock() + if called { + e.cond.L.Unlock() + panic("Enqueue already called") } - callbackCalled = true - e.queue = append(e.queue, f) - e.registeredCallbacks-- - e.lock.Unlock() - e.wakeup() + e.queue = append(e.queue, job) // Add the job to the queue + called = true + e.enqueue-- + e.cond.Signal() // Signal the condition variable + e.cond.L.Unlock() } } -func (e *EventLoop) promiseRejectionTracker(p *goja.Promise, op goja.PromiseRejectionOperation) { - // No locking necessary here as the goja runtime will call this synchronously - // Read Notes on https://tc39.es/ecma262/#sec-host-promise-rejection-tracker - if op == goja.PromiseRejectionReject { - e.pendingPromiseRejections[p] = struct{}{} - } else { // goja.PromiseRejectionHandle so a promise that was previously rejected without handler now got one - delete(e.pendingPromiseRejections, p) - } -} - -func (e *EventLoop) popAll() (queue []func() error, awaiting bool) { - e.lock.Lock() - queue = e.queue - e.queue = make([]func() error, 0, len(queue)) - awaiting = e.registeredCallbacks != 0 - e.lock.Unlock() - return -} - -func (e *EventLoop) putInfront(queue []func() error) { - e.lock.Lock() - e.queue = append(queue, e.queue...) - e.lock.Unlock() -} - -// Start will run the event loop until it's empty and there are no uninvoked registered callbacks -// or a queued function returns an error. The provided firstCallback will be the first thing executed. -// After Start returns the event loop can be reused as long as waitOnRegistered is called. -func (e *EventLoop) Start(firstCallback func() error) error { - e.pendingPromiseRejections = make(map[*goja.Promise]struct{}) - e.queue = []func() error{firstCallback} - for { - queue, awaiting := e.popAll() - - if len(queue) == 0 { - if !awaiting { - return nil - } - <-e.wakeupCh - continue - } +// Wait until all queue in the event loop are completed +func (e *EventLoop) Wait() { + e.cond.L.Lock() + defer e.cond.L.Unlock() - for i, f := range queue { - if err := f(); err != nil { - e.putInfront(queue[i+1:]) - return err - } - } - - // This will get a random unhandled rejection instead of the first one, for example. - // But that seems to be the case in other tools as well so it seems to not be that big of a problem. - for promise := range e.pendingPromiseRejections { - value := promise.Result() - if !goja.IsNull(value) && !goja.IsUndefined(value) { - if o := value.ToObject(e.runtime); o != nil { - if stack := o.Get("stack"); stack != nil { - value = stack - } - } - } - // this is the de facto wording in both firefox and deno at least - return fmt.Errorf("Uncaught (in promise) %s", value) //nolint:stylecheck - } + for e.enqueue > 0 { + e.cond.Wait() } } -// WaitOnRegistered waits on all registered callbacks so we know nothing is still doing work. -// This does call back the callbacks and more can be queued over time. -// A different mechanism needs to be used to tell the users that the event loop has errored out or winding down for a -// different reason. -func (e *EventLoop) WaitOnRegistered() { - for { - queue, awaiting := e.popAll() - if len(queue) == 0 { - if !awaiting { - return - } - <-e.wakeupCh - continue - } +// OnDone add a function to execute when done. +func (e *EventLoop) OnDone(job func()) { + e.cond.L.Lock() + defer e.cond.L.Unlock() - for _, f := range queue { - if err := f(); err != nil { - // TODO figure out if we should buffer all errors happening or send them on a channel - continue - } - } - } + e.doneJobs = append(e.doneJobs, job) } diff --git a/js/eventloop_test.go b/js/eventloop_test.go index cbbab5c..c5383aa 100644 --- a/js/eventloop_test.go +++ b/js/eventloop_test.go @@ -1,8 +1,6 @@ package js import ( - "errors" - "fmt" "sync/atomic" "testing" "time" @@ -10,226 +8,99 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBasicEventLoop(t *testing.T) { +func TestEventLoop(t *testing.T) { t.Parallel() - loop := NewEventLoop(NewTestVM(t).Runtime()) - var ran int - f := func() error { //nolint:unparam - ran++ - return nil - } - assert.NoError(t, loop.Start(f)) - assert.Equal(t, 1, ran) - assert.NoError(t, loop.Start(f)) - assert.Equal(t, 2, ran) - assert.Error(t, loop.Start(func() error { - _ = f() - loop.RegisterCallback()(f) - return errors.New("something") - })) - assert.Equal(t, 3, ran) -} - -func TestEventLoopRegistered(t *testing.T) { - t.Parallel() - loop := NewEventLoop(NewTestVM(t).Runtime()) - var ran int - f := func() error { - ran++ - r := loop.RegisterCallback() - go func() { - time.Sleep(time.Second) - r(func() error { - ran++ - return nil - }) - }() - return nil - } - start := time.Now() - assert.NoError(t, loop.Start(f)) - took := time.Since(start) - assert.Equal(t, 2, ran) - assert.Less(t, time.Second, took) - assert.Greater(t, time.Second+time.Millisecond*100, took) + loop := NewEventLoop() + var i int + f := func() { i++ } + loop.Start(f) + assert.Equal(t, 1, i) + loop.Start(f) + assert.Equal(t, 2, i) } -func TestEventLoopWaitOnRegistered(t *testing.T) { +func TestEventLoopEnqueue(t *testing.T) { t.Parallel() - var ran int - loop := NewEventLoop(NewTestVM(t).Runtime()) - f := func() error { - ran++ - r := loop.RegisterCallback() + loop := NewEventLoop() + sleep := time.Millisecond * 500 + var i int + f := func() { + i++ + r := loop.EnqueueJob() go func() { - time.Sleep(time.Second) - r(func() error { - ran++ - return nil - }) + time.Sleep(sleep) + r(func() { i++ }) }() - return fmt.Errorf("expected") } start := time.Now() - assert.Error(t, loop.Start(f)) + loop.Start(f) took := time.Since(start) - loop.WaitOnRegistered() - took2 := time.Since(start) - assert.Equal(t, 2, ran) - assert.Greater(t, time.Millisecond*50, took) - assert.Less(t, time.Second, took2) - assert.Greater(t, time.Second+time.Millisecond*100, took2) + assert.Equal(t, 2, i) + assert.Less(t, sleep, took) } -func TestEventLoopAllCallbacksGetCalled(t *testing.T) { +func TestEventLoopAllJobCalled(t *testing.T) { t.Parallel() sleepTime := time.Millisecond * 500 - loop := NewEventLoop(NewTestVM(t).Runtime()) + loop := NewEventLoop() var called int64 - f := func() error { - for i := 0; i < 100; i++ { - bad := i == 99 - r := loop.RegisterCallback() + f := func() { + for i := 0; i < 10; i++ { + bad := i == 9 + e := loop.EnqueueJob() go func() { if !bad { time.Sleep(sleepTime) } - r(func() error { - if bad { - return errors.New("something") - } - atomic.AddInt64(&called, 1) - return nil - }) + e(func() { atomic.AddInt64(&called, 1) }) }() } - return fmt.Errorf("expected") } + all := time.Now() for i := 0; i < 3; i++ { called = 0 start := time.Now() - assert.Error(t, loop.Start(f)) + loop.Start(f) took := time.Since(start) - loop.WaitOnRegistered() + loop.Wait() took2 := time.Since(start) - assert.Greater(t, time.Millisecond*50, took) + assert.Less(t, time.Millisecond*500, took) assert.Less(t, sleepTime, took2) assert.Greater(t, sleepTime+time.Millisecond*100, took2) - assert.EqualValues(t, called, 99) + assert.EqualValues(t, 10, called) } + took := time.Since(all) + assert.Less(t, time.Millisecond*500*3, took) } -func TestEventLoopPanicOnDoubleCallback(t *testing.T) { +func TestEventLoopPanicOnDoubleEnqueue(t *testing.T) { t.Parallel() - loop := NewEventLoop(NewTestVM(t).Runtime()) - var ran int - f := func() error { - ran++ - r := loop.RegisterCallback() + loop := NewEventLoop() + var i int + f := func() { + i++ + e := loop.EnqueueJob() go func() { time.Sleep(time.Second) - r(func() error { - ran++ - return nil - }) + e(func() { i++ }) - assert.Panics(t, func() { r(func() error { return nil }) }) + assert.Panics(t, func() { e(func() {}) }) }() - return nil } start := time.Now() - assert.NoError(t, loop.Start(f)) + loop.Start(f) took := time.Since(start) - assert.Equal(t, 2, ran) + assert.Equal(t, 2, i) assert.Less(t, time.Second, took) assert.Greater(t, time.Second+time.Millisecond*100, took) } -func TestEventLoopRejectUndefined(t *testing.T) { - t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString("Promise.reject()") - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) undefined") -} - -func TestEventLoopRejectString(t *testing.T) { - t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString("Promise.reject('some string')") - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) some string") -} - -func TestEventLoopRejectSyntaxError(t *testing.T) { - t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {some.syntax.error})") - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) ReferenceError: some is not defined\n\tat :1:30(1)\n") -} - -func TestEventLoopRejectGoError(t *testing.T) { - t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - rt := vm.Runtime() - assert.NoError(t, rt.Set("f", rt.ToValue(func() error { - return errors.New("some error") - }))) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {f()})") - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) GoError: some error\n\tat github.com/shiroyk/cloudcat/js.TestEventLoopRejectGoError.func1 (native)\n\tat :1:31(2)\n") -} - -func TestEventLoopRejectThrow(t *testing.T) { - t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - rt := vm.Runtime() - assert.NoError(t, rt.Set("f", rt.ToValue(func() error { - Throw(rt, errors.New("throw error")) - return nil - }))) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {f()})") - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) throw error") -} - -func TestEventLoopAsyncAwait(t *testing.T) { +func TestEventLoopOnDone(t *testing.T) { t.Parallel() - vm := NewTestVM(t) - loop := NewEventLoop(vm.Runtime()) - err := loop.Start(func() error { - _, err := vm.Runtime().RunString(` - async function a() { - some.error.here - } - Promise.resolve().then(async () => { - await a(); - }) - `) - return err - }) - loop.WaitOnRegistered() - assert.EqualError(t, err, "Uncaught (in promise) ReferenceError: some is not defined\n\tat a (:3:13(1))\n\tat :6:20(2)\n") + loop := NewEventLoop() + var i int + loop.Start(func() { loop.OnDone(func() { i++ }) }) + loop.Wait() + assert.Equal(t, 1, i) } diff --git a/js/format.go b/js/format.go deleted file mode 100644 index 18ad29b..0000000 --- a/js/format.go +++ /dev/null @@ -1,81 +0,0 @@ -package js - -import ( - "bytes" - - "github.com/dop251/goja" -) - -func runeFormat(vm *goja.Runtime, f rune, val goja.Value, w *bytes.Buffer) bool { - switch f { - case 's': - w.WriteString(val.String()) - case 'd': - w.WriteString(val.ToNumber().String()) - case 'j': - if json, ok := vm.Get("JSON").(*goja.Object); ok { - if stringify, ok := goja.AssertFunction(json.Get("stringify")); ok { - res, err := stringify(json, val) - if err != nil { - panic(err) - } - w.WriteString(res.String()) - } - } - case '%': - w.WriteByte('%') - return false - default: - w.WriteByte('%') - w.WriteRune(f) - return false - } - return true -} - -func bufferFormat(vm *goja.Runtime, b *bytes.Buffer, f string, args ...goja.Value) { - pct := false - argNum := 0 - for _, chr := range f { - if pct { //nolint:nestif - if argNum < len(args) { - if runeFormat(vm, chr, args[argNum], b) { - argNum++ - } - } else { - b.WriteByte('%') - b.WriteRune(chr) - } - pct = false - } else { - if chr == '%' { - pct = true - } else { - b.WriteRune(chr) - } - } - } - - for _, arg := range args[argNum:] { - b.WriteByte(' ') - b.WriteString(arg.String()) - } -} - -// Format implements js format -func Format(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - var b bytes.Buffer - var fmt string - - if arg := call.Argument(0); !goja.IsUndefined(arg) { - fmt = arg.String() - } - - var args []goja.Value - if len(call.Arguments) > 0 { - args = call.Arguments[1:] - } - bufferFormat(vm, &b, fmt, args...) - - return vm.ToValue(b.String()) -} diff --git a/js/js.go b/js/js.go deleted file mode 100644 index c71f52a..0000000 --- a/js/js.go +++ /dev/null @@ -1,144 +0,0 @@ -package js - -import ( - "context" - "errors" - "fmt" - "runtime" - "sync" - "sync/atomic" - "time" - - "log/slog" - - "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" -) - -const ( - // DefaultMaxTimeToWaitGetVM default retries time - DefaultMaxTimeToWaitGetVM = 500 * time.Millisecond - // DefaultMaxRetriesGetVM default retries times - DefaultMaxRetriesGetVM = 3 -) - -var ( - schedulerDefault = sync.OnceValue[Scheduler](func() Scheduler { - scheduler, err := cloudcat.Resolve[Scheduler]() - if err != nil { - scheduler = NewScheduler(Options{InitialVMs: 2, MaxVMs: runtime.GOMAXPROCS(0)}) - cloudcat.Provide(scheduler) - } - return scheduler - }) - // ErrSchedulerClosed the scheduler is closed error - ErrSchedulerClosed = errors.New("scheduler is closed") -) - -// RunString the js string -func RunString(ctx context.Context, script string) (goja.Value, error) { - tr, err := schedulerDefault().Get() - if err != nil { - return nil, err - } - return tr.RunString(ctx, script) -} - -// RunModule the goja.CyclicModuleRecord -func RunModule(ctx context.Context, module goja.CyclicModuleRecord) (goja.Value, error) { - tr, err := schedulerDefault().Get() - if err != nil { - return nil, err - } - return tr.RunModule(ctx, module) -} - -// Scheduler the VM scheduler -type Scheduler interface { - // Get the VM - Get() (VM, error) - // Release the VM - Release(VM) - // Close the scheduler - Close() error -} - -// Options Scheduler options -type Options struct { - InitialVMs int `yaml:"initial-vms"` - MaxVMs int `yaml:"max-vms"` - MaxRetriesGetVM int `yaml:"max-retries-get-vm"` - MaxTimeToWaitGetVM time.Duration `yaml:"max-time-to-wait-get-vm"` -} - -type schedulerImpl struct { - mu *sync.Mutex - vms chan VM - initVMs, maxVMs, maxRetriesGetVM int - unInitVMs *atomic.Int64 - closed *atomic.Bool - maxTimeToWaitGetVM time.Duration -} - -// NewScheduler returns a new Scheduler -func NewScheduler(opt Options) Scheduler { - scheduler := &schedulerImpl{ - mu: new(sync.Mutex), - closed: new(atomic.Bool), - unInitVMs: new(atomic.Int64), - maxVMs: cloudcat.ZeroOr(opt.MaxVMs, 1), - initVMs: cloudcat.ZeroOr(opt.InitialVMs, 1), - maxRetriesGetVM: cloudcat.ZeroOr(opt.MaxRetriesGetVM, DefaultMaxRetriesGetVM), - maxTimeToWaitGetVM: cloudcat.ZeroOr(opt.MaxTimeToWaitGetVM, DefaultMaxTimeToWaitGetVM), - } - scheduler.vms = make(chan VM, scheduler.maxVMs) - for i := 0; i < scheduler.initVMs; i++ { - scheduler.vms <- NewVM() - } - scheduler.unInitVMs.Store(int64(scheduler.maxVMs - scheduler.initVMs)) - return scheduler -} - -// Close the scheduler -func (s *schedulerImpl) Close() error { - s.closed.Store(true) - close(s.vms) - return nil -} - -// Get the VM -func (s *schedulerImpl) Get() (VM, error) { - timer := time.NewTimer(s.maxTimeToWaitGetVM) - - defer func() { - timer.Stop() - }() - - for i := 1; i <= s.maxRetriesGetVM; i++ { - select { - case vm, ok := <-s.vms: - if !ok { - return nil, ErrSchedulerClosed - } - return vm, nil - case <-timer.C: - if s.unInitVMs.Add(-1) >= 0 { - return NewVM(), nil - } - s.unInitVMs.Add(1) - slog.Warn(fmt.Sprintf("could not get VM in %v", time.Duration(i)*s.maxTimeToWaitGetVM)) - timer.Reset(s.maxTimeToWaitGetVM) - } - } - return nil, fmt.Errorf("could not get VM in %v", - time.Duration(s.maxRetriesGetVM)*s.maxTimeToWaitGetVM) -} - -// Release the VM -func (s *schedulerImpl) Release(vm VM) { - if s.closed.Load() { - return - } - - s.vms <- vm -} diff --git a/js/js_test.go b/js/js_test.go deleted file mode 100644 index 61a1548..0000000 --- a/js/js_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package js - -import ( - "context" - "errors" - "sync" - "testing" - "time" - - "github.com/shiroyk/cloudcat" -) - -func TestScheduler(t *testing.T) { - goroutineNum := 20 - blockNum := 4 - scheduler := NewScheduler(Options{InitialVMs: 2, MaxVMs: 4}) - cloudcat.Provide(scheduler) - wg := new(sync.WaitGroup) - - for i := 1; i <= goroutineNum; i++ { - wg.Add(1) - go func(i int) { - timeout := time.Second - script := "1" - if i < blockNum { - script = `while(true){}` - timeout *= 2 - } - - ctx, _ := context.WithTimeout(context.Background(), timeout) - defer func() { - wg.Done() - }() - - vm, err := scheduler.Get() - if err != nil { - t.Errorf("%v: %v", i, err) - return - } - _, err = vm.RunString(ctx, script) - if err != nil && !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("%v: %v", i, err) - } - }(i) - } - wg.Wait() -} - -func BenchmarkScheduler(b *testing.B) { - b.ResetTimer() - b.ReportAllocs() - wg := sync.WaitGroup{} - for n := 0; n < b.N; n++ { - wg.Add(1) - go func() { - _, _ = RunString(context.Background(), `1`) - wg.Done() - }() - } - b.StopTimer() -} diff --git a/js/loader.go b/js/loader.go index 01faf67..c7b4d38 100644 --- a/js/loader.go +++ b/js/loader.go @@ -12,12 +12,12 @@ import ( "path" "path/filepath" "strings" + "sync" "text/template" "github.com/dop251/goja" "github.com/dop251/goja/parser" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski" ) var ( @@ -25,76 +25,77 @@ var ( ErrInvalidModule = errors.New("invalid module") // ErrIllegalModuleName module name is illegal ErrIllegalModuleName = errors.New("illegal module name") + // ErrNotFoundModule not found module + ErrNotFoundModule = errors.New("not found module") ) type ( // ModuleLoader the js module loader. ModuleLoader interface { - // EnableRequire enable the global function require to the goja.Runtime. - EnableRequire(rt *goja.Runtime) + // CompileModule compile module from source string (cjs/esm). + CompileModule(name, source string) (goja.CyclicModuleRecord, error) // ResolveModule resolve the module returns the goja.ModuleRecord. ResolveModule(any, string) (goja.ModuleRecord, error) - // ImportModuleDynamically goja runtime SetImportModuleDynamically - ImportModuleDynamically(rt *goja.Runtime) + // EnableRequire enable the global function require to the goja.Runtime. + EnableRequire(*goja.Runtime) ModuleLoader + // EnableImportModuleDynamically goja runtime SetImportModuleDynamically + EnableImportModuleDynamically(*goja.Runtime) ModuleLoader } + // LoaderOption the default moduleLoader options. + LoaderOption func(*moduleLoader) + // FileLoader is a type alias for a function that returns the contents of the referenced file. FileLoader func(specifier *url.URL, name string) ([]byte, error) -) -// Option the default moduleLoader options. -type Option func(*moduleLoader) + // emptyLoader + emptyLoader struct{} +) -// WithBase the base directory of module loader. -func WithBase(base *url.URL) Option { - return func(o *moduleLoader) { - o.base = base - } +// WithBaseLoader the base directory of module loader. +func WithBaseLoader(base *url.URL) LoaderOption { + return func(o *moduleLoader) { o.base = base } } // WithFileLoader the file loader of module loader. -func WithFileLoader(fileLoader FileLoader) Option { - return func(o *moduleLoader) { - o.fileLoader = fileLoader - } +func WithFileLoader(loader FileLoader) LoaderOption { + return func(o *moduleLoader) { o.fileLoader = loader } } // WithSourceMapLoader the source map loader of module loader. -func WithSourceMapLoader(loader func(path string) ([]byte, error)) Option { - return func(o *moduleLoader) { - o.sourceLoader = parser.WithSourceMapLoader(loader) - } +func WithSourceMapLoader(loader func(path string) ([]byte, error)) LoaderOption { + return func(o *moduleLoader) { o.sourceLoader = parser.WithSourceMapLoader(loader) } } // NewModuleLoader returns a new module resolver // if the fileLoader option not provided, uses the default DefaultFileLoader. -func NewModuleLoader(opts ...Option) ModuleLoader { - mr := &moduleLoader{ +func NewModuleLoader(opts ...LoaderOption) ModuleLoader { + ml := &moduleLoader{ modules: make(map[string]moduleCache), goModules: make(map[string]goja.CyclicModuleRecord), + parsers: make(map[string]goja.CyclicModuleRecord), reverse: make(map[goja.ModuleRecord]*url.URL), } for _, option := range opts { - option(mr) + option(ml) } - if mr.base == nil { - mr.base = &url.URL{Scheme: "file", Path: "."} + if ml.base == nil { + ml.base = &url.URL{Scheme: "file", Path: "."} } - if mr.fileLoader == nil { - mr.fileLoader = DefaultFileLoader() + if ml.fileLoader == nil { + ml.fileLoader = DefaultFileLoader(ski.NewFetch()) } - if mr.sourceLoader == nil { - mr.sourceLoader = parser.WithDisableSourceMaps + if ml.sourceLoader == nil { + ml.sourceLoader = parser.WithDisableSourceMaps } - return mr + return ml } // DefaultFileLoader the default file loader. // Supports file and HTTP scheme loading. -func DefaultFileLoader() FileLoader { - fetch := cloudcat.MustResolveLazy[cloudcat.Fetch]() +func DefaultFileLoader(fetch ski.Fetch) FileLoader { return func(specifier *url.URL, name string) ([]byte, error) { switch specifier.Scheme { case "http", "https": @@ -102,7 +103,7 @@ func DefaultFileLoader() FileLoader { if err != nil { return nil, err } - res, err := fetch().Do(req) + res, err := fetch.Do(req) if err != nil { return nil, err } @@ -121,9 +122,12 @@ type ( // moduleLoader the ModuleLoader implement. // Allows loading and interop between ES module and CommonJS module. moduleLoader struct { - modules map[string]moduleCache - goModules map[string]goja.CyclicModuleRecord - reverse map[goja.ModuleRecord]*url.URL + sync.Mutex + modules map[string]moduleCache + goModules map[string]goja.CyclicModuleRecord + parsers map[string]goja.CyclicModuleRecord + reverse map[goja.ModuleRecord]*url.URL + fileLoader FileLoader base *url.URL @@ -131,74 +135,112 @@ type ( } moduleCache struct { - mod goja.ModuleRecord + mod goja.CyclicModuleRecord err error } ) // EnableRequire enable the global function require to the goja.Runtime. -func (ml *moduleLoader) EnableRequire(rt *goja.Runtime) { _ = rt.Set("require", ml.require) } +func (ml *moduleLoader) EnableRequire(rt *goja.Runtime) ModuleLoader { + _ = rt.Set("require", ml.require) + return ml +} // require resolve the module instance. func (ml *moduleLoader) require(call goja.FunctionCall, rt *goja.Runtime) goja.Value { name := call.Argument(0).String() - module, err := ml.ResolveModule(ml.getCurrentModuleRecord(rt), name) + mod, err := ml.ResolveModule(ml.getCurrentModuleRecord(rt), name) if err != nil { - panic(rt.ToValue(err)) - } - if nm, ok := module.(*goModule); ok { - return rt.ToValue(nm.mod.Exports()) - } - if err = module.Link(); err != nil { - panic(rt.ToValue(err)) + Throw(rt, err) } - cm, ok := module.(goja.CyclicModuleRecord) - if !ok { - panic(rt.ToValue(ErrInvalidModule)) + if mod, ok := mod.(*goModule); ok { + instance, err := mod.mod.Instantiate(rt) + if err != nil { + Throw(rt, err) + } + return instance } - promise := rt.CyclicModuleRecordEvaluate(cm, ml.ResolveModule) - if promise.State() == goja.PromiseStateRejected { - panic(promise.Result()) + + instance := rt.GetModuleInstance(mod) + if instance == nil { + if err = mod.Link(); err != nil { + Throw(rt, err) + } + cm, ok := mod.(goja.CyclicModuleRecord) + if !ok { + Throw(rt, ErrInvalidModule) + } + promise := rt.CyclicModuleRecordEvaluate(cm, ml.ResolveModule) + if promise.State() == goja.PromiseStateRejected { + panic(promise.Result()) + } + instance = rt.GetModuleInstance(mod) } - if cjs, ok := module.(*cjsModule); ok { - return rt.GetModuleInstance(cjs).(*cjsModuleInstance).exports + + switch mod.(type) { + case *cjsModule: + return instance.(*cjsModuleInstance).GetBindingValue("default") + case *goja.SourceTextModuleRecord: + if v := instance.GetBindingValue("default"); v != nil { + return v + } } - return rt.NamespaceObjectFor(cm) + + return rt.NamespaceObjectFor(mod) } -func (ml *moduleLoader) ImportModuleDynamically(rt *goja.Runtime) { +func (ml *moduleLoader) EnableImportModuleDynamically(rt *goja.Runtime) ModuleLoader { rt.SetImportModuleDynamically(func(referencingScriptOrModule any, specifier goja.Value, promiseCapability any) { - NewEnqueueCallback(rt)(func() error { - module, err := ml.ResolveModule(referencingScriptOrModule, specifier.String()) - rt.FinishLoadingImportModule(referencingScriptOrModule, specifier, promiseCapability, module, err) - return nil - }) + NewPromise(rt, + func() (goja.ModuleRecord, error) { + return ml.ResolveModule(referencingScriptOrModule, specifier.String()) + }, + func(module goja.ModuleRecord, err error) (any, error) { + rt.FinishLoadingImportModule(referencingScriptOrModule, specifier, promiseCapability, module, err) + return nil, err + }) }) + return ml } func (ml *moduleLoader) getCurrentModuleRecord(rt *goja.Runtime) goja.ModuleRecord { - var parent string var buf [2]goja.StackFrame frames := rt.CaptureCallStack(2, buf[:0]) - parent = frames[1].SrcName() - - module, _ := ml.ResolveModule(nil, parent) - return module + if len(frames) == 0 { + return nil + } + mod, _ := ml.ResolveModule(nil, frames[1].SrcName()) + return mod } // ResolveModule resolve the module returns the goja.ModuleRecord. func (ml *moduleLoader) ResolveModule(referencingScriptOrModule any, name string) (goja.ModuleRecord, error) { switch { - case strings.HasPrefix(name, jsmodule.ExtPrefix): + case strings.HasPrefix(name, modulePrefix): + ml.Lock() + defer ml.Unlock() if mod, ok := ml.goModules[name]; ok { return mod, nil } - if e, ok := jsmodule.GetModule(name); ok { + if e, ok := GetModule(name); ok { mod := &goModule{mod: e} ml.goModules[name] = mod return mod, nil } - return nil, ErrIllegalModuleName + return nil, ErrNotFoundModule + case strings.HasPrefix(name, parserPrefix): + ml.Lock() + defer ml.Unlock() + name = strings.TrimPrefix(name, parserPrefix) + if mod, ok := ml.parsers[name]; ok { + return mod, nil + } + if p, ok := ski.GetParser(name); ok { + mod := &goModule{mod: &jsParser{p}} + ml.parsers[name] = mod + return mod, nil + } + return nil, ErrNotFoundModule default: return ml.resolve(ml.reversePath(referencingScriptOrModule), name) } @@ -232,11 +274,16 @@ func (ml *moduleLoader) reversePath(referencingScriptOrModule any) *url.URL { if referencingScriptOrModule == nil { return ml.base } - p, ok := ml.reverse[referencingScriptOrModule.(goja.ModuleRecord)] + mod, ok := referencingScriptOrModule.(goja.ModuleRecord) + if !ok { + return ml.base + } + + ml.Lock() + p, ok := ml.reverse[mod] + ml.Unlock() + if !ok { - if referencingScriptOrModule != nil { - // TODO fix this - } return ml.base } @@ -312,6 +359,10 @@ func (ml *moduleLoader) loadNodeModules(modName string) (mod goja.ModuleRecord, func (ml *moduleLoader) loadModule(modPath *url.URL, modName string) (goja.ModuleRecord, error) { file := modPath.JoinPath(modName) specifier := file.String() + + ml.Lock() + defer ml.Unlock() + cache, exists := ml.modules[specifier] if exists { return cache.mod, cache.err @@ -321,7 +372,7 @@ func (ml *moduleLoader) loadModule(modPath *url.URL, modName string) (goja.Modul if err != nil { return nil, err } - mod, err := ml.compileModule(specifier, string(buf)) + mod, err := ml.CompileModule(specifier, string(buf)) if err == nil { file.Path = filepath.Dir(file.Path) ml.reverse[mod] = file @@ -330,29 +381,29 @@ func (ml *moduleLoader) loadModule(modPath *url.URL, modName string) (goja.Modul return mod, err } -func (ml *moduleLoader) compileModule(path, source string) (goja.ModuleRecord, error) { - if filepath.Ext(path) == ".json" { +func (ml *moduleLoader) CompileModule(name, source string) (goja.CyclicModuleRecord, error) { + if filepath.Ext(name) == ".json" { source = "module.exports = JSON.parse('" + template.JSEscapeString(source) + "')" - return ml.compileCjsModule(path, source) + return ml.compileCjsModule(name, source) } - ast, err := goja.Parse(path, source, parser.IsModule, ml.sourceLoader) + ast, err := goja.Parse(name, source, parser.IsModule, ml.sourceLoader) if err != nil { return nil, err } isModule := len(ast.ExportEntries) > 0 || len(ast.ImportEntries) > 0 || ast.HasTLA if !isModule { - return ml.compileCjsModule(path, source) + return ml.compileCjsModule(name, source) } return goja.ModuleFromAST(ast, ml.ResolveModule) } -func (ml *moduleLoader) compileCjsModule(path, source string) (goja.ModuleRecord, error) { +func (ml *moduleLoader) compileCjsModule(name, source string) (goja.CyclicModuleRecord, error) { source = "(function(exports, require, module) {" + source + "\n})" - ast, err := goja.Parse(path, source, ml.sourceLoader) + ast, err := goja.Parse(name, source, ml.sourceLoader) if err != nil { return nil, err } @@ -365,9 +416,33 @@ func (ml *moduleLoader) compileCjsModule(path, source string) (goja.ModuleRecord return &cjsModule{prg: prg}, nil } -func isBasePath(modPath string) bool { - return strings.HasPrefix(modPath, "./") || - strings.HasPrefix(modPath, "/") || - strings.HasPrefix(modPath, "../") || - modPath == "." || modPath == ".." +func isBasePath(path string) bool { + return strings.HasPrefix(path, "/") || + strings.HasPrefix(path, "./") || + strings.HasPrefix(path, "../") || + path == "." || path == ".." +} + +var errNotSupport = errors.New("js.ModuleLoader not provided, require and module not working") + +func (e emptyLoader) CompileModule(name string, source string) (goja.CyclicModuleRecord, error) { + return goja.ParseModule(name, source, e.ResolveModule) +} +func (emptyLoader) ResolveModule(any, string) (goja.ModuleRecord, error) { + return nil, errNotSupport +} +func (e emptyLoader) EnableRequire(rt *goja.Runtime) ModuleLoader { + _ = rt.Set("require", func() { + panic(rt.NewGoError(errNotSupport)) + }) + return e +} +func (e emptyLoader) EnableImportModuleDynamically(rt *goja.Runtime) ModuleLoader { + rt.SetImportModuleDynamically(func(referencingScriptOrModule any, specifier goja.Value, promiseCapability any) { + NewPromise(rt, + func() (goja.ModuleRecord, error) { + return nil, errNotSupport + }) + }) + return e } diff --git a/js/loader_test.go b/js/loader_test.go index 5261cd0..c0bd962 100644 --- a/js/loader_test.go +++ b/js/loader_test.go @@ -7,22 +7,25 @@ import ( "io/fs" "net/http" "net/url" + "strconv" "strings" + "sync" "testing" "testing/fstest" + _ "unsafe" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski" "github.com/stretchr/testify/assert" ) -type testModuleFetch struct{} +type fetch struct{} -func (*testModuleFetch) Do(req *http.Request) (*http.Response, error) { - source := `module.exports = { foo: 'bar' + require('cloudcat/gomod1').key }` +func (*fetch) Do(req *http.Request) (*http.Response, error) { + source := `module.exports = { foo: 'bar' + require('ski/gomod1').key }` if req.URL.Query().Get("type") == "esm" { source = ` -import gomod1 from "cloudcat/gomod1"; +import gomod1 from "ski/gomod1"; const a = async () => 4; export default async () => gomod1.key + 1 + (await a())` } @@ -31,28 +34,32 @@ export default async () => gomod1.key + 1 + (await a())` type gomod1 struct{} -func (gomod1) Exports() any { return map[string]string{"key": "gomod1"} } +func (gomod1) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(map[string]string{"key": "gomod1"}), nil +} type gomod2 struct{} -func (gomod2) Exports() any { - return struct { +func (gomod2) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(struct { Key string `js:"key"` - }{Key: "gomod2"} + }{Key: "gomod2"}), nil } type gomod3 struct{} -func (gomod3) Exports() any { return map[string]string{"key": "gomod3"} } +func (gomod3) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(map[string]string{"key": "gomod3"}), nil +} func (gomod3) Global() {} -func TestModule(t *testing.T) { +func TestModuleLoader(t *testing.T) { t.Parallel() - fetch := new(testModuleFetch) + fetch := new(fetch) mfs := fstest.MapFS{ "node_modules/module1/index.js": &fstest.MapFile{ - Data: []byte(`export default function() { return "module1" };`), + Data: []byte(`module.exports = function() { return "module1" };`), }, "node_modules/module2/index.js": &fstest.MapFile{ Data: []byte(` @@ -107,7 +114,7 @@ func TestModule(t *testing.T) { Data: []byte(`export const value = () => 555;`), }, "cjs_script1.js": &fstest.MapFile{ - Data: []byte(`exports.default = () => { return require('module4').default() + "/cjs_script1" };`), + Data: []byte(`module.exports = () => { return require('module4')() + "/cjs_script1" };`), }, "cjs_script2.js": &fstest.MapFile{ Data: []byte(` @@ -119,7 +126,7 @@ func TestModule(t *testing.T) { Data: []byte(`{"key": "json1"}`), }, } - resolver := NewModuleLoader(WithFileLoader(func(specifier *url.URL, name string) ([]byte, error) { + loader := NewModuleLoader(WithFileLoader(func(specifier *url.URL, name string) ([]byte, error) { switch specifier.Scheme { case "http", "https": res, err := fetch.Do(&http.Request{URL: specifier}) @@ -129,54 +136,56 @@ func TestModule(t *testing.T) { body, err := io.ReadAll(res.Body) return body, err case "file": - return fs.ReadFile(mfs, specifier.Path) + return mfs.ReadFile(specifier.Path) default: return nil, fmt.Errorf("unexpected scheme %s", specifier.Scheme) } })) - jsmodule.Register("gomod1", new(gomod1)) - jsmodule.Register("gomod2", new(gomod2)) - jsmodule.Register("gomod3", new(gomod3)) - vm := NewTestVM(t, resolver) + Register("gomod1", new(gomod1)) + Register("gomod2", new(gomod2)) + Register("gomod3", new(gomod3)) + vm := NewTestVM(t, WithModuleLoader(loader)) { scriptCases := []struct{ name, s string }{ - {"gomod1", `assert.equal(require("cloudcat/gomod1").key, "gomod1")`}, - {"gomod2", `assert.equal(require("cloudcat/gomod2").key, "gomod2")`}, + {"gomod1", `assert.equal(require("ski/gomod1").key, "gomod1")`}, + {"gomod2", `assert.equal(require("ski/gomod2").key, "gomod2")`}, {"gomod3", `assert.equal(gomod3.key, "gomod3")`}, - {"remote cjs", `assert.equal(require("https://foo.com/foo.min.js?type=cjs").foo, "bargomod1")`}, - {"remote esm", `async () => assert.equal(await require("https://foo.com/foo.min.js?type=esm").default(), "gomod114")`}, - {"module1", `assert.equal(require("module1").default(), "module1")`}, - {"module2", `assert.equal(require("module2").default(), "module1/module2")`}, - {"module3", `assert.equal(require("module3").default(), "module1/module2/module3")`}, - {"module4", `assert.equal(require("module4").default(), "/module4")`}, - {"module5", `assert.equal(require("module5").default(), "/module5/module6")`}, - {"module6", `assert.equal(require("module6").default(), "/module6/module5")`}, - {"module7", `async () => assert.equal(await require("module7").default(), "dynamic import /module6")`}, - {"es_script1", `assert.equal(require("./es_script1").default(), "module1/module2/module3/es_script1")`}, + {"remote cjs", `assert.equal(require("http://foo.com/foo.min.js?type=cjs").foo, "bargomod1")`}, + {"remote esm", `(async () => assert.equal(await require("http://foo.com/foo.min.js?type=esm")(), "gomod114"))()`}, + {"module1", `assert.equal(require("module1")(), "module1")`}, + {"module2", `assert.equal(require("module2")(), "module1/module2")`}, + {"module3", `assert.equal(require("module3")(), "module1/module2/module3")`}, + {"module4", `assert.equal(require("module4")(), "/module4")`}, + {"module5", `assert.equal(require("module5")(), "/module5/module6")`}, + {"module6", `assert.equal(require("module6")(), "/module6/module5")`}, + {"module7", `(async () => assert.equal(await require("module7")(), "dynamic import /module6"))()`}, + {"es_script1", `assert.equal(require("./es_script1")(), "module1/module2/module3/es_script1")`}, {"es_script2", `assert.equal(require("./es_script2").value(), 555)`}, - {"cjs_script1", `assert.equal(require("./cjs_script1").default(), "/module4/cjs_script1")`}, + {"cjs_script1", `assert.equal(require("./cjs_script1")(), "/module4/cjs_script1")`}, {"cjs_script2", `assert.equal(require("./cjs_script2").value(), 555)`}, {"json1", `assert.equal(require("./json1.json").key, "json1")`}, } for _, script := range scriptCases { t.Run(fmt.Sprintf("script %s", script.name), func(t *testing.T) { - _, err := vm.RunString(context.Background(), script.s) - assert.NoError(t, err) + vm.Run(context.Background(), func() { + _, err := vm.Runtime().RunString(script.s) + assert.NoError(t, err) + }) }) } } { moduleCases := []struct{ name, s string }{ - {"gomod1", `import gomod1 from "cloudcat/gomod1"; + {"gomod1", `import gomod1 from "ski/gomod1"; export default () => assert.equal(gomod1.key, "gomod1")`}, - {"gomod2", `import gomod2 from "cloudcat/gomod2"; + {"gomod2", `import gomod2 from "ski/gomod2"; export default () => assert.equal(gomod2.key, "gomod2")`}, {"gomod3", `export default () => assert.equal(gomod3.key, "gomod3")`}, - {"remote cjs", `import foo from "https://foo.com/foo.min.js?type=cjs"; + {"remote cjs", `import foo from "http://foo.com/foo.min.js?type=cjs"; export default () => assert.equal(foo.foo, "bargomod1")`}, - {"remote esm", `import foo from "https://foo.com/foo.min.js?type=esm"; + {"remote esm", `import foo from "http://foo.com/foo.min.js?type=esm"; export default async () => assert.equal(await foo(), "gomod114")`}, {"module1", `import module1 from "module1"; export default () => assert.equal(module1(), "module1");`}, @@ -206,12 +215,96 @@ func TestModule(t *testing.T) { for _, script := range moduleCases { t.Run(fmt.Sprintf("module %v", script.name), func(t *testing.T) { - module, err := goja.ParseModule("", script.s, resolver.ResolveModule) + mod, err := loader.CompileModule("", script.s) if assert.NoError(t, err) { - _, err = vm.RunModule(context.Background(), module) + _, err = vm.RunModule(context.Background(), mod) assert.NoError(t, err) } }) } } } + +func TestConcurrentLoader(t *testing.T) { + t.Parallel() + num := 8 + + mfs := make(fstest.MapFS, num) + for i := 0; i < num; i++ { + mfs[fmt.Sprintf("module%d.js", i)] = &fstest.MapFile{Data: []byte(`export default () => ` + strconv.Itoa(i))} + } + + fileLoader := WithFileLoader(func(specifier *url.URL, name string) ([]byte, error) { + return fs.ReadFile(mfs, specifier.Path) + }) + scheduler := NewScheduler(SchedulerOptions{ + InitialVMs: 2, + Loader: NewModuleLoader(fileLoader), + }) + + var wg sync.WaitGroup + + for i := 0; i < num; i++ { + wg.Add(1) + go func(j int) { + defer wg.Done() + + vm, err := scheduler.Get() + if assert.NoError(t, err) { + vm.Run(context.Background(), func() { + v, err := vm.Runtime().RunString(fmt.Sprintf("require('./module%d.js')()", j)) + if assert.NoError(t, err) { + assert.Equal(t, int64(j), v.ToInteger()) + } + }) + } + }(i) + } + + wg.Wait() +} + +type testParser struct{} + +func (testParser) Value(s string) (ski.Executor, error) { return ski.Raw(s), nil } +func (testParser) Element(s string) (ski.Executor, error) { return ski.Raw(s), nil } +func (testParser) Elements(s string) (ski.Executor, error) { return ski.Raw([]string{s}), nil } + +func TestParser(t *testing.T) { + ski.Register("loader_parser", new(testParser)) + vm := NewTestVM(t, WithModuleLoader(NewModuleLoader())) + + for i, s := range []string{ + `assert.equal(require("parser/loader_parser")('foo').exec(''), 'foo');`, + `assert.equal(require("parser/loader_parser").value('foo').exec(''), 'foo');`, + `assert.equal(require("parser/loader_parser").element('bar').exec(''), 'bar');`, + `assert.equal(require("parser/loader_parser").elements('bar').exec('')[0], 'bar');`, + } { + t.Run(strconv.Itoa(i), func(t *testing.T) { + _, err := vm.Runtime().RunString(s) + assert.NoError(t, err) + }) + } +} + +func TestESMParserValue(t *testing.T) { + p := Parser{NewModuleLoader()} + executor, err := p.Value(`export default (ctx) => ctx.get('content') + 1`) + if assert.NoError(t, err) { + v, err := executor.Exec(context.Background(), "a") + if assert.NoError(t, err) { + assert.Equal(t, "a1", v) + } + } +} + +func NewTestVM(t *testing.T, opts ...Option) VM { + vm := NewVM(opts...) + p := vm.Runtime().NewObject() + _ = p.Set("equal", func(call goja.FunctionCall) goja.Value { + assert.Equal(t, call.Argument(1).Export(), call.Argument(0).Export(), call.Argument(2).String()) + return goja.Undefined() + }) + _ = vm.Runtime().Set("assert", p) + return vm +} diff --git a/js/module.go b/js/module.go index 22f64eb..1690301 100644 --- a/js/module.go +++ b/js/module.go @@ -1,13 +1,67 @@ package js import ( + "context" "errors" + "maps" "sync" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski" ) +// Module is what a module needs to return +type Module interface { + Instantiate(*goja.Runtime) (goja.Value, error) +} + +// Global implements the interface will load into global when the VM initialize (InitGlobalModule). +type Global interface { + Module + Global() // is it a global module +} + +// Register the given mod as an external JavaScript module that can be imported +// by name. +func Register(name string, mod Module) { + if _, ok := mod.(Global); !ok { + name = modulePrefix + name + } + registry.Lock() + registry.native[name] = mod + registry.Unlock() +} + +// GetModule get the module +func GetModule(name string) (Module, bool) { + registry.RLock() + defer registry.RUnlock() + module, ok := registry.native[name] + return module, ok +} + +func RemoveModule(name string) { + registry.Lock() + delete(registry.native, name) + registry.Unlock() +} + +// AllModule get all module +func AllModule() map[string]Module { + registry.RLock() + defer registry.RUnlock() + return maps.Clone(registry.native) +} + +const modulePrefix = "ski/" + +var registry = struct { + sync.RWMutex + native map[string]Module +}{ + native: make(map[string]Module), +} + type cjsModule struct { prg *goja.Program exportedNames []string @@ -18,23 +72,15 @@ func (cm *cjsModule) Link() error { return nil } func (cm *cjsModule) InitializeEnvironment() error { return nil } -func (cm *cjsModule) Instantiate(rt *goja.Runtime) (goja.CyclicModuleInstance, error) { - return &cjsModuleInstance{rt: rt, m: cm}, nil +func (cm *cjsModule) Instantiate(_ *goja.Runtime) (goja.CyclicModuleInstance, error) { + return &cjsModuleInstance{m: cm}, nil } func (cm *cjsModule) RequestedModules() []string { return nil } -func (cm *cjsModule) Evaluate(_ *goja.Runtime) *goja.Promise { - panic("this shouldn't be called in the current implementation") -} +func (cm *cjsModule) Evaluate(_ *goja.Runtime) *goja.Promise { return nil } -func (cm *cjsModule) GetExportedNames(_ ...goja.ModuleRecord) []string { - cm.o.Do(func() { - panic("somehow we first got to GetExportedNames of a commonjs module before they were set" + - "- this should never happen and is some kind of a bug") - }) - return cm.exportedNames -} +func (cm *cjsModule) GetExportedNames(_ ...goja.ModuleRecord) []string { return cm.exportedNames } func (cm *cjsModule) ResolveExport(exportName string, _ ...goja.ResolveSetElement) (*goja.ResolvedBinding, bool) { return &goja.ResolvedBinding{ @@ -44,18 +90,15 @@ func (cm *cjsModule) ResolveExport(exportName string, _ ...goja.ResolveSetElemen } type cjsModuleInstance struct { - rt *goja.Runtime - m *cjsModule - exports *goja.Object - isEsModuleMarked bool + m *cjsModule + exports *goja.Object } func (cmi *cjsModuleInstance) HasTLA() bool { return false } func (cmi *cjsModuleInstance) GetBindingValue(name string) goja.Value { if name == "default" { - d := cmi.exports.Get("default") - if d != nil { + if d := cmi.exports.Get("default"); d != nil { return d } return cmi.exports @@ -64,38 +107,40 @@ func (cmi *cjsModuleInstance) GetBindingValue(name string) goja.Value { } func (cmi *cjsModuleInstance) ExecuteModule(rt *goja.Runtime, _, _ func(any)) (goja.CyclicModuleInstance, error) { - v, err := rt.RunProgram(cmi.m.prg) + f, err := rt.RunProgram(cmi.m.prg) if err != nil { return nil, err } - module := rt.NewObject() + jsModule := rt.NewObject() cmi.exports = rt.NewObject() - _ = module.Set("exports", cmi.exports) - jsRequire := rt.Get("require") - call, ok := goja.AssertFunction(v) - if !ok { - return nil, errors.New("somehow a commonjs module is not wrapped in a function") - } - if _, err = call(cmi.exports, cmi.exports, jsRequire, module); err != nil { - return nil, err - } - exportsV := module.Get("exports") - if goja.IsNull(exportsV) { - return nil, errors.New("exports must be an object") // TODO make this message more specific for commonjs + _ = jsModule.Set("exports", cmi.exports) + if call, ok := goja.AssertFunction(f); ok { + jsRequire := rt.Get("require") + + // Run the module source, with "cmi.exports" as "this", + // "cmi.exports" as the "exports" variable, "jsRequire" + // as the "require" variable and "jsModule" as the + // "module" variable (Nodejs capable). + _, err = call(cmi.exports, cmi.exports, jsRequire, jsModule) + if err != nil { + return nil, err + } } - cmi.exports = exportsV.ToObject(rt) + exports := jsModule.Get("exports") + if goja.IsNull(exports) { + return nil, ErrInvalidModule + } + cmi.exports = exports.ToObject(rt) cmi.m.o.Do(func() { cmi.m.exportedNames = cmi.exports.Keys() }) - __esModule := cmi.exports.Get("__esModule") //nolint:revive,stylecheck - cmi.isEsModuleMarked = __esModule != nil && __esModule.ToBoolean() return cmi, nil } type goModule struct { - mod jsmodule.Module + mod Module once sync.Once exportedNames []string } @@ -107,11 +152,13 @@ func (gm *goModule) RequestedModules() []string { return nil } func (gm *goModule) InitializeEnvironment() error { return nil } func (gm *goModule) Instantiate(rt *goja.Runtime) (goja.CyclicModuleInstance, error) { - object := rt.ToValue(gm.mod.Exports()).ToObject(rt) - gm.once.Do(func() { - gm.exportedNames = object.Keys() - }) - return &goModuleInstance{object}, nil + instance, err := gm.mod.Instantiate(rt) + if err != nil { + return nil, err + } + exports := instance.ToObject(rt) + gm.once.Do(func() { gm.exportedNames = exports.Keys() }) + return &goModuleInstance{exports}, nil } func (gm *goModule) GetExportedNames(_ ...goja.ModuleRecord) []string { @@ -125,18 +172,21 @@ func (gm *goModule) ResolveExport(exportName string, _ ...goja.ResolveSetElement }, false } -func (gm *goModule) Evaluate(_ *goja.Runtime) *goja.Promise { panic("this shouldn't happen") } +func (gm *goModule) Evaluate(_ *goja.Runtime) *goja.Promise { return nil } -type goModuleInstance struct{ export *goja.Object } +type goModuleInstance struct{ *goja.Object } func (gmi *goModuleInstance) GetBindingValue(name string) goja.Value { - if name == "default" { - return gmi.export - } - if gmi.export == nil { + if gmi.Object == nil { return nil } - return gmi.export.Get(name) + if name == "default" { + if v := gmi.Get("default"); v != nil { + return v + } + return gmi.Object + } + return gmi.Get(name) } func (gmi *goModuleInstance) HasTLA() bool { return false } @@ -144,3 +194,89 @@ func (gmi *goModuleInstance) HasTLA() bool { return false } func (gmi *goModuleInstance) ExecuteModule(_ *goja.Runtime, _, _ func(any)) (goja.CyclicModuleInstance, error) { return gmi, nil } + +const parserPrefix = "parser/" + +type jsParser struct{ ski.Parser } + +type exec struct { + e ski.Executor +} + +func (e exec) Exec(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + v, err := e.e.Exec(Context(rt), call.Argument(0).Export()) + if err != nil { + return goja.Null() + } + return rt.ToValue(v) +} + +func (m *jsParser) Instantiate(rt *goja.Runtime) (goja.Value, error) { + object := rt.ToValue(m.Value).ToObject(rt) + _ = object.SetPrototype(rt.ToValue(map[string]func(call goja.FunctionCall, rt *goja.Runtime) goja.Value{ + "value": m.Value, + "element": m.Element, + "elements": m.Elements, + }).ToObject(rt)) + return object, nil +} + +func (m *jsParser) Value(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + executor, err := m.Parser.Value(call.Argument(0).String()) + if err != nil { + Throw(rt, err) + } + return rt.ToValue(exec{executor}) +} + +func (m *jsParser) Element(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + p, ok := m.Parser.(ski.ElementParser) + if !ok { + return goja.Null() + } + executor, err := p.Element(call.Argument(0).String()) + if err != nil { + Throw(rt, err) + } + return rt.ToValue(exec{executor}) +} + +func (m *jsParser) Elements(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + p, ok := m.Parser.(ski.ElementParser) + if !ok { + return goja.Null() + } + executor, err := p.Elements(call.Argument(0).String()) + if err != nil { + Throw(rt, err) + } + return rt.ToValue(exec{executor}) +} + +// Parser the esm parser of ski.Parser +type Parser struct{ ModuleLoader } + +func (p Parser) Value(arg string) (ski.Executor, error) { + if p.ModuleLoader == nil { + return nil, errors.New("ModuleLoader can not be nil") + } + module, err := p.CompileModule("", arg) + if err != nil { + return nil, err + } + return _mod{module}, nil +} + +// ModExec return a ski.Executor +func ModExec(cm goja.CyclicModuleRecord) ski.Executor { return _mod{cm} } + +type _mod struct{ goja.CyclicModuleRecord } + +func (m _mod) Exec(ctx context.Context, arg any) (any, error) { + value, err := RunModule(ski.WithValue(ctx, "content", arg), m) + if err != nil { + return nil, err + } + + return Unwrap(value) +} diff --git a/js/modules/cache/cache.go b/js/modules/cache/cache.go index 9cbfb01..0a56c20 100644 --- a/js/modules/cache/cache.go +++ b/js/modules/cache/cache.go @@ -2,34 +2,37 @@ package cache import ( + "errors" "time" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" ) -// Module js module -type Module struct{} - -// Exports returns the module instance -func (*Module) Exports() any { - return &Cache{cloudcat.MustResolve[cloudcat.Cache]()} -} - func init() { - jsmodule.Register("cache", new(Module)) + js.Register("cache", &Cache{ski.NewCache()}) } // Cache interface is used to store string or bytes. -type Cache struct { - cache cloudcat.Cache +type Cache struct{ ski.Cache } + +func (c *Cache) Instantiate(rt *goja.Runtime) (goja.Value, error) { + if c.Cache == nil { + return nil, errors.New("Cache can not nil") + } + return rt.ToValue(map[string]func(call goja.FunctionCall, vm *goja.Runtime) goja.Value{ + "get": c.Get, + "getBytes": c.GetBytes, + "set": c.Set, + "setBytes": c.SetBytes, + "del": c.Del, + }), nil } // Get returns string. func (c *Cache) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - if bytes, ok := c.cache.Get(js.VMContext(vm), call.Argument(0).String()); ok { + if bytes, err := c.Cache.Get(js.Context(vm), call.Argument(0).String()); err == nil && bytes != nil { return vm.ToValue(string(bytes)) } return goja.Undefined() @@ -37,7 +40,7 @@ func (c *Cache) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value { // GetBytes returns ArrayBuffer. func (c *Cache) GetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - if bytes, ok := c.cache.Get(js.VMContext(vm), call.Argument(0).String()); ok { + if bytes, err := c.Cache.Get(js.Context(vm), call.Argument(0).String()); err == nil && bytes != nil { return vm.ToValue(vm.NewArrayBuffer(bytes)) } return goja.Undefined() @@ -45,29 +48,32 @@ func (c *Cache) GetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value { // Set saves string to the cache with key. func (c *Cache) Set(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - ctx := js.VMContext(vm) + ctx := js.Context(vm) if !goja.IsUndefined(call.Argument(2)) { timeout, err := time.ParseDuration(call.Argument(2).String()) if err != nil { js.Throw(vm, err) } - ctx = cloudcat.WithCacheTimeout(ctx, timeout) + ctx = ski.WithCacheTimeout(ctx, timeout) } - c.cache.Set(ctx, call.Argument(0).String(), []byte(call.Argument(1).String())) + err := c.Cache.Set(ctx, call.Argument(0).String(), []byte(call.Argument(1).String())) + if err != nil { + js.Throw(vm, err) + } return goja.Undefined() } // SetBytes saves ArrayBuffer to the cache with key. func (c *Cache) SetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - ctx := js.VMContext(vm) + ctx := js.Context(vm) if !goja.IsUndefined(call.Argument(2)) { timeout, err := time.ParseDuration(call.Argument(2).String()) if err != nil { js.Throw(vm, err) } - ctx = cloudcat.WithCacheTimeout(ctx, timeout) + ctx = ski.WithCacheTimeout(ctx, timeout) } value, err := js.ToBytes(call.Argument(1).Export()) @@ -75,13 +81,19 @@ func (c *Cache) SetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value { js.Throw(vm, err) } - c.cache.Set(ctx, call.Argument(0).String(), value) + err = c.Cache.Set(ctx, call.Argument(0).String(), value) + if err != nil { + js.Throw(vm, err) + } return goja.Undefined() } // Del removes key from the cache. func (c *Cache) Del(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - c.cache.Del(js.VMContext(vm), call.Argument(0).String()) + err := c.Cache.Del(js.Context(vm), call.Argument(0).String()) + if err != nil { + js.Throw(vm, err) + } return goja.Undefined() } diff --git a/js/modules/cache/cache_test.go b/js/modules/cache/cache_test.go index bf9014b..df9a874 100644 --- a/js/modules/cache/cache_test.go +++ b/js/modules/cache/cache_test.go @@ -1,27 +1,32 @@ package cache import ( - "context" "testing" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/dop251/goja" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestCache(t *testing.T) { t.Parallel() - cloudcat.Provide[cloudcat.Cache](cloudcat.NewCache()) - ctx := context.Background() - vm := modulestest.New(t) + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + cache := Cache{ski.NewCache()} + instantiate, err := cache.Instantiate(rt) + if err != nil { + t.Fatal(err) + } + _ = rt.Set("cache", instantiate) + })) - _, err := vm.RunString(ctx, ` - const cache = require('cloudcat/cache'); + _, err := vm.Runtime().RunString(` cache.set("cache1", "1"); cache.del("cache1"); assert.true(!cache.get("cache1"), "cache should be deleted"); cache.set("cache2", "2", "1s"); - cache.get("cache2"); + assert.equal(cache.get("not exists"), undefined); assert.equal(cache.get("not exists"), undefined); assert.equal(cache.get("cache2"), "2"); cache.setBytes("cache3", new Uint8Array([50])); diff --git a/js/modules/cookie/cookie.go b/js/modules/cookie/cookie.go deleted file mode 100644 index 61e27e3..0000000 --- a/js/modules/cookie/cookie.go +++ /dev/null @@ -1,68 +0,0 @@ -// Package cookie the cookie JS implementation -package cookie - -import ( - "net/url" - - "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin/jsmodule" -) - -// Module js module -type Module struct{} - -// Exports returns module instance -func (*Module) Exports() any { - return &Cookie{cloudcat.MustResolve[cloudcat.Cookie]()} -} - -func init() { - jsmodule.Register("cookie", new(Module)) -} - -// Cookie manages storage and use of cookies in HTTP requests. -type Cookie struct { - cookie cloudcat.Cookie -} - -// Get returns the cookies string for the given URL. -func (c *Cookie) Get(uri string) ([]string, error) { - u, err := url.Parse(uri) - if err != nil { - return nil, err - } - return c.cookie.CookieString(u), nil -} - -// Set handles the receipt of the cookies strung in a reply for the given URL. -func (c *Cookie) Set(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { - u, err := url.Parse(call.Argument(0).String()) - if err != nil { - js.Throw(vm, err) - } - - str, err := js.ToStrings(call.Argument(1).Export()) - if err != nil { - js.Throw(vm, err) - } - - switch cookie := str.(type) { - case string: - c.cookie.SetCookies(u, cloudcat.ParseCookie(cookie)) - case []string: - c.cookie.SetCookies(u, cloudcat.ParseSetCookie(cookie...)) - } - return -} - -// Del handles the receipt of the cookies in a reply for the given URL. -func (c *Cookie) Del(uri string) error { - u, err := url.Parse(uri) - if err != nil { - return err - } - c.cookie.DeleteCookie(u) - return nil -} diff --git a/js/modules/cookie/cookie_test.go b/js/modules/cookie/cookie_test.go deleted file mode 100644 index 2b99880..0000000 --- a/js/modules/cookie/cookie_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package cookie - -import ( - "context" - "testing" - - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js/modulestest" - "github.com/stretchr/testify/assert" -) - -func TestCookie(t *testing.T) { - t.Parallel() - cloudcat.Provide[cloudcat.Cookie](cloudcat.NewCookie()) - ctx := context.Background() - vm := modulestest.New(t) - - _, _ = vm.Runtime().RunString(`const cookie = require('cloudcat/cookie')`) - - var err error - errScript := []string{`cookie.set('\x0000', "");`, `cookie.get('\x0000');`, `cookie.del('\x0000');`} - for _, s := range errScript { - _, err = vm.RunString(ctx, s) - assert.ErrorContains(t, err, "net/url: invalid control character in URL") - } - - _, err = vm.RunString(ctx, ` - cookie.set("https://github.com", ["test=1; path=/; secure; HttpOnly;"]); - cookie.del("https://github.com"); - assert.true(!cookie.get("https://github.com").length, "cookie should be deleted"); - cookie.set("https://github.com", ["has_recent_activity=1; path=/; secure; HttpOnly; SameSite=Lax"]); - assert.equal("has_recent_activity=1", cookie.get("https://github.com")[0]); - `) - assert.NoError(t, err) -} diff --git a/js/modules/crypto/crypto.go b/js/modules/crypto/crypto.go index ef93620..3a6f6c0 100644 --- a/js/modules/crypto/crypto.go +++ b/js/modules/crypto/crypto.go @@ -6,22 +6,25 @@ import ( "encoding/hex" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski/js" ) -// Module js module -type Module struct{} +func init() { + js.Register("crypto", new(Crypto)) +} -// Exports returns module instance -func (*Module) Exports() any { - return map[string]any{ +// Crypto js module +type Crypto struct{} + +// Instantiate returns module instance +func (*Crypto) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(map[string]any{ "aes": Aes, "createCipher": CreateCipher, "createHash": CreateHash, "createHMAC": CreateHMAC, "des": Des, "hmac": Hmac, - "md4": Md4, "md5": Md5, "randomBytes": RandomBytes, "ripemd160": Ripemd160, @@ -32,42 +35,20 @@ func (*Module) Exports() any { "sha512": Sha512, "sha512_224": Sha512_224, "sha512_256": Sha512_256, - } -} - -func init() { - jsmodule.Register("crypto", new(Module)) + }), nil } // Encoder the encoded -type Encoder struct { - data []byte -} +type Encoder struct{ data []byte } // Base64 encode to base64 -func (e *Encoder) Base64() string { - return base64.StdEncoding.EncodeToString(e.data) -} - -// Base64url encode to base64url -func (e *Encoder) Base64url() string { - return base64.URLEncoding.EncodeToString(e.data) -} - -// Base64rawurl encode to base64rawurl -func (e *Encoder) Base64rawurl() string { - return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(e.data) -} +func (e *Encoder) Base64() string { return base64.StdEncoding.EncodeToString(e.data) } // Hex encode to hex -func (e *Encoder) Hex() string { - return hex.EncodeToString(e.data) -} +func (e *Encoder) Hex() string { return hex.EncodeToString(e.data) } // String encode to string -func (e *Encoder) String() string { - return string(e.data) -} +func (e *Encoder) String() string { return string(e.data) } // Binary encode to arraybuffer func (e *Encoder) Binary(_ goja.FunctionCall, vm *goja.Runtime) goja.Value { diff --git a/js/modules/crypto/digest.go b/js/modules/crypto/digest.go index 1426c15..81cbe59 100644 --- a/js/modules/crypto/digest.go +++ b/js/modules/crypto/digest.go @@ -2,9 +2,9 @@ package crypto import ( "crypto/hmac" - "crypto/md5" //nolint:gosec + "crypto/md5" "crypto/rand" - "crypto/sha1" //nolint:gosec + "crypto/sha1" "crypto/sha256" "crypto/sha512" "errors" @@ -12,74 +12,50 @@ import ( "hash" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/js" - "golang.org/x/crypto/md4" //nolint:staticcheck - "golang.org/x/crypto/ripemd160" //nolint:staticcheck + "github.com/shiroyk/ski/js" + "golang.org/x/crypto/ripemd160" ) -// Copyright grafana/k6, licensed under the AGPL License. - -// RandomBytes returns random data of the given size. -func RandomBytes(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { +// RandomBytes returns a random ArrayBuffer of the given size. +func RandomBytes(call goja.FunctionCall, rt *goja.Runtime) goja.Value { size := int(call.Argument(0).ToInteger()) if size < 1 { - js.Throw(vm, errors.New("invalid size")) + js.Throw(rt, errors.New("invalid size")) } bytes := make([]byte, size) if _, err := rand.Read(bytes); err != nil { - js.Throw(vm, err) + js.Throw(rt, err) } - return vm.ToValue(vm.NewArrayBuffer(bytes)) -} - -// Md4 returns the MD4 Hash of input in the given encoding. -func Md4(input any) (any, error) { - return Hash("md4", input) + return rt.ToValue(rt.NewArrayBuffer(bytes)) } // Md5 returns the MD5 Hash of input in the given encoding. -func Md5(input any) (any, error) { - return Hash("md5", input) -} +func Md5(input any) (any, error) { return Hash("md5", input) } // Sha1 returns the SHA1 Hash of input in the given encoding. -func Sha1(input any) (any, error) { - return Hash("sha1", input) -} +func Sha1(input any) (any, error) { return Hash("sha1", input) } // Sha256 returns the SHA256 Hash of input in the given encoding. -func Sha256(input any) (any, error) { - return Hash("sha256", input) -} +func Sha256(input any) (any, error) { return Hash("sha256", input) } // Sha384 returns the SHA384 Hash of input in the given encoding. -func Sha384(input any) (any, error) { - return Hash("sha384", input) -} +func Sha384(input any) (any, error) { return Hash("sha384", input) } // Sha512 returns the SHA512 Hash of input in the given encoding. -func Sha512(input any) (any, error) { - return Hash("sha512", input) -} +func Sha512(input any) (any, error) { return Hash("sha512", input) } // Sha512_224 returns the SHA512/224 Hash of input in the given encoding. -func Sha512_224(input any) (any, error) { - return Hash("sha512_224", input) -} +func Sha512_224(input any) (any, error) { return Hash("sha512_224", input) } // Sha512_256 returns the SHA512/256 Hash of input in the given encoding. -func Sha512_256(input any) (any, error) { - return Hash("sha512_256", input) -} +func Sha512_256(input any) (any, error) { return Hash("sha512_256", input) } // Ripemd160 returns the RIPEMD160 Hash of input in the given encoding. -func Ripemd160(input any) (any, error) { - return Hash("ripemd160_256", input) -} +func Ripemd160(input any) (any, error) { return Hash("ripemd160", input) } // CreateHash returns a Hasher instance that uses the given algorithm. func CreateHash(algorithm string) (*Hasher, error) { - h := parseHashFunc(algorithm) //nolint:ifshort + h := hashFunc(algorithm) if h == nil { return nil, fmt.Errorf("invalid algorithm: %s", algorithm) } @@ -97,16 +73,16 @@ func Hash(algorithm string, input any) (*Encoder, error) { // CreateHMAC returns a new HMAC Hash using the given algorithm and key. func CreateHMAC(algorithm string, key any) (*Hasher, error) { - h := parseHashFunc(algorithm) + h := hashFunc(algorithm) if h == nil { return nil, fmt.Errorf("invalid algorithm: %s", algorithm) } - kb, err := js.ToBytes(key) + data, err := js.ToBytes(key) if err != nil { return nil, err } - return &Hasher{hmac.New(h, kb)}, nil + return &Hasher{hmac.New(h, data)}, nil } // Hmac returns a new Encoder of input using the given algorithm and key. @@ -118,29 +94,27 @@ func Hmac(algorithm string, key, input any) (*Encoder, error) { return hasher.Encrypt(input) } -func parseHashFunc(a string) func() hash.Hash { - var h func() hash.Hash - switch a { - case "md4": - h = md4.New +func hashFunc(name string) func() hash.Hash { + switch name { case "md5": - h = md5.New + return md5.New case "sha1": - h = sha1.New + return sha1.New case "sha256": - h = sha256.New + return sha256.New case "sha384": - h = sha512.New384 + return sha512.New384 + case "sha512": + return sha512.New case "sha512_224": - h = sha512.New512_224 + return sha512.New512_224 case "sha512_256": - h = sha512.New512_256 - case "sha512": - h = sha512.New - case "ripemd160_256": - h = ripemd160.New + return sha512.New512_256 + case "ripemd160": + return ripemd160.New + default: + return nil } - return h } // Hasher wraps a hash.Hash. diff --git a/js/modules/crypto/digest_test.go b/js/modules/crypto/digest_test.go index d8b470b..7e75fd4 100644 --- a/js/modules/crypto/digest_test.go +++ b/js/modules/crypto/digest_test.go @@ -1,347 +1,75 @@ package crypto import ( - "context" - "crypto/rand" - "errors" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/dop251/goja" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) -type MockReader struct{} - -func (MockReader) Read(_ []byte) (n int, err error) { - return -1, errors.New("contrived failure") -} - func TestHashAlgorithms(t *testing.T) { - if testing.Short() { - return - } - - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - `) - - t.Run("RandomBytesSuccess", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - let buf = crypto.randomBytes(5); - assert.equal(5, buf.byteLength); - `) - - assert.NoError(t, err) - }) - - t.Run("RandomBytesInvalidSize", func(t *testing.T) { - _, err := vm.RunString(context.Background(), `crypto.randomBytes(-1);`) - - assert.Error(t, err) - }) - - t.Run("RandomBytesFailure", func(t *testing.T) { - SavedReader := rand.Reader - rand.Reader = MockReader{} - _, err := vm.RunString(context.Background(), `crypto.randomBytes(5);`) - rand.Reader = SavedReader - - assert.Error(t, err) - }) - - t.Run("MD4", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "aa010fbc1d14c795d86ef98c95479d17"; - var hash = crypto.md4("hello world").hex(); - assert.equal(correct, hash); - `) - assert.NoError(t, err) - }) - - t.Run("MD5", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "5eb63bbbe01eeed093cb22bb8f5acdc3"; - var hash = crypto.md5("hello world").hex(); - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA1", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed"; - var hash = crypto.sha1("hello world").hex(); - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA256", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; - var hash = crypto.sha256("hello world").hex(); - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA384", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd"; - var hash = crypto.sha384("hello world").hex(); - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA512", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correct = "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"; - var hash = crypto.sha512("hello world").hex(); - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA512_224", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var hash = crypto.sha512_224("hello world").hex(); - var correct = "22e0d52336f64a998085078b05a6e37b26f8120f43bf4db4c43a64ee"; - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("SHA512_256", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var hash = crypto.sha512_256("hello world").hex(); - var correct = "0ac561fac838104e3f2e4ad107b4bee3e938bf15f2b15f009ccccd61a913f017"; - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) - - t.Run("RIPEMD160", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var hash = crypto.ripemd160("hello world").hex(); - var correct = "98c615784ccb5fe5936fbc0cbe9dfdb408d92f0f"; - assert.equal(correct, hash); - `) - - assert.NoError(t, err) - }) -} - -func TestStreamingApi(t *testing.T) { - if testing.Short() { - return + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + c := new(Crypto) + instance, _ := c.Instantiate(rt) + _ = rt.Set("crypto", instance) + })) + + testCases := []struct { + algorithm, origin, want string + }{ + {"md5", "hello md5", "741fc6b1878e208346359af502dd11c5"}, + {"ripemd160", "hello ripemd160", "6fb0548fc1acb266457d6ddae686905295b47a2a"}, + {"sha1", "hello sha1", "64faca92dec81be17500f67d521fbd32bb3a6968"}, + {"sha256", "hello sha256", "433855b7d2b96c23a6f60e70c655eb4305e8806b682a9596a200642f947259b1"}, + {"sha384", "hello sha384", "5a37b3a56f9a5ae7b267d25303801d2a610c329d799e9a61879fe35b8108ccb8a4c1154c420ea69fdb6d177fbf6db8b6"}, + {"sha512", "hello sha512", "ae9ae8f823f9b841bd94062d0af09c2dcffc04a705a89e5415330ed1279f369ea990ca92d63adda838696efe28436c0c14d8e805cd0f04b6c6a0e25127de838c"}, + {"sha512_224", "hello sha512_224", "60765c29a50404c4ff1797540fd5bd38383a24d1232e39030638e647"}, + {"sha512_256", "hello sha512_256", "b5e03d2c411178f6c174370e2f420d274cd20b9635ae7a41e40120d826a4b23b"}, } - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - `) - - // Empty strings are still hashable - t.Run("Empty", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correctHex = "d41d8cd98f00b204e9800998ecf8427e"; - var hasher = crypto.createHash("md5"); - assert.equal(correctHex, hasher.digest().hex()); - `) - - assert.NoError(t, err) - }) - - t.Run("UpdateOnce", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3"; - - var hasher = crypto.createHash("md5"); - hasher.update("hello world"); - assert.equal(correctHex, hasher.digest().hex()); - `) - - assert.NoError(t, err) - }) - - t.Run("UpdateMultiple", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3"; - - var hasher = crypto.createHash("md5"); - hasher.update("hello"); - hasher.update(" "); - hasher.update("world"); - - assert.equal(correctHex, hasher.digest().hex()); - `) - - assert.NoError(t, err) - }) -} - -func TestOutputEncoding(t *testing.T) { - if testing.Short() { - return + for _, testCase := range testCases { + t.Run(testCase.algorithm, func(t *testing.T) { + _, err := vm.Runtime().RunString(`{ + let correct = "` + testCase.want + `"; + let hash = crypto.` + testCase.algorithm + `("` + testCase.origin + `").hex(); + assert.equal(hash, correct); + }`) + assert.NoError(t, err) + }) } - - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - `) - - t.Run("Valid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - let correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3"; - let correctBase64 = "XrY7u+Ae7tCTyyK7j1rNww=="; - let correctBase64URL = "XrY7u-Ae7tCTyyK7j1rNww==" - let correctBase64RawURL = "XrY7u-Ae7tCTyyK7j1rNww"; - let correctBinary = new Uint8Array([94,182,59,187,224,30,238,208,147,203,34,187,143,90,205,195]); - - let hasher = crypto.createHash("md5"); - let encoder = hasher.encrypt("hello world"); - - assert.equal(correctHex, encoder.hex()); - assert.equal(correctBase64, encoder.base64()); - assert.equal(correctBase64URL, encoder.base64url()); - assert.equal(correctBase64RawURL, encoder.base64rawurl()); - assert.equal(correctBinary, encoder.binary()); - `) - - assert.NoError(t, err) - }) - - t.Run("Invalid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - crypto.createHash("md5").encrypt("hello world").someInvalidEncoding(); - `) - assert.ErrorContains(t, err, "Object has no member 'someInvalidEncoding'") - }) } func TestHMac(t *testing.T) { - if testing.Short() { - return + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + c := new(Crypto) + instance, _ := c.Instantiate(rt) + _ = rt.Set("crypto", instance) + })) + + testCases := []struct { + algorithm, origin, want string + }{ + {"md5", "hello hmac md5", "6c241e7c650d8a839aeff9a7a28db599"}, + {"ripemd160", "hello hmac ripemd160", "dfbd49aebc8a7cc33ffd3f6e16ab922a23329c2d"}, + {"sha1", "hello hmac sha1", "754cfe3b0dc73755f9d7cfa90ec979e2c1d42f08"}, + {"sha256", "hello hmac sha256", "1d103c86749c67b0c5531bcf4b1125f32540a3bad4165f4efe804a1a5b4dd9f1"}, + {"sha384", "hello hmac sha384", "bc19f1775949f93a53909fb674c65e6978d6fa80173ead68717543d5e01c229ae0d7f6c5f8901147e9998dd477c701cb"}, + {"sha512", "hello hmac sha512", "1f893eec7580ed74a38053c88d0a380c99213f7cb727984692b25f318e49b3e4f0b9c5ae9c5ba942287738d8d812608c0223e1a599bf4b1429a2972cb2a7844a"}, + {"sha512_224", "hello hmac sha512_224", "5f4a8c8cb6404ad3ff85ccbde756d231ff2544be3be702a4706c8a9b"}, + {"sha512_256", "hello hmac sha512_256", "e466b90580a96d60c34a4fb164afc725840c94d30ce1bdafaa00f8f830771dd8"}, } - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - `) - - testData := map[string]string{ - "md4": "92d8f5c302cf04cca0144d7a9feb1596", - "md5": "e04f2ec05c8b12e19e46936b171c9d03", - "sha1": "c113b62711ff5d8e8100bbb17b998591af81dc24", - "sha256": "7fd04df92f636fd450bc841c9418e5825c17f33ad9c87c518115a45971f7f77e", - "sha384": "d331e169e2dcfc742e80a3bf4dcc76d0e6425ab3777a3ac217ac6b2552aad5529ed4d40135b06e53a495ac7425d1e462", - "sha512_224": "bac4e6256bdbf81d029aec48af4fdd4b14001db6721f07c429a80817", - "sha512_256": "e3d0763ba92a4f40676c3d5b234d9842b71951e6e0767082cfb3f5e14c124b22", - "sha512": "cd3146f96a3005024108ff56b025517552435589a4c218411f165da0a368b6f47228b20a1a4bf081e4aae6f07e2790f27194fc77f0addc890e98ce1951cacc9f", - "ripemd160_256": "00bb4ce0d6afd4c7424c9d01b8a6caa3e749b08b", - } - for algorithm, value := range testData { - _ = vm.Runtime().Set("correctHex", vm.Runtime().ToValue(value)) - _ = vm.Runtime().Set("algorithm", vm.Runtime().ToValue(algorithm)) - - t.Run(algorithm+" hasher: valid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var hasher = crypto.createHMAC(algorithm, "a secret"); - assert.equal(correctHex, hasher.encrypt("some data to hash").hex()); - `) - - assert.NoError(t, err) - }) - - t.Run(algorithm+" wrapper: valid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var resultHex = crypto.hmac(algorithm, "a secret", "some data to hash").hex(); - assert.equal(correctHex, resultHex); - `) - + for _, testCase := range testCases { + t.Run(testCase.algorithm, func(t *testing.T) { + _, err := vm.Runtime().RunString(`{ + let correct = "` + testCase.want + `"; + let origin = "` + testCase.origin + `"; + let hasher = crypto.createHMAC("` + testCase.algorithm + `", "some secret"); + assert.equal(hasher.encrypt(origin).hex(), correct); + }`) assert.NoError(t, err) }) - - t.Run(algorithm+" ArrayBuffer: valid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var data = new Uint8Array([115,111,109,101,32,100,97,116,97,32,116, - 111,32,104,97,115,104]); - var resultHex = crypto.hmac(algorithm, "a secret", data).hex(); - assert.equal(correctHex, resultHex); - `) - - assert.NoError(t, err) - }) - } - - // Algorithms not supported or typing error - invalidData := map[string]string{ - "md6": "e04f2ec05c8b12e19e46936b171c9d03", - "sha526": "7fd04df92f636fd450bc841c9418e5825c17f33ad9c87c518115a45971f7f77e", - "sha348": "d331e169e2dcfc742e80a3bf4dcc76d0e6425ab3777a3ac217ac6b2552aad5529ed4d40135b06e53a495ac7425d1e462", } - for algorithm, value := range invalidData { - algorithm := algorithm - _ = vm.Runtime().Set("correctHex", vm.Runtime().ToValue(value)) - _ = vm.Runtime().Set("algorithm", vm.Runtime().ToValue(algorithm)) - t.Run(algorithm+" hasher: invalid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var hasher = crypto.createHMAC(algorithm, "a secret"); - assert.equal(correctHex, hasher.hash("some data to hash").hex()) - `) - - assert.Contains(t, err.Error(), "invalid algorithm: "+algorithm) - }) - - t.Run(algorithm+" wrapper: invalid", func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var resultHex = crypto.hmac(algorithm, "a secret", "some data to hash").hex(); - assert.equal(correctHex, resultHex); - `) - - assert.Contains(t, err.Error(), "invalid algorithm: "+algorithm) - }) - } -} - -func TestAWSv4(t *testing.T) { - // example values from https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html - vm := modulestest.New(t) - - _, err := vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - let hmacSHA256 = function(data, key) { - return crypto.hmac("sha256", key, data); - }; - - let expectedKDate = '969fbb94feb542b71ede6f87fe4d5fa29c789342b0f407474670f0c2489e0a0d' - let expectedKRegion = '69daa0209cd9c5ff5c8ced464a696fd4252e981430b10e3d3fd8e2f197d7a70c' - let expectedKService = 'f72cfd46f26bc4643f06a11eabb6c0ba18780c19a8da0c31ace671265e3c87fa' - let expectedKSigning = 'f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d' - - let key = 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'; - let dateStamp = '20120215'; - let regionName = 'us-east-1'; - let serviceName = 'iam'; - - let kDate = hmacSHA256(dateStamp, "AWS4" + key); - let kRegion = hmacSHA256(regionName, kDate.binary()); - let kService = hmacSHA256(serviceName, kRegion.binary()); - let kSigning = hmacSHA256("aws4_request", kService.binary()); - - assert.equal(expectedKDate, kDate.hex()); - assert.equal(expectedKRegion, kRegion.hex()); - assert.equal(expectedKService, kService.hex()); - assert.equal(expectedKSigning, kSigning.hex()); - `) - assert.NoError(t, err) } diff --git a/js/modules/crypto/symmetric.go b/js/modules/crypto/symmetric.go index de38ef7..519b970 100644 --- a/js/modules/crypto/symmetric.go +++ b/js/modules/crypto/symmetric.go @@ -9,7 +9,7 @@ import ( "fmt" "strings" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski/js" ) // Aes returns a new AES cipher diff --git a/js/modules/crypto/symmetric_test.go b/js/modules/crypto/symmetric_test.go index 3bf6176..ba59145 100644 --- a/js/modules/crypto/symmetric_test.go +++ b/js/modules/crypto/symmetric_test.go @@ -1,10 +1,11 @@ package crypto import ( - "context" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/dop251/goja" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) @@ -14,10 +15,24 @@ func TestCipherAlgorithm(t *testing.T) { return } - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const crypto = require('cloudcat/crypto'); - `) + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + c := new(Crypto) + instance, _ := c.Instantiate(rt) + _ = rt.Set("crypto", instance) + })) + + t.Run("Cipher", func(t *testing.T) { + _, err := vm.Runtime().RunString(`{ + let key = "1111111111111111"; + let iv = "1111111111111111"; + let text = "hello aes"; + let aes = crypto.createCipher("AES/ECB/ZERO", key, iv); + let result = aes.encrypt(text); + let decrypt = aes.decrypt(result.binary()).string(); + assert.equal(text, decrypt); + }`) + assert.NoError(t, err) + }) t.Run("AES", func(t *testing.T) { mode := []string{"ECB", "CBC", "CFB", "OFB", "CTR", "GCM"} @@ -27,15 +42,15 @@ func TestCipherAlgorithm(t *testing.T) { for _, p := range padding { _ = vm.Runtime().Set("P", p) t.Run(m+"/"+p, func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var key = "1111111111111111"; - var iv = "1111111111111111"; - var text = "hello aes"; - var aes = crypto.aes(key, iv, 'AES'+'/'+M+'/'+P); - var result = aes.encrypt(text); - var decrypt = aes.decrypt(result.binary()).string(); + _, err := vm.Runtime().RunString(`{ + let key = "1111111111111111"; + let iv = "1111111111111111"; + let text = "hello aes"; + let aes = crypto.aes(key, iv, 'AES'+'/'+M+'/'+P); + let result = aes.encrypt(text); + let decrypt = aes.decrypt(result.binary()).string(); assert.equal(text, decrypt); - `) + }`) assert.NoError(t, err) }) } @@ -50,15 +65,15 @@ func TestCipherAlgorithm(t *testing.T) { for _, p := range padding { _ = vm.Runtime().Set("P", p) t.Run(m+"/"+p, func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var key = "11111111"; - var iv = "11111111"; - var text = "hello des"; - var des = crypto.des(key, iv, 'DES'+'/'+M+'/'+P); - var result = des.encrypt(text); - var decrypt = des.decrypt(result.binary()).string(); + _, err := vm.Runtime().RunString(`{ + let key = "11111111"; + let iv = "11111111"; + let text = "hello des"; + let des = crypto.des(key, iv, 'DES'+'/'+M+'/'+P); + let result = des.encrypt(text); + let decrypt = des.decrypt(result.binary()).string(); assert.equal(text, decrypt); - `) + }`) assert.NoError(t, err) }) } @@ -73,14 +88,14 @@ func TestCipherAlgorithm(t *testing.T) { for _, p := range padding { _ = vm.Runtime().Set("P", p) t.Run(m+"/"+p, func(t *testing.T) { - _, err := vm.RunString(context.Background(), ` - var key = "111111111111111111111111"; - var text = "hello des"; - var des = crypto.tripleDes(key, null, 'TripleDes'+'/'+M+'/'+P); - var result = des.encrypt(text); - var decrypt = des.decrypt(result.binary()).string(); + _, err := vm.Runtime().RunString(`{ + let key = "111111111111111111111111"; + let text = "hello des"; + let des = crypto.tripleDes(key, null, 'TripleDes'+'/'+M+'/'+P); + let result = des.encrypt(text); + let decrypt = des.decrypt(result.binary()).string(); assert.equal(text, decrypt); - `) + }`) assert.NoError(t, err) }) } diff --git a/js/modules/encoding/encoding.go b/js/modules/encoding/encoding.go index 13dddf0..c1531d2 100644 --- a/js/modules/encoding/encoding.go +++ b/js/modules/encoding/encoding.go @@ -6,22 +6,21 @@ import ( "strings" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski/js" ) -// Module js module -type Module struct{} - -// Exports returns module instance -func (*Module) Exports() any { - return map[string]any{ - "base64": &Base64{}, - } +func init() { + js.Register("encoding", new(Encoding)) } -func init() { - jsmodule.Register("encoding", new(Module)) +// Encoding js module +type Encoding struct{} + +// Instantiate returns Encoding module instance +func (*Encoding) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(map[string]any{ + "base64": new(Base64), + }), nil } // Base64 encoding and decoding diff --git a/js/modules/encoding/encoding_test.go b/js/modules/encoding/encoding_test.go index 8f6720f..8d5dd2a 100644 --- a/js/modules/encoding/encoding_test.go +++ b/js/modules/encoding/encoding_test.go @@ -1,25 +1,22 @@ package encoding import ( - "context" "fmt" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/dop251/goja" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestEncodingBase64(t *testing.T) { t.Parallel() - if testing.Short() { - return - } - - vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(` - const encoding = require('cloudcat/encoding'); - `) + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + instantiate, _ := new(Encoding).Instantiate(rt) + _ = rt.Set("encoding", instantiate) + })) buffer := vm.Runtime().NewArrayBuffer([]byte{100, 97, 110, 107, 111, 103, 97, 105}) @@ -50,7 +47,7 @@ func TestEncodingBase64(t *testing.T) { if testCase.url { code += "URI" } - _, err := vm.RunString(context.Background(), code+"(raw, padding);assert.equal(want, result);") + _, err := vm.Runtime().RunString(code + "(raw, padding);assert.equal(want, result);") assert.NoError(t, err) }) } @@ -73,7 +70,7 @@ func TestEncodingBase64(t *testing.T) { _ = vm.Runtime().Set("want", testCase.want) _ = vm.Runtime().Set("toBuffer", testCase.toBuffer) - _, err := vm.RunString(context.Background(), ` + _, err := vm.Runtime().RunString(` assert.equal(want, encoding.base64.decode(raw, toBuffer)); `) assert.NoError(t, err) diff --git a/js/modules/http/cookiejar.go b/js/modules/http/cookiejar.go new file mode 100644 index 0000000..4c9e4af --- /dev/null +++ b/js/modules/http/cookiejar.go @@ -0,0 +1,156 @@ +package http + +import ( + "errors" + "net/http" + "net/url" + "time" + + "github.com/dop251/goja" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" + "github.com/spf13/cast" +) + +// CookieJar manages storage and use of cookies in HTTP requests. +type CookieJar struct{ ski.CookieJar } + +func (j *CookieJar) Instantiate(rt *goja.Runtime) (goja.Value, error) { + if j.CookieJar == nil { + return nil, errors.New("CookieJar can not nil") + } + return rt.ToValue(map[string]func(call goja.FunctionCall, rt *goja.Runtime) goja.Value{ + "get": j.Get, + "getAll": j.GetAll, + "set": j.Set, + "del": j.Del, + }), nil +} + +// Get returns the cookie for the given option. +func (j *CookieJar) Get(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + opt, err := cast.ToStringMapStringE(call.Argument(0).Export()) + if err != nil { + js.Throw(rt, errors.New("get parameter must be an object containing name, url")) + } + u, err := url.Parse(opt["url"]) + if err != nil { + js.Throw(rt, err) + } + cookies := j.CookieJar.Cookies(u) + name := opt["name"] + for _, cookie := range cookies { + if cookie.Name == name { + return toObj(cookie, rt) + } + } + if len(cookies) > 0 { + return toObj(cookies[0], rt) + } + return goja.Null() +} + +// GetAll returns the cookies for the given option. +func (j *CookieJar) GetAll(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + opt, err := cast.ToStringMapStringE(call.Argument(0).Export()) + if err != nil { + js.Throw(rt, errors.New("getAll parameter must be an object containing name, url")) + } + u, err := url.Parse(opt["url"]) + if err != nil { + js.Throw(rt, err) + } + return toObjs(j.CookieJar.Cookies(u), rt) +} + +// Set handles the receipt of the cookies in a reply for the given option. +func (j *CookieJar) Set(call goja.FunctionCall, rt *goja.Runtime) (ret goja.Value) { + u, err := url.Parse(call.Argument(0).String()) + if err != nil { + js.Throw(rt, errors.New("set first parameter must be url string")) + } + var cookies []*http.Cookie + switch e := call.Argument(1).Export().(type) { + case map[string]any: + cookies = append(cookies, toCookie(e)) + case []any: + for _, cookie := range cookies { + cookies = append(cookies, toCookie(cast.ToStringMap(cookie))) + } + default: + js.Throw(rt, errors.New("set second parameter must be cookie object")) + } + if len(cookies) == 0 { + return goja.Undefined() + } + + j.CookieJar.SetCookies(u, cookies) + return goja.Undefined() +} + +// Del handles the receipt of the cookies in a reply for the given URL. +func (j *CookieJar) Del(call goja.FunctionCall, rt *goja.Runtime) goja.Value { + u, err := url.Parse(call.Argument(0).String()) + if err != nil { + js.Throw(rt, err) + } + j.CookieJar.RemoveCookie(u) + return goja.Undefined() +} + +var sameSiteMapping = [...]string{ + http.SameSiteDefaultMode: "", + http.SameSiteLaxMode: "lax", + http.SameSiteStrictMode: "strict", + http.SameSiteNoneMode: "none", +} + +func toObj(cookie *http.Cookie, rt *goja.Runtime) goja.Value { + o := rt.NewObject() + _ = o.Set("domain", rt.ToValue(cookie.Domain)) + _ = o.Set("expires", rt.ToValue(cookie.Expires.Unix())) + _ = o.Set("name", rt.ToValue(cookie.Name)) + _ = o.Set("path", rt.ToValue(cookie.Path)) + _ = o.Set("sameSite", rt.ToValue(sameSiteMapping[cookie.SameSite])) + _ = o.Set("secure", rt.ToValue(cookie.Secure)) + _ = o.Set("value", rt.ToValue(cookie.Value)) + _ = o.Set("toString", func(goja.FunctionCall) goja.Value { + return rt.ToValue(cookie.String()) + }) + return o +} + +func toObjs(cookies []*http.Cookie, rt *goja.Runtime) goja.Value { + ret := make([]goja.Value, 0, len(cookies)) + for _, cookie := range cookies { + ret = append(ret, toObj(cookie, rt)) + } + return rt.ToValue(ret) +} + +func toCookie(o map[string]any) *http.Cookie { + var sameSite = http.SameSiteDefaultMode + switch cast.ToString(o["sameSite"]) { + case "lax": + sameSite = http.SameSiteLaxMode + case "strict": + sameSite = http.SameSiteStrictMode + case "none": + sameSite = http.SameSiteNoneMode + } + expires := cast.ToInt64(o["expires"]) + if expires == 0 { + expires = time.Now().Add(time.Hour * 72).Unix() + } + return &http.Cookie{ + Domain: cast.ToString(o["domain"]), + Expires: time.Unix(expires, 0), + Name: cast.ToString(o["name"]), + Path: cast.ToString(o["path"]), + SameSite: sameSite, + Value: cast.ToString(o["value"]), + MaxAge: cast.ToInt(o["maxAge"]), + Secure: cast.ToBool(o["secure"]), + HttpOnly: cast.ToBool(o["httpOnly"]), + } +} diff --git a/js/modules/http/cookiejar_test.go b/js/modules/http/cookiejar_test.go new file mode 100644 index 0000000..d5b255b --- /dev/null +++ b/js/modules/http/cookiejar_test.go @@ -0,0 +1,54 @@ +package http + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/dop251/goja" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" + "github.com/stretchr/testify/assert" +) + +func TestCookie(t *testing.T) { + t.Parallel() + vm := modulestest.New(t, js.WithInitial(func(rt *goja.Runtime) { + jar := CookieJar{ski.NewCookieJar()} + instantiate, err := jar.Instantiate(rt) + if err != nil { + t.Fatal(err) + } + _ = rt.Set("cookieJar", instantiate) + client := http.Client{Jar: jar} + instance, _ := (&Http{&client}).Instantiate(rt) + _ = rt.Set("http", instance) + })) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if cookie, err := r.Cookie("foo"); err == nil { + _, err = fmt.Fprint(w, cookie.String()) + assert.NoError(t, err) + return + } + w.WriteHeader(http.StatusOK) + })) + _ = vm.Runtime().Set("url", ts.URL) + + _, err := vm.RunString(context.Background(), ` + cookieJar.set("https://github.com", { name: "foo", value: "bar", path: "/", maxAge: 7200 }); + assert.equal("bar", cookieJar.get({ url: "https://github.com" }).value); + cookieJar.del("https://github.com"); + assert.true(!cookieJar.get({ url: "https://github.com" }), "cookie should be deleted"); + cookieJar.set(url, { name: "foo", value: "bar", path: "/", maxAge: 7200 }); + const res1 = http.get(url); + assert.equal(res1.text(), "foo=bar"); + cookieJar.del(url); + const res2 = http.get(url); + assert.equal(res2.text(), ""); + `) + assert.NoError(t, err) +} diff --git a/js/modules/http/form_data.go b/js/modules/http/form_data.go index ea51061..6042b57 100644 --- a/js/modules/http/form_data.go +++ b/js/modules/http/form_data.go @@ -2,98 +2,119 @@ package http import ( "fmt" + "slices" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" ) -// FileData wraps the file data and filename -type FileData struct { - Data []byte - Filename string +// fileData wraps the file data and filename +type fileData struct { + data []byte + filename string } -// FormData provides a way to construct a set of key/value pairs representing form fields and their values. +// formData provides a way to construct a set of key/value pairs representing form fields and their values. // which can be sent using the http() method and encoding type were set to "multipart/form-data". // Implement the https://developer.mozilla.org/en-US/docs/Web/API/FormData -type FormData struct { +type formData struct { + keys []string data map[string][]any } -// FormDataConstructor FormData Constructor -type FormDataConstructor struct{} +// FormData Constructor +type FormData struct{} -// Exports returns module instance -func (*FormDataConstructor) Exports() any { - return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object { - param := call.Argument(0) +// Instantiate returns module instance +func (*FormData) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(func(call goja.ConstructorCall) *goja.Object { + params := call.Argument(0) - if goja.IsUndefined(param) { - return vm.ToValue(FormData{make(map[string][]any)}).ToObject(vm) + var ret formData + if goja.IsUndefined(params) { + ret.data = make(map[string][]any) + return ret.object(rt) } - var pa map[string]any - var ok bool - pa, ok = param.Export().(map[string]any) - if !ok { - js.Throw(vm, fmt.Errorf("unsupported type %T", param.Export())) - } - - data := make(map[string][]any, len(pa)) + object := params.ToObject(rt) + keys := object.Keys() + ret.keys = make([]string, 0, len(keys)) + ret.data = make(map[string][]any, len(keys)) - for k, v := range pa { - switch ve := v.(type) { + for _, key := range keys { + value, _ := js.Unwrap(object.Get(key)) + switch ve := value.(type) { case []byte: // Default filename "blob". - data[k] = []any{FileData{ - Data: ve, - Filename: "blob", + ret.data[key] = []any{fileData{ + data: ve, + filename: "blob", }} case goja.ArrayBuffer: // Default filename "blob". - data[k] = []any{FileData{ - Data: ve.Bytes(), - Filename: "blob", + ret.data[key] = []any{fileData{ + data: ve.Bytes(), + filename: "blob", }} case []any: - data[k] = ve + ret.data[key] = ve default: - data[k] = []any{fmt.Sprintf("%v", ve)} + ret.data[key] = []any{fmt.Sprintf("%s", ve)} } + ret.keys = append(ret.keys, key) } - return vm.ToValue(FormData{data}).ToObject(vm) - } + return ret.object(rt) + }), nil } // Global it is a global module -func (*FormDataConstructor) Global() {} +func (*FormData) Global() {} + +func (f *formData) object(rt *goja.Runtime) *goja.Object { + obj := rt.ToValue(f).ToObject(rt) + + _ = obj.SetSymbol(goja.SymIterator, func(goja.ConstructorCall) *goja.Object { + var i int + it := rt.NewObject() + _ = it.Set("next", func(goja.FunctionCall) goja.Value { + if i < len(f.keys) { + key := f.keys[i] + i++ + return rt.ToValue(iter{Value: rt.ToValue([2]any{key, f.data[key]})}) + } + return rt.ToValue(iter{Done: true}) + }) + return it + }) + return obj +} -// Append method of the FormData interface appends a new value onto an existing key inside a FormData object, +// Append method of the formData interface appends a new value onto an existing key inside a formData object, // or adds the key if it does not already exist. -func (f *FormData) Append(name string, value any, filename string) (ret goja.Value) { +func (f *formData) Append(name string, value any, filename string) goja.Value { if filename == "" { // Default filename "blob". filename = "blob" } - var ele []any - var ok bool - if ele, ok = f.data[name]; !ok { + ele, ok := f.data[name] + if !ok { + f.keys = append(f.keys, name) ele = make([]any, 0) } switch v := value.(type) { case []byte: - ele = append(ele, FileData{ - Data: v, - Filename: filename, + ele = append(ele, fileData{ + data: v, + filename: filename, }) case goja.ArrayBuffer: - ele = append(ele, FileData{ - Data: v.Bytes(), - Filename: filename, + ele = append(ele, fileData{ + data: v.Bytes(), + filename: filename, }) default: ele = append(ele, fmt.Sprintf("%v", v)) @@ -101,36 +122,37 @@ func (f *FormData) Append(name string, value any, filename string) (ret goja.Val f.data[name] = ele - return + return goja.Undefined() } -// Delete method of the FormData interface deletes a key and its value(s) from a FormData object. -func (f *FormData) Delete(name string) { +// Delete method of the formData interface deletes a key and its value(s) from a formData object. +func (f *formData) Delete(name string) { + f.keys = slices.DeleteFunc(f.keys, func(k string) bool { return k == name }) delete(f.data, name) } -// Entries method returns an iterator which iterates through all key/value pairs contained in the FormData. -func (f *FormData) Entries() any { - entries := make([][2]any, 0, len(f.data)) - for k, v := range f.data { - entries = append(entries, [2]any{k, v}) +// Entries method returns an iterator which iterates through all key/value pairs contained in the formData. +func (f *formData) Entries() any { + entries := make([][2]any, 0, len(f.keys)) + for _, key := range f.keys { + entries = append(entries, [2]any{key, f.data[key]}) } return entries } -// Get method of the FormData interface returns the first value associated -// with a given key from within a FormData object. +// Get method of the formData interface returns the first value associated +// with a given key from within a formData object. // If you expect multiple values and want all of them, use the getAll() method instead. -func (f *FormData) Get(name string) any { +func (f *formData) Get(name string) any { if v, ok := f.data[name]; ok { return v[0] } return nil } -// GetAll method of the FormData interface returns all the values associated -// with a given key from within a FormData object. -func (f *FormData) GetAll(name string) any { +// GetAll method of the formData interface returns all the values associated +// with a given key from within a formData object. +func (f *formData) GetAll(name string) any { v, ok := f.data[name] if ok { return v @@ -138,29 +160,33 @@ func (f *FormData) GetAll(name string) any { return [0]any{} } -// Has method of the FormData interface returns whether a FormData object contains a certain key. -func (f *FormData) Has(name string) bool { +// Has method of the formData interface returns whether a formData object contains a certain key. +func (f *formData) Has(name string) bool { _, ok := f.data[name] return ok } -// Keys method returns an iterator which iterates through all keys contained in the FormData. +// Keys method returns an iterator which iterates through all keys contained in the formData. // The keys are strings. -func (f *FormData) Keys() any { return cloudcat.MapKeys(f.data) } +func (f *formData) Keys() any { return f.keys } -// Set method of the FormData interface sets a new value for an existing key inside a FormData object, +// Set method of the formData interface sets a new value for an existing key inside a formData object, // or adds the key/value if it does not already exist. -func (f *FormData) Set(name string, value any, filename string) { +func (f *formData) Set(name string, value any, filename string) { if filename == "" { filename = "blob" } + if _, ok := f.data[name]; !ok { + f.keys = append(f.keys, name) + } + switch v := value.(type) { case goja.ArrayBuffer: f.data[name] = []any{ - FileData{ - Data: v.Bytes(), - Filename: filename, + fileData{ + data: v.Bytes(), + filename: filename, }, } default: @@ -168,5 +194,5 @@ func (f *FormData) Set(name string, value any, filename string) { } } -// Values method returns an iterator which iterates through all values contained in the FormData. -func (f *FormData) Values() any { return cloudcat.MapValues(f.data) } +// Values method returns an iterator which iterates through all values contained in the formData. +func (f *formData) Values() any { return ski.MapValues(f.data) } diff --git a/js/modules/http/form_data_test.go b/js/modules/http/form_data_test.go index c68f2a2..5dcf543 100644 --- a/js/modules/http/form_data_test.go +++ b/js/modules/http/form_data_test.go @@ -2,46 +2,43 @@ package http import ( "context" - "fmt" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestFormData(t *testing.T) { - ctx := context.Background() vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(`const mp = new FormData({ + _, err := vm.RunString(context.Background(), ` + const form = new FormData({ 'file': new Uint8Array([50]), 'name': 'foo' - });`) - - testCase := []string{ - `try { + }); + try { new FormData(0); - } catch (e) { + } catch (e) { assert.true(e.toString().includes('unsupported type')) - }`, - `assert.equal(mp.get('name'), 'foo')`, - `mp.append('file', new Uint8Array([51]).buffer); - assert.equal(mp.getAll('file').length, 2)`, - `mp.append('name', 'bar'); - assert.equal(mp.keys().length, 2); - assert.equal(mp.get('name'), 'foo');`, - `assert.equal(mp.entries().length, 2)`, - `mp.delete('name'); - assert.equal(mp.getAll('name').length, 0)`, - `assert.true(!mp.has('name'))`, - `mp.set('name', 'foobar'); - assert.equal(mp.values().length, 2)`, - } - - for i, s := range testCase { - t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) - assert.NoError(t, err) - }) - } + } + assert.equal(form.get('name'), 'foo') + form.delete('name'); + assert.equal(form.get('name'), null); + form.append('file', new Uint8Array([51]).buffer); + assert.equal(form.getAll('file').length, 2) + form.append('name', 'bar'); + assert.equal(form.keys().length, 2); + assert.equal(form.get('name'), 'bar'); + assert.equal(form.entries().length, 2) + form.delete('name'); + assert.equal(form.getAll('name').length, 0) + assert.true(!form.has('name')) + form.set('name', 'foobar'); + assert.equal(form.values().length, 2) + let str = ""; + for (const [key, value] of form) { + str += key + ","; + } + assert.equal(str, 'file,name,')`) + assert.NoError(t, err) } diff --git a/js/modules/http/http.go b/js/modules/http/http.go index 725d523..558280d 100644 --- a/js/modules/http/http.go +++ b/js/modules/http/http.go @@ -14,175 +14,186 @@ import ( "strings" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin/jsmodule" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" "github.com/spf13/cast" - "golang.org/x/net/http/httpguts" ) -// Module js module -type Module struct{} - -// Exports returns module instance -func (*Module) Exports() any { - return &Http{cloudcat.MustResolve[cloudcat.Fetch]()} -} - func init() { - jsmodule.Register("http", new(Module)) - jsmodule.Register("fetch", new(FetchModule)) - jsmodule.Register("FormData", new(FormDataConstructor)) - jsmodule.Register("URLSearchParams", new(URLSearchParamsConstructor)) - jsmodule.Register("AbortController", new(AbortControllerConstructor)) - jsmodule.Register("AbortSignal", new(AbortSignalModule)) + jar := ski.NewCookieJar() + fetch := ski.NewFetch().(*http.Client) + fetch.Jar = jar + js.Register("cookieJar", &CookieJar{jar}) + js.Register("http", &Http{fetch}) + js.Register("fetch", &Fetch{fetch}) + js.Register("FormData", new(FormData)) + js.Register("URLSearchParams", new(URLSearchParams)) + js.Register("AbortController", new(AbortController)) + js.Register("AbortSignal", new(AbortSignal)) } -// FetchModule the global fetch() method starts the process of +// Fetch the global Fetch() method starts the process of // fetching a resource from the network, returning a promise // which is fulfilled once the response is available. // https://developer.mozilla.org/en-US/docs/Web/API/fetch -type FetchModule struct{} +type Fetch struct{ ski.Fetch } -func (*FetchModule) Exports() any { - fetch := cloudcat.MustResolveLazy[cloudcat.Fetch]() - return func(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - req, signal := buildRequest(http.MethodGet, call, vm) - return vm.ToValue(js.NewPromise(vm, func() (*http.Response, error) { - if signal != nil { - defer signal.abort() // release resources - } - return fetch().Do(req) - }, func(res *http.Response, err error) (any, error) { - if err != nil { - return nil, err - } - return NewAsyncResponse(vm, res), nil - })) +func (fetch *Fetch) Instantiate(rt *goja.Runtime) (goja.Value, error) { + if fetch.Fetch == nil { + return nil, errors.New("Fetch can not nil") } + return rt.ToValue(func(call goja.FunctionCall, vm *goja.Runtime) goja.Value { + req, signal := buildRequest(http.MethodGet, call, vm) + return vm.ToValue(js.NewPromise(vm, + func() (*http.Response, error) { + if signal != nil { + defer signal.abort() // release resources + } + return fetch.Do(req) + }, + func(res *http.Response, err error) (any, error) { + if err != nil { + return nil, err + } + return NewAsyncResponse(vm, res), nil + })) + }), nil } -func (*FetchModule) Global() {} +func (*Fetch) Global() {} // Http module for fetching resources (including across the network). -type Http struct { //nolint - fetch cloudcat.Fetch +type Http struct{ ski.Fetch } + +func (h *Http) Instantiate(rt *goja.Runtime) (goja.Value, error) { + if h.Fetch == nil { + return nil, errors.New("Fetch can not nil") + } + return rt.ToValue(map[string]func(call goja.FunctionCall, vm *goja.Runtime) goja.Value{ + "get": h.Get, + "post": h.Post, + "put": h.Put, + "delete": h.Delete, + "patch": h.Patch, + "request": h.Request, + "head": h.Head, + }), nil } // Get Make a HTTP GET request. func (h *Http) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodGet, call, vm) + return h.do(call, vm, http.MethodGet) } // Post Make a HTTP POST. // Send POST with multipart: -// http.post(url, { body: new FormData({'bytes': new Uint8Array([0]).buffer}) }) +// http.post(url, { body: new formData({'bytes': new Uint8Array([0]).buffer}) }) // Send POST with x-www-form-urlencoded: -// http.post(url, { body: new URLSearchParams({'key': 'foo', 'value': 'bar'}) }) +// http.post(url, { body: new urlSearchParams({'key': 'foo', 'value': 'bar'}) }) // Send POST with json: // http.post(url, { body: {'key': 'foo'} }) func (h *Http) Post(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodPost, call, vm) + return h.do(call, vm, http.MethodPost) } // Put Make a HTTP PUT request. func (h *Http) Put(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodPut, call, vm) + return h.do(call, vm, http.MethodPut) } // Delete Make a HTTP DELETE request. func (h *Http) Delete(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodDelete, call, vm) + return h.do(call, vm, http.MethodDelete) } // Patch Make a HTTP PATCH request. func (h *Http) Patch(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodPatch, call, vm) + return h.do(call, vm, http.MethodPatch) } // Request Make a HTTP request. func (h *Http) Request(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodGet, call, vm) + return h.do(call, vm, http.MethodGet) } // Head Make a HTTP HEAD request. func (h *Http) Head(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return doRequest(h.fetch, http.MethodHead, call, vm) + return h.do(call, vm, http.MethodHead) +} + +func (h *Http) do(call goja.FunctionCall, vm *goja.Runtime, method string) goja.Value { + req, signal := buildRequest(method, call, vm) + if signal != nil { + defer signal.abort() // release resources + } + + res, err := h.Do(req) + if err != nil { + js.Throw(vm, err) + } + + return NewResponse(vm, res) } func buildRequest( method string, call goja.FunctionCall, vm *goja.Runtime, -) (req *http.Request, signal *AbortSignal) { - url := call.Argument(0).String() - opt := call.Argument(1) - var body io.Reader - var headers = make(map[string]string) - var proxy *urlpkg.URL - var err error +) (req *http.Request, signal *abortSignal) { + var ( + ctx = context.Background() + url = call.Argument(0).String() + options = call.Argument(1) + opt *goja.Object + body io.Reader + headers = make(map[string]string) + err error + ) - if opt != nil && !goja.IsUndefined(opt) { - options, assert := opt.Export().(map[string]any) - if !assert { - js.Throw(vm, errors.New("request options is invalid")) - } - if v, ok := options["method"]; ok { - method, err = cast.ToStringE(v) - if err != nil { - js.Throw(vm, errors.New("request options method is invalid string")) - } - method = strings.ToUpper(method) - if !validMethod(method) { - js.Throw(vm, fmt.Errorf("request options method %v is invalid HTTP method", method)) - } - } - if v, ok := options["headers"]; ok { - headers, err = cast.ToStringMapStringE(v) - if err != nil { - js.Throw(vm, errors.New("request options headers is invalid")) - } + if goja.IsUndefined(options) || goja.IsNull(options) { + ctx = js.Context(vm) + goto NEW + } + + opt = options.ToObject(vm) + if v := opt.Get("method"); v != nil { + method = strings.ToUpper(v.String()) + } + if v := opt.Get("headers"); v != nil { + if headers, err = cast.ToStringMapStringE(v.Export()); err != nil { + js.Throw(vm, fmt.Errorf("options headers is invalid, %s", err)) } - if v, ok := options["body"]; ok { - body, err = handleBody(v, headers) - if err != nil { + } + if method != http.MethodGet && method != http.MethodHead { + if v := opt.Get("body"); v != nil { + if body, err = processBody(v.Export(), headers); err != nil { js.Throw(vm, err) } } - if v, ok := options["cache"]; ok { - str, err := cast.ToStringE(v) - if err != nil { - js.Throw(vm, errors.New("request options cache is invalid string")) - } - headers["Cache-Control"] = str - headers["Pragma"] = str - } - if v, ok := options["proxy"]; ok { - str, err := cast.ToStringE(v) - if err != nil { - js.Throw(vm, errors.New("request options proxy is invalid string")) - } - proxy, err = urlpkg.Parse(str) - if err != nil { - js.Throw(vm, errors.Join(errors.New("request options proxy is invalid URL"), err)) - } - } - if v, ok := options["signal"]; ok { - signal, ok = v.(*AbortSignal) - if !ok { - js.Throw(vm, errors.New("request options signal is invalid AbortSignal")) - } - } } - - var parent context.Context - if signal != nil { - parent = signal.ctx + if v := opt.Get("cache"); v != nil { + str := v.String() + headers["Cache-Control"] = str + headers["Pragma"] = str + } + if v := opt.Get("signal"); v != nil { + var ok bool + if signal, ok = v.Export().(*abortSignal); !ok { + js.Throw(vm, errors.New("options signal is not AbortSignal")) + } + ctx = signal.ctx } else { - parent = js.VMContext(vm) + ctx = js.Context(vm) + } + if v := opt.Get("proxy"); v != nil { + proxy, err := urlpkg.Parse(v.String()) + if err != nil { + js.Throw(vm, fmt.Errorf("options proxy is invalid URL, %s", err)) + } + ctx = ski.WithProxyURL(ctx, proxy) } - ctx := cloudcat.WithProxyURL(parent, proxy) +NEW: req, err = http.NewRequestWithContext(ctx, method, url, body) if err != nil { js.Throw(vm, err) @@ -195,65 +206,27 @@ func buildRequest( return } -func doRequest( - fetch cloudcat.Fetch, - method string, - call goja.FunctionCall, - vm *goja.Runtime, -) goja.Value { - req, signal := buildRequest(method, call, vm) - if signal != nil { - defer signal.abort() // release resources - } - - res, err := fetch.Do(req) - if err != nil { - js.Throw(vm, err) - } - - return NewResponse(vm, res) -} - -func validMethod(method string) bool { - /* - Method = "OPTIONS" ; Section 9.2 - | "GET" ; Section 9.3 - | "HEAD" ; Section 9.4 - | "POST" ; Section 9.5 - | "PUT" ; Section 9.6 - | "DELETE" ; Section 9.7 - | "TRACE" ; Section 9.8 - | "CONNECT" ; Section 9.9 - | extension-method - extension-method = token - token = 1* - */ - return len(method) > 0 && strings.IndexFunc(method, func(r rune) bool { - return !httpguts.IsTokenRune(r) - }) == -1 -} - -// handleBody process the send request body and set the content-type -func handleBody(body any, headers map[string]string) (io.Reader, error) { +// processBody process the send request body and set the content-type +func processBody(body any, headers map[string]string) (io.Reader, error) { switch data := body.(type) { - case FormData: + case *formData: buf := new(bytes.Buffer) mpw := multipart.NewWriter(buf) - for k, v := range data.data { - for _, ve := range v { - if f, ok := ve.(FileData); ok { + for _, key := range data.keys { + for _, value := range data.data[key] { + if f, ok := value.(fileData); ok { // Creates a new form-data header with the provided field name and file name. - fw, err := mpw.CreateFormFile(k, f.Filename) + fw, err := mpw.CreateFormFile(key, f.filename) if err != nil { return nil, err } // Write bytes to the part - if _, err := fw.Write(f.Data); err != nil { + if _, err = fw.Write(f.data); err != nil { return nil, err } } else { // Write string value - if err := mpw.WriteField(k, fmt.Sprintf("%v", v)); err != nil { + if err := mpw.WriteField(key, fmt.Sprintf("%v", key)); err != nil { return nil, err } } @@ -264,7 +237,7 @@ func handleBody(body any, headers map[string]string) (io.Reader, error) { return nil, err } return buf, nil - case URLSearchParams: + case *urlSearchParams: headers["Content-Type"] = "application/x-www-form-url" return strings.NewReader(data.encode()), nil case string: diff --git a/js/modules/http/http_test.go b/js/modules/http/http_test.go index bc69c21..9e83403 100644 --- a/js/modules/http/http_test.go +++ b/js/modules/http/http_test.go @@ -1,7 +1,6 @@ package http import ( - "context" "fmt" "io" "net/http" @@ -11,14 +10,14 @@ import ( "testing" "time" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/dop251/goja" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestHttp(t *testing.T) { - cloudcat.Provide[cloudcat.Fetch](&http.Client{Transport: &http.Transport{Proxy: cloudcat.ProxyFromRequest}}) vm := createVM(t) testCase := []string{ `assert.equal(http.get(url).text(), "");`, @@ -65,14 +64,22 @@ func TestHttp(t *testing.T) { for i, s := range testCase { t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(context.Background(), s) + _, err := vm.Runtime().RunString(s) assert.NoError(t, err) }) } } -func createVM(t *testing.T) js.VM { - vm := modulestest.New(t) +var initial = js.WithInitial(func(rt *goja.Runtime) { + client := http.Client{Transport: &http.Transport{Proxy: ski.ProxyFromRequest}} + instance, _ := (&Http{&client}).Instantiate(rt) + _ = rt.Set("http", instance) + f, _ := (&Fetch{&client}).Instantiate(rt) + _ = rt.Set("fetch", f) +}) + +func createVM(t *testing.T) modulestest.VM { + vm := modulestest.New(t, initial) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPut { @@ -123,7 +130,6 @@ func createVM(t *testing.T) js.VM { }) _, _ = vm.Runtime().RunString(fmt.Sprintf(` - const http = require('cloudcat/http'); const url = "%s"; const proxyURL = "%s"; const fa = new Uint8Array([226, 153, 130, 239, 184, 142])`, ts.URL, proxy.URL)) diff --git a/js/modules/http/response.go b/js/modules/http/response.go index 2551da6..c98603e 100644 --- a/js/modules/http/response.go +++ b/js/modules/http/response.go @@ -8,49 +8,72 @@ import ( "strings" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski/js" ) -func defineAccessorProperty(r *goja.Runtime, o *goja.Object, name string, v any) { - _ = o.DefineAccessorProperty(name, r.ToValue(func(call goja.FunctionCall) goja.Value { return r.ToValue(v) }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE) +var errBodyAlreadyRead = errors.New("body stream already read") + +func defineGetter(r *goja.Runtime, o *goja.Object, name string, v func() any) { + _ = o.DefineAccessorProperty(name, r.ToValue(func(goja.FunctionCall) goja.Value { + return r.ToValue(v()) + }), nil, goja.FLAG_FALSE, goja.FLAG_TRUE) } // NewResponse returns a new Response -func NewResponse(vm *goja.Runtime, res *http.Response) goja.Value { - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - js.Throw(vm, err) +func NewResponse(rt *goja.Runtime, res *http.Response) goja.Value { + var bodyUsed bool + js.OnDone(rt, func() { + if !bodyUsed { + res.Body.Close() + } + }) + readBody := func() []byte { + if bodyUsed { + js.Throw(rt, errBodyAlreadyRead) + } + bodyUsed = true + defer res.Body.Close() + data, err := io.ReadAll(res.Body) + if err != nil { + js.Throw(rt, err) + } + return data } - object := vm.NewObject() - _ = object.DefineAccessorProperty("body", vm.ToValue(func(call goja.FunctionCall) goja.Value { - return vm.ToValue(vm.NewArrayBuffer(body)) - }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE) - defineAccessorProperty(vm, object, "bodyUsed", true) - defineAccessorProperty(vm, object, "headers", joinHeader(res.Header)) - defineAccessorProperty(vm, object, "status", res.StatusCode) - defineAccessorProperty(vm, object, "statusText", res.Status) - defineAccessorProperty(vm, object, "ok", res.StatusCode >= 200 || res.StatusCode < 300) - _ = object.Set("text", func(goja.FunctionCall) goja.Value { return vm.ToValue(string(body)) }) - _ = object.Set("json", func(goja.FunctionCall) goja.Value { - var j any - if err = json.Unmarshal(body, &j); err != nil { - js.Throw(vm, err) + + object := rt.NewObject() + defineGetter(rt, object, "body", func() any { return rt.NewArrayBuffer(readBody()) }) + defineGetter(rt, object, "bodyUsed", func() any { return bodyUsed }) + defineGetter(rt, object, "headers", func() any { return joinHeader(res.Header) }) + defineGetter(rt, object, "status", func() any { return res.StatusCode }) + defineGetter(rt, object, "statusText", func() any { return res.Status }) + defineGetter(rt, object, "ok", func() any { + return res.StatusCode >= 200 && res.StatusCode < 300 + }) + _ = object.Set("text", func(goja.FunctionCall) goja.Value { return rt.ToValue(string(readBody())) }) + _ = object.Set("json", func(call goja.FunctionCall) goja.Value { + var data any + if err := json.Unmarshal(readBody(), &data); err != nil { + js.Throw(rt, err) } - return vm.ToValue(j) + return rt.ToValue(data) }) - _ = object.Set("arrayBuffer", func(goja.FunctionCall) goja.Value { return vm.ToValue(vm.NewArrayBuffer(body)) }) + _ = object.Set("arrayBuffer", func(goja.FunctionCall) goja.Value { return rt.ToValue(rt.NewArrayBuffer(readBody())) }) return object } // NewAsyncResponse returns a new async Response -func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value { +func NewAsyncResponse(rt *goja.Runtime, res *http.Response) goja.Value { var bodyUsed bool - object := vm.NewObject() + js.OnDone(rt, func() { + if !bodyUsed { + res.Body.Close() + } + }) + object := rt.NewObject() readBody := func() ([]byte, error) { if bodyUsed { - return nil, errors.New("body used already for") + return nil, errBodyAlreadyRead } bodyUsed = true defer res.Body.Close() @@ -61,20 +84,21 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value { return data, nil } - _ = object.DefineAccessorProperty("body", vm.ToValue(func(call goja.FunctionCall) goja.Value { + defineGetter(rt, object, "body", func() any { if bodyUsed { - js.Throw(vm, errors.New("body used already for")) + js.Throw(rt, errBodyAlreadyRead) } - return NewReadableStream(res.Body, vm, &bodyUsed) - }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE) - defineAccessorProperty(vm, object, "bodyUsed", &bodyUsed) - defineAccessorProperty(vm, object, "headers", joinHeader(res.Header)) - defineAccessorProperty(vm, object, "status", res.StatusCode) - defineAccessorProperty(vm, object, "statusText", res.Status) - defineAccessorProperty(vm, object, "ok", res.StatusCode >= 200 || res.StatusCode < 300) - + return NewReadableStream(res.Body, rt, &bodyUsed) + }) + defineGetter(rt, object, "bodyUsed", func() any { return bodyUsed }) + defineGetter(rt, object, "headers", func() any { return joinHeader(res.Header) }) + defineGetter(rt, object, "status", func() any { return res.StatusCode }) + defineGetter(rt, object, "statusText", func() any { return res.Status }) + defineGetter(rt, object, "ok", func() any { + return res.StatusCode >= 200 && res.StatusCode < 300 + }) _ = object.Set("text", func(goja.FunctionCall) goja.Value { - return vm.ToValue(js.NewPromise(vm, func() (any, error) { + return rt.ToValue(js.NewPromise(rt, func() (any, error) { data, err := readBody() if err != nil { return nil, err @@ -83,7 +107,7 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value { })) }) _ = object.Set("json", func(goja.FunctionCall) goja.Value { - return vm.ToValue(js.NewPromise(vm, func() (any, error) { + return rt.ToValue(js.NewPromise(rt, func() (any, error) { data, err := readBody() if err != nil { return nil, err @@ -96,12 +120,12 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value { })) }) _ = object.Set("arrayBuffer", func(goja.FunctionCall) goja.Value { - return vm.ToValue(js.NewPromise(vm, func() (any, error) { + return rt.ToValue(js.NewPromise(rt, func() (any, error) { data, err := readBody() if err != nil { return nil, err } - return vm.NewArrayBuffer(data), nil + return rt.NewArrayBuffer(data), nil })) }) return object @@ -127,7 +151,7 @@ func NewReadableStream(body io.ReadCloser, vm *goja.Runtime, bodyUsed *bool) *go }) _ = object.Set("getReader", func(call goja.FunctionCall) goja.Value { if *bodyUsed { - js.Throw(vm, errors.New("body used already for")) + js.Throw(vm, errBodyAlreadyRead) } *bodyUsed = true if lock { @@ -144,7 +168,7 @@ func NewReadableStream(body io.ReadCloser, vm *goja.Runtime, bodyUsed *bool) *go return object } -type chunk struct { +type iter struct { Value goja.Value Done bool } @@ -154,7 +178,7 @@ type chunk struct { // https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamBYOBReader func NewReadableStreamDefaultReader(body io.ReadCloser, vm *goja.Runtime, lock *bool) *goja.Object { object := vm.NewObject() - defineAccessorProperty(vm, object, "locked", &lock) + defineGetter(vm, object, "locked", func() any { return &lock }) _ = object.Set("cancel", func() { if err := body.Close(); err != nil { js.Throw(vm, err) @@ -175,11 +199,12 @@ func NewReadableStreamDefaultReader(body io.ReadCloser, vm *goja.Runtime, lock * value = call.Argument(0).ToObject(vm) } - return vm.ToValue(js.NewPromise(vm, func() (int, error) { return body.Read(buffer) }, + return vm.ToValue(js.NewPromise(vm, + func() (int, error) { return body.Read(buffer) }, func(n int, err error) (any, error) { if err != nil { if errors.Is(err, io.EOF) { - return chunk{goja.Undefined(), true}, nil + return iter{goja.Undefined(), true}, nil } return nil, err } @@ -190,7 +215,7 @@ func NewReadableStreamDefaultReader(body io.ReadCloser, vm *goja.Runtime, lock * js.Throw(vm, err) } } - return chunk{value, false}, nil + return iter{value, false}, nil })) }) diff --git a/js/modules/http/response_test.go b/js/modules/http/response_test.go index daf1be8..3268656 100644 --- a/js/modules/http/response_test.go +++ b/js/modules/http/response_test.go @@ -9,15 +9,12 @@ import ( "testing" "time" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestResponse(t *testing.T) { - ctx := context.Background() - vm := modulestest.New(t) - cloudcat.Provide[cloudcat.Fetch](http.DefaultClient) + vm := modulestest.New(t, initial) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -37,9 +34,10 @@ func TestResponse(t *testing.T) { })) _ = vm.Runtime().Set("url", ts.URL) - _, _ = vm.Runtime().RunString(`const http = require("cloudcat/http");`) testCase := []string{ + `const res = http.get(url+'/array'); + assert.true(res.ok);`, `const res = http.get(url+'/json'); assert.equal(res.json(), { "foo": "bar", "test": true }); assert.true(res.bodyUsed); @@ -55,27 +53,28 @@ func TestResponse(t *testing.T) { assert.equal(res.statusText, "200 OK"); assert.equal(res.headers["Content-Type"], "application/json");`, `const res = http.get(url+'/text'); - assert.true(res.bodyUsed); + assert.true(!res.bodyUsed); assert.true(res.ok); assert.equal(res.statusText, "200 OK"); assert.equal(res.text(), "foo"); - assert.equal(res.arrayBuffer(), new Uint8Array([102, 111, 111])); - assert.equal(res.arrayBuffer(), res.body); - assert.equal(res.headers["Content-Type"], "text/plain");`, + assert.true(res.bodyUsed); + try { + res.text(); + } catch (e) { + assert.true(e && e.toString().includes("body stream already read")); + }`, } for i, s := range testCase { t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) + _, err := vm.Runtime().RunString(fmt.Sprintf(`{%s}`, s)) assert.NoError(t, err) }) } } func TestAsyncResponse(t *testing.T) { - ctx := context.Background() - vm := modulestest.New(t) - cloudcat.Provide[cloudcat.Fetch](http.DefaultClient) + vm := modulestest.New(t, initial) ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -169,8 +168,34 @@ func TestAsyncResponse(t *testing.T) { for i, s := range testCase { t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) + _, err := vm.RunString(context.Background(), s) assert.NoError(t, err) }) } } + +type testBody struct { + closed bool +} + +func (*testBody) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (b *testBody) Close() error { + b.closed = true + return nil +} + +func TestAutoClose(t *testing.T) { + vm := modulestest.New(t) + body := new(testBody) + res := NewResponse(vm.Runtime(), &http.Response{Body: body, StatusCode: http.StatusOK}) + ctx := context.WithValue(context.Background(), "res", res) + assert.False(t, body.closed) + v, err := vm.RunModule(ctx, `export default (ctx) => ctx.get('res').ok`) + if assert.NoError(t, err) { + assert.True(t, v.ToBoolean()) + assert.True(t, body.closed) + } +} diff --git a/js/modules/http/signal.go b/js/modules/http/signal.go index 20ac48e..0622214 100644 --- a/js/modules/http/signal.go +++ b/js/modules/http/signal.go @@ -6,44 +6,43 @@ import ( "time" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski/js" ) -// AbortController interface represents a controller object +// abortController interface represents a controller object // that allows you to abort one or more Web requests as and when desired. // https://developer.mozilla.org/en-US/docs/Web/API/AbortController. -type AbortController struct { - Signal *AbortSignal +type abortController struct { + Signal *abortSignal Aborted bool Reason string } -func (c *AbortController) Abort() { +func (c *abortController) Abort() { c.Signal.abort() c.Aborted = c.Signal.Aborted c.Reason = c.Signal.Reason } -// AbortControllerConstructor AbortController Constructor -type AbortControllerConstructor struct{} +// AbortController Constructor +type AbortController struct{} -// Exports AbortController Constructor -func (*AbortControllerConstructor) Exports() any { - return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object { - signal := new(AbortSignal) - parent := js.VMContext(vm) - signal.ctx, signal.cancel = context.WithCancel(parent) - return vm.ToValue(&AbortController{Signal: signal}).ToObject(vm) - } +// Instantiate module +func (*AbortController) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object { + signal := new(abortSignal) + signal.ctx, signal.cancel = context.WithCancel(js.Context(vm)) + return vm.ToValue(&abortController{Signal: signal}).ToObject(vm) + }), nil } // Global it is a global module -func (*AbortControllerConstructor) Global() {} +func (*AbortController) Global() {} -// AbortSignal represents a signal object that allows you to communicate +// abortSignal represents a signal object that allows you to communicate // with http request and abort it. // https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal -type AbortSignal struct { +type abortSignal struct { ctx context.Context cancel context.CancelFunc once sync.Once @@ -51,7 +50,7 @@ type AbortSignal struct { Reason string } -func (s *AbortSignal) abort() { +func (s *abortSignal) abort() { s.once.Do(func() { s.Aborted = true s.cancel() @@ -61,24 +60,23 @@ func (s *AbortSignal) abort() { }) } -type AbortSignalModule struct{} +type AbortSignal struct{} -func (*AbortSignalModule) Exports() any { return new(abortSignalInstance) } - -func (*AbortSignalModule) Global() {} - -type abortSignalInstance struct{} - -func (s *abortSignalInstance) Abort(_ goja.FunctionCall, vm *goja.Runtime) goja.Value { - signal := new(AbortSignal) - signal.ctx, signal.cancel = context.WithCancel(context.Background()) - signal.abort() - return vm.ToValue(signal).ToObject(vm) +func (*AbortSignal) Instantiate(rt *goja.Runtime) (goja.Value, error) { + object := rt.NewObject() + _ = object.Set("abort", func(_ goja.FunctionCall) goja.Value { + signal := new(abortSignal) + signal.ctx, signal.cancel = context.WithCancel(context.Background()) + signal.abort() + return rt.ToValue(signal).ToObject(rt) + }) + _ = object.Set("timeout", func(call goja.FunctionCall) goja.Value { + timeout := call.Argument(0).ToInteger() + signal := new(abortSignal) + signal.ctx, signal.cancel = context.WithTimeout(js.Context(rt), time.Duration(timeout)) + return rt.ToValue(signal).ToObject(rt) + }) + return object, nil } -func (s *abortSignalInstance) Timeout(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - timeout := call.Argument(0).ToInteger() - signal := new(AbortSignal) - signal.ctx, signal.cancel = context.WithTimeout(js.VMContext(vm), time.Duration(timeout)) - return vm.ToValue(signal).ToObject(vm) -} +func (*AbortSignal) Global() {} diff --git a/js/modules/http/signal_test.go b/js/modules/http/signal_test.go index 12d6007..def118a 100644 --- a/js/modules/http/signal_test.go +++ b/js/modules/http/signal_test.go @@ -1,19 +1,17 @@ package http import ( - "context" "fmt" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestAbortSignal(t *testing.T) { - ctx := context.Background() vm := modulestest.New(t) - testCase := []string{ + testCases := []string{ `const controller = new AbortController(); controller.abort(); assert.equal(controller.reason, "context canceled"); @@ -26,9 +24,9 @@ func TestAbortSignal(t *testing.T) { assert.true(!signal.aborted);`, } - for i, s := range testCase { + for i, s := range testCases { t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) + _, err := vm.Runtime().RunString(fmt.Sprintf(`{%s}`, s)) assert.NoError(t, err) }) } diff --git a/js/modules/http/url_search_params.go b/js/modules/http/url_search_params.go index f9b9065..9970757 100644 --- a/js/modules/http/url_search_params.go +++ b/js/modules/http/url_search_params.go @@ -3,69 +3,88 @@ package http import ( "fmt" "net/url" - "sort" + "slices" "strings" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" "github.com/spf13/cast" ) -// The URLSearchParams defines utility methods to work with the query string of a URL, +// The urlSearchParams defines utility methods to work with the query string of a URL, // which can be sent using the http() method and encoding type were set to "application/x-www-form-url". // Implement the https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams -type URLSearchParams struct { +type urlSearchParams struct { + keys []string data map[string][]string } -// URLSearchParamsConstructor Constructor -type URLSearchParamsConstructor struct{} +// URLSearchParams Constructor +type URLSearchParams struct{} -// Exports instance URLSearchParams module -func (*URLSearchParamsConstructor) Exports() any { - return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object { - param := call.Argument(0) +// Instantiate instance module +func (*URLSearchParams) Instantiate(rt *goja.Runtime) (goja.Value, error) { + return rt.ToValue(func(call goja.ConstructorCall) *goja.Object { + params := call.Argument(0) - if goja.IsUndefined(param) { - return vm.ToValue(URLSearchParams{data: make(url.Values)}).ToObject(vm) + var ret urlSearchParams + if goja.IsUndefined(params) { + ret.data = make(map[string][]string) + return ret.object(rt) } - var pa map[string]any - var ok bool - pa, ok = param.Export().(map[string]any) - if !ok { - js.Throw(vm, fmt.Errorf("unsupported type %T", param.Export())) - } + object := params.ToObject(rt) + keys := object.Keys() + ret.keys = make([]string, 0, len(keys)) + ret.data = make(map[string][]string, len(keys)) - data := make(map[string][]string, len(pa)) - for k, v := range pa { - if s, ok := v.([]any); ok { - data[k] = cast.ToStringSlice(s) + for _, key := range keys { + value, _ := js.Unwrap(object.Get(key)) + if s, ok := value.([]any); ok { + ret.data[key] = cast.ToStringSlice(s) } else { - data[k] = []string{fmt.Sprintf("%v", v)} + ret.data[key] = []string{fmt.Sprintf("%s", value)} } + ret.keys = append(ret.keys, key) } - return vm.ToValue(URLSearchParams{data: data}).ToObject(vm) - } + return ret.object(rt) + }), nil } // Global it is a global module -func (*URLSearchParamsConstructor) Global() {} +func (*URLSearchParams) Global() {} + +func (u *urlSearchParams) object(rt *goja.Runtime) *goja.Object { + obj := rt.ToValue(u).ToObject(rt) + + _ = obj.SetSymbol(goja.SymIterator, func(goja.ConstructorCall) *goja.Object { + var i int + it := rt.NewObject() + _ = it.Set("next", func(goja.FunctionCall) goja.Value { + if i < len(u.keys) { + key := u.keys[i] + i++ + return rt.ToValue(iter{Value: rt.ToValue([2]any{key, u.data[key]})}) + } + return rt.ToValue(iter{Done: true}) + }) + return it + }) + return obj +} // encode encodes the values into “URL encoded” form // ("bar=baz&foo=qux") sorted by key. -func (u *URLSearchParams) encode() string { +func (u *urlSearchParams) encode() string { if u.data == nil { return "" } var buf strings.Builder - keys := cloudcat.MapKeys(u.data) - sort.Strings(keys) - for _, k := range keys { - vs := u.data[k] - keyEscaped := url.QueryEscape(k) + for _, key := range u.keys { + vs := u.data[key] + keyEscaped := url.QueryEscape(key) for _, v := range vs { if buf.Len() > 0 { buf.WriteByte('&') @@ -78,38 +97,41 @@ func (u *URLSearchParams) encode() string { return buf.String() } -// Append method of the URLSearchParams interface appends a specified key/value pair as a new search parameter. -func (u *URLSearchParams) Append(name, value string) { - u.data[name] = append(u.data[name], value) +// Append method of the urlSearchParams interface appends a specified key/value pair as a new search parameter. +func (u *urlSearchParams) Append(name, value string) { + values, ok := u.data[name] + if !ok { + u.keys = append(u.keys, name) + } + u.data[name] = append(values, value) } -// Delete method of the URLSearchParams interface deletes the given search parameter and all its associated values, +// Delete method of the urlSearchParams interface deletes the given search parameter and all its associated values, // from the list of all search parameters. -func (u *URLSearchParams) Delete(name string) { +func (u *urlSearchParams) Delete(name string) { + u.keys = slices.DeleteFunc(u.keys, func(k string) bool { return k == name }) delete(u.data, name) } -// Entries method of the URLSearchParams interface returns an iterator allowing iteration +// Entries method of the urlSearchParams interface returns an iterator allowing iteration // through all key/value pairs contained in this object. // The iterator returns key/value pairs in the same order as they appear in the query string. // The key and value of each pair are string objects. -func (u *URLSearchParams) Entries() any { - entries := make([][2]string, 0, len(u.data)) - for k, v := range u.data { - for _, ve := range v { - entries = append(entries, [2]string{k, ve}) - } +func (u *urlSearchParams) Entries() any { + entries := make([][2]any, 0, len(u.keys)) + for _, key := range u.keys { + entries = append(entries, [2]any{key, u.data[key]}) } return entries } -// ForEach method of the URLSearchParams interface allows iteration +// ForEach method of the urlSearchParams interface allows iteration // through all values contained in this object via a callback function. -func (u *URLSearchParams) ForEach(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { +func (u *urlSearchParams) ForEach(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { arg := call.Argument(0) if callback, ok := goja.AssertFunction(arg); ok { - for k, v := range u.data { - if _, err := callback(goja.Undefined(), vm.ToValue(v), vm.ToValue(k), vm.ToValue(u)); err != nil { + for _, key := range u.keys { + if _, err := callback(goja.Undefined(), vm.ToValue(u.data[key]), vm.ToValue(key), vm.ToValue(u)); err != nil { panic(vm.ToValue(err)) } } @@ -117,56 +139,55 @@ func (u *URLSearchParams) ForEach(call goja.FunctionCall, vm *goja.Runtime) (ret return } -// Get method of the URLSearchParams interface returns the first value associated to the given search parameter. -func (u *URLSearchParams) Get(name string) any { +// Get method of the urlSearchParams interface returns the first value associated to the given search parameter. +func (u *urlSearchParams) Get(name string) any { if v, ok := u.data[name]; ok { return v[0] } return nil } -// GetAll method of the URLSearchParams interface returns all the values associated +// GetAll method of the urlSearchParams interface returns all the values associated // with a given search parameter as an array. -func (u *URLSearchParams) GetAll(name string) []string { +func (u *urlSearchParams) GetAll(name string) []string { if v, ok := u.data[name]; ok { return v } return []string{} } -// Has method of the URLSearchParams interface returns a boolean value that indicates whether +// Has method of the urlSearchParams interface returns a boolean value that indicates whether // a parameter with the specified name exists. -func (u *URLSearchParams) Has(name string) bool { +func (u *urlSearchParams) Has(name string) bool { _, ok := u.data[name] return ok } -// Keys method of the URLSearchParams interface returns an iterator allowing iteration +// Keys method of the urlSearchParams interface returns an iterator allowing iteration // through all keys contained in this object. The keys are string objects. -func (u *URLSearchParams) Keys() []string { - return cloudcat.MapKeys(u.data) -} +func (u *urlSearchParams) Keys() []string { return u.keys } -// Set method of the URLSearchParams interface sets the value associated +// Set method of the urlSearchParams interface sets the value associated // with a given search parameter to the given value. // If there were several matching values, this method deletes the others. // If the search parameter doesn't exist, this method creates it. -func (u *URLSearchParams) Set(name, value string) { +func (u *urlSearchParams) Set(name, value string) { + if _, ok := u.data[name]; !ok { + u.keys = append(u.keys, name) + } u.data[name] = []string{value} } // Sort method sorts all key/value pairs contained in this object in place and returns undefined. -func (u *URLSearchParams) Sort() { - // Not implemented -} +func (u *urlSearchParams) Sort() { slices.Sort(u.keys) } -// ToString method of the URLSearchParams interface returns a query string suitable for use in a URL. -func (u *URLSearchParams) ToString() string { +// ToString method of the urlSearchParams interface returns a query string suitable for use in a URL. +func (u *urlSearchParams) ToString() string { return u.encode() } -// Values method of the URLSearchParams interface returns an iterator allowing iteration through +// Values method of the urlSearchParams interface returns an iterator allowing iteration through // all values contained in this object. The values are string objects. -func (u *URLSearchParams) Values() [][]string { - return cloudcat.MapValues(u.data) +func (u *urlSearchParams) Values() [][]string { + return ski.MapValues(u.data) } diff --git a/js/modules/http/url_search_params_test.go b/js/modules/http/url_search_params_test.go index 23916a7..790b663 100644 --- a/js/modules/http/url_search_params_test.go +++ b/js/modules/http/url_search_params_test.go @@ -1,19 +1,17 @@ package http import ( - "context" "fmt" "testing" - "github.com/shiroyk/cloudcat/js/modulestest" + "github.com/shiroyk/ski/js/modulestest" "github.com/stretchr/testify/assert" ) func TestURLSearchParams(t *testing.T) { - ctx := context.Background() vm := modulestest.New(t) - _, _ = vm.Runtime().RunString(`const form = new URLSearchParams({'name': 'foo'});form.sort();`) + _, _ = vm.Runtime().RunString(`const params = new URLSearchParams({'name': 'foo'});params.sort();`) testCase := []string{ `try { @@ -21,24 +19,32 @@ func TestURLSearchParams(t *testing.T) { } catch (e) { assert.true(e.toString().includes('unsupported type')) }`, - `form.forEach((v, k) => assert.true(v.length == 1)) - assert.equal(form.get('name'), 'foo')`, - `form.append('name', 'bar'); - assert.equal(form.getAll('name').length, 2)`, - `assert.equal(form.toString(), 'name=foo&name=bar')`, - `form.append('value', 'zoo'); - assert.true(form.keys(), ['name', 'value'])`, - `assert.equal(form.entries().length, 3)`, - `form.delete('name'); - assert.equal(form.getAll('name').length, 0)`, - `assert.true(!form.has('name'))`, - `form.set('name', 'foobar'); - assert.equal(form.values().length, 2)`, + `params.forEach((v, k) => assert.true(v.length == 1)) + assert.equal(params.get('name'), 'foo')`, + `params.append('name', 'bar'); + assert.equal(params.getAll('name').length, 2)`, + `assert.equal(params.toString(), 'name=foo&name=bar')`, + `params.append('value', 'zoo'); + assert.true(params.keys(), ['name', 'value'])`, + `assert.equal(params.entries().length, 2)`, + `params.delete('name'); + assert.equal(params.getAll('name').length, 0)`, + `assert.true(!params.has('name'))`, + `params.set('name', 'foobar'); + assert.equal(params.values().length, 2)`, + `params.append('000', '114'); + params.sort(); + assert.equal(params.toString(), '000=114&name=foobar&value=zoo')`, + `let str = ""; + for (const [key, value] of params) { + str += key + "=" + value + ","; + } + assert.equal(str, '000=114,name=foobar,value=zoo,')`, } for i, s := range testCase { t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) { - _, err := vm.RunString(ctx, s) + _, err := vm.Runtime().RunString(s) assert.NoError(t, err) }) } diff --git a/js/modules/main.go b/js/modules/main.go deleted file mode 100644 index b8e7a77..0000000 --- a/js/modules/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package jsmodules - -import ( - _ "github.com/shiroyk/cloudcat/js/modules/cache" // deep - _ "github.com/shiroyk/cloudcat/js/modules/cookie" - _ "github.com/shiroyk/cloudcat/js/modules/crypto" - _ "github.com/shiroyk/cloudcat/js/modules/encoding" - _ "github.com/shiroyk/cloudcat/js/modules/http" -) diff --git a/js/modulestest/vm.go b/js/modulestest/vm.go index 5346d70..87519d6 100644 --- a/js/modulestest/vm.go +++ b/js/modulestest/vm.go @@ -2,20 +2,36 @@ package modulestest import ( + "context" "errors" "testing" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/js" + "github.com/shiroyk/ski/js" "github.com/stretchr/testify/assert" ) -// New returns a test VM instance -func New(t *testing.T) js.VM { - vm := js.NewVM() - runtime := vm.Runtime() +type VM struct{ js.VM } - assertObject := runtime.NewObject() +func (vm *VM) RunString(ctx context.Context, source string) (ret goja.Value, err error) { + vm.Run(ctx, func() { + ret, err = vm.Runtime().RunString(source) + }) + return +} + +func (vm *VM) RunModule(ctx context.Context, source string) (ret goja.Value, err error) { + module, err := vm.Loader().CompileModule("", source) + if err != nil { + return + } + return vm.VM.RunModule(ctx, module) +} + +// New returns a test VM instance +func New(t *testing.T, opts ...js.Option) VM { + vm := js.NewVM(append([]js.Option{js.WithModuleLoader(js.NewModuleLoader())}, opts...)...) + assertObject := vm.Runtime().NewObject() _ = assertObject.Set("equal", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { a, err := js.Unwrap(call.Argument(0)) if err != nil { @@ -45,7 +61,6 @@ func New(t *testing.T) js.VM { return }) - _ = runtime.Set("assert", assertObject) - - return vm + _ = vm.Runtime().Set("assert", assertObject) + return VM{vm} } diff --git a/js/scheduler.go b/js/scheduler.go new file mode 100644 index 0000000..01ccbd1 --- /dev/null +++ b/js/scheduler.go @@ -0,0 +1,192 @@ +package js + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "runtime" + "sync/atomic" + "time" + + "log/slog" + + "github.com/dop251/goja" + "github.com/shiroyk/ski" +) + +const ( + // DefaultMaxTimeToWaitGetVM default retries time + DefaultMaxTimeToWaitGetVM = 500 * time.Millisecond + // DefaultMaxRetriesGetVM default retries times + DefaultMaxRetriesGetVM = 3 +) + +var ( + _scheduler = new(atomic.Value) + // ErrSchedulerClosed the scheduler is closed error + ErrSchedulerClosed = errors.New("scheduler is closed") +) + +func init() { + loader := NewModuleLoader() + ski.Register("js", Parser{loader}) + _scheduler.Store(NewScheduler(SchedulerOptions{ + MaxVMs: uint(runtime.GOMAXPROCS(0)), + Loader: loader, + })) +} + +// SetScheduler set the default Scheduler +func SetScheduler(scheduler Scheduler) { _scheduler.Store(scheduler) } + +// RunModule the goja.CyclicModuleRecord +func RunModule(ctx context.Context, module goja.CyclicModuleRecord) (goja.Value, error) { + vm, err := GetScheduler().Get() + if err != nil { + return nil, err + } + return vm.RunModule(ctx, module) +} + +// GetScheduler get the default Scheduler +func GetScheduler() Scheduler { return _scheduler.Load().(Scheduler) } + +// Scheduler the VM scheduler +type Scheduler interface { + // Get the VM + Get() (VM, error) + // Shrink the available VM + Shrink() + // Loader the ModuleLoader + Loader() ModuleLoader + // Close the scheduler + Close() error +} + +// SchedulerOptions options +type SchedulerOptions struct { + InitialVMs uint `yaml:"initial-vms" json:"initialVMs"` + MaxVMs uint `yaml:"max-vms" json:"maxVMs"` + MaxRetriesGetVM uint `yaml:"max-retries-get-vm" json:"maxRetriesGetVM"` + MaxTimeToWaitGetVM time.Duration `yaml:"max-time-to-wait-get-vm" json:"maxTimeToWaitGetVM"` + Loader ModuleLoader `yaml:"-"` // module loader + VMOptions []Option `yaml:"-"` // options for NewVM +} + +// NewScheduler returns a new Scheduler +func NewScheduler(opt SchedulerOptions) Scheduler { + s := &schedulerImpl{ + closed: new(atomic.Bool), + unInitVMs: new(atomic.Int32), + maxVMs: opt.MaxVMs, + maxRetriesGetVM: opt.MaxRetriesGetVM, + maxTimeToWaitGetVM: opt.MaxTimeToWaitGetVM, + loader: opt.Loader, + } + if s.maxVMs == 0 { + s.maxVMs = 1 + } + if s.maxRetriesGetVM == 0 { + s.maxRetriesGetVM = DefaultMaxRetriesGetVM + } + if s.maxTimeToWaitGetVM == 0 { + s.maxTimeToWaitGetVM = DefaultMaxTimeToWaitGetVM + } + if s.loader == nil { + slog.Warn("js.ModuleLoader not provided, require and module will not working") + s.loader = emptyLoader{} + } + s.maxVMs = max(s.maxVMs, opt.InitialVMs) + s.vms = make(chan VM, s.maxVMs) + s.vmOpt = append(opt.VMOptions, + func(vm *vmImpl) { + vm.release = func() { s.release(vm) } + }, WithModuleLoader(opt.Loader)) + for i := uint(0); i < opt.InitialVMs; i++ { + s.vms <- NewVM(s.vmOpt...) + } + s.unInitVMs.Store(int32(s.maxVMs - opt.InitialVMs)) + return s +} + +type schedulerImpl struct { + vms chan VM + maxVMs, maxRetriesGetVM uint + unInitVMs *atomic.Int32 + closed *atomic.Bool + maxTimeToWaitGetVM time.Duration + loader ModuleLoader + vmOpt []Option +} + +func (s *schedulerImpl) Loader() ModuleLoader { return s.loader } + +func (s *schedulerImpl) String() string { + text, _ := s.MarshalText() + return string(text) +} + +func (s *schedulerImpl) MarshalText() ([]byte, error) { + return json.Marshal(map[string]any{ + "available": len(s.vms), + "max": int(s.maxVMs), + "unInit": int(s.unInitVMs.Load()), + }) +} + +// Close the scheduler +func (s *schedulerImpl) Close() error { + s.closed.Store(true) + close(s.vms) + return nil +} + +// Get the VM +func (s *schedulerImpl) Get() (VM, error) { + if s.unInitVMs.CompareAndSwap(int32(s.maxVMs), int32(s.maxVMs-1)) { + return NewVM(s.vmOpt...), nil + } + + timer := time.NewTimer(s.maxTimeToWaitGetVM) + + defer timer.Stop() + + for i := uint(1); i <= s.maxRetriesGetVM; i++ { + select { + case vm, ok := <-s.vms: + if !ok { + return nil, ErrSchedulerClosed + } + return vm, nil + case <-timer.C: + if s.unInitVMs.Add(-1) >= 0 { + return NewVM(s.vmOpt...), nil + } + s.unInitVMs.Add(1) + slog.Warn(fmt.Sprintf("could not get VM in %v", time.Duration(i)*s.maxTimeToWaitGetVM)) + timer.Reset(s.maxTimeToWaitGetVM) + } + } + return nil, fmt.Errorf("could not get VM in %v", + time.Duration(s.maxRetriesGetVM)*s.maxTimeToWaitGetVM) +} + +func (s *schedulerImpl) Shrink() { + if len(s.vms) == 0 { + return + } + s.unInitVMs.Store(int32(s.maxVMs)) + for i := 0; i <= len(s.vms); i++ { + _ = <-s.vms + } +} + +// Release the VM +func (s *schedulerImpl) release(vm VM) { + if s.closed.Load() { + return + } + + s.vms <- vm +} diff --git a/js/scheduler_test.go b/js/scheduler_test.go new file mode 100644 index 0000000..943d0a9 --- /dev/null +++ b/js/scheduler_test.go @@ -0,0 +1,61 @@ +package js + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestScheduler(t *testing.T) { + scheduler := NewScheduler(SchedulerOptions{InitialVMs: 2, MaxVMs: 4}) + goroutineNum := 12 + blockNum := 4 + wg := new(sync.WaitGroup) + + for i := 1; i <= goroutineNum; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + timeout := time.Millisecond * 400 + script := "1" + if i < blockNum { + script = `while(true){}` + timeout *= 2 + } + + vm, err := scheduler.Get() + if err != nil { + t.Errorf("scheduler %v: %v", i, err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + vm.Run(ctx, func() { + _, err := vm.Runtime().RunString(script) + if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + t.Errorf("run string %v: %v", i, err) + } + }) + }(i) + } + wg.Wait() +} + +func TestSchedulerShrink(t *testing.T) { + scheduler := NewScheduler(SchedulerOptions{InitialVMs: 2, MaxVMs: 4}) + scheduler.Shrink() + assert.Equal(t, `{"available":0,"max":4,"unInit":4}`, scheduler.(fmt.Stringer).String()) + start := time.Now() + _, _ = scheduler.Get() + _, _ = scheduler.Get() + took := time.Since(start) + assert.Equal(t, `{"available":0,"max":4,"unInit":2}`, scheduler.(fmt.Stringer).String()) + assert.True(t, took < time.Millisecond*600) +} diff --git a/js/type.go b/js/type.go deleted file mode 100644 index 648feb3..0000000 --- a/js/type.go +++ /dev/null @@ -1,27 +0,0 @@ -package js - -import ( - "reflect" - "strings" -) - -// FieldNameMapper provides custom mapping between Go and JavaScript property names. -type FieldNameMapper struct{} - -// FieldName returns a JavaScript name for the given struct field in the given type. -// If this method returns "" the field becomes hidden. -func (FieldNameMapper) FieldName(_ reflect.Type, f reflect.StructField) string { - if v, ok := f.Tag.Lookup("js"); ok { - if v == "-" { - return "" - } - return v - } - return strings.ToLower(f.Name[0:1]) + f.Name[1:] -} - -// MethodName returns a JavaScript name for the given method in the given type. -// If this method returns "" the method becomes hidden. -func (FieldNameMapper) MethodName(_ reflect.Type, m reflect.Method) string { - return strings.ToLower(m.Name[0:1]) + m.Name[1:] -} diff --git a/js/utils.go b/js/utils.go index 42a1453..b49764d 100644 --- a/js/utils.go +++ b/js/utils.go @@ -4,18 +4,20 @@ import ( "context" "errors" "fmt" + "log/slog" + "reflect" + "strings" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin/jsmodule" - "github.com/spf13/cast" ) // Throw js exception -func Throw(vm *goja.Runtime, err error) { - if e, ok := err.(*goja.Exception); ok { //nolint:errorlint - panic(e) +func Throw(rt *goja.Runtime, err error) { + var ex *goja.Exception + if errors.As(err, &ex) { //nolint:errorlint + panic(ex) } - panic(vm.ToValue(err)) + panic(rt.ToValue(err)) } // ToBytes tries to return a byte slice from compatible types. @@ -28,25 +30,7 @@ func ToBytes(data any) ([]byte, error) { case goja.ArrayBuffer: return dt.Bytes(), nil default: - return nil, fmt.Errorf("invalid type %T, expected string, []byte or ArrayBuffer", data) - } -} - -// ToStrings tries to return a string slice or string from compatible types. -func ToStrings(data any) (s any, err error) { - switch dt := data.(type) { - case string: - return dt, nil - case []string: - return dt, nil - case []byte: - return string(dt), nil - case []any: - return cast.ToStringSliceE(dt) - case goja.ArrayBuffer: - return string(dt.Bytes()), nil - default: - return nil, fmt.Errorf("invalid type %T, expected string, string array or ArrayBuffer", data) + return nil, fmt.Errorf("expected string, []byte or ArrayBuffer, but got %T, ", data) } } @@ -72,22 +56,83 @@ func Unwrap(value goja.Value) (any, error) { } } -// VMContext returns the current context of the goja.Runtime -func VMContext(runtime *goja.Runtime) context.Context { - if v := runtime.GlobalObject().Get("__ctx__"); v != nil { - if vc, ok := v.Export().(vmctx); ok { - return vc.ctx +// ModuleCallable run the goja.CyclicModuleRecord default export as goja.Callable. +func ModuleCallable(rt *goja.Runtime, resolve goja.HostResolveImportedModuleFunc, module goja.CyclicModuleRecord) (goja.Callable, error) { + instance := rt.GetModuleInstance(module) + if instance == nil { + if err := module.Link(); err != nil { + return nil, err } + promise := rt.CyclicModuleRecordEvaluate(module, resolve) + switch promise.State() { + case goja.PromiseStateRejected: + return nil, promise.Result().Export().(error) + case goja.PromiseStateFulfilled: + default: + } + instance = rt.GetModuleInstance(module) + } + value := instance.GetBindingValue("default") + call, ok := goja.AssertFunction(value) + if !ok { + return nil, errors.New("module default export is not a function") + } + return call, nil +} + +// Context returns the current context of the goja.Runtime +func Context(rt *goja.Runtime) context.Context { + if v := self(rt).ctx.Export().(*vmctx).ctx; v != nil { + return v } return context.Background() } -// InitGlobalModule init all global modules -func InitGlobalModule(runtime *goja.Runtime) { - // Init global modules - for _, extension := range jsmodule.AllModules() { - if mod, ok := extension.Module.(jsmodule.Global); ok { - _ = runtime.Set(extension.Name, mod.Exports()) +// OnDone add a function to execute when the VM has finished running. +// eg: close resources... +func OnDone(rt *goja.Runtime, job func()) { self(rt).eventloop.OnDone(job) } + +// InitGlobalModule init all implement the Global modules +func InitGlobalModule(rt *goja.Runtime) { + for name, mod := range AllModule() { + if mod, ok := mod.(Global); ok { + instance, err := mod.Instantiate(rt) + if err != nil { + slog.Warn(fmt.Sprintf("instantiate global js module %s failed: %s", name, err)) + continue + } + _ = rt.Set(name, instance) + } + } +} + +func FreezeObject(rt *goja.Runtime, obj goja.Value) error { + global := rt.GlobalObject().Get("Object").ToObject(rt) + freeze, ok := goja.AssertFunction(global.Get("freeze")) + if !ok { + panic("failed to get the Object.freeze function from the runtime") + } + _, err := freeze(goja.Undefined(), obj) + return err +} + +// FieldNameMapper provides custom mapping between Go and JavaScript property names. +type FieldNameMapper struct{} + +// FieldName returns a JavaScript name for the given struct field in the given type. +// If this method returns "" the field becomes hidden. +func (FieldNameMapper) FieldName(_ reflect.Type, f reflect.StructField) string { + if v, ok := f.Tag.Lookup("js"); ok { + if v == "-" { + return "" } + return v } + return strings.ToLower(f.Name[0:1]) + f.Name[1:] +} + +// MethodName returns a JavaScript name for the given method in the given type. +// If this method returns "" the method becomes hidden. +func (FieldNameMapper) MethodName(_ reflect.Type, m reflect.Method) string { + return strings.ToLower(m.Name[0:1]) + m.Name[1:] } diff --git a/js/vm.go b/js/vm.go index 14d429a..e8a9209 100644 --- a/js/vm.go +++ b/js/vm.go @@ -3,74 +3,113 @@ package js import ( "bytes" "context" - "errors" "fmt" "log/slog" + "reflect" "runtime/debug" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/plugin" + "github.com/shiroyk/ski" ) -var errInitExecutor = errors.New("initializing JavaScript VM executor failed") - // VM the js runtime. // An instance of VM can only be used by a single goroutine at a time. type VM interface { // RunModule run the goja.CyclicModuleRecord. - // The module default export must be a function. - // To compile the module, goja.ParseModule("name", "module", resolver.ResolveModule) + // To compile the module, goja.ParseModule or ModuleLoader.CompileModule RunModule(ctx context.Context, module goja.CyclicModuleRecord) (goja.Value, error) - // RunString run the script string - RunString(ctx context.Context, src string) (goja.Value, error) + // Run execute the given function in the EventLoop. + // when context done interrupt VM execution and release the VM. + // This is usually used when needs to be called repeatedly many times. + // like this: + // + // func main() { + // scheduler := js.NewScheduler(js.SchedulerOptions{ + // InitialVMs: 2, + // Loader: js.NewModuleLoader(), + // }) + // run := func(ctx context.Context, scheduler js.Scheduler) int64 { + // vm, err := scheduler.Get() + // if err != nil { + // panic(err) + // } + // rt := vm.Runtime() + // + // module, err := scheduler.Loader().CompileModule("sum", "module.exports = (a, b) => a + b") + // if err != nil { + // panic(module) + // } + // + // sum, err := js.ModuleCallable(rt, module) + // if err != nil { + // panic(err) + // } + // + // var total int64 + // vm.Run(ctx, func() { + // for i := 0; i < 8; i++ { + // v, err := sum(goja.Undefined(), rt.ToValue(i), rt.ToValue(total)) + // if err != nil { + // panic(err) + // } + // total = v.ToInteger() + // } + // }) + // + // return total + // } + // + // ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + // defer cancel() + // + // fmt.Println(run(ctx, scheduler)) + // } + Run(context.Context, func()) + // Context return the js context of NewContext + Context() goja.Value + // Loader return the ModuleLoader + Loader() ModuleLoader // Runtime return the js runtime Runtime() *goja.Runtime } +type Option func(*vmImpl) + +// WithInitial call goja.Runtime on VM create, be care require and module not working when init. +func WithInitial(fn func(*goja.Runtime)) Option { + return func(vm *vmImpl) { fn(vm.runtime) } +} + +// WithModuleLoader set a ModuleLoader, if not present require and es module will not work. +func WithModuleLoader(loader ModuleLoader) Option { + return func(vm *vmImpl) { vm.loader = loader } +} + // NewVM creates a new JavaScript VM -// Initialize the EventLoop, require, global module, console. -// If loader.ModuleLoader not declared, use the default loader.NewModuleLoader(). -func NewVM() VM { +// Initialize the EventLoop, global module, console. +func NewVM(opts ...Option) VM { rt := goja.New() rt.SetFieldNameMapper(FieldNameMapper{}) - InitGlobalModule(rt) - mr, err := cloudcat.Resolve[ModuleLoader]() - if err != nil { - slog.Warn(fmt.Sprintf("ModuleLoader not declared, using default")) - mr = NewModuleLoader() - cloudcat.Provide(mr) - } - mr.EnableRequire(rt) - mr.ImportModuleDynamically(rt) EnableConsole(rt) - - eval := `(ctx, code)=>eval(code)` - program := goja.MustCompile("", eval, false) - callable, err := rt.RunProgram(program) - if err != nil { - panic(errInitExecutor) - } - executor, ok := goja.AssertFunction(callable) - if !ok { - panic(errInitExecutor) - } - + InitGlobalModule(rt) vm := &vmImpl{ runtime: rt, - eventloop: NewEventLoop(rt), - executor: executor, - done: make(chan struct{}, 1), - loader: mr, + eventloop: NewEventLoop(), + ctx: NewContext(context.Background(), rt), } - scheduler := cloudcat.ResolveLazy[Scheduler]() - vm.release = func() { - s, err := scheduler() - if err != nil { - return - } - s.Release(vm) + for _, opt := range opts { + opt(vm) + } + if vm.release == nil { + vm.release = func() {} } + if vm.loader == nil { + vm.loader = new(emptyLoader) + } + + vm.loader.EnableRequire(rt).EnableImportModuleDynamically(rt) + _ = rt.GlobalObject().SetSymbol(symbolVM, &vmself{vm}) + return vm } @@ -79,117 +118,150 @@ type ( runtime *goja.Runtime eventloop *EventLoop executor goja.Callable - done chan struct{} + ctx goja.Value release func() loader ModuleLoader } vmctx struct{ ctx context.Context } + + vmself struct{ vm *vmImpl } ) -// RunModule run the goja.CyclicModuleRecord. -// The module default export must be a function. -func (vm *vmImpl) RunModule(ctx context.Context, module goja.CyclicModuleRecord) (goja.Value, error) { - if err := module.Link(); err != nil { - vm.release() - return nil, err - } - promise := vm.runtime.CyclicModuleRecordEvaluate(module, vm.loader.ResolveModule) - switch promise.State() { - case goja.PromiseStateRejected: - vm.release() - return nil, promise.Result().Export().(error) - case goja.PromiseStateFulfilled: - default: - } - value := vm.runtime.GetModuleInstance(module).GetBindingValue("default") - fn, ok := goja.AssertFunction(value) - if !ok { - vm.release() - return value, nil - } +// Loader return the ModuleLoader +func (vm *vmImpl) Loader() ModuleLoader { return vm.loader } - if pc, ok := ctx.(*plugin.Context); ok { - return vm.run(ctx, fn, NewCtxWrapper(vm, pc)) - } - return vm.run(ctx, fn) -} +// Runtime return the js runtime +func (vm *vmImpl) Runtime() *goja.Runtime { return vm.runtime } -// RunString run the script string -func (vm *vmImpl) RunString(ctx context.Context, src string) (goja.Value, error) { - if pc, ok := ctx.(*plugin.Context); ok { - return vm.run(ctx, vm.executor, NewCtxWrapper(vm, pc), vm.runtime.ToValue(src)) - } - return vm.run(ctx, vm.executor, goja.Undefined(), vm.runtime.ToValue(src)) +func (vm *vmImpl) Context() goja.Value { return vm.ctx } + +// RunModule run the goja.CyclicModuleRecord. +// The module default export must be a function. +func (vm *vmImpl) RunModule(ctx context.Context, module goja.CyclicModuleRecord) (ret goja.Value, err error) { + vm.Run(ctx, func() { + var call goja.Callable + call, err = ModuleCallable(vm.runtime, vm.loader.ResolveModule, module) + if err != nil { + return + } + ret, err = call(goja.Undefined(), vm.ctx) + }) + return } -func (vm *vmImpl) run(ctx context.Context, call goja.Callable, args ...goja.Value) (ret goja.Value, err error) { - // resets the interrupt flag. - vm.runtime.ClearInterrupt() +// Run execute the given function in the EventLoop. +// when context done interrupt VM execution and release the VM. +// This is usually used when needs to be called repeatedly many times. +// like this: +// +// func main() { +// scheduler := js.NewScheduler(js.SchedulerOptions{ +// InitialVMs: 2, +// Loader: js.NewModuleLoader(), +// }) +// run := func(ctx context.Context, scheduler js.Scheduler) int64 { +// vm, err := scheduler.Get() +// if err != nil { +// panic(err) +// } +// rt := vm.Runtime() +// +// module, err := scheduler.Loader().CompileModule("sum", "module.exports = (a, b) => a + b") +// if err != nil { +// panic(module) +// } +// +// sum, err := js.ModuleCallable(rt, module) +// if err != nil { +// panic(err) +// } +// +// var total int64 +// vm.Run(ctx, func() { +// for i := 0; i < 8; i++ { +// v, err := sum(goja.Undefined(), rt.ToValue(i), rt.ToValue(total)) +// if err != nil { +// panic(err) +// } +// total = v.ToInteger() +// } +// }) +// +// return total +// } +// +// ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) +// defer cancel() +// +// fmt.Println(run(ctx, scheduler)) +// } +func (vm *vmImpl) Run(ctx context.Context, task func()) { defer func() { - vm.eventloop.WaitOnRegistered() - - if r := recover(); r != nil { + if x := recover(); x != nil { stack := vm.runtime.CaptureCallStack(20, nil) buf := new(bytes.Buffer) for _, frame := range stack { frame.Write(buf) } - slog.Error(fmt.Sprintf("vm run error %s", r), - "stack", string(debug.Stack()), "js stack", buf.String()) + ski.Logger(ctx).Error(fmt.Sprintf("vm run error: %s", x), + slog.String("go_stack", string(debug.Stack())), + slog.String("js_stack", buf.String())) } - - _ = vm.runtime.GlobalObject().Delete("__ctx__") - vm.done <- struct{}{} // End of run + vm.ctx.Export().(*vmctx).ctx = context.Background() + vm.release() }() + // resets the interrupt flag. + vm.runtime.ClearInterrupt() + vm.ctx.Export().(*vmctx).ctx = ctx go func() { select { case <-ctx.Done(): - // Interrupt running JavaScript. + // interrupt the running JavaScript. vm.runtime.Interrupt(ctx.Err()) - // Release vm - vm.release() - return - case <-vm.done: - // Release vm - vm.release() return } }() - _ = vm.runtime.GlobalObject().Set("__ctx__", vmctx{ctx}) - - err = vm.eventloop.Start(func() error { - ret, err = call(goja.Undefined(), args...) - return err - }) - return + vm.eventloop.Start(task) + vm.eventloop.Wait() } -// Runtime return the js runtime -func (vm *vmImpl) Runtime() *goja.Runtime { return vm.runtime } - -// NewPromise returns the new promise with the async function. -// must be called on the EventLoop. +// NewPromise return a goja.Promise object. +// The second argument is a long-running asynchronous task that will be executed in a child goroutine. +// The third optional argument is a callback that will be executed in the main goroutine. +// Additional arguments will be ignored. // like this: // // func main() { +// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// w.WriteHeader(http.StatusOK) +// _, _ = w.Write([]byte(`{"foo":"bar"}`)) +// })) +// defer server.Close() +// // vm := js.NewVM() // ctx, cancel := context.WithTimeout(context.Background(), time.Minute) // defer cancel() // -// goFunc := func(call goja.FunctionCall, rt *goja.Runtime) goja.Value { -// return rt.ToValue(js.NewPromise(rt, func() (any, error) { -// time.Sleep(time.Second) -// return max(call.Argument(0).ToInteger(), call.Argument(1).ToInteger()), nil -// })) +// fetch := func(call goja.FunctionCall, rt *goja.Runtime) goja.Value { +// return rt.ToValue(js.NewPromise(rt, +// func() (*http.Response, error) { return http.Get(call.Argument(0).String()) }, +// func(res *http.Response, err error) (any, error) { +// defer res.Body.Close() +// data, err := io.ReadAll(res.Body) +// if err != nil { +// return nil, err +// } +// return string(data), nil +// })) // } -// _ = vm.Runtime().Set("max", goFunc) +// _ = vm.Runtime().Set("fetch", fetch) // // start := time.Now() // -// result, err := vm.RunString(ctx, `max(1, 2)`) +// result, err := vm.RunString(ctx, fmt.Sprintf(`fetch("%s")`, server.URL)) // if err != nil { // panic(err) // } @@ -199,21 +271,24 @@ func (vm *vmImpl) Runtime() *goja.Runtime { return vm.runtime } // } // // fmt.Println(value) -// fmt.Println(time.Now().Sub(start)) +// fmt.Println(time.Since(start)) // } func NewPromise[T any](runtime *goja.Runtime, async func() (T, error), then ...func(T, error) (any, error)) *goja.Promise { - callback := NewEnqueueCallback(runtime) + enqueue := self(runtime).eventloop.EnqueueJob() promise, resolve, reject := runtime.NewPromise() - thenFun := func(r T, e error) (any, error) { - return r, e - } + thenFun := func(r T, e error) (any, error) { return r, e } if len(then) > 0 { thenFun = then[0] } go func() { + defer func() { + if x := recover(); x != nil { + reject(x) + } + }() result, err := async() - callback(func() error { + enqueue(func() { var value any = result value, err = thenFun(result, err) if err != nil { @@ -221,40 +296,60 @@ func NewPromise[T any](runtime *goja.Runtime, async func() (T, error), then ...f } else { resolve(value) } - return nil }) }() return promise } -// NewEnqueueCallback signals to the event loop that you are going to do some -// asynchronous work off the main thread and that you may need to execute some -// code back on the main thread when you are done. -// see EventLoop.RegisterCallback. -// -// func doAsyncWork(runtime *goja.Runtime) *goja.Promise { -// enqueueCallback := js.NewEnqueueCallback(runtime) -// promise, resolve, reject := runtime.NewPromise() -// -// // Do the actual async work in a new independent goroutine, but make -// // sure that the Promise resolution is done on the main thread: -// -// go func() { -// // Also make sure to abort early if the context is cancelled, so -// // the VM is not stuck when the scenario ends or Ctrl+C is used: -// result, err := doTheActualAsyncWork() -// enqueueCallback(func() error { -// if err != nil { -// reject(err) -// } else { -// resolve(result) -// } -// return nil // do not abort the iteration -// }) -// }() -// return promise -// } -func NewEnqueueCallback(runtime *goja.Runtime) EnqueueCallback { - return runtime.GlobalObject().GetSymbol(enqueueCallbackSymbol).Export().(func() EnqueueCallback)() +var ( + reflectTypeCtx = reflect.TypeOf((*vmctx)(nil)) + reflectTypeVmself = reflect.TypeOf((*vmself)(nil)) + symbolVM = goja.NewSymbol("Symbol.__vm__") +) + +// NewContext create the context object +func NewContext(ctx context.Context, rt *goja.Runtime) *goja.Object { + ret := rt.ToValue(&vmctx{ctx}).ToObject(rt) + proto := rt.NewObject() + _ = ret.SetPrototype(proto) + err := FreezeObject(rt, ret) + if err != nil { + panic(err) + } + + _ = proto.Set("get", func(call goja.FunctionCall) goja.Value { + return rt.ToValue(toCtx(rt, call.This).Value(call.Argument(0).Export())) + }) + _ = proto.Set("set", func(call goja.FunctionCall) goja.Value { + e := toCtx(rt, call.This) + if c, ok := e.(ski.Context); ok { + c.SetValue(call.Argument(0).Export(), call.Argument(1).Export()) + return rt.ToValue(true) + } + return rt.ToValue(false) + }) + _ = proto.Set("toString", func(call goja.FunctionCall) goja.Value { + return rt.ToValue("[context]") + }) + return ret +} + +func toCtx(rt *goja.Runtime, v goja.Value) context.Context { + if v.ExportType() == reflectTypeCtx { + if u := v.Export().(*vmctx); u != nil && u.ctx != nil { + return u.ctx + } + } + panic(rt.NewTypeError(`value of "this" must be of type vmctx`)) +} + +// self get VM self +func self(rt *goja.Runtime) *vmImpl { + value := rt.GlobalObject().GetSymbol(symbolVM) + if value.ExportType() == reflectTypeVmself { + return value.Export().(*vmself).vm + } + panic(rt.NewTypeError(`symbol value of "VM" must be of type vmself, ` + + `this shouldn't happen, maybe not call from VM.Runtime`)) } diff --git a/js/vm_test.go b/js/vm_test.go index 3581982..7cd59ca 100644 --- a/js/vm_test.go +++ b/js/vm_test.go @@ -1,102 +1,55 @@ package js import ( + "bytes" "context" - "errors" - "strconv" + "log/slog" "testing" "time" + _ "unsafe" "github.com/dop251/goja" - "github.com/shiroyk/cloudcat/plugin" + "github.com/shiroyk/ski" "github.com/stretchr/testify/assert" ) -func TestVMRunString(t *testing.T) { +func TestVMContext(t *testing.T) { t.Parallel() - vm := NewTestVM(t) + ctx := context.WithValue(context.Background(), "foo", "bar") + vm := NewVM(WithModuleLoader(NewModuleLoader())) - testCases := []struct { - script string - want any - }{ - {"2", 2}, - {"let a = 1; a + 2", 3}, - {"(() => 4)()", 4}, - {"[5]", []any{int64(5)}}, - {"let a = {'key':'foo'}; a", map[string]any{"key": "foo"}}, - {"JSON.stringify({'key':7})", `{"key":7}`}, - {"JSON.stringify([8])", `[8]`}, - {"(async () => 9)()", 9}, - } - - for _, c := range testCases { - t.Run(c.script, func(t *testing.T) { - v, err := vm.RunString(context.Background(), c.script) - assert.NoError(t, err) - vv, err := Unwrap(v) - assert.NoError(t, err) - assert.EqualValues(t, c.want, vv) - }) + v, err := runMod(ctx, vm, `module.exports = (ctx) => ctx.get('foo')`) + if assert.NoError(t, err) { + assert.Equal(t, "bar", v.Export()) + assert.Equal(t, context.Background(), Context(vm.Runtime())) } } func TestVMRunModule(t *testing.T) { t.Parallel() - resolver := NewModuleLoader() - vm := NewTestVM(t, resolver) - - { - testCases := []struct { - script string - want any - }{ - {"export default () => 1", 1}, - {"export default function () {let a = 1; return a + 1}", 2}, - {"export default async () => 3", 3}, - {"const a = async () => 5; let b = await a(); export default () => b - 1", 4}, - {"export default 3 + 2", 5}, - } + vm := NewVM() - for i, c := range testCases { - module, err := goja.ParseModule(strconv.Itoa(i), c.script, resolver.ResolveModule) - assert.NoError(t, err) - t.Run(c.script, func(t *testing.T) { - v, err := vm.RunModule(context.Background(), module) - assert.NoError(t, err) - vv, err := Unwrap(v) - assert.NoError(t, err) - assert.EqualValues(t, c.want, vv) - }) - } + testCases := []struct { + script string + want any + }{ + {"export default () => 1", 1}, + {"export default function () {let a = 1; return a + 1}", 2}, + {"export default async () => 3", 3}, + {"const a = async () => 5; let b = await a(); export default () => b - 1", 4}, + {"export default () => 3 + 2", 5}, } - { - ctx := plugin.NewContext(plugin.ContextOptions{Values: map[any]any{ - "v1": 1, - "v2": []string{"2"}, - "v3": map[string]any{"key": 3}, - }}) - testCases := []struct { - script string - want any - }{ - {"export default (ctx) => ctx.get('v1')", 1}, - {"export default function (ctx) {return ctx.get('v2')[0]}", "2"}, - {"export default async (ctx) => ctx.get('v3').key", 3}, - {"const a = async () => 5; let b = await a(); export default (ctx) => b - ctx.get('v1')", 4}, - } - for i, c := range testCases { - module, err := goja.ParseModule(strconv.Itoa(i), c.script, resolver.ResolveModule) - assert.NoError(t, err) - t.Run(c.script, func(t *testing.T) { - v, err := vm.RunModule(ctx, module) - assert.NoError(t, err) + for _, c := range testCases { + t.Run(c.script, func(t *testing.T) { + v, err := runMod(context.Background(), vm, c.script) + if assert.NoError(t, err) { vv, err := Unwrap(v) - assert.NoError(t, err) - assert.EqualValues(t, c.want, vv) - }) - } + if assert.NoError(t, err) { + assert.EqualValues(t, c.want, vv) + } + } + }) } } @@ -105,40 +58,27 @@ func TestTimeout(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*200) defer cancel() - _, err := NewTestVM(t).RunString(ctx, `while(true){}`) + start := time.Now() + _, err := runMod(ctx, NewVM(), "export default () => {while(true){}}") + took := time.Since(start) assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Greater(t, time.Millisecond*300, took) } -func TestVMRunWithContext(t *testing.T) { +func TestWithInitial(t *testing.T) { t.Parallel() - { - vm := NewTestVM(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - _ = vm.Runtime().Set("testContext", func(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return vm.ToValue(VMContext(vm)) - }) - v, err := vm.RunString(ctx, "testContext()") - assert.NoError(t, err) - assert.Equal(t, ctx, v.Export()) - assert.Equal(t, context.Background(), VMContext(vm.Runtime())) - } - { - vm := NewTestVM(t) - ctx := plugin.NewContext(plugin.ContextOptions{}) - _ = vm.Runtime().Set("testContext", func(call goja.FunctionCall, vm *goja.Runtime) goja.Value { - return vm.ToValue(VMContext(vm)) - }) - v, err := vm.RunString(ctx, "testContext()") - assert.NoError(t, err) - assert.Equal(t, ctx, v.Export()) - assert.Equal(t, context.Background(), VMContext(vm.Runtime())) + vm := NewVM(WithInitial(func(rt *goja.Runtime) { + _ = rt.Set("init", true) + })) + v, err := runMod(context.Background(), vm, `export default () => init`) + if assert.NoError(t, err) { + assert.Equal(t, true, v.Export()) } } func TestNewPromise(t *testing.T) { t.Parallel() - vm := NewTestVM(t) + vm := NewVM() ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -153,82 +93,53 @@ func TestNewPromise(t *testing.T) { start := time.Now() - result, err := vm.RunString(ctx, `max(1, 2)`) - if err != nil { - assert.NoError(t, err) - } - value, err := Unwrap(result) - if err != nil { - assert.NoError(t, err) + result, err := runMod(ctx, vm, `export default () => max(1, 2)`) + if assert.NoError(t, err) { + value, err := Unwrap(result) + if assert.NoError(t, err) { + assert.EqualValues(t, 2, value) + assert.EqualValues(t, 1, int(time.Now().Sub(start).Seconds())) + } } - assert.EqualValues(t, 2, value) - assert.EqualValues(t, 1, int(time.Now().Sub(start).Seconds())) } -func NewTestVM(t *testing.T, m ...ModuleLoader) VM { - rt := goja.New() - rt.SetFieldNameMapper(FieldNameMapper{}) - InitGlobalModule(rt) - EnableConsole(rt) +type testScheduler struct{ vm VM } - eval := `(ctx, code)=>eval(code)` - program := goja.MustCompile("", eval, false) - callable, err := rt.RunProgram(program) - if err != nil { - panic(errInitExecutor) - } - executor, ok := goja.AssertFunction(callable) - if !ok { - panic(errInitExecutor) - } - var ml ModuleLoader - if len(m) > 0 { - ml = m[0] - } else { - ml = NewModuleLoader() - } - ml.EnableRequire(rt) - ml.ImportModuleDynamically(rt) - - vm := &vmImpl{ - runtime: rt, - eventloop: NewEventLoop(rt), - executor: executor, - done: make(chan struct{}, 1), - loader: ml, - release: func() {}, - } +func (t *testScheduler) release(vm VM) { t.vm = vm } +func (*testScheduler) Get() (VM, error) { return nil, nil } +func (*testScheduler) Close() error { return nil } - assertObject := vm.Runtime().NewObject() - _ = assertObject.Set("equal", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { - a, err := Unwrap(call.Argument(0)) - if err != nil { - Throw(vm, err) - } - b, err := Unwrap(call.Argument(1)) - if err != nil { - Throw(vm, err) - } - var msg string - if !goja.IsUndefined(call.Argument(2)) { - msg = call.Argument(2).String() - } - if !assert.Equal(t, b, a, msg) { - Throw(vm, errors.New("not equal")) - } - return +func TestVMPanic(t *testing.T) { + t.Parallel() + scheduler := new(testScheduler) + vm := NewVM(func(vm *vmImpl) { + vm.release = func() { scheduler.release(vm) } }) - _ = assertObject.Set("true", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) { - var msg string - if !goja.IsUndefined(call.Argument(1)) { - msg = call.Argument(1).String() - } - if !assert.True(t, call.Argument(0).ToBoolean(), msg) { - Throw(vm, errors.New("should be true")) - } - return + + ctx, cancel := context.WithTimeout(ski.NewContext(context.Background(), nil), time.Second) + defer cancel() + + log := new(bytes.Buffer) + + logger := slog.New(slog.NewTextHandler(log, nil)) + + _ = vm.Runtime().Set("some", func() { + OnDone(vm.Runtime(), func() { assert.Equal(t, Context(vm.Runtime()), ctx) }) + OnDone(vm.Runtime(), func() { panic("some panic") }) }) - _ = vm.Runtime().Set("assert", assertObject) + _, err := runMod(ski.WithLogger(ctx, logger), vm, `export default () => {some(); (() => other.error)()}`) + if assert.Error(t, err) { + assert.ErrorContains(t, err, "other is not defined") + assert.NotNil(t, scheduler.vm) + assert.Equal(t, context.Background(), Context(vm.Runtime())) + assert.Contains(t, log.String(), "vm run error: some panic") + } +} - return vm +func runMod(ctx context.Context, vm VM, script string) (goja.Value, error) { + mod, err := vm.Loader().CompileModule("", script) + if err != nil { + return nil, err + } + return vm.RunModule(ctx, mod) } diff --git a/parser.go b/parser.go new file mode 100644 index 0000000..39337c6 --- /dev/null +++ b/parser.go @@ -0,0 +1,69 @@ +package ski + +import ( + "maps" + "sync" +) + +// Parser compile the selector return the Executor +type Parser interface { + // Value get the value of the content with the given argument. + // + // content := `
  • 1
  • 2
` + // e, _ := Value("ul li") + // e.Exec(ctx, content) // "1\n2" + Value(string) (Executor, error) +} + +// ElementParser compile the selector return the Executor +type ElementParser interface { + Parser + + // Element get the element of the content with the given argument. + // + // content := `
  • 1
  • 2
` + // e, _ := Element("ul li") + // e.Exec(ctx, content) // "
  • 1
    • \n
  • 2
    • " + Element(string) (Executor, error) + + // Elements get the elements of the content with the given argument. + // + // content := `
    • 1
    • 2
    ` + // e, _ := Elements("ul li") + // e.Exec(ctx, content) // []string{"
  • 1
    • ", "
  • 2
    • "} + Elements(string) (Executor, error) +} + +// Register registers the Parser with the given key Parser +func Register(key string, parser Parser) { + parsers.Lock() + defer parsers.Unlock() + parsers.registry[key] = parser +} + +// GetParser returns a Parser with the given key +func GetParser(key string) (Parser, bool) { + parsers.RLock() + defer parsers.RUnlock() + parser, ok := parsers.registry[key] + return parser, ok +} + +func RemoveParser(key string) { + parsers.Lock() + defer parsers.Unlock() + delete(parsers.registry, key) +} + +func AllParser() map[string]Parser { + parsers.RLock() + defer parsers.RUnlock() + return maps.Clone(parsers.registry) +} + +var parsers = struct { + sync.RWMutex + registry map[string]Parser +}{ + registry: make(map[string]Parser), +} diff --git a/parsers/gq/bench_gq_test.go b/parsers/gq/bench_gq_test.go index 6bf3d22..e2b74f5 100644 --- a/parsers/gq/bench_gq_test.go +++ b/parsers/gq/bench_gq_test.go @@ -7,7 +7,7 @@ import ( func BenchmarkParser(b *testing.B) { b.StartTimer() for i := 0; i < b.N; i++ { - _, err := gq.GetString(ctx, content, `.body ul a -> parent(li) -> slice(0) -> next(.selected) -> join(-)`) + _, err := gq.Value(`.body ul a -> parent(li) -> slice(0) -> next(.selected) -> join(-)`) if err != nil { b.Fatal(err) } diff --git a/parsers/gq/buildin_function.go b/parsers/gq/buildin_function.go index 219d36c..ae48351 100644 --- a/parsers/gq/buildin_function.go +++ b/parsers/gq/buildin_function.go @@ -1,31 +1,29 @@ package gq import ( + "context" "errors" "fmt" "net/url" "strings" "github.com/PuerkitoBio/goquery" - "github.com/shiroyk/cloudcat/plugin" "github.com/spf13/cast" ) type ( - // GFunc is the type of gq parse function. - GFunc func(ctx *plugin.Context, content any, args ...string) (any, error) + // Func is the type of gq parse function. + Func func(ctx context.Context, content any, args ...string) (any, error) // FuncMap is the type of the map defining the mapping from names to functions. - FuncMap map[string]GFunc + FuncMap map[string]Func ) func builtins() FuncMap { return FuncMap{ - "get": Get, - "set": Set, + "zip": Zip, "attr": Attr, "href": Href, "html": Html, - "join": Join, "prev": Prev, "text": Text, "next": Next, @@ -50,10 +48,14 @@ func contentToString(content any, fn func(*goquery.Selection) (string, error)) ( list[i] = result return true }) - if len(list) == 1 { + switch len(list) { + case 0: + return nil, nil + case 1: return list[0], nil + default: + return list, nil } - return list, nil case string: return c, nil case []string: @@ -61,90 +63,25 @@ func contentToString(content any, fn func(*goquery.Selection) (string, error)) ( return c[0], nil } return c, nil + case nil: + return nil, nil default: return nil, fmt.Errorf("unexpected type %T", content) } } -// Get returns the value associated with this context for key, or nil -// if no value is associated with key. -func Get(ctx *plugin.Context, _ any, args ...string) (any, error) { - if len(args) == 0 { - return nil, fmt.Errorf("get function must has one argment") - } - - key, err := cast.ToStringE(args[0]) - if err != nil { - return nil, err - } - - return ctx.Value(key), nil -} - -// Set value associated with key is val. -// The first argument is the key, and the second argument is value. -// if the value is present will store the content. -func Set(ctx *plugin.Context, content any, args ...string) (any, error) { - if len(args) == 0 { - return nil, fmt.Errorf("set function must has least one argment") - } - - key, err := cast.ToStringE(args[0]) - if err != nil { - return nil, err - } - - if len(args) > 1 { - ctx.SetValue(key, args[1]) - } else { - ctx.SetValue(key, content) - } - - return content, nil -} - // Text gets the combined text contents of each element in the set of matched // elements, including their descendants. -func Text(_ *plugin.Context, content any, _ ...string) (any, error) { +func Text(_ context.Context, content any, _ ...string) (any, error) { return contentToString(content, func(node *goquery.Selection) (string, error) { return strings.TrimSpace(node.Text()), nil }) } -// Join the text with the separator, if not present separator uses the default separator ", ". -func Join(ctx *plugin.Context, content any, args ...string) (any, error) { - if str, ok := content.(string); ok { - return str, nil - } - - if node, ok := content.(*goquery.Selection); ok { - text, err := Text(ctx, node) - if err != nil { - return nil, err - } - if str, ok := text.(string); ok { - return str, nil - } - content = text - } - - list, err := cast.ToStringSliceE(content) - if err != nil { - return nil, err - } - - sep := ", " - if len(args) > 0 { - sep = args[0] - } - - return strings.Join(list, sep), nil -} - // Attr gets the specified attribute's value for the first element in the // Selection. // The first argument is the name of the attribute, the second is the default value -func Attr(_ *plugin.Context, content any, args ...string) (any, error) { +func Attr(_ context.Context, content any, args ...string) (any, error) { if len(args) == 0 { return "", fmt.Errorf("attr(name) must has name") } @@ -161,33 +98,43 @@ func Attr(_ *plugin.Context, content any, args ...string) (any, error) { } // Href gets the href attribute's value, if URL is not absolute returns the absolute URL. -func Href(ctx *plugin.Context, content any, args ...string) (any, error) { +func Href(ctx context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { - var path string href, exists := node.Attr("href") if !exists { return nil, errors.New("href attribute's value is not exist") } - if len(args) > 0 { - path = args[0] - if !strings.HasSuffix(path, "/") { - path = path + "/" - } - href = strings.TrimPrefix(href, "/") - } - hrefURL, err := url.Parse(href) - if err != nil { - return nil, err + if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { + return href, nil } + var base string - baseURL, err := url.Parse(ctx.BaseURL()) - if err != nil { - return nil, err + if v := ctx.Value("baseURL"); v != nil { + base = v.(string) + } else if len(args) > 0 { + base = args[0] + } + if len(base) > 0 { + if !strings.HasPrefix(href, ".") { + if !strings.HasPrefix(href, "/") { + href = "/" + href + } + return strings.TrimSuffix(base, "/") + href, nil + } + hrefURL, err := url.Parse(href) + if err != nil { + return nil, err + } + baseURL, err := url.Parse(base) + if err != nil { + return nil, err + } + return baseURL.ResolveReference(hrefURL).String(), nil } - return baseURL.JoinPath(path).ResolveReference(hrefURL).String(), nil + return href, nil } - return nil, fmt.Errorf("unexpected content type %T", content) + return nil, fmt.Errorf("href: unexpected content type %T", content) } // Html the first argument is outer. @@ -195,7 +142,7 @@ func Href(ctx *plugin.Context, content any, args ...string) (any, error) { // the selection - that is, the HTML including the first element's // tag and attributes, or gets the HTML contents of the first element // in the set of matched elements. It includes text and comment nodes; -func Html(_ *plugin.Context, content any, args ...string) (any, error) { //nolint +func Html(_ context.Context, content any, args ...string) (any, error) { //nolint var err error var outer bool @@ -229,7 +176,7 @@ func Html(_ *plugin.Context, content any, args ...string) (any, error) { //nolin // Selection. // If present selector gets all preceding siblings of each element up to but not // including the element matched by the selector. -func Prev(_ *plugin.Context, content any, args ...string) (any, error) { +func Prev(_ context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { if len(args) > 0 { return node.PrevUntil(args[0]), nil @@ -237,14 +184,14 @@ func Prev(_ *plugin.Context, content any, args ...string) (any, error) { return node.Prev(), nil } - return nil, fmt.Errorf("unexpected content type %T", content) + return nil, fmt.Errorf("prev: unexpected content type %T", content) } // Next gets the immediately following sibling of each element in the // Selection. // If present selector gets all following siblings of each element up to but not // including the element matched by the selector. -func Next(_ *plugin.Context, content any, args ...string) (any, error) { +func Next(_ context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { if len(args) > 0 { return node.NextUntil(args[0]), nil @@ -252,7 +199,7 @@ func Next(_ *plugin.Context, content any, args ...string) (any, error) { return node.Next(), nil } - return nil, fmt.Errorf("unexpected type %T", content) + return nil, fmt.Errorf("next: unexpected type %T", content) } // Slice reduces the set of matched elements to a subset specified by a range @@ -265,7 +212,7 @@ func Next(_ *plugin.Context, content any, args ...string) (any, error) { // // The indices may be negative, in which case they represent an offset from the // end of the selection. -func Slice(_ *plugin.Context, content any, args ...string) (any, error) { +func Slice(_ context.Context, content any, args ...string) (any, error) { if len(args) == 0 { return nil, fmt.Errorf("slice(start, end) must have at least one int argument") } @@ -286,12 +233,12 @@ func Slice(_ *plugin.Context, content any, args ...string) (any, error) { return node.Eq(start), nil } - return nil, fmt.Errorf("unexpected type %T", content) + return nil, fmt.Errorf("slice: unexpected type %T", content) } // Child gets the child elements of each element in the Selection. // If present the selector will return filtered by the specified selector. -func Child(_ *plugin.Context, content any, args ...string) (any, error) { +func Child(_ context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { if len(args) > 0 { return node.ChildrenFiltered(args[0]), nil @@ -299,12 +246,12 @@ func Child(_ *plugin.Context, content any, args ...string) (any, error) { return node.Children(), nil } - return nil, fmt.Errorf("unexpected type %T", content) + return nil, fmt.Errorf("child: unexpected type %T", content) } // Parent gets the parent of each element in the Selection. // if present the selector will return filtered by a selector. -func Parent(_ *plugin.Context, content any, args ...string) (any, error) { +func Parent(_ context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { if len(args) > 0 { return node.ParentFiltered(args[0]), nil @@ -312,12 +259,12 @@ func Parent(_ *plugin.Context, content any, args ...string) (any, error) { return node.Parent(), nil } - return nil, fmt.Errorf("unexpected type %T", content) + return nil, fmt.Errorf("parent: unexpected type %T", content) } // Parents gets the ancestors of each element in the current Selection. // if present the selector will return filtered by a selector. -func Parents(_ *plugin.Context, content any, args ...string) (any, error) { +func Parents(_ context.Context, content any, args ...string) (any, error) { if node, ok := content.(*goquery.Selection); ok { //nolint:nestif if len(args) > 0 { if len(args) > 1 { @@ -334,10 +281,50 @@ func Parents(_ *plugin.Context, content any, args ...string) (any, error) { return node.Parents(), nil } - return nil, fmt.Errorf("unexpected type %T", content) + return nil, fmt.Errorf("parents: unexpected type %T", content) +} + +// Zip returns an element array of first selector element length, +// the first of which contains the first elements of the given selector, +// the second of which contains the second elements of the given selector, and so on. +func Zip(_ context.Context, content any, args ...string) (any, error) { + sel, ok := content.(*goquery.Selection) + if !ok { + return nil, fmt.Errorf("zip: unexpected type %T", content) + } + + if len(args) == 0 { + return nil, fmt.Errorf("zip(selector) must have at least one string argument") + } + + first := sel.Find(args[0]) + length := first.Length() + zip := make([]string, 0, length*len(args)) + first.Each(func(i int, s *goquery.Selection) { + html, _ := goquery.OuterHtml(s) + zip = append(zip, html) + }) + + for _, arg := range args[1:] { + sel.Find(arg).Each(func(i int, s *goquery.Selection) { + html, _ := goquery.OuterHtml(s) + zip = append(zip, html) + }) + } + + ret := make([]string, 0, length) + for i := 0; i < length; i++ { + var s string + for j := 0; j < len(args); j++ { + s += zip[i+j*length] + } + ret = append(ret, s) + } + + return ret, nil } -func Prefix(_ *plugin.Context, content any, args ...string) (ret any, err error) { +func Prefix(_ context.Context, content any, args ...string) (ret any, err error) { if len(args) == 0 { return content, nil } @@ -356,7 +343,7 @@ func Prefix(_ *plugin.Context, content any, args ...string) (ret any, err error) } } -func Suffix(_ *plugin.Context, content any, args ...string) (ret any, err error) { +func Suffix(_ context.Context, content any, args ...string) (ret any, err error) { if len(args) == 0 { return content, nil } diff --git a/parsers/gq/buildin_function_test.go b/parsers/gq/buildin_function_test.go index cd2960f..719d25b 100644 --- a/parsers/gq/buildin_function_test.go +++ b/parsers/gq/buildin_function_test.go @@ -2,194 +2,135 @@ package gq import ( "testing" - - "github.com/stretchr/testify/assert" ) -func TestBuildInFuncGet(t *testing.T) { - t.Parallel() - if _, err := gq.GetString(ctx, content, `-> get`); err == nil { - t.Error("Unexpected function error") - } - - if _, err := gq.GetString(ctx, content, `.body #a1 -> set(key111)`); err != nil { - t.Error(err) - } - - assertGetString(t, `-> get(key111) -> child`, "Google") -} - -func TestBuildInFuncSet(t *testing.T) { - t.Parallel() - if _, err := gq.GetString(ctx, content, `-> set`); err == nil { - t.Fatal("Unexpected function error") - } - - if _, err := gq.GetString(ctx, content, `-> set(v1, 'v1')`); err != nil { - t.Error(err) - } - - if _, err := gq.GetString(ctx, content, `.body #a1 -> text -> set(key222)`); err != nil { - t.Error(err) - } -} - func TestBuildInFuncText(t *testing.T) { t.Parallel() - assertGetString(t, `#main #n1 -> text`, "1") + assertValue(t, `#main #n1 -> text`, "1") - assertGetString(t, `#main #n1`, "1") + assertValue(t, `#main #n1`, "1") } func TestBuildInFuncAttr(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `#main #n1 -> text -> attr`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `#main #n1 -> text -> attr`, "attr(name) must has name") - if _, err := gq.GetString(ctx, content, `-> attr()`); err == nil { - t.Fatal("Unexpected null argument") - } + assertError(t, `#main -> attr()`, "attr(name) must has name") - assertGetString(t, `#main #n1 -> attr(class)`, "one even row") + assertValue(t, `#main #n1 -> attr(class)`, "one even row") - assertGetString(t, `#main #n1 -> attr(empty, default)`, "default") -} - -func TestBuildInFuncJoin(t *testing.T) { - t.Parallel() - assertGetString(t, `#main div -> join(' < ')`, "1 < 2 < 3 < 4 < 5 < 6") - - assertGetString(t, `#main div -> join("")`, "123456") - - assertGetString(t, `#main div -> join('')`, "123456") + assertValue(t, `#main #n1 -> attr(empty, default)`, "default") } func TestBuildInFuncHref(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `.body ul #a4 -> text -> href`); err == nil { - t.Fatal("Unexpected function error") - } - - assertGetString(t, `.body ul #a4 a -> href`, "https://localhost/home") - - assertGetString(t, `.body ul #a4 a -> href(path)`, "https://localhost/path/home") + assertError(t, `.body ul #a4 -> text -> href`, "unexpected content type string") - assertGetString(t, `.body ul #a4 a -> href(path/)`, "https://localhost/path/home") + assertValue(t, `.body ul #a4 a -> href(https://localhost)`, "https://localhost/home") - assertGetString(t, `.body ul #a4 a -> href(/path/)`, "https://localhost/path/home") - - _, err := gq.GetString(ctx, content, `#main #n1 -> href`) - assert.Error(t, err) + assertValue(t, `.body ul #a4 a -> href(https://localhost/path/)`, "https://localhost/path/home") } func TestBuildInFuncHtml(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `-> html(test)`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `.body -> html(test)`, "html(outer) `outer` must bool type value: true/false") - assertGetString(t, `.body ul a -> html`, "Google\nGithub\nGolang\nHome") + assertValue(t, `.body ul a -> html`, []string{"Google", "Github", "Golang", "Home"}) - assertGetString(t, `.body ul a -> slice(0) -> html(true)`, - `Google`) + assertValue(t, `.body ul a -> slice(0,2) -> html(true)`, + []string{ + "Google", + "Github"}) } func TestBuildInFuncPrev(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `#foot #nf3 -> text -> prev`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `#foot #nf3 -> text -> prev`, "unexpected content type string") - assertGetString(t, `#foot #nf3 -> prev`, "f2") + assertValue(t, `#foot #nf3 -> prev`, "f2") - assertGetString(t, `#foot #nf3 -> prev(#nf1)`, "f2") + assertValue(t, `#foot #nf3 -> prev(#nf1)`, "f2") } func TestBuildInFuncNext(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `#foot #nf2 -> text -> next`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `#foot #nf2 -> text -> next`, "unexpected type string") - assertGetString(t, `#foot #nf2 -> next`, "f3") + assertValue(t, `#foot #nf2 -> next`, "f3") - assertGetString(t, `#foot #nf2 -> next(#nf4)`, "f3") + assertValue(t, `#foot #nf2 -> next(#nf4)`, "f3") } func TestBuildInFuncSlice(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `-> slice`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `#main -> slice`, "slice(start, end) must have at least one int argument") - if _, err := gq.GetString(ctx, content, `#main div -> text -> slice(0)`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `#main div -> text -> slice(0)`, "slice: unexpected type []string") - assertGetString(t, `#main div -> slice(0)`, "1") + assertValue(t, `#main div -> slice(0)`, "1") - assertGetString(t, `#main div -> slice(-1)`, "6") + assertValue(t, `#main div -> slice(-1)`, "6") - assertGetString(t, `#main div -> slice(0, 3)`, "1\n2\n3") + assertValue(t, `#main div -> slice(0, 3)`, []string{"1", "2", "3"}) - assertGetString(t, `#main div -> slice(0, -2)`, "1\n2\n3\n4") + assertValue(t, `#main div -> slice(0, -2)`, []string{"1", "2", "3", "4"}) } func TestBuildInFuncChild(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `.body ul -> text -> child`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `.body ul -> text -> child`, "unexpected type string") - assertGetString(t, `.body ul li -> child(a)`, "Google\nGithub\nGolang\nHome") + assertValue(t, `.body ul li -> child(a)`, []string{"Google", "Github", "Golang", "Home"}) - assertGetString(t, `.body ul li -> child`, "Google\nGithub\nGolang\nHome") + assertValue(t, `.body ul li -> child`, []string{"Google", "Github", "Golang", "Home"}) } func TestBuildInFuncParent(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `.body ul -> text -> parent`); err == nil { - t.Fatal("Unexpected function error") - } + assertError(t, `.body ul -> text -> parent`, "unexpected type string") - assertGetString(t, `.body ul a -> parent(#a1) -> attr(id)`, "a1") + assertValue(t, `.body ul a -> parent(#a1) -> attr(id)`, "a1") - assertGetString(t, `.body ul a -> parent -> attr(id)`, "a1\na2\na3\na4") + assertValue(t, `.body ul a -> parent -> attr(id)`, []string{"a1", "a2", "a3", "a4"}) } func TestBuildInFuncParents(t *testing.T) { t.Parallel() - if _, err := gq.GetString(ctx, content, `.body ul -> text -> parents`); err == nil { - t.Fatal("Unexpected type") - } + assertError(t, `.body ul -> text -> parents`, "unexpected type string") - if _, err := gq.GetString(ctx, content, `.body ul .selected -> parents(div, test)`); err == nil { - t.Fatal("Unexpected argument") - } + assertError(t, `.body ul .selected -> parents(div, test)`, "parents(selector, until) `until` must bool type value: true/false") - assertGetString(t, `.body ul .selected -> parents(div, true) -> attr(id)`, "url") + assertValue(t, `.body ul .selected -> parents(div, true) -> attr(id)`, "url") - assertGetString(t, `.body ul .selected -> parents -> slice(0) -> attr(id)`, "url") + assertValue(t, `.body ul .selected -> parents -> slice(0) -> attr(id)`, "url") } func TestBuildInFuncPrefix(t *testing.T) { t.Parallel() - assertGetString(t, `#main #n1 -> text -> prefix(A)`, "A1") - - assertGetString(t, `#main #n1 -> prefix(B)`, "B1") + assertValue(t, `#main #n1 -> text -> prefix(A)`, "A1") - assertGetStrings(t, `#main div -> slice(0, 2) -> text -> prefix(-)`, []string{"-1", "-2"}) + assertValue(t, `#main #n1 -> prefix(B)`, "B1") } func TestBuildInFuncSuffix(t *testing.T) { t.Parallel() - assertGetString(t, `#main #n1 -> text -> suffix(A)`, "1A") + assertValue(t, `#main #n1 -> text -> suffix(A)`, "1A") - assertGetString(t, `#main #n1 -> suffix(B)`, "1B") + assertValue(t, `#main #n1 -> suffix(B)`, "1B") +} + +func TestBuildInZip(t *testing.T) { + t.Parallel() - assertGetStrings(t, `.body a -> slice(0, 2) -> text -> suffix(.com)`, []string{"Google.com", "Github.com"}) + assertElements(t, `-> zip('#main div', '#foot div')`, []string{ + `
    1
    f1
    `, + `
    2
    f2
    `, + `
    3
    f3
    `, + `
    4
    f4
    `, + `
    5
    f5
    `, + `
    6
    f6
    `, + }) } diff --git a/parsers/gq/gq.go b/parsers/gq/gq.go index b80823d..17d8e0c 100644 --- a/parsers/gq/gq.go +++ b/parsers/gq/gq.go @@ -2,189 +2,199 @@ package gq import ( + "context" + "fmt" + "maps" "strings" "github.com/PuerkitoBio/goquery" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/spf13/cast" + "github.com/andybalholm/cascadia" + "github.com/shiroyk/ski" + "golang.org/x/net/html" ) -// Key the gq parser register key. -const Key string = "gq" +// parser the goquery parser +type parser struct{ funcs FuncMap } -// Parser the goquery parser -type Parser struct { - parseFuncs FuncMap +// NewParser creates a new goquery parser with the given FuncMap. +func NewParser(m FuncMap) ski.ElementParser { + funcs := maps.Clone(builtins()) + maps.Copy(funcs, m) + return &parser{funcs} } -// NewParser creates a new goquery Parser with the given FuncMap. -func NewParser(funcs FuncMap) parser.Parser { - p := &Parser{parseFuncs: builtins()} - for k, v := range funcs { - p.parseFuncs[k] = v - } - return p -} - -// init register the goquery Parser with default builtins funcs. func init() { - parser.Register(Key, &Parser{parseFuncs: builtins()}) + ski.Register("gq", NewParser(nil)) } -// GetString gets the string of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetString(ctx, content, "ul li") returns "1\n2" -func (p *Parser) GetString(ctx *plugin.Context, content any, arg string) (ret string, err error) { - nodes, err := getSelection(content) +func (p *parser) Value(arg string) (ski.Executor, error) { + ret, err := p.compile(arg) if err != nil { - return + return nil, err } + ret.calls = append(ret.calls, call{fn: value}) + return ret, nil +} - rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg) +func (p *parser) Element(arg string) (ski.Executor, error) { + ret, err := p.compile(arg) if err != nil { - return - } - - var node any = nodes.Find(rule) - - for _, fun := range funcs { - node, err = p.parseFuncs[fun.name](ctx, node, fun.args...) - if err != nil || node == nil { - return ret, err - } + return nil, err } + ret.calls = append(ret.calls, call{fn: element}) + return ret, nil +} - node, err = Join(ctx, node, "\n") +func (p *parser) Elements(arg string) (ski.Executor, error) { + ret, err := p.compile(arg) if err != nil { - return + return nil, err } - - return node.(string), nil + ret.calls = append(ret.calls, call{fn: elements}) + return ret, nil } -// GetStrings gets the strings of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetStrings(ctx, content, "ul li") returns []string{"1", "2"} -func (p *Parser) GetStrings(ctx *plugin.Context, content any, arg string) (ret []string, err error) { - nodes, err := getSelection(content) - if err != nil { +func (p *parser) compile(raw string) (ret matcher, err error) { + funcs := strings.Split(raw, "->") + if len(funcs) == 1 { + ret.Matcher, err = cascadia.Compile(funcs[0]) return } - - rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg) - if err != nil { - return + selector := strings.TrimSpace(funcs[0]) + if len(selector) == 0 { + ret.Matcher = new(emptyMatcher) + } else { + ret.Matcher, err = cascadia.Compile(selector) + if err != nil { + return + } } - var node any = nodes.Find(rule) + ret.calls = make([]call, 0, len(funcs)-1) - for _, fun := range funcs { - node, err = p.parseFuncs[fun.name](ctx, node, fun.args...) - if err != nil || node == nil { - return nil, err + for _, function := range funcs[1:] { + function = strings.TrimSpace(function) + if function == "" { + continue + } + name, args, err := parseFuncArguments(function) + if err != nil { + return ret, err } + fn, ok := p.funcs[name] + if !ok { + return ret, fmt.Errorf("function %s not exists", name) + } + ret.calls = append(ret.calls, call{fn, args}) } - if sel, ok := node.(*goquery.Selection); ok { - str := make([]string, sel.Length()) - sel.EachWithBreak(func(i int, sel *goquery.Selection) bool { - str[i] = strings.TrimSpace(sel.Text()) - return true - }) - return str, nil - } - return cast.ToStringSliceE(node) + return } -// GetElement gets the element of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetElement(ctx, content, "ul li") returns "
  • 1
    • \n
  • 2
    • " -func (p *Parser) GetElement(ctx *plugin.Context, content any, arg string) (ret string, err error) { - nodes, err := getSelection(content) - if err != nil { - return - } +type call struct { + fn Func + args []string +} + +type matcher struct { + goquery.Matcher + calls []call +} - rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg) +func (f matcher) Exec(ctx context.Context, arg any) (any, error) { + nodes, err := selection(arg) if err != nil { - return + return nil, err } - var node any = nodes.Find(rule) + var node any = nodes.FindMatcher(f) - for _, fun := range funcs { - node, err = p.parseFuncs[fun.name](ctx, node, fun.args...) + for _, c := range f.calls { + node, err = c.fn(ctx, node, c.args...) if err != nil || node == nil { - return ret, err + return nil, err } } - if sel, ok := node.(*goquery.Selection); ok { - return goquery.OuterHtml(sel) - } - - return cast.ToStringE(node) + return node, nil } -// GetElements gets the elements of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetElements(ctx, content, "ul li") returns []string{"
  • 1
    • ", "
  • 2
    • "} -func (p *Parser) GetElements(ctx *plugin.Context, content any, arg string) (ret []string, err error) { - nodes, err := getSelection(content) - if err != nil { - return - } - - rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg) - if err != nil { - return +func value(ctx context.Context, node any, _ ...string) (any, error) { + v, err := Text(ctx, node) + if node == nil || err != nil { + return nil, err } + return v, nil +} - var node any = nodes.Find(rule) - - for _, fun := range funcs { - node, err = p.parseFuncs[fun.name](ctx, node, fun.args...) - if err != nil || node == nil { - return nil, err +func element(_ context.Context, node any, _ ...string) (any, error) { + switch t := node.(type) { + default: + return nil, fmt.Errorf("unexpected type %T", node) + case string, []string, *html.Node, nil: + return t, nil + case *goquery.Selection: + if len(t.Nodes) == 0 { + return nil, nil + } + return t.Nodes[0], nil + case []*html.Node: + if len(t) == 0 { + return nil, nil } + return t[0], nil } +} - if sel, ok := node.(*goquery.Selection); ok { - objs := make([]string, sel.Length()) - sel.EachWithBreak(func(i int, sel *goquery.Selection) bool { - if objs[i], err = goquery.OuterHtml(sel); err != nil { - return false - } - return true - }) - if err != nil { - return +func elements(_ context.Context, node any, _ ...string) (any, error) { + switch t := node.(type) { + default: + return nil, fmt.Errorf("unexpected type %T", node) + case string, []string, *html.Node, nil: + return t, nil + case *goquery.Selection: + ele := make([]any, t.Length()) + for i, n := range t.Nodes { + ele[i] = n } - return objs, nil + return ele, nil + case []*html.Node: + ele := make([]any, len(t)) + for i, n := range t { + ele[i] = n + } + return ele, nil } - return cast.ToStringSliceE(node) } -// getSelection converts content to goquery.Selection -func getSelection(content any) (*goquery.Selection, error) { +// selection converts content to goquery.Selection +func selection(content any) (*goquery.Selection, error) { switch data := content.(type) { default: - str, err := cast.ToStringE(content) - if err != nil { - return nil, err + return nil, fmt.Errorf("unexpected type %T", content) + case nil: + return new(goquery.Selection), nil + case *html.Node: + return goquery.NewDocumentFromNode(data).Selection, nil + case []any: + if len(data) == 0 { + return nil, nil } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(str)) - if err != nil { - return nil, err + root := &html.Node{Type: html.DocumentNode} + doc := goquery.NewDocumentFromNode(root) + doc.Selection.Nodes = make([]*html.Node, len(data)) + for i, v := range data { + n, ok := v.(*html.Node) + if !ok { + return nil, fmt.Errorf("expected type *html.Node, but got %T", v) + } + n.Parent = nil + n.PrevSibling = nil + n.NextSibling = nil + root.AppendChild(n) + doc.Selection.Nodes[i] = n } return doc.Selection, nil - case nil: - return new(goquery.Selection), nil case []string: doc, err := goquery.NewDocumentFromReader(strings.NewReader(strings.Join(data, "\n"))) if err != nil { @@ -199,3 +209,11 @@ func getSelection(content any) (*goquery.Selection, error) { return doc.Selection, nil } } + +type emptyMatcher struct{} + +func (emptyMatcher) Match(*html.Node) bool { return true } + +func (emptyMatcher) MatchAll(node *html.Node) []*html.Node { return []*html.Node{node} } + +func (emptyMatcher) Filter(nodes []*html.Node) []*html.Node { return nodes } diff --git a/parsers/gq/gq_test.go b/parsers/gq/gq_test.go index 58dfeac..7585cbb 100644 --- a/parsers/gq/gq_test.go +++ b/parsers/gq/gq_test.go @@ -1,21 +1,20 @@ package gq import ( - "flag" + "bytes" + "context" "fmt" - "os" "testing" "log/slog" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" "github.com/stretchr/testify/assert" + "golang.org/x/net/html" ) var ( - gq Parser - ctx *plugin.Context + gq = parser{funcs: builtins()} + ctx = context.Background() content = ` @@ -58,134 +57,122 @@ var ( ` ) -func TestMain(m *testing.M) { - flag.Parse() - ctx = plugin.NewContext(plugin.ContextOptions{ - URL: "https://localhost", - }) - gq = Parser{parseFuncs: builtins()} - code := m.Run() - os.Exit(code) -} - -func assertGetString(t *testing.T, arg string, expected string) { - str, err := gq.GetString(ctx, content, arg) - if err != nil { - t.Error(err) - } - - assert.Equal(t, expected, str) -} - -func assertGetStrings(t *testing.T, arg string, expected []string) { - str, err := gq.GetStrings(ctx, content, arg) - if err != nil { - t.Fatal(err) +func assertError(t *testing.T, arg string, contains string) { + executor, err := gq.Value(arg) + if assert.NoError(t, err) { + _, err = executor.Exec(ctx, content) + assert.ErrorContains(t, err, contains) } - - assert.Equal(t, expected, str) } -func assertGetElement(t *testing.T, arg string, expected string) { - ele, err := gq.GetElement(ctx, content, arg) - if err != nil { - t.Fatal(err) +func assertValue(t *testing.T, arg string, expected any) { + executor, err := gq.Value(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + assert.Equal(t, expected, v) + } } - - assert.Equal(t, expected, ele) } -func assertGetElements(t *testing.T, arg string, expected []string) { - objs, err := gq.GetElements(ctx, content, arg) - if err != nil { - t.Fatal(err) +func assertElement(t *testing.T, arg string, expected string) { + executor, err := gq.Element(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + switch c := v.(type) { + case *html.Node: + b := new(bytes.Buffer) + if assert.NoError(t, html.Render(b, c)) { + assert.Equal(t, expected, b.String()) + } + default: + assert.Equal(t, expected, v) + } + } } - - assert.Equal(t, expected, objs) } -func TestParser(t *testing.T) { - t.Parallel() - if _, ok := parser.GetParser(Key); !ok { - t.Fatal("schema not registered") +func assertElements(t *testing.T, arg string, expected []string) { + executor, err := gq.Elements(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + switch c := v.(type) { + case []any: + ele := make([]string, len(c)) + for i, v := range c { + var b bytes.Buffer + if assert.NoError(t, html.Render(&b, v.(*html.Node))) { + ele[i] = b.String() + } + } + assert.Equal(t, expected, ele) + default: + assert.Equal(t, expected, v) + } + } } - - _, err := gq.GetString(ctx, 0x11, `0x11`) - assert.NoError(t, err) - - _, err = gq.GetString(ctx, nil, ``) - assert.NoError(t, err) - - _, err = gq.GetString(ctx, []string{"
    "}, ``) - assert.NoError(t, err) - - _, err = gq.GetString(ctx, `Golang`, ``) - assert.NoError(t, err) - - sel, _ := gq.GetElement(ctx, content, `#main .row`) - _, err = gq.GetString(ctx, sel, ``) - assert.NoError(t, err) } -func TestGetString(t *testing.T) { +func TestValue(t *testing.T) { t.Parallel() - assertGetString(t, `#main .row -> text`, "1\n2\n3\n4\n5\n6") - - assertGetString(t, `.body ul a -> parent(li) -> attr(id) -> join(-)`, "a1-a2-a3-a4") + assertValue(t, `#main .row -> text`, []string{"1", "2", "3", "4", "5", "6"}) - assertGetString(t, `script -> slice(0) -> attr(type)`, "text/javascript") -} - -func TestGetStrings(t *testing.T) { - t.Parallel() - assertGetStrings(t, `.body ul li -> child(a) -> attr(title)`, []string{"Google page", "Github page", "Golang page", "Home page"}) + assertValue(t, `.body ul a -> parent(li) -> attr(id)`, []string{"a1", "a2", "a3", "a4"}) - assertGetStrings(t, `.body ul a`, []string{"Google", "Github", "Golang", "Home"}) + assertValue(t, `script -> slice(0) -> attr(type)`, "text/javascript") } -func TestGetElement(t *testing.T) { +func TestElement(t *testing.T) { t.Parallel() - assertGetElement(t, `.body ul a -> parents(li)`, `
  • Google
    • `) + assertElement(t, `.body ul a -> parents(li)`, `
  • Google
    • `) - assertGetElement(t, `.body ul a -> slice(1) -> text`, `Github`) + assertElement(t, `.body ul a -> slice(1) -> text`, `Github`) } -func TestGetElements(t *testing.T) { +func TestElements(t *testing.T) { t.Parallel() - assertGetElements(t, `#foot div -> slice(0, 3)`, []string{ + assertElements(t, `#foot div -> slice(0, 3)`, []string{ `
    f1
    `, `
    f2
    `, `
    f3
    `, }) - assertGetElements(t, `#foot div -> slice(0, 3) -> text`, []string{"f1", "f2", "f3"}) + assertElements(t, `#foot div -> slice(0, 3) -> text`, []string{"f1", "f2", "f3"}) } func TestExternalFunc(t *testing.T) { { - fun := func(logger *slog.Logger) GFunc { - return func(_ *plugin.Context, content any, args ...string) (any, error) { + fun := func(logger *slog.Logger) Func { + return func(_ context.Context, content any, args ...string) (any, error) { logger.Info(fmt.Sprintf("result type was %T", content)) return content, nil } } - p := NewParser(FuncMap{"logger": fun(slog.Default())}) - _, err := p.GetString(ctx, content, ".body ul a -> logger -> text") - assert.NoError(t, err) + data := new(bytes.Buffer) + p := NewParser(FuncMap{"logger": fun(slog.New(slog.NewTextHandler(data, nil)))}) + executor, err := p.Value(".body ul a -> logger -> text") + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + assert.Equal(t, []string{"Google", "Github", "Golang", "Home"}, v) + } + } + assert.Contains(t, data.String(), `result type was *goquery.Selection`) } { - fun := func(_ *plugin.Context, content any, args ...string) (any, error) { + fun := func(_ context.Context, content any, args ...string) (any, error) { return nil, nil } p := NewParser(FuncMap{"nil": fun}) - _, err := p.GetString(ctx, content, ".body ul a -> nil -> text") - assert.NoError(t, err) - _, err = p.GetStrings(ctx, content, ".body ul a -> nil -> text") - assert.NoError(t, err) - _, err = p.GetElement(ctx, content, ".body ul a -> nil -> text") - assert.NoError(t, err) - _, err = p.GetElements(ctx, content, ".body ul a -> nil -> text") - assert.NoError(t, err) + executor, err := p.Value(".body ul a -> nil -> text") + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + assert.Equal(t, nil, v) + } + } } } diff --git a/parsers/gq/tokenizer.go b/parsers/gq/tokenizer.go index fc40295..03e6dd6 100644 --- a/parsers/gq/tokenizer.go +++ b/parsers/gq/tokenizer.go @@ -13,50 +13,19 @@ const ( doubleQuoteState ) -type ruleFunc struct { - name string - args []string -} - -func parseRuleFunctions(funcMap FuncMap, ruleStr string) (rule string, funcs []ruleFunc, err error) { - ruleFuncs := strings.Split(ruleStr, "->") - if len(ruleFuncs) == 1 { - return ruleFuncs[0], funcs, nil - } - rule = strings.TrimSpace(ruleFuncs[0]) - - for _, function := range ruleFuncs[1:] { - function = strings.TrimSpace(function) - if function == "" { - continue - } - fn, err := parseFuncArguments(function) - if err != nil { - return "", nil, err - } - if _, ok := funcMap[fn.name]; !ok { - return "", nil, fmt.Errorf("function %s not exists", fn.name) - } - funcs = append(funcs, fn) - } - - return -} - -func parseFuncArguments(s string) (ret ruleFunc, err error) { +func parseFuncArguments(s string) (name string, args []string, err error) { openBracket := strings.IndexByte(s, '(') closeBracket := strings.LastIndexByte(s, ')') if openBracket == -1 { - return ruleFunc{name: s}, nil + return s, nil, nil } if closeBracket == -1 { - return ret, fmt.Errorf("unexpected function %s not close bracket", s) + return name, nil, fmt.Errorf("unexpected function %s not close bracket", s) } - funcName := s[0:openBracket] - args := make([]string, 0) + name = s[0:openBracket] arg := strings.Builder{} state := commonState offset := openBracket + 1 @@ -107,7 +76,7 @@ func parseFuncArguments(s string) (ret ruleFunc, err error) { } if state == singleQuoteState || state == doubleQuoteState { - return ret, fmt.Errorf("unexpected function %s argument quote not closed", s) + return name, nil, fmt.Errorf("unexpected function %s argument quote not closed", s) } if arg.Cap() > 0 { @@ -115,8 +84,5 @@ func parseFuncArguments(s string) (ret ruleFunc, err error) { arg.Reset() } - return ruleFunc{ - name: funcName, - args: args, - }, nil + return } diff --git a/parsers/gq/tokenizer_test.go b/parsers/gq/tokenizer_test.go index 454c2d8..ddb5211 100644 --- a/parsers/gq/tokenizer_test.go +++ b/parsers/gq/tokenizer_test.go @@ -4,16 +4,15 @@ import ( "testing" ) -func TestParseRuleFunction(t *testing.T) { +func TestParseFuncArguments(t *testing.T) { t.Parallel() rules := []string{ - `-> -> unknown`, `-> text(`, `-> text(")`, + `-> text(`, `-> text(")`, `-> text("')`, `-> text('")`, `-> text(' ", ")`, `-> text("\")`, `-> text('\')`, `-> text(" ", ')`, } - funcs := builtins() for _, rule := range rules { - if _, _, err := parseRuleFunctions(funcs, rule); err == nil { + if _, _, err := parseFuncArguments(rule); err == nil { t.Fatalf("Unexpected function and argument parse %s", rule) } } diff --git a/parsers/json/README.md b/parsers/jq/README.md similarity index 100% rename from parsers/json/README.md rename to parsers/jq/README.md diff --git a/parsers/jq/jq.go b/parsers/jq/jq.go new file mode 100644 index 0000000..d81c223 --- /dev/null +++ b/parsers/jq/jq.go @@ -0,0 +1,63 @@ +// Package jq the json path parser +package jq + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ohler55/ojg/jp" + "github.com/ohler55/ojg/oj" + "github.com/shiroyk/ski" +) + +// Parser the json path parser +type Parser struct{} + +func init() { + ski.Register("jq", new(Parser)) +} + +func (p Parser) Value(arg string) (ski.Executor, error) { + x, err := jp.ParseString(arg) + if err != nil { + return nil, err + } + return expr{x, x.Normal()}, nil +} + +type expr struct { + jp.Expr + normal bool +} + +func (e expr) Exec(_ context.Context, arg any) (any, error) { + obj, err := doc(arg) + if err != nil { + return nil, err + } + if e.normal { + return e.First(obj), nil + } + return e.Get(obj), nil +} + +func doc(content any) (any, error) { + switch data := content.(type) { + default: + return content, nil + case fmt.Stringer: + return oj.ParseString(data.String()) + case json.RawMessage: + return oj.Parse(data) + case []byte: + return oj.Parse(data) + case []string: + if len(data) == 0 { + return nil, nil + } + return oj.ParseString(data[0]) + case string: + return oj.ParseString(data) + } +} diff --git a/parsers/jq/jq_test.go b/parsers/jq/jq_test.go new file mode 100644 index 0000000..65f4eb6 --- /dev/null +++ b/parsers/jq/jq_test.go @@ -0,0 +1,67 @@ +package jq + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + jq Parser + content = ` +{ + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + }, + "expensive": 10 +}` +) + +func assertValue(t *testing.T, arg string, expected any) { + executor, err := jq.Value(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(context.Background(), content) + if assert.NoError(t, err) { + assert.Equal(t, expected, v) + } + } +} + +func TestGetString(t *testing.T) { + t.Parallel() + assertValue(t, `$.store.book[-1].price`, 22.99) + assertValue(t, `$.store.book[*].author`, []any{"Nigel Rees", "Evelyn Waugh", "Herman Melville", "J. R. R. Tolkien"}) + assertValue(t, `$.store.book[?(@.price < 10)].isbn`, []any{`0-553-21311-3`}) +} diff --git a/parsers/js/README.md b/parsers/js/README.md deleted file mode 100644 index b688aa7..0000000 --- a/parsers/js/README.md +++ /dev/null @@ -1,3 +0,0 @@ - -## References -- [goja](https://github.com/dop251/goja) \ No newline at end of file diff --git a/parsers/js/esm.go b/parsers/js/esm.go deleted file mode 100644 index 9e90de4..0000000 --- a/parsers/js/esm.go +++ /dev/null @@ -1,104 +0,0 @@ -// Package js the js parser -package js - -import ( - "hash/maphash" - "sync" - - "github.com/dop251/goja" - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/parsers/js/lru" - "github.com/shiroyk/cloudcat/plugin" -) - -// ESMParser the js parser with es module -type ESMParser struct { - mu *sync.Mutex - cache *lru.Cache[uint64, goja.CyclicModuleRecord] - hash *maphash.Hash - load func() js.ModuleLoader -} - -// NewESMParser returns a new ESMParser -func NewESMParser(maxCache int) *ESMParser { - return &ESMParser{ - new(sync.Mutex), - lru.New[uint64, goja.CyclicModuleRecord](maxCache), - new(maphash.Hash), - cloudcat.MustResolveLazy[js.ModuleLoader](), - } -} - -// GetString gets the string of the content with the given arguments. -// returns the string result. -func (p *ESMParser) GetString(ctx *plugin.Context, content any, arg string) (ret string, err error) { - v, err := p.run(ctx, content, arg) - if err != nil { - return "", err - } - return toString(v) -} - -// GetStrings gets the strings of the content with the given arguments. -// returns the slice of string result. -func (p *ESMParser) GetStrings(ctx *plugin.Context, content any, arg string) (ret []string, err error) { - v, err := p.run(ctx, content, arg) - if err != nil { - return nil, err - } - return toStrings(v) -} - -// GetElement gets the element of the content with the given arguments. -// returns the string result. -func (p *ESMParser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return p.GetString(ctx, content, arg) -} - -// GetElements gets the elements of the content with the given arguments. -// returns the slice of string result. -func (p *ESMParser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return p.GetStrings(ctx, content, arg) -} - -// ClearCache clear the module cache -func (p *ESMParser) ClearCache() { - p.mu.Lock() - defer p.mu.Unlock() - p.cache.Clear() -} - -// LenCache size the module cache -func (p *ESMParser) LenCache() int { - p.mu.Lock() - defer p.mu.Unlock() - return p.cache.Len() -} - -func (p *ESMParser) run(ctx *plugin.Context, content any, script string) (any, error) { - ctx.SetValue("content", content) - - p.mu.Lock() - defer p.mu.Unlock() - _, _ = p.hash.WriteString(script) - hash := p.hash.Sum64() - p.hash.Reset() - - mod, ok := p.cache.Get(hash) - if !ok { - var err error - mod, err = goja.ParseModule("", script, p.load().ResolveModule) - if err != nil { - return nil, err - } - p.cache.Add(hash, mod) - } - - result, err := js.RunModule(ctx, mod) - if err != nil { - return nil, err - } - - return js.Unwrap(result) -} diff --git a/parsers/js/esm_test.go b/parsers/js/esm_test.go deleted file mode 100644 index 1f85c94..0000000 --- a/parsers/js/esm_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package js - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -var esmParser = NewESMParser(1) - -func TestESMCache(t *testing.T) { - _, err := esmParser.GetString(ctx, ``, `export default 1;`) - assert.NoError(t, err) - assert.Equal(t, 1, esmParser.cache.Len()) - _, err = esmParser.GetString(ctx, ``, `export default 1;`) - assert.NoError(t, err) - assert.Equal(t, 1, esmParser.cache.Len()) - _, err = esmParser.GetString(ctx, ``, `export default 2;`) - assert.NoError(t, err) - assert.Equal(t, 1, esmParser.cache.Len()) - esmParser.ClearCache() - assert.Equal(t, 0, esmParser.cache.Len()) -} - -func TestESMGetString(t *testing.T) { - { - str, err := esmParser.GetString(ctx, "a", `export default (ctx) => ctx.get('content') + 1`) - assert.NoError(t, err) - assert.Equal(t, "a1", str) - } - - { - str, err := esmParser.GetString(ctx, "", `export default () => ({"test":"1"})`) - assert.NoError(t, err) - assert.JSONEq(t, `{"test":"1"}`, str) - } -} - -func TestESMGetStrings(t *testing.T) { - { - str, err := esmParser.GetStrings(ctx, `["a1"]`, - `export default function (ctx) { - return new Promise((r, j) => { - let s = JSON.parse(ctx.get('content')); - s.push('a2'); - r(s) - }); - }`) - assert.NoError(t, err) - assert.Equal(t, []string{"a1", "a2"}, str) - } - - { - str, err := esmParser.GetStrings(ctx, "", `export default [{"foo":"1"}, {"bar":"1"}, 19]`) - assert.NoError(t, err) - assert.Equal(t, []string{`{"foo":"1"}`, `{"bar":"1"}`, "19"}, str) - } -} - -func TestESMGetElement(t *testing.T) { - ele, err := esmParser.GetElement(ctx, ``, ` - export default (ctx) => { - ctx.set('esm_size', 1 + 2); - return ctx.get('esm_size'); - } - `) - assert.NoError(t, err) - assert.Equal(t, "3", ele) -} - -func TestESMGetElements(t *testing.T) { - ele, err := esmParser.GetElements(ctx, ``, `export default [1, 2];`) - assert.NoError(t, err) - assert.Equal(t, []string{"1", "2"}, ele) -} diff --git a/parsers/js/js.go b/parsers/js/js.go deleted file mode 100644 index 75587b5..0000000 --- a/parsers/js/js.go +++ /dev/null @@ -1,98 +0,0 @@ -// Package js the js parser -package js - -import ( - "encoding/json" - - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/spf13/cast" -) - -// Parser the js parser -type Parser struct{} - -const key string = "js" - -func init() { - parser.Register(key, new(Parser)) -} - -// GetString gets the string of the content with the given arguments. -// returns the string result. -func (p *Parser) GetString(ctx *plugin.Context, content any, arg string) (string, error) { - v, err := p.run(ctx, content, arg) - if err != nil { - return "", err - } - return toString(v) -} - -// GetStrings gets the strings of the content with the given arguments. -// returns the slice of string result. -func (p *Parser) GetStrings(ctx *plugin.Context, content any, arg string) ([]string, error) { - v, err := p.run(ctx, content, arg) - if err != nil { - return nil, err - } - return toStrings(v) -} - -// GetElement gets the element of the content with the given arguments. -// returns the string result. -func (p *Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return p.GetString(ctx, content, arg) -} - -// GetElements gets the elements of the content with the given arguments. -// returns the slice of string result. -func (p *Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return p.GetStrings(ctx, content, arg) -} - -func (p *Parser) run(ctx *plugin.Context, content any, script string) (any, error) { - ctx.SetValue("content", content) - result, err := js.RunString(ctx, script) - if err != nil { - return nil, err - } - return js.Unwrap(result) -} - -func toString(value any) (ret string, err error) { - switch value.(type) { - case map[string]any, []any: - bytes, err := json.Marshal(value) - if err != nil { - return ret, err - } - return string(bytes), nil - case nil: - return ret, nil - default: - return cast.ToStringE(value) - } -} - -func toStrings(value any) (ret []string, err error) { - if value == nil { - return nil, nil - } - - slice, ok := value.([]any) - if !ok { - slice = []any{value} - } - - ret = make([]string, len(slice)) - for i, v := range slice { - if s, ok := v.(string); ok { - ret[i] = s - } else { - bytes, _ := json.Marshal(v) - ret[i] = string(bytes) - } - } - return -} diff --git a/parsers/js/js_test.go b/parsers/js/js_test.go deleted file mode 100644 index f6f8eb3..0000000 --- a/parsers/js/js_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package js - -import ( - "flag" - "os" - "testing" - - "github.com/shiroyk/cloudcat" - "github.com/shiroyk/cloudcat/js" - "github.com/shiroyk/cloudcat/plugin" - "github.com/stretchr/testify/assert" -) - -var ( - jsParser Parser - ctx *plugin.Context -) - -func TestMain(m *testing.M) { - flag.Parse() - cloudcat.Provide(js.NewModuleLoader()) - ctx = plugin.NewContext(plugin.ContextOptions{ - URL: "http://localhost/home", - }) - code := m.Run() - os.Exit(code) -} - -func TestGetString(t *testing.T) { - { - str, err := jsParser.GetString(ctx, "a", `(async () => ctx.get('content') + 1)()`) - assert.NoError(t, err) - assert.Equal(t, "a1", str) - } - - { - str, err := jsParser.GetString(ctx, "", `(async () => ({"test":"1"}))()`) - assert.NoError(t, err) - assert.JSONEq(t, `{"test":"1"}`, str) - } -} - -func TestGetStrings(t *testing.T) { - { - str, err := jsParser.GetStrings(ctx, `["a1"]`, - `new Promise((r, j) => { - let s = JSON.parse(ctx.get('content')); - s.push('a2'); - r(s) - });`) - assert.NoError(t, err) - assert.Equal(t, []string{"a1", "a2"}, str) - } - - { - str, err := jsParser.GetStrings(ctx, "", `[{"foo":"1"}, {"bar":"1"}, 19]`) - assert.NoError(t, err) - assert.Equal(t, []string{`{"foo":"1"}`, `{"bar":"1"}`, "19"}, str) - } -} - -func TestGetElement(t *testing.T) { - ele, err := jsParser.GetElement(ctx, ``, `ctx.set('size', 1 + 2);ctx.get('size');`) - assert.NoError(t, err) - assert.Equal(t, "3", ele) -} - -func TestGetElements(t *testing.T) { - t.Parallel() - ele, err := jsParser.GetElements(ctx, ``, `[1, 2]`) - assert.NoError(t, err) - assert.Equal(t, []string{"1", "2"}, ele) -} diff --git a/parsers/js/lru/lru.go b/parsers/js/lru/lru.go deleted file mode 100644 index 85d38a6..0000000 --- a/parsers/js/lru/lru.go +++ /dev/null @@ -1,130 +0,0 @@ -/* -Copyright 2013 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package lru implements an LRU cache. -package lru - -import "container/list" - -// Cache is an LRU cache. It is not safe for concurrent access. -type Cache[K comparable, V any] struct { - // MaxEntries is the maximum number of cache entries before - // an item is evicted. Zero means no limit. - MaxEntries int - - // OnEvicted optionally specifies a callback function to be - // executed when an entry is purged from the cache. - OnEvicted func(key K, value V) - - ll *list.List - cache map[K]*list.Element -} - -type entry[K comparable, V any] struct { - key K - value V -} - -// New creates a new Cache. -// If maxEntries is zero, the cache has no limit and it's assumed -// that eviction is done by the caller. -func New[K comparable, V any](maxEntries int) *Cache[K, V] { - return &Cache[K, V]{ - MaxEntries: maxEntries, - ll: list.New(), - cache: make(map[K]*list.Element), - } -} - -// Add adds a value to the cache. -func (c *Cache[K, V]) Add(key K, value V) { - if c.cache == nil { - c.cache = make(map[K]*list.Element) - c.ll = list.New() - } - if ee, ok := c.cache[key]; ok { - c.ll.MoveToFront(ee) - ee.Value.(*entry[K, V]).value = value - return - } - ele := c.ll.PushFront(&entry[K, V]{key, value}) - c.cache[key] = ele - if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { - c.RemoveOldest() - } -} - -// Get looks up a key's value from the cache. -func (c *Cache[K, V]) Get(key K) (value V, ok bool) { - if c.cache == nil { - return - } - if ele, hit := c.cache[key]; hit { - c.ll.MoveToFront(ele) - return ele.Value.(*entry[K, V]).value, true - } - return -} - -// Remove removes the provided key from the cache. -func (c *Cache[K, V]) Remove(key K) { - if c.cache == nil { - return - } - if ele, hit := c.cache[key]; hit { - c.removeElement(ele) - } -} - -// RemoveOldest removes the oldest item from the cache. -func (c *Cache[K, V]) RemoveOldest() { - if c.cache == nil { - return - } - ele := c.ll.Back() - if ele != nil { - c.removeElement(ele) - } -} - -func (c *Cache[K, V]) removeElement(e *list.Element) { - c.ll.Remove(e) - kv := e.Value.(*entry[K, V]) - delete(c.cache, kv.key) - if c.OnEvicted != nil { - c.OnEvicted(kv.key, kv.value) - } -} - -// Len returns the number of items in the cache. -func (c *Cache[K, V]) Len() int { - if c.cache == nil { - return 0 - } - return c.ll.Len() -} - -// Clear purges all stored items from the cache. -func (c *Cache[K, V]) Clear() { - if c.OnEvicted != nil { - for _, e := range c.cache { - kv := e.Value.(*entry[K, V]) - c.OnEvicted(kv.key, kv.value) - } - } - c.ll = nil - c.cache = nil -} diff --git a/parsers/js/lru/lru_test.go b/parsers/js/lru/lru_test.go deleted file mode 100644 index a14f439..0000000 --- a/parsers/js/lru/lru_test.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2013 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package lru - -import ( - "fmt" - "testing" -) - -type simpleStruct struct { - int - string -} - -type complexStruct struct { - int - simpleStruct -} - -var getTests = []struct { - name string - keyToAdd interface{} - keyToGet interface{} - expectedOk bool -}{ - {"string_hit", "myKey", "myKey", true}, - {"string_miss", "myKey", "nonsense", false}, - {"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true}, - {"simple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false}, - {"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}}, - complexStruct{1, simpleStruct{2, "three"}}, true}, -} - -func TestGet(t *testing.T) { - for _, tt := range getTests { - lru := New(0) - lru.Add(tt.keyToAdd, 1234) - val, ok := lru.Get(tt.keyToGet) - if ok != tt.expectedOk { - t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok) - } else if ok && val != 1234 { - t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val) - } - } -} - -func TestRemove(t *testing.T) { - lru := New(0) - lru.Add("myKey", 1234) - if val, ok := lru.Get("myKey"); !ok { - t.Fatal("TestRemove returned no match") - } else if val != 1234 { - t.Fatalf("TestRemove failed. Expected %d, got %v", 1234, val) - } - - lru.Remove("myKey") - if _, ok := lru.Get("myKey"); ok { - t.Fatal("TestRemove returned a removed entry") - } -} - -func TestEvict(t *testing.T) { - evictedKeys := make([]Key, 0) - onEvictedFun := func(key Key, value interface{}) { - evictedKeys = append(evictedKeys, key) - } - - lru := New(20) - lru.OnEvicted = onEvictedFun - for i := 0; i < 22; i++ { - lru.Add(fmt.Sprintf("myKey%d", i), 1234) - } - - if len(evictedKeys) != 2 { - t.Fatalf("got %d evicted keys; want 2", len(evictedKeys)) - } - if evictedKeys[0] != Key("myKey0") { - t.Fatalf("got %v in first evicted key; want %s", evictedKeys[0], "myKey0") - } - if evictedKeys[1] != Key("myKey1") { - t.Fatalf("got %v in second evicted key; want %s", evictedKeys[1], "myKey1") - } -} diff --git a/parsers/json/json.go b/parsers/json/json.go deleted file mode 100644 index 4304fd6..0000000 --- a/parsers/json/json.go +++ /dev/null @@ -1,113 +0,0 @@ -// Package json the json parser -package json - -import ( - "fmt" - "strings" - - "github.com/ohler55/ojg/jp" - "github.com/ohler55/ojg/oj" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/spf13/cast" -) - -// Parser the json parser -type Parser struct{} - -const key string = "json" - -func init() { - parser.Register(key, new(Parser)) -} - -// GetString gets the string of the content with the given arguments. -// -// content := `{"keys": [{"key":"foo"},{"key":"bar"}]}` -// GetString(ctx, content, "$.key[*].key") returns "foo\nbar" -func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - obj, err := getDoc(content, arg) - if err != nil { - return "", err - } - - str := make([]string, len(obj)) - var ok bool - - for i, o := range obj { - if str[i], ok = o.(string); !ok { - str[i] = oj.JSON(o) - } - } - - return strings.Join(str, "\n"), nil -} - -// GetStrings gets the strings of the content with the given arguments. -// -// content := `{"keys": [{"key":"foo"},{"key":"bar"}]}` -// GetStrings(ctx, content, "$.key[*].key") returns []string{"foo", "bar"} -func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - obj, err := getDoc(content, arg) - if err != nil { - return nil, err - } - - str := make([]string, len(obj)) - var ok bool - - for i, o := range obj { - if str[i], ok = o.(string); !ok { - str[i] = oj.JSON(o) - } - } - - return str, nil -} - -// GetElement gets the element of the content with the given arguments. -// sames as the GetString. -func (p Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return p.GetString(ctx, content, arg) -} - -// GetElements gets the elements of the content with the given arguments. -// sames as the GetStrings. -func (p Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return p.GetStrings(ctx, content, arg) -} - -func getDoc(content any, arg string) ([]any, error) { - var err error - var doc any - switch data := content.(type) { - default: - str, err := cast.ToStringE(content) - if err != nil { - return nil, err - } - if doc, err = oj.ParseString(str); err != nil { - return nil, err - } - case nil: - return nil, nil - case []string: - if len(data) == 0 { - return nil, fmt.Errorf("unexpected content %s", content) - } - if doc, err = oj.ParseString(data[0]); err != nil { - return nil, err - } - case string: - if doc, err = oj.ParseString(data); err != nil { - return nil, err - } - } - - x, err := jp.ParseString(arg) - if err != nil { - return nil, err - } - - return x.Get(doc), nil -} diff --git a/parsers/json/json_test.go b/parsers/json/json_test.go deleted file mode 100644 index ccc1b95..0000000 --- a/parsers/json/json_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package json - -import ( - "flag" - "os" - "testing" - - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/stretchr/testify/assert" -) - -var ( - json Parser - ctx *plugin.Context - content = ` -{ - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - }, - "expensive": 10 -}` -) - -func TestMain(m *testing.M) { - flag.Parse() - ctx = plugin.NewContext(plugin.ContextOptions{}) - code := m.Run() - os.Exit(code) -} - -func assertString(t *testing.T, arg string, expected string) { - str, err := json.GetString(ctx, content, arg) - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, expected, str) -} - -func assertStrings(t *testing.T, arg string, expected []string) { - str, err := json.GetStrings(ctx, content, arg) - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, expected, str) -} - -func TestParser(t *testing.T) { - t.Parallel() - if _, ok := parser.GetParser(key); !ok { - t.Fatal("schema not registered") - } - - contents := []any{`][`, `}{`} - for _, ct := range contents { - _, err := json.GetString(ctx, ct, ``) - assert.ErrorContains(t, err, "unexpected") - } - - if _, err := json.GetString(ctx, &contents[len(contents)-1], ""); err == nil { - t.Fatal("Unexpected type") - } -} - -func TestGetString(t *testing.T) { - t.Parallel() - assertString(t, `$.store.book[*].author`, "Nigel Rees\nEvelyn Waugh\nHerman Melville\nJ. R. R. Tolkien") -} - -func TestGetStrings(t *testing.T) { - t.Parallel() - assertStrings(t, `$...book[0].price`, []string{"8.95"}) - - assertStrings(t, `$...book[-1].price`, []string{"22.99"}) -} - -func TestGetElement(t *testing.T) { - t.Parallel() - if _, err := json.GetElement(ctx, content, `$$$`); err == nil { - t.Fatal("Unexpected path") - } - - assertString(t, `$.store.book[-1].price`, "22.99") - - str1, err := json.GetElement(ctx, content, `$.store.book[?(@.price > 20)]`) - if err != nil { - t.Fatal(err) - } - - str2, err := json.GetElement(ctx, str1, `$.title`) - if err != nil { - t.Fatal(err) - } - if str2 != `The Lord of the Rings` { - t.Fatalf("Unexpected string %s", str2) - } -} - -func TestGetElements(t *testing.T) { - t.Parallel() - assertStrings(t, `$.store.book[?(@.price < 10)].isbn`, []string{`0-553-21311-3`}) - - str1, err := json.GetElements(ctx, content, `$.store.book[3]`) - if err != nil { - t.Fatal(err) - } - - str2, err := json.GetElement(ctx, str1[0], `$.category`) - if err != nil { - t.Fatal(err) - } - if str2 != `fiction` { - t.Fatalf("Unexpected string %s", str2) - } -} diff --git a/parsers/main.go b/parsers/main.go deleted file mode 100644 index 8469318..0000000 --- a/parsers/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package parsers - -import ( - _ "github.com/shiroyk/cloudcat/parsers/gq" // deep - _ "github.com/shiroyk/cloudcat/parsers/js" - _ "github.com/shiroyk/cloudcat/parsers/json" - _ "github.com/shiroyk/cloudcat/parsers/regex" - _ "github.com/shiroyk/cloudcat/parsers/xpath" -) diff --git a/parsers/regex/regex.go b/parsers/regex/regex.go index 6464fcc..1edd837 100644 --- a/parsers/regex/regex.go +++ b/parsers/regex/regex.go @@ -2,72 +2,89 @@ package regex import ( + "context" "fmt" "strconv" "strings" "github.com/dlclark/regexp2" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/spf13/cast" + "github.com/shiroyk/ski" ) // Parser the regexp2 parser type Parser struct{} -const key string = "regex" - func init() { - parser.Register(key, new(Parser)) + ski.Register("regex", new(Parser)) } -// GetString gets the string of the content with the given arguments. -// replace the string with the given regexp. -func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - re, replace, start, count, err := parseRegexp(arg) +func (p Parser) Value(arg string) (ski.Executor, error) { + ret, err := compile(arg) if err != nil { - return "", err + return nil, err } + ret.exec = ret.string + return ret, nil +} - var str string - switch conv := content.(type) { - case string: - str = conv - case []string: - str = strings.Join(conv, "") - default: - str, err = cast.ToStringE(conv) - if err != nil { - return "", err - } +func (p Parser) Element(arg string) (ski.Executor, error) { return p.Value(arg) } + +func (p Parser) Elements(arg string) (ski.Executor, error) { + ret, err := compile(arg) + if err != nil { + return nil, err } + ret.exec = ret.strings + return ret, nil +} - return re.Replace(str, replace, start, count) +type regexp struct { + re *regexp2.Regexp + start, count int + replace string + exec func(any) (any, error) } -// GetStrings gets the strings of the content with the given arguments. -// replace each string of the slice with the given regexp. -func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - re, replace, start, count, err := parseRegexp(arg) - if err != nil { - return nil, err +func (r regexp) Exec(_ context.Context, arg any) (any, error) { return r.exec(arg) } + +func (r regexp) string(arg any) (any, error) { + switch conv := arg.(type) { + case string: + return r.re.Replace(conv, r.replace, r.start, r.count) + case []string: + var err error + for i := 0; i < len(conv); i++ { + conv[i], err = r.re.Replace(conv[i], r.replace, r.start, r.count) + if err != nil { + return nil, err + } + } + return conv, nil + case fmt.Stringer: + return r.re.Replace(conv.String(), r.replace, r.start, r.count) + default: + return nil, fmt.Errorf("unexpected type %T", arg) } +} - var str []string - switch conv := content.(type) { +func (r regexp) strings(arg any) (any, error) { + var ( + str []string + err error + ) + switch conv := arg.(type) { case string: str = []string{conv} case []string: str = conv + case fmt.Stringer: + str = []string{conv.String()} default: - str, err = cast.ToStringSliceE(conv) - if err != nil { - return nil, err - } + return nil, fmt.Errorf("unexpected type %T", arg) } for i := 0; i < len(str); i++ { - str[i], err = re.Replace(str[i], replace, start, count) + str[i], err = r.re.Replace(str[i], r.replace, r.start, r.count) if err != nil { return nil, err } @@ -75,18 +92,6 @@ func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string return str, nil } -// GetElement gets the element of the content with the given arguments. -// sames as GetString. -func (p Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return p.GetString(ctx, content, arg) -} - -// GetElements gets the elements of the content with the given arguments. -// sames as GetStrings. -func (p Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return p.GetStrings(ctx, content, arg) -} - type tokenState int const ( @@ -109,12 +114,11 @@ var reOptMap = map[string]regexp2.RegexOptions{ "u": regexp2.Unicode, } -//nolint:gocognit -func parseRegexp(arg string) (re *regexp2.Regexp, replace string, start, count int, err error) { +func compile(arg string) (ret regexp, err error) { state := commonState pattern := strings.Builder{} - start = -1 - count = -1 + ret.start = -1 + ret.count = -1 var offset int var regex string var reOpt int32 @@ -150,25 +154,28 @@ func parseRegexp(arg string) (re *regexp2.Regexp, replace string, start, count i pattern.Reset() case replaceState: state = flagState - replace = pattern.String() + ret.replace = pattern.String() pattern.Reset() default: - return nil, "", start, count, fmt.Errorf("/ character must escaped") + return ret, fmt.Errorf("/ character must escaped") } } } if pattern.Len() > 0 { s1, s2, _ := strings.Cut(pattern.String(), ",") - start, err = strconv.Atoi(s1) + ret.start, err = strconv.Atoi(s1) if err != nil { - start = -1 + ret.start = -1 + err = nil } - count, err = strconv.Atoi(s2) + ret.count, err = strconv.Atoi(s2) if err != nil { - count = -1 + ret.count = -1 + err = nil } } - return regexp2.MustCompile(regex, regexp2.RegexOptions(reOpt)), replace, start, count, nil + ret.re, err = regexp2.Compile(regex, regexp2.RegexOptions(reOpt)) + return } diff --git a/parsers/regex/regex_test.go b/parsers/regex/regex_test.go index d212ff9..93d305d 100644 --- a/parsers/regex/regex_test.go +++ b/parsers/regex/regex_test.go @@ -1,9 +1,9 @@ package regex import ( + "context" "testing" - "github.com/shiroyk/cloudcat/plugin/parser" "github.com/stretchr/testify/assert" ) @@ -25,34 +25,32 @@ var ( } ) -func TestParser(t *testing.T) { - if _, ok := parser.GetParser(key); !ok { - t.Fatal("parser not registered") - } -} - -func TestGetString(t *testing.T) { +func TestValue(t *testing.T) { t.Parallel() for _, s := range testCase { t.Run(s.re, func(t *testing.T) { - str, err := re.GetString(nil, s.str, s.re) - if err != nil { - t.Error(err) + executor, err := re.Value(s.re) + if assert.NoError(t, err) { + v, err := executor.Exec(context.Background(), s.str) + if assert.NoError(t, err) { + assert.Equal(t, s.want, v) + } } - assert.Equal(t, s.want, str) }) } } -func TestGetStrings(t *testing.T) { +func TestElements(t *testing.T) { t.Parallel() for _, s := range testCase { t.Run(s.re, func(t *testing.T) { - str, err := re.GetStrings(nil, []string{s.str}, s.re) - if err != nil { - t.Error(err) + executor, err := re.Elements(s.re) + if assert.NoError(t, err) { + v, err := executor.Exec(context.Background(), s.str) + if assert.NoError(t, err) { + assert.Equal(t, s.want, v.([]string)[0]) + } } - assert.Equal(t, s.want, str[0]) }) } } diff --git a/parsers/xpath/bench_xpath_test.go b/parsers/xpath/bench_xpath_test.go index 5d2318f..a37e074 100644 --- a/parsers/xpath/bench_xpath_test.go +++ b/parsers/xpath/bench_xpath_test.go @@ -7,7 +7,7 @@ import ( func BenchmarkParser(b *testing.B) { b.StartTimer() for i := 0; i < b.N; i++ { - _, err := xpath.GetString(ctx, ``, `//div[@class="body"]/ul//a/@title`) + _, err := p.Value(`//div[@class="body"]/ul//a/@title`) if err != nil { b.Fatal(err) } diff --git a/parsers/xpath/xpath.go b/parsers/xpath/xpath.go index f0c4857..f401a3c 100644 --- a/parsers/xpath/xpath.go +++ b/parsers/xpath/xpath.go @@ -2,147 +2,105 @@ package xpath import ( + "context" + "fmt" "strings" "github.com/antchfx/htmlquery" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" - "github.com/spf13/cast" + "github.com/antchfx/xpath" + "github.com/shiroyk/ski" "golang.org/x/net/html" ) // Parser the xpath parser type Parser struct{} -const key string = "xpath" - func init() { - parser.Register(key, new(Parser)) + ski.Register("xpath", new(Parser)) } -// GetString gets the string of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetString(ctx, content, "//li/text()") returns "1\n2" -func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - nodes, err := getHTMLNode(content, arg) +func (p Parser) Value(arg string) (ski.Executor, error) { + ex, err := xpath.Compile(arg) if err != nil { - return "", err + return nil, err } + return expr{ex, value}, nil +} - if len(nodes) == 0 { - return "", nil +func (p Parser) Element(arg string) (ski.Executor, error) { + ex, err := xpath.Compile(arg) + if err != nil { + return nil, err } - - str := strings.Builder{} - str.WriteString(htmlquery.InnerText(nodes[0])) - for _, node := range nodes[1:] { - str.WriteString("\n") - str.WriteString(htmlquery.InnerText(node)) + return expr{ex, element}, nil +} +func (p Parser) Elements(arg string) (ski.Executor, error) { + ex, err := xpath.Compile(arg) + if err != nil { + return nil, err } + return expr{ex, elements}, nil +} - return str.String(), nil +type expr struct { + *xpath.Expr + ret func([]*html.Node) (any, error) } -// GetStrings gets the strings of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetStrings(ctx, content, "//li/text()") returns []string{"1", "2"} -func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - nodes, err := getHTMLNode(content, arg) +func (e expr) Exec(_ context.Context, arg any) (any, error) { + node, err := htmlNode(arg) if err != nil { return nil, err } + return e.ret(htmlquery.QuerySelectorAll(node, e.Expr)) +} - if len(nodes) == 0 { +func value(nodes []*html.Node) (any, error) { + switch len(nodes) { + case 0: return nil, nil + case 1: + return htmlquery.InnerText(nodes[0]), nil + default: + str := make([]string, len(nodes)) + for i, node := range nodes { + str[i] = htmlquery.InnerText(node) + } + return str, nil } - - result := make([]string, len(nodes)) - for i, node := range nodes { - result[i] = htmlquery.InnerText(node) - } - - return result, err } -// GetElement gets the element of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetStrings(ctx, content, "//li..") returns "
  • 1
    • \n
  • 2
    • " -func (p Parser) GetElement(_ *plugin.Context, content any, arg string) (string, error) { - nodes, err := getHTMLNode(content, arg) - if err != nil { - return "", err - } - +func element(nodes []*html.Node) (any, error) { if len(nodes) == 0 { - return "", nil - } - - str := strings.Builder{} - str.WriteString(htmlquery.OutputHTML(nodes[0], true)) - for _, node := range nodes[1:] { - str.WriteString("\n") - str.WriteString(htmlquery.OutputHTML(node, true)) + return nil, nil } - - return str.String(), nil + return nodes[0], nil } -// GetElements gets the elements of the content with the given arguments. -// -// content := `
    • 1
    • 2
    ` -// GetStrings(ctx, content, "//li..") returns []string{"
  • 1
    • ", "
  • 2
    • "} -func (p Parser) GetElements(_ *plugin.Context, content any, arg string) ([]string, error) { - nodes, err := getHTMLNode(content, arg) - if err != nil { - return nil, err - } - +func elements(nodes []*html.Node) (any, error) { if len(nodes) == 0 { return nil, nil } - str := make([]string, len(nodes)) + ret := make([]any, len(nodes)) for i, node := range nodes { - str[i] = htmlquery.OutputHTML(node, true) + ret[i] = node } - return str, nil + return ret, nil } -func getHTMLNode(content any, arg string) ([]*html.Node, error) { - var err error - var node *html.Node +func htmlNode(content any) (node *html.Node, err error) { switch data := content.(type) { default: - str, err := cast.ToStringE(content) - if err != nil { - return nil, err - } - node, err = html.Parse(strings.NewReader(str)) - if err != nil { - return nil, err - } + return nil, fmt.Errorf("unexpected type %T", content) case nil: return nil, nil + case *html.Node: + return data, nil case []string: - node, err = html.Parse(strings.NewReader(strings.Join(data, "\n"))) - if err != nil { - return nil, err - } + return html.Parse(strings.NewReader(strings.Join(data, "\n"))) case string: - node, err = html.Parse(strings.NewReader(data)) - if err != nil { - return nil, err - } + return html.Parse(strings.NewReader(data)) } - - htmlNode, err := htmlquery.QueryAll(node, arg) - if err != nil { - return nil, err - } - - return htmlNode, nil } diff --git a/parsers/xpath/xpath_test.go b/parsers/xpath/xpath_test.go index e1e5849..e487874 100644 --- a/parsers/xpath/xpath_test.go +++ b/parsers/xpath/xpath_test.go @@ -1,18 +1,17 @@ package xpath import ( - "flag" - "os" + "bytes" + "context" "testing" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" "github.com/stretchr/testify/assert" + "golang.org/x/net/html" ) var ( - xpath Parser - ctx *plugin.Context + p Parser + ctx = context.Background() content = ` @@ -43,120 +42,90 @@ var (
    f5
    f6
    - + ` ) -func TestMain(m *testing.M) { - flag.Parse() - ctx = plugin.NewContext(plugin.ContextOptions{}) - code := m.Run() - os.Exit(code) +func assertError(t *testing.T, arg string, contains string) { + _, err := p.Value(arg) + assert.ErrorContains(t, err, contains) } -func TestParser(t *testing.T) { - t.Parallel() - if _, ok := parser.GetParser(key); !ok { - t.Fatal("schema not registered") - } - - _, err := xpath.GetString(ctx, 1, ``) - if err == nil { - t.Fatal("error should not be nil") - } - - _, err = xpath.GetString(ctx, `Golang`, `//a`) - if err != nil { - t.Error(err) - } - - sel, _ := xpath.GetElement(ctx, content, `//div[@class="body"]`) - _, err = xpath.GetString(ctx, sel, `//a/text()`) - if err != nil { - t.Error(err) +func assertValue(t *testing.T, arg string, expected any) { + executor, err := p.Value(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + assert.Equal(t, expected, v) + } } } -func TestGetString(t *testing.T) { - t.Parallel() - if o, _ := xpath.GetStrings(ctx, content, `///`); o != nil { - t.Fatal("Unexpected type") +func assertElement(t *testing.T, arg string, expected string) { + executor, err := p.Element(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + switch c := v.(type) { + case *html.Node: + b := new(bytes.Buffer) + if assert.NoError(t, html.Render(b, c)) { + assert.Equal(t, expected, b.String()) + } + default: + assert.Equal(t, expected, v) + } + } } +} - str1, err := xpath.GetString(ctx, content, `//div[@id="main"]/div[contains(@class, "row")]/text()`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "1\n2\n3\n4\n5\n6", str1) - - str2, err := xpath.GetString(ctx, content, `//div[@class="body"]/ul/li/@id`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "a1\na2\na3", str2) - - js, err := xpath.GetString(ctx, content, `//script[1]`) - if err != nil { - t.Fatal(err) +func assertElements(t *testing.T, arg string, expected []string) { + executor, err := p.Elements(arg) + if assert.NoError(t, err) { + v, err := executor.Exec(ctx, content) + if assert.NoError(t, err) { + switch c := v.(type) { + case []any: + ele := make([]string, len(c)) + for i, v := range c { + var b bytes.Buffer + if assert.NoError(t, html.Render(&b, v.(*html.Node))) { + ele[i] = b.String() + } + } + assert.Equal(t, expected, ele) + default: + assert.Equal(t, expected, v) + } + } } - assert.NotEmpty(t, js) } -func TestGetStrings(t *testing.T) { +func TestValue(t *testing.T) { t.Parallel() - if o, _ := xpath.GetStrings(ctx, content, `//unknown`); o != nil { - t.Fatal("Unexpected type") - } + assertError(t, `///`, "expression must evaluate to a node-set") - str1, err := xpath.GetStrings(ctx, content, `//div[@class="body"]/ul//a/@title`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, []string{"Google page", "Github page", "Golang page"}, str1) + assertValue(t, `//div[@id="main"]/div[contains(@class, "row")]/text()`, []string{"1", "2", "3", "4", "5", "6"}) - str2, err := xpath.GetStrings(ctx, content, `//div[@class="body"]/ul//a`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, []string{"Google", "Github", "Golang"}, str2) + assertValue(t, `//div[@class="body"]/ul/li/@id`, []string{"a1", "a2", "a3"}) + + assertValue(t, `//script[1]`, `(function() {})();`) } -func TestGetElement(t *testing.T) { +func TestElement(t *testing.T) { t.Parallel() - if o, _ := xpath.GetElement(ctx, content, `//unknown`); o != "" { - t.Fatal("Unexpected type") - } - object, err := xpath.GetElement(ctx, content, `//div[@class="body"]/ul//a/..`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, `
  • Google
    • -
  • Github
    • -
  • Golang
    • `, object) + assertElement(t, `//div[@class="body"]/ul//a/..`, `
  • Google
    • `) } -func TestGetElements(t *testing.T) { +func TestElements(t *testing.T) { t.Parallel() - if o, _ := xpath.GetElements(ctx, content, `//unknown`); o != nil { - t.Fatal("Unexpected type") - } - objects, err := xpath.GetElements(ctx, content, `//div[@id="foot"]/div/@class`) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, []string{ + assertElements(t, `//div[@id="foot"]/div/@class`, []string{ "one even row", "two odd row", "three even row", "four odd row", "five even row odder", "six odd row", - }, objects) + }) } diff --git a/plugin/context.go b/plugin/context.go deleted file mode 100644 index 46b125f..0000000 --- a/plugin/context.go +++ /dev/null @@ -1,115 +0,0 @@ -package plugin - -import ( - "context" - "fmt" - "log/slog" - "net/url" - "sync" - "time" -) - -const ( - // DefaultTimeout The Context default timeout one minute. - DefaultTimeout = time.Minute -) - -// Context The Parser context -type Context struct { - context.Context // set to non-nil by the first cancel call - parent context.Context // the parent context - cancelFunc context.CancelFunc - logger *slog.Logger - value *sync.Map - baseURL, url string -} - -// ContextOptions The Context options -type ContextOptions struct { - Parent context.Context // the parent context - Timeout time.Duration // the context timeout, default DefaultTimeout. - Logger *slog.Logger // the context logger, default slog.Default if nil. - Values map[any]any // the values - URL string // the analyzer URL -} - -// NewContext creates a new Context with ContextOptions -func NewContext(opt ContextOptions) *Context { - ctx := &Context{ - value: new(sync.Map), - logger: opt.Logger, - parent: opt.Parent, - } - if ctx.logger == nil { - ctx.logger = slog.Default() - } - if ctx.parent == nil { - ctx.parent = context.Background() - } - for k, v := range opt.Values { - ctx.value.Store(k, v) - } - if opt.URL != "" { - ctx.url = opt.URL - if u, err := url.Parse(ctx.url); err == nil { - ctx.baseURL = fmt.Sprintf("%s://%s", u.Scheme, u.Host) - } - } - - timeout := DefaultTimeout - if opt.Timeout > 0 { - timeout = opt.Timeout - } - ctx.Context, ctx.cancelFunc = context.WithTimeout(ctx.parent, timeout) - return ctx -} - -// Cancel this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func (c *Context) Cancel() { - if c.cancelFunc != nil { - c.cancelFunc() - } -} - -// ClearValue clean all values -func (c *Context) ClearValue() { - c.value = new(sync.Map) -} - -// Value returns the value associated with this context for key, or nil -// if no value is associated with key. Successive calls to Value with -// the same key returns the same result. -func (c *Context) Value(key any) any { - if v, ok := c.value.Load(key); ok { - return v - } - return c.parent.Value(key) -} - -// GetValue returns the value associated with this context for key, or nil -// if no value is associated with key. Successive calls to Value with -// the same key returns the same result. -func (c *Context) GetValue(key any) (any, bool) { - return c.value.Load(key) -} - -// SetValue value associated with key is val. -func (c *Context) SetValue(key any, value any) { - c.value.Store(key, value) -} - -// Logger returns the logger, if ContextOptions.Logger is nil return slog.Default -func (c *Context) Logger() *slog.Logger { - return c.logger -} - -// BaseURL returns the baseURL string -func (c *Context) BaseURL() string { - return c.baseURL -} - -// URL returns the absolute URL string -func (c *Context) URL() string { - return c.url -} diff --git a/plugin/context_test.go b/plugin/context_test.go deleted file mode 100644 index 14eaed8..0000000 --- a/plugin/context_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package plugin - -import ( - "context" - "log/slog" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestContext(t *testing.T) { - t.Parallel() - url := "https://example.com/some/path?offset=1" - baseURL := "https://example.com" - logger := slog.Default().With(slog.String("source", "ctx")) - ctx := NewContext(ContextOptions{ - URL: url, - Logger: logger, - Timeout: time.Minute, - Values: map[any]any{ - "key1": "value1", - }, - }) - defer ctx.Cancel() - - assert.NotNil(t, ctx.Logger()) - assert.Equal(t, ctx.Logger(), logger) - assert.Equal(t, ctx.URL(), url) - assert.Equal(t, ctx.BaseURL(), baseURL) - assert.Equal(t, ctx.Value("key1"), "value1") - assert.Nil(t, ctx.Value("notExists")) - - if _, ok := ctx.Deadline(); !ok { - t.Error("deadline not set") - } - key := "test" - value := "1" - ctx.SetValue(key, value) - if v, ok := ctx.GetValue(key); ok { - assert.Equalf(t, v, value, "want %v, got %v", value, v) - } - if v := ctx.Value(key); v != value { - t.Errorf("want %v, got %v", value, v) - } - ctx.ClearValue() - assert.Nil(t, ctx.Value(key), "values should be nil") - - ctx.Cancel() - assert.ErrorIs(t, ctx.Err(), context.Canceled) - - <-ctx.Done() - - ctx1 := NewContext(ContextOptions{Timeout: time.Nanosecond}) - <-ctx1.Done() - assert.ErrorIs(t, ctx1.Err(), context.DeadlineExceeded) -} - -func TestParentContext(t *testing.T) { - t.Parallel() - type k string - key := k("parentKey") - value := "foo" - valueCtx := context.WithValue(context.Background(), key, value) - parent, cancel := context.WithTimeout(valueCtx, time.Minute) - - ctx := NewContext(ContextOptions{Parent: parent}) - assert.Equal(t, value, ctx.Value(key)) - cancel() - - time.Sleep(time.Millisecond) - - assert.ErrorIs(t, ctx.Err(), context.Canceled) -} diff --git a/plugin/go.mod b/plugin/go.mod deleted file mode 100644 index 225d4e8..0000000 --- a/plugin/go.mod +++ /dev/null @@ -1,13 +0,0 @@ -module github.com/shiroyk/cloudcat/plugin - -go 1.21 - -require github.com/stretchr/testify v1.8.4 - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/plugin/go.sum b/plugin/go.sum deleted file mode 100644 index 9b7f750..0000000 --- a/plugin/go.sum +++ /dev/null @@ -1,22 +0,0 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/plugin/internal/ext/extension.go b/plugin/internal/ext/extension.go deleted file mode 100644 index 3de4dbc..0000000 --- a/plugin/internal/ext/extension.go +++ /dev/null @@ -1,176 +0,0 @@ -// Package ext the extension manager -package ext - -import ( - "encoding/json" - "errors" - "fmt" - "reflect" - "runtime" - "runtime/debug" - "strings" - "sync" -) - -var ( - mx sync.RWMutex - extensions = make(map[ExtensionType]map[string]*Extension) -) - -// ExtensionType The type of extension -type ExtensionType uint - -const ( - // JSExtension The modules.Module - JSExtension ExtensionType = iota + 1 - // ParserExtension The parser.Parser. - ParserExtension -) - -func (e ExtensionType) String() string { - switch e { - case JSExtension: - return "js" - case ParserExtension: - return "parser" - default: - return "" - } -} - -// Extension a generic container. -type Extension struct { - Name, Path, Version string - Type ExtensionType - Module any -} - -func (e Extension) String() string { - return fmt.Sprintf("%s [%s] %s %s ", e.Name, e.Type, e.Version, e.Path) -} - -// MarshalJSON encodes to JSON -func (e Extension) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]string{ - "name": e.Name, - "path": e.Path, - "version": e.Version, - "type": e.Type.String(), - }) -} - -// Register a new extension with the given name and type. This function will -// panic if an unsupported extension type is provided, or if an extension of the -// same type and name is already registered. -func Register(name string, typ ExtensionType, mod any) { - mx.Lock() - defer mx.Unlock() - - if mod == nil { - panic(errors.New("extension cannot be nil")) - } - - exts, ok := extensions[typ] - if !ok { - panic(fmt.Sprintf("unsupported extension type: %T", typ)) - } - - path, version := extractModuleInfo(mod) - - exts[name] = &Extension{ - Name: name, - Type: typ, - Module: mod, - Path: path, - Version: version, - } -} - -// Get returns all extensions of the specified type. -func Get(typ ExtensionType) map[string]*Extension { - mx.RLock() - defer mx.RUnlock() - - exts, ok := extensions[typ] - if !ok { - panic(fmt.Sprintf("unsupported extension type: %T", typ)) - } - - result := make(map[string]*Extension, len(exts)) - - for name, ext := range exts { - result[name] = ext - } - - return result -} - -// GetName returns extension of the specified type and name. -func GetName(typ ExtensionType, name string) (ext *Extension, ok bool) { - mx.RLock() - defer mx.RUnlock() - - exts, ok := extensions[typ] - if !ok { - panic(fmt.Sprintf("unsupported extension type: %T", typ)) - } - ext, ok = exts[name] - return -} - -// GetAll returns all extensions. -func GetAll() []*Extension { - mx.RLock() - defer mx.RUnlock() - - js, parser := extensions[JSExtension], extensions[ParserExtension] - result := make([]*Extension, 0, len(js)+len(parser)) - - for _, e := range js { - result = append(result, e) - } - for _, e := range parser { - result = append(result, e) - } - - return result -} - -// extractModuleInfo attempts to return the package path and version of the Go -// module that created the given value. -func extractModuleInfo(mod any) (path, version string) { - t := reflect.TypeOf(mod) - - switch t.Kind() { - case reflect.Ptr, reflect.Struct: - if t.Elem() != nil { - path = t.Elem().PkgPath() - } - case reflect.Func: - path = runtime.FuncForPC(reflect.ValueOf(mod).Pointer()).Name() - default: - return - } - - buildInfo, ok := debug.ReadBuildInfo() - if !ok { - return - } - - for _, dep := range buildInfo.Deps { - depPath := strings.TrimSpace(dep.Path) - if strings.HasPrefix(path, depPath) { - if dep.Replace != nil { - return depPath, dep.Replace.Version - } - return depPath, dep.Version - } - } - - return -} - -func init() { - extensions[JSExtension] = make(map[string]*Extension) - extensions[ParserExtension] = make(map[string]*Extension) -} diff --git a/plugin/jsmodule/module.go b/plugin/jsmodule/module.go deleted file mode 100644 index f71f0f9..0000000 --- a/plugin/jsmodule/module.go +++ /dev/null @@ -1,44 +0,0 @@ -// Package jsmodule the JS module -package jsmodule - -import ( - "github.com/shiroyk/cloudcat/plugin/internal/ext" -) - -const ( - // ExtPrefix common module prefix - ExtPrefix = "cloudcat/" -) - -// Module is what a module needs to return -type Module interface { - Exports() any // module instance -} - -// Global is it a global module -// When the module implements the interface it will be loaded into the global -// when the js.VM is initialized. -type Global interface { - Module - Global() // is it a global module -} - -// Register the given mod as an external JavaScript module that can be imported -// by name. -func Register(name string, mod Module) { - if _, ok := mod.(any).(Global); !ok { - name = ExtPrefix + name - } - ext.Register(name, ext.JSExtension, mod) -} - -func GetModule(name string) (Module, bool) { - if m, ok := ext.GetName(ext.JSExtension, name); ok { - return m.Module.(Module), true - } - return nil, false -} - -func AllModules() map[string]*ext.Extension { - return ext.Get(ext.JSExtension) -} diff --git a/plugin/jsmodule/module_test.go b/plugin/jsmodule/module_test.go deleted file mode 100644 index 22c6067..0000000 --- a/plugin/jsmodule/module_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package jsmodule - -import ( - "testing" -) - -type testModule struct{} - -func (t testModule) Exports() any { return map[string]string{"foo": "module"} } - -type testGlobalModule struct{} - -func (t testGlobalModule) Exports() any { return map[string]string{"foo": "global"} } -func (t testGlobalModule) Global() {} - -func TestModule(t *testing.T) { - t.Parallel() - - moduleKey := "testModule" - if _, ok := GetModule(ExtPrefix + moduleKey); !ok { - Register(moduleKey, new(testModule)) - } - if _, ok := GetModule(ExtPrefix + moduleKey); !ok { - t.Fatal("unable get module") - } - - globalModuleKey := "testModule" - if _, ok := GetModule(globalModuleKey); !ok { - Register(globalModuleKey, new(testGlobalModule)) - } - if _, ok := GetModule(globalModuleKey); !ok { - t.Fatal("unable get global module") - } -} diff --git a/plugin/parser/parser.go b/plugin/parser/parser.go deleted file mode 100644 index 220fc3f..0000000 --- a/plugin/parser/parser.go +++ /dev/null @@ -1,56 +0,0 @@ -// Package parser the schema parser -package parser - -import ( - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/internal/ext" -) - -// Parser the content schema -type Parser interface { - // GetString gets the string of the content with the given arguments. - // e.g.: - // - // content := `
    • 1
    • 2
    ` - // GetString(ctx, content, "ul li") returns "1\n2" - // - GetString(*plugin.Context, any, string) (string, error) - // GetStrings gets the strings of the content with the given arguments. - // e.g.: - // - // content := `
    • 1
    • 2
    ` - // GetStrings(ctx, content, "ul li") returns []string{"1", "2"} - // - GetStrings(*plugin.Context, any, string) ([]string, error) - // GetElement gets the element of the content with the given arguments. - // e.g.: - // - // content := `
    • 1
    • 2
    ` - // GetElement(ctx, content, "ul li") returns "
  • 1
    • \n
  • 2
    • " - // - GetElement(*plugin.Context, any, string) (string, error) - // GetElements gets the elements of the content with the given arguments. - // e.g.: - // - // content := `
    • 1
    • 2
    ` - // GetElements(ctx, content, "ul li") returns []string{"
  • 1
    • ", "
  • 2
    • "} - // - GetElements(*plugin.Context, any, string) ([]string, error) -} - -// Register registers the Parser with the given key Parser -func Register(key string, parser Parser) { - ext.Register(key, ext.ParserExtension, parser) -} - -// GetParser returns a Parser with the given key -func GetParser(key string) (Parser, bool) { - if p, ok := ext.GetName(ext.ParserExtension, key); ok { - return p.Module.(Parser), true - } - return nil, false -} - -func AllParsers() map[string]*ext.Extension { - return ext.Get(ext.ParserExtension) -} diff --git a/plugin/parser/parser_test.go b/plugin/parser/parser_test.go deleted file mode 100644 index 276bd98..0000000 --- a/plugin/parser/parser_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package parser - -import ( - "testing" - - "github.com/shiroyk/cloudcat/plugin" -) - -type testParser struct{} - -func (t *testParser) GetString(*plugin.Context, any, string) (string, error) { - return "", nil -} - -func (t *testParser) GetStrings(*plugin.Context, any, string) ([]string, error) { - return nil, nil -} - -func (t *testParser) GetElement(*plugin.Context, any, string) (string, error) { - return "", nil -} - -func (t *testParser) GetElements(*plugin.Context, any, string) ([]string, error) { - return nil, nil -} - -func TestRegister(t *testing.T) { - t.Parallel() - if _, ok := GetParser("test"); !ok { - Register("test", new(testParser)) - } - if _, ok := GetParser("test"); !ok { - t.Fatal("unable get parser") - } -} diff --git a/plugin/plugin.go b/plugin/plugin.go deleted file mode 100644 index 86a5ea0..0000000 --- a/plugin/plugin.go +++ /dev/null @@ -1,8 +0,0 @@ -package plugin - -import ( - "github.com/shiroyk/cloudcat/plugin/internal/ext" -) - -// GetAll returns all plugins. -func GetAll() []*ext.Extension { return ext.GetAll() } diff --git a/plugin/plugin_unix.go b/plugin/plugin_unix.go deleted file mode 100644 index 9b61dbd..0000000 --- a/plugin/plugin_unix.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build unix - -package plugin - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "plugin" -) - -func LoadPlugin(dir string) (size int, err error) { - entries, err := os.ReadDir(dir) - if err != nil { - return size, err - } - loadErr := make([]error, 0) - for _, entry := range entries { - if entry.IsDir() || filepath.Ext(entry.Name()) != ".so" { - continue - } - _, err = plugin.Open(filepath.Join(dir, entry.Name())) - if err != nil { - loadErr = append(loadErr, fmt.Errorf("error opening %s: %v", entry.Name(), err)) - continue - } - size++ - } - return size, errors.Join(loadErr...) -} diff --git a/plugin/plugin_windows.go b/plugin/plugin_windows.go deleted file mode 100644 index 0e5eac8..0000000 --- a/plugin/plugin_windows.go +++ /dev/null @@ -1,12 +0,0 @@ -//go:build windows - -package plugin - -import ( - "log/slog" -) - -func LoadPlugin(dir string) (size int, err error) { - slog.Warn("plugin are only supported on Linux, FreeBSD, and macOS. see https://pkg.go.dev/plugin") - return -} diff --git a/sample/env/README.md b/sample/env/README.md deleted file mode 100644 index 6445e45..0000000 --- a/sample/env/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Env Plugin -A cloudcat js plugin for reading environment variables. -### Build the plugin -```shell -go build -buildmode=plugin -o env.so -``` -### Plugin usage -```shell -export FOO=BAR -cat << EOF | cloudcat -p $(pwd) -d -s - -require("cloudcat/env").get("FOO") -EOF -# "BAR" -``` \ No newline at end of file diff --git a/sample/env/env.go b/sample/env/env.go deleted file mode 100644 index 2e356a9..0000000 --- a/sample/env/env.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "os" - - "github.com/shiroyk/cloudcat/plugin/jsmodule" -) - -type Module struct{} - -func (m Module) Exports() any { return new(Env) } - -func init() { - jsmodule.Register("env", new(Module)) -} - -type Env struct{} - -func (e Env) Get(key string) string { return os.Getenv(key) } diff --git a/sample/prefix/README.md b/sample/prefix/README.md deleted file mode 100644 index b00b5cd..0000000 --- a/sample/prefix/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Prefix Plugin -A cloudcat parser plugin for adding string prefix. -### Build the plugin -```shell -go build -buildmode=plugin -o prefix.so -``` -### Plugin usage -```shell -cat << EOF | cloudcat -p $(pwd) run -s - -cat.getString("prefix", "...", "test"); -EOF -# "...test" -``` \ No newline at end of file diff --git a/sample/prefix/prefix.go b/sample/prefix/prefix.go deleted file mode 100644 index 195667e..0000000 --- a/sample/prefix/prefix.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" -) - -type Parser struct{} - -func init() { - parser.Register("prefix", new(Parser)) -} - -func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - if str, ok := content.(string); ok { - return arg + str, nil - } - return "", fmt.Errorf("content must be a string") -} - -func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - if str, ok := content.([]string); ok { - for i := range str { - str[i] = arg + str[i] - } - } - return nil, fmt.Errorf("content must be a string slice") -} - -func (p Parser) GetElement(_ *plugin.Context, content any, arg string) (string, error) { - return p.GetString(nil, content, arg) -} - -func (p Parser) GetElements(_ *plugin.Context, content any, arg string) ([]string, error) { - return p.GetStrings(nil, content, arg) -} diff --git a/schema.go b/schema.go index 96d5e2e..c192705 100644 --- a/schema.go +++ b/schema.go @@ -1,581 +1,502 @@ -package cloudcat +package ski import ( + "context" + "encoding/json" "errors" "fmt" - "slices" + "log/slog" + "maps" "strings" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" + "github.com/spf13/cast" "gopkg.in/yaml.v3" ) -var ( - // ErrInvalidSchema invalid schema error - ErrInvalidSchema = errors.New("invalid schema") - // ErrInvalidAction invalid action error - ErrInvalidAction = errors.New("invalid action") - // ErrInvalidStep invalid step error - ErrInvalidStep = errors.New("invalid step") -) - -// Type The property type. -type Type string +type Kind uint const ( - // StringType The Type of string. - StringType Type = "string" - // NumberType The Type of number. - NumberType Type = "number" - // IntegerType The Type of integer. - IntegerType Type = "integer" - // BooleanType The Type of boolean. - BooleanType Type = "boolean" - // ObjectType The Type of object. - ObjectType Type = "object" - // ArrayType The Type of array. - ArrayType Type = "array" + KindAny Kind = iota + KindBool + KindInt // int32 + KindInt64 + KindFloat // float 32 + KindFloat64 + KindString ) -// ToType parses the schema type. -func ToType(s any) (Type, error) { - switch s { +var kindNames = [...]string{ + KindAny: "any", + KindBool: "bool", + KindInt: "int", + KindInt64: "int64", + KindFloat: "float", + KindFloat64: "float64", + KindString: "string", +} + +func (k Kind) String() string { return kindNames[k] } + +func (k Kind) MarshalText() (text []byte, err error) { return []byte(kindNames[k]), nil } + +func (k *Kind) UnmarshalText(text []byte) error { + switch string(text) { + case "", "any": + *k = KindAny + case "bool": + *k = KindBool + case "int", "int32": + *k = KindInt + case "int64": + *k = KindInt64 + case "float", "float32": + *k = KindFloat + case "float64": + *k = KindFloat64 case "string": - return StringType, nil - case "array": - return ArrayType, nil - case "object": - return ObjectType, nil - case "number": - return NumberType, nil - case "integer": - return IntegerType, nil - case "boolean": - return BooleanType, nil + *k = KindString + default: + return fmt.Errorf("unknown kind %s", text) } - return "", fmt.Errorf("invalid type %s", s) + return nil } -// Operator The Action operator. -type Operator string +func (k Kind) Exec(_ context.Context, v any) (any, error) { + switch k { + case KindBool: + return cast.ToBoolE(v) + case KindInt: + return cast.ToInt32E(v) + case KindInt64: + return cast.ToInt64E(v) + case KindFloat: + return cast.ToFloat32E(v) + case KindFloat64: + return cast.ToFloat64E(v) + case KindString: + return cast.ToStringE(v) + default: + return v, nil + } +} -const ( - // OperatorAnd The Operator of and. - // Action result A, B; Join the A + B. - OperatorAnd Operator = "and" - // OperatorOr The Operator of or. - // Action result A, B; if result A is nil return B else return A. - OperatorOr Operator = "or" - // OperatorNot The Operator of not. - // Action result A, B; if result A is not nil return B else return nil. - OperatorNot Operator = "not" +type ( + // Executor accept the argument and output result + Executor interface { + Exec(context.Context, any) (any, error) + } + + // ExecutorMap map of the Executor init function + ExecutorMap map[string]func(args ...Executor) (Executor, error) ) -// Schema The schema. -type Schema struct { - Type Type `yaml:"type"` - Format Type `yaml:"format,omitempty"` - Init Action `yaml:"init,omitempty"` - Rule Action `yaml:"rule,omitempty"` - Properties Property `yaml:"properties,omitempty"` +type compiler struct { + funcs ExecutorMap + meta func(node *yaml.Node, exec Executor, isParser bool) Executor } -// Property The Schema property. -type Property map[string]Schema - -// NewSchema returns a new Schema with the given Type. -// The first argument is the Schema.Type, second is the Schema.Format. -func NewSchema(types ...Type) *Schema { - switch { - case len(types) == 0: - panic("schema must have type") - case len(types) == 1: - return &Schema{ - Type: types[0], - } - default: - return &Schema{ - Type: types[0], - Format: types[1], - } +func (c compiler) newError(message string, node *yaml.Node, err error) error { + if err != nil { + message = fmt.Sprintf("%s: %s", message, err) } + return fmt.Errorf("line %d column %d %s", node.Line, node.Column, message) } -// SetProperty set the Property to Schema.Properties. -func (schema *Schema) SetProperty(m Property) *Schema { - schema.Properties = m - return schema +// compile the Executor from the YAML string. +func (c compiler) compile(str string) (Executor, error) { + node := new(yaml.Node) + if err := yaml.Unmarshal([]byte(str), node); err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML: %s", err) + } + if node.Kind != yaml.DocumentNode || len(node.Content) != 1 { + return nil, errors.New("invalid YAML schema: document node is missing or incorrect") + } + exec, err := c.compileNode(node.Content[0]) + if err != nil { + return nil, err + } + if len(exec) == 1 { + return exec[0], nil + } + return c.piping(exec), nil } -// AddProperty append a field string with Schema to Schema.Properties. -func (schema *Schema) AddProperty(field string, s Schema) *Schema { - if schema.Properties == nil { - property := make(map[string]Schema) - schema.Properties = property +// piping return the first arg if the length is 1, else return _pipe +func (c compiler) piping(args []Executor) Executor { + if len(args) == 1 { + return args[0] } + return _pipe(args) +} - schema.Properties[field] = s +// compileExecutor return the Executor with the key and values +func (c compiler) compileExecutor(k, v *yaml.Node) (Executor, error) { + key := strings.TrimPrefix(k.Value, "$") + init, ok := c.funcs[key] + if ok { + args, err := c.compileNode(v) + if err != nil { + return nil, err + } + exec, err := init(args...) + if err != nil { + return nil, c.newError(key, k, err) + } + if c.meta != nil { + return c.meta(k, exec, false), nil + } + return exec, nil + } - return schema -} + name, method, found := strings.Cut(key, ".") + parser, ok := GetParser(name) + if !ok { + return nil, c.newError("function or parser not found", k, errors.New(key)) + } -// SetInit set the Init Action to Schema.Init. -func (schema *Schema) SetInit(action Action) *Schema { - schema.Init = action - return schema -} + var ( + exec Executor + err error + ) -// SetRule set the Init Action to Schema.Rule. -func (schema *Schema) SetRule(action Action) *Schema { - schema.Rule = action - return schema -} + if found && method != "value" { + parser, ok := parser.(ElementParser) + if !ok { + return nil, c.newError("method not found", k, errors.New(key)) + } + switch method { + case "element": + exec, err = parser.Element(v.Value) + case "elements": + exec, err = parser.Elements(v.Value) + default: + return nil, c.newError("method not found", k, errors.New(key)) + } + } else { + exec, err = parser.Value(v.Value) + } -// UnmarshalYAML decodes the Schema from yaml -func (schema *Schema) UnmarshalYAML(node *yaml.Node) (err error) { - *schema, err = buildSchema(node) - return + if err != nil { + return nil, c.newError(key, k, err) + } + if c.meta != nil { + return c.meta(k, exec, true), nil + } + return exec, nil } -// buildSchema builds a Schema -func buildSchema(node *yaml.Node) (schema Schema, err error) { +func (c compiler) compileNode(node *yaml.Node) ([]Executor, error) { switch node.Kind { - case yaml.SequenceNode: - return buildStringSchema(node) case yaml.MappingNode: - typed := slices.ContainsFunc(node.Content, - func(node *yaml.Node) bool { - return node.Value == "type" - }) - if typed { - return buildTypedSchema(node) - } - return buildStringSchema(node) + return c.compileMapping(node) + case yaml.SequenceNode: + return c.compileSequence(node) + case yaml.ScalarNode: + return []Executor{String(node.Value)}, nil case yaml.AliasNode: - return buildSchema(node.Alias) + return c.compileNode(node.Alias) default: - err = ErrInvalidSchema + return nil, c.newError("invalid node type", node, nil) } - return } -// buildStringSchema builds a StringType Schema -func buildStringSchema(node *yaml.Node) (schema Schema, err error) { - schema.Type = StringType - schema.Rule, err = actionDecode(node) - if err != nil { - return - } - if len(node.Tag) > 2 && node.Tag[0] == '!' && node.Tag[1] != '!' { - tags := strings.Split(node.Tag[1:], "/") - schema.Type, err = ToType(tags[0]) - if len(tags) > 1 { - schema.Format, err = ToType(tags[1]) - } +func (c compiler) compileSequence(node *yaml.Node) ([]Executor, error) { + args := make([]Executor, 0, len(node.Content)) + for _, item := range node.Content { + items, err := c.compileNode(item) if err != nil { - return schema, fmt.Errorf("invalid tag %s", node.Tag) + return nil, err } + args = append(args, c.piping(items)) } - return + return args, nil } -// buildTypedSchema builds a specific Type Schema -// -//nolint:nakedret -func buildTypedSchema(node *yaml.Node) (schema Schema, err error) { +func (c compiler) compileMapping(node *yaml.Node) ([]Executor, error) { + if len(node.Content) == 0 || len(node.Content)%2 != 0 { + return nil, c.newError("mapping node requires at least two elements", node, nil) + } + + if strings.HasPrefix(node.Content[0].Value, "$") { + ret := make([]Executor, 0, len(node.Content)/2) + for i := 0; i < len(node.Content); i += 2 { + exec, err := c.compileExecutor(node.Content[i], node.Content[i+1]) + if err != nil { + return nil, err + } + ret = append(ret, exec) + } + return ret, nil + } + + ret := make([]Executor, 0, len(node.Content)/2) for i := 0; i < len(node.Content); i += 2 { - field, value := node.Content[i], node.Content[i+1] - switch field.Value { - case "type": - schema.Type, err = ToType(value.Value) - case "format": - schema.Format, err = ToType(value.Value) - case "init": - schema.Init, err = actionDecode(value) - case "rule": - schema.Rule, err = actionDecode(value) - case "properties": - if len(value.Content) == 2 { - schema.Properties = make(Property, 2) - k, v := value.Content[0], value.Content[1] - if k.Kind == yaml.MappingNode { - schema.Properties["$key"], err = buildSchema(k) - schema.Properties["$value"], err = buildSchema(v) - return - } - schema.Properties[k.Value], err = buildSchema(v) - return + keyNode, valueNode := node.Content[i], node.Content[i+1] + key := String(keyNode.Value) + + if valueNode.Kind != yaml.MappingNode { + child, err := c.compileNode(valueNode) + if err != nil { + return nil, err } - schema.Properties = make(Property, len(value.Content)/2) - for j := 0; j < len(value.Content); j += 2 { - k, v := value.Content[j], value.Content[j+1] - schema.Properties[k.Value], err = buildSchema(v) - if err != nil { - return - } + ret = append(ret, key, c.piping(child)) + continue + } + + if len(valueNode.Content) == 2 { + exec, err := c.compileExecutor(valueNode.Content[0], valueNode.Content[1]) + if err != nil { + return nil, err } + ret = append(ret, key, exec) + continue } - if err != nil { - return + + pipe := make(_pipe, 0, len(valueNode.Content)/2) + for j := 0; j < len(valueNode.Content); j += 2 { + exec, err := c.compileExecutor(valueNode.Content[j], valueNode.Content[j+1]) + if err != nil { + return nil, err + } + pipe = append(pipe, exec) } + ret = append(ret, key, pipe) } - - return + return ret, nil } -// MarshalYAML encodes the Schema -func (schema Schema) MarshalYAML() (any, error) { - if schema.Type == StringType && - schema.Init == nil { - return schema.Rule, nil - } - s := make(map[string]any, 5) - s["type"] = schema.Type - if schema.Format != "" { - s["format"] = schema.Format - } - if schema.Init != nil { - s["init"] = schema.Init - } - if schema.Rule != nil { - s["rule"] = schema.Rule - } - if len(schema.Properties) > 0 { - s["properties"] = schema.Properties - } - return s, nil -} +type Option func(*compiler) -// MarshalText encodes the receiver into UTF-8-encoded text and returns the result. -func (schema Schema) MarshalText() ([]byte, error) { - if schema.Type == "" { - return nil, nil +// WithExecutorMap external ExecutorMap +func WithExecutorMap(fn ExecutorMap) Option { + return func(parser *compiler) { + funcs := maps.Clone(buildInExecutor) + maps.Copy(funcs, fn) + parser.funcs = funcs } - return yaml.Marshal(schema) } -// UnmarshalText must be able to decode the form generated by MarshalText. -func (schema *Schema) UnmarshalText(text []byte) error { - return yaml.Unmarshal(text, schema) -} +type Meta = func(node *yaml.Node, exec Executor, isParser bool) Executor -// Action The Schema Action -type Action interface { - // Left returns the left Action - Left() Action - // Right returns the right Action - Right() Action +// WithMeta with the meta message +func WithMeta(meta Meta) Option { + return func(parser *compiler) { parser.meta = meta } } -// Step The Action of step -type Step struct{ K, V string } +// Compile the Executor with the Option. +func Compile(str string, opts ...Option) (Executor, error) { + c := new(compiler) + for _, opt := range opts { + opt(c) + } + if c.funcs == nil { + c.funcs = buildInExecutor + } + return c.compile(str) +} -// MarshalYAML encodes to yaml -func (s Step) MarshalYAML() (any, error) { - return map[string]string{s.K: s.V}, nil +var buildInExecutor = ExecutorMap{ + "debug": func(args ...Executor) (Executor, error) { + if len(args) > 0 { + return _debug(ToString(args[0])), nil + } + return _debug(""), nil + }, + "kind": func(args ...Executor) (Executor, error) { + if len(args) != 1 { + return nil, errors.New("kind needs 1 parameter") + } + var k Kind + return k, k.UnmarshalText([]byte(ToString(args[0]))) + }, + "each": func(args ...Executor) (Executor, error) { + if len(args) != 1 { + return nil, errors.New("each needs 1 parameter") + } + return _each{args[0]}, nil + }, + "json.parse": func(args ...Executor) (Executor, error) { return _jsonParse{}, nil }, + "json.string": func(args ...Executor) (Executor, error) { return _jsonString{}, nil }, + "map": func(args ...Executor) (Executor, error) { + kv := _map(args) + if len(kv)%2 != 0 { + kv = append(kv, Raw(nil)) + } + return kv, nil + }, + "or": func(args ...Executor) (Executor, error) { return _or(args), nil }, + "string.join": func(args ...Executor) (Executor, error) { + if len(args) > 0 { + return _stringJoin(ToString(args[0])), nil + } + return _stringJoin(""), nil + }, + "pipe": func(args ...Executor) (Executor, error) { return _pipe(args), nil }, } -// Steps slice of Step -type Steps []Step +// String the Executor for string value +type String string -// NewSteps return new Steps -func NewSteps(str ...string) *Steps { - if len(str)%2 != 0 { - panic(ErrInvalidStep) - } - steps := make(Steps, 0, len(str)/2) - for i := 0; i < len(str); i += 2 { - steps = append(steps, Step{str[i], str[i+1]}) - } - return &steps -} +func (k String) Exec(_ context.Context, _ any) (any, error) { return k.String(), nil } -// Left returns the left Action -func (s *Steps) Left() Action { return nil } +func (k String) String() string { return string(k) } -// Right returns the right Action -func (s *Steps) Right() Action { return nil } +type _map []Executor -func (s *Steps) UnmarshalYAML(value *yaml.Node) error { - switch value.Kind { - case yaml.MappingNode: - *s = make(Steps, 0, len(value.Content)/2) - for i := 0; i < len(value.Content); i += 2 { - k, v := value.Content[i], value.Content[i+1] - if v.Kind == yaml.AliasNode { - v = v.Alias +func (m _map) Exec(ctx context.Context, arg any) (any, error) { + var ret map[string]any + + exec := func(a any) { + for i := 0; i < len(m); i += 2 { + k, err := m[i].Exec(ctx, a) + if err != nil { + continue } - if k.Kind != yaml.ScalarNode || v.Kind != yaml.ScalarNode { - return ErrInvalidStep + ks, err := cast.ToStringE(k) + if err != nil { + continue } - *s = append(*s, Step{k.Value, v.Value}) + v, _ := m[i+1].Exec(ctx, a) + ret[ks] = v } - case yaml.SequenceNode: - *s = make(Steps, 0, len(value.Content)) - for _, node := range value.Content { - if node.Kind != yaml.MappingNode { - return ErrInvalidStep - } - steps := new(Steps) - if err := node.Decode(steps); err != nil { - return err - } - *s = append(*s, *steps...) + } + + switch s := arg.(type) { + case []any: + ret = make(map[string]any, len(s)) + for _, a := range s { + exec(a) + } + return ret, nil + case []string: + ret = make(map[string]any, len(s)) + for _, a := range s { + exec(a) } + return ret, nil default: - return ErrInvalidStep + ret = make(map[string]any, len(m)/2) + exec(arg) + return ret, nil } - return nil } -func (s Steps) MarshalYAML() (any, error) { - if len(s) == 1 { - return s[0], nil - } - return s, nil -} +type _each struct{ Executor } -// And Action node of Operator and -type And struct{ l, r Action } - -// NewAnd create new And action with left and right Action -func NewAnd(left, right Action) *And { return &And{left, right} } -func (a And) Left() Action { return a.l } -func (a And) Right() Action { return a.r } -func (a And) String() string { return string(OperatorAnd) } -func (a And) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil } - -// Or Action node of Operator or -type Or struct{ l, r Action } - -// NewOr create new Or action with left and right Action -func NewOr(left, right Action) *Or { return &Or{left, right} } -func (a Or) Left() Action { return a.l } -func (a Or) Right() Action { return a.r } -func (a Or) String() string { return string(OperatorOr) } -func (a Or) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil } - -// Not Action node of Operator not -type Not struct{ l, r Action } - -// NewNot create new Not action with left and right Action -func NewNot(left, right Action) *Not { return &Not{left, right} } -func (a Not) Left() Action { return a.l } -func (a Not) Right() Action { return a.r } -func (a Not) String() string { return string(OperatorNot) } -func (a Not) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil } - -// actionDecode decodes the Action from yaml.node -func actionDecode(value *yaml.Node) (ret Action, err error) { - if value.Kind == yaml.DocumentNode { - return nil, ErrInvalidAction - } - if value.Kind == yaml.AliasNode { - value = value.Alias - } - multiStep := value.Kind == yaml.SequenceNode && - !slices.ContainsFunc(value.Content, func(e *yaml.Node) bool { - return e.Kind == yaml.ScalarNode - }) - if value.Kind == yaml.MappingNode || multiStep { - steps := new(Steps) - if err = value.Decode(steps); err != nil { - return +func (each _each) Exec(ctx context.Context, arg any) (any, error) { + switch s := arg.(type) { + case []any: + ret := make([]any, 0, len(s)) + for _, i := range s { + v, _ := each.Executor.Exec(ctx, i) + ret = append(ret, v) } - return steps, nil - } - - var op string - var left Action - for _, node := range value.Content { - switch node.Kind { - case yaml.MappingNode, yaml.SequenceNode: - var leaf Action - leaf, err = actionDecode(node) - if err != nil { - return - } - if left == nil { - left = leaf - continue - } - ret, err = toActionOp(op, left, leaf) - left = nil - case yaml.ScalarNode: - op = node.Value - default: - continue + return ret, nil + case []string: + ret := make([]any, 0, len(s)) + for _, i := range s { + v, _ := each.Executor.Exec(ctx, i) + ret = append(ret, v) } - + return ret, nil + default: + v, err := each.Executor.Exec(ctx, arg) if err != nil { - return + return []any{}, nil } + return []any{v}, nil } +} - if left != nil { - return toActionOp(op, ret, left) - } +// Raw the Executor for raw value, return the original value +func Raw(arg any) Executor { return _raw{arg} } - return -} +type _raw struct{ any } + +func (raw _raw) Exec(context.Context, any) (any, error) { return raw.any, nil } -// toActionOp parser the Operator string returns an operator Action -func toActionOp(op string, left, right Action) (Action, error) { - switch Operator(strings.ToLower(op)) { - case OperatorAnd: - return NewAnd(left, right), nil - case OperatorOr: - return NewOr(left, right), nil - case OperatorNot: - return NewNot(left, right), nil +type _pipe []Executor + +func (pipe _pipe) Exec(ctx context.Context, v any) (any, error) { + switch len(pipe) { + case 0: + return nil, nil + case 1: + return pipe[0].Exec(ctx, v) default: - return nil, fmt.Errorf("invalid operator %v", op) + ret, err := pipe[0].Exec(ctx, v) + if err != nil { + return nil, err + } + for _, s := range pipe[1:] { + ret, err = s.Exec(ctx, ret) + if err != nil { + return nil, err + } + } + return ret, nil } } -// GetString run the action returns a string -func GetString(ctx *plugin.Context, node Action, content any) (string, error) { - ret, err := WalkAction(node, func(steps Steps) ([]string, error) { - var err error - cur := content - for _, step := range steps { - p, ok := parser.GetParser(step.K) - if !ok { - return nil, fmt.Errorf("parser %s not found", step.K) - } - cur, err = p.GetString(ctx, cur, step.V) - if err != nil { - return nil, fmt.Errorf("parser %s: %s", step.K, err) - } +type _or []Executor + +func (or _or) Exec(ctx context.Context, arg any) (any, error) { + for _, exec := range or { + v, err := exec.Exec(ctx, arg) + if err != nil { + continue } - ret := cur.(string) - if len(ret) == 0 { - return nil, nil + if v != nil { + return v, nil } - return []string{ret}, nil - }) - if err != nil { - return "", err } - return strings.Join(ret, ""), nil + return nil, nil } -// GetStrings run the action returns a slice of string -func GetStrings(ctx *plugin.Context, node Action, content any) ([]string, error) { - return WalkAction(node, func(steps Steps) ([]string, error) { - var err error - ret := content - for _, step := range steps { - p, ok := parser.GetParser(step.K) - if !ok { - return nil, fmt.Errorf("parser %s not found", step.K) - } - ret, err = p.GetStrings(ctx, ret, step.V) - if err != nil { - return nil, fmt.Errorf("parser %s: %s", step.K, err) - } - } - return ret.([]string), nil - }) +type _debug string + +func (debug _debug) Exec(ctx context.Context, v any) (any, error) { + Logger(ctx).LogAttrs(ctx, slog.LevelDebug, string(debug), slog.Any("value", v)) + return v, nil } -// GetElement run the action returns an element string -func GetElement(ctx *plugin.Context, node Action, content any) (string, error) { - ret, err := WalkAction(node, func(steps Steps) ([]string, error) { - var err error - cur := content - for _, step := range steps { - p, ok := parser.GetParser(step.K) - if !ok { - return nil, fmt.Errorf("parser %s not found", step.K) - } - cur, err = p.GetElement(ctx, cur, step.V) - if err != nil { - return nil, fmt.Errorf("parser %s: %s", step.K, err) - } - } - ret := cur.(string) - if len(ret) == 0 { - return nil, nil +type _stringJoin string + +func (sep _stringJoin) Exec(_ context.Context, arg any) (any, error) { + switch s := arg.(type) { + case []any: + str, err := cast.ToStringSliceE(s) + if err != nil { + return nil, fmt.Errorf("expected string or []string, but got type %T", arg) } - return []string{ret}, nil - }) - if err != nil { - return "", err + return strings.Join(str, string(sep)), nil + case []string: + return strings.Join(s, string(sep)), nil + case string: + return s, nil + default: + return nil, fmt.Errorf("expected string or []string, but got type %T", arg) } - return strings.Join(ret, ""), nil } -// GetElements run the action returns a slice of element string -func GetElements(ctx *plugin.Context, node Action, content any) ([]string, error) { - return WalkAction(node, func(steps Steps) ([]string, error) { - var err error - ret := content - for _, step := range steps { - p, ok := parser.GetParser(step.K) - if !ok { - return nil, fmt.Errorf("parser %s not found", step.K) - } - ret, err = p.GetElements(ctx, ret, step.V) - if err != nil { - return nil, fmt.Errorf("parser %s: %s", step.K, err) - } - } - return ret.([]string), nil - }) +type _jsonParse struct{} + +func (_jsonParse) Exec(_ context.Context, v any) (any, error) { + s, err := cast.ToStringE(v) + if err != nil { + return nil, err + } + var ret any + err = json.Unmarshal([]byte(s), &ret) + return ret, err } -// WalkAction preorder traversal of Action. -func WalkAction(node Action, walk func(Steps) ([]string, error)) (ret []string, err error) { - var left []string - var stack []Action - for len(stack) > 0 || node != nil { - // traverse the left subtree and push the node to the stack - for node != nil { - stack = append(stack, node) - node = node.Left() - } - // pop the stack and walk the node - node = stack[len(stack)-1] - stack = stack[:len(stack)-1] - switch node.(type) { - case *And: - if len(stack) == 0 { - // join the left subtree result to ret - ret = append(ret, left...) - left = nil - } - case *Or: - if len(left) > 0 { - // discard right subtree if left subtree result is not empty - node = nil - if len(stack) == 0 { - ret = append(ret, left...) - left = nil - } - continue - } - case *Not: - if len(left) == 0 { - // discard right subtree if left subtree result is empty - node = nil - continue - } - left = nil - case *Steps: - var cur []string - cur, err = walk(*node.(*Steps)) - if err != nil { - return nil, err - } - if len(stack) == 0 { - ret = append(ret, cur...) - } else { - left = append(left, cur...) - } - } - node = node.Right() +type _jsonString struct{} + +func (_jsonString) Exec(_ context.Context, v any) (any, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err } - return + return string(data), nil } diff --git a/schema_test.go b/schema_test.go index 0dfe62b..3cb39c6 100644 --- a/schema_test.go +++ b/schema_test.go @@ -1,598 +1,155 @@ -package cloudcat +package ski import ( + "bytes" + "context" + "fmt" + "log/slog" "strconv" "testing" - "github.com/shiroyk/cloudcat/plugin" - "github.com/shiroyk/cloudcat/plugin/parser" + "github.com/spf13/cast" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) -func TestSchemaYaml(t *testing.T) { - t.Parallel() +type _inc struct{} +func (_inc) Exec(_ context.Context, v any) (ret any, err error) { + return cast.ToInt(v) + 1, nil +} + +type _dec struct{} + +func (_dec) Exec(_ context.Context, v any) (ret any, err error) { + return cast.ToInt(v) - 1, nil +} + +func TestExecutor(t *testing.T) { + ctx := context.WithValue(context.Background(), "foo", "bar") testCases := []struct { - Yaml string - Schema *Schema + e Executor + arg any + want any }{ - { - ` -{ p: foo }`, NewSchema(StringType). - SetRule(NewSteps("p", "foo")), - }, - { - ` -- p: foo - p: bar`, NewSchema(StringType). - SetRule(NewSteps("p", "foo", "p", "bar")), - }, - { - ` -- p: foo -- p: bar -- p: title`, NewSchema(StringType). - SetRule(NewSteps("p", "foo", "p", "bar", "p", "title")), - }, - { - ` -- p: foo -- not -- p: title`, NewSchema(StringType). - SetRule(NewNot(NewSteps("p", "foo"), NewSteps("p", "title"))), - }, - { - ` -- p: foo -- or -- p: title`, NewSchema(StringType). - SetRule(NewOr(NewSteps("p", "foo"), NewSteps("p", "title"))), - }, - { - ` -- p: foo -- and -- p: bar -- or -- p: body`, NewSchema(StringType). - SetRule(NewOr( - NewAnd(NewSteps("p", "foo"), NewSteps("p", "bar")), - NewSteps("p", "body"), - )), - }, - { - ` -- p: foo -- not -- p: bar -- or -- p: body`, NewSchema(StringType). - SetRule(NewOr( - NewNot(NewSteps("p", "foo"), NewSteps("p", "bar")), - NewSteps("p", "body"), - )), - }, - { - ` -- - p: foo - - p: bar -- or -- - p: title - - p: body`, NewSchema(StringType). - SetRule(NewOr(NewSteps("p", "foo", "p", "bar"), NewSteps("p", "title", "p", "body"))), - }, - { - ` -!integer { p: foo }`, NewSchema(IntegerType). - SetRule(NewSteps("p", "foo")), - }, - { - ` -type: integer -rule: { p: foo }`, NewSchema(IntegerType). - SetRule(NewSteps("p", "foo")), - }, - { - ` -type: number -rule: { p: foo }`, NewSchema(NumberType). - SetRule(NewSteps("p", "foo")), - }, - { - ` -type: boolean -rule: { p: foo }`, NewSchema(BooleanType). - SetRule(NewSteps("p", "foo")), - }, - { - ` -type: object -properties: - context: - type: string - format: boolean - rule: { p: foo }`, NewSchema(ObjectType). - AddProperty("context", *NewSchema(StringType, BooleanType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -properties: - context: !string/boolean { p: foo }`, NewSchema(ObjectType). - AddProperty("context", *NewSchema(StringType, BooleanType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: array -init: { p: foo } -properties: - context: - type: string - format: integer - rule: { p: foo }`, NewSchema(ArrayType). - SetInit(NewSteps("p", "foo")). - AddProperty("context", *NewSchema(StringType, IntegerType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -init: { p: foo } -properties: - context: - type: number - rule: { p: foo }`, NewSchema(ObjectType). - SetInit(NewSteps("p", "foo")). - AddProperty("context", *NewSchema(NumberType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -init: { p: foo } -properties: - context: !number { p: foo }`, NewSchema(ObjectType). - SetInit(NewSteps("p", "foo")). - AddProperty("context", *NewSchema(NumberType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -properties: - ? p: foo - : p: bar`, NewSchema(ObjectType). - AddProperty("$key", *NewSchema(StringType). - SetRule(NewSteps("p", "foo"))). - AddProperty("$value", *NewSchema(StringType). - SetRule(NewSteps("p", "bar"))), - }, - { - ` -type: object -properties: - ? p: foo - : type: integer - rule: { p: bar }`, NewSchema(ObjectType). - AddProperty("$key", *NewSchema(StringType). - SetRule(NewSteps("p", "foo"))). - AddProperty("$value", *NewSchema(IntegerType). - SetRule(NewSteps("p", "bar"))), - }, - { - ` -type: object -properties: - $key: { p: foo } - $value: { p: bar }`, NewSchema(ObjectType). - AddProperty("$key", *NewSchema(StringType). - SetRule(NewSteps("p", "foo"))). - AddProperty("$value", *NewSchema(StringType). - SetRule(NewSteps("p", "bar"))), - }, - { - ` -type: object -init: - - p: foo - - not - - p: bar -properties: - context: - type: number - rule: { p: foo }`, NewSchema(ObjectType). - SetInit(NewNot(NewSteps("p", "foo"), NewSteps("p", "bar"))). - AddProperty("context", *NewSchema(NumberType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -init: - - p: foo - - or - - p: bar -properties: - context: - type: number - rule: { p: foo }`, NewSchema(ObjectType). - SetInit(NewOr(NewSteps("p", "foo"), NewSteps("p", "bar"))). - AddProperty("context", *NewSchema(NumberType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -init: - - p: foo - - or - - p: bar -properties: - a: !number { p: foo } - b: - type: boolean - rule: { p: foo }`, NewSchema(ObjectType). - SetInit(NewOr(NewSteps("p", "foo"), NewSteps("p", "bar"))). - AddProperty("a", *NewSchema(NumberType). - SetRule(NewSteps("p", "foo"))). - AddProperty("b", *NewSchema(BooleanType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -properties: - a: !boolean { p: &p foo } - b: { p: *p }`, NewSchema(ObjectType). - AddProperty("a", *NewSchema(BooleanType). - SetRule(NewSteps("p", "foo"))). - AddProperty("b", *NewSchema(StringType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -properties: - a: &a - type: boolean - rule: { p: foo } - b: *a`, NewSchema(ObjectType). - AddProperty("a", *NewSchema(BooleanType). - SetRule(NewSteps("p", "foo"))). - AddProperty("b", *NewSchema(BooleanType). - SetRule(NewSteps("p", "foo"))), - }, - { - ` -type: object -properties: - a: - type: array - rule: &a - - { p: foo } - - and - - { p: foo } - b: - type: array - rule: *a`, NewSchema(ObjectType). - AddProperty("a", *NewSchema(ArrayType). - SetRule(NewAnd(NewSteps("p", "foo"), NewSteps("p", "foo")))). - AddProperty("b", *NewSchema(ArrayType). - SetRule(NewAnd(NewSteps("p", "foo"), NewSteps("p", "foo")))), - }, + {_raw{nil}, 1, nil}, + {_map{_raw{"k"}, _raw{nil}}, 1, map[string]any{"k": nil}}, + {_pipe{_inc{}, _inc{}, _dec{}}, 1, 2}, + {KindInt, "1", int32(1)}, + {_or{_raw{nil}, _raw{"b"}}, nil, "b"}, + {_map{_raw{"k"}, _inc{}}, 0, map[string]any{"k": 1}}, + {_each{_inc{}}, []string{"1", "2", "3"}, []any{2, 3, 4}}, + {_map{_raw{"k"}, _inc{}}, []any{1}, map[string]any{"k": 2}}, + {_stringJoin(""), []string{"1", "2", "3"}, "123"}, + {_pipe{_each{KindString}, _stringJoin("")}, []any{1, 2, 3}, "123"}, + {_pipe{_each{_inc{}}, _each{_inc{}}}, []any{1, 2, 3}, []any{3, 4, 5}}, + {_each{_map{_raw{"k"}, _inc{}}}, []any{1}, []any{map[string]any{"k": 2}}}, + {_map{_raw{"k"}, _jsonParse{}}, `{"foo": "bar"}`, map[string]any{"k": map[string]any{"foo": "bar"}}}, } - - for i, test := range testCases { + for i, c := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { - s := new(Schema) - err := yaml.Unmarshal([]byte(test.Yaml), s) - if err != nil { - t.Fatal(err) + v, err := c.e.Exec(ctx, c.arg) + if assert.NoError(t, err) { + assert.Equal(t, c.want, v) } - assert.Equal(t, test.Schema, s) }) } } -type testParser struct{} - -func (t *testParser) GetString(_ *plugin.Context, content any, arg string) (string, error) { - if str, ok := content.(string); ok { - if str == arg { - return str, nil - } +func TestDebug(t *testing.T) { + data := new(bytes.Buffer) + ctx := WithLogger(context.Background(), slog.New(slog.NewTextHandler(data, &slog.HandlerOptions{Level: slog.LevelDebug}))) + v, err := _pipe{_debug("before"), _inc{}, _debug("after")}.Exec(ctx, 1) + if assert.NoError(t, err) { + assert.Equal(t, v, 2) } - return "", nil + assert.Regexp(t, "msg=before value=1 | msg=after value=2", data.String()) } -func (t *testParser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) { - if str, ok := content.(string); ok { - if str == arg { - return []string{str}, nil - } - } - return nil, nil +type meta struct { + exec Executor + line, column int } -func (t *testParser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) { - return t.GetString(ctx, content, arg) +func (m *meta) Exec(ctx context.Context, arg any) (any, error) { + v, err := m.exec.Exec(ctx, arg) + if err != nil { + return nil, fmt.Errorf("line %d column %d: %s", m.line, m.column, err) + } + return v, nil } -func (t *testParser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) { - return t.GetStrings(ctx, content, arg) -} +type errexec struct{} -type unknown struct{ act Action } +func (errexec) Exec(context.Context, any) (any, error) { + return nil, fmt.Errorf("some error") +} -func (u *unknown) UnmarshalYAML(value *yaml.Node) (err error) { - u.act, err = actionDecode(value) - return +func TestWithMetaWrap(t *testing.T) { + v, err := Compile(`$error: ...`, + WithMeta(func(node *yaml.Node, exec Executor, isParser bool) Executor { + return &meta{exec, node.Line, node.Column} + }), + WithExecutorMap(ExecutorMap{"error": func(args ...Executor) (Executor, error) { + return errexec{}, nil + }})) + if assert.NoError(t, err) { + _, err = v.Exec(context.Background(), ``) + assert.ErrorContains(t, err, "line 1 column 1: some error") + } } -func TestActions(t *testing.T) { - t.Parallel() +type p struct{} - parser.Register("act", new(testParser)) - ctx := plugin.NewContext(plugin.ContextOptions{}) +func (p) Value(string) (Executor, error) { return String("p.value"), nil } +func (p) Element(string) (Executor, error) { return String("p.element"), nil } +func (p) Elements(string) (Executor, error) { return String("p.elements"), nil } +func TestCompile(t *testing.T) { + Register("p", p{}) testCases := []struct { - acts string - want any - str bool + s string + e Executor }{ - { - ` -- act: 1 -`, `1`, true, - }, - { - ` -- act: 2 -`, ``, true, - }, - { - ` -- act: 1 -- and -- act: 1 -`, `11`, true, - }, - { - ` -- act: 1 -- not -- act: 1 -`, `1`, true, - }, - { - ` -- act: 2 -- not -- act: 1 -`, ``, true, - }, - { - ` -- act: 1 -- and -- act: 1 -- and -- act: 1 -`, `111`, true, - }, - { - ` -- act: 1 -- and -- act: 1 -`, []string{`1`, `1`}, false, - }, - { - ` -- act: 2 -- or -- act: 1 -`, `1`, true, - }, - { - ` -- act: 2 -- or -- act: 2 -- or -- act: 1 -`, `1`, true, - }, - { - ` -- act: 2 -- or -- act: 1 -`, []string{`1`}, false, - }, - { - ` -- act: 1 -- or -- act: 2 -- and -- act: 1 -`, `11`, true, - }, - { - ` -- act: 1 -- and -- act: 2 -- or -- act: 1 -`, `1`, true, - }, - { - ` -- act: 1 -- and -- act: 2 -- or -- act: 1 -`, []string{`1`}, false, - }, - { - ` -- - act: 1 - - and - - act: 1 -- and -- act: 2 -- or -- - act: 2 - - or - - act: 1 -`, `11`, true, - }, - { - ` -- - act: 2 - - or - - act: 1 -- and -- act: 1 -- or -- - act: 2 - - and - - act: 1 -`, `11`, true, - }, - { - ` -- - act: 2 - - or - - act: 1 -- or -- - act: 1 - - and - - act: 1 -`, `1`, true, - }, - { - ` -- - act: 1 - - or - - act: 2 -- or -- - act: 2 - - or - - act: 1 -`, `1`, true, - }, - { - ` -- - act: 2 - - and - - act: 2 -- or -- - act: 2 - - or - - act: 1 -`, `1`, true, - }, - { - ` -- - act: 2 - - and - - act: 1 -- and -- - act: 2 - - or - - act: 1 -`, `11`, true, - }, - { - ` -- - act: 2 - - or - - act: 1 -- not -- - act: 2 - - or - - act: 1 -`, `1`, true, - }, - { - ` -- - act: 2 - - and - - act: 2 -- not -- - act: 2 - - or - - act: 1 -`, ``, true, - }, - { - ` -- - act: 1 - - and - - act: 1 -- and -- - act: 2 - - or - - act: 2 - - or - - act: 1 -`, `111`, true, - }, - { - ` -- - act: 1 - - and - - act: 1 -- and -- act: 1 -- and -- - act: 1 - - and - - act: 1 -`, `11111`, true, - }, - { - ` -- - act: 1 - - and - - act: 1 -- and -- act: 1 -- and -- - act: 1 - - and - - act: 1 -`, []string{`1`, `1`, `1`, `1`, `1`}, false, - }, + {`$p: foo`, String("p.value")}, + {`$p.value: foo`, String("p.value")}, + {` +- $map: &alias + title: + $p: text +- $map: *alias`, + _pipe{ + _map{String("title"), String("p.value")}, + _map{String("title"), String("p.value")}}}, + {` +$map: + size: + $p: text + $debug: the size + $kind: int64`, _map{String("size"), + _pipe{String("p.value"), _debug("the size"), KindInt64}}}, + {` +$map: + size: + - $p: text + - $debug: the size + - $kind: int64`, _map{String("size"), + _pipe{String("p.value"), _debug("the size"), KindInt64}}}, + {` +$map: + size: + $pipe: + - $p: text + - $debug: the size + - $kind: int32`, _map{String("size"), + _pipe{String("p.value"), _debug("the size"), KindInt}}}, } - - for i, testCase := range testCases { + for i, c := range testCases { t.Run(strconv.Itoa(i), func(t *testing.T) { - u := new(unknown) - err := yaml.Unmarshal([]byte(testCase.acts), u) - if err != nil { - t.Fatal(err) - } - var result any - if testCase.str { - result, err = GetString(ctx, u.act, "1") - if err != nil { - t.Error(err) - } - } else { - result, err = GetStrings(ctx, u.act, "1") - if err != nil { - t.Error(err) - } + v, err := Compile(c.s) + if assert.NoError(t, err) { + assert.Equal(t, c.e, v) } - assert.EqualValues(t, testCase.want, result, testCase.acts) }) } } diff --git a/ski/README.md b/ski/README.md new file mode 100644 index 0000000..af0a095 --- /dev/null +++ b/ski/README.md @@ -0,0 +1,64 @@ +# ski + +## Install +```shell +go install github.com/shiroyk/ski/ski +``` + +## Run model +```shell +cat << 'EOF' | ski -m - +$fetch: https://news.ycombinator.com/best +$xpath.element: //*[@id="hnmain"]/tbody/tr[3]/td/table +$gq.elements: -> zip('.athing', '.athing + tr') +$each: + $map: + index: + $gq: .rank + $regex: /[^\d]/ + $kind: int + title: + $gq: .titleline>:first-child + by: + $gq: .hnuser + age: + $gq: .age + comments: + $gq: .subline>:last-child + $regex: /[^\d]/ + $kind: int +EOF +``` + +## Run script +```shell +cat << EOF | ski -s - +import http from "ski/http"; +import gq from "parser/gq"; + +export default () => { + let res = http.get('https://news.ycombinator.com/best'); + + const index = gq('.rank'); + const title = gq('.titleline>:first-child'); + const by = gq('.hnuser'); + const age = gq('.age'); + const comments = gq('.subline>:last-child'); + + const stories = gq.elements("#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')").exec(res.text()); + return stories?.reduce((acc, v, i, arr) => { + if (i % 2 === 0) { + let item = arr.slice(i, i + 2); + acc.push({ + index: parseInt(index.exec(item)?.replace(/[^\d]+/g, ''), 10), + title: title.exec(item), + by: by.exec(item), + age: age.exec(item), + comments: parseInt(comments.exec(item)?.replace(/[^\d]+/g, ''), 10) + }); + } + return acc; + }, []); +} +EOF +``` \ No newline at end of file diff --git a/ski/main.go b/ski/main.go new file mode 100644 index 0000000..3feabd3 --- /dev/null +++ b/ski/main.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/shiroyk/ski" + "github.com/shiroyk/ski/js" + + _ "github.com/shiroyk/ski/js/modules/cache" + _ "github.com/shiroyk/ski/js/modules/crypto" + _ "github.com/shiroyk/ski/js/modules/encoding" + _ "github.com/shiroyk/ski/js/modules/http" + + _ "github.com/shiroyk/ski/parsers/gq" + _ "github.com/shiroyk/ski/parsers/jq" + _ "github.com/shiroyk/ski/parsers/regex" + _ "github.com/shiroyk/ski/parsers/xpath" +) + +const defaultTimeout = time.Minute + +var ( + scriptFlag = flag.String("s", "", "run script") + modelFlag = flag.String("m", "", "run model") + timeoutFlag = flag.Duration("t", defaultTimeout, "run timeout") + outputFlag = flag.String("o", "", "write to file instead of stdout") + versionFlag = flag.Bool("v", false, "output version") +) + +type _fetch string + +func (f _fetch) Exec(ctx context.Context, _ any) (any, error) { + method, url, found := strings.Cut(string(f), " ") + if !found { + url = string(f) + method = http.MethodGet + } + + req, err := http.NewRequestWithContext(ctx, method, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "ski") + + res, err := ski.NewFetch().Do(req) + if err != nil { + return nil, err + } + + defer res.Body.Close() + data, err := io.ReadAll(res.Body) + if err != nil { + return nil, err + } + return string(data), nil +} + +func runModel() (err error) { + var bytes []byte + if *modelFlag == "-" { + bytes, err = io.ReadAll(os.Stdin) + } else { + bytes, err = os.ReadFile(*modelFlag) //nolint:gosec + } + if err != nil { + return + } + fmt.Println(string(bytes)) + + executor, err := ski.Compile(string(bytes), + ski.WithExecutorMap(ski.ExecutorMap{ + "fetch": func(args ...ski.Executor) (ski.Executor, error) { + if len(args) != 1 { + return nil, errors.New("fetch needs 1 parameter") + } + return _fetch(ski.ToString(args[0])), nil + }, + })) + if err != nil { + return err + } + + timeout := defaultTimeout + if timeoutFlag != nil { + timeout = *timeoutFlag + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + ret, err := executor.Exec(ski.WithLogger(ctx, slog.New(loggerHandler())), nil) + if err != nil { + return err + } + + return outputJSON(ret) +} + +func runScript() (err error) { + var bytes []byte + if *scriptFlag == "-" { + bytes, err = io.ReadAll(os.Stdin) + } else { + bytes, err = os.ReadFile(*scriptFlag) //nolint:gosec + } + if err != nil { + return + } + + timeout := defaultTimeout + if timeoutFlag != nil { + timeout = *timeoutFlag + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + ctx = ski.NewContext(ctx, nil) + + vm, err := js.GetScheduler().Get() + if err != nil { + return err + } + module, err := vm.Loader().CompileModule("js", string(bytes)) + if err != nil { + return err + } + + ret, err := vm.RunModule(ski.WithLogger(ctx, slog.New(loggerHandler())), module) + if err != nil { + return err + } + + v, err := js.Unwrap(ret) + if err != nil { + return err + } + + return outputJSON(v) +} + +func loggerHandler() slog.Handler { + return slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}) +} + +func outputJSON(data any) (err error) { + bytes, err := json.MarshalIndent(data, "", "\t") + if err != nil { + return err + } + + if *outputFlag == "" { + fmt.Println(string(bytes)) //nolint:forbidigo + return + } + + ext := filepath.Ext(*outputFlag) + if ext == "" { + *outputFlag += ".json" + } + return os.WriteFile(*outputFlag, bytes, 0o600) +} + +func main() { + flag.Parse() + + if *versionFlag { + fmt.Println(fmt.Sprintf("ski %v/%v", Version, CommitSHA)) + os.Exit(0) + return + } + + if scriptFlag != nil && *scriptFlag != "" { + if err := runScript(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + } else if modelFlag != nil && *modelFlag != "" { + if err := runModel(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + } else { + flag.Usage() + } +} diff --git a/cmd/version.go b/ski/version.go similarity index 100% rename from cmd/version.go rename to ski/version.go diff --git a/utils.go b/utils.go index d4c280c..369ce44 100644 --- a/utils.go +++ b/utils.go @@ -1,24 +1,37 @@ -package cloudcat +package ski import ( - "net/http" + "context" + "fmt" + "log/slog" ) -// ZeroOr if value is zero value returns the defaultValue -func ZeroOr[T comparable](value, defaultValue T) T { - var zero T - if zero == value { - return defaultValue +var loggerKey byte + +// Logger get slog.Logger from the context +func Logger(ctx context.Context) *slog.Logger { + if logger := ctx.Value(&loggerKey); logger != nil { + return logger.(*slog.Logger) } - return value + return slog.Default() +} + +// WithLogger set the slog.Logger to context +func WithLogger(ctx context.Context, logger *slog.Logger) context.Context { + return WithValue(ctx, &loggerKey, logger) } -// EmptyOr if slice is empty returns the defaultValue -func EmptyOr[T any](value, defaultValue []T) []T { - if len(value) == 0 { - return defaultValue +// ToString convert Executor to string if it implements fmt.Stringer +func ToString(exec Executor) string { + switch t := exec.(type) { + case fmt.Stringer: + return t.String() + case _raw: + if s, ok := t.any.(string); ok { + return s + } } - return value + return "" } // MapKeys returns the keys of the map m. @@ -40,37 +53,3 @@ func MapValues[M ~map[K]V, K comparable, V any](m M) []V { } return r } - -// ParseCookie parses the cookie string and return a slice http.Cookie. -func ParseCookie(cookies string) []*http.Cookie { - header := http.Header{} - header.Add("Cookie", cookies) - req := http.Request{Header: header} - return req.Cookies() -} - -// ParseSetCookie parses the set-cookie strings and return a slice http.Cookie. -func ParseSetCookie(cookies ...string) []*http.Cookie { - header := http.Header{} - for _, cookie := range cookies { - header.Add("Set-Cookie", cookie) - } - res := http.Response{Header: header} - return res.Cookies() -} - -// CookieToString returns the slice string of the slice http.Cookie. -func CookieToString(cookies []*http.Cookie) []string { - switch len(cookies) { - case 0: - return nil - case 1: - return []string{cookies[0].String()} - } - - ret := make([]string, 0, len(cookies)) - for _, cookie := range cookies { - ret = append(ret, cookie.String()) - } - return ret -} diff --git a/utils_test.go b/utils_test.go deleted file mode 100644 index ae237c0..0000000 --- a/utils_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package cloudcat - -import ( - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestZeroOr(t *testing.T) { - assert.Equal(t, 1, ZeroOr(0, 1)) -} - -func TestEmptyOr(t *testing.T) { - assert.Equal(t, []int{1}, EmptyOr([]int{}, []int{1})) -} - -func TestParseCookie(t *testing.T) { - maxAge := "name=Test;id=123" - assert.Equal(t, []string{"name=Test", "id=123"}, CookieToString(ParseCookie(maxAge))) -} - -func TestParseSetCookie(t *testing.T) { - var parseCookiesTests = []struct { - String string - Cookies []*http.Cookie - }{ - { - "Cookie-1=v$1", - []*http.Cookie{{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}}, - }, - { - "NID=99=MaIh2c9H-Mzwz-; expires=Wed, 07-Jun-2023 19:52:03 GMT; path=/; domain=.google.com; HttpOnly", - []*http.Cookie{{ - Name: "NID", - Value: "99=MaIh2c9H-Mzwz-", - Path: "/", - Domain: ".google.com", - HttpOnly: true, - Expires: time.Date(2023, 6, 7, 19, 52, 3, 0, time.UTC), - RawExpires: "Wed, 07-Jun-2023 19:52:03 GMT", - Raw: "NID=99=MaIh2c9H-Mzwz-; expires=Wed, 07-Jun-2023 19:52:03 GMT; path=/; domain=.google.com; HttpOnly", - }}, - }, - { - ".ASPXAUTH=7E3AA; expires=Wed, 07-Jun-2023 19:58:08 GMT; path=/; HttpOnly", - []*http.Cookie{{ - Name: ".ASPXAUTH", - Value: "7E3AA", - Path: "/", - Expires: time.Date(2023, 6, 7, 19, 58, 8, 0, time.UTC), - RawExpires: "Wed, 07-Jun-2023 19:58:08 GMT", - HttpOnly: true, - Raw: ".ASPXAUTH=7E3AA; expires=Wed, 07-Jun-2023 19:58:08 GMT; path=/; HttpOnly", - }}, - }, - } - for _, tt := range parseCookiesTests { - assert.Equal(t, tt.Cookies, ParseSetCookie(tt.String)) - } -}