diff --git a/.gitignore b/.gitignore index 349f315..900790d 100644 --- a/.gitignore +++ b/.gitignore @@ -134,8 +134,13 @@ node_modules/ /yolov7-tracker-example/data/dataset1_2024_06_19/vott-json-export.zip -/test +# /test + + # /yolov7-setup +/test/yolov7-tracker/cfg/training/yolov7x_dataset2fps.yaml +/test/yolov7-tracker/data/dataset2fps +/test/yolov7-tracker/tracker/config_files/dataset2fps.yaml # /yolov7-tracker-example # /yolov7-tracker-example/cfg/training/yolov7x_dataset1_2024_06_19.yaml diff --git a/test/yolov7-tracker/LICENSE.md b/test/yolov7-tracker/LICENSE.md new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/test/yolov7-tracker/LICENSE.md @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/test/yolov7-tracker/README.md b/test/yolov7-tracker/README.md new file mode 100644 index 0000000..401c94d --- /dev/null +++ b/test/yolov7-tracker/README.md @@ -0,0 +1,194 @@ +# YOLO detector and SOTA Multi-object tracker Toolbox + +## ❗❗Important Notes + +Compared to the previous version, this is an ***entirely new version (branch v2)***!!! + +**Please use this version directly, as I have almost rewritten all the code to ensure better readability and improved results, as well as to correct some errors in the past code.** + +```bash +git clone https://github.com/JackWoo0831/Yolov7-tracker.git +git checkout v2 # change to v2 branch !! +``` + +🙌 ***If you have any suggestions for adding trackers***, please leave a comment in the Issues section with the paper title or link! Everyone is welcome to contribute to making this repo better. + +
+ +**Language**: English | [简体中文](README_CN.md) + +
+ +## ❤️ Introduction + +This repo is a toolbox that implements the **tracking-by-detection paradigm multi-object tracker**. The detector supports: + +- YOLOX +- YOLO v7 +- YOLO v8, + +and the tracker supports: + +- SORT +- DeepSORT +- ByteTrack ([ECCV2022](https://arxiv.org/pdf/2110.06864)) +- Bot-SORT ([arxiv2206](https://arxiv.org/pdf/2206.14651.pdf)) +- OCSORT ([CVPR2023](https://openaccess.thecvf.com/content/CVPR2023/papers/Cao_Observation-Centric_SORT_Rethinking_SORT_for_Robust_Multi-Object_Tracking_CVPR_2023_paper.pdf)) +- C_BIoU Track ([arxiv2211](https://arxiv.org/pdf/2211.14317v2.pdf)) +- Strong SORT ([IEEE TMM 2023](https://arxiv.org/pdf/2202.13514)) +- Sparse Track ([arxiv 2306](https://arxiv.org/pdf/2306.05238)) + +and the reid model supports: + +- OSNet +- Extractor from DeepSort + +The highlights are: +- Supporting more trackers than MMTracking +- Rewrite multiple trackers with a ***unified code style***, without the need to configure multiple environments for each tracker +- Modular design, which ***decouples*** the detector, tracker, reid model and Kalman filter for easy conducting experiments + +![gif](figure/demo.gif) + +## 🗺️ Roadmap + +- [ x ] Add StrongSort and SparseTrack +- [ x ] Add save video function +- [ x ] Add timer function to calculate fps +- [] Add more ReID modules. + +## 🔨 Installation + +The basic env is: +- Ubuntu 18.04 +- Python：3.9, Pytorch: 1.12 + +Run following commond to install other packages: + +```bash +pip3 install -r requirements.txt +``` + +### 🔍 Detector installation + +1. YOLOX: + +The version of YOLOX is **0.1.0 (same as ByteTrack)**. To install it, you can clone the ByteTrack repo somewhere, and run: + +``` bash +https://github.com/ifzhang/ByteTrack.git + +python3 setup.py develop +``` + +2. YOLO v7: + +There is no need to execute addtional steps as the repo itself is based on YOLOv7. + +3. YOLO v8: + +Please run: + +```bash +pip3 install ultralytics==8.0.94 +``` + +### 📑 Data preparation + +***If you do not want to test on the specific dataset, instead, you only want to run demos, please skip this section.*** + +***No matter what dataset you want to test, please organize it in the following way (YOLO style):*** + +``` +dataset_name + |---images + |---train + |---sequence_name1 + |---000001.jpg + |---000002.jpg ... + |---val ... + |---test ... + + | + +``` + +You can refer to the codes in `./tools` to see how to organize the datasets. + +***Then, you need to prepare a `yaml` file to indicate the path so that the code can find the images.*** + +Some examples are in `tracker/config_files`. The important keys are: + +``` +DATASET_ROOT: '/data/xxxx/datasets/MOT17' # your dataset root +SPLIT: test # train, test or val +CATEGORY_NAMES: # same in YOLO training + - 'pedestrian' + +CATEGORY_DICT: + 0: 'pedestrian' +``` + + + +## 🚗 Practice + +### 🏃 Training + +Trackers generally do not require parameters to be trained. Please refer to the training methods of different detectors to train YOLOs. + +Some references may help you: + +- YOLOX: `tracker/yolox_utils/train_yolox.py` + +- YOLO v7: + +```shell +python train_aux.py --dataset visdrone --workers 8 --device <$GPU_id$> --batch-size 16 --data data/visdrone_all.yaml --img 1280 1280 --cfg cfg/training/yolov7-w6.yaml --weights <$YOLO v7 pretrained model path$> --name yolov7-w6-custom --hyp data/hyp.scratch.custom.yaml +``` + +- YOLO v8: `tracker/yolov8_utils/train_yolov8.py` + + + +### 😊 Tracking ! + +If you only want to run a demo: + +```bash +python tracker/track_demo.py --obj ${video path or images folder path} --detector ${yolox, yolov8 or yolov7} --tracker ${tracker name} --kalman_format ${kalman format, sort, byte, ...} --detector_model_path ${detector weight path} --save_images +``` + +For example: + +```bash +python tracker/track_demo.py --obj M0203.mp4 --detector yolov8 --tracker deepsort --kalman_format byte --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt --save_images +``` + +If you want to run trackers on dataset: + +```bash +python tracker/track.py --dataset ${dataset name, related with the yaml file} --detector ${yolox, yolov8 or yolov7} --tracker ${tracker name} --kalman_format ${kalman format, sort, byte, ...} --detector_model_path ${detector weight path} +``` + +For example: + +- SORT: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker sort --kalman_format sort --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt ` + +- DeepSORT: `python tracker/track.py --dataset uavdt --detector yolov7 --tracker deepsort --kalman_format byte --detector_model_path weights/yolov7_UAVDT_35epochs_20230507.pt` + +- ByteTrack: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker bytetrack --kalman_format byte --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- OCSort: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker ocsort --kalman_format ocsort --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- C-BIoU Track: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker c_bioutrack --kalman_format bot --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- BoT-SORT: `python tracker/track.py --dataset uavdt --detector yolox --tracker botsort --kalman_format bot --detector_model_path weights/yolox_m_uavdt_50epochs.pth.tar` + +- Strong SORT: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker strongsort --kalman_format strongsort --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- Sparse Track: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker sparsetrack --kalman_format bot --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +### ✅ Evaluation + +Coming Soon. As an alternative, after obtaining the result txt file, you can use the [Easier to use TrackEval repo](https://github.com/JackWoo0831/Easier_To_Use_TrackEval). \ No newline at end of file diff --git a/test/yolov7-tracker/README_CN.md b/test/yolov7-tracker/README_CN.md new file mode 100644 index 0000000..f0ac80c --- /dev/null +++ b/test/yolov7-tracker/README_CN.md @@ -0,0 +1,186 @@ +# YOLO检测器与SOTA多目标跟踪工具箱 + +## ❗❗重要提示 + +与之前的版本相比，这是一个***全新的版本（分支v2）***！！！ + +**请直接使用这个版本，因为我几乎重写了所有代码，以确保更好的可读性和改进的结果，并修正了以往代码中的一些错误。** + +```bash +git clone https://github.com/JackWoo0831/Yolov7-tracker.git +git checkout v2 # change to v2 branch !! +``` + +🙌 ***如果您有任何关于添加跟踪器的建议***，请在Issues部分留言并附上论文标题或链接！欢迎大家一起来让这个repo变得更好 + + + +## ❤️ 介绍 + +这个仓库是一个实现了***检测后跟踪范式***多目标跟踪器的工具箱。检测器支持： + +- YOLOX +- YOLO v7 +- YOLO v8, + +跟踪器支持: + +- SORT +- DeepSORT +- ByteTrack ([ECCV2022](https://arxiv.org/pdf/2110.06864)) +- Bot-SORT ([arxiv2206](https://arxiv.org/pdf/2206.14651.pdf)) +- OCSORT ([CVPR2023](https://openaccess.thecvf.com/content/CVPR2023/papers/Cao_Observation-Centric_SORT_Rethinking_SORT_for_Robust_Multi-Object_Tracking_CVPR_2023_paper.pdf)) +- C_BIoU Track ([arxiv2211](https://arxiv.org/pdf/2211.14317v2.pdf)) +- Strong SORT ([IEEE TMM 2023](https://arxiv.org/pdf/2202.13514)) +- Sparse Track ([arxiv 2306](https://arxiv.org/pdf/2306.05238)) + +REID模型支持： + +- OSNet +- DeepSORT中的 + +亮点包括: +- 支持的跟踪器比MMTracking多 +- 用***统一的代码风格***重写了多个跟踪器，无需为每个跟踪器配置多个环境 +- 模块化设计，将检测器、跟踪器、外观提取模块和卡尔曼滤波器**解耦**，便于进行实验 + +![gif](figure/demo.gif) + +## 🗺️ 路线图 + +- [ x ] Add StrongSort and SparseTrack +- [ x ] Add save video function +- [ x ] Add timer function to calculate fps +- [] Add more ReID modules.mer function to calculate fps + +## 🔨 安装 + +基本环境是： +- Ubuntu 18.04 +- Python：3.9, Pytorch: 1.12 + +运行以下命令安装其他包： + +```bash +pip3 install -r requirements.txt +``` + +### 🔍 检测器安装 + +1. YOLOX: + +YOLOX的版本是0.1.0（与ByteTrack相同）。要安装它，你可以在某处克隆ByteTrack仓库，然后运行： + +``` bash +https://github.com/ifzhang/ByteTrack.git + +python3 setup.py develop +``` + +2. YOLO v7: + +由于仓库本身就是基于YOLOv7的，因此无需执行额外的步骤。 + +3. YOLO v8: + +请运行： + +```bash +pip3 install ultralytics==8.0.94 +``` + +### 📑 数据准备 + +***如果你不想在特定数据集上测试，而只想运行演示，请跳过这一部分。*** + +***无论你想测试哪个数据集，请按以下方式（YOLO风格）组织：*** + +``` +dataset_name + |---images + |---train + |---sequence_name1 + |---000001.jpg + |---000002.jpg ... + |---val ... + |---test ... + + | + +``` + +你可以参考`./tools`中的代码来了解如何组织数据集。 + +***然后，你需要准备一个yaml文件来指明路径，以便代码能够找到图像*** + +一些示例在tracker/config_files中。重要的键包括： + +``` +DATASET_ROOT: '/data/xxxx/datasets/MOT17' # your dataset root +SPLIT: test # train, test or val +CATEGORY_NAMES: # same in YOLO training + - 'pedestrian' + +CATEGORY_DICT: + 0: 'pedestrian' +``` + + + +## 🚗 实践 + +### 🏃 训练 + +跟踪器通常不需要训练参数。请参考不同检测器的训练方法来训练YOLOs。 + +以下参考资料可能对你有帮助： + +- YOLOX: `tracker/yolox_utils/train_yolox.py` + +- YOLO v7: + +```shell +python train_aux.py --dataset visdrone --workers 8 --device <$GPU_id$> --batch-size 16 --data data/visdrone_all.yaml --img 1280 1280 --cfg cfg/training/yolov7-w6.yaml --weights <$YOLO v7 pretrained model path$> --name yolov7-w6-custom --hyp data/hyp.scratch.custom.yaml +``` + +- YOLO v8: `tracker/yolov8_utils/train_yolov8.py` + + + +### 😊 跟踪！ + +如果你只是想运行一个demo: + +```bash +python tracker/track_demo.py --obj ${video path or images folder path} --detector ${yolox, yolov8 or yolov7} --tracker ${tracker name} --kalman_format ${kalman format, sort, byte, ...} --detector_model_path ${detector weight path} --save_images +``` + +例如: + +```bash +python tracker/track_demo.py --obj M0203.mp4 --detector yolov8 --tracker deepsort --kalman_format byte --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt --save_images +``` + +如果你想在数据集上测试: + +```bash +python tracker/track.py --dataset ${dataset name, related with the yaml file} --detector ${yolox, yolov8 or yolov7} --tracker ${tracker name} --kalman_format ${kalman format, sort, byte, ...} --detector_model_path ${detector weight path} +``` + +例如: + +- SORT: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker sort --kalman_format sort --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt ` + +- DeepSORT: `python tracker/track.py --dataset uavdt --detector yolov7 --tracker deepsort --kalman_format byte --detector_model_path weights/yolov7_UAVDT_35epochs_20230507.pt` + +- ByteTrack: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker bytetrack --kalman_format byte --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- OCSort: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker ocsort --kalman_format ocsort --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- C-BIoU Track: `python tracker/track.py --dataset uavdt --detector yolov8 --tracker c_bioutrack --kalman_format bot --detector_model_path weights/yolov8l_UAVDT_60epochs_20230509.pt` + +- BoT-SORT: `python tracker/track.py --dataset uavdt --detector yolox --tracker botsort --kalman_format bot --detector_model_path weights/yolox_m_uavdt_50epochs.pth.tar` + +### ✅ 评估 + +马上推出！作为备选项，你可以使用这个repo： [Easier to use TrackEval repo](https://github.com/JackWoo0831/Easier_To_Use_TrackEval). \ No newline at end of file diff --git a/test/yolov7-tracker/cfg/baseline/r50-csp.yaml b/test/yolov7-tracker/cfg/baseline/r50-csp.yaml new file mode 100644 index 0000000..94559f7 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/r50-csp.yaml @@ -0,0 +1,49 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# CSP-ResNet backbone +backbone: + # [from, number, module, args] + [[-1, 1, Stem, [128]], # 0-P1/2 + [-1, 3, ResCSPC, [128]], + [-1, 1, Conv, [256, 3, 2]], # 2-P3/8 + [-1, 4, ResCSPC, [256]], + [-1, 1, Conv, [512, 3, 2]], # 4-P3/8 + [-1, 6, ResCSPC, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 6-P3/8 + [-1, 3, ResCSPC, [1024]], # 7 + ] + +# CSP-Res-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 8 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [5, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 2, ResCSPB, [256]], # 13 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [3, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 2, ResCSPB, [128]], # 18 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 13], 1, Concat, [1]], # cat + [-1, 2, ResCSPB, [256]], # 22 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 8], 1, Concat, [1]], # cat + [-1, 2, ResCSPB, [512]], # 26 + [-1, 1, Conv, [1024, 3, 1]], + + [[19,23,27], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/x50-csp.yaml b/test/yolov7-tracker/cfg/baseline/x50-csp.yaml new file mode 100644 index 0000000..8de14f8 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/x50-csp.yaml @@ -0,0 +1,49 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# CSP-ResNeXt backbone +backbone: + # [from, number, module, args] + [[-1, 1, Stem, [128]], # 0-P1/2 + [-1, 3, ResXCSPC, [128]], + [-1, 1, Conv, [256, 3, 2]], # 2-P3/8 + [-1, 4, ResXCSPC, [256]], + [-1, 1, Conv, [512, 3, 2]], # 4-P3/8 + [-1, 6, ResXCSPC, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 6-P3/8 + [-1, 3, ResXCSPC, [1024]], # 7 + ] + +# CSP-ResX-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 8 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [5, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 2, ResXCSPB, [256]], # 13 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [3, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 2, ResXCSPB, [128]], # 18 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 13], 1, Concat, [1]], # cat + [-1, 2, ResXCSPB, [256]], # 22 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 8], 1, Concat, [1]], # cat + [-1, 2, ResXCSPB, [512]], # 26 + [-1, 1, Conv, [1024, 3, 1]], + + [[19,23,27], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/yolor-csp-x.yaml b/test/yolov7-tracker/cfg/baseline/yolor-csp-x.yaml new file mode 100644 index 0000000..6e234c5 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-csp-x.yaml @@ -0,0 +1,52 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.33 # model depth multiple +width_multiple: 1.25 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, BottleneckCSPC, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, BottleneckCSPC, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, BottleneckCSPC, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, BottleneckCSPC, [1024]], # 10 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 11 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [8, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [256]], # 16 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [6, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [128]], # 21 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [256]], # 25 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 11], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [512]], # 29 + [-1, 1, Conv, [1024, 3, 1]], + + [[22,26,30], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/yolor-csp.yaml b/test/yolov7-tracker/cfg/baseline/yolor-csp.yaml new file mode 100644 index 0000000..3beecf3 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-csp.yaml @@ -0,0 +1,52 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, BottleneckCSPC, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, BottleneckCSPC, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, BottleneckCSPC, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, BottleneckCSPC, [1024]], # 10 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 11 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [8, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [256]], # 16 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [6, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [128]], # 21 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [256]], # 25 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 11], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [512]], # 29 + [-1, 1, Conv, [1024, 3, 1]], + + [[22,26,30], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/yolor-d6.yaml b/test/yolov7-tracker/cfg/baseline/yolor-d6.yaml new file mode 100644 index 0000000..297b0d1 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-d6.yaml @@ -0,0 +1,63 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # expand model depth +width_multiple: 1.25 # expand layer channels + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + [-1, 1, DownC, [128]], # 2-P2/4 + [-1, 3, BottleneckCSPA, [128]], + [-1, 1, DownC, [256]], # 4-P3/8 + [-1, 15, BottleneckCSPA, [256]], + [-1, 1, DownC, [512]], # 6-P4/16 + [-1, 15, BottleneckCSPA, [512]], + [-1, 1, DownC, [768]], # 8-P5/32 + [-1, 7, BottleneckCSPA, [768]], + [-1, 1, DownC, [1024]], # 10-P6/64 + [-1, 7, BottleneckCSPA, [1024]], # 11 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 12 + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-6, 1, Conv, [384, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [384]], # 17 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-13, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [256]], # 22 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-20, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [128]], # 27 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, DownC, [256]], + [[-1, 22], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [256]], # 31 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, DownC, [384]], + [[-1, 17], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [384]], # 35 + [-1, 1, Conv, [768, 3, 1]], + [-2, 1, DownC, [512]], + [[-1, 12], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [512]], # 39 + [-1, 1, Conv, [1024, 3, 1]], + + [[28,32,36,40], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] \ No newline at end of file diff --git a/test/yolov7-tracker/cfg/baseline/yolor-e6.yaml b/test/yolov7-tracker/cfg/baseline/yolor-e6.yaml new file mode 100644 index 0000000..58afc5b --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-e6.yaml @@ -0,0 +1,63 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # expand model depth +width_multiple: 1.25 # expand layer channels + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + [-1, 1, DownC, [128]], # 2-P2/4 + [-1, 3, BottleneckCSPA, [128]], + [-1, 1, DownC, [256]], # 4-P3/8 + [-1, 7, BottleneckCSPA, [256]], + [-1, 1, DownC, [512]], # 6-P4/16 + [-1, 7, BottleneckCSPA, [512]], + [-1, 1, DownC, [768]], # 8-P5/32 + [-1, 3, BottleneckCSPA, [768]], + [-1, 1, DownC, [1024]], # 10-P6/64 + [-1, 3, BottleneckCSPA, [1024]], # 11 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 12 + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-6, 1, Conv, [384, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [384]], # 17 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-13, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [256]], # 22 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-20, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [128]], # 27 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, DownC, [256]], + [[-1, 22], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [256]], # 31 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, DownC, [384]], + [[-1, 17], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [384]], # 35 + [-1, 1, Conv, [768, 3, 1]], + [-2, 1, DownC, [512]], + [[-1, 12], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [512]], # 39 + [-1, 1, Conv, [1024, 3, 1]], + + [[28,32,36,40], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] \ No newline at end of file diff --git a/test/yolov7-tracker/cfg/baseline/yolor-p6.yaml b/test/yolov7-tracker/cfg/baseline/yolor-p6.yaml new file mode 100644 index 0000000..924cf5c --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-p6.yaml @@ -0,0 +1,63 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # expand model depth +width_multiple: 1.0 # expand layer channels + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 2-P2/4 + [-1, 3, BottleneckCSPA, [128]], + [-1, 1, Conv, [256, 3, 2]], # 4-P3/8 + [-1, 7, BottleneckCSPA, [256]], + [-1, 1, Conv, [384, 3, 2]], # 6-P4/16 + [-1, 7, BottleneckCSPA, [384]], + [-1, 1, Conv, [512, 3, 2]], # 8-P5/32 + [-1, 3, BottleneckCSPA, [512]], + [-1, 1, Conv, [640, 3, 2]], # 10-P6/64 + [-1, 3, BottleneckCSPA, [640]], # 11 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [320]], # 12 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-6, 1, Conv, [256, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [256]], # 17 + [-1, 1, Conv, [192, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-13, 1, Conv, [192, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [192]], # 22 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-20, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [128]], # 27 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [192, 3, 2]], + [[-1, 22], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [192]], # 31 + [-1, 1, Conv, [384, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 17], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [256]], # 35 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [320, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [320]], # 39 + [-1, 1, Conv, [640, 3, 1]], + + [[28,32,36,40], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] \ No newline at end of file diff --git a/test/yolov7-tracker/cfg/baseline/yolor-w6.yaml b/test/yolov7-tracker/cfg/baseline/yolor-w6.yaml new file mode 100644 index 0000000..a2fc969 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolor-w6.yaml @@ -0,0 +1,63 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # expand model depth +width_multiple: 1.0 # expand layer channels + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 2-P2/4 + [-1, 3, BottleneckCSPA, [128]], + [-1, 1, Conv, [256, 3, 2]], # 4-P3/8 + [-1, 7, BottleneckCSPA, [256]], + [-1, 1, Conv, [512, 3, 2]], # 6-P4/16 + [-1, 7, BottleneckCSPA, [512]], + [-1, 1, Conv, [768, 3, 2]], # 8-P5/32 + [-1, 3, BottleneckCSPA, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 10-P6/64 + [-1, 3, BottleneckCSPA, [1024]], # 11 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 12 + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-6, 1, Conv, [384, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [384]], # 17 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-13, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [256]], # 22 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [-20, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 3, BottleneckCSPB, [128]], # 27 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 22], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [256]], # 31 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [384, 3, 2]], + [[-1, 17], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [384]], # 35 + [-1, 1, Conv, [768, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat + [-1, 3, BottleneckCSPB, [512]], # 39 + [-1, 1, Conv, [1024, 3, 1]], + + [[28,32,36,40], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] \ No newline at end of file diff --git a/test/yolov7-tracker/cfg/baseline/yolov3-spp.yaml b/test/yolov7-tracker/cfg/baseline/yolov3-spp.yaml new file mode 100644 index 0000000..38dcc44 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolov3-spp.yaml @@ -0,0 +1,51 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# darknet53 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, Bottleneck, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, Bottleneck, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, Bottleneck, [1024]], # 10 + ] + +# YOLOv3-SPP head +head: + [[-1, 1, Bottleneck, [1024, False]], + [-1, 1, SPP, [512, [5, 9, 13]]], + [-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Bottleneck, [256, False]], + [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small) + + [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/yolov3.yaml b/test/yolov7-tracker/cfg/baseline/yolov3.yaml new file mode 100644 index 0000000..f2e7613 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolov3.yaml @@ -0,0 +1,51 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# darknet53 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, Bottleneck, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, Bottleneck, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, Bottleneck, [1024]], # 10 + ] + +# YOLOv3 head +head: + [[-1, 1, Bottleneck, [1024, False]], + [-1, 1, Conv, [512, [1, 1]]], + [-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Bottleneck, [256, False]], + [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small) + + [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/baseline/yolov4-csp.yaml b/test/yolov7-tracker/cfg/baseline/yolov4-csp.yaml new file mode 100644 index 0000000..3c908c7 --- /dev/null +++ b/test/yolov7-tracker/cfg/baseline/yolov4-csp.yaml @@ -0,0 +1,52 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# CSP-Darknet backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, BottleneckCSPC, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, BottleneckCSPC, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, BottleneckCSPC, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, BottleneckCSPC, [1024]], # 10 + ] + +# CSP-Dark-PAN head +head: + [[-1, 1, SPPCSPC, [512]], # 11 + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [8, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [256]], # 16 + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [6, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + [-1, 2, BottleneckCSPB, [128]], # 21 + [-1, 1, Conv, [256, 3, 1]], + [-2, 1, Conv, [256, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [256]], # 25 + [-1, 1, Conv, [512, 3, 1]], + [-2, 1, Conv, [512, 3, 2]], + [[-1, 11], 1, Concat, [1]], # cat + [-1, 2, BottleneckCSPB, [512]], # 29 + [-1, 1, Conv, [1024, 3, 1]], + + [[22,26,30], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-d6.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-d6.yaml new file mode 100644 index 0000000..75a8cf5 --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-d6.yaml @@ -0,0 +1,202 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7-d6 backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [96, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [192]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [192, 1, 1]], # 14 + + [-1, 1, DownC, [384]], # 15-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 27 + + [-1, 1, DownC, [768]], # 28-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 40 + + [-1, 1, DownC, [1152]], # 41-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [1152, 1, 1]], # 53 + + [-1, 1, DownC, [1536]], # 54-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [1536, 1, 1]], # 66 + ] + +# yolov7-d6 head +head: + [[-1, 1, SPPCSPC, [768]], # 67 + + [-1, 1, Conv, [576, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [53, 1, Conv, [576, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [576, 1, 1]], # 83 + + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [40, 1, Conv, [384, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 99 + + [-1, 1, Conv, [192, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [27, 1, Conv, [192, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [192, 1, 1]], # 115 + + [-1, 1, DownC, [384]], + [[-1, 99], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 129 + + [-1, 1, DownC, [576]], + [[-1, 83], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [576, 1, 1]], # 143 + + [-1, 1, DownC, [768]], + [[-1, 67], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 157 + + [115, 1, Conv, [384, 3, 1]], + [129, 1, Conv, [768, 3, 1]], + [143, 1, Conv, [1152, 3, 1]], + [157, 1, Conv, [1536, 3, 1]], + + [[158,159,160,161], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-e6.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-e6.yaml new file mode 100644 index 0000000..e680406 --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-e6.yaml @@ -0,0 +1,180 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7-e6 backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [80, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [160]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 12 + + [-1, 1, DownC, [320]], # 13-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 23 + + [-1, 1, DownC, [640]], # 24-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 34 + + [-1, 1, DownC, [960]], # 35-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 45 + + [-1, 1, DownC, [1280]], # 46-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 56 + ] + +# yolov7-e6 head +head: + [[-1, 1, SPPCSPC, [640]], # 57 + + [-1, 1, Conv, [480, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [45, 1, Conv, [480, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 71 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [34, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 85 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [23, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 99 + + [-1, 1, DownC, [320]], + [[-1, 85], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 111 + + [-1, 1, DownC, [480]], + [[-1, 71], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 123 + + [-1, 1, DownC, [640]], + [[-1, 57], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 135 + + [99, 1, Conv, [320, 3, 1]], + [111, 1, Conv, [640, 3, 1]], + [123, 1, Conv, [960, 3, 1]], + [135, 1, Conv, [1280, 3, 1]], + + [[136,137,138,139], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-e6e.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-e6e.yaml new file mode 100644 index 0000000..135990d --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-e6e.yaml @@ -0,0 +1,301 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7-e6e backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [80, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [160]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 12 + [-11, 1, Conv, [64, 1, 1]], + [-12, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 22 + [[-1, -11], 1, Shortcut, [1]], # 23 + + [-1, 1, DownC, [320]], # 24-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 34 + [-11, 1, Conv, [128, 1, 1]], + [-12, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 44 + [[-1, -11], 1, Shortcut, [1]], # 45 + + [-1, 1, DownC, [640]], # 46-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 56 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 66 + [[-1, -11], 1, Shortcut, [1]], # 67 + + [-1, 1, DownC, [960]], # 68-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 78 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 88 + [[-1, -11], 1, Shortcut, [1]], # 89 + + [-1, 1, DownC, [1280]], # 90-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 100 + [-11, 1, Conv, [512, 1, 1]], + [-12, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 110 + [[-1, -11], 1, Shortcut, [1]], # 111 + ] + +# yolov7-e6e head +head: + [[-1, 1, SPPCSPC, [640]], # 112 + + [-1, 1, Conv, [480, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [89, 1, Conv, [480, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 126 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 136 + [[-1, -11], 1, Shortcut, [1]], # 137 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [67, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 151 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 161 + [[-1, -11], 1, Shortcut, [1]], # 162 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [45, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 176 + [-11, 1, Conv, [128, 1, 1]], + [-12, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 186 + [[-1, -11], 1, Shortcut, [1]], # 187 + + [-1, 1, DownC, [320]], + [[-1, 162], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 199 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 209 + [[-1, -11], 1, Shortcut, [1]], # 210 + + [-1, 1, DownC, [480]], + [[-1, 137], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 222 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 232 + [[-1, -11], 1, Shortcut, [1]], # 233 + + [-1, 1, DownC, [640]], + [[-1, 112], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 245 + [-11, 1, Conv, [512, 1, 1]], + [-12, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 255 + [[-1, -11], 1, Shortcut, [1]], # 256 + + [187, 1, Conv, [320, 3, 1]], + [210, 1, Conv, [640, 3, 1]], + [233, 1, Conv, [960, 3, 1]], + [256, 1, Conv, [1280, 3, 1]], + + [[257,258,259,260], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-tiny-silu.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-tiny-silu.yaml new file mode 100644 index 0000000..9250573 --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-tiny-silu.yaml @@ -0,0 +1,112 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv7-tiny backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 2]], # 0-P1/2 + + [-1, 1, Conv, [64, 3, 2]], # 1-P2/4 + + [-1, 1, Conv, [32, 1, 1]], + [-2, 1, Conv, [32, 1, 1]], + [-1, 1, Conv, [32, 3, 1]], + [-1, 1, Conv, [32, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1]], # 7 + + [-1, 1, MP, []], # 8-P3/8 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 14 + + [-1, 1, MP, []], # 15-P4/16 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 21 + + [-1, 1, MP, []], # 22-P5/32 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 28 + ] + +# YOLOv7-tiny head +head: + [[-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, SP, [5]], + [-2, 1, SP, [9]], + [-3, 1, SP, [13]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], + [[-1, -7], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 37 + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [21, 1, Conv, [128, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 47 + + [-1, 1, Conv, [64, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [14, 1, Conv, [64, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [32, 1, 1]], + [-2, 1, Conv, [32, 1, 1]], + [-1, 1, Conv, [32, 3, 1]], + [-1, 1, Conv, [32, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1]], # 57 + + [-1, 1, Conv, [128, 3, 2]], + [[-1, 47], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 65 + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 37], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 73 + + [57, 1, Conv, [128, 3, 1]], + [65, 1, Conv, [256, 3, 1]], + [73, 1, Conv, [512, 3, 1]], + + [[74,75,76], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-tiny.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-tiny.yaml new file mode 100644 index 0000000..b09f130 --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-tiny.yaml @@ -0,0 +1,112 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# yolov7-tiny backbone +backbone: + # [from, number, module, args] c2, k=1, s=1, p=None, g=1, act=True + [[-1, 1, Conv, [32, 3, 2, None, 1, nn.LeakyReLU(0.1)]], # 0-P1/2 + + [-1, 1, Conv, [64, 3, 2, None, 1, nn.LeakyReLU(0.1)]], # 1-P2/4 + + [-1, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 7 + + [-1, 1, MP, []], # 8-P3/8 + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 14 + + [-1, 1, MP, []], # 15-P4/16 + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 21 + + [-1, 1, MP, []], # 22-P5/32 + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 28 + ] + +# yolov7-tiny head +head: + [[-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, SP, [5]], + [-2, 1, SP, [9]], + [-3, 1, SP, [13]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -7], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 37 + + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [21, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 47 + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [14, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 57 + + [-1, 1, Conv, [128, 3, 2, None, 1, nn.LeakyReLU(0.1)]], + [[-1, 47], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 65 + + [-1, 1, Conv, [256, 3, 2, None, 1, nn.LeakyReLU(0.1)]], + [[-1, 37], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 73 + + [57, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [65, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [73, 1, Conv, [512, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + + [[74,75,76], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7-w6.yaml b/test/yolov7-tracker/cfg/deploy/yolov7-w6.yaml new file mode 100644 index 0000000..5637a61 --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7-w6.yaml @@ -0,0 +1,158 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7-w6 backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + + [-1, 1, Conv, [128, 3, 2]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 10 + + [-1, 1, Conv, [256, 3, 2]], # 11-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 19 + + [-1, 1, Conv, [512, 3, 2]], # 20-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 28 + + [-1, 1, Conv, [768, 3, 2]], # 29-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 37 + + [-1, 1, Conv, [1024, 3, 2]], # 38-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 46 + ] + +# yolov7-w6 head +head: + [[-1, 1, SPPCSPC, [512]], # 47 + + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [37, 1, Conv, [384, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 59 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [28, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 71 + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [19, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 83 + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 71], 1, Concat, [1]], # cat + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 93 + + [-1, 1, Conv, [384, 3, 2]], + [[-1, 59], 1, Concat, [1]], # cat + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 103 + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 47], 1, Concat, [1]], # cat + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 113 + + [83, 1, Conv, [256, 3, 1]], + [93, 1, Conv, [512, 3, 1]], + [103, 1, Conv, [768, 3, 1]], + [113, 1, Conv, [1024, 3, 1]], + + [[114,115,116,117], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7.yaml b/test/yolov7-tracker/cfg/deploy/yolov7.yaml new file mode 100644 index 0000000..201f98d --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7.yaml @@ -0,0 +1,140 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# yolov7 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Conv, [64, 3, 1]], + + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 11 + + [-1, 1, MP, []], + [-1, 1, Conv, [128, 1, 1]], + [-3, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 16-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 24 + + [-1, 1, MP, []], + [-1, 1, Conv, [256, 1, 1]], + [-3, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 29-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 37 + + [-1, 1, MP, []], + [-1, 1, Conv, [512, 1, 1]], + [-3, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 42-P5/32 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 50 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [512]], # 51 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [37, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 63 + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [24, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 75 + + [-1, 1, MP, []], + [-1, 1, Conv, [128, 1, 1]], + [-3, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 2]], + [[-1, -3, 63], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 88 + + [-1, 1, MP, []], + [-1, 1, Conv, [256, 1, 1]], + [-3, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 2]], + [[-1, -3, 51], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 101 + + [75, 1, RepConv, [256, 3, 1]], + [88, 1, RepConv, [512, 3, 1]], + [101, 1, RepConv, [1024, 3, 1]], + + [[102,103,104], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/deploy/yolov7x.yaml b/test/yolov7-tracker/cfg/deploy/yolov7x.yaml new file mode 100644 index 0000000..c1b4acc --- /dev/null +++ b/test/yolov7-tracker/cfg/deploy/yolov7x.yaml @@ -0,0 +1,156 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# yolov7x backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [40, 3, 1]], # 0 + + [-1, 1, Conv, [80, 3, 2]], # 1-P1/2 + [-1, 1, Conv, [80, 3, 1]], + + [-1, 1, Conv, [160, 3, 2]], # 3-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 13 + + [-1, 1, MP, []], + [-1, 1, Conv, [160, 1, 1]], + [-3, 1, Conv, [160, 1, 1]], + [-1, 1, Conv, [160, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 18-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 28 + + [-1, 1, MP, []], + [-1, 1, Conv, [320, 1, 1]], + [-3, 1, Conv, [320, 1, 1]], + [-1, 1, Conv, [320, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 33-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 43 + + [-1, 1, MP, []], + [-1, 1, Conv, [640, 1, 1]], + [-3, 1, Conv, [640, 1, 1]], + [-1, 1, Conv, [640, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 48-P5/32 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 58 + ] + +# yolov7x head +head: + [[-1, 1, SPPCSPC, [640]], # 59 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [43, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 73 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [28, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 87 + + [-1, 1, MP, []], + [-1, 1, Conv, [160, 1, 1]], + [-3, 1, Conv, [160, 1, 1]], + [-1, 1, Conv, [160, 3, 2]], + [[-1, -3, 73], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 102 + + [-1, 1, MP, []], + [-1, 1, Conv, [320, 1, 1]], + [-3, 1, Conv, [320, 1, 1]], + [-1, 1, Conv, [320, 3, 2]], + [[-1, -3, 59], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 117 + + [87, 1, Conv, [320, 3, 1]], + [102, 1, Conv, [640, 3, 1]], + [117, 1, Conv, [1280, 3, 1]], + + [[118,119,120], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7-d6.yaml b/test/yolov7-tracker/cfg/training/yolov7-d6.yaml new file mode 100644 index 0000000..4faedda --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7-d6.yaml @@ -0,0 +1,207 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7 backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [96, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [192]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [192, 1, 1]], # 14 + + [-1, 1, DownC, [384]], # 15-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 27 + + [-1, 1, DownC, [768]], # 28-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 40 + + [-1, 1, DownC, [1152]], # 41-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [1152, 1, 1]], # 53 + + [-1, 1, DownC, [1536]], # 54-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [1536, 1, 1]], # 66 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [768]], # 67 + + [-1, 1, Conv, [576, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [53, 1, Conv, [576, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [576, 1, 1]], # 83 + + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [40, 1, Conv, [384, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 99 + + [-1, 1, Conv, [192, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [27, 1, Conv, [192, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [192, 1, 1]], # 115 + + [-1, 1, DownC, [384]], + [[-1, 99], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 129 + + [-1, 1, DownC, [576]], + [[-1, 83], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [576, 1, 1]], # 143 + + [-1, 1, DownC, [768]], + [[-1, 67], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8, -9, -10], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 157 + + [115, 1, Conv, [384, 3, 1]], + [129, 1, Conv, [768, 3, 1]], + [143, 1, Conv, [1152, 3, 1]], + [157, 1, Conv, [1536, 3, 1]], + + [115, 1, Conv, [384, 3, 1]], + [99, 1, Conv, [768, 3, 1]], + [83, 1, Conv, [1152, 3, 1]], + [67, 1, Conv, [1536, 3, 1]], + + [[158,159,160,161,162,163,164,165], 1, IAuxDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7-e6.yaml b/test/yolov7-tracker/cfg/training/yolov7-e6.yaml new file mode 100644 index 0000000..58b27f0 --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7-e6.yaml @@ -0,0 +1,185 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7 backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [80, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [160]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 12 + + [-1, 1, DownC, [320]], # 13-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 23 + + [-1, 1, DownC, [640]], # 24-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 34 + + [-1, 1, DownC, [960]], # 35-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 45 + + [-1, 1, DownC, [1280]], # 46-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 56 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [640]], # 57 + + [-1, 1, Conv, [480, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [45, 1, Conv, [480, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 71 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [34, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 85 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [23, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 99 + + [-1, 1, DownC, [320]], + [[-1, 85], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 111 + + [-1, 1, DownC, [480]], + [[-1, 71], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 123 + + [-1, 1, DownC, [640]], + [[-1, 57], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 135 + + [99, 1, Conv, [320, 3, 1]], + [111, 1, Conv, [640, 3, 1]], + [123, 1, Conv, [960, 3, 1]], + [135, 1, Conv, [1280, 3, 1]], + + [99, 1, Conv, [320, 3, 1]], + [85, 1, Conv, [640, 3, 1]], + [71, 1, Conv, [960, 3, 1]], + [57, 1, Conv, [1280, 3, 1]], + + [[136,137,138,139,140,141,142,143], 1, IAuxDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7-e6e.yaml b/test/yolov7-tracker/cfg/training/yolov7-e6e.yaml new file mode 100644 index 0000000..3c83661 --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7-e6e.yaml @@ -0,0 +1,306 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7 backbone +backbone: + # [from, number, module, args], + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [80, 3, 1]], # 1-P1/2 + + [-1, 1, DownC, [160]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 12 + [-11, 1, Conv, [64, 1, 1]], + [-12, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 22 + [[-1, -11], 1, Shortcut, [1]], # 23 + + [-1, 1, DownC, [320]], # 24-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 34 + [-11, 1, Conv, [128, 1, 1]], + [-12, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 44 + [[-1, -11], 1, Shortcut, [1]], # 45 + + [-1, 1, DownC, [640]], # 46-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 56 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 66 + [[-1, -11], 1, Shortcut, [1]], # 67 + + [-1, 1, DownC, [960]], # 68-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 78 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [960, 1, 1]], # 88 + [[-1, -11], 1, Shortcut, [1]], # 89 + + [-1, 1, DownC, [1280]], # 90-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 100 + [-11, 1, Conv, [512, 1, 1]], + [-12, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 110 + [[-1, -11], 1, Shortcut, [1]], # 111 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [640]], # 112 + + [-1, 1, Conv, [480, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [89, 1, Conv, [480, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 126 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 136 + [[-1, -11], 1, Shortcut, [1]], # 137 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [67, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 151 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 161 + [[-1, -11], 1, Shortcut, [1]], # 162 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [45, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 176 + [-11, 1, Conv, [128, 1, 1]], + [-12, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 186 + [[-1, -11], 1, Shortcut, [1]], # 187 + + [-1, 1, DownC, [320]], + [[-1, 162], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 199 + [-11, 1, Conv, [256, 1, 1]], + [-12, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 209 + [[-1, -11], 1, Shortcut, [1]], # 210 + + [-1, 1, DownC, [480]], + [[-1, 137], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 222 + [-11, 1, Conv, [384, 1, 1]], + [-12, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [480, 1, 1]], # 232 + [[-1, -11], 1, Shortcut, [1]], # 233 + + [-1, 1, DownC, [640]], + [[-1, 112], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 245 + [-11, 1, Conv, [512, 1, 1]], + [-12, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 255 + [[-1, -11], 1, Shortcut, [1]], # 256 + + [187, 1, Conv, [320, 3, 1]], + [210, 1, Conv, [640, 3, 1]], + [233, 1, Conv, [960, 3, 1]], + [256, 1, Conv, [1280, 3, 1]], + + [186, 1, Conv, [320, 3, 1]], + [161, 1, Conv, [640, 3, 1]], + [136, 1, Conv, [960, 3, 1]], + [112, 1, Conv, [1280, 3, 1]], + + [[257,258,259,260,261,262,263,264], 1, IAuxDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7-tiny.yaml b/test/yolov7-tracker/cfg/training/yolov7-tiny.yaml new file mode 100644 index 0000000..3679b0d --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7-tiny.yaml @@ -0,0 +1,112 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# yolov7-tiny backbone +backbone: + # [from, number, module, args] c2, k=1, s=1, p=None, g=1, act=True + [[-1, 1, Conv, [32, 3, 2, None, 1, nn.LeakyReLU(0.1)]], # 0-P1/2 + + [-1, 1, Conv, [64, 3, 2, None, 1, nn.LeakyReLU(0.1)]], # 1-P2/4 + + [-1, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 7 + + [-1, 1, MP, []], # 8-P3/8 + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 14 + + [-1, 1, MP, []], # 15-P4/16 + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 21 + + [-1, 1, MP, []], # 22-P5/32 + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 28 + ] + +# yolov7-tiny head +head: + [[-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, SP, [5]], + [-2, 1, SP, [9]], + [-3, 1, SP, [13]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -7], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 37 + + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [21, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 47 + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [14, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [32, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [32, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 57 + + [-1, 1, Conv, [128, 3, 2, None, 1, nn.LeakyReLU(0.1)]], + [[-1, 47], 1, Concat, [1]], + + [-1, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [64, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [64, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 65 + + [-1, 1, Conv, [256, 3, 2, None, 1, nn.LeakyReLU(0.1)]], + [[-1, 37], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-2, 1, Conv, [128, 1, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [-1, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [[-1, -2, -3, -4], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1, None, 1, nn.LeakyReLU(0.1)]], # 73 + + [57, 1, Conv, [128, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [65, 1, Conv, [256, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + [73, 1, Conv, [512, 3, 1, None, 1, nn.LeakyReLU(0.1)]], + + [[74,75,76], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7-w6.yaml b/test/yolov7-tracker/cfg/training/yolov7-w6.yaml new file mode 100644 index 0000000..4b9c013 --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7-w6.yaml @@ -0,0 +1,163 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [ 19,27, 44,40, 38,94 ] # P3/8 + - [ 96,68, 86,152, 180,137 ] # P4/16 + - [ 140,301, 303,264, 238,542 ] # P5/32 + - [ 436,615, 739,380, 925,792 ] # P6/64 + +# yolov7 backbone +backbone: + # [from, number, module, args] + [[-1, 1, ReOrg, []], # 0 + [-1, 1, Conv, [64, 3, 1]], # 1-P1/2 + + [-1, 1, Conv, [128, 3, 2]], # 2-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 10 + + [-1, 1, Conv, [256, 3, 2]], # 11-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 19 + + [-1, 1, Conv, [512, 3, 2]], # 20-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 28 + + [-1, 1, Conv, [768, 3, 2]], # 29-P5/32 + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [-1, 1, Conv, [384, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [768, 1, 1]], # 37 + + [-1, 1, Conv, [1024, 3, 2]], # 38-P6/64 + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 46 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [512]], # 47 + + [-1, 1, Conv, [384, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [37, 1, Conv, [384, 1, 1]], # route backbone P5 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 59 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [28, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 71 + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [19, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 83 + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 71], 1, Concat, [1]], # cat + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 93 + + [-1, 1, Conv, [384, 3, 2]], + [[-1, 59], 1, Concat, [1]], # cat + + [-1, 1, Conv, [384, 1, 1]], + [-2, 1, Conv, [384, 1, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [-1, 1, Conv, [192, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [384, 1, 1]], # 103 + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 47], 1, Concat, [1]], # cat + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 113 + + [83, 1, Conv, [256, 3, 1]], + [93, 1, Conv, [512, 3, 1]], + [103, 1, Conv, [768, 3, 1]], + [113, 1, Conv, [1024, 3, 1]], + + [83, 1, Conv, [320, 3, 1]], + [71, 1, Conv, [640, 3, 1]], + [59, 1, Conv, [960, 3, 1]], + [47, 1, Conv, [1280, 3, 1]], + + [[114,115,116,117,118,119,120,121], 1, IAuxDetect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7.yaml b/test/yolov7-tracker/cfg/training/yolov7.yaml new file mode 100644 index 0000000..9a807e5 --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7.yaml @@ -0,0 +1,140 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# yolov7 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Conv, [64, 3, 1]], + + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 11 + + [-1, 1, MP, []], + [-1, 1, Conv, [128, 1, 1]], + [-3, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 16-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 24 + + [-1, 1, MP, []], + [-1, 1, Conv, [256, 1, 1]], + [-3, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 29-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 37 + + [-1, 1, MP, []], + [-1, 1, Conv, [512, 1, 1]], + [-3, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 42-P5/32 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [1024, 1, 1]], # 50 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [512]], # 51 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [37, 1, Conv, [256, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 63 + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [24, 1, Conv, [128, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [128, 1, 1]], # 75 + + [-1, 1, MP, []], + [-1, 1, Conv, [128, 1, 1]], + [-3, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 2]], + [[-1, -3, 63], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [256, 1, 1]], # 88 + + [-1, 1, MP, []], + [-1, 1, Conv, [256, 1, 1]], + [-3, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 2]], + [[-1, -3, 51], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -2, -3, -4, -5, -6], 1, Concat, [1]], + [-1, 1, Conv, [512, 1, 1]], # 101 + + [75, 1, RepConv, [256, 3, 1]], + [88, 1, RepConv, [512, 3, 1]], + [101, 1, RepConv, [1024, 3, 1]], + + [[102,103,104], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/cfg/training/yolov7x.yaml b/test/yolov7-tracker/cfg/training/yolov7x.yaml new file mode 100644 index 0000000..207be88 --- /dev/null +++ b/test/yolov7-tracker/cfg/training/yolov7x.yaml @@ -0,0 +1,156 @@ +# parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# anchors +anchors: + - [12,16, 19,36, 40,28] # P3/8 + - [36,75, 76,55, 72,146] # P4/16 + - [142,110, 192,243, 459,401] # P5/32 + +# yolov7 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [40, 3, 1]], # 0 + + [-1, 1, Conv, [80, 3, 2]], # 1-P1/2 + [-1, 1, Conv, [80, 3, 1]], + + [-1, 1, Conv, [160, 3, 2]], # 3-P2/4 + [-1, 1, Conv, [64, 1, 1]], + [-2, 1, Conv, [64, 1, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, Conv, [64, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 13 + + [-1, 1, MP, []], + [-1, 1, Conv, [160, 1, 1]], + [-3, 1, Conv, [160, 1, 1]], + [-1, 1, Conv, [160, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 18-P3/8 + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 28 + + [-1, 1, MP, []], + [-1, 1, Conv, [320, 1, 1]], + [-3, 1, Conv, [320, 1, 1]], + [-1, 1, Conv, [320, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 33-P4/16 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 43 + + [-1, 1, MP, []], + [-1, 1, Conv, [640, 1, 1]], + [-3, 1, Conv, [640, 1, 1]], + [-1, 1, Conv, [640, 3, 2]], + [[-1, -3], 1, Concat, [1]], # 48-P5/32 + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [1280, 1, 1]], # 58 + ] + +# yolov7 head +head: + [[-1, 1, SPPCSPC, [640]], # 59 + + [-1, 1, Conv, [320, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [43, 1, Conv, [320, 1, 1]], # route backbone P4 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 73 + + [-1, 1, Conv, [160, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [28, 1, Conv, [160, 1, 1]], # route backbone P3 + [[-1, -2], 1, Concat, [1]], + + [-1, 1, Conv, [128, 1, 1]], + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, Conv, [128, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [160, 1, 1]], # 87 + + [-1, 1, MP, []], + [-1, 1, Conv, [160, 1, 1]], + [-3, 1, Conv, [160, 1, 1]], + [-1, 1, Conv, [160, 3, 2]], + [[-1, -3, 73], 1, Concat, [1]], + + [-1, 1, Conv, [256, 1, 1]], + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, Conv, [256, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [320, 1, 1]], # 102 + + [-1, 1, MP, []], + [-1, 1, Conv, [320, 1, 1]], + [-3, 1, Conv, [320, 1, 1]], + [-1, 1, Conv, [320, 3, 2]], + [[-1, -3, 59], 1, Concat, [1]], + + [-1, 1, Conv, [512, 1, 1]], + [-2, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, Conv, [512, 3, 1]], + [[-1, -3, -5, -7, -8], 1, Concat, [1]], + [-1, 1, Conv, [640, 1, 1]], # 117 + + [87, 1, Conv, [320, 3, 1]], + [102, 1, Conv, [640, 3, 1]], + [117, 1, Conv, [1280, 3, 1]], + + [[118,119,120], 1, IDetect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/test/yolov7-tracker/data/coco.yaml b/test/yolov7-tracker/data/coco.yaml new file mode 100644 index 0000000..a1d126c --- /dev/null +++ b/test/yolov7-tracker/data/coco.yaml @@ -0,0 +1,23 @@ +# COCO 2017 dataset http://cocodataset.org + +# download command/URL (optional) +download: bash ./scripts/get_coco.sh + +# train and val data as 1) directory: path/images/, 2) file: path/images.txt, or 3) list: [path1/images/, path2/images/] +train: ./coco/train2017.txt # 118287 images +val: ./coco/val2017.txt # 5000 images +test: ./coco/test-dev2017.txt # 20288 of 40670 images, submit to https://competitions.codalab.org/competitions/20794 + +# number of classes +nc: 80 + +# class names +names: [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', + 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', + 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', + 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', + 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', + 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', + 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', + 'hair drier', 'toothbrush' ] diff --git a/test/yolov7-tracker/data/hyp.scratch.custom.yaml b/test/yolov7-tracker/data/hyp.scratch.custom.yaml new file mode 100644 index 0000000..0ec17fe --- /dev/null +++ b/test/yolov7-tracker/data/hyp.scratch.custom.yaml @@ -0,0 +1,29 @@ +lr0: 0.01 # initial learning rate (SGD=1E-2, Adam=1E-3) +lrf: 0.1 # final OneCycleLR learning rate (lr0 * lrf) +momentum: 0.937 # SGD momentum/Adam beta1 +weight_decay: 0.0005 # optimizer weight decay 5e-4 +warmup_epochs: 3.0 # warmup epochs (fractions ok) +warmup_momentum: 0.8 # warmup initial momentum +warmup_bias_lr: 0.1 # warmup initial bias lr +box: 0.05 # box loss gain +cls: 0.3 # cls loss gain +cls_pw: 1.0 # cls BCELoss positive_weight +obj: 0.7 # obj loss gain (scale with pixels) +obj_pw: 1.0 # obj BCELoss positive_weight +iou_t: 0.20 # IoU training threshold +anchor_t: 4.0 # anchor-multiple threshold +fl_gamma: 0.0 # focal loss gamma (efficientDet default gamma=1.5) +hsv_h: 0.015 # image HSV-Hue augmentation (fraction) +hsv_s: 0.7 # image HSV-Saturation augmentation (fraction) +hsv_v: 0.4 # image HSV-Value augmentation (fraction) +degrees: 0.0 # image rotation (+/- deg) +translate: 0.2 # image translation (+/- fraction) +scale: 0.5 # image scale (+/- gain) +shear: 0.0 # image shear (+/- deg) +perspective: 0.0 # image perspective (+/- fraction), range 0-0.001 +flipud: 0.0 # image flip up-down (probability) +fliplr: 0.5 # image flip left-right (probability) +mosaic: 1.0 # image mosaic (probability) +mixup: 0.0 # image mixup (probability) +copy_paste: 0.0 # image copy paste (probability) +paste_in: 0.0 # image copy paste (probability) diff --git a/test/yolov7-tracker/data/hyp.scratch.p5.yaml b/test/yolov7-tracker/data/hyp.scratch.p5.yaml new file mode 100644 index 0000000..ca512b7 --- /dev/null +++ b/test/yolov7-tracker/data/hyp.scratch.p5.yaml @@ -0,0 +1,29 @@ +lr0: 0.01 # initial learning rate (SGD=1E-2, Adam=1E-3) +lrf: 0.1 # final OneCycleLR learning rate (lr0 * lrf) +momentum: 0.937 # SGD momentum/Adam beta1 +weight_decay: 0.0005 # optimizer weight decay 5e-4 +warmup_epochs: 3.0 # warmup epochs (fractions ok) +warmup_momentum: 0.8 # warmup initial momentum +warmup_bias_lr: 0.1 # warmup initial bias lr +box: 0.05 # box loss gain +cls: 0.3 # cls loss gain +cls_pw: 1.0 # cls BCELoss positive_weight +obj: 0.7 # obj loss gain (scale with pixels) +obj_pw: 1.0 # obj BCELoss positive_weight +iou_t: 0.20 # IoU training threshold +anchor_t: 4.0 # anchor-multiple threshold +fl_gamma: 0.0 # focal loss gamma (efficientDet default gamma=1.5) +hsv_h: 0.015 # image HSV-Hue augmentation (fraction) +hsv_s: 0.7 # image HSV-Saturation augmentation (fraction) +hsv_v: 0.4 # image HSV-Value augmentation (fraction) +degrees: 0.0 # image rotation (+/- deg) +translate: 0.2 # image translation (+/- fraction) +scale: 0.9 # image scale (+/- gain) +shear: 0.0 # image shear (+/- deg) +perspective: 0.0 # image perspective (+/- fraction), range 0-0.001 +flipud: 0.0 # image flip up-down (probability) +fliplr: 0.5 # image flip left-right (probability) +mosaic: 1.0 # image mosaic (probability) +mixup: 0.15 # image mixup (probability) +copy_paste: 0.0 # image copy paste (probability) +paste_in: 0.15 # image copy paste (probability) diff --git a/test/yolov7-tracker/data/hyp.scratch.p6.yaml b/test/yolov7-tracker/data/hyp.scratch.p6.yaml new file mode 100644 index 0000000..dcb55d6 --- /dev/null +++ b/test/yolov7-tracker/data/hyp.scratch.p6.yaml @@ -0,0 +1,29 @@ +lr0: 0.01 # initial learning rate (SGD=1E-2, Adam=1E-3) +lrf: 0.2 # final OneCycleLR learning rate (lr0 * lrf) +momentum: 0.937 # SGD momentum/Adam beta1 +weight_decay: 0.0005 # optimizer weight decay 5e-4 +warmup_epochs: 3.0 # warmup epochs (fractions ok) +warmup_momentum: 0.8 # warmup initial momentum +warmup_bias_lr: 0.1 # warmup initial bias lr +box: 0.05 # box loss gain +cls: 0.3 # cls loss gain +cls_pw: 1.0 # cls BCELoss positive_weight +obj: 0.7 # obj loss gain (scale with pixels) +obj_pw: 1.0 # obj BCELoss positive_weight +iou_t: 0.20 # IoU training threshold +anchor_t: 4.0 # anchor-multiple threshold +fl_gamma: 0.0 # focal loss gamma (efficientDet default gamma=1.5) +hsv_h: 0.015 # image HSV-Hue augmentation (fraction) +hsv_s: 0.7 # image HSV-Saturation augmentation (fraction) +hsv_v: 0.4 # image HSV-Value augmentation (fraction) +degrees: 0.0 # image rotation (+/- deg) +translate: 0.2 # image translation (+/- fraction) +scale: 0.9 # image scale (+/- gain) +shear: 0.0 # image shear (+/- deg) +perspective: 0.0 # image perspective (+/- fraction), range 0-0.001 +flipud: 0.0 # image flip up-down (probability) +fliplr: 0.5 # image flip left-right (probability) +mosaic: 1.0 # image mosaic (probability) +mixup: 0.15 # image mixup (probability) +copy_paste: 0.0 # image copy paste (probability) +paste_in: 0.15 # image copy paste (probability) diff --git a/test/yolov7-tracker/data/hyp.scratch.tiny.yaml b/test/yolov7-tracker/data/hyp.scratch.tiny.yaml new file mode 100644 index 0000000..b84fbfa --- /dev/null +++ b/test/yolov7-tracker/data/hyp.scratch.tiny.yaml @@ -0,0 +1,29 @@ +lr0: 0.01 # initial learning rate (SGD=1E-2, Adam=1E-3) +lrf: 0.01 # final OneCycleLR learning rate (lr0 * lrf) +momentum: 0.937 # SGD momentum/Adam beta1 +weight_decay: 0.0005 # optimizer weight decay 5e-4 +warmup_epochs: 3.0 # warmup epochs (fractions ok) +warmup_momentum: 0.8 # warmup initial momentum +warmup_bias_lr: 0.1 # warmup initial bias lr +box: 0.05 # box loss gain +cls: 0.5 # cls loss gain +cls_pw: 1.0 # cls BCELoss positive_weight +obj: 1.0 # obj loss gain (scale with pixels) +obj_pw: 1.0 # obj BCELoss positive_weight +iou_t: 0.20 # IoU training threshold +anchor_t: 4.0 # anchor-multiple threshold +fl_gamma: 0.0 # focal loss gamma (efficientDet default gamma=1.5) +hsv_h: 0.015 # image HSV-Hue augmentation (fraction) +hsv_s: 0.7 # image HSV-Saturation augmentation (fraction) +hsv_v: 0.4 # image HSV-Value augmentation (fraction) +degrees: 0.0 # image rotation (+/- deg) +translate: 0.1 # image translation (+/- fraction) +scale: 0.5 # image scale (+/- gain) +shear: 0.0 # image shear (+/- deg) +perspective: 0.0 # image perspective (+/- fraction), range 0-0.001 +flipud: 0.0 # image flip up-down (probability) +fliplr: 0.5 # image flip left-right (probability) +mosaic: 1.0 # image mosaic (probability) +mixup: 0.05 # image mixup (probability) +copy_paste: 0.0 # image copy paste (probability) +paste_in: 0.05 # image copy paste (probability) diff --git a/test/yolov7-tracker/data/mot17.yaml b/test/yolov7-tracker/data/mot17.yaml new file mode 100644 index 0000000..c21a4c9 --- /dev/null +++ b/test/yolov7-tracker/data/mot17.yaml @@ -0,0 +1,7 @@ +train: ./mot17/train.txt +val: ./mot17/val.txt +test: ./mot17/val.txt + +nc: 1 + +names: ['pedestrain'] \ No newline at end of file diff --git a/test/yolov7-tracker/data/uavdt.yaml b/test/yolov7-tracker/data/uavdt.yaml new file mode 100644 index 0000000..3f0e02e --- /dev/null +++ b/test/yolov7-tracker/data/uavdt.yaml @@ -0,0 +1,7 @@ +train: ./uavdt/train.txt +val: ./uavdt/test.txt +test: ./uavdt/test.txt + +nc: 1 + +names: ['car'] \ No newline at end of file diff --git a/test/yolov7-tracker/data/visdrone_all.yaml b/test/yolov7-tracker/data/visdrone_all.yaml new file mode 100644 index 0000000..99636d6 --- /dev/null +++ b/test/yolov7-tracker/data/visdrone_all.yaml @@ -0,0 +1,8 @@ + +train: ./visdrone/train.txt +val: ./visdrone/val.txt +test: ./visdrone/test.txt + +nc: 10 + +names: ['pedestrain', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor'] \ No newline at end of file diff --git a/test/yolov7-tracker/data/visdrone_half_car.yaml b/test/yolov7-tracker/data/visdrone_half_car.yaml new file mode 100644 index 0000000..800aa96 --- /dev/null +++ b/test/yolov7-tracker/data/visdrone_half_car.yaml @@ -0,0 +1,8 @@ + +train: ./visdrone/train.txt +val: ./visdrone/val.txt +test: ./visdrone/test.txt + +nc: 4 + +names: ['car', 'van', 'truck', 'bus'] \ No newline at end of file diff --git a/test/yolov7-tracker/detect.py b/test/yolov7-tracker/detect.py new file mode 100644 index 0000000..53b63eb --- /dev/null +++ b/test/yolov7-tracker/detect.py @@ -0,0 +1,184 @@ +import argparse +import time +from pathlib import Path + +import cv2 +import torch +import torch.backends.cudnn as cudnn +from numpy import random + +from models.experimental import attempt_load +from utils.datasets import LoadStreams, LoadImages +from utils.general import check_img_size, check_requirements, check_imshow, non_max_suppression, apply_classifier, \ + scale_coords, xyxy2xywh, strip_optimizer, set_logging, increment_path +from utils.plots import plot_one_box +from utils.torch_utils import select_device, load_classifier, time_synchronized, TracedModel + + +def detect(save_img=False): + source, weights, view_img, save_txt, imgsz, trace = opt.source, opt.weights, opt.view_img, opt.save_txt, opt.img_size, not opt.no_trace + save_img = not opt.nosave and not source.endswith('.txt') # save inference images + webcam = source.isnumeric() or source.endswith('.txt') or source.lower().startswith( + ('rtsp://', 'rtmp://', 'http://', 'https://')) + + # Directories + save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Initialize + set_logging() + device = select_device(opt.device) + half = device.type != 'cpu' # half precision only supported on CUDA + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + stride = int(model.stride.max()) # model stride + imgsz = check_img_size(imgsz, s=stride) # check img_size + + if trace: + model = TracedModel(model, device, opt.img_size) + + if half: + model.half() # to FP16 + + # Second-stage classifier + classify = False + if classify: + modelc = load_classifier(name='resnet101', n=2) # initialize + modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location=device)['model']).to(device).eval() + + # Set Dataloader + vid_path, vid_writer = None, None + if webcam: + view_img = check_imshow() + cudnn.benchmark = True # set True to speed up constant image size inference + dataset = LoadStreams(source, img_size=imgsz, stride=stride) + else: + dataset = LoadImages(source, img_size=imgsz, stride=stride) + + # Get names and colors + names = model.module.names if hasattr(model, 'module') else model.names + colors = [[random.randint(0, 255) for _ in range(3)] for _ in names] + + # Run inference + if device.type != 'cpu': + model(torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once + t0 = time.time() + for path, img, im0s, vid_cap in dataset: + img = torch.from_numpy(img).to(device) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + if img.ndimension() == 3: + img = img.unsqueeze(0) + + # Inference + t1 = time_synchronized() + pred = model(img, augment=opt.augment)[0] + + # Apply NMS + pred = non_max_suppression(pred, opt.conf_thres, opt.iou_thres, classes=opt.classes, agnostic=opt.agnostic_nms) + t2 = time_synchronized() + + # Apply Classifier + if classify: + pred = apply_classifier(pred, modelc, img, im0s) + + # Process detections + for i, det in enumerate(pred): # detections per image + if webcam: # batch_size >= 1 + p, s, im0, frame = path[i], '%g: ' % i, im0s[i].copy(), dataset.count + else: + p, s, im0, frame = path, '', im0s, getattr(dataset, 'frame', 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # img.jpg + txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # img.txt + s += '%gx%g ' % img.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round() + + # Print results + for c in det[:, -1].unique(): + n = (det[:, -1] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Write results + for *xyxy, conf, cls in reversed(det): + if save_txt: # Write to file + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if opt.save_conf else (cls, *xywh) # label format + with open(txt_path + '.txt', 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + if save_img or view_img: # Add bbox to image + label = f'{names[int(cls)]} {conf:.2f}' + plot_one_box(xyxy, im0, label=label, color=colors[int(cls)], line_thickness=3) + + # Print time (inference + NMS) + #print(f'{s}Done. ({t2 - t1:.3f}s)') + + # Stream results + if view_img: + cv2.imshow(str(p), im0) + cv2.waitKey(1) # 1 millisecond + + # Save results (image with detections) + if save_img: + if dataset.mode == 'image': + cv2.imwrite(save_path, im0) + print(f" The image with the result is saved in: {save_path}") + else: # 'video' or 'stream' + if vid_path != save_path: # new video + vid_path = save_path + if isinstance(vid_writer, cv2.VideoWriter): + vid_writer.release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path += '.mp4' + vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer.write(im0) + + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + #print(f"Results saved to {save_dir}{s}") + + print(f'Done. ({time.time() - t0:.3f}s)') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default='yolov7.pt', help='model.pt path(s)') + parser.add_argument('--source', type=str, default='inference/images', help='source') # file/folder, 0 for webcam + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.25, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='IOU threshold for NMS') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='display results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --class 0, or --class 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default='runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--no-trace', action='store_true', help='don`t trace model') + opt = parser.parse_args() + print(opt) + #check_requirements(exclude=('pycocotools', 'thop')) + + with torch.no_grad(): + if opt.update: # update all models (to fix SourceChangeWarning) + for opt.weights in ['yolov7.pt']: + detect() + strip_optimizer(opt.weights) + else: + detect() diff --git a/test/yolov7-tracker/figure/demo.gif b/test/yolov7-tracker/figure/demo.gif new file mode 100644 index 0000000..f05ede9 Binary files /dev/null and b/test/yolov7-tracker/figure/demo.gif differ diff --git a/test/yolov7-tracker/figure/horses_prediction.jpg b/test/yolov7-tracker/figure/horses_prediction.jpg new file mode 100644 index 0000000..0b95070 Binary files /dev/null and b/test/yolov7-tracker/figure/horses_prediction.jpg differ diff --git a/test/yolov7-tracker/figure/mask.png b/test/yolov7-tracker/figure/mask.png new file mode 100644 index 0000000..1a2743a Binary files /dev/null and b/test/yolov7-tracker/figure/mask.png differ diff --git a/test/yolov7-tracker/figure/performance.png b/test/yolov7-tracker/figure/performance.png new file mode 100644 index 0000000..58c0698 Binary files /dev/null and b/test/yolov7-tracker/figure/performance.png differ diff --git a/test/yolov7-tracker/figure/pose.png b/test/yolov7-tracker/figure/pose.png new file mode 100644 index 0000000..7bf288e Binary files /dev/null and b/test/yolov7-tracker/figure/pose.png differ diff --git a/test/yolov7-tracker/hubconf.py b/test/yolov7-tracker/hubconf.py new file mode 100644 index 0000000..f8a8cbe --- /dev/null +++ b/test/yolov7-tracker/hubconf.py @@ -0,0 +1,97 @@ +"""PyTorch Hub models + +Usage: + import torch + model = torch.hub.load('repo', 'model') +""" + +from pathlib import Path + +import torch + +from models.yolo import Model +from utils.general import check_requirements, set_logging +from utils.google_utils import attempt_download +from utils.torch_utils import select_device + +dependencies = ['torch', 'yaml'] +check_requirements(Path(__file__).parent / 'requirements.txt', exclude=('pycocotools', 'thop')) +set_logging() + + +def create(name, pretrained, channels, classes, autoshape): + """Creates a specified model + + Arguments: + name (str): name of model, i.e. 'yolov7' + pretrained (bool): load pretrained weights into the model + channels (int): number of input channels + classes (int): number of model classes + + Returns: + pytorch model + """ + try: + cfg = list((Path(__file__).parent / 'cfg').rglob(f'{name}.yaml'))[0] # model.yaml path + model = Model(cfg, channels, classes) + if pretrained: + fname = f'{name}.pt' # checkpoint filename + attempt_download(fname) # download if not found locally + ckpt = torch.load(fname, map_location=torch.device('cpu')) # load + msd = model.state_dict() # model state_dict + csd = ckpt['model'].float().state_dict() # checkpoint state_dict as FP32 + csd = {k: v for k, v in csd.items() if msd[k].shape == v.shape} # filter + model.load_state_dict(csd, strict=False) # load + if len(ckpt['model'].names) == classes: + model.names = ckpt['model'].names # set class names attribute + if autoshape: + model = model.autoshape() # for file/URI/PIL/cv2/np inputs and NMS + device = select_device('0' if torch.cuda.is_available() else 'cpu') # default to GPU if available + return model.to(device) + + except Exception as e: + s = 'Cache maybe be out of date, try force_reload=True.' + raise Exception(s) from e + + +def custom(path_or_model='path/to/model.pt', autoshape=True): + """custom mode + + Arguments (3 options): + path_or_model (str): 'path/to/model.pt' + path_or_model (dict): torch.load('path/to/model.pt') + path_or_model (nn.Module): torch.load('path/to/model.pt')['model'] + + Returns: + pytorch model + """ + model = torch.load(path_or_model) if isinstance(path_or_model, str) else path_or_model # load checkpoint + if isinstance(model, dict): + model = model['ema' if model.get('ema') else 'model'] # load model + + hub_model = Model(model.yaml).to(next(model.parameters()).device) # create + hub_model.load_state_dict(model.float().state_dict()) # load state_dict + hub_model.names = model.names # class names + if autoshape: + hub_model = hub_model.autoshape() # for file/URI/PIL/cv2/np inputs and NMS + device = select_device('0' if torch.cuda.is_available() else 'cpu') # default to GPU if available + return hub_model.to(device) + + +def yolov7(pretrained=True, channels=3, classes=80, autoshape=True): + return create('yolov7', pretrained, channels, classes, autoshape) + + +if __name__ == '__main__': + model = custom(path_or_model='yolov7.pt') # custom example + # model = create(name='yolov7', pretrained=True, channels=3, classes=80, autoshape=True) # pretrained example + + # Verify inference + import numpy as np + from PIL import Image + + imgs = [np.zeros((640, 480, 3))] + + results = model(imgs) # batched inference + results.print() + results.save() diff --git a/test/yolov7-tracker/inference/images/horses.jpg b/test/yolov7-tracker/inference/images/horses.jpg new file mode 100644 index 0000000..3a761f4 Binary files /dev/null and b/test/yolov7-tracker/inference/images/horses.jpg differ diff --git a/test/yolov7-tracker/models/__init__.py b/test/yolov7-tracker/models/__init__.py new file mode 100644 index 0000000..84952a8 --- /dev/null +++ b/test/yolov7-tracker/models/__init__.py @@ -0,0 +1 @@ +# init \ No newline at end of file diff --git a/test/yolov7-tracker/models/common.py b/test/yolov7-tracker/models/common.py new file mode 100644 index 0000000..53e3f87 --- /dev/null +++ b/test/yolov7-tracker/models/common.py @@ -0,0 +1,2019 @@ +import math +from copy import copy +from pathlib import Path + +import numpy as np +import pandas as pd +import requests +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.ops import DeformConv2d +from PIL import Image +from torch.cuda import amp + +from utils.datasets import letterbox +from utils.general import non_max_suppression, make_divisible, scale_coords, increment_path, xyxy2xywh +from utils.plots import color_list, plot_one_box +from utils.torch_utils import time_synchronized + + +##### basic #### + +def autopad(k, p=None): # kernel, padding + # Pad to 'same' + if p is None: + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad + return p + + +class MP(nn.Module): + def __init__(self, k=2): + super(MP, self).__init__() + self.m = nn.MaxPool2d(kernel_size=k, stride=k) + + def forward(self, x): + return self.m(x) + + +class SP(nn.Module): + def __init__(self, k=3, s=1): + super(SP, self).__init__() + self.m = nn.MaxPool2d(kernel_size=k, stride=s, padding=k // 2) + + def forward(self, x): + return self.m(x) + + +class ReOrg(nn.Module): + def __init__(self): + super(ReOrg, self).__init__() + + def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2) + return torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1) + + +class Concat(nn.Module): + def __init__(self, dimension=1): + super(Concat, self).__init__() + self.d = dimension + + def forward(self, x): + return torch.cat(x, self.d) + + +class Chuncat(nn.Module): + def __init__(self, dimension=1): + super(Chuncat, self).__init__() + self.d = dimension + + def forward(self, x): + x1 = [] + x2 = [] + for xi in x: + xi1, xi2 = xi.chunk(2, self.d) + x1.append(xi1) + x2.append(xi2) + return torch.cat(x1+x2, self.d) + + +class Shortcut(nn.Module): + def __init__(self, dimension=0): + super(Shortcut, self).__init__() + self.d = dimension + + def forward(self, x): + return x[0]+x[1] + + +class Foldcut(nn.Module): + def __init__(self, dimension=0): + super(Foldcut, self).__init__() + self.d = dimension + + def forward(self, x): + x1, x2 = x.chunk(2, self.d) + return x1+x2 + + +class Conv(nn.Module): + # Standard convolution + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super(Conv, self).__init__() + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False) + self.bn = nn.BatchNorm2d(c2) + self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity()) + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def fuseforward(self, x): + return self.act(self.conv(x)) + + +class RobustConv(nn.Module): + # Robust convolution (use high kernel size 7-11 for: downsampling and other layers). Train for 300 - 450 epochs. + def __init__(self, c1, c2, k=7, s=1, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups + super(RobustConv, self).__init__() + self.conv_dw = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act) + self.conv1x1 = nn.Conv2d(c1, c2, 1, 1, 0, groups=1, bias=True) + self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None + + def forward(self, x): + x = x.to(memory_format=torch.channels_last) + x = self.conv1x1(self.conv_dw(x)) + if self.gamma is not None: + x = x.mul(self.gamma.reshape(1, -1, 1, 1)) + return x + + +class RobustConv2(nn.Module): + # Robust convolution 2 (use [32, 5, 2] or [32, 7, 4] or [32, 11, 8] for one of the paths in CSP). + def __init__(self, c1, c2, k=7, s=4, p=None, g=1, act=True, layer_scale_init_value=1e-6): # ch_in, ch_out, kernel, stride, padding, groups + super(RobustConv2, self).__init__() + self.conv_strided = Conv(c1, c1, k=k, s=s, p=p, g=c1, act=act) + self.conv_deconv = nn.ConvTranspose2d(in_channels=c1, out_channels=c2, kernel_size=s, stride=s, + padding=0, bias=True, dilation=1, groups=1 + ) + self.gamma = nn.Parameter(layer_scale_init_value * torch.ones(c2)) if layer_scale_init_value > 0 else None + + def forward(self, x): + x = self.conv_deconv(self.conv_strided(x)) + if self.gamma is not None: + x = x.mul(self.gamma.reshape(1, -1, 1, 1)) + return x + + +def DWConv(c1, c2, k=1, s=1, act=True): + # Depthwise convolution + return Conv(c1, c2, k, s, g=math.gcd(c1, c2), act=act) + + +class GhostConv(nn.Module): + # Ghost Convolution https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups + super(GhostConv, self).__init__() + c_ = c2 // 2 # hidden channels + self.cv1 = Conv(c1, c_, k, s, None, g, act) + self.cv2 = Conv(c_, c_, 5, 1, None, c_, act) + + def forward(self, x): + y = self.cv1(x) + return torch.cat([y, self.cv2(y)], 1) + + +class Stem(nn.Module): + # Stem + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super(Stem, self).__init__() + c_ = int(c2/2) # hidden channels + self.cv1 = Conv(c1, c_, 3, 2) + self.cv2 = Conv(c_, c_, 1, 1) + self.cv3 = Conv(c_, c_, 3, 2) + self.pool = torch.nn.MaxPool2d(2, stride=2) + self.cv4 = Conv(2 * c_, c2, 1, 1) + + def forward(self, x): + x = self.cv1(x) + return self.cv4(torch.cat((self.cv3(self.cv2(x)), self.pool(x)), dim=1)) + + +class DownC(nn.Module): + # Spatial pyramid pooling layer used in YOLOv3-SPP + def __init__(self, c1, c2, n=1, k=2): + super(DownC, self).__init__() + c_ = int(c1) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c2//2, 3, k) + self.cv3 = Conv(c1, c2//2, 1, 1) + self.mp = nn.MaxPool2d(kernel_size=k, stride=k) + + def forward(self, x): + return torch.cat((self.cv2(self.cv1(x)), self.cv3(self.mp(x))), dim=1) + + +class SPP(nn.Module): + # Spatial pyramid pooling layer used in YOLOv3-SPP + def __init__(self, c1, c2, k=(5, 9, 13)): + super(SPP, self).__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1) + self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k]) + + def forward(self, x): + x = self.cv1(x) + return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1)) + + +class Bottleneck(nn.Module): + # Darknet bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super(Bottleneck, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c2, 3, 1, g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class Res(nn.Module): + # ResNet bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super(Res, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c_, 3, 1, g=g) + self.cv3 = Conv(c_, c2, 1, 1) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv3(self.cv2(self.cv1(x))) if self.add else self.cv3(self.cv2(self.cv1(x))) + + +class ResX(Res): + # ResNet bottleneck + def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super().__init__(c1, c2, shortcu, g, e) + c_ = int(c2 * e) # hidden channels + + +class Ghost(nn.Module): + # Ghost Bottleneck https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k=3, s=1): # ch_in, ch_out, kernel, stride + super(Ghost, self).__init__() + c_ = c2 // 2 + self.conv = nn.Sequential(GhostConv(c1, c_, 1, 1), # pw + DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw + GhostConv(c_, c2, 1, 1, act=False)) # pw-linear + self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False), + Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity() + + def forward(self, x): + return self.conv(x) + self.shortcut(x) + +##### end of basic ##### + + +##### cspnet ##### + +class SPPCSPC(nn.Module): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)): + super(SPPCSPC, self).__init__() + c_ = int(2 * c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(c_, c_, 3, 1) + self.cv4 = Conv(c_, c_, 1, 1) + self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k]) + self.cv5 = Conv(4 * c_, c_, 1, 1) + self.cv6 = Conv(c_, c_, 3, 1) + self.cv7 = Conv(2 * c_, c2, 1, 1) + + def forward(self, x): + x1 = self.cv4(self.cv3(self.cv1(x))) + y1 = self.cv6(self.cv5(torch.cat([x1] + [m(x1) for m in self.m], 1))) + y2 = self.cv2(x) + return self.cv7(torch.cat((y1, y2), dim=1)) + +class GhostSPPCSPC(SPPCSPC): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, k=(5, 9, 13)): + super().__init__(c1, c2, n, shortcut, g, e, k) + c_ = int(2 * c2 * e) # hidden channels + self.cv1 = GhostConv(c1, c_, 1, 1) + self.cv2 = GhostConv(c1, c_, 1, 1) + self.cv3 = GhostConv(c_, c_, 3, 1) + self.cv4 = GhostConv(c_, c_, 1, 1) + self.cv5 = GhostConv(4 * c_, c_, 1, 1) + self.cv6 = GhostConv(c_, c_, 3, 1) + self.cv7 = GhostConv(2 * c_, c2, 1, 1) + + +class GhostStem(Stem): + # Stem + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super().__init__(c1, c2, k, s, p, g, act) + c_ = int(c2/2) # hidden channels + self.cv1 = GhostConv(c1, c_, 3, 2) + self.cv2 = GhostConv(c_, c_, 1, 1) + self.cv3 = GhostConv(c_, c_, 3, 2) + self.cv4 = GhostConv(2 * c_, c2, 1, 1) + + +class BottleneckCSPA(nn.Module): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(BottleneckCSPA, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.m(self.cv1(x)) + y2 = self.cv2(x) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class BottleneckCSPB(nn.Module): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(BottleneckCSPB, self).__init__() + c_ = int(c2) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + x1 = self.cv1(x) + y1 = self.m(x1) + y2 = self.cv2(x1) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class BottleneckCSPC(nn.Module): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(BottleneckCSPC, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(c_, c_, 1, 1) + self.cv4 = Conv(2 * c_, c2, 1, 1) + self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(torch.cat((y1, y2), dim=1)) + + +class ResCSPA(BottleneckCSPA): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class ResCSPB(BottleneckCSPB): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class ResCSPC(BottleneckCSPC): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class ResXCSPA(ResCSPA): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class ResXCSPB(ResCSPB): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class ResXCSPC(ResCSPC): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Res(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class GhostCSPA(BottleneckCSPA): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)]) + + +class GhostCSPB(BottleneckCSPB): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)]) + + +class GhostCSPC(BottleneckCSPC): + # CSP https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[Ghost(c_, c_) for _ in range(n)]) + +##### end of cspnet ##### + + +##### yolor ##### + +class ImplicitA(nn.Module): + def __init__(self, channel, mean=0., std=.02): + super(ImplicitA, self).__init__() + self.channel = channel + self.mean = mean + self.std = std + self.implicit = nn.Parameter(torch.zeros(1, channel, 1, 1)) + nn.init.normal_(self.implicit, mean=self.mean, std=self.std) + + def forward(self, x): + return self.implicit + x + + +class ImplicitM(nn.Module): + def __init__(self, channel, mean=0., std=.02): + super(ImplicitM, self).__init__() + self.channel = channel + self.mean = mean + self.std = std + self.implicit = nn.Parameter(torch.ones(1, channel, 1, 1)) + nn.init.normal_(self.implicit, mean=self.mean, std=self.std) + + def forward(self, x): + return self.implicit * x + +##### end of yolor ##### + + +##### repvgg ##### + +class RepConv(nn.Module): + # Represented convolution + # https://arxiv.org/abs/2101.03697 + + def __init__(self, c1, c2, k=3, s=1, p=None, g=1, act=True, deploy=False): + super(RepConv, self).__init__() + + self.deploy = deploy + self.groups = g + self.in_channels = c1 + self.out_channels = c2 + + assert k == 3 + assert autopad(k, p) == 1 + + padding_11 = autopad(k, p) - k // 2 + + self.act = nn.SiLU() if act is True else (act if isinstance(act, nn.Module) else nn.Identity()) + + if deploy: + self.rbr_reparam = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=True) + + else: + self.rbr_identity = (nn.BatchNorm2d(num_features=c1) if c2 == c1 and s == 1 else None) + + self.rbr_dense = nn.Sequential( + nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g, bias=False), + nn.BatchNorm2d(num_features=c2), + ) + + self.rbr_1x1 = nn.Sequential( + nn.Conv2d( c1, c2, 1, s, padding_11, groups=g, bias=False), + nn.BatchNorm2d(num_features=c2), + ) + + def forward(self, inputs): + if hasattr(self, "rbr_reparam"): + return self.act(self.rbr_reparam(inputs)) + + if self.rbr_identity is None: + id_out = 0 + else: + id_out = self.rbr_identity(inputs) + + return self.act(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out) + + def get_equivalent_kernel_bias(self): + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) + kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) + return ( + kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, + bias3x3 + bias1x1 + biasid, + ) + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if isinstance(branch, nn.Sequential): + kernel = branch[0].weight + running_mean = branch[1].running_mean + running_var = branch[1].running_var + gamma = branch[1].weight + beta = branch[1].bias + eps = branch[1].eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, "id_tensor"): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros( + (self.in_channels, input_dim, 3, 3), dtype=np.float32 + ) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def repvgg_convert(self): + kernel, bias = self.get_equivalent_kernel_bias() + return ( + kernel.detach().cpu().numpy(), + bias.detach().cpu().numpy(), + ) + + def fuse_conv_bn(self, conv, bn): + + std = (bn.running_var + bn.eps).sqrt() + bias = bn.bias - bn.running_mean * bn.weight / std + + t = (bn.weight / std).reshape(-1, 1, 1, 1) + weights = conv.weight * t + + bn = nn.Identity() + conv = nn.Conv2d(in_channels = conv.in_channels, + out_channels = conv.out_channels, + kernel_size = conv.kernel_size, + stride=conv.stride, + padding = conv.padding, + dilation = conv.dilation, + groups = conv.groups, + bias = True, + padding_mode = conv.padding_mode) + + conv.weight = torch.nn.Parameter(weights) + conv.bias = torch.nn.Parameter(bias) + return conv + + def fuse_repvgg_block(self): + if self.deploy: + return + print(f"RepConv.fuse_repvgg_block") + + self.rbr_dense = self.fuse_conv_bn(self.rbr_dense[0], self.rbr_dense[1]) + + self.rbr_1x1 = self.fuse_conv_bn(self.rbr_1x1[0], self.rbr_1x1[1]) + rbr_1x1_bias = self.rbr_1x1.bias + weight_1x1_expanded = torch.nn.functional.pad(self.rbr_1x1.weight, [1, 1, 1, 1]) + + # Fuse self.rbr_identity + if (isinstance(self.rbr_identity, nn.BatchNorm2d) or isinstance(self.rbr_identity, nn.modules.batchnorm.SyncBatchNorm)): + # print(f"fuse: rbr_identity == BatchNorm2d or SyncBatchNorm") + identity_conv_1x1 = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=1, + stride=1, + padding=0, + groups=self.groups, + bias=False) + identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.to(self.rbr_1x1.weight.data.device) + identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.squeeze().squeeze() + # print(f" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}") + identity_conv_1x1.weight.data.fill_(0.0) + identity_conv_1x1.weight.data.fill_diagonal_(1.0) + identity_conv_1x1.weight.data = identity_conv_1x1.weight.data.unsqueeze(2).unsqueeze(3) + # print(f" identity_conv_1x1.weight = {identity_conv_1x1.weight.shape}") + + identity_conv_1x1 = self.fuse_conv_bn(identity_conv_1x1, self.rbr_identity) + bias_identity_expanded = identity_conv_1x1.bias + weight_identity_expanded = torch.nn.functional.pad(identity_conv_1x1.weight, [1, 1, 1, 1]) + else: + # print(f"fuse: rbr_identity != BatchNorm2d, rbr_identity = {self.rbr_identity}") + bias_identity_expanded = torch.nn.Parameter( torch.zeros_like(rbr_1x1_bias) ) + weight_identity_expanded = torch.nn.Parameter( torch.zeros_like(weight_1x1_expanded) ) + + + #print(f"self.rbr_1x1.weight = {self.rbr_1x1.weight.shape}, ") + #print(f"weight_1x1_expanded = {weight_1x1_expanded.shape}, ") + #print(f"self.rbr_dense.weight = {self.rbr_dense.weight.shape}, ") + + self.rbr_dense.weight = torch.nn.Parameter(self.rbr_dense.weight + weight_1x1_expanded + weight_identity_expanded) + self.rbr_dense.bias = torch.nn.Parameter(self.rbr_dense.bias + rbr_1x1_bias + bias_identity_expanded) + + self.rbr_reparam = self.rbr_dense + self.deploy = True + + if self.rbr_identity is not None: + del self.rbr_identity + self.rbr_identity = None + + if self.rbr_1x1 is not None: + del self.rbr_1x1 + self.rbr_1x1 = None + + if self.rbr_dense is not None: + del self.rbr_dense + self.rbr_dense = None + + +class RepBottleneck(Bottleneck): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super().__init__(c1, c2, shortcut=True, g=1, e=0.5) + c_ = int(c2 * e) # hidden channels + self.cv2 = RepConv(c_, c2, 3, 1, g=g) + + +class RepBottleneckCSPA(BottleneckCSPA): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class RepBottleneckCSPB(BottleneckCSPB): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class RepBottleneckCSPC(BottleneckCSPC): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + +class RepRes(Res): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super().__init__(c1, c2, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.cv2 = RepConv(c_, c_, 3, 1, g=g) + + +class RepResCSPA(ResCSPA): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class RepResCSPB(ResCSPB): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class RepResCSPC(ResCSPC): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepRes(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class RepResX(ResX): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=32, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super().__init__(c1, c2, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.cv2 = RepConv(c_, c_, 3, 1, g=g) + + +class RepResXCSPA(ResXCSPA): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class RepResXCSPB(ResXCSPB): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2) # hidden channels + self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + + +class RepResXCSPC(ResXCSPC): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=32, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*[RepResX(c_, c_, shortcut, g, e=0.5) for _ in range(n)]) + +##### end of repvgg ##### + + +##### transformer ##### + +class TransformerLayer(nn.Module): + # Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance) + def __init__(self, c, num_heads): + super().__init__() + self.q = nn.Linear(c, c, bias=False) + self.k = nn.Linear(c, c, bias=False) + self.v = nn.Linear(c, c, bias=False) + self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads) + self.fc1 = nn.Linear(c, c, bias=False) + self.fc2 = nn.Linear(c, c, bias=False) + + def forward(self, x): + x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x + x = self.fc2(self.fc1(x)) + x + return x + + +class TransformerBlock(nn.Module): + # Vision Transformer https://arxiv.org/abs/2010.11929 + def __init__(self, c1, c2, num_heads, num_layers): + super().__init__() + self.conv = None + if c1 != c2: + self.conv = Conv(c1, c2) + self.linear = nn.Linear(c2, c2) # learnable position embedding + self.tr = nn.Sequential(*[TransformerLayer(c2, num_heads) for _ in range(num_layers)]) + self.c2 = c2 + + def forward(self, x): + if self.conv is not None: + x = self.conv(x) + b, _, w, h = x.shape + p = x.flatten(2) + p = p.unsqueeze(0) + p = p.transpose(0, 3) + p = p.squeeze(3) + e = self.linear(p) + x = p + e + + x = self.tr(x) + x = x.unsqueeze(3) + x = x.transpose(0, 3) + x = x.reshape(b, self.c2, w, h) + return x + +##### end of transformer ##### + + +##### yolov5 ##### + +class Focus(nn.Module): + # Focus wh information into c-space + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super(Focus, self).__init__() + self.conv = Conv(c1 * 4, c2, k, s, p, g, act) + # self.contract = Contract(gain=2) + + def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2) + return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1)) + # return self.conv(self.contract(x)) + + +class SPPF(nn.Module): + # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher + def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13)) + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * 4, c2, 1, 1) + self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + + def forward(self, x): + x = self.cv1(x) + y1 = self.m(x) + y2 = self.m(y1) + return self.cv2(torch.cat([x, y1, y2, self.m(y2)], 1)) + + +class Contract(nn.Module): + # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40) + def __init__(self, gain=2): + super().__init__() + self.gain = gain + + def forward(self, x): + N, C, H, W = x.size() # assert (H / s == 0) and (W / s == 0), 'Indivisible gain' + s = self.gain + x = x.view(N, C, H // s, s, W // s, s) # x(1,64,40,2,40,2) + x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40) + return x.view(N, C * s * s, H // s, W // s) # x(1,256,40,40) + + +class Expand(nn.Module): + # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160) + def __init__(self, gain=2): + super().__init__() + self.gain = gain + + def forward(self, x): + N, C, H, W = x.size() # assert C / s ** 2 == 0, 'Indivisible gain' + s = self.gain + x = x.view(N, s, s, C // s ** 2, H, W) # x(1,2,2,16,80,80) + x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2) + return x.view(N, C // s ** 2, H * s, W * s) # x(1,16,160,160) + + +class NMS(nn.Module): + # Non-Maximum Suppression (NMS) module + conf = 0.25 # confidence threshold + iou = 0.45 # IoU threshold + classes = None # (optional list) filter by class + + def __init__(self): + super(NMS, self).__init__() + + def forward(self, x): + return non_max_suppression(x[0], conf_thres=self.conf, iou_thres=self.iou, classes=self.classes) + + +class autoShape(nn.Module): + # input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS + conf = 0.25 # NMS confidence threshold + iou = 0.45 # NMS IoU threshold + classes = None # (optional list) filter by class + + def __init__(self, model): + super(autoShape, self).__init__() + self.model = model.eval() + + def autoshape(self): + print('autoShape already enabled, skipping... ') # model already converted to model.autoshape() + return self + + @torch.no_grad() + def forward(self, imgs, size=640, augment=False, profile=False): + # Inference from various sources. For height=640, width=1280, RGB images example inputs are: + # filename: imgs = 'data/samples/zidane.jpg' + # URI: = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/zidane.jpg' + # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3) + # PIL: = Image.open('image.jpg') # HWC x(640,1280,3) + # numpy: = np.zeros((640,1280,3)) # HWC + # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values) + # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images + + t = [time_synchronized()] + p = next(self.model.parameters()) # for device and type + if isinstance(imgs, torch.Tensor): # torch + with amp.autocast(enabled=p.device.type != 'cpu'): + return self.model(imgs.to(p.device).type_as(p), augment, profile) # inference + + # Pre-process + n, imgs = (len(imgs), imgs) if isinstance(imgs, list) else (1, [imgs]) # number of images, list of images + shape0, shape1, files = [], [], [] # image and inference shapes, filenames + for i, im in enumerate(imgs): + f = f'image{i}' # filename + if isinstance(im, str): # filename or uri + im, f = np.asarray(Image.open(requests.get(im, stream=True).raw if im.startswith('http') else im)), im + elif isinstance(im, Image.Image): # PIL Image + im, f = np.asarray(im), getattr(im, 'filename', f) or f + files.append(Path(f).with_suffix('.jpg').name) + if im.shape[0] < 5: # image in CHW + im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1) + im = im[:, :, :3] if im.ndim == 3 else np.tile(im[:, :, None], 3) # enforce 3ch input + s = im.shape[:2] # HWC + shape0.append(s) # image shape + g = (size / max(s)) # gain + shape1.append([y * g for y in s]) + imgs[i] = im # update + shape1 = [make_divisible(x, int(self.stride.max())) for x in np.stack(shape1, 0).max(0)] # inference shape + x = [letterbox(im, new_shape=shape1, auto=False)[0] for im in imgs] # pad + x = np.stack(x, 0) if n > 1 else x[0][None] # stack + x = np.ascontiguousarray(x.transpose((0, 3, 1, 2))) # BHWC to BCHW + x = torch.from_numpy(x).to(p.device).type_as(p) / 255. # uint8 to fp16/32 + t.append(time_synchronized()) + + with amp.autocast(enabled=p.device.type != 'cpu'): + # Inference + y = self.model(x, augment, profile)[0] # forward + t.append(time_synchronized()) + + # Post-process + y = non_max_suppression(y, conf_thres=self.conf, iou_thres=self.iou, classes=self.classes) # NMS + for i in range(n): + scale_coords(shape1, y[i][:, :4], shape0[i]) + + t.append(time_synchronized()) + return Detections(imgs, y, files, t, self.names, x.shape) + + +class Detections: + # detections class for YOLOv5 inference results + def __init__(self, imgs, pred, files, times=None, names=None, shape=None): + super(Detections, self).__init__() + d = pred[0].device # device + gn = [torch.tensor([*[im.shape[i] for i in [1, 0, 1, 0]], 1., 1.], device=d) for im in imgs] # normalizations + self.imgs = imgs # list of images as numpy arrays + self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls) + self.names = names # class names + self.files = files # image filenames + self.xyxy = pred # xyxy pixels + self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels + self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized + self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized + self.n = len(self.pred) # number of images (batch size) + self.t = tuple((times[i + 1] - times[i]) * 1000 / self.n for i in range(3)) # timestamps (ms) + self.s = shape # inference BCHW shape + + def display(self, pprint=False, show=False, save=False, render=False, save_dir=''): + colors = color_list() + for i, (img, pred) in enumerate(zip(self.imgs, self.pred)): + str = f'image {i + 1}/{len(self.pred)}: {img.shape[0]}x{img.shape[1]} ' + if pred is not None: + for c in pred[:, -1].unique(): + n = (pred[:, -1] == c).sum() # detections per class + str += f"{n} {self.names[int(c)]}{'s' * (n > 1)}, " # add to string + if show or save or render: + for *box, conf, cls in pred: # xyxy, confidence, class + label = f'{self.names[int(cls)]} {conf:.2f}' + plot_one_box(box, img, label=label, color=colors[int(cls) % 10]) + img = Image.fromarray(img.astype(np.uint8)) if isinstance(img, np.ndarray) else img # from np + if pprint: + print(str.rstrip(', ')) + if show: + img.show(self.files[i]) # show + if save: + f = self.files[i] + img.save(Path(save_dir) / f) # save + print(f"{'Saved' * (i == 0)} {f}", end=',' if i < self.n - 1 else f' to {save_dir}\n') + if render: + self.imgs[i] = np.asarray(img) + + def print(self): + self.display(pprint=True) # print results + print(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {tuple(self.s)}' % self.t) + + def show(self): + self.display(show=True) # show results + + def save(self, save_dir='runs/hub/exp'): + save_dir = increment_path(save_dir, exist_ok=save_dir != 'runs/hub/exp') # increment save_dir + Path(save_dir).mkdir(parents=True, exist_ok=True) + self.display(save=True, save_dir=save_dir) # save results + + def render(self): + self.display(render=True) # render results + return self.imgs + + def pandas(self): + # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0]) + new = copy(self) # return copy + ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns + cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns + for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]): + a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update + setattr(new, k, [pd.DataFrame(x, columns=c) for x in a]) + return new + + def tolist(self): + # return a list of Detections objects, i.e. 'for result in results.tolist():' + x = [Detections([self.imgs[i]], [self.pred[i]], self.names, self.s) for i in range(self.n)] + for d in x: + for k in ['imgs', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']: + setattr(d, k, getattr(d, k)[0]) # pop out of list + return x + + def __len__(self): + return self.n + + +class Classify(nn.Module): + # Classification head, i.e. x(b,c1,20,20) to x(b,c2) + def __init__(self, c1, c2, k=1, s=1, p=None, g=1): # ch_in, ch_out, kernel, stride, padding, groups + super(Classify, self).__init__() + self.aap = nn.AdaptiveAvgPool2d(1) # to x(b,c1,1,1) + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p), groups=g) # to x(b,c2,1,1) + self.flat = nn.Flatten() + + def forward(self, x): + z = torch.cat([self.aap(y) for y in (x if isinstance(x, list) else [x])], 1) # cat if list + return self.flat(self.conv(z)) # flatten to x(b,c2) + +##### end of yolov5 ###### + + +##### orepa ##### + +def transI_fusebn(kernel, bn): + gamma = bn.weight + std = (bn.running_var + bn.eps).sqrt() + return kernel * ((gamma / std).reshape(-1, 1, 1, 1)), bn.bias - bn.running_mean * gamma / std + + +class ConvBN(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, + stride=1, padding=0, dilation=1, groups=1, deploy=False, nonlinear=None): + super().__init__() + if nonlinear is None: + self.nonlinear = nn.Identity() + else: + self.nonlinear = nonlinear + if deploy: + self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, + stride=stride, padding=padding, dilation=dilation, groups=groups, bias=True) + else: + self.conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, + stride=stride, padding=padding, dilation=dilation, groups=groups, bias=False) + self.bn = nn.BatchNorm2d(num_features=out_channels) + + def forward(self, x): + if hasattr(self, 'bn'): + return self.nonlinear(self.bn(self.conv(x))) + else: + return self.nonlinear(self.conv(x)) + + def switch_to_deploy(self): + kernel, bias = transI_fusebn(self.conv.weight, self.bn) + conv = nn.Conv2d(in_channels=self.conv.in_channels, out_channels=self.conv.out_channels, kernel_size=self.conv.kernel_size, + stride=self.conv.stride, padding=self.conv.padding, dilation=self.conv.dilation, groups=self.conv.groups, bias=True) + conv.weight.data = kernel + conv.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('conv') + self.__delattr__('bn') + self.conv = conv + +class OREPA_3x3_RepConv(nn.Module): + + def __init__(self, in_channels, out_channels, kernel_size, + stride=1, padding=0, dilation=1, groups=1, + internal_channels_1x1_3x3=None, + deploy=False, nonlinear=None, single_init=False): + super(OREPA_3x3_RepConv, self).__init__() + self.deploy = deploy + + if nonlinear is None: + self.nonlinear = nn.Identity() + else: + self.nonlinear = nonlinear + + self.kernel_size = kernel_size + self.in_channels = in_channels + self.out_channels = out_channels + self.groups = groups + assert padding == kernel_size // 2 + + self.stride = stride + self.padding = padding + self.dilation = dilation + + self.branch_counter = 0 + + self.weight_rbr_origin = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), kernel_size, kernel_size)) + nn.init.kaiming_uniform_(self.weight_rbr_origin, a=math.sqrt(1.0)) + self.branch_counter += 1 + + + if groups < out_channels: + self.weight_rbr_avg_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1)) + self.weight_rbr_pfir_conv = nn.Parameter(torch.Tensor(out_channels, int(in_channels/self.groups), 1, 1)) + nn.init.kaiming_uniform_(self.weight_rbr_avg_conv, a=1.0) + nn.init.kaiming_uniform_(self.weight_rbr_pfir_conv, a=1.0) + self.weight_rbr_avg_conv.data + self.weight_rbr_pfir_conv.data + self.register_buffer('weight_rbr_avg_avg', torch.ones(kernel_size, kernel_size).mul(1.0/kernel_size/kernel_size)) + self.branch_counter += 1 + + else: + raise NotImplementedError + self.branch_counter += 1 + + if internal_channels_1x1_3x3 is None: + internal_channels_1x1_3x3 = in_channels if groups < out_channels else 2 * in_channels # For mobilenet, it is better to have 2X internal channels + + if internal_channels_1x1_3x3 == in_channels: + self.weight_rbr_1x1_kxk_idconv1 = nn.Parameter(torch.zeros(in_channels, int(in_channels/self.groups), 1, 1)) + id_value = np.zeros((in_channels, int(in_channels/self.groups), 1, 1)) + for i in range(in_channels): + id_value[i, i % int(in_channels/self.groups), 0, 0] = 1 + id_tensor = torch.from_numpy(id_value).type_as(self.weight_rbr_1x1_kxk_idconv1) + self.register_buffer('id_tensor', id_tensor) + + else: + self.weight_rbr_1x1_kxk_conv1 = nn.Parameter(torch.Tensor(internal_channels_1x1_3x3, int(in_channels/self.groups), 1, 1)) + nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv1, a=math.sqrt(1.0)) + self.weight_rbr_1x1_kxk_conv2 = nn.Parameter(torch.Tensor(out_channels, int(internal_channels_1x1_3x3/self.groups), kernel_size, kernel_size)) + nn.init.kaiming_uniform_(self.weight_rbr_1x1_kxk_conv2, a=math.sqrt(1.0)) + self.branch_counter += 1 + + expand_ratio = 8 + self.weight_rbr_gconv_dw = nn.Parameter(torch.Tensor(in_channels*expand_ratio, 1, kernel_size, kernel_size)) + self.weight_rbr_gconv_pw = nn.Parameter(torch.Tensor(out_channels, in_channels*expand_ratio, 1, 1)) + nn.init.kaiming_uniform_(self.weight_rbr_gconv_dw, a=math.sqrt(1.0)) + nn.init.kaiming_uniform_(self.weight_rbr_gconv_pw, a=math.sqrt(1.0)) + self.branch_counter += 1 + + if out_channels == in_channels and stride == 1: + self.branch_counter += 1 + + self.vector = nn.Parameter(torch.Tensor(self.branch_counter, self.out_channels)) + self.bn = nn.BatchNorm2d(out_channels) + + self.fre_init() + + nn.init.constant_(self.vector[0, :], 0.25) #origin + nn.init.constant_(self.vector[1, :], 0.25) #avg + nn.init.constant_(self.vector[2, :], 0.0) #prior + nn.init.constant_(self.vector[3, :], 0.5) #1x1_kxk + nn.init.constant_(self.vector[4, :], 0.5) #dws_conv + + + def fre_init(self): + prior_tensor = torch.Tensor(self.out_channels, self.kernel_size, self.kernel_size) + half_fg = self.out_channels/2 + for i in range(self.out_channels): + for h in range(3): + for w in range(3): + if i < half_fg: + prior_tensor[i, h, w] = math.cos(math.pi*(h+0.5)*(i+1)/3) + else: + prior_tensor[i, h, w] = math.cos(math.pi*(w+0.5)*(i+1-half_fg)/3) + + self.register_buffer('weight_rbr_prior', prior_tensor) + + def weight_gen(self): + + weight_rbr_origin = torch.einsum('oihw,o->oihw', self.weight_rbr_origin, self.vector[0, :]) + + weight_rbr_avg = torch.einsum('oihw,o->oihw', torch.einsum('oihw,hw->oihw', self.weight_rbr_avg_conv, self.weight_rbr_avg_avg), self.vector[1, :]) + + weight_rbr_pfir = torch.einsum('oihw,o->oihw', torch.einsum('oihw,ohw->oihw', self.weight_rbr_pfir_conv, self.weight_rbr_prior), self.vector[2, :]) + + weight_rbr_1x1_kxk_conv1 = None + if hasattr(self, 'weight_rbr_1x1_kxk_idconv1'): + weight_rbr_1x1_kxk_conv1 = (self.weight_rbr_1x1_kxk_idconv1 + self.id_tensor).squeeze() + elif hasattr(self, 'weight_rbr_1x1_kxk_conv1'): + weight_rbr_1x1_kxk_conv1 = self.weight_rbr_1x1_kxk_conv1.squeeze() + else: + raise NotImplementedError + weight_rbr_1x1_kxk_conv2 = self.weight_rbr_1x1_kxk_conv2 + + if self.groups > 1: + g = self.groups + t, ig = weight_rbr_1x1_kxk_conv1.size() + o, tg, h, w = weight_rbr_1x1_kxk_conv2.size() + weight_rbr_1x1_kxk_conv1 = weight_rbr_1x1_kxk_conv1.view(g, int(t/g), ig) + weight_rbr_1x1_kxk_conv2 = weight_rbr_1x1_kxk_conv2.view(g, int(o/g), tg, h, w) + weight_rbr_1x1_kxk = torch.einsum('gti,gothw->goihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2).view(o, ig, h, w) + else: + weight_rbr_1x1_kxk = torch.einsum('ti,othw->oihw', weight_rbr_1x1_kxk_conv1, weight_rbr_1x1_kxk_conv2) + + weight_rbr_1x1_kxk = torch.einsum('oihw,o->oihw', weight_rbr_1x1_kxk, self.vector[3, :]) + + weight_rbr_gconv = self.dwsc2full(self.weight_rbr_gconv_dw, self.weight_rbr_gconv_pw, self.in_channels) + weight_rbr_gconv = torch.einsum('oihw,o->oihw', weight_rbr_gconv, self.vector[4, :]) + + weight = weight_rbr_origin + weight_rbr_avg + weight_rbr_1x1_kxk + weight_rbr_pfir + weight_rbr_gconv + + return weight + + def dwsc2full(self, weight_dw, weight_pw, groups): + + t, ig, h, w = weight_dw.size() + o, _, _, _ = weight_pw.size() + tg = int(t/groups) + i = int(ig*groups) + weight_dw = weight_dw.view(groups, tg, ig, h, w) + weight_pw = weight_pw.squeeze().view(o, groups, tg) + + weight_dsc = torch.einsum('gtihw,ogt->ogihw', weight_dw, weight_pw) + return weight_dsc.view(o, i, h, w) + + def forward(self, inputs): + weight = self.weight_gen() + out = F.conv2d(inputs, weight, bias=None, stride=self.stride, padding=self.padding, dilation=self.dilation, groups=self.groups) + + return self.nonlinear(self.bn(out)) + +class RepConv_OREPA(nn.Module): + + def __init__(self, c1, c2, k=3, s=1, padding=1, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False, nonlinear=nn.SiLU()): + super(RepConv_OREPA, self).__init__() + self.deploy = deploy + self.groups = groups + self.in_channels = c1 + self.out_channels = c2 + + self.padding = padding + self.dilation = dilation + self.groups = groups + + assert k == 3 + assert padding == 1 + + padding_11 = padding - k // 2 + + if nonlinear is None: + self.nonlinearity = nn.Identity() + else: + self.nonlinearity = nonlinear + + if use_se: + self.se = SEBlock(self.out_channels, internal_neurons=self.out_channels // 16) + else: + self.se = nn.Identity() + + if deploy: + self.rbr_reparam = nn.Conv2d(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s, + padding=padding, dilation=dilation, groups=groups, bias=True, padding_mode=padding_mode) + + else: + self.rbr_identity = nn.BatchNorm2d(num_features=self.in_channels) if self.out_channels == self.in_channels and s == 1 else None + self.rbr_dense = OREPA_3x3_RepConv(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=k, stride=s, padding=padding, groups=groups, dilation=1) + self.rbr_1x1 = ConvBN(in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=1, stride=s, padding=padding_11, groups=groups, dilation=1) + print('RepVGG Block, identity = ', self.rbr_identity) + + + def forward(self, inputs): + if hasattr(self, 'rbr_reparam'): + return self.nonlinearity(self.se(self.rbr_reparam(inputs))) + + if self.rbr_identity is None: + id_out = 0 + else: + id_out = self.rbr_identity(inputs) + + out1 = self.rbr_dense(inputs) + out2 = self.rbr_1x1(inputs) + out3 = id_out + out = out1 + out2 + out3 + + return self.nonlinearity(self.se(out)) + + + # Optional. This improves the accuracy and facilitates quantization. + # 1. Cancel the original weight decay on rbr_dense.conv.weight and rbr_1x1.conv.weight. + # 2. Use like this. + # loss = criterion(....) + # for every RepVGGBlock blk: + # loss += weight_decay_coefficient * 0.5 * blk.get_cust_L2() + # optimizer.zero_grad() + # loss.backward() + + # Not used for OREPA + def get_custom_L2(self): + K3 = self.rbr_dense.weight_gen() + K1 = self.rbr_1x1.conv.weight + t3 = (self.rbr_dense.bn.weight / ((self.rbr_dense.bn.running_var + self.rbr_dense.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach() + t1 = (self.rbr_1x1.bn.weight / ((self.rbr_1x1.bn.running_var + self.rbr_1x1.bn.eps).sqrt())).reshape(-1, 1, 1, 1).detach() + + l2_loss_circle = (K3 ** 2).sum() - (K3[:, :, 1:2, 1:2] ** 2).sum() # The L2 loss of the "circle" of weights in 3x3 kernel. Use regular L2 on them. + eq_kernel = K3[:, :, 1:2, 1:2] * t3 + K1 * t1 # The equivalent resultant central point of 3x3 kernel. + l2_loss_eq_kernel = (eq_kernel ** 2 / (t3 ** 2 + t1 ** 2)).sum() # Normalize for an L2 coefficient comparable to regular L2. + return l2_loss_eq_kernel + l2_loss_circle + + def get_equivalent_kernel_bias(self): + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1) + kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity) + return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1,1,1,1]) + + def _fuse_bn_tensor(self, branch): + if branch is None: + return 0, 0 + if not isinstance(branch, nn.BatchNorm2d): + if isinstance(branch, OREPA_3x3_RepConv): + kernel = branch.weight_gen() + elif isinstance(branch, ConvBN): + kernel = branch.conv.weight + else: + raise NotImplementedError + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + if not hasattr(self, 'id_tensor'): + input_dim = self.in_channels // self.groups + kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32) + for i in range(self.in_channels): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def switch_to_deploy(self): + if hasattr(self, 'rbr_reparam'): + return + print(f"RepConv_OREPA.switch_to_deploy") + kernel, bias = self.get_equivalent_kernel_bias() + self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.in_channels, out_channels=self.rbr_dense.out_channels, + kernel_size=self.rbr_dense.kernel_size, stride=self.rbr_dense.stride, + padding=self.rbr_dense.padding, dilation=self.rbr_dense.dilation, groups=self.rbr_dense.groups, bias=True) + self.rbr_reparam.weight.data = kernel + self.rbr_reparam.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__('rbr_dense') + self.__delattr__('rbr_1x1') + if hasattr(self, 'rbr_identity'): + self.__delattr__('rbr_identity') + +##### end of orepa ##### + + +##### swin transformer ##### + +class WindowAttention(nn.Module): + + def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + nn.init.normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + # print(attn.dtype, v.dtype) + try: + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + except: + #print(attn.dtype, v.dtype) + x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class Mlp(nn.Module): + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + +def window_partition(x, window_size): + + B, H, W, C = x.shape + assert H % window_size == 0, 'feature map h and w can not divide by window size' + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + +def window_reverse(windows, window_size, H, W): + + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class SwinTransformerLayer(nn.Module): + + def __init__(self, dim, num_heads, window_size=8, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.SiLU, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + # if min(self.input_resolution) <= self.window_size: + # # if window size is larger than input resolution, we don't partition windows + # self.shift_size = 0 + # self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, window_size=(self.window_size, self.window_size), num_heads=num_heads, + qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def create_mask(self, H, W): + # calculate attention mask for SW-MSA + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) + + return attn_mask + + def forward(self, x): + # reshape x[b c h w] to x[b l c] + _, _, H_, W_ = x.shape + + Padding = False + if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0: + Padding = True + # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.') + pad_r = (self.window_size - W_ % self.window_size) % self.window_size + pad_b = (self.window_size - H_ % self.window_size) % self.window_size + x = F.pad(x, (0, pad_r, 0, pad_b)) + + # print('2', x.shape) + B, C, H, W = x.shape + L = H * W + x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c + + # create mask from init to forward + if self.shift_size > 0: + attn_mask = self.create_mask(H, W).to(x.device) + else: + attn_mask = None + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w + + if Padding: + x = x[:, :, :H_, :W_] # reverse padding + + return x + + +class SwinTransformerBlock(nn.Module): + def __init__(self, c1, c2, num_heads, num_layers, window_size=8): + super().__init__() + self.conv = None + if c1 != c2: + self.conv = Conv(c1, c2) + + # remove input_resolution + self.blocks = nn.Sequential(*[SwinTransformerLayer(dim=c2, num_heads=num_heads, window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)]) + + def forward(self, x): + if self.conv is not None: + x = self.conv(x) + x = self.blocks(x) + return x + + +class STCSPA(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(STCSPA, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformerBlock(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.m(self.cv1(x)) + y2 = self.cv2(x) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class STCSPB(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(STCSPB, self).__init__() + c_ = int(c2) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformerBlock(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + x1 = self.cv1(x) + y1 = self.m(x1) + y2 = self.cv2(x1) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class STCSPC(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(STCSPC, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(c_, c_, 1, 1) + self.cv4 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformerBlock(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(torch.cat((y1, y2), dim=1)) + +##### end of swin transformer ##### + + +##### swin transformer v2 ##### + +class WindowAttention_v2(nn.Module): + + def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0., + pretrained_window_size=[0, 0]): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.pretrained_window_size = pretrained_window_size + self.num_heads = num_heads + + self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True) + + # mlp to generate continuous relative position bias + self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True), + nn.ReLU(inplace=True), + nn.Linear(512, num_heads, bias=False)) + + # get relative_coords_table + relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32) + relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32) + relative_coords_table = torch.stack( + torch.meshgrid([relative_coords_h, + relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0) # 1, 2*Wh-1, 2*Ww-1, 2 + if pretrained_window_size[0] > 0: + relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1) + relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1) + else: + relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1) + relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1) + relative_coords_table *= 8 # normalize to -8, 8 + relative_coords_table = torch.sign(relative_coords_table) * torch.log2( + torch.abs(relative_coords_table) + 1.0) / np.log2(8) + + self.register_buffer("relative_coords_table", relative_coords_table) + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=False) + if qkv_bias: + self.q_bias = nn.Parameter(torch.zeros(dim)) + self.v_bias = nn.Parameter(torch.zeros(dim)) + else: + self.q_bias = None + self.v_bias = None + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + + B_, N, C = x.shape + qkv_bias = None + if self.q_bias is not None: + qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias)) + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + # cosine attention + attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1)) + logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp() + attn = attn * logit_scale + + relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads) + relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + relative_position_bias = 16 * torch.sigmoid(relative_position_bias) + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + try: + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + except: + x = (attn.half() @ v).transpose(1, 2).reshape(B_, N, C) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f'dim={self.dim}, window_size={self.window_size}, ' \ + f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}' + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + +class Mlp_v2(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition_v2(x, window_size): + + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse_v2(windows, window_size, H, W): + + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class SwinTransformerLayer_v2(nn.Module): + + def __init__(self, dim, num_heads, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.SiLU, norm_layer=nn.LayerNorm, pretrained_window_size=0): + super().__init__() + self.dim = dim + #self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + #if min(self.input_resolution) <= self.window_size: + # # if window size is larger than input resolution, we don't partition windows + # self.shift_size = 0 + # self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention_v2( + dim, window_size=(self.window_size, self.window_size), num_heads=num_heads, + qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop, + pretrained_window_size=(pretrained_window_size, pretrained_window_size)) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp_v2(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def create_mask(self, H, W): + # calculate attention mask for SW-MSA + img_mask = torch.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) + + return attn_mask + + def forward(self, x): + # reshape x[b c h w] to x[b l c] + _, _, H_, W_ = x.shape + + Padding = False + if min(H_, W_) < self.window_size or H_ % self.window_size!=0 or W_ % self.window_size!=0: + Padding = True + # print(f'img_size {min(H_, W_)} is less than (or not divided by) window_size {self.window_size}, Padding.') + pad_r = (self.window_size - W_ % self.window_size) % self.window_size + pad_b = (self.window_size - H_ % self.window_size) % self.window_size + x = F.pad(x, (0, pad_r, 0, pad_b)) + + # print('2', x.shape) + B, C, H, W = x.shape + L = H * W + x = x.permute(0, 2, 3, 1).contiguous().view(B, L, C) # b, L, c + + # create mask from init to forward + if self.shift_size > 0: + attn_mask = self.create_mask(H, W).to(x.device) + else: + attn_mask = None + + shortcut = x + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition_v2(shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse_v2(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + x = shortcut + self.drop_path(self.norm1(x)) + + # FFN + x = x + self.drop_path(self.norm2(self.mlp(x))) + x = x.permute(0, 2, 1).contiguous().view(-1, C, H, W) # b c h w + + if Padding: + x = x[:, :, :H_, :W_] # reverse padding + + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " \ + f"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}" + + def flops(self): + flops = 0 + H, W = self.input_resolution + # norm1 + flops += self.dim * H * W + # W-MSA/SW-MSA + nW = H * W / self.window_size / self.window_size + flops += nW * self.attn.flops(self.window_size * self.window_size) + # mlp + flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + return flops + + +class SwinTransformer2Block(nn.Module): + def __init__(self, c1, c2, num_heads, num_layers, window_size=7): + super().__init__() + self.conv = None + if c1 != c2: + self.conv = Conv(c1, c2) + + # remove input_resolution + self.blocks = nn.Sequential(*[SwinTransformerLayer_v2(dim=c2, num_heads=num_heads, window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2) for i in range(num_layers)]) + + def forward(self, x): + if self.conv is not None: + x = self.conv(x) + x = self.blocks(x) + return x + + +class ST2CSPA(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(ST2CSPA, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformer2Block(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.m(self.cv1(x)) + y2 = self.cv2(x) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class ST2CSPB(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(ST2CSPB, self).__init__() + c_ = int(c2) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformer2Block(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + x1 = self.cv1(x) + y1 = self.m(x1) + y2 = self.cv2(x1) + return self.cv3(torch.cat((y1, y2), dim=1)) + + +class ST2CSPC(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super(ST2CSPC, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(c_, c_, 1, 1) + self.cv4 = Conv(2 * c_, c2, 1, 1) + num_heads = c_ // 32 + self.m = SwinTransformer2Block(c_, c_, num_heads, n) + #self.m = nn.Sequential(*[Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n)]) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(torch.cat((y1, y2), dim=1)) + +##### end of swin transformer v2 ##### diff --git a/test/yolov7-tracker/models/experimental.py b/test/yolov7-tracker/models/experimental.py new file mode 100644 index 0000000..a14d496 --- /dev/null +++ b/test/yolov7-tracker/models/experimental.py @@ -0,0 +1,106 @@ +import numpy as np +import torch +import torch.nn as nn + +from models.common import Conv, DWConv +from utils.google_utils import attempt_download + + +class CrossConv(nn.Module): + # Cross Convolution Downsample + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False): + # ch_in, ch_out, kernel, stride, groups, expansion, shortcut + super(CrossConv, self).__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, (1, k), (1, s)) + self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class Sum(nn.Module): + # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070 + def __init__(self, n, weight=False): # n: number of inputs + super(Sum, self).__init__() + self.weight = weight # apply weights boolean + self.iter = range(n - 1) # iter object + if weight: + self.w = nn.Parameter(-torch.arange(1., n) / 2, requires_grad=True) # layer weights + + def forward(self, x): + y = x[0] # no weight + if self.weight: + w = torch.sigmoid(self.w) * 2 + for i in self.iter: + y = y + x[i + 1] * w[i] + else: + for i in self.iter: + y = y + x[i + 1] + return y + + +class MixConv2d(nn.Module): + # Mixed Depthwise Conv https://arxiv.org/abs/1907.09595 + def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True): + super(MixConv2d, self).__init__() + groups = len(k) + if equal_ch: # equal c_ per group + i = torch.linspace(0, groups - 1E-6, c2).floor() # c2 indices + c_ = [(i == g).sum() for g in range(groups)] # intermediate channels + else: # equal weight.numel() per group + b = [c2] + [0] * groups + a = np.eye(groups + 1, groups, k=-1) + a -= np.roll(a, 1, axis=1) + a *= np.array(k) ** 2 + a[0] = 1 + c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b + + self.m = nn.ModuleList([nn.Conv2d(c1, int(c_[g]), k[g], s, k[g] // 2, bias=False) for g in range(groups)]) + self.bn = nn.BatchNorm2d(c2) + self.act = nn.LeakyReLU(0.1, inplace=True) + + def forward(self, x): + return x + self.act(self.bn(torch.cat([m(x) for m in self.m], 1))) + + +class Ensemble(nn.ModuleList): + # Ensemble of models + def __init__(self): + super(Ensemble, self).__init__() + + def forward(self, x, augment=False): + y = [] + for module in self: + y.append(module(x, augment)[0]) + # y = torch.stack(y).max(0)[0] # max ensemble + # y = torch.stack(y).mean(0) # mean ensemble + y = torch.cat(y, 1) # nms ensemble + return y, None # inference, train output + + +def attempt_load(weights, map_location=None): + # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a + model = Ensemble() + for w in weights if isinstance(weights, list) else [weights]: + # attempt_download(w) + ckpt = torch.load(w, map_location=map_location) # load + model.append(ckpt['ema' if ckpt.get('ema') else 'model'].float().fuse().eval()) # FP32 model + + # Compatibility updates + for m in model.modules(): + if type(m) in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]: + m.inplace = True # pytorch 1.7.0 compatibility + elif type(m) is nn.Upsample: + m.recompute_scale_factor = None # torch 1.11.0 compatibility + elif type(m) is Conv: + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility + + if len(model) == 1: + return model[-1] # return model + else: + print('Ensemble created with %s\n' % weights) + for k in ['names', 'stride']: + setattr(model, k, getattr(model[-1], k)) + return model # return ensemble diff --git a/test/yolov7-tracker/models/export.py b/test/yolov7-tracker/models/export.py new file mode 100644 index 0000000..dc12559 --- /dev/null +++ b/test/yolov7-tracker/models/export.py @@ -0,0 +1,98 @@ +import argparse +import sys +import time + +sys.path.append('./') # to run '$ python *.py' files in subdirectories + +import torch +import torch.nn as nn + +import models +from models.experimental import attempt_load +from utils.activations import Hardswish, SiLU +from utils.general import set_logging, check_img_size +from utils.torch_utils import select_device + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default='./yolor-csp-c.pt', help='weights path') + parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='image size') # height, width + parser.add_argument('--batch-size', type=int, default=1, help='batch size') + parser.add_argument('--dynamic', action='store_true', help='dynamic ONNX axes') + parser.add_argument('--grid', action='store_true', help='export Detect() layer grid') + parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + opt = parser.parse_args() + opt.img_size *= 2 if len(opt.img_size) == 1 else 1 # expand + print(opt) + set_logging() + t = time.time() + + # Load PyTorch model + device = select_device(opt.device) + model = attempt_load(opt.weights, map_location=device) # load FP32 model + labels = model.names + + # Checks + gs = int(max(model.stride)) # grid size (max stride) + opt.img_size = [check_img_size(x, gs) for x in opt.img_size] # verify img_size are gs-multiples + + # Input + img = torch.zeros(opt.batch_size, 3, *opt.img_size).to(device) # image size(1,3,320,192) iDetection + + # Update model + for k, m in model.named_modules(): + m._non_persistent_buffers_set = set() # pytorch 1.6.0 compatibility + if isinstance(m, models.common.Conv): # assign export-friendly activations + if isinstance(m.act, nn.Hardswish): + m.act = Hardswish() + elif isinstance(m.act, nn.SiLU): + m.act = SiLU() + # elif isinstance(m, models.yolo.Detect): + # m.forward = m.forward_export # assign forward (optional) + model.model[-1].export = not opt.grid # set Detect() layer grid export + y = model(img) # dry run + + # TorchScript export + try: + print('\nStarting TorchScript export with torch %s...' % torch.__version__) + f = opt.weights.replace('.pt', '.torchscript.pt') # filename + ts = torch.jit.trace(model, img, strict=False) + ts.save(f) + print('TorchScript export success, saved as %s' % f) + except Exception as e: + print('TorchScript export failure: %s' % e) + + # ONNX export + try: + import onnx + + print('\nStarting ONNX export with onnx %s...' % onnx.__version__) + f = opt.weights.replace('.pt', '.onnx') # filename + torch.onnx.export(model, img, f, verbose=False, opset_version=12, input_names=['images'], + output_names=['classes', 'boxes'] if y is None else ['output'], + dynamic_axes={'images': {0: 'batch', 2: 'height', 3: 'width'}, # size(1,3,640,640) + 'output': {0: 'batch', 2: 'y', 3: 'x'}} if opt.dynamic else None) + + # Checks + onnx_model = onnx.load(f) # load onnx model + onnx.checker.check_model(onnx_model) # check onnx model + # print(onnx.helper.printable_graph(onnx_model.graph)) # print a human readable model + print('ONNX export success, saved as %s' % f) + except Exception as e: + print('ONNX export failure: %s' % e) + + # CoreML export + try: + import coremltools as ct + + print('\nStarting CoreML export with coremltools %s...' % ct.__version__) + # convert model from torchscript and apply pixel scaling as per detect.py + model = ct.convert(ts, inputs=[ct.ImageType(name='image', shape=img.shape, scale=1 / 255.0, bias=[0, 0, 0])]) + f = opt.weights.replace('.pt', '.mlmodel') # filename + model.save(f) + print('CoreML export success, saved as %s' % f) + except Exception as e: + print('CoreML export failure: %s' % e) + + # Finish + print('\nExport complete (%.2fs). Visualize with https://github.com/lutzroeder/netron.' % (time.time() - t)) diff --git a/test/yolov7-tracker/models/yolo.py b/test/yolov7-tracker/models/yolo.py new file mode 100644 index 0000000..7e1b3da --- /dev/null +++ b/test/yolov7-tracker/models/yolo.py @@ -0,0 +1,550 @@ +import argparse +import logging +import sys +from copy import deepcopy + +sys.path.append('./') # to run '$ python *.py' files in subdirectories +logger = logging.getLogger(__name__) + +from models.common import * +from models.experimental import * +from utils.autoanchor import check_anchor_order +from utils.general import make_divisible, check_file, set_logging +from utils.torch_utils import time_synchronized, fuse_conv_and_bn, model_info, scale_img, initialize_weights, \ + select_device, copy_attr +from utils.loss import SigmoidBin + +try: + import thop # for FLOPS computation +except ImportError: + thop = None + + +class Detect(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, nc=80, anchors=(), ch=()): # detection layer + super(Detect, self).__init__() + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.zeros(1)] * self.nl # init grid + a = torch.tensor(anchors).float().view(self.nl, -1, 2) + self.register_buffer('anchors', a) # shape(nl,na,2) + self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + + def forward(self, x): + # x = x.copy() # for profiling + z = [] # inference output + self.training |= self.export + for i in range(self.nl): + x[i] = self.m[i](x[i]) # conv + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i] = self._make_grid(nx, ny).to(x[i].device) + + y = x[i].sigmoid() + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy + y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh + z.append(y.view(bs, -1, self.no)) + + return x if self.training else (torch.cat(z, 1), x) + + @staticmethod + def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +class IDetect(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, nc=80, anchors=(), ch=()): # detection layer + super(IDetect, self).__init__() + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.zeros(1)] * self.nl # init grid + a = torch.tensor(anchors).float().view(self.nl, -1, 2) + self.register_buffer('anchors', a) # shape(nl,na,2) + self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + + self.ia = nn.ModuleList(ImplicitA(x) for x in ch) + self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch) + + def forward(self, x): + # x = x.copy() # for profiling + z = [] # inference output + self.training |= self.export + for i in range(self.nl): + x[i] = self.m[i](self.ia[i](x[i])) # conv + x[i] = self.im[i](x[i]) + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i] = self._make_grid(nx, ny).to(x[i].device) + + y = x[i].sigmoid() + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy + y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh + z.append(y.view(bs, -1, self.no)) + + return x if self.training else (torch.cat(z, 1), x) + + @staticmethod + def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +class IAuxDetect(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, nc=80, anchors=(), ch=()): # detection layer + super(IAuxDetect, self).__init__() + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.zeros(1)] * self.nl # init grid + a = torch.tensor(anchors).float().view(self.nl, -1, 2) + self.register_buffer('anchors', a) # shape(nl,na,2) + self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch[:self.nl]) # output conv + self.m2 = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch[self.nl:]) # output conv + + self.ia = nn.ModuleList(ImplicitA(x) for x in ch[:self.nl]) + self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch[:self.nl]) + + def forward(self, x): + # x = x.copy() # for profiling + z = [] # inference output + self.training |= self.export + for i in range(self.nl): + x[i] = self.m[i](self.ia[i](x[i])) # conv + x[i] = self.im[i](x[i]) + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + x[i+self.nl] = self.m2[i](x[i+self.nl]) + x[i+self.nl] = x[i+self.nl].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i] = self._make_grid(nx, ny).to(x[i].device) + + y = x[i].sigmoid() + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy + y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh + z.append(y.view(bs, -1, self.no)) + + return x if self.training else (torch.cat(z, 1), x[:self.nl]) + + @staticmethod + def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +class IBin(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, nc=80, anchors=(), ch=(), bin_count=21): # detection layer + super(IBin, self).__init__() + self.nc = nc # number of classes + self.bin_count = bin_count + + self.w_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0) + self.h_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0) + # classes, x,y,obj + self.no = nc + 3 + \ + self.w_bin_sigmoid.get_length() + self.h_bin_sigmoid.get_length() # w-bce, h-bce + # + self.x_bin_sigmoid.get_length() + self.y_bin_sigmoid.get_length() + + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.zeros(1)] * self.nl # init grid + a = torch.tensor(anchors).float().view(self.nl, -1, 2) + self.register_buffer('anchors', a) # shape(nl,na,2) + self.register_buffer('anchor_grid', a.clone().view(self.nl, 1, -1, 1, 1, 2)) # shape(nl,1,na,1,1,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + + self.ia = nn.ModuleList(ImplicitA(x) for x in ch) + self.im = nn.ModuleList(ImplicitM(self.no * self.na) for _ in ch) + + def forward(self, x): + + #self.x_bin_sigmoid.use_fw_regression = True + #self.y_bin_sigmoid.use_fw_regression = True + self.w_bin_sigmoid.use_fw_regression = True + self.h_bin_sigmoid.use_fw_regression = True + + # x = x.copy() # for profiling + z = [] # inference output + self.training |= self.export + for i in range(self.nl): + x[i] = self.m[i](self.ia[i](x[i])) # conv + x[i] = self.im[i](x[i]) + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i] = self._make_grid(nx, ny).to(x[i].device) + + y = x[i].sigmoid() + y[..., 0:2] = (y[..., 0:2] * 2. - 0.5 + self.grid[i]) * self.stride[i] # xy + #y[..., 2:4] = (y[..., 2:4] * 2) ** 2 * self.anchor_grid[i] # wh + + + #px = (self.x_bin_sigmoid.forward(y[..., 0:12]) + self.grid[i][..., 0]) * self.stride[i] + #py = (self.y_bin_sigmoid.forward(y[..., 12:24]) + self.grid[i][..., 1]) * self.stride[i] + + pw = self.w_bin_sigmoid.forward(y[..., 2:24]) * self.anchor_grid[i][..., 0] + ph = self.h_bin_sigmoid.forward(y[..., 24:46]) * self.anchor_grid[i][..., 1] + + #y[..., 0] = px + #y[..., 1] = py + y[..., 2] = pw + y[..., 3] = ph + + y = torch.cat((y[..., 0:4], y[..., 46:]), dim=-1) + + z.append(y.view(bs, -1, y.shape[-1])) + + return x if self.training else (torch.cat(z, 1), x) + + @staticmethod + def _make_grid(nx=20, ny=20): + yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + + +class Model(nn.Module): + def __init__(self, cfg='yolor-csp-c.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes + super(Model, self).__init__() + self.traced = False + if isinstance(cfg, dict): + self.yaml = cfg # model dict + else: # is *.yaml + import yaml # for torch hub + self.yaml_file = Path(cfg).name + with open(cfg) as f: + self.yaml = yaml.load(f, Loader=yaml.SafeLoader) # model dict + + # Define model + ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels + if nc and nc != self.yaml['nc']: + logger.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}") + self.yaml['nc'] = nc # override yaml value + if anchors: + logger.info(f'Overriding model.yaml anchors with anchors={anchors}') + self.yaml['anchors'] = round(anchors) # override yaml value + self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist + self.names = [str(i) for i in range(self.yaml['nc'])] # default names + # print([x.shape for x in self.forward(torch.zeros(1, ch, 64, 64))]) + + # Build strides, anchors + m = self.model[-1] # Detect() + if isinstance(m, Detect): + s = 256 # 2x min stride + m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward + m.anchors /= m.stride.view(-1, 1, 1) + check_anchor_order(m) + self.stride = m.stride + self._initialize_biases() # only run once + # print('Strides: %s' % m.stride.tolist()) + if isinstance(m, IDetect): + s = 256 # 2x min stride + m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward + m.anchors /= m.stride.view(-1, 1, 1) + check_anchor_order(m) + self.stride = m.stride + self._initialize_biases() # only run once + # print('Strides: %s' % m.stride.tolist()) + if isinstance(m, IAuxDetect): + s = 256 # 2x min stride + m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))[:4]]) # forward + #print(m.stride) + m.anchors /= m.stride.view(-1, 1, 1) + check_anchor_order(m) + self.stride = m.stride + self._initialize_aux_biases() # only run once + # print('Strides: %s' % m.stride.tolist()) + if isinstance(m, IBin): + s = 256 # 2x min stride + m.stride = torch.tensor([s / x.shape[-2] for x in self.forward(torch.zeros(1, ch, s, s))]) # forward + m.anchors /= m.stride.view(-1, 1, 1) + check_anchor_order(m) + self.stride = m.stride + self._initialize_biases_bin() # only run once + # print('Strides: %s' % m.stride.tolist()) + + # Init weights, biases + initialize_weights(self) + self.info() + logger.info('') + + def forward(self, x, augment=False, profile=False): + if augment: + img_size = x.shape[-2:] # height, width + s = [1, 0.83, 0.67] # scales + f = [None, 3, None] # flips (2-ud, 3-lr) + y = [] # outputs + for si, fi in zip(s, f): + xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max())) + yi = self.forward_once(xi)[0] # forward + # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save + yi[..., :4] /= si # de-scale + if fi == 2: + yi[..., 1] = img_size[0] - yi[..., 1] # de-flip ud + elif fi == 3: + yi[..., 0] = img_size[1] - yi[..., 0] # de-flip lr + y.append(yi) + return torch.cat(y, 1), None # augmented inference, train + else: + return self.forward_once(x, profile) # single-scale inference, train + + def forward_once(self, x, profile=False): + y, dt = [], [] # outputs + for m in self.model: + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + + if not hasattr(self, 'traced'): + self.traced=False + + if self.traced: + if isinstance(m, Detect) or isinstance(m, IDetect) or isinstance(m, IAuxDetect): + break + + if profile: + c = isinstance(m, (Detect, IDetect, IAuxDetect, IBin)) + o = thop.profile(m, inputs=(x.copy() if c else x,), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPS + for _ in range(10): + m(x.copy() if c else x) + t = time_synchronized() + for _ in range(10): + m(x.copy() if c else x) + dt.append((time_synchronized() - t) * 100) + print('%10.1f%10.0f%10.1fms %-40s' % (o, m.np, dt[-1], m.type)) + + x = m(x) # run + + y.append(x if m.i in self.save else None) # save output + + if profile: + print('%.1fms total' % sum(dt)) + return x + + def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency + # https://arxiv.org/abs/1708.02002 section 3.3 + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1. + m = self.model[-1] # Detect() module + for mi, s in zip(m.m, m.stride): # from + b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85) + b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls + mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + def _initialize_aux_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency + # https://arxiv.org/abs/1708.02002 section 3.3 + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1. + m = self.model[-1] # Detect() module + for mi, mi2, s in zip(m.m, m.m2, m.stride): # from + b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85) + b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls + mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + b2 = mi2.bias.view(m.na, -1) # conv.bias(255) to (3,85) + b2.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b2.data[:, 5:] += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls + mi2.bias = torch.nn.Parameter(b2.view(-1), requires_grad=True) + + def _initialize_biases_bin(self, cf=None): # initialize biases into Detect(), cf is class frequency + # https://arxiv.org/abs/1708.02002 section 3.3 + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1. + m = self.model[-1] # Bin() module + bc = m.bin_count + for mi, s in zip(m.m, m.stride): # from + b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85) + old = b[:, (0,1,2,bc+3)].data + obj_idx = 2*bc+4 + b[:, :obj_idx].data += math.log(0.6 / (bc + 1 - 0.99)) + b[:, obj_idx].data += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b[:, (obj_idx+1):].data += math.log(0.6 / (m.nc - 0.99)) if cf is None else torch.log(cf / cf.sum()) # cls + b[:, (0,1,2,bc+3)].data = old + mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + def _print_biases(self): + m = self.model[-1] # Detect() module + for mi in m.m: # from + b = mi.bias.detach().view(m.na, -1).T # conv.bias(255) to (3,85) + print(('%6g Conv2d.bias:' + '%10.3g' * 6) % (mi.weight.shape[1], *b[:5].mean(1).tolist(), b[5:].mean())) + + # def _print_weights(self): + # for m in self.model.modules(): + # if type(m) is Bottleneck: + # print('%10.3g' % (m.w.detach().sigmoid() * 2)) # shortcut weights + + def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers + print('Fusing layers... ') + for m in self.model.modules(): + if isinstance(m, RepConv): + #print(f" fuse_repvgg_block") + m.fuse_repvgg_block() + elif isinstance(m, RepConv_OREPA): + #print(f" switch_to_deploy") + m.switch_to_deploy() + elif type(m) is Conv and hasattr(m, 'bn'): + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, 'bn') # remove batchnorm + m.forward = m.fuseforward # update forward + self.info() + return self + + def nms(self, mode=True): # add or remove NMS module + present = type(self.model[-1]) is NMS # last layer is NMS + if mode and not present: + print('Adding NMS... ') + m = NMS() # module + m.f = -1 # from + m.i = self.model[-1].i + 1 # index + self.model.add_module(name='%s' % m.i, module=m) # add + self.eval() + elif not mode and present: + print('Removing NMS... ') + self.model = self.model[:-1] # remove + return self + + def autoshape(self): # add autoShape module + print('Adding autoShape... ') + m = autoShape(self) # wrap model + copy_attr(m, self, include=('yaml', 'nc', 'hyp', 'names', 'stride'), exclude=()) # copy attributes + return m + + def info(self, verbose=False, img_size=640): # print model information + model_info(self, verbose, img_size) + + +def parse_model(d, ch): # model_dict, input_channels(3) + logger.info('\n%3s%18s%3s%10s %-40s%-30s' % ('', 'from', 'n', 'params', 'module', 'arguments')) + anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'] + na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors + no = na * (nc + 5) # number of outputs = anchors * (classes + 5) + + layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out + for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args + m = eval(m) if isinstance(m, str) else m # eval strings + for j, a in enumerate(args): + try: + args[j] = eval(a) if isinstance(a, str) else a # eval strings + except: + pass + + n = max(round(n * gd), 1) if n > 1 else n # depth gain + if m in [nn.Conv2d, Conv, RobustConv, RobustConv2, DWConv, GhostConv, RepConv, RepConv_OREPA, DownC, + SPP, SPPF, SPPCSPC, GhostSPPCSPC, MixConv2d, Focus, Stem, GhostStem, CrossConv, + Bottleneck, BottleneckCSPA, BottleneckCSPB, BottleneckCSPC, + RepBottleneck, RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC, + Res, ResCSPA, ResCSPB, ResCSPC, + RepRes, RepResCSPA, RepResCSPB, RepResCSPC, + ResX, ResXCSPA, ResXCSPB, ResXCSPC, + RepResX, RepResXCSPA, RepResXCSPB, RepResXCSPC, + Ghost, GhostCSPA, GhostCSPB, GhostCSPC, + SwinTransformerBlock, STCSPA, STCSPB, STCSPC, + SwinTransformer2Block, ST2CSPA, ST2CSPB, ST2CSPC]: + c1, c2 = ch[f], args[0] + if c2 != no: # if not output + c2 = make_divisible(c2 * gw, 8) + + args = [c1, c2, *args[1:]] + if m in [DownC, SPPCSPC, GhostSPPCSPC, + BottleneckCSPA, BottleneckCSPB, BottleneckCSPC, + RepBottleneckCSPA, RepBottleneckCSPB, RepBottleneckCSPC, + ResCSPA, ResCSPB, ResCSPC, + RepResCSPA, RepResCSPB, RepResCSPC, + ResXCSPA, ResXCSPB, ResXCSPC, + RepResXCSPA, RepResXCSPB, RepResXCSPC, + GhostCSPA, GhostCSPB, GhostCSPC, + STCSPA, STCSPB, STCSPC, + ST2CSPA, ST2CSPB, ST2CSPC]: + args.insert(2, n) # number of repeats + n = 1 + elif m is nn.BatchNorm2d: + args = [ch[f]] + elif m is Concat: + c2 = sum([ch[x] for x in f]) + elif m is Chuncat: + c2 = sum([ch[x] for x in f]) + elif m is Shortcut: + c2 = ch[f[0]] + elif m is Foldcut: + c2 = ch[f] // 2 + elif m in [Detect, IDetect, IAuxDetect, IBin]: + args.append([ch[x] for x in f]) + if isinstance(args[1], int): # number of anchors + args[1] = [list(range(args[1] * 2))] * len(f) + elif m is ReOrg: + c2 = ch[f] * 4 + elif m is Contract: + c2 = ch[f] * args[0] ** 2 + elif m is Expand: + c2 = ch[f] // args[0] ** 2 + else: + c2 = ch[f] + + m_ = nn.Sequential(*[m(*args) for _ in range(n)]) if n > 1 else m(*args) # module + t = str(m)[8:-2].replace('__main__.', '') # module type + np = sum([x.numel() for x in m_.parameters()]) # number params + m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params + logger.info('%3s%18s%3s%10.0f %-40s%-30s' % (i, f, n, np, t, args)) # print + save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist + layers.append(m_) + if i == 0: + ch = [] + ch.append(c2) + return nn.Sequential(*layers), sorted(save) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cfg', type=str, default='yolor-csp-c.yaml', help='model.yaml') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--profile', action='store_true', help='profile model speed') + opt = parser.parse_args() + opt.cfg = check_file(opt.cfg) # check file + set_logging() + device = select_device(opt.device) + + # Create model + model = Model(opt.cfg).to(device) + model.train() + + if opt.profile: + img = torch.rand(1, 3, 640, 640).to(device) + y = model(img, profile=True) + + # Profile + # img = torch.rand(8 if torch.cuda.is_available() else 1, 3, 640, 640).to(device) + # y = model(img, profile=True) + + # Tensorboard + # from torch.utils.tensorboard import SummaryWriter + # tb_writer = SummaryWriter() + # print("Run 'tensorboard --logdir=models/runs' to view tensorboard at http://localhost:6006/") + # tb_writer.add_graph(model.model, img) # add model to tensorboard + # tb_writer.add_image('test', img[0], dataformats='CWH') # add model to tensorboard diff --git a/test/yolov7-tracker/requirements.txt b/test/yolov7-tracker/requirements.txt new file mode 100644 index 0000000..1f9c9d0 --- /dev/null +++ b/test/yolov7-tracker/requirements.txt @@ -0,0 +1,29 @@ +albumentations==1.4.11 +coremltools==7.2 +cython_bbox==0.1.5 +filterpy==1.4.5 +gdown==5.2.0 +lap==0.4.0 +loguru==0.7.2 +matplotlib==3.9.1 +numpy==2.0.0 +onnx==1.16.1 +pafy==0.5.5 +pandas==2.2.2 +Pillow==10.4.0 +pycocotools==2.0.8 +PyYAML==6.0.1 +PyYAML==6.0.1 +Requests==2.32.3 +scipy==1.14.0 +seaborn==0.13.2 +setuptools==69.5.1 +skimage==0.0 +tabulate==0.9.0 +thop==0.1.1.post2209072238 +torch==1.12.1 +torchvision==0.18.1 +tqdm==4.66.4 +ultralytics==8.2.58 +wandb==0.17.4 +yolox==0.3.0 diff --git a/test/yolov7-tracker/requirements.txt.save b/test/yolov7-tracker/requirements.txt.save new file mode 100644 index 0000000..92fcc8f --- /dev/null +++ b/test/yolov7-tracker/requirements.txt.save @@ -0,0 +1,24 @@ +numpy +cython-bbox==0.1.3 +loguru +motmetrics==1.4.0 +ninja + +pandas +Pillow + +PyYAML + +scikit-learn +scipy +seaborn + +thop +tensorboard +lap +tabulate +tqdm + +wandb + +gdown diff --git a/test/yolov7-tracker/scripts/get_coco.sh b/test/yolov7-tracker/scripts/get_coco.sh new file mode 100644 index 0000000..524f8dd --- /dev/null +++ b/test/yolov7-tracker/scripts/get_coco.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# COCO 2017 dataset http://cocodataset.org +# Download command: bash ./scripts/get_coco.sh + +# Download/unzip labels +d='./' # unzip directory +url=https://github.com/ultralytics/yolov5/releases/download/v1.0/ +f='coco2017labels-segments.zip' # or 'coco2017labels.zip', 68 MB +echo 'Downloading' $url$f ' ...' +curl -L $url$f -o $f && unzip -q $f -d $d && rm $f & # download, unzip, remove in background + +# Download/unzip images +d='./coco/images' # unzip directory +url=http://images.cocodataset.org/zips/ +f1='train2017.zip' # 19G, 118k images +f2='val2017.zip' # 1G, 5k images +f3='test2017.zip' # 7G, 41k images (optional) +for f in $f1 $f2 $f3; do + echo 'Downloading' $url$f '...' + curl -L $url$f -o $f && unzip -q $f -d $d && rm $f & # download, unzip, remove in background +done +wait # finish background tasks diff --git a/test/yolov7-tracker/test.py b/test/yolov7-tracker/test.py new file mode 100644 index 0000000..87ae7e0 --- /dev/null +++ b/test/yolov7-tracker/test.py @@ -0,0 +1,350 @@ +import argparse +import json +import os +from pathlib import Path +from threading import Thread + +import numpy as np +import torch +import yaml +from tqdm import tqdm + +from models.experimental import attempt_load +from utils.datasets import create_dataloader +from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, check_requirements, \ + box_iou, non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, set_logging, increment_path, colorstr +from utils.metrics import ap_per_class, ConfusionMatrix +from utils.plots import plot_images, output_to_target, plot_study_txt +from utils.torch_utils import select_device, time_synchronized, TracedModel + + +def test(data, + weights=None, + batch_size=32, + imgsz=640, + conf_thres=0.001, + iou_thres=0.6, # for NMS + save_json=False, + single_cls=False, + augment=False, + verbose=False, + model=None, + dataloader=None, + save_dir=Path(''), # for saving images + save_txt=False, # for auto-labelling + save_hybrid=False, # for hybrid auto-labelling + save_conf=False, # save auto-label confidences + plots=True, + wandb_logger=None, + compute_loss=None, + half_precision=True, + trace=False, + is_coco=False): + # Initialize/load model and set device + training = model is not None + if training: # called by train.py + device = next(model.parameters()).device # get model device + + else: # called directly + set_logging() + device = select_device(opt.device, batch_size=batch_size) + + # Directories + save_dir = Path(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + model = attempt_load(weights, map_location=device) # load FP32 model + gs = max(int(model.stride.max()), 32) # grid size (max stride) + imgsz = check_img_size(imgsz, s=gs) # check img_size + + if trace: + model = TracedModel(model, device, opt.img_size) + + # Half + half = device.type != 'cpu' and half_precision # half precision only supported on CUDA + if half: + model.half() + + # Configure + model.eval() + if isinstance(data, str): + is_coco = data.endswith('coco.yaml') + with open(data) as f: + data = yaml.load(f, Loader=yaml.SafeLoader) + check_dataset(data) # check + nc = 1 if single_cls else int(data['nc']) # number of classes + iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for mAP@0.5:0.95 + niou = iouv.numel() + + # Logging + log_imgs = 0 + if wandb_logger and wandb_logger.wandb: + log_imgs = min(wandb_logger.log_imgs, 100) + # Dataloader + if not training: + if device.type != 'cpu': + model(torch.zeros(1, 3, imgsz, imgsz).to(device).type_as(next(model.parameters()))) # run once + task = opt.task if opt.task in ('train', 'val', 'test') else 'val' # path to train/val/test images + dataloader = create_dataloader(data[task], imgsz, batch_size, gs, opt, pad=0.5, rect=True, + prefix=colorstr(f'{task}: '))[0] + + seen = 0 + + confusion_matrix = ConfusionMatrix(nc=nc) + names = {k: v for k, v in enumerate(model.names if hasattr(model, 'names') else model.module.names)} + coco91class = coco80_to_coco91_class() + s = ('%20s' + '%12s' * 6) % ('Class', 'Images', 'Labels', 'P', 'R', 'mAP@.5', 'mAP@.5:.95') + p, r, f1, mp, mr, map50, map, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0. + loss = torch.zeros(3, device=device) + jdict, stats, ap, ap_class, wandb_images = [], [], [], [], [] + for batch_i, (img, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)): + img = img.to(device, non_blocking=True) + img = img.half() if half else img.float() # uint8 to fp16/32 + img /= 255.0 # 0 - 255 to 0.0 - 1.0 + targets = targets.to(device) + nb, _, height, width = img.shape # batch size, channels, height, width + + with torch.no_grad(): + # Run model + t = time_synchronized() + out, train_out = model(img, augment=augment) # inference and training outputs + t0 += time_synchronized() - t + + # Compute loss + if compute_loss: + loss += compute_loss([x.float() for x in train_out], targets)[1][:3] # box, obj, cls + + # Run NMS + targets[:, 2:] *= torch.Tensor([width, height, width, height]).to(device) # to pixels + lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling + t = time_synchronized() + out = non_max_suppression(out, conf_thres=conf_thres, iou_thres=iou_thres, labels=lb, multi_label=True) + t1 += time_synchronized() - t + + # Statistics per image + for si, pred in enumerate(out): + labels = targets[targets[:, 0] == si, 1:] + nl = len(labels) + tcls = labels[:, 0].tolist() if nl else [] # target class + path = Path(paths[si]) + seen += 1 + + if len(pred) == 0: + if nl: + stats.append((torch.zeros(0, niou, dtype=torch.bool), torch.Tensor(), torch.Tensor(), tcls)) + continue + + # Predictions + predn = pred.clone() + scale_coords(img[si].shape[1:], predn[:, :4], shapes[si][0], shapes[si][1]) # native-space pred + + # Append to text file + if save_txt: + gn = torch.tensor(shapes[si][0])[[1, 0, 1, 0]] # normalization gain whwh + for *xyxy, conf, cls in predn.tolist(): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(save_dir / 'labels' / (path.stem + '.txt'), 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + # W&B logging - Media Panel Plots + if len(wandb_images) < log_imgs and wandb_logger.current_epoch > 0: # Check for test operation + if wandb_logger.current_epoch % wandb_logger.bbox_interval == 0: + box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, + "class_id": int(cls), + "box_caption": "%s %.3f" % (names[cls], conf), + "scores": {"class_score": conf}, + "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()] + boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space + wandb_images.append(wandb_logger.wandb.Image(img[si], boxes=boxes, caption=path.name)) + wandb_logger.log_training_progress(predn, path, names) if wandb_logger and wandb_logger.wandb_run else None + + # Append to pycocotools JSON dictionary + if save_json: + # [{"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}, ... + image_id = int(path.stem) if path.stem.isnumeric() else path.stem + box = xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(pred.tolist(), box.tolist()): + jdict.append({'image_id': image_id, + 'category_id': coco91class[int(p[5])] if is_coco else int(p[5]), + 'bbox': [round(x, 3) for x in b], + 'score': round(p[4], 5)}) + + # Assign all predictions as incorrect + correct = torch.zeros(pred.shape[0], niou, dtype=torch.bool, device=device) + if nl: + detected = [] # target indices + tcls_tensor = labels[:, 0] + + # target boxes + tbox = xywh2xyxy(labels[:, 1:5]) + scale_coords(img[si].shape[1:], tbox, shapes[si][0], shapes[si][1]) # native-space labels + if plots: + confusion_matrix.process_batch(predn, torch.cat((labels[:, 0:1], tbox), 1)) + + # Per target class + for cls in torch.unique(tcls_tensor): + ti = (cls == tcls_tensor).nonzero(as_tuple=False).view(-1) # prediction indices + pi = (cls == pred[:, 5]).nonzero(as_tuple=False).view(-1) # target indices + + # Search for detections + if pi.shape[0]: + # Prediction to target ious + ious, i = box_iou(predn[pi, :4], tbox[ti]).max(1) # best ious, indices + + # Append detections + detected_set = set() + for j in (ious > iouv[0]).nonzero(as_tuple=False): + d = ti[i[j]] # detected target + if d.item() not in detected_set: + detected_set.add(d.item()) + detected.append(d) + correct[pi[j]] = ious[j] > iouv # iou_thres is 1xn + if len(detected) == nl: # all targets already located in image + break + + # Append statistics (correct, conf, pcls, tcls) + stats.append((correct.cpu(), pred[:, 4].cpu(), pred[:, 5].cpu(), tcls)) + + # Plot images + if plots and batch_i < 3: + f = save_dir / f'test_batch{batch_i}_labels.jpg' # labels + Thread(target=plot_images, args=(img, targets, paths, f, names), daemon=True).start() + f = save_dir / f'test_batch{batch_i}_pred.jpg' # predictions + Thread(target=plot_images, args=(img, output_to_target(out), paths, f, names), daemon=True).start() + + # Compute statistics + stats = [np.concatenate(x, 0) for x in zip(*stats)] # to numpy + if len(stats) and stats[0].any(): + p, r, ap, f1, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) + ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 + mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() + nt = np.bincount(stats[3].astype(np.int64), minlength=nc) # number of targets per class + else: + nt = torch.zeros(1) + + # Print results + pf = '%20s' + '%12i' * 2 + '%12.3g' * 4 # print format + print(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) + + # Print results per class + if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): + for i, c in enumerate(ap_class): + print(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) + + # Print speeds + t = tuple(x / seen * 1E3 for x in (t0, t1, t0 + t1)) + (imgsz, imgsz, batch_size) # tuple + if not training: + print('Speed: %.1f/%.1f/%.1f ms inference/NMS/total per %gx%g image at batch-size %g' % t) + + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) + if wandb_logger and wandb_logger.wandb: + val_batches = [wandb_logger.wandb.Image(str(f), caption=f.name) for f in sorted(save_dir.glob('test*.jpg'))] + wandb_logger.log({"Validation": val_batches}) + if wandb_images: + wandb_logger.log({"Bounding Box Debugger/Images": wandb_images}) + + # Save JSON + if save_json and len(jdict): + w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else '' # weights + anno_json = './coco/annotations/instances_val2017.json' # annotations json + pred_json = str(save_dir / f"{w}_predictions.json") # predictions json + print('\nEvaluating pycocotools mAP... saving %s...' % pred_json) + with open(pred_json, 'w') as f: + json.dump(jdict, f) + + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + + anno = COCO(anno_json) # init annotations api + pred = anno.loadRes(pred_json) # init predictions api + eval = COCOeval(anno, pred, 'bbox') + if is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.img_files] # image IDs to evaluate + eval.evaluate() + eval.accumulate() + eval.summarize() + map, map50 = eval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) + except Exception as e: + print(f'pycocotools unable to run: {e}') + + # Return results + model.float() # for training + if not training: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + print(f"Results saved to {save_dir}{s}") + maps = np.zeros(nc) + map + for i, c in enumerate(ap_class): + maps[c] = ap[i] + return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(prog='test.py') + parser.add_argument('--dataset', type=str, default='COCO', help='dataset name') + + parser.add_argument('--weights', nargs='+', type=str, default='yolov7.pt', help='model.pt path(s)') + parser.add_argument('--data', type=str, default='data/coco.yaml', help='*.data path') + parser.add_argument('--batch-size', type=int, default=32, help='size of each image batch') + parser.add_argument('--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.001, help='object confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.65, help='IOU threshold for NMS') + parser.add_argument('--task', default='val', help='train, val, test, speed or study') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--verbose', action='store_true', help='report mAP by class') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-json', action='store_true', help='save a cocoapi-compatible JSON results file') + parser.add_argument('--project', default='runs/test', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--no-trace', action='store_true', help='don`t trace model') + opt = parser.parse_args() + opt.save_json |= opt.data.endswith('coco.yaml') + opt.data = check_file(opt.data) # check file + print(opt) + #check_requirements() + + if opt.task in ('train', 'val', 'test'): # run normally + test(opt.data, + opt.weights, + opt.batch_size, + opt.img_size, + opt.conf_thres, + opt.iou_thres, + opt.save_json, + opt.single_cls, + opt.augment, + opt.verbose, + save_txt=opt.save_txt | opt.save_hybrid, + save_hybrid=opt.save_hybrid, + save_conf=opt.save_conf, + trace=not opt.no_trace, + ) + + elif opt.task == 'speed': # speed benchmarks + for w in opt.weights: + test(opt.data, w, opt.batch_size, opt.img_size, 0.25, 0.45, save_json=False, plots=False) + + elif opt.task == 'study': # run over a range of settings and save/plot + # python test.py --task study --data coco.yaml --iou 0.65 --weights yolov7.pt + x = list(range(256, 1536 + 128, 128)) # x axis (image sizes) + for w in opt.weights: + f = f'study_{Path(opt.data).stem}_{Path(w).stem}.txt' # filename to save to + y = [] # y axis + for i in x: # img-size + print(f'\nRunning {f} point {i}...') + r, _, t = test(opt.data, w, opt.batch_size, i, opt.conf_thres, opt.iou_thres, opt.save_json, + plots=False) + y.append(r + t) # results and times + np.savetxt(f, y, fmt='%10.4g') # save + os.system('zip -r study.zip study_*.txt') + plot_study_txt(x=x) # plot diff --git a/test/yolov7-tracker/tools/convert_MOT17_to_yolo.py b/test/yolov7-tracker/tools/convert_MOT17_to_yolo.py new file mode 100644 index 0000000..e042094 --- /dev/null +++ b/test/yolov7-tracker/tools/convert_MOT17_to_yolo.py @@ -0,0 +1,180 @@ +""" +将UAVDT转换为yolo v5格式 +class_id, xc_norm, yc_norm, w_norm, h_norm +""" + +import os +import os.path as osp +import argparse +import cv2 +import glob +import numpy as np +import random + +DATA_ROOT = '/data/wujiapeng/datasets/MOT17/' + +image_wh_dict = {} # seq->(w,h) 字典 用于归一化 + +def generate_imgs_and_labels(opts): + """ + 产生图片路径的txt文件以及yolo格式真值文件 + """ + if opts.split == 'test': + seq_list = os.listdir(osp.join(DATA_ROOT, 'test')) + else: + seq_list = os.listdir(osp.join(DATA_ROOT, 'train')) + seq_list = [item for item in seq_list if 'FRCNN' in item] # 只取一个FRCNN即可 + if 'val' in opts.split: opts.half = True # 验证集取训练集的一半 + + print('--------------------------') + print(f'Total {len(seq_list)} seqs!!') + print(seq_list) + + if opts.random: + random.shuffle(seq_list) + + # 定义类别 MOT只有一类 + CATEGOTY_ID = 0 # pedestrian + + # 定义帧数范围 + frame_range = {'start': 0.0, 'end': 1.0} + if opts.half: # half 截取一半 + frame_range['end'] = 0.5 + + if opts.split == 'test': + process_train_test(seqs=seq_list, frame_range=frame_range, cat_id=CATEGOTY_ID, split='test') + else: + process_train_test(seqs=seq_list, frame_range=frame_range, cat_id=CATEGOTY_ID, split=opts.split) + + +def process_train_test(seqs: list, frame_range: dict, cat_id: int = 0, split: str = 'trian') -> None: + """ + 处理MOT17的train 或 test + 由于操作相似 故另写函数 + + """ + + for seq in seqs: + print(f'Dealing with {split} dataset...') + + img_dir = osp.join(DATA_ROOT, 'train', seq, 'img1') if split != 'test' else osp.join(DATA_ROOT, 'test', seq, 'img1') # 图片路径 + imgs = sorted(os.listdir(img_dir)) # 所有图片的相对路径 + seq_length = len(imgs) # 序列长度 + + if split != 'test': + + # 求解图片高宽 + img_eg = cv2.imread(osp.join(img_dir, imgs[0])) + w0, h0 = img_eg.shape[1], img_eg.shape[0] # 原始高宽 + + ann_of_seq_path = os.path.join(img_dir, '../', 'gt', 'gt.txt') # GT文件路径 + ann_of_seq = np.loadtxt(ann_of_seq_path, dtype=np.float32, delimiter=',') # GT内容 + + gt_to_path = osp.join(DATA_ROOT, 'labels', split, seq) # 要写入的真值文件夹 + # 如果不存在就创建 + if not osp.exists(gt_to_path): + os.makedirs(gt_to_path) + + exist_gts = [] # 初始化该列表 每个元素对应该seq的frame中有无真值框 + # 如果没有 就在train.txt产生图片路径 + + for idx, img in enumerate(imgs): + # img 形如: img000001.jpg + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + # 第一步 产生图片软链接 + # print('step1, creating imgs symlink...') + if opts.generate_imgs: + img_to_path = osp.join(DATA_ROOT, 'images', split, seq) # 该序列图片存储位置 + + if not osp.exists(img_to_path): + os.makedirs(img_to_path) + + os.symlink(osp.join(img_dir, img), + osp.join(img_to_path, img)) # 创建软链接 + + # 第二步 产生真值文件 + # print('step2, generating gt files...') + ann_of_current_frame = ann_of_seq[ann_of_seq[:, 0] == float(idx + 1), :] # 筛选真值文件里本帧的目标信息 + exist_gts.append(True if ann_of_current_frame.shape[0] != 0 else False) + + gt_to_file = osp.join(gt_to_path, img[: -4] + '.txt') + + with open(gt_to_file, 'w') as f_gt: + for i in range(ann_of_current_frame.shape[0]): + if int(ann_of_current_frame[i][6]) == 1 and int(ann_of_current_frame[i][7]) == 1 \ + and float(ann_of_current_frame[i][8]) > 0.25: + # bbox xywh + x0, y0 = int(ann_of_current_frame[i][2]), int(ann_of_current_frame[i][3]) + x0, y0 = max(x0, 0), max(y0, 0) + w, h = int(ann_of_current_frame[i][4]), int(ann_of_current_frame[i][5]) + + xc, yc = x0 + w // 2, y0 + h // 2 # 中心点 x y + + # 归一化 + xc, yc = xc / w0, yc / h0 + xc, yc = min(xc, 1.0), min(yc, 1.0) + w, h = w / w0, h / h0 + w, h = min(w, 1.0), min(h, 1.0) + assert w <= 1 and h <= 1, f'{w}, {h} must be normed, original size{w0}, {h0}' + assert xc >= 0 and yc >= 0, f'{x0}, {y0} must be positve' + assert xc <= 1 and yc <= 1, f'{x0}, {y0} must be le than 1' + category_id = cat_id + + write_line = '{:d} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + category_id, xc, yc, w, h) + + f_gt.write(write_line) + + f_gt.close() + + else: # test 只产生图片软链接 + for idx, img in enumerate(imgs): + # img 形如: img000001.jpg + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + # 第一步 产生图片软链接 + # print('step1, creating imgs symlink...') + if opts.generate_imgs: + img_to_path = osp.join(DATA_ROOT, 'images', split, seq) # 该序列图片存储位置 + + if not osp.exists(img_to_path): + os.makedirs(img_to_path) + + os.symlink(osp.join(img_dir, img), + osp.join(img_to_path, img)) # 创建软链接 + + # 第三步 产生图片索引train.txt等 + print(f'generating img index file of {seq}') + to_file = os.path.join('./mot17/', split + '.txt') + with open(to_file, 'a') as f: + for idx, img in enumerate(imgs): + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + if split == 'test' or exist_gts[idx]: + f.write('MOT17/' + 'images/' + split + '/' \ + + seq + '/' + img + '\n') + + f.close() + + + +if __name__ == '__main__': + if not osp.exists('./mot17'): + os.system('mkdir mot17') + + parser = argparse.ArgumentParser() + parser.add_argument('--split', type=str, default='train', help='train, test or val') + parser.add_argument('--generate_imgs', action='store_true', help='whether generate soft link of imgs') + parser.add_argument('--certain_seqs', action='store_true', help='for debug') + parser.add_argument('--half', action='store_true', help='half frames') + parser.add_argument('--ratio', type=float, default=0.8, help='ratio of test dataset devide train dataset') + parser.add_argument('--random', action='store_true', help='random split train and test') + + opts = parser.parse_args() + + generate_imgs_and_labels(opts) + # python tools/convert_MOT17_to_yolo.py --split train --generate_imgs \ No newline at end of file diff --git a/test/yolov7-tracker/tools/convert_UAVDT_to_yolo.py b/test/yolov7-tracker/tools/convert_UAVDT_to_yolo.py new file mode 100644 index 0000000..7793c4c --- /dev/null +++ b/test/yolov7-tracker/tools/convert_UAVDT_to_yolo.py @@ -0,0 +1,159 @@ +""" +将UAVDT转换为yolo v5格式 +class_id, xc_norm, yc_norm, w_norm, h_norm +""" + +import os +import os.path as osp +import argparse +import cv2 +import glob +import numpy as np +import random + +DATA_ROOT = '/data/wujiapeng/datasets/UAVDT/' + +image_wh_dict = {} # seq->(w,h) 字典 用于归一化 + +def generate_imgs_and_labels(opts): + """ + 产生图片路径的txt文件以及yolo格式真值文件 + """ + seq_list = os.listdir(osp.join(DATA_ROOT, 'UAV-benchmark-M')) + print('--------------------------') + print(f'Total {len(seq_list)} seqs!!') + # 划分train test + if opts.random: + random.shuffle(seq_list) + + bound = int(opts.ratio * len(seq_list)) + train_seq_list = seq_list[: bound] + test_seq_list = seq_list[bound:] + del bound + print(f'train dataset: {train_seq_list}') + print(f'test dataset: {test_seq_list}') + print('--------------------------') + + if not osp.exists('./uavdt/'): + os.makedirs('./uavdt/') + + # 定义类别 UAVDT只有一类 + CATEGOTY_ID = 0 # car + + # 定义帧数范围 + frame_range = {'start': 0.0, 'end': 1.0} + if opts.half: # half 截取一半 + frame_range['end'] = 0.5 + + # 分别处理train与test + process_train_test(train_seq_list, frame_range, CATEGOTY_ID, 'train') + process_train_test(test_seq_list, {'start': 0.0, 'end': 1.0}, CATEGOTY_ID, 'test') + print('All Done!!') + + +def process_train_test(seqs: list, frame_range: dict, cat_id: int = 0, split: str = 'trian') -> None: + """ + 处理UAVDT的train 或 test + 由于操作相似 故另写函数 + + """ + + for seq in seqs: + print('Dealing with train dataset...') + + img_dir = osp.join(DATA_ROOT, 'UAV-benchmark-M', seq, 'img1') # 图片路径 + imgs = sorted(os.listdir(img_dir)) # 所有图片的相对路径 + seq_length = len(imgs) # 序列长度 + + # 求解图片高宽 + img_eg = cv2.imread(osp.join(img_dir, imgs[0])) + w0, h0 = img_eg.shape[1], img_eg.shape[0] # 原始高宽 + + ann_of_seq_path = os.path.join(img_dir, '../', 'gt', 'gt.txt') # GT文件路径 + ann_of_seq = np.loadtxt(ann_of_seq_path, dtype=np.float32, delimiter=',') # GT内容 + + gt_to_path = osp.join(DATA_ROOT, 'labels', split, seq) # 要写入的真值文件夹 + # 如果不存在就创建 + if not osp.exists(gt_to_path): + os.makedirs(gt_to_path) + + exist_gts = [] # 初始化该列表 每个元素对应该seq的frame中有无真值框 + # 如果没有 就在train.txt产生图片路径 + + for idx, img in enumerate(imgs): + # img 形如: img000001.jpg + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + # 第一步 产生图片软链接 + # print('step1, creating imgs symlink...') + if opts.generate_imgs: + img_to_path = osp.join(DATA_ROOT, 'images', split, seq) # 该序列图片存储位置 + + if not osp.exists(img_to_path): + os.makedirs(img_to_path) + + os.symlink(osp.join(img_dir, img), + osp.join(img_to_path, img)) # 创建软链接 + + # 第二步 产生真值文件 + # print('step2, generating gt files...') + ann_of_current_frame = ann_of_seq[ann_of_seq[:, 0] == float(idx + 1), :] # 筛选真值文件里本帧的目标信息 + exist_gts.append(True if ann_of_current_frame.shape[0] != 0 else False) + + gt_to_file = osp.join(gt_to_path, img[:-4] + '.txt') + + with open(gt_to_file, 'w') as f_gt: + for i in range(ann_of_current_frame.shape[0]): + if int(ann_of_current_frame[i][6]) == 1: + # bbox xywh + x0, y0 = int(ann_of_current_frame[i][2]), int(ann_of_current_frame[i][3]) + w, h = int(ann_of_current_frame[i][4]), int(ann_of_current_frame[i][5]) + + xc, yc = x0 + w // 2, y0 + h // 2 # 中心点 x y + + # 归一化 + xc, yc = xc / w0, yc / h0 + w, h = w / w0, h / h0 + category_id = cat_id + + write_line = '{:d} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + category_id, xc, yc, w, h) + + f_gt.write(write_line) + + f_gt.close() + + # 第三步 产生图片索引train.txt等 + print(f'generating img index file of {seq}') + to_file = os.path.join('./uavdt/', split + '.txt') + with open(to_file, 'a') as f: + for idx, img in enumerate(imgs): + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + if exist_gts[idx]: + f.write('UAVDT/' + 'images/' + split + '/' \ + + seq + '/' + img + '\n') + + f.close() + + + +if __name__ == '__main__': + if not osp.exists('./uavdt'): + os.system('mkdir ./uavdt') + else: + os.system('rm -rf ./uavdt/*') + + parser = argparse.ArgumentParser() + parser.add_argument('--generate_imgs', action='store_true', help='whether generate soft link of imgs') + parser.add_argument('--certain_seqs', action='store_true', help='for debug') + parser.add_argument('--half', action='store_true', help='half frames') + parser.add_argument('--ratio', type=float, default=0.8, help='ratio of test dataset devide train dataset') + parser.add_argument('--random', action='store_true', help='random split train and test') + + opts = parser.parse_args() + + generate_imgs_and_labels(opts) + # python tools/convert_UAVDT_to_yolo.py --generate_imgs --half --random \ No newline at end of file diff --git a/test/yolov7-tracker/tools/convert_VisDrone_to_yolo.py b/test/yolov7-tracker/tools/convert_VisDrone_to_yolo.py new file mode 100644 index 0000000..c96e56d --- /dev/null +++ b/test/yolov7-tracker/tools/convert_VisDrone_to_yolo.py @@ -0,0 +1,182 @@ +""" +将VisDrone转换为yolo v5格式 +class_id, xc_norm, yc_norm, w_norm, h_norm +""" +import os +import os.path as osp +import argparse +import cv2 +import glob + +DATA_ROOT = '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' + + +# 以下两个seqs只跟踪车的时候有用 +certain_seqs = ['uav0000071_03240_v', 'uav0000072_04488_v','uav0000072_05448_v', 'uav0000072_06432_v','uav0000124_00944_v','uav0000126_00001_v','uav0000138_00000_v','uav0000145_00000_v','uav0000150_02310_v','uav0000222_03150_v','uav0000239_12336_v','uav0000243_00001_v', +'uav0000248_00001_v','uav0000263_03289_v','uav0000266_03598_v','uav0000273_00001_v','uav0000279_00001_v','uav0000281_00460_v','uav0000289_00001_v','uav0000289_06922_v','uav0000307_00000_v', +'uav0000308_00000_v','uav0000308_01380_v','uav0000326_01035_v','uav0000329_04715_v','uav0000361_02323_v','uav0000366_00001_v'] + +ignored_seqs = ['uav0000013_00000_v', 'uav0000013_01073_v', 'uav0000013_01392_v', + 'uav0000020_00406_v', 'uav0000079_00480_v', + 'uav0000084_00000_v', 'uav0000099_02109_v', 'uav0000086_00000_v', + 'uav0000073_00600_v', 'uav0000073_04464_v', 'uav0000088_00290_v'] + +image_wh_dict = {} # seq->(w,h) 字典 用于归一化 + +def generate_imgs(split_name='VisDrone2019-MOT-train', generate_imgs=True, if_certain_seqs=False, car_only=False): + """ + 产生图片文件夹 例如 VisDrone/images/VisDrone2019-MOT-train/uav0000076_00720_v/000010.jpg + 同时产生序列->高,宽的字典 便于后续 + + split: str, 'VisDrone2019-MOT-train', 'VisDrone2019-MOT-val' or 'VisDrone2019-MOT-test-dev' + if_certain_seqs: bool, use for debug. + """ + + if not if_certain_seqs: + seq_list = os.listdir(osp.join(DATA_ROOT, split_name, 'sequences')) # 所有序列名称 + else: + seq_list = certain_seqs + + if car_only: # 只跟踪车就忽略行人多的视频 + seq_list = [seq for seq in seq_list if seq not in ignored_seqs] + + # 遍历所有序列 给图片创建软链接 同时更新seq->(w,h)字典 + if_write_txt = True if glob.glob('./visdrone/*.txt') else False + # if_write_txt = True if not osp.exists(f'./visdrone/.txt') else False # 是否需要写txt 用于生成visdrone.train + + if not if_write_txt: + for seq in seq_list: + img_dir = osp.join(DATA_ROOT, split_name, 'sequences', seq) # 该序列下所有图片路径 + + imgs = sorted(os.listdir(img_dir)) # 所有图片 + + if generate_imgs: + to_path = osp.join(DATA_ROOT, 'images', split_name, seq) # 该序列图片存储位置 + if not osp.exists(to_path): + os.makedirs(to_path) + + for img in imgs: # 遍历该序列下的图片 + os.symlink(osp.join(img_dir, img), + osp.join(to_path, img)) # 创建软链接 + + img_sample = cv2.imread(osp.join(img_dir, imgs[0])) # 每个序列第一张图片 用于获取w, h + w, h = img_sample.shape[1], img_sample.shape[0] # w, h + + image_wh_dict[seq] = (w, h) # 更新seq->(w,h) 字典 + + # print(image_wh_dict) + # return + else: + with open('./visdrone.txt', 'a') as f: + for seq in seq_list: + img_dir = osp.join(DATA_ROOT, split_name, 'sequences', seq) # 该序列下所有图片路径 + + imgs = sorted(os.listdir(img_dir)) # 所有图片 + + if generate_imgs: + to_path = osp.join(DATA_ROOT, 'images', split_name, seq) # 该序列图片存储位置 + if not osp.exists(to_path): + os.makedirs(to_path) + + for img in imgs: # 遍历该序列下的图片 + + f.write('VisDrone2019/' + 'VisDrone2019/' + 'images/' + split_name + '/' \ + + seq + '/' + img + '\n') + + os.symlink(osp.join(img_dir, img), + osp.join(to_path, img)) # 创建软链接 + + img_sample = cv2.imread(osp.join(img_dir, imgs[0])) # 每个序列第一张图片 用于获取w, h + w, h = img_sample.shape[1], img_sample.shape[0] # w, h + + image_wh_dict[seq] = (w, h) # 更新seq->(w,h) 字典 + f.close() + if if_certain_seqs: # for debug + print(image_wh_dict) + + +def generate_labels(split='VisDrone2019-MOT-train', if_certain_seqs=False, car_only=False): + """ + split: str, 'train', 'val' or 'test' + if_certain_seqs: bool, use for debug. + """ + # from choose_anchors import image_wh_dict + # print(image_wh_dict) + if not if_certain_seqs: + seq_list = os.listdir(osp.join(DATA_ROOT, split, 'sequences')) # 序列列表 + else: + seq_list = certain_seqs + + if car_only: # 只跟踪车就忽略行人多的视频 + seq_list = [seq for seq in seq_list if seq not in ignored_seqs] + category_list = ['4', '5', '6', '9'] + else: + category_list = [str(i) for i in range(1, 11)] + + # 类别ID 从0开始 + category_dict = {category_list[idx]: idx for idx in range(len(category_list))} + # 每张图片分配一个txt + # 要从sequence的txt里分出来 + for seq in seq_list: + seq_dir = osp.join(DATA_ROOT, split, 'annotations', seq + '.txt') # 真值文件 + with open(seq_dir, 'r') as f: + lines = f.readlines() + + for row in lines: + + current_line = row.split(',') + + frame = current_line[0] # 第几帧 + if current_line[6] == '0' or current_line[7] not in category_list: + continue + + to_file = osp.join(DATA_ROOT, 'labels', split, seq) # 要写入的文件名 + # 如果不存在就创建 + if not osp.exists(to_file): + os.makedirs(to_file) + + to_file = osp.join(to_file, frame.zfill(7) + '.txt') + + category_id = category_dict[current_line[7]] + x0, y0 = int(current_line[2]), int(current_line[3]) # 左上角 x y + w, h = int(current_line[4]), int(current_line[5]) # 宽 高 + + x_c, y_c = x0 + w // 2, y0 + h // 2 # 中心点 x y + + image_w, image_h = image_wh_dict[seq][0], image_wh_dict[seq][1] # 图像高宽 + # 归一化 + w, h = w / image_w, h / image_h + x_c, y_c = x_c / image_w, y_c / image_h + + + with open(to_file, 'a') as f_to: + write_line = '{:d} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + category_id, x_c, y_c, w, h) + + f_to.write(write_line) + + f_to.close() + + + f.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--split', type=str, default='VisDrone2019-MOT-train', help='train or test') + parser.add_argument('--generate_imgs', action='store_true', help='whether generate soft link of imgs') + parser.add_argument('--car_only', action='store_true', help='only cars') + parser.add_argument('--if_certain_seqs', action='store_true', help='for debug') + + opt = parser.parse_args() + print('generating images...') + generate_imgs(opt.split, opt.generate_imgs, opt.if_certain_seqs, opt.car_only) + + print('generating labels...') + generate_labels(opt.split, opt.if_certain_seqs, opt.car_only) + + print('Done!') + + + # python convert_VisDrone_to_yolo.py --split VisDrone2019-MOT-train + # python convert_VisDrone_to_yolo.py --split VisDrone2019-MOT-train --car_only --if_certain_seqs \ No newline at end of file diff --git a/test/yolov7-tracker/tools/convert_VisDrone_to_yolov2.py b/test/yolov7-tracker/tools/convert_VisDrone_to_yolov2.py new file mode 100644 index 0000000..b48a4f3 --- /dev/null +++ b/test/yolov7-tracker/tools/convert_VisDrone_to_yolov2.py @@ -0,0 +1,168 @@ +""" +将VisDrone转换为yolo v5格式 +class_id, xc_norm, yc_norm, w_norm, h_norm + +改动: +1. 将产生img和label函数合成一个 +2. 增加如果无label就不产生当前img路径的功能 +3. 增加half选项 每个视频截取一半 +""" +import os +import os.path as osp +import argparse +import cv2 +import glob +import numpy as np + +DATA_ROOT = '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' + + +# 以下两个seqs只跟踪车的时候有用 +certain_seqs = ['uav0000071_03240_v', 'uav0000072_04488_v','uav0000072_05448_v', 'uav0000072_06432_v','uav0000124_00944_v','uav0000126_00001_v','uav0000138_00000_v','uav0000145_00000_v','uav0000150_02310_v','uav0000222_03150_v','uav0000239_12336_v','uav0000243_00001_v', +'uav0000248_00001_v','uav0000263_03289_v','uav0000266_03598_v','uav0000273_00001_v','uav0000279_00001_v','uav0000281_00460_v','uav0000289_00001_v','uav0000289_06922_v','uav0000307_00000_v', +'uav0000308_00000_v','uav0000308_01380_v','uav0000326_01035_v','uav0000329_04715_v','uav0000361_02323_v','uav0000366_00001_v'] + +ignored_seqs = ['uav0000013_00000_v', 'uav0000013_01073_v', 'uav0000013_01392_v', + 'uav0000020_00406_v', 'uav0000079_00480_v', + 'uav0000084_00000_v', 'uav0000099_02109_v', 'uav0000086_00000_v', + 'uav0000073_00600_v', 'uav0000073_04464_v', 'uav0000088_00290_v'] + +image_wh_dict = {} # seq->(w,h) 字典 用于归一化 + +def generate_imgs_and_labels(opts): + """ + 产生图片路径的txt文件以及yolo格式真值文件 + """ + if not opts.certain_seqs: + seq_list = os.listdir(osp.join(DATA_ROOT, opts.split_name, 'sequences')) # 所有序列名称 + else: + seq_list = certain_seqs + + if opts.car_only: # 只跟踪车就忽略行人多的视频 + seq_list = [seq for seq in seq_list if seq not in ignored_seqs] + category_list = [4, 5, 6, 9] # 感兴趣的类别编号 List[int] + else: + category_list = [i for i in range(1, 11)] + + print(f'Total {len(seq_list)} seqs!!') + if not osp.exists('./visdrone/'): + os.makedirs('./visdrone/') + + # 类别ID 从0开始 + category_dict = {category_list[idx]: idx for idx in range(len(category_list))} + + txt_name_dict = {'VisDrone2019-MOT-train': 'train', + 'VisDrone2019-MOT-val': 'val', + 'VisDrone2019-MOT-test-dev': 'test'} # 产生txt文件名称对应关系 + + # 如果已经存在就不写了 + write_txt = False if os.path.isfile(os.path.join('./visdrone', txt_name_dict[opts.split_name] + '.txt')) else True + print(f'write txt is {write_txt}') + + frame_range = {'start': 0.0, 'end': 1.0} + if opts.half: # VisDrone-half 截取一半 + frame_range['end'] = 0.5 + + # 以序列为单位进行处理 + for seq in seq_list: + img_dir = osp.join(DATA_ROOT, opts.split_name, 'sequences', seq) # 该序列下所有图片路径 + + imgs = sorted(os.listdir(img_dir)) # 所有图片 + seq_length = len(imgs) # 序列长度 + + img_eg = cv2.imread(os.path.join(img_dir, imgs[0])) # 序列的第一张图 用以计算高宽 + w0, h0 = img_eg.shape[1], img_eg.shape[0] # 原始高宽 + + ann_of_seq_path = os.path.join(DATA_ROOT, opts.split_name, 'annotations', seq + '.txt') # GT文件路径 + ann_of_seq = np.loadtxt(ann_of_seq_path, dtype=np.float32, delimiter=',') # GT内容 + + gt_to_path = osp.join(DATA_ROOT, 'labels', opts.split_name, seq) # 要写入的真值文件夹 + # 如果不存在就创建 + if not osp.exists(gt_to_path): + os.makedirs(gt_to_path) + + exist_gts = [] # 初始化该列表 每个元素对应该seq的frame中有无真值框 + # 如果没有 就在train.txt产生图片路径 + + for idx, img in enumerate(imgs): + # img: 相对路径 即 图片名称 0000001.jpg + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + # 第一步 产生图片软链接 + # print('step1, creating imgs symlink...') + if opts.generate_imgs: + img_to_path = osp.join(DATA_ROOT, 'images', opts.split_name, seq) # 该序列图片存储位置 + + if not osp.exists(img_to_path): + os.makedirs(img_to_path) + + os.symlink(osp.join(img_dir, img), + osp.join(img_to_path, img)) # 创建软链接 + # print('Done!\n') + + # 第二步 产生真值文件 + # print('step2, generating gt files...') + + # 根据本序列的真值文件读取 + # ann_idx = int(ann_of_seq[:, 0]) == idx + 1 + ann_of_current_frame = ann_of_seq[ann_of_seq[:, 0] == float(idx + 1), :] # 筛选真值文件里本帧的目标信息 + exist_gts.append(True if ann_of_current_frame.shape[0] != 0 else False) + + gt_to_file = osp.join(gt_to_path, img[:-4] + '.txt') + + with open(gt_to_file, 'a') as f_gt: + for i in range(ann_of_current_frame.shape[0]): + + category = int(ann_of_current_frame[i][7]) + if int(ann_of_current_frame[i][6]) == 1 and category in category_list: + + # bbox xywh + x0, y0 = int(ann_of_current_frame[i][2]), int(ann_of_current_frame[i][3]) + w, h = int(ann_of_current_frame[i][4]), int(ann_of_current_frame[i][5]) + + xc, yc = x0 + w // 2, y0 + h // 2 # 中心点 x y + + # 归一化 + xc, yc = xc / w0, yc / h0 + w, h = w / w0, h / h0 + + category_id = category_dict[category] + + write_line = '{:d} {:.6f} {:.6f} {:.6f} {:.6f}\n'.format( + category_id, xc, yc, w, h) + + f_gt.write(write_line) + + f_gt.close() + # print('Done!\n') + print(f'img symlink and gt files of seq {seq} Done!') + # 第三步 产生图片索引train.txt等 + print(f'generating img index file of {seq}') + if write_txt: + to_file = os.path.join('./visdrone', txt_name_dict[opts.split_name] + '.txt') + with open(to_file, 'a') as f: + for idx, img in enumerate(imgs): + if idx < int(seq_length * frame_range['start']) or idx > int(seq_length * frame_range['end']): + continue + + if exist_gts[idx]: + f.write('VisDrone2019/' + 'VisDrone2019/' + 'images/' + opts.split_name + '/' \ + + seq + '/' + img + '\n') + + f.close() + + print('All done!!') + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--split_name', type=str, default='VisDrone2019-MOT-train', help='train or test') + parser.add_argument('--generate_imgs', action='store_true', help='whether generate soft link of imgs') + parser.add_argument('--car_only', action='store_true', help='only cars') + parser.add_argument('--certain_seqs', action='store_true', help='for debug') + parser.add_argument('--half', action='store_true', help='half frames') + + opts = parser.parse_args() + + generate_imgs_and_labels(opts) + # python tools/convert_VisDrone_to_yolov2.py --split_name VisDrone2019-MOT-train --generate_imgs --car_only --half \ No newline at end of file diff --git a/test/yolov7-tracker/tools/reparameterization.ipynb b/test/yolov7-tracker/tools/reparameterization.ipynb new file mode 100644 index 0000000..4e9a810 --- /dev/null +++ b/test/yolov7-tracker/tools/reparameterization.ipynb @@ -0,0 +1,479 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d7cbe5ee", + "metadata": {}, + "source": [ + "# Reparameterization" + ] + }, + { + "cell_type": "markdown", + "id": "13393b70", + "metadata": {}, + "source": [ + "## YOLOv7 reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf53becf", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.105.m.0.weight'].data[i, :, :, :] *= state_dict['model.105.im.0.implicit'].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.105.m.1.weight'].data[i, :, :, :] *= state_dict['model.105.im.1.implicit'].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.105.m.2.weight'].data[i, :, :, :] *= state_dict['model.105.im.2.implicit'].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.105.m.0.bias'].data += state_dict['model.105.m.0.weight'].mul(state_dict['model.105.ia.0.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.105.m.1.bias'].data += state_dict['model.105.m.1.weight'].mul(state_dict['model.105.ia.1.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.105.m.2.bias'].data += state_dict['model.105.m.2.weight'].mul(state_dict['model.105.ia.2.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.105.m.0.bias'].data *= state_dict['model.105.im.0.implicit'].data.squeeze()\n", + "model.state_dict()['model.105.m.1.bias'].data *= state_dict['model.105.im.1.implicit'].data.squeeze()\n", + "model.state_dict()['model.105.m.2.bias'].data *= state_dict['model.105.im.2.implicit'].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7.pt')\n" + ] + }, + { + "cell_type": "markdown", + "id": "5b396a53", + "metadata": {}, + "source": [ + "## YOLOv7x reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d54d17f", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7x.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7x.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.121.m.0.weight'].data[i, :, :, :] *= state_dict['model.121.im.0.implicit'].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.121.m.1.weight'].data[i, :, :, :] *= state_dict['model.121.im.1.implicit'].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.121.m.2.weight'].data[i, :, :, :] *= state_dict['model.121.im.2.implicit'].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.121.m.0.bias'].data += state_dict['model.121.m.0.weight'].mul(state_dict['model.121.ia.0.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.121.m.1.bias'].data += state_dict['model.121.m.1.weight'].mul(state_dict['model.121.ia.1.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.121.m.2.bias'].data += state_dict['model.121.m.2.weight'].mul(state_dict['model.121.ia.2.implicit']).sum(1).squeeze()\n", + "model.state_dict()['model.121.m.0.bias'].data *= state_dict['model.121.im.0.implicit'].data.squeeze()\n", + "model.state_dict()['model.121.m.1.bias'].data *= state_dict['model.121.im.1.implicit'].data.squeeze()\n", + "model.state_dict()['model.121.m.2.bias'].data *= state_dict['model.121.im.2.implicit'].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7x.pt')\n" + ] + }, + { + "cell_type": "markdown", + "id": "11a9108e", + "metadata": {}, + "source": [ + "## YOLOv7-W6 reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d032c629", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7-w6.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7-w6.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "idx = 118\n", + "idx2 = 122\n", + "\n", + "# copy weights of lead head\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data -= model.state_dict()['model.{}.m.0.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data -= model.state_dict()['model.{}.m.1.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data -= model.state_dict()['model.{}.m.2.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data -= model.state_dict()['model.{}.m.3.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data -= model.state_dict()['model.{}.m.0.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data -= model.state_dict()['model.{}.m.1.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data -= model.state_dict()['model.{}.m.2.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data -= model.state_dict()['model.{}.m.3.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.bias'.format(idx2)].data\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.{}.m.0.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.0.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.1.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.1.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.2.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.2.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.3.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.3.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].mul(state_dict['model.{}.ia.0.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].mul(state_dict['model.{}.ia.1.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].mul(state_dict['model.{}.ia.2.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].mul(state_dict['model.{}.ia.3.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data *= state_dict['model.{}.im.0.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data *= state_dict['model.{}.im.1.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data *= state_dict['model.{}.im.2.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data *= state_dict['model.{}.im.3.implicit'.format(idx2)].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7-w6.pt')\n" + ] + }, + { + "cell_type": "markdown", + "id": "5f093d43", + "metadata": {}, + "source": [ + "## YOLOv7-E6 reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa2b2142", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7-e6.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7-e6.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "idx = 140\n", + "idx2 = 144\n", + "\n", + "# copy weights of lead head\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data -= model.state_dict()['model.{}.m.0.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data -= model.state_dict()['model.{}.m.1.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data -= model.state_dict()['model.{}.m.2.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data -= model.state_dict()['model.{}.m.3.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data -= model.state_dict()['model.{}.m.0.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data -= model.state_dict()['model.{}.m.1.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data -= model.state_dict()['model.{}.m.2.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data -= model.state_dict()['model.{}.m.3.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.bias'.format(idx2)].data\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.{}.m.0.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.0.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.1.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.1.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.2.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.2.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.3.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.3.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].mul(state_dict['model.{}.ia.0.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].mul(state_dict['model.{}.ia.1.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].mul(state_dict['model.{}.ia.2.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].mul(state_dict['model.{}.ia.3.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data *= state_dict['model.{}.im.0.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data *= state_dict['model.{}.im.1.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data *= state_dict['model.{}.im.2.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data *= state_dict['model.{}.im.3.implicit'.format(idx2)].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7-e6.pt')\n" + ] + }, + { + "cell_type": "markdown", + "id": "a3bccf89", + "metadata": {}, + "source": [ + "## YOLOv7-D6 reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5216b70", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7-d6.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7-d6.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "idx = 162\n", + "idx2 = 166\n", + "\n", + "# copy weights of lead head\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data -= model.state_dict()['model.{}.m.0.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data -= model.state_dict()['model.{}.m.1.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data -= model.state_dict()['model.{}.m.2.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data -= model.state_dict()['model.{}.m.3.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data -= model.state_dict()['model.{}.m.0.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data -= model.state_dict()['model.{}.m.1.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data -= model.state_dict()['model.{}.m.2.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data -= model.state_dict()['model.{}.m.3.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.bias'.format(idx2)].data\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.{}.m.0.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.0.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.1.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.1.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.2.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.2.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.3.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.3.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].mul(state_dict['model.{}.ia.0.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].mul(state_dict['model.{}.ia.1.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].mul(state_dict['model.{}.ia.2.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].mul(state_dict['model.{}.ia.3.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data *= state_dict['model.{}.im.0.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data *= state_dict['model.{}.im.1.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data *= state_dict['model.{}.im.2.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data *= state_dict['model.{}.im.3.implicit'.format(idx2)].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7-d6.pt')\n" + ] + }, + { + "cell_type": "markdown", + "id": "334c273b", + "metadata": {}, + "source": [ + "## YOLOv7-E6E reparameterization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "635fd8d2", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "from copy import deepcopy\n", + "from models.yolo import Model\n", + "import torch\n", + "from utils.torch_utils import select_device, is_parallel\n", + "\n", + "device = select_device('0', batch_size=1)\n", + "# model trained by cfg/training/*.yaml\n", + "ckpt = torch.load('cfg/training/yolov7-e6e.pt', map_location=device)\n", + "# reparameterized model in cfg/deploy/*.yaml\n", + "model = Model('cfg/deploy/yolov7-e6e.yaml', ch=3, nc=80).to(device)\n", + "\n", + "# copy intersect weights\n", + "state_dict = ckpt['model'].float().state_dict()\n", + "exclude = []\n", + "intersect_state_dict = {k: v for k, v in state_dict.items() if k in model.state_dict() and not any(x in k for x in exclude) and v.shape == model.state_dict()[k].shape}\n", + "model.load_state_dict(intersect_state_dict, strict=False)\n", + "model.names = ckpt['model'].names\n", + "model.nc = ckpt['model'].nc\n", + "\n", + "idx = 261\n", + "idx2 = 265\n", + "\n", + "# copy weights of lead head\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data -= model.state_dict()['model.{}.m.0.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data -= model.state_dict()['model.{}.m.1.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data -= model.state_dict()['model.{}.m.2.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data -= model.state_dict()['model.{}.m.3.weight'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.weight'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.weight'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.weight'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.weight'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data -= model.state_dict()['model.{}.m.0.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data -= model.state_dict()['model.{}.m.1.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data -= model.state_dict()['model.{}.m.2.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data -= model.state_dict()['model.{}.m.3.bias'.format(idx)].data\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.bias'.format(idx2)].data\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.bias'.format(idx2)].data\n", + "\n", + "# reparametrized YOLOR\n", + "for i in range(255):\n", + " model.state_dict()['model.{}.m.0.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.0.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.1.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.1.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.2.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.2.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + " model.state_dict()['model.{}.m.3.weight'.format(idx)].data[i, :, :, :] *= state_dict['model.{}.im.3.implicit'.format(idx2)].data[:, i, : :].squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data += state_dict['model.{}.m.0.weight'.format(idx2)].mul(state_dict['model.{}.ia.0.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data += state_dict['model.{}.m.1.weight'.format(idx2)].mul(state_dict['model.{}.ia.1.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data += state_dict['model.{}.m.2.weight'.format(idx2)].mul(state_dict['model.{}.ia.2.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data += state_dict['model.{}.m.3.weight'.format(idx2)].mul(state_dict['model.{}.ia.3.implicit'.format(idx2)]).sum(1).squeeze()\n", + "model.state_dict()['model.{}.m.0.bias'.format(idx)].data *= state_dict['model.{}.im.0.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.1.bias'.format(idx)].data *= state_dict['model.{}.im.1.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.2.bias'.format(idx)].data *= state_dict['model.{}.im.2.implicit'.format(idx2)].data.squeeze()\n", + "model.state_dict()['model.{}.m.3.bias'.format(idx)].data *= state_dict['model.{}.im.3.implicit'.format(idx2)].data.squeeze()\n", + "\n", + "# model to be saved\n", + "ckpt = {'model': deepcopy(model.module if is_parallel(model) else model).half(),\n", + " 'optimizer': None,\n", + " 'training_results': None,\n", + " 'epoch': -1}\n", + "\n", + "# save reparameterized model\n", + "torch.save(ckpt, 'cfg/deploy/yolov7-e6e.pt')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63a62625", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/yolov7-tracker/tracker/config_files/mot17.yaml b/test/yolov7-tracker/tracker/config_files/mot17.yaml new file mode 100644 index 0000000..360bd0c --- /dev/null +++ b/test/yolov7-tracker/tracker/config_files/mot17.yaml @@ -0,0 +1,32 @@ +# Config file of MOT17 dataset + +DATASET_ROOT: '/data/wujiapeng/datasets/MOT17' # your dataset root +SPLIT: test +CATEGORY_NAMES: # category names to show + - 'pedestrian' + +CATEGORY_DICT: + 0: 'pedestrian' + +CERTAIN_SEQS: + - +IGNORE_SEQS: # Seqs you want to ignore + - + +YAML_DICT: '' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend) + +TRACK_EVAL: # If use TrackEval to evaluate, use these configs + 'DISPLAY_LESS_PROGRESS': False + 'GT_FOLDER': '/data/wujiapeng/datasets/MOT17/train' + 'TRACKERS_FOLDER': './tracker/results' + 'SKIP_SPLIT_FOL': True + 'TRACKER_SUB_FOLDER': '' + 'SEQ_INFO': + 'MOT17-02-SDP': null + 'MOT17-04-SDP': null + 'MOT17-05-SDP': null + 'MOT17-09-SDP': null + 'MOT17-10-SDP': null + 'MOT17-11-SDP': null + 'MOT17-13-SDP': null + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt' diff --git a/test/yolov7-tracker/tracker/config_files/uavdt.yaml b/test/yolov7-tracker/tracker/config_files/uavdt.yaml new file mode 100644 index 0000000..a2aabc3 --- /dev/null +++ b/test/yolov7-tracker/tracker/config_files/uavdt.yaml @@ -0,0 +1,26 @@ +# Config file of UAVDT dataset + +DATASET_ROOT: '/data/wujiapeng/datasets/UAVDT' # your dataset root +SPLIT: test +CATEGORY_NAMES: # category names to show + - 'car' + +CATEGORY_DICT: + 0: 'car' + +CERTAIN_SEQS: + - +IGNORE_SEQS: # Seqs you want to ignore + - + +YAML_DICT: './data/UAVDT.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend) + +TRACK_EVAL: # If use TrackEval to evaluate, use these configs + 'DISPLAY_LESS_PROGRESS': False + 'GT_FOLDER': '/data/wujiapeng/datasets/UAVDT/UAV-benchmark-M' + 'TRACKERS_FOLDER': './tracker/results' + 'SKIP_SPLIT_FOL': True + 'TRACKER_SUB_FOLDER': '' + 'SEQ_INFO': + 'M0101': 407 + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt' \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/config_files/visdrone.yaml b/test/yolov7-tracker/tracker/config_files/visdrone.yaml new file mode 100644 index 0000000..bf0636f --- /dev/null +++ b/test/yolov7-tracker/tracker/config_files/visdrone.yaml @@ -0,0 +1,61 @@ +# Config file of VisDrone dataset + +DATASET_ROOT: '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' +SPLIT: test +CATEGORY_NAMES: + - 'pedestrain' + - 'people' + - 'bicycle' + - 'car' + - 'van' + - 'truck' + - 'tricycle' + - 'awning-tricycle' + - 'bus' + - 'motor' + +CATEGORY_DICT: + 0: 'pedestrain' + 1: 'people' + 2: 'bicycle' + 3: 'car' + 4: 'van' + 5: 'truck' + 6: 'tricycle' + 7: 'awning-tricycle' + 8: 'bus' + 9: 'motor' + +CERTAIN_SEQS: + - + +IGNORE_SEQS: # Seqs you want to ignore + - + +YAML_DICT: './data/Visdrone_all.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend) + +TRACK_EVAL: # If use TrackEval to evaluate, use these configs + 'DISPLAY_LESS_PROGRESS': False + 'GT_FOLDER': '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019/VisDrone2019-MOT-test-dev/annotations' + 'TRACKERS_FOLDER': './tracker/results' + 'SKIP_SPLIT_FOL': True + 'TRACKER_SUB_FOLDER': '' + 'SEQ_INFO': + 'uav0000009_03358_v': 219 + 'uav0000073_00600_v': 328 + 'uav0000073_04464_v': 312 + 'uav0000077_00720_v': 780 + 'uav0000088_00290_v': 296 + 'uav0000119_02301_v': 179 + 'uav0000120_04775_v': 1000 + 'uav0000161_00000_v': 308 + 'uav0000188_00000_v': 260 + 'uav0000201_00000_v': 677 + 'uav0000249_00001_v': 360 + 'uav0000249_02688_v': 244 + 'uav0000297_00000_v': 146 + 'uav0000297_02761_v': 373 + 'uav0000306_00230_v': 420 + 'uav0000355_00001_v': 468 + 'uav0000370_00001_v': 265 + 'GT_LOC_FORMAT': '{gt_folder}/{seq}.txt' \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/config_files/visdrone_part.yaml b/test/yolov7-tracker/tracker/config_files/visdrone_part.yaml new file mode 100644 index 0000000..5b2ea60 --- /dev/null +++ b/test/yolov7-tracker/tracker/config_files/visdrone_part.yaml @@ -0,0 +1,51 @@ +# Config file of VisDrone dataset + +DATASET_ROOT: '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' +SPLIT: test +CATEGORY_NAMES: + - 'pedestrain' + - 'car' + - 'van' + - 'truck' + - 'bus' + +CATEGORY_DICT: + 0: 'pedestrain' + 1: 'car' + 2: 'van' + 3: 'truck' + 4: 'bus' + +CERTAIN_SEQS: + - + +IGNORE_SEQS: # Seqs you want to ignore + - + +YAML_DICT: './data/Visdrone_all.yaml' # NOTE: ONLY for yolo v5 model loader(func DetectMultiBackend) + +TRACK_EVAL: # If use TrackEval to evaluate, use these configs + 'DISPLAY_LESS_PROGRESS': False + 'GT_FOLDER': '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019/VisDrone2019-MOT-test-dev/annotations' + 'TRACKERS_FOLDER': './tracker/results' + 'SKIP_SPLIT_FOL': True + 'TRACKER_SUB_FOLDER': '' + 'SEQ_INFO': + 'uav0000009_03358_v': 219 + 'uav0000073_00600_v': 328 + 'uav0000073_04464_v': 312 + 'uav0000077_00720_v': 780 + 'uav0000088_00290_v': 296 + 'uav0000119_02301_v': 179 + 'uav0000120_04775_v': 1000 + 'uav0000161_00000_v': 308 + 'uav0000188_00000_v': 260 + 'uav0000201_00000_v': 677 + 'uav0000249_00001_v': 360 + 'uav0000249_02688_v': 244 + 'uav0000297_00000_v': 146 + 'uav0000297_02761_v': 373 + 'uav0000306_00230_v': 420 + 'uav0000355_00001_v': 468 + 'uav0000370_00001_v': 265 + 'GT_LOC_FORMAT': '{gt_folder}/{seq}.txt' \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/my_timer.py b/test/yolov7-tracker/tracker/my_timer.py new file mode 100644 index 0000000..c9b15fb --- /dev/null +++ b/test/yolov7-tracker/tracker/my_timer.py @@ -0,0 +1,37 @@ +import time + + +class Timer(object): + """A simple timer.""" + def __init__(self): + self.total_time = 0. + self.calls = 0 + self.start_time = 0. + self.diff = 0. + self.average_time = 0. + + self.duration = 0. + + def tic(self): + # using time.time instead of time.clock because time time.clock + # does not normalize for multithreading + self.start_time = time.time() + + def toc(self, average=True): + self.diff = time.time() - self.start_time + self.total_time += self.diff + self.calls += 1 + self.average_time = self.total_time / self.calls + if average: + self.duration = self.average_time + else: + self.duration = self.diff + return self.duration + + def clear(self): + self.total_time = 0. + self.calls = 0 + self.start_time = 0. + self.diff = 0. + self.average_time = 0. + self.duration = 0. \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/track.py b/test/yolov7-tracker/tracker/track.py new file mode 100644 index 0000000..5774835 --- /dev/null +++ b/test/yolov7-tracker/tracker/track.py @@ -0,0 +1,305 @@ +""" +main code for track +""" +import sys, os +import numpy as np +import torch +import cv2 +from PIL import Image +from tqdm import tqdm +import yaml + +from loguru import logger +import argparse + +from tracking_utils.envs import select_device +from tracking_utils.tools import * +from tracking_utils.visualization import plot_img, save_video +from my_timer import Timer + +from tracker_dataloader import TestDataset + +# trackers +from trackers.byte_tracker import ByteTracker +from trackers.sort_tracker import SortTracker +from trackers.botsort_tracker import BotTracker +from trackers.c_biou_tracker import C_BIoUTracker +from trackers.ocsort_tracker import OCSortTracker +from trackers.deepsort_tracker import DeepSortTracker +from trackers.strongsort_tracker import StrongSortTracker +from trackers.sparse_tracker import SparseTracker + +# YOLOX modules +try: + from yolox.exp import get_exp + from yolox_utils.postprocess import postprocess_yolox + from yolox.utils import fuse_model +except Exception as e: + logger.warning(e) + logger.warning('Load yolox fail. If you want to use yolox, please check the installation.') + pass + +# YOLOv7 modules +try: + sys.path.append(os.getcwd()) + from models.experimental import attempt_load + from utils.torch_utils import select_device, time_synchronized, TracedModel + from utils.general import non_max_suppression, scale_coords, check_img_size + from yolov7_utils.postprocess import postprocess as postprocess_yolov7 + +except Exception as e: + logger.warning(e) + logger.warning('Load yolov7 fail. If you want to use yolov7, please check the installation.') + pass + +# YOLOv8 modules +try: + from ultralytics import YOLO + from yolov8_utils.postprocess import postprocess as postprocess_yolov8 + +except Exception as e: + logger.warning(e) + logger.warning('Load yolov8 fail. If you want to use yolov8, please check the installation.') + pass + +TRACKER_DICT = { + 'sort': SortTracker, + 'bytetrack': ByteTracker, + 'botsort': BotTracker, + 'c_bioutrack': C_BIoUTracker, + 'ocsort': OCSortTracker, + 'deepsort': DeepSortTracker, + 'strongsort': StrongSortTracker, + 'sparsetrack': SparseTracker +} + +def get_args(): + + parser = argparse.ArgumentParser() + + """general""" + parser.add_argument('--dataset', type=str, default='visdrone_part', help='visdrone, mot17, etc.') + parser.add_argument('--detector', type=str, default='yolov8', help='yolov7, yolox, etc.') + parser.add_argument('--tracker', type=str, default='sort', help='sort, deepsort, etc') + parser.add_argument('--reid_model', type=str, default='osnet_x0_25', help='osnet or deppsort') + + parser.add_argument('--kalman_format', type=str, default='default', help='use what kind of Kalman, sort, deepsort, byte, etc.') + parser.add_argument('--img_size', type=int, default=1280, help='image size, [h, w]') + + parser.add_argument('--conf_thresh', type=float, default=0.2, help='filter tracks') + parser.add_argument('--nms_thresh', type=float, default=0.7, help='thresh for NMS') + parser.add_argument('--iou_thresh', type=float, default=0.5, help='IOU thresh to filter tracks') + + parser.add_argument('--device', type=str, default='6', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + + """yolox""" + parser.add_argument('--yolox_exp_file', type=str, default='./tracker/yolox_utils/yolox_m.py') + + """model path""" + parser.add_argument('--detector_model_path', type=str, default='./weights/best.pt', help='model path') + parser.add_argument('--trace', type=bool, default=False, help='traced model of YOLO v7') + # other model path + parser.add_argument('--reid_model_path', type=str, default='./weights/osnet_x0_25.pth', help='path for reid model path') + parser.add_argument('--dhn_path', type=str, default='./weights/DHN.pth', help='path of DHN path for DeepMOT') + + + """other options""" + parser.add_argument('--discard_reid', action='store_true', help='discard reid model, only work in bot-sort etc. which need a reid part') + parser.add_argument('--track_buffer', type=int, default=30, help='tracking buffer') + parser.add_argument('--gamma', type=float, default=0.1, help='param to control fusing motion and apperance dist') + parser.add_argument('--min_area', type=float, default=150, help='use to filter small bboxs') + + parser.add_argument('--save_dir', type=str, default='track_results/{dataset_name}/{split}') + parser.add_argument('--save_images', action='store_true', help='save tracking results (image)') + parser.add_argument('--save_videos', action='store_true', help='save tracking results (video)') + + parser.add_argument('--track_eval', type=bool, default=True, help='Use TrackEval to evaluate') + + return parser.parse_args() + +def main(args, dataset_cfgs): + + """1. set some params""" + + # NOTE: if save video, you must save image + if args.save_videos: + args.save_images = True + + """2. load detector""" + device = select_device(args.device) + + if args.detector == 'yolox': + + exp = get_exp(args.yolox_exp_file, None) # TODO: modify num_classes etc. for specific dataset + model_img_size = exp.input_size + model = exp.get_model() + model.to(device) + model.eval() + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + ckpt = torch.load(args.detector_model_path, map_location=device) + model.load_state_dict(ckpt['model']) + logger.info("loaded checkpoint done") + model = fuse_model(model) + + stride = None # match with yolo v7 + + logger.info(f'Now detector is on device {next(model.parameters()).device}') + + elif args.detector == 'yolov7': + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + model = attempt_load(args.detector_model_path, map_location=device) + + # get inference img size + stride = int(model.stride.max()) # model stride + model_img_size = check_img_size(args.img_size, s=stride) # check img_size + + # Traced model + model = TracedModel(model, device=device, img_size=args.img_size) + # model.half() + + logger.info("loaded checkpoint done") + + logger.info(f'Now detector is on device {next(model.parameters()).device}') + + elif args.detector == 'yolov8': + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + model = YOLO(args.detector_model_path) + + model_img_size = [None, None] + stride = None + + logger.info("loaded checkpoint done") + + else: + logger.error(f"detector {args.detector} is not supprted") + exit(0) + + """3. load sequences""" + DATA_ROOT = dataset_cfgs['DATASET_ROOT'] + SPLIT = dataset_cfgs['SPLIT'] + + seqs = sorted(os.listdir(os.path.join(DATA_ROOT, 'images', SPLIT))) + seqs = [seq for seq in seqs if seq not in dataset_cfgs['IGNORE_SEQS']] + if not None in dataset_cfgs['CERTAIN_SEQS']: + seqs = dataset_cfgs['CERTAIN_SEQS'] + + logger.info(f'Total {len(seqs)} seqs will be tracked: {seqs}') + + save_dir = args.save_dir.format(dataset_name=args.dataset, split=SPLIT) + + + """4. Tracking""" + + # set timer + timer = Timer() + seq_fps = [] + + for seq in seqs: + logger.info(f'--------------tracking seq {seq}--------------') + + dataset = TestDataset(DATA_ROOT, SPLIT, seq_name=seq, img_size=model_img_size, model=args.detector, stride=stride) + + data_loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False) + + tracker = TRACKER_DICT[args.tracker](args, ) + + process_bar = enumerate(data_loader) + process_bar = tqdm(process_bar, total=len(data_loader), ncols=150) + + results = [] + + for frame_idx, (ori_img, img) in process_bar: + + # start timing this frame + timer.tic() + + if args.detector == 'yolov8': + img = img.squeeze(0).cpu().numpy() + + else: + img = img.to(device) # (1, C, H, W) + img = img.float() + + ori_img = ori_img.squeeze(0) + + # get detector output + with torch.no_grad(): + if args.detector == 'yolov8': + output = model.predict(img, conf=args.conf_thresh, iou=args.nms_thresh) + else: + output = model(img) + + # postprocess output to original scales + if args.detector == 'yolox': + output = postprocess_yolox(output, len(dataset_cfgs['CATEGORY_NAMES']), conf_thresh=args.conf_thresh, + img=img, ori_img=ori_img) + + elif args.detector == 'yolov7': + output = postprocess_yolov7(output, args.conf_thresh, args.nms_thresh, img.shape[2:], ori_img.shape) + + elif args.detector == 'yolov8': + output = postprocess_yolov8(output) + + else: raise NotImplementedError + + # output: (tlbr, conf, cls) + # convert tlbr to tlwh + if isinstance(output, torch.Tensor): + output = output.detach().cpu().numpy() + output[:, 2] -= output[:, 0] + output[:, 3] -= output[:, 1] + current_tracks = tracker.update(output, img, ori_img.cpu().numpy()) + + # save results + cur_tlwh, cur_id, cur_cls, cur_score = [], [], [], [] + for trk in current_tracks: + bbox = trk.tlwh + id = trk.track_id + cls = trk.category + score = trk.score + + # filter low area bbox + if bbox[2] * bbox[3] > args.min_area: + cur_tlwh.append(bbox) + cur_id.append(id) + cur_cls.append(cls) + cur_score.append(score) + # results.append((frame_id + 1, id, bbox, cls)) + + results.append((frame_idx + 1, cur_id, cur_tlwh, cur_cls, cur_score)) + + timer.toc() + + if args.save_images: + plot_img(img=ori_img, frame_id=frame_idx, results=[cur_tlwh, cur_id, cur_cls], + save_dir=os.path.join(save_dir, 'vis_results')) + + save_results(folder_name=os.path.join(args.dataset, SPLIT), + seq_name=seq, + results=results) + + # show the fps + seq_fps.append(frame_idx / timer.total_time) + logger.info(f'fps of seq {seq}: {seq_fps[-1]}') + timer.clear() + + if args.save_videos: + save_video(images_path=os.path.join(save_dir, 'vis_results')) + logger.info(f'save video of {seq} done') + + # show the average fps + logger.info(f'average fps: {np.mean(seq_fps)}') + + +if __name__ == '__main__': + + args = get_args() + + with open(f'./tracker/config_files/{args.dataset}.yaml', 'r') as f: + cfgs = yaml.load(f, Loader=yaml.FullLoader) + + + main(args, cfgs) diff --git a/test/yolov7-tracker/tracker/track_demo.py b/test/yolov7-tracker/tracker/track_demo.py new file mode 100644 index 0000000..40a940c --- /dev/null +++ b/test/yolov7-tracker/tracker/track_demo.py @@ -0,0 +1,266 @@ +""" +main code for track +""" +import sys, os +import numpy as np +import torch +import cv2 +from PIL import Image +from tqdm import tqdm +import yaml + +from loguru import logger +import argparse + +from tracking_utils.envs import select_device +from tracking_utils.tools import * +from tracking_utils.visualization import plot_img, save_video + +from tracker_dataloader import TestDataset, DemoDataset + +# trackers +from trackers.byte_tracker import ByteTracker +from trackers.sort_tracker import SortTracker +from trackers.botsort_tracker import BotTracker +from trackers.c_biou_tracker import C_BIoUTracker +from trackers.ocsort_tracker import OCSortTracker +from trackers.deepsort_tracker import DeepSortTracker + +# YOLOX modules +try: + from yolox.exp import get_exp + from yolox_utils.postprocess import postprocess_yolox + from yolox.utils import fuse_model +except Exception as e: + logger.warning(e) + logger.warning('Load yolox fail. If you want to use yolox, please check the installation.') + pass + +# YOLOv7 modules +try: + sys.path.append(os.getcwd()) + from models.experimental import attempt_load + from utils.torch_utils import select_device, time_synchronized, TracedModel + from utils.general import non_max_suppression, scale_coords, check_img_size + from yolov7_utils.postprocess import postprocess as postprocess_yolov7 + +except Exception as e: + logger.warning(e) + logger.warning('Load yolov7 fail. If you want to use yolov7, please check the installation.') + pass + +# YOLOv8 modules +try: + from ultralytics import YOLO + from yolov8_utils.postprocess import postprocess as postprocess_yolov8 + +except Exception as e: + logger.warning(e) + logger.warning('Load yolov8 fail. If you want to use yolov8, please check the installation.') + pass + +TRACKER_DICT = { + 'sort': SortTracker, + 'bytetrack': ByteTracker, + 'botsort': BotTracker, + 'c_bioutrack': C_BIoUTracker, + 'ocsort': OCSortTracker, + 'deepsort': DeepSortTracker +} + +def get_args(): + + parser = argparse.ArgumentParser() + + """general""" + parser.add_argument('--obj', type=str, required=True, default='demo.mp4', help='video or images folder PATH') + + parser.add_argument('--detector', type=str, default='yolov8', help='yolov7, yolox, etc.') + parser.add_argument('--tracker', type=str, default='sort', help='sort, deepsort, etc') + parser.add_argument('--reid_model', type=str, default='osnet_x0_25', help='osnet or deppsort') + + parser.add_argument('--kalman_format', type=str, default='default', help='use what kind of Kalman, sort, deepsort, byte, etc.') + parser.add_argument('--img_size', type=int, default=1280, help='image size, [h, w]') + + parser.add_argument('--conf_thresh', type=float, default=0.2, help='filter tracks') + parser.add_argument('--nms_thresh', type=float, default=0.7, help='thresh for NMS') + parser.add_argument('--iou_thresh', type=float, default=0.5, help='IOU thresh to filter tracks') + + parser.add_argument('--device', type=str, default='6', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + + """yolox""" + parser.add_argument('--num_classes', type=int, default=1) + parser.add_argument('--yolox_exp_file', type=str, default='./tracker/yolox_utils/yolox_m.py') + + """model path""" + parser.add_argument('--detector_model_path', type=str, default='./weights/best.pt', help='model path') + parser.add_argument('--trace', type=bool, default=False, help='traced model of YOLO v7') + # other model path + parser.add_argument('--reid_model_path', type=str, default='./weights/osnet_x0_25.pth', help='path for reid model path') + parser.add_argument('--dhn_path', type=str, default='./weights/DHN.pth', help='path of DHN path for DeepMOT') + + + """other options""" + parser.add_argument('--discard_reid', action='store_true', help='discard reid model, only work in bot-sort etc. which need a reid part') + parser.add_argument('--track_buffer', type=int, default=30, help='tracking buffer') + parser.add_argument('--gamma', type=float, default=0.1, help='param to control fusing motion and apperance dist') + parser.add_argument('--min_area', type=float, default=150, help='use to filter small bboxs') + + parser.add_argument('--save_dir', type=str, default='track_demo_results') + parser.add_argument('--save_images', action='store_true', help='save tracking results (image)') + parser.add_argument('--save_videos', action='store_true', help='save tracking results (video)') + + parser.add_argument('--track_eval', type=bool, default=True, help='Use TrackEval to evaluate') + + return parser.parse_args() + +def main(args): + + """1. set some params""" + + # NOTE: if save video, you must save image + if args.save_videos: + args.save_images = True + + """2. load detector""" + device = select_device(args.device) + + if args.detector == 'yolox': + + exp = get_exp(args.yolox_exp_file, None) # TODO: modify num_classes etc. for specific dataset + model_img_size = exp.input_size + model = exp.get_model() + model.to(device) + model.eval() + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + ckpt = torch.load(args.detector_model_path, map_location=device) + model.load_state_dict(ckpt['model']) + logger.info("loaded checkpoint done") + model = fuse_model(model) + + stride = None # match with yolo v7 + + logger.info(f'Now detector is on device {next(model.parameters()).device}') + + elif args.detector == 'yolov7': + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + model = attempt_load(args.detector_model_path, map_location=device) + + # get inference img size + stride = int(model.stride.max()) # model stride + model_img_size = check_img_size(args.img_size, s=stride) # check img_size + + # Traced model + model = TracedModel(model, device=device, img_size=args.img_size) + # model.half() + + logger.info("loaded checkpoint done") + + logger.info(f'Now detector is on device {next(model.parameters()).device}') + + elif args.detector == 'yolov8': + + logger.info(f"loading detector {args.detector} checkpoint {args.detector_model_path}") + model = YOLO(args.detector_model_path) + + model_img_size = [None, None] + stride = None + + logger.info("loaded checkpoint done") + + else: + logger.error(f"detector {args.detector} is not supprted") + exit(0) + + """3. load sequences""" + + dataset = DemoDataset(file_name=args.obj, img_size=model_img_size, model=args.detector, stride=stride, ) + data_loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False) + + tracker = TRACKER_DICT[args.tracker](args, ) + + + save_dir = args.save_dir + + process_bar = enumerate(data_loader) + process_bar = tqdm(process_bar, total=len(data_loader), ncols=150) + + results = [] + + """4. Tracking""" + + for frame_idx, (ori_img, img) in process_bar: + if args.detector == 'yolov8': + img = img.squeeze(0).cpu().numpy() + + else: + img = img.to(device) # (1, C, H, W) + img = img.float() + + ori_img = ori_img.squeeze(0) + + # get detector output + with torch.no_grad(): + if args.detector == 'yolov8': + output = model.predict(img, conf=args.conf_thresh, iou=args.nms_thresh) + else: + output = model(img) + + # postprocess output to original scales + if args.detector == 'yolox': + output = postprocess_yolox(output, args.num_classes, conf_thresh=args.conf_thresh, + img=img, ori_img=ori_img) + + elif args.detector == 'yolov7': + output = postprocess_yolov7(output, args.conf_thresh, args.nms_thresh, img.shape[2:], ori_img.shape) + + elif args.detector == 'yolov8': + output = postprocess_yolov8(output) + + else: raise NotImplementedError + + # output: (tlbr, conf, cls) + # convert tlbr to tlwh + if isinstance(output, torch.Tensor): + output = output.detach().cpu().numpy() + output[:, 2] -= output[:, 0] + output[:, 3] -= output[:, 1] + current_tracks = tracker.update(output, img, ori_img.cpu().numpy()) + + # save results + cur_tlwh, cur_id, cur_cls, cur_score = [], [], [], [] + for trk in current_tracks: + bbox = trk.tlwh + id = trk.track_id + cls = trk.category + score = trk.score + + # filter low area bbox + if bbox[2] * bbox[3] > args.min_area: + cur_tlwh.append(bbox) + cur_id.append(id) + cur_cls.append(cls) + cur_score.append(score) + # results.append((frame_id + 1, id, bbox, cls)) + + results.append((frame_idx + 1, cur_id, cur_tlwh, cur_cls, cur_score)) + + if args.save_images: + plot_img(img=ori_img, frame_id=frame_idx, results=[cur_tlwh, cur_id, cur_cls], + save_dir=os.path.join(save_dir, 'vis_results')) + + save_results(folder_name=os.path.join(save_dir, 'txt_results'), + seq_name='demo', + results=results) + + if args.save_videos: + save_video(images_path=os.path.join(save_dir, 'vis_results')) + logger.info(f'save video done') + +if __name__ == '__main__': + + args = get_args() + + main(args) diff --git a/test/yolov7-tracker/tracker/tracker_dataloader.py b/test/yolov7-tracker/tracker/tracker_dataloader.py new file mode 100644 index 0000000..523b140 --- /dev/null +++ b/test/yolov7-tracker/tracker/tracker_dataloader.py @@ -0,0 +1,223 @@ +import numpy as np +import torch +import cv2 +import os +import os.path as osp + +from torch.utils.data import Dataset + + +class TestDataset(Dataset): + """ This class generate origin image, preprocessed image for inference + NOTE: for every sequence, initialize a TestDataset class + + """ + + def __init__(self, data_root, split, seq_name, img_size=[640, 640], legacy_yolox=True, model='yolox', **kwargs) -> None: + """ + Args: + data_root: path for entire dataset + seq_name: name of sequence + img_size: List[int, int] | Tuple[int, int] image size for detection model + legacy_yolox: bool, to be compatible with older versions of yolox + model: detection model, currently support x, v7, v8 + """ + super().__init__() + + self.model = model + + self.data_root = data_root + self.seq_name = seq_name + self.img_size = img_size + self.split = split + + self.seq_path = osp.join(self.data_root, 'images', self.split, self.seq_name) + self.imgs_in_seq = sorted(os.listdir(self.seq_path)) + + self.legacy = legacy_yolox + + self.other_param = kwargs + + def __getitem__(self, idx): + + if self.model == 'yolox': + return self._getitem_yolox(idx) + elif self.model == 'yolov7': + return self._getitem_yolov7(idx) + elif self.model == 'yolov8': + return self._getitem_yolov8(idx) + + def _getitem_yolox(self, idx): + + img = cv2.imread(osp.join(self.seq_path, self.imgs_in_seq[idx])) + img_resized, _ = self._preprocess_yolox(img, self.img_size, ) + if self.legacy: + img_resized = img_resized[::-1, :, :].copy() # BGR -> RGB + img_resized /= 255.0 + img_resized -= np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1) + img_resized /= np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1) + + return torch.from_numpy(img), torch.from_numpy(img_resized) + + def _getitem_yolov7(self, idx): + + img = cv2.imread(osp.join(self.seq_path, self.imgs_in_seq[idx])) + + img_resized = self._preprocess_yolov7(img, ) # torch.Tensor + + return torch.from_numpy(img), img_resized + + def _getitem_yolov8(self, idx): + + img = cv2.imread(osp.join(self.seq_path, self.imgs_in_seq[idx])) # (h, w, c) + # img = self._preprocess_yolov8(img) + + return torch.from_numpy(img), torch.from_numpy(img) + + + def _preprocess_yolox(self, img, size, swap=(2, 0, 1)): + """ convert origin image to resized image, YOLOX-manner + + Args: + img: np.ndarray + size: List[int, int] | Tuple[int, int] + swap: (H, W, C) -> (C, H, W) + + Returns: + np.ndarray, float + + """ + if len(img.shape) == 3: + padded_img = np.ones((size[0], size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(size, dtype=np.uint8) * 114 + + r = min(size[0] / img.shape[0], size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + def _preprocess_yolov7(self, img, ): + + img_resized = self._letterbox(img, new_shape=self.img_size, stride=self.other_param['stride'], )[0] + img_resized = img_resized[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB + img_resized = np.ascontiguousarray(img_resized) + + img_resized = torch.from_numpy(img_resized).float() + img_resized /= 255.0 + + return img_resized + + def _preprocess_yolov8(self, img, ): + + img = img.transpose((2, 0, 1)) + img = np.ascontiguousarray(img) + + return img + + + def _letterbox(self, img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32): + # Resize and pad image while meeting stride-multiple constraints + shape = img.shape[:2] # current shape [height, width] + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + if not scaleup: # only scale down, do not scale up (for better test mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding + if auto: # minimum rectangle + dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding + elif scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios + + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) + top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) + img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border + return img, ratio, (dw, dh) + + def __len__(self, ): + return len(self.imgs_in_seq) + + +class DemoDataset(TestDataset): + """ + dataset for demo + """ + def __init__(self, file_name, img_size=[640, 640], model='yolox', legacy_yolox=True, **kwargs) -> None: + + self.file_name = file_name + self.model = model + self.img_size = img_size + + self.is_video = '.mp4' in file_name or '.avi' in file_name + + if not self.is_video: + self.imgs_in_seq = sorted(os.listdir(file_name)) + else: + self.imgs_in_seq = [] + self.cap = cv2.VideoCapture(file_name) + + while True: + ret, frame = self.cap.read() + if not ret: break + + self.imgs_in_seq.append(frame) + + self.legacy = legacy_yolox + + def __getitem__(self, idx): + + if not self.is_video: + img = cv2.imread(osp.join(self.file_name, self.imgs_in_seq[idx])) + else: + img = self.imgs_in_seq[idx] + + if self.model == 'yolox': + return self._getitem_yolox(img) + elif self.model == 'yolov7': + return self._getitem_yolov7(img) + elif self.model == 'yolov8': + return self._getitem_yolov8(img) + + def _getitem_yolox(self, img): + + img_resized, _ = self._preprocess_yolox(img, self.img_size, ) + if self.legacy: + img_resized = img_resized[::-1, :, :].copy() # BGR -> RGB + img_resized /= 255.0 + img_resized -= np.array([0.485, 0.456, 0.406]).reshape(3, 1, 1) + img_resized /= np.array([0.229, 0.224, 0.225]).reshape(3, 1, 1) + + return torch.from_numpy(img), torch.from_numpy(img_resized) + + def _getitem_yolov7(self, img): + + img_resized = self._preprocess_yolov7(img, ) # torch.Tensor + + return torch.from_numpy(img), img_resized + + def _getitem_yolov8(self, img): + + # img = self._preprocess_yolov8(img) + + return torch.from_numpy(img), torch.from_numpy(img) \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/basetrack.py b/test/yolov7-tracker/tracker/trackers/basetrack.py new file mode 100644 index 0000000..23afa9c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/basetrack.py @@ -0,0 +1,133 @@ +import numpy as np +from collections import OrderedDict + + +class TrackState(object): + New = 0 + Tracked = 1 + Lost = 2 + Removed = 3 + + +class BaseTrack(object): + _count = 0 + + track_id = 0 + is_activated = False + state = TrackState.New + + history = OrderedDict() + features = [] + curr_feature = None + score = 0 + start_frame = 0 + frame_id = 0 + time_since_update = 0 + + # multi-camera + location = (np.inf, np.inf) + + @property + def end_frame(self): + return self.frame_id + + @staticmethod + def next_id(): + BaseTrack._count += 1 + return BaseTrack._count + + def activate(self, *args): + raise NotImplementedError + + def predict(self): + raise NotImplementedError + + def update(self, *args, **kwargs): + raise NotImplementedError + + def mark_lost(self): + self.state = TrackState.Lost + + def mark_removed(self): + self.state = TrackState.Removed + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[:2] -= ret[2:] / 2 + return ret + + @property + def tlbr(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + @property + def xywh(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[:2] += ret[2:] / 2.0 + return ret + + @staticmethod + # @jit(nopython=True) + def tlwh_to_xyah(tlwh): + """Convert bounding box to format `(center x, center y, aspect ratio, + height)`, where the aspect ratio is `width / height`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + return ret + + @staticmethod + def tlwh_to_xywh(tlwh): + """Convert bounding box to format `(center x, center y, width, + height)`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + return ret + + @staticmethod + def tlwh_to_xysa(tlwh): + """Convert bounding box to format `(center x, center y, width, + height)`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] = tlwh[2] * tlwh[3] + ret[3] = tlwh[2] / tlwh[3] + return ret + + def to_xyah(self): + return self.tlwh_to_xyah(self.tlwh) + + def to_xywh(self): + return self.tlwh_to_xywh(self.tlwh) + + @staticmethod + def tlbr_to_tlwh(tlbr): + ret = np.asarray(tlbr).copy() + ret[2:] -= ret[:2] + return ret + + @staticmethod + # @jit(nopython=True) + def tlwh_to_tlbr(tlwh): + ret = np.asarray(tlwh).copy() + ret[2:] += ret[:2] + return ret + + def __repr__(self): + return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame) diff --git a/test/yolov7-tracker/tracker/trackers/botsort_tracker.py b/test/yolov7-tracker/tracker/trackers/botsort_tracker.py new file mode 100644 index 0000000..30204e5 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/botsort_tracker.py @@ -0,0 +1,329 @@ +""" +Bot sort +""" + +import numpy as np +import torch +from torchvision.ops import nms + +import cv2 +import torchvision.transforms as T + +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_reid +from .matching import * + +from .reid_models.OSNet import * +from .reid_models.load_model_tools import load_pretrained_weights +from .reid_models.deepsort_reid import Extractor + +from .camera_motion_compensation import GMC + +REID_MODEL_DICT = { + 'osnet_x1_0': osnet_x1_0, + 'osnet_x0_75': osnet_x0_75, + 'osnet_x0_5': osnet_x0_5, + 'osnet_x0_25': osnet_x0_25, + 'deepsort': Extractor +} + + +def load_reid_model(reid_model, reid_model_path): + + if 'osnet' in reid_model: + func = REID_MODEL_DICT[reid_model] + model = func(num_classes=1, pretrained=False, ) + load_pretrained_weights(model, reid_model_path) + model.cuda().eval() + + elif 'deepsort' in reid_model: + model = REID_MODEL_DICT[reid_model](reid_model_path, use_cuda=True) + + else: + raise NotImplementedError + + return model + +class BotTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + self.with_reid = not args.discard_reid + + self.reid_model, self.crop_transforms = None, None + if self.with_reid: + self.reid_model = load_reid_model(args.reid_model, args.reid_model_path) + self.crop_transforms = T.Compose([ + # T.ToPILImage(), + # T.Resize(size=(256, 128)), + T.ToTensor(), # (c, 128, 256) + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + + # camera motion compensation module + self.gmc = GMC(method='orb', downscale=2, verbose=None) + + def reid_preprocess(self, obj_bbox): + """ + preprocess cropped object bboxes + + obj_bbox: np.ndarray, shape=(h_obj, w_obj, c) + + return: + torch.Tensor of shape (c, 128, 256) + """ + obj_bbox = cv2.resize(obj_bbox.astype(np.float32) / 255.0, dsize=(128, 128)) # shape: (128, 256, c) + + return self.crop_transforms(obj_bbox) + + def get_feature(self, tlwhs, ori_img): + """ + get apperance feature of an object + tlwhs: shape (num_of_objects, 4) + ori_img: original image, np.ndarray, shape(H, W, C) + """ + obj_bbox = [] + + for tlwh in tlwhs: + tlwh = list(map(int, tlwh)) + # if any(tlbr_ == -1 for tlbr_ in tlwh): + # print(tlwh) + + tlbr_tensor = self.reid_preprocess(ori_img[tlwh[1]: tlwh[1] + tlwh[3], tlwh[0]: tlwh[0] + tlwh[2]]) + obj_bbox.append(tlbr_tensor) + + if not obj_bbox: + return np.array([]) + + obj_bbox = torch.stack(obj_bbox, dim=0) + obj_bbox = obj_bbox.cuda() + + features = self.reid_model(obj_bbox) # shape: (num_of_objects, feature_dim) + return features.cpu().detach().numpy() + + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlwh format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + inds_low = scores > 0.1 + inds_high = scores < self.args.conf_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + cates_second = categories[inds_second] + + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + + """Step 1: Extract reid features""" + if self.with_reid: + features_keep = self.get_feature(tlwhs=dets[:, :4], ori_img=ori_img) + + if len(dets) > 0: + if self.with_reid: + detections = [Tracklet_w_reid(tlwh, s, cate, motion=self.motion, feat=feat) for + (tlwh, s, cate, feat) in zip(dets, scores_keep, cates, features_keep)] + else: + detections = [Tracklet(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with high score detection boxes''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + # Camera motion compensation + warp = self.gmc.apply(ori_img, dets) + self.gmc.multi_gmc(tracklet_pool, warp) + self.gmc.multi_gmc(unconfirmed, warp) + + ious_dists = iou_distance(tracklet_pool, detections) + ious_dists_mask = (ious_dists > 0.5) # high conf iou + + if self.with_reid: + # mixed cost matrix + emb_dists = embedding_distance(tracklet_pool, detections) / 2.0 + raw_emb_dists = emb_dists.copy() + emb_dists[emb_dists > 0.25] = 1.0 + emb_dists[ious_dists_mask] = 1.0 + dists = np.minimum(ious_dists, emb_dists) + + else: + dists = ious_dists + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + ''' Step 3: Second association, with low score detection boxes''' + # association the untrack to the low score detections + if len(dets_second) > 0: + '''Detections''' + detections_second = [Tracklet(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets_second, scores_second, cates_second)] + else: + detections_second = [] + + r_tracked_tracklets = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + dists = iou_distance(r_tracked_tracklets, detections_second) + matches, u_track, u_detection_second = linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_tracklets[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = r_tracked_tracklets[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + ious_dists = iou_distance(unconfirmed, detections) + ious_dists_mask = (ious_dists > 0.5) + + if self.with_reid: + emb_dists = embedding_distance(unconfirmed, detections) / 2.0 + raw_emb_dists = emb_dists.copy() + emb_dists[emb_dists > 0.25] = 1.0 + emb_dists[ious_dists_mask] = 1.0 + dists = np.minimum(ious_dists, emb_dists) + else: + dists = ious_dists + + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/byte_tracker.py b/test/yolov7-tracker/tracker/trackers/byte_tracker.py new file mode 100644 index 0000000..c820bd4 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/byte_tracker.py @@ -0,0 +1,201 @@ +""" +ByteTrack +""" + +import numpy as np +from collections import deque +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet +from .matching import * + +class ByteTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + inds_low = scores > 0.1 + inds_high = scores < self.args.conf_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + cates_second = categories[inds_second] + + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with high score detection boxes''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + dists = iou_distance(tracklet_pool, detections) + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + ''' Step 3: Second association, with low score detection boxes''' + # association the untrack to the low score detections + if len(dets_second) > 0: + '''Detections''' + detections_second = [Tracklet(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets_second, scores_second, cates_second)] + else: + detections_second = [] + r_tracked_tracklets = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + dists = iou_distance(r_tracked_tracklets, detections_second) + matches, u_track, u_detection_second = linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_tracklets[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = r_tracked_tracklets[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/c_biou_tracker.py b/test/yolov7-tracker/tracker/trackers/c_biou_tracker.py new file mode 100644 index 0000000..e0f4b77 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/c_biou_tracker.py @@ -0,0 +1,204 @@ +""" +C_BIoU Track +""" + +import numpy as np +from collections import deque +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_bbox_buffer +from .matching import * + +class C_BIoUTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + inds_low = scores > 0.1 + inds_high = scores < self.args.conf_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + cates_second = categories[inds_second] + + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet_w_bbox_buffer(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with high score detection boxes''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + dists = buffered_iou_distance(tracklet_pool, detections, level=1) + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + ''' Step 3: Second association, with low score detection boxes''' + # association the untrack to the low score detections + if len(dets_second) > 0: + '''Detections''' + detections_second = [Tracklet_w_bbox_buffer(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets_second, scores_second, cates_second)] + else: + detections_second = [] + r_tracked_tracklets = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + + + dists = buffered_iou_distance(r_tracked_tracklets, detections_second, level=2) + + matches, u_track, u_detection_second = linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_tracklets[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = r_tracked_tracklets[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = buffered_iou_distance(unconfirmed, detections, level=1) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/camera_motion_compensation.py b/test/yolov7-tracker/tracker/trackers/camera_motion_compensation.py new file mode 100644 index 0000000..69c97b2 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/camera_motion_compensation.py @@ -0,0 +1,264 @@ +import cv2 +import numpy as np +import copy +import matplotlib.pyplot as plt + +"""GMC Module""" +class GMC: + def __init__(self, method='orb', downscale=2, verbose=None): + super(GMC, self).__init__() + + self.method = method + self.downscale = max(1, int(downscale)) + + if self.method == 'orb': + self.detector = cv2.FastFeatureDetector_create(20) + self.extractor = cv2.ORB_create() + self.matcher = cv2.BFMatcher(cv2.NORM_HAMMING) + + elif self.method == 'sift': + self.detector = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.extractor = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.matcher = cv2.BFMatcher(cv2.NORM_L2) + + elif self.method == 'ecc': + number_of_iterations = 100 + termination_eps = 1e-5 + self.warp_mode = cv2.MOTION_EUCLIDEAN + self.criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, number_of_iterations, termination_eps) + + elif self.method == 'file' or self.method == 'files': + seqName = verbose[0] + ablation = verbose[1] + if ablation: + filePath = r'tracker/GMC_files/MOT17_ablation' + else: + filePath = r'tracker/GMC_files/MOTChallenge' + + if '-FRCNN' in seqName: + seqName = seqName[:-6] + elif '-DPM' in seqName: + seqName = seqName[:-4] + elif '-SDP' in seqName: + seqName = seqName[:-4] + + self.gmcFile = open(filePath + "/GMC-" + seqName + ".txt", 'r') + + if self.gmcFile is None: + raise ValueError("Error: Unable to open GMC file in directory:" + filePath) + elif self.method == 'none' or self.method == 'None': + self.method = 'none' + else: + raise ValueError("Error: Unknown CMC method:" + method) + + self.prevFrame = None + self.prevKeyPoints = None + self.prevDescriptors = None + + self.initializedFirstFrame = False + + def apply(self, raw_frame, detections=None): + if self.method == 'orb' or self.method == 'sift': + return self.applyFeaures(raw_frame, detections) + elif self.method == 'ecc': + return self.applyEcc(raw_frame, detections) + elif self.method == 'file': + return self.applyFile(raw_frame, detections) + elif self.method == 'none': + return np.eye(2, 3) + else: + return np.eye(2, 3) + + def applyEcc(self, raw_frame, detections=None): + + # Initialize + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3, dtype=np.float32) + + # Downscale image (TODO: consider using pyramids) + if self.downscale > 1.0: + frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Run the ECC algorithm. The results are stored in warp_matrix. + # (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria) + try: + (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria, None, 1) + except: + print('Warning: find transform failed. Set warp as identity') + + return H + + def applyFeaures(self, raw_frame, detections=None): + + # Initialize + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3) + + # Downscale image (TODO: consider using pyramids) + if self.downscale > 1.0: + # frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # find the keypoints + mask = np.zeros_like(frame) + # mask[int(0.05 * height): int(0.95 * height), int(0.05 * width): int(0.95 * width)] = 255 + mask[int(0.02 * height): int(0.98 * height), int(0.02 * width): int(0.98 * width)] = 255 + if detections is not None: + for det in detections: + tlbr = (det[:4] / self.downscale).astype(np.int_) + mask[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2]] = 0 + + keypoints = self.detector.detect(frame, mask) + + # compute the descriptors + keypoints, descriptors = self.extractor.compute(frame, keypoints) + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Match descriptors. + knnMatches = self.matcher.knnMatch(self.prevDescriptors, descriptors, 2) + + # Filtered matches based on smallest spatial distance + matches = [] + spatialDistances = [] + + maxSpatialDistance = 0.25 * np.array([width, height]) + + # Handle empty matches case + if len(knnMatches) == 0: + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + for m, n in knnMatches: + if m.distance < 0.9 * n.distance: + prevKeyPointLocation = self.prevKeyPoints[m.queryIdx].pt + currKeyPointLocation = keypoints[m.trainIdx].pt + + spatialDistance = (prevKeyPointLocation[0] - currKeyPointLocation[0], + prevKeyPointLocation[1] - currKeyPointLocation[1]) + + if (np.abs(spatialDistance[0]) < maxSpatialDistance[0]) and \ + (np.abs(spatialDistance[1]) < maxSpatialDistance[1]): + spatialDistances.append(spatialDistance) + matches.append(m) + + meanSpatialDistances = np.mean(spatialDistances, 0) + stdSpatialDistances = np.std(spatialDistances, 0) + + inliesrs = (spatialDistances - meanSpatialDistances) < 2.5 * stdSpatialDistances + + goodMatches = [] + prevPoints = [] + currPoints = [] + for i in range(len(matches)): + if inliesrs[i, 0] and inliesrs[i, 1]: + goodMatches.append(matches[i]) + prevPoints.append(self.prevKeyPoints[matches[i].queryIdx].pt) + currPoints.append(keypoints[matches[i].trainIdx].pt) + + prevPoints = np.array(prevPoints) + currPoints = np.array(currPoints) + + # Draw the keypoint matches on the output image + if 0: + matches_img = np.hstack((self.prevFrame, frame)) + matches_img = cv2.cvtColor(matches_img, cv2.COLOR_GRAY2BGR) + W = np.size(self.prevFrame, 1) + for m in goodMatches: + prev_pt = np.array(self.prevKeyPoints[m.queryIdx].pt, dtype=np.int_) + curr_pt = np.array(keypoints[m.trainIdx].pt, dtype=np.int_) + curr_pt[0] += W + color = np.random.randint(0, 255, (3,)) + color = (int(color[0]), int(color[1]), int(color[2])) + + matches_img = cv2.line(matches_img, prev_pt, curr_pt, tuple(color), 1, cv2.LINE_AA) + matches_img = cv2.circle(matches_img, prev_pt, 2, tuple(color), -1) + matches_img = cv2.circle(matches_img, curr_pt, 2, tuple(color), -1) + + plt.figure() + plt.imshow(matches_img) + plt.show() + + # Find rigid matrix + if (np.size(prevPoints, 0) > 4) and (np.size(prevPoints, 0) == np.size(prevPoints, 0)): + H, inliesrs = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) + + # Handle downscale + if self.downscale > 1.0: + H[0, 2] *= self.downscale + H[1, 2] *= self.downscale + else: + print('Warning: not enough matching points') + + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + def applyFile(self, raw_frame, detections=None): + line = self.gmcFile.readline() + tokens = line.split("\t") + H = np.eye(2, 3, dtype=np.float_) + H[0, 0] = float(tokens[1]) + H[0, 1] = float(tokens[2]) + H[0, 2] = float(tokens[3]) + H[1, 0] = float(tokens[4]) + H[1, 1] = float(tokens[5]) + H[1, 2] = float(tokens[6]) + + return H + + @staticmethod + def multi_gmc(stracks, H=np.eye(2, 3)): + """ + GMC module prediction + :param stracks: List[Strack] + """ + if len(stracks) > 0: + multi_mean = np.asarray([st.kalman_filter.kf.x.copy() for st in stracks]) + multi_covariance = np.asarray([st.kalman_filter.kf.P for st in stracks]) + + R = H[:2, :2] + R8x8 = np.kron(np.eye(4, dtype=float), R) + t = H[:2, 2] + + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + mean = R8x8.dot(mean) + mean[:2] += t + cov = R8x8.dot(cov).dot(R8x8.transpose()) + + stracks[i].kalman_filter.kf.x = mean + stracks[i].kalman_filter.kf.P = cov \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/deepsort_tracker.py b/test/yolov7-tracker/tracker/trackers/deepsort_tracker.py new file mode 100644 index 0000000..e81b145 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/deepsort_tracker.py @@ -0,0 +1,327 @@ +""" +Deep Sort +""" + +import numpy as np +import torch +from torchvision.ops import nms + +import cv2 +import torchvision.transforms as T + +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_reid +from .matching import * + +from .reid_models.OSNet import * +from .reid_models.load_model_tools import load_pretrained_weights +from .reid_models.deepsort_reid import Extractor + +REID_MODEL_DICT = { + 'osnet_x1_0': osnet_x1_0, + 'osnet_x0_75': osnet_x0_75, + 'osnet_x0_5': osnet_x0_5, + 'osnet_x0_25': osnet_x0_25, + 'deepsort': Extractor +} + + +def load_reid_model(reid_model, reid_model_path): + + if 'osnet' in reid_model: + func = REID_MODEL_DICT[reid_model] + model = func(num_classes=1, pretrained=False, ) + load_pretrained_weights(model, reid_model_path) + model.cuda().eval() + + elif 'deepsort' in reid_model: + model = REID_MODEL_DICT[reid_model](reid_model_path, use_cuda=True) + + else: + raise NotImplementedError + + return model + + +class DeepSortTracker(object): + + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + self.with_reid = not args.discard_reid + + self.reid_model, self.crop_transforms = None, None + if self.with_reid: + self.reid_model = load_reid_model(args.reid_model, args.reid_model_path) + self.crop_transforms = T.Compose([ + # T.ToPILImage(), + # T.Resize(size=(256, 128)), + T.ToTensor(), # (c, 128, 256) + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + self.bbox_crop_size = (64, 128) if 'deepsort' in args.reid_model else (128, 128) + + + def reid_preprocess(self, obj_bbox): + """ + preprocess cropped object bboxes + + obj_bbox: np.ndarray, shape=(h_obj, w_obj, c) + + return: + torch.Tensor of shape (c, 128, 256) + """ + + obj_bbox = cv2.resize(obj_bbox.astype(np.float32) / 255.0, dsize=self.bbox_crop_size) # shape: (h, w, c) + + return self.crop_transforms(obj_bbox) + + def get_feature(self, tlwhs, ori_img): + """ + get apperance feature of an object + tlwhs: shape (num_of_objects, 4) + ori_img: original image, np.ndarray, shape(H, W, C) + """ + obj_bbox = [] + + for tlwh in tlwhs: + tlwh = list(map(int, tlwh)) + + # limit to the legal range + tlwh[0], tlwh[1] = max(tlwh[0], 0), max(tlwh[1], 0) + + tlbr_tensor = self.reid_preprocess(ori_img[tlwh[1]: tlwh[1] + tlwh[3], tlwh[0]: tlwh[0] + tlwh[2]]) + + obj_bbox.append(tlbr_tensor) + + if not obj_bbox: + return np.array([]) + + obj_bbox = torch.stack(obj_bbox, dim=0) + obj_bbox = obj_bbox.cuda() + + features = self.reid_model(obj_bbox) # shape: (num_of_objects, feature_dim) + return features.cpu().detach().numpy() + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + + scores_keep = scores[remain_inds] + + features_keep = self.get_feature(tlwhs=dets[:, :4], ori_img=ori_img) + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet_w_reid(tlwh, s, cate, motion=self.motion, feat=feat) for + (tlwh, s, cate, feat) in zip(dets, scores_keep, cates, features_keep)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with appearance''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + + matches, u_track, u_detection = matching_cascade(distance_metric=self.gated_metric, + matching_thresh=0.9, + cascade_depth=30, + tracks=tracklet_pool, + detections=detections + ) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + '''Step 3: Second association, with iou''' + tracklet_for_iou = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + detection_for_iou = [detections[i] for i in u_detection] + + dists = iou_distance(tracklet_for_iou, detection_for_iou) + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.5) + + for itracked, idet in matches: + track = tracklet_for_iou[itracked] + det = detection_for_iou[idet] + if track.state == TrackState.Tracked: + track.update(detection_for_iou[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = tracklet_for_iou[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detection_for_iou[i] for i in u_detection] + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + def gated_metric(self, tracks, dets): + """ + get cost matrix, firstly calculate apperence cost, then filter by Kalman state. + + tracks: List[STrack] + dets: List[STrack] + """ + apperance_dist = nearest_embedding_distance(tracks=tracks, detections=dets, metric='cosine') + cost_matrix = self.gate_cost_matrix(apperance_dist, tracks, dets, ) + return cost_matrix + + def gate_cost_matrix(self, cost_matrix, tracks, dets, max_apperance_thresh=0.15, gated_cost=1e5, only_position=False): + """ + gate cost matrix by calculating the Kalman state distance and constrainted by + 0.95 confidence interval of x2 distribution + + cost_matrix: np.ndarray, shape (len(tracks), len(dets)) + tracks: List[STrack] + dets: List[STrack] + gated_cost: a very largt const to infeasible associations + only_position: use [xc, yc, a, h] as state vector or only use [xc, yc] + + return: + updated cost_matirx, np.ndarray + """ + gating_dim = 2 if only_position else 4 + gating_threshold = chi2inv95[gating_dim] + measurements = np.asarray([Tracklet.tlwh_to_xyah(det.tlwh) for det in dets]) # (len(dets), 4) + + cost_matrix[cost_matrix > max_apperance_thresh] = gated_cost + for row, track in enumerate(tracks): + gating_distance = track.kalman_filter.gating_distance(measurements, ) + cost_matrix[row, gating_distance > gating_threshold] = gated_cost + return cost_matrix + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/base_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/base_kalman.py new file mode 100644 index 0000000..5257e84 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/base_kalman.py @@ -0,0 +1,74 @@ +from filterpy.kalman import KalmanFilter +import numpy as np +import scipy + +class BaseKalman: + + def __init__(self, + state_dim: int = 8, + observation_dim: int = 4, + F: np.ndarray = np.zeros((0, )), + P: np.ndarray = np.zeros((0, )), + Q: np.ndarray = np.zeros((0, )), + H: np.ndarray = np.zeros((0, )), + R: np.ndarray = np.zeros((0, )), + ) -> None: + + self.kf = KalmanFilter(dim_x=state_dim, dim_z=observation_dim, dim_u=0) + if F.shape[0] > 0: self.kf.F = F # if valid + if P.shape[0] > 0: self.kf.P = P + if Q.shape[0] > 0: self.kf.Q = Q + if H.shape[0] > 0: self.kf.H = H + if R.shape[0] > 0: self.kf.R = R + + def initialize(self, observation): + return NotImplementedError + + def predict(self, ): + self.kf.predict() + + def update(self, observation, **kwargs): + self.kf.update(observation, self.R, self.H) + + def get_state(self, ): + return self.kf.x + + def gating_distance(self, measurements, only_position=False): + """Compute gating distance between state distribution and measurements. + A suitable distance threshold can be obtained from `chi2inv95`. If + `only_position` is False, the chi-square distribution has 4 degrees of + freedom, otherwise 2. + Parameters + ---------- + measurements : ndarray + An Nx4 dimensional matrix of N measurements, note the format (whether xywh or xyah or others) + should be identical to state definition + only_position : Optional[bool] + If True, distance computation is done with respect to the bounding + box center position only. + Returns + ------- + ndarray + Returns an array of length N, where the i-th element contains the + squared Mahalanobis distance between (mean, covariance) and + `measurements[i]`. + """ + + # map state space to measurement space + mean = self.kf.x.copy() + mean = np.dot(self.kf.H, mean) + covariance = np.linalg.multi_dot((self.kf.H, self.kf.P, self.kf.H.T)) + + if only_position: + mean, covariance = mean[:2], covariance[:2, :2] + measurements = measurements[:, :2] + + cholesky_factor = np.linalg.cholesky(covariance) + d = measurements - mean + z = scipy.linalg.solve_triangular( + cholesky_factor, d.T, lower=True, check_finite=False, + overwrite_b=True) + squared_maha = np.sum(z * z, axis=0) + return squared_maha + + \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/botsort_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/botsort_kalman.py new file mode 100644 index 0000000..f9fdfe8 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/botsort_kalman.py @@ -0,0 +1,99 @@ +from numpy.core.multiarray import zeros as zeros +from .base_kalman import BaseKalman +import numpy as np +import cv2 + +class BotKalman(BaseKalman): + + def __init__(self, ): + + state_dim = 8 # [x, y, w, h, vx, vy, vw, vh] + observation_dim = 4 + + F = np.eye(state_dim, state_dim) + ''' + [1, 0, 0, 0, 1, 0, 0] + [0, 1, 0, 0, 0, 1, 0] + ... + ''' + for i in range(state_dim // 2): + F[i, i + state_dim // 2] = 1 + + H = np.eye(state_dim // 2, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 + + def initialize(self, observation): + """ init x, P, Q, R + + Args: + observation: x-y-w-h format + """ + # init x, P, Q, R + + mean_pos = observation + mean_vel = np.zeros_like(observation) + self.kf.x = np.r_[mean_pos, mean_vel] # x_{0, 0} + + std = [ + 2 * self._std_weight_position * observation[2], # related to h + 2 * self._std_weight_position * observation[3], + 2 * self._std_weight_position * observation[2], + 2 * self._std_weight_position * observation[3], + 10 * self._std_weight_velocity * observation[2], + 10 * self._std_weight_velocity * observation[3], + 10 * self._std_weight_velocity * observation[2], + 10 * self._std_weight_velocity * observation[3], + ] + + self.kf.P = np.diag(np.square(std)) # P_{0, 0} + + def predict(self, ): + """ predict step + + x_{n + 1, n} = F * x_{n, n} + P_{n + 1, n} = F * P_{n, n} * F^T + Q + + """ + std_pos = [ + self._std_weight_position * self.kf.x[2], + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[2], + self._std_weight_position * self.kf.x[3]] + std_vel = [ + self._std_weight_velocity * self.kf.x[2], + self._std_weight_velocity * self.kf.x[3], + self._std_weight_velocity * self.kf.x[2], + self._std_weight_velocity * self.kf.x[3]] + + Q = np.diag(np.square(np.r_[std_pos, std_vel])) + + self.kf.predict(Q=Q) + + def update(self, z): + """ update step + + Args: + z: observation x-y-a-h format + + K_n = P_{n, n - 1} * H^T * (H P_{n, n - 1} H^T + R)^{-1} + x_{n, n} = x_{n, n - 1} + K_n * (z - H * x_{n, n - 1}) + P_{n, n} = (I - K_n * H) P_{n, n - 1} (I - K_n * H)^T + K_n R_n + + """ + + std = [ + self._std_weight_position * self.kf.x[2], + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[2], + self._std_weight_position * self.kf.x[3]] + + R = np.diag(np.square(std)) + + self.kf.update(z=z, R=R) diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/bytetrack_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/bytetrack_kalman.py new file mode 100644 index 0000000..e37d3aa --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/bytetrack_kalman.py @@ -0,0 +1,97 @@ +from .base_kalman import BaseKalman +import numpy as np + +class ByteKalman(BaseKalman): + + def __init__(self, ): + + state_dim = 8 # [x, y, a, h, vx, vy, va, vh] + observation_dim = 4 + + F = np.eye(state_dim, state_dim) + ''' + [1, 0, 0, 0, 1, 0, 0] + [0, 1, 0, 0, 0, 1, 0] + ... + ''' + for i in range(state_dim // 2): + F[i, i + state_dim // 2] = 1 + + H = np.eye(state_dim // 2, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 + + def initialize(self, observation): + """ init x, P, Q, R + + Args: + observation: x-y-a-h format + """ + # init x, P, Q, R + + mean_pos = observation + mean_vel = np.zeros_like(observation) + self.kf.x = np.r_[mean_pos, mean_vel] # x_{0, 0} + + std = [ + 2 * self._std_weight_position * observation[3], # related to h + 2 * self._std_weight_position * observation[3], + 1e-2, + 2 * self._std_weight_position * observation[3], + 10 * self._std_weight_velocity * observation[3], + 10 * self._std_weight_velocity * observation[3], + 1e-5, + 10 * self._std_weight_velocity * observation[3], + ] + + self.kf.P = np.diag(np.square(std)) # P_{0, 0} + + def predict(self, ): + """ predict step + + x_{n + 1, n} = F * x_{n, n} + P_{n + 1, n} = F * P_{n, n} * F^T + Q + + """ + std_pos = [ + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[3], + 1e-2, + self._std_weight_position * self.kf.x[3]] + std_vel = [ + self._std_weight_velocity * self.kf.x[3], + self._std_weight_velocity * self.kf.x[3], + 1e-5, + self._std_weight_velocity * self.kf.x[3]] + + Q = np.diag(np.square(np.r_[std_pos, std_vel])) + + self.kf.predict(Q=Q) + + def update(self, z): + """ update step + + Args: + z: observation x-y-a-h format + + K_n = P_{n, n - 1} * H^T * (H P_{n, n - 1} H^T + R)^{-1} + x_{n, n} = x_{n, n - 1} + K_n * (z - H * x_{n, n - 1}) + P_{n, n} = (I - K_n * H) P_{n, n - 1} (I - K_n * H)^T + K_n R_n + + """ + + std = [ + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[3], + 1e-1, + self._std_weight_position * self.kf.x[3]] + + R = np.diag(np.square(std)) + + self.kf.update(z=z, R=R) diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/ocsort_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/ocsort_kalman.py new file mode 100644 index 0000000..a83b5b7 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/ocsort_kalman.py @@ -0,0 +1,144 @@ +from numpy.core.multiarray import zeros as zeros +from .base_kalman import BaseKalman +import numpy as np +from copy import deepcopy + +class OCSORTKalman(BaseKalman): + + def __init__(self, ): + + state_dim = 7 # [x, y, s, a, vx, vy, vs] s: area + observation_dim = 4 + + F = np.array([[1, 0, 0, 0, 1, 0, 0], + [0, 1, 0, 0, 0, 1, 0], + [0, 0, 1, 0, 0, 0, 1], + [0, 0, 0, 1, 0, 0, 0], + [0, 0, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 1]]) + + H = np.eye(state_dim // 2 + 1, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + # TODO check + # give high uncertainty to the unobservable initial velocities + self.kf.R[2:, 2:] *= 10 # [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 10, 0], [0, 0, 0, 10]] + self.kf.P[4:, 4:] *= 1000 + self.kf.P *= 10 + self.kf.Q[-1, -1] *= 0.01 + self.kf.Q[4:, 4:] *= 0.01 + + # keep all observations + self.history_obs = [] + self.attr_saved = None + self.observed = False + + def initialize(self, observation): + """ + Args: + observation: x-y-s-a + """ + self.kf.x = self.kf.x.flatten() + self.kf.x[:4] = observation + + + def predict(self, ): + """ predict step + + """ + + # s + vs + if (self.kf.x[6] + self.kf.x[2] <= 0): + self.kf.x[6] *= 0.0 + + self.kf.predict() + + def _freeze(self, ): + """ freeze all the param of Kalman + + """ + self.attr_saved = deepcopy(self.kf.__dict__) + + def _unfreeze(self, ): + """ when observe an lost object again, use the virtual trajectory + + """ + if self.attr_saved is not None: + new_history = deepcopy(self.history_obs) + self.kf.__dict__ = self.attr_saved + + self.history_obs = self.history_obs[:-1] + + occur = [int(d is None) for d in new_history] + indices = np.where(np.array(occur)==0)[0] + index1 = indices[-2] + index2 = indices[-1] + box1 = new_history[index1] + x1, y1, s1, r1 = box1 + w1 = np.sqrt(s1 * r1) + h1 = np.sqrt(s1 / r1) + box2 = new_history[index2] + x2, y2, s2, r2 = box2 + w2 = np.sqrt(s2 * r2) + h2 = np.sqrt(s2 / r2) + time_gap = index2 - index1 + dx = (x2-x1)/time_gap + dy = (y2-y1)/time_gap + dw = (w2-w1)/time_gap + dh = (h2-h1)/time_gap + for i in range(index2 - index1): + """ + The default virtual trajectory generation is by linear + motion (constant speed hypothesis), you could modify this + part to implement your own. + """ + x = x1 + (i+1) * dx + y = y1 + (i+1) * dy + w = w1 + (i+1) * dw + h = h1 + (i+1) * dh + s = w * h + r = w / float(h) + new_box = np.array([x, y, s, r]).reshape((4, 1)) + """ + I still use predict-update loop here to refresh the parameters, + but this can be faster by directly modifying the internal parameters + as suggested in the paper. I keep this naive but slow way for + easy read and understanding + """ + self.kf.update(new_box) + if not i == (index2-index1-1): + self.kf.predict() + + + def update(self, z): + """ update step + + For simplicity, directly change the self.kf as OCSORT modify the intrinsic Kalman + + Args: + z: observation x-y-s-a format + """ + + self.history_obs.append(z) + + if z is None: + if self.observed: + self._freeze() + self.observed = False + + self.kf.update(z) + + else: + if not self.observed: # Get observation, use online smoothing to re-update parameters + self._unfreeze() + + self.kf.update(z) + + self.observed = True + + diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/sort_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/sort_kalman.py new file mode 100644 index 0000000..c593bfa --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/sort_kalman.py @@ -0,0 +1,73 @@ +from numpy.core.multiarray import zeros as zeros +from .base_kalman import BaseKalman +import numpy as np +from copy import deepcopy + +class SORTKalman(BaseKalman): + + def __init__(self, ): + + state_dim = 7 # [x, y, s, a, vx, vy, vs] s: area + observation_dim = 4 + + F = np.array([[1, 0, 0, 0, 1, 0, 0], + [0, 1, 0, 0, 0, 1, 0], + [0, 0, 1, 0, 0, 0, 1], + [0, 0, 0, 1, 0, 0, 0], + [0, 0, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 1]]) + + H = np.eye(state_dim // 2 + 1, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + # TODO check + # give high uncertainty to the unobservable initial velocities + self.kf.R[2:, 2:] *= 10 # [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 10, 0], [0, 0, 0, 10]] + self.kf.P[4:, 4:] *= 1000 + self.kf.P *= 10 + self.kf.Q[-1, -1] *= 0.01 + self.kf.Q[4:, 4:] *= 0.01 + + # keep all observations + self.history_obs = [] + self.attr_saved = None + self.observed = False + + def initialize(self, observation): + """ + Args: + observation: x-y-s-a + """ + self.kf.x = self.kf.x.flatten() + self.kf.x[:4] = observation + + + def predict(self, ): + """ predict step + + """ + + # s + vs + if (self.kf.x[6] + self.kf.x[2] <= 0): + self.kf.x[6] *= 0.0 + + self.kf.predict() + + def update(self, z): + """ update step + + For simplicity, directly change the self.kf as OCSORT modify the intrinsic Kalman + + Args: + z: observation x-y-s-a format + """ + + self.kf.update(z) + + + diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/strongsort_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/strongsort_kalman.py new file mode 100644 index 0000000..dee8394 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/strongsort_kalman.py @@ -0,0 +1,101 @@ +from .base_kalman import BaseKalman +import numpy as np + +class NSAKalman(BaseKalman): + + def __init__(self, ): + + state_dim = 8 # [x, y, a, h, vx, vy, va, vh] + observation_dim = 4 + + F = np.eye(state_dim, state_dim) + ''' + [1, 0, 0, 0, 1, 0, 0] + [0, 1, 0, 0, 0, 1, 0] + ... + ''' + for i in range(state_dim // 2): + F[i, i + state_dim // 2] = 1 + + H = np.eye(state_dim // 2, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 + + def initialize(self, observation): + """ init x, P, Q, R + + Args: + observation: x-y-a-h format + """ + # init x, P, Q, R + + mean_pos = observation + mean_vel = np.zeros_like(observation) + self.kf.x = np.r_[mean_pos, mean_vel] # x_{0, 0} + + std = [ + 2 * self._std_weight_position * observation[3], # related to h + 2 * self._std_weight_position * observation[3], + 1e-2, + 2 * self._std_weight_position * observation[3], + 10 * self._std_weight_velocity * observation[3], + 10 * self._std_weight_velocity * observation[3], + 1e-5, + 10 * self._std_weight_velocity * observation[3], + ] + + self.kf.P = np.diag(np.square(std)) # P_{0, 0} + + def predict(self, ): + """ predict step + + x_{n + 1, n} = F * x_{n, n} + P_{n + 1, n} = F * P_{n, n} * F^T + Q + + """ + std_pos = [ + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[3], + 1e-2, + self._std_weight_position * self.kf.x[3]] + std_vel = [ + self._std_weight_velocity * self.kf.x[3], + self._std_weight_velocity * self.kf.x[3], + 1e-5, + self._std_weight_velocity * self.kf.x[3]] + + Q = np.diag(np.square(np.r_[std_pos, std_vel])) + + self.kf.predict(Q=Q) + + def update(self, z, score): + """ update step + + Args: + z: observation x-y-a-h format + score: the detection score/confidence required by NSA kalman + + K_n = P_{n, n - 1} * H^T * (H P_{n, n - 1} H^T + R)^{-1} + x_{n, n} = x_{n, n - 1} + K_n * (z - H * x_{n, n - 1}) + P_{n, n} = (I - K_n * H) P_{n, n - 1} (I - K_n * H)^T + K_n R_n + + """ + + std = [ + self._std_weight_position * self.kf.x[3], + self._std_weight_position * self.kf.x[3], + 1e-1, + self._std_weight_position * self.kf.x[3]] + + # NSA + std = [(1. - score) * x for x in std] + + R = np.diag(np.square(std)) + + self.kf.update(z=z, R=R) diff --git a/test/yolov7-tracker/tracker/trackers/kalman_filters/ucmctrack_kalman.py b/test/yolov7-tracker/tracker/trackers/kalman_filters/ucmctrack_kalman.py new file mode 100644 index 0000000..f7a2786 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/kalman_filters/ucmctrack_kalman.py @@ -0,0 +1,27 @@ +from .base_kalman import BaseKalman +import numpy as np + +class UCMCKalman(BaseKalman): + def __init__(self, ): + + state_dim = 8 + observation_dim = 4 + + F = np.eye(state_dim, state_dim) + ''' + [1, 0, 0, 0, 1, 0, 0] + [0, 1, 0, 0, 0, 1, 0] + ... + ''' + for i in range(state_dim // 2): + F[i, i + state_dim // 2] = 1 + + H = np.eye(state_dim // 2, state_dim) + + super().__init__(state_dim=state_dim, + observation_dim=observation_dim, + F=F, + H=H) + + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/matching.py b/test/yolov7-tracker/tracker/trackers/matching.py new file mode 100644 index 0000000..4eb1997 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/matching.py @@ -0,0 +1,388 @@ +import cv2 +import numpy as np +import scipy +import lap +from scipy.spatial.distance import cdist +import math +from cython_bbox import bbox_overlaps as bbox_ious +import time + +chi2inv95 = { + 1: 3.8415, + 2: 5.9915, + 3: 7.8147, + 4: 9.4877, + 5: 11.070, + 6: 12.592, + 7: 14.067, + 8: 15.507, + 9: 16.919} + + +def merge_matches(m1, m2, shape): + O,P,Q = shape + m1 = np.asarray(m1) + m2 = np.asarray(m2) + + M1 = scipy.sparse.coo_matrix((np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P)) + M2 = scipy.sparse.coo_matrix((np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q)) + + mask = M1*M2 + match = mask.nonzero() + match = list(zip(match[0], match[1])) + unmatched_O = tuple(set(range(O)) - set([i for i, j in match])) + unmatched_Q = tuple(set(range(Q)) - set([j for i, j in match])) + + return match, unmatched_O, unmatched_Q + + +def _indices_to_matches(cost_matrix, indices, thresh): + matched_cost = cost_matrix[tuple(zip(*indices))] + matched_mask = (matched_cost <= thresh) + + matches = indices[matched_mask] + unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0])) + unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1])) + + return matches, unmatched_a, unmatched_b + + +def linear_assignment(cost_matrix, thresh): + if cost_matrix.size == 0: + return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1])) + matches, unmatched_a, unmatched_b = [], [], [] + cost, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh) + for ix, mx in enumerate(x): + if mx >= 0: + matches.append([ix, mx]) + unmatched_a = np.where(x < 0)[0] + unmatched_b = np.where(y < 0)[0] + matches = np.asarray(matches) + return matches, unmatched_a, unmatched_b + + +def ious(atlbrs, btlbrs): + """ + Compute cost based on IoU + :type atlbrs: list[tlbr] | np.ndarray + :type atlbrs: list[tlbr] | np.ndarray + + :rtype ious np.ndarray + """ + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + if ious.size == 0: + return ious + + ious = bbox_ious( + np.ascontiguousarray(atlbrs, dtype=np.float), + np.ascontiguousarray(btlbrs, dtype=np.float) + ) + + return ious + + +def iou_distance(atracks, btracks): + """ + Compute cost based on IoU + :type atracks: list[STrack] + :type btracks: list[STrack] + + :rtype cost_matrix np.ndarray + """ + + if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)): + atlbrs = atracks + btlbrs = btracks + else: + atlbrs = [track.tlbr for track in atracks] + btlbrs = [track.tlbr for track in btracks] + _ious = ious(atlbrs, btlbrs) + cost_matrix = 1 - _ious + + return cost_matrix + +def v_iou_distance(atracks, btracks): + """ + Compute cost based on IoU + :type atracks: list[STrack] + :type btracks: list[STrack] + + :rtype cost_matrix np.ndarray + """ + + if (len(atracks)>0 and isinstance(atracks[0], np.ndarray)) or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)): + atlbrs = atracks + btlbrs = btracks + else: + atlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in atracks] + btlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in btracks] + _ious = ious(atlbrs, btlbrs) + cost_matrix = 1 - _ious + + return cost_matrix + +def embedding_distance(tracks, detections, metric='cosine'): + """ + :param tracks: list[STrack] + :param detections: list[BaseTrack] + :param metric: + :return: cost_matrix np.ndarray + """ + + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + if cost_matrix.size == 0: + return cost_matrix + det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + #for i, track in enumerate(tracks): + #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features + return cost_matrix + + +def fuse_motion(kf, cost_matrix, tracks, detections, only_position=False, lambda_=0.98): + if cost_matrix.size == 0: + return cost_matrix + gating_dim = 2 if only_position else 4 + gating_threshold = chi2inv95[gating_dim] + measurements = np.asarray([det.to_xyah() for det in detections]) + for row, track in enumerate(tracks): + gating_distance = kf.gating_distance( + track.mean, track.covariance, measurements, only_position, metric='maha') + cost_matrix[row, gating_distance > gating_threshold] = np.inf + cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_) * gating_distance + return cost_matrix + + +def fuse_iou(cost_matrix, tracks, detections): + if cost_matrix.size == 0: + return cost_matrix + reid_sim = 1 - cost_matrix + iou_dist = iou_distance(tracks, detections) + iou_sim = 1 - iou_dist + fuse_sim = reid_sim * (1 + iou_sim) / 2 + det_scores = np.array([det.score for det in detections]) + det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) + #fuse_sim = fuse_sim * (1 + det_scores) / 2 + fuse_cost = 1 - fuse_sim + return fuse_cost + + +def fuse_score(cost_matrix, detections): + if cost_matrix.size == 0: + return cost_matrix + iou_sim = 1 - cost_matrix + det_scores = np.array([det.score for det in detections]) + det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) + fuse_sim = iou_sim * det_scores + fuse_cost = 1 - fuse_sim + return fuse_cost + + +def greedy_assignment_iou(dist, thresh): + matched_indices = [] + if dist.shape[1] == 0: + return np.array(matched_indices, np.int32).reshape(-1, 2) + for i in range(dist.shape[0]): + j = dist[i].argmin() + if dist[i][j] < thresh: + dist[:, j] = 1. + matched_indices.append([j, i]) + return np.array(matched_indices, np.int32).reshape(-1, 2) + +def greedy_assignment(dists, threshs): + matches = greedy_assignment_iou(dists.T, threshs) + u_det = [d for d in range(dists.shape[1]) if not (d in matches[:, 1])] + u_track = [d for d in range(dists.shape[0]) if not (d in matches[:, 0])] + return matches, u_track, u_det + +def fuse_score_matrix(cost_matrix, detections, tracks): + if cost_matrix.size == 0: + return cost_matrix + iou_sim = 1 - cost_matrix + + det_scores = np.array([det.score for det in detections]) + det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) + trk_scores = np.array([trk.score for trk in tracks]) + trk_scores = np.expand_dims(trk_scores, axis=1).repeat(cost_matrix.shape[1], axis=1) + mid_scores = (det_scores + trk_scores) / 2 + fuse_sim = iou_sim * mid_scores + fuse_cost = 1 - fuse_sim + + return fuse_cost + +""" +calculate buffered IoU, used in C_BIoU_Tracker +""" +def buffered_iou_distance(atracks, btracks, level=1): + """ + atracks: list[C_BIoUSTrack], tracks + btracks: list[C_BIoUSTrack], detections + level: cascade level, 1 or 2 + """ + assert level in [1, 2], 'level must be 1 or 2' + if level == 1: # use motion_state1(tracks) and buffer_bbox1(detections) to calculate + atlbrs = [track.tlwh_to_tlbr(track.motion_state1) for track in atracks] + btlbrs = [det.tlwh_to_tlbr(det.buffer_bbox1) for det in btracks] + else: + atlbrs = [track.tlwh_to_tlbr(track.motion_state2) for track in atracks] + btlbrs = [det.tlwh_to_tlbr(det.buffer_bbox2) for det in btracks] + _ious = ious(atlbrs, btlbrs) + + cost_matrix = 1 - _ious + return cost_matrix + +""" +observation centric association, with velocity, for OC Sort +""" +def observation_centric_association(tracklets, detections, iou_threshold, velocities, previous_obs, vdc_weight): + + if(len(tracklets) == 0): + return np.empty((0, 2), dtype=int), tuple(range(len(tracklets))), tuple(range(len(detections))) + + # get numpy format bboxes + trk_tlbrs = np.array([track.tlbr for track in tracklets]) + det_tlbrs = np.array([det.tlbr for det in detections]) + det_scores = np.array([det.score for det in detections]) + + iou_matrix = bbox_ious(trk_tlbrs, det_tlbrs) + + Y, X = speed_direction_batch(det_tlbrs, previous_obs) + inertia_Y, inertia_X = velocities[:,0], velocities[:,1] + inertia_Y = np.repeat(inertia_Y[:, np.newaxis], Y.shape[1], axis=1) + inertia_X = np.repeat(inertia_X[:, np.newaxis], X.shape[1], axis=1) + diff_angle_cos = inertia_X * X + inertia_Y * Y + diff_angle_cos = np.clip(diff_angle_cos, a_min=-1, a_max=1) + diff_angle = np.arccos(diff_angle_cos) + diff_angle = (np.pi / 2.0 - np.abs(diff_angle)) / np.pi + + valid_mask = np.ones(previous_obs.shape[0]) + valid_mask[np.where(previous_obs[:, 4] < 0)] = 0 + + scores = np.repeat(det_scores[:, np.newaxis], trk_tlbrs.shape[0], axis=1) + valid_mask = np.repeat(valid_mask[:, np.newaxis], X.shape[1], axis=1) + + angle_diff_cost = (valid_mask * diff_angle) * vdc_weight + angle_diff_cost = angle_diff_cost * scores.T + + matches, unmatched_a, unmatched_b = linear_assignment(- (iou_matrix + angle_diff_cost), thresh=0.9) + + + return matches, unmatched_a, unmatched_b + +""" +helper func of observation_centric_association +""" +def speed_direction_batch(dets, tracks): + tracks = tracks[..., np.newaxis] + CX1, CY1 = (dets[:, 0] + dets[:, 2]) / 2.0, (dets[:,1] + dets[:,3]) / 2.0 + CX2, CY2 = (tracks[:, 0] + tracks[:, 2]) / 2.0, (tracks[:, 1] + tracks[:, 3]) / 2.0 + dx = CX2 - CX1 + dy = CY2 - CY1 + norm = np.sqrt(dx**2 + dy**2) + 1e-6 + dx = dx / norm + dy = dy / norm + return dy, dx # size: num_track x num_det + + +def matching_cascade( + distance_metric, matching_thresh, cascade_depth, tracks, detections, + track_indices=None, detection_indices=None): + """ + Run matching cascade in DeepSORT + + distance_metirc: function that calculate the cost matrix + matching_thresh: float, Associations with cost larger than this value are disregarded. + cascade_path: int, equal to max_age of a tracklet + tracks: List[STrack], current tracks + detections: List[STrack], current detections + track_indices: List[int], tracks that will be calculated, Default None + detection_indices: List[int], detections that will be calculated, Default None + + return: + matched pair, unmatched tracks, unmatced detections: List[int], List[int], List[int] + """ + if track_indices is None: + track_indices = list(range(len(tracks))) + if detection_indices is None: + detection_indices = list(range(len(detections))) + + detections_to_match = detection_indices + matches = [] + + for level in range(cascade_depth): + """ + match new track with detection firstly + """ + if not len(detections_to_match): # No detections left + break + + track_indices_l = [ + k for k in track_indices + if tracks[k].time_since_update == 1 + level + ] # filter tracks whose age is equal to level + 1 (The age of Newest track = 1) + + if not len(track_indices_l): # Nothing to match at this level + continue + + # tracks and detections which will be mathcted in current level + track_l = [tracks[idx] for idx in track_indices_l] # List[STrack] + det_l = [detections[idx] for idx in detections_to_match] # List[STrack] + + # calculate the cost matrix + cost_matrix = distance_metric(track_l, det_l) + + # solve the linear assignment problem + matched_row_col, umatched_row, umatched_col = \ + linear_assignment(cost_matrix, matching_thresh) + + for row, col in matched_row_col: # for those who matched + matches.append((track_indices_l[row], detections_to_match[col])) + + umatched_detecion_l = [] # current detections not matched + for col in umatched_col: # for detections not matched + umatched_detecion_l.append(detections_to_match[col]) + + detections_to_match = umatched_detecion_l # update detections to match for next level + unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches)) + + return matches, unmatched_tracks, detections_to_match + +def nearest_embedding_distance(tracks, detections, metric='cosine'): + """ + different from embedding distance, this func calculate the + nearest distance among all track history features and detections + + tracks: list[STrack] + detections: list[STrack] + metric: str, cosine or euclidean + TODO: support euclidean distance + + return: + cost_matrix, np.ndarray, shape(len(tracks), len(detections)) + """ + cost_matrix = np.zeros((len(tracks), len(detections))) + det_features = np.asarray([det.features[-1] for det in detections]) + + for row, track in enumerate(tracks): + track_history_features = np.asarray(track.features) + dist = 1. - cal_cosine_distance(track_history_features, det_features) + dist = dist.min(axis=0) + cost_matrix[row, :] = dist + + return cost_matrix + +def cal_cosine_distance(mat1, mat2): + """ + simple func to calculate cosine distance between 2 matrixs + + :param mat1: np.ndarray, shape(M, dim) + :param mat2: np.ndarray, shape(N, dim) + :return: np.ndarray, shape(M, N) + """ + # result = mat1·mat2^T / |mat1|·|mat2| + # norm mat1 and mat2 + mat1 = mat1 / np.linalg.norm(mat1, axis=1, keepdims=True) + mat2 = mat2 / np.linalg.norm(mat2, axis=1, keepdims=True) + + return np.dot(mat1, mat2.T) \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/ocsort_tracker.py b/test/yolov7-tracker/tracker/trackers/ocsort_tracker.py new file mode 100644 index 0000000..cccbc84 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/ocsort_tracker.py @@ -0,0 +1,237 @@ +""" +OC Sort +""" + +import numpy as np +from collections import deque +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_velocity +from .matching import * + +from cython_bbox import bbox_overlaps as bbox_ious + +class OCSortTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + self.delta_t = 3 + + @staticmethod + def k_previous_obs(observations, cur_age, k): + if len(observations) == 0: + return [-1, -1, -1, -1, -1] + for i in range(k): + dt = k - i + if cur_age - dt in observations: + return observations[cur_age-dt] + max_age = max(observations.keys()) + return observations[max_age] + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + inds_low = scores > 0.1 + inds_high = scores < self.args.conf_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + cates_second = categories[inds_second] + + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet_w_velocity(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, Observation Centric Momentum''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + velocities = np.array( + [trk.velocity if trk.velocity is not None else np.array((0, 0)) for trk in tracklet_pool]) + + # last observation, obervation-centric + # last_boxes = np.array([trk.last_observation for trk in tracklet_pool]) + + # historical observations + k_observations = np.array( + [self.k_previous_obs(trk.observations, trk.age, self.delta_t) for trk in tracklet_pool]) + + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + # Observation centric cost matrix and assignment + matches, u_track, u_detection = observation_centric_association( + tracklets=tracklet_pool, detections=detections, iou_threshold=0.3, + velocities=velocities, previous_obs=k_observations, vdc_weight=0.2 + ) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + ''' Step 3: Second association, with low score detection boxes''' + # association the untrack to the low score detections + if len(dets_second) > 0: + '''Detections''' + detections_second = [Tracklet_w_velocity(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets_second, scores_second, cates_second)] + else: + detections_second = [] + r_tracked_tracklets = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + + # for unmatched tracks in the first round, use last obervation + r_tracked_tracklets_last_observ = [tracklet_pool[i].last_observation[:4] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + detections_second_bbox = [det.tlbr for det in detections_second] + + dists = 1. - ious(r_tracked_tracklets_last_observ, detections_second_bbox) + + matches, u_track, u_detection_second = linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_tracklets[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = r_tracked_tracklets[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/reid_models/AFLink.py b/test/yolov7-tracker/tracker/trackers/reid_models/AFLink.py new file mode 100644 index 0000000..143534c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/reid_models/AFLink.py @@ -0,0 +1,98 @@ +""" +AFLink code in StrongSORT(StrongSORT: Make DeepSORT Great Again(arxiv)) + +copied from origin repo +""" +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import cv2 +import logging +import torchvision.transforms as transforms + + +class TemporalBlock(nn.Module): + def __init__(self, cin, cout): + super(TemporalBlock, self).__init__() + self.conv = nn.Conv2d(cin, cout, (7, 1), bias=False) + self.relu = nn.ReLU(inplace=True) + self.bnf = nn.BatchNorm1d(cout) + self.bnx = nn.BatchNorm1d(cout) + self.bny = nn.BatchNorm1d(cout) + + def bn(self, x): + x[:, :, :, 0] = self.bnf(x[:, :, :, 0]) + x[:, :, :, 1] = self.bnx(x[:, :, :, 1]) + x[:, :, :, 2] = self.bny(x[:, :, :, 2]) + return x + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class FusionBlock(nn.Module): + def __init__(self, cin, cout): + super(FusionBlock, self).__init__() + self.conv = nn.Conv2d(cin, cout, (1, 3), bias=False) + self.bn = nn.BatchNorm2d(cout) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class Classifier(nn.Module): + def __init__(self, cin): + super(Classifier, self).__init__() + self.fc1 = nn.Linear(cin*2, cin//2) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Linear(cin//2, 2) + + def forward(self, x1, x2): + x = torch.cat((x1, x2), dim=1) + x = self.fc1(x) + x = self.relu(x) + x = self.fc2(x) + return x + + +class PostLinker(nn.Module): + def __init__(self): + super(PostLinker, self).__init__() + self.TemporalModule_1 = nn.Sequential( + TemporalBlock(1, 32), + TemporalBlock(32, 64), + TemporalBlock(64, 128), + TemporalBlock(128, 256) + ) + self.TemporalModule_2 = nn.Sequential( + TemporalBlock(1, 32), + TemporalBlock(32, 64), + TemporalBlock(64, 128), + TemporalBlock(128, 256) + ) + self.FusionBlock_1 = FusionBlock(256, 256) + self.FusionBlock_2 = FusionBlock(256, 256) + self.pooling = nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = Classifier(256) + + def forward(self, x1, x2): + x1 = x1[:, :, :, :3] + x2 = x2[:, :, :, :3] + x1 = self.TemporalModule_1(x1) # [B,1,30,3] -> [B,256,6,3] + x2 = self.TemporalModule_2(x2) + x1 = self.FusionBlock_1(x1) + x2 = self.FusionBlock_2(x2) + x1 = self.pooling(x1).squeeze(-1).squeeze(-1) + x2 = self.pooling(x2).squeeze(-1).squeeze(-1) + y = self.classifier(x1, x2) + if not self.training: + y = torch.softmax(y, dim=1) + return y \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/reid_models/OSNet.py b/test/yolov7-tracker/tracker/trackers/reid_models/OSNet.py new file mode 100644 index 0000000..b77388f --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/reid_models/OSNet.py @@ -0,0 +1,598 @@ +from __future__ import division, absolute_import +import warnings +import torch +from torch import nn +from torch.nn import functional as F + +__all__ = [ + 'osnet_x1_0', 'osnet_x0_75', 'osnet_x0_5', 'osnet_x0_25', 'osnet_ibn_x1_0' +] + +pretrained_urls = { + 'osnet_x1_0': + 'https://drive.google.com/uc?id=1LaG1EJpHrxdAxKnSCJ_i0u-nbxSAeiFY', + 'osnet_x0_75': + 'https://drive.google.com/uc?id=1uwA9fElHOk3ZogwbeY5GkLI6QPTX70Hq', + 'osnet_x0_5': + 'https://drive.google.com/uc?id=16DGLbZukvVYgINws8u8deSaOqjybZ83i', + 'osnet_x0_25': + 'https://drive.google.com/uc?id=1rb8UN5ZzPKRc_xvtHlyDh-cSz88YX9hs', + 'osnet_ibn_x1_0': + 'https://drive.google.com/uc?id=1sr90V6irlYYDd4_4ISU2iruoRG8J__6l' +} + + +########## +# Basic layers +########## +class ConvLayer(nn.Module): + """Convolution layer (conv + bn + relu).""" + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + groups=1, + IN=False + ): + super(ConvLayer, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + bias=False, + groups=groups + ) + if IN: + self.bn = nn.InstanceNorm2d(out_channels, affine=True) + else: + self.bn = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class Conv1x1(nn.Module): + """1x1 convolution + bn + relu.""" + + def __init__(self, in_channels, out_channels, stride=1, groups=1): + super(Conv1x1, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + 1, + stride=stride, + padding=0, + bias=False, + groups=groups + ) + self.bn = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class Conv1x1Linear(nn.Module): + """1x1 convolution + bn (w/o non-linearity).""" + + def __init__(self, in_channels, out_channels, stride=1): + super(Conv1x1Linear, self).__init__() + self.conv = nn.Conv2d( + in_channels, out_channels, 1, stride=stride, padding=0, bias=False + ) + self.bn = nn.BatchNorm2d(out_channels) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class Conv3x3(nn.Module): + """3x3 convolution + bn + relu.""" + + def __init__(self, in_channels, out_channels, stride=1, groups=1): + super(Conv3x3, self).__init__() + self.conv = nn.Conv2d( + in_channels, + out_channels, + 3, + stride=stride, + padding=1, + bias=False, + groups=groups + ) + self.bn = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class LightConv3x3(nn.Module): + """Lightweight 3x3 convolution. + + 1x1 (linear) + dw 3x3 (nonlinear). + """ + + def __init__(self, in_channels, out_channels): + super(LightConv3x3, self).__init__() + self.conv1 = nn.Conv2d( + in_channels, out_channels, 1, stride=1, padding=0, bias=False + ) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + 3, + stride=1, + padding=1, + bias=False, + groups=out_channels + ) + self.bn = nn.BatchNorm2d(out_channels) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.bn(x) + x = self.relu(x) + return x + + +########## +# Building blocks for omni-scale feature learning +########## +class ChannelGate(nn.Module): + """A mini-network that generates channel-wise gates conditioned on input tensor.""" + + def __init__( + self, + in_channels, + num_gates=None, + return_gates=False, + gate_activation='sigmoid', + reduction=16, + layer_norm=False + ): + super(ChannelGate, self).__init__() + if num_gates is None: + num_gates = in_channels + self.return_gates = return_gates + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + self.fc1 = nn.Conv2d( + in_channels, + in_channels // reduction, + kernel_size=1, + bias=True, + padding=0 + ) + self.norm1 = None + if layer_norm: + self.norm1 = nn.LayerNorm((in_channels // reduction, 1, 1)) + self.relu = nn.ReLU(inplace=True) + self.fc2 = nn.Conv2d( + in_channels // reduction, + num_gates, + kernel_size=1, + bias=True, + padding=0 + ) + if gate_activation == 'sigmoid': + self.gate_activation = nn.Sigmoid() + elif gate_activation == 'relu': + self.gate_activation = nn.ReLU(inplace=True) + elif gate_activation == 'linear': + self.gate_activation = None + else: + raise RuntimeError( + "Unknown gate activation: {}".format(gate_activation) + ) + + def forward(self, x): + input = x + x = self.global_avgpool(x) + x = self.fc1(x) + if self.norm1 is not None: + x = self.norm1(x) + x = self.relu(x) + x = self.fc2(x) + if self.gate_activation is not None: + x = self.gate_activation(x) + if self.return_gates: + return x + return input * x + + +class OSBlock(nn.Module): + """Omni-scale feature learning block.""" + + def __init__( + self, + in_channels, + out_channels, + IN=False, + bottleneck_reduction=4, + **kwargs + ): + super(OSBlock, self).__init__() + mid_channels = out_channels // bottleneck_reduction + self.conv1 = Conv1x1(in_channels, mid_channels) + self.conv2a = LightConv3x3(mid_channels, mid_channels) + self.conv2b = nn.Sequential( + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + ) + self.conv2c = nn.Sequential( + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + ) + self.conv2d = nn.Sequential( + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + LightConv3x3(mid_channels, mid_channels), + ) + self.gate = ChannelGate(mid_channels) + self.conv3 = Conv1x1Linear(mid_channels, out_channels) + self.downsample = None + if in_channels != out_channels: + self.downsample = Conv1x1Linear(in_channels, out_channels) + self.IN = None + if IN: + self.IN = nn.InstanceNorm2d(out_channels, affine=True) + + def forward(self, x): + identity = x + x1 = self.conv1(x) + x2a = self.conv2a(x1) + x2b = self.conv2b(x1) + x2c = self.conv2c(x1) + x2d = self.conv2d(x1) + x2 = self.gate(x2a) + self.gate(x2b) + self.gate(x2c) + self.gate(x2d) + x3 = self.conv3(x2) + if self.downsample is not None: + identity = self.downsample(identity) + out = x3 + identity + if self.IN is not None: + out = self.IN(out) + return F.relu(out) + + +########## +# Network architecture +########## +class OSNet(nn.Module): + """Omni-Scale Network. + + Reference: + - Zhou et al. Omni-Scale Feature Learning for Person Re-Identification. ICCV, 2019. + - Zhou et al. Learning Generalisable Omni-Scale Representations + for Person Re-Identification. TPAMI, 2021. + """ + + def __init__( + self, + num_classes, + blocks, + layers, + channels, + feature_dim=512, + loss='softmax', + IN=False, + **kwargs + ): + super(OSNet, self).__init__() + num_blocks = len(blocks) + assert num_blocks == len(layers) + assert num_blocks == len(channels) - 1 + self.loss = loss + self.feature_dim = feature_dim + + # convolutional backbone + self.conv1 = ConvLayer(3, channels[0], 7, stride=2, padding=3, IN=IN) + self.maxpool = nn.MaxPool2d(3, stride=2, padding=1) + self.conv2 = self._make_layer( + blocks[0], + layers[0], + channels[0], + channels[1], + reduce_spatial_size=True, + IN=IN + ) + self.conv3 = self._make_layer( + blocks[1], + layers[1], + channels[1], + channels[2], + reduce_spatial_size=True + ) + self.conv4 = self._make_layer( + blocks[2], + layers[2], + channels[2], + channels[3], + reduce_spatial_size=False + ) + self.conv5 = Conv1x1(channels[3], channels[3]) + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + # fully connected layer + self.fc = self._construct_fc_layer( + self.feature_dim, channels[3], dropout_p=None + ) + # identity classification layer + self.classifier = nn.Linear(self.feature_dim, num_classes) + + self._init_params() + + def _make_layer( + self, + block, + layer, + in_channels, + out_channels, + reduce_spatial_size, + IN=False + ): + layers = [] + + layers.append(block(in_channels, out_channels, IN=IN)) + for i in range(1, layer): + layers.append(block(out_channels, out_channels, IN=IN)) + + if reduce_spatial_size: + layers.append( + nn.Sequential( + Conv1x1(out_channels, out_channels), + nn.AvgPool2d(2, stride=2) + ) + ) + + return nn.Sequential(*layers) + + def _construct_fc_layer(self, fc_dims, input_dim, dropout_p=None): + if fc_dims is None or fc_dims < 0: + self.feature_dim = input_dim + return None + + if isinstance(fc_dims, int): + fc_dims = [fc_dims] + + layers = [] + for dim in fc_dims: + layers.append(nn.Linear(input_dim, dim)) + layers.append(nn.BatchNorm1d(dim)) + layers.append(nn.ReLU(inplace=True)) + if dropout_p is not None: + layers.append(nn.Dropout(p=dropout_p)) + input_dim = dim + + self.feature_dim = fc_dims[-1] + + return nn.Sequential(*layers) + + def _init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_( + m.weight, mode='fan_out', nonlinearity='relu' + ) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + elif isinstance(m, nn.BatchNorm1d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + def featuremaps(self, x): + x = self.conv1(x) + x = self.maxpool(x) + x = self.conv2(x) + x = self.conv3(x) + x = self.conv4(x) + x = self.conv5(x) + return x + + def forward(self, x, return_featuremaps=False): + x = self.featuremaps(x) + if return_featuremaps: + return x + v = self.global_avgpool(x) + v = v.view(v.size(0), -1) + if self.fc is not None: + v = self.fc(v) + if not self.training: + return v + y = self.classifier(v) + if self.loss == 'softmax': + return y + elif self.loss == 'triplet': + return y, v + else: + raise KeyError("Unsupported loss: {}".format(self.loss)) + + +def init_pretrained_weights(model, key=''): + """Initializes model with pretrained weights. + + Layers that don't match with pretrained layers in name or size are kept unchanged. + """ + import os + import errno + import gdown + from collections import OrderedDict + + def _get_torch_home(): + ENV_TORCH_HOME = 'TORCH_HOME' + ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' + DEFAULT_CACHE_DIR = '~/.cache' + torch_home = os.path.expanduser( + os.getenv( + ENV_TORCH_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'torch' + ) + ) + ) + return torch_home + + torch_home = _get_torch_home() + model_dir = os.path.join(torch_home, 'checkpoints') + try: + os.makedirs(model_dir) + except OSError as e: + if e.errno == errno.EEXIST: + # Directory already exists, ignore. + pass + else: + # Unexpected OSError, re-raise. + raise + filename = key + '_imagenet.pth' + cached_file = os.path.join(model_dir, filename) + + if not os.path.exists(cached_file): + gdown.download(pretrained_urls[key], cached_file, quiet=False) + + state_dict = torch.load(cached_file) + model_dict = model.state_dict() + new_state_dict = OrderedDict() + matched_layers, discarded_layers = [], [] + + for k, v in state_dict.items(): + if k.startswith('module.'): + k = k[7:] # discard module. + + if k in model_dict and model_dict[k].size() == v.size(): + new_state_dict[k] = v + matched_layers.append(k) + else: + discarded_layers.append(k) + + model_dict.update(new_state_dict) + model.load_state_dict(model_dict) + + if len(matched_layers) == 0: + warnings.warn( + 'The pretrained weights from "{}" cannot be loaded, ' + 'please check the key names manually ' + '(** ignored and continue **)'.format(cached_file) + ) + else: + print( + 'Successfully loaded imagenet pretrained weights from "{}"'. + format(cached_file) + ) + if len(discarded_layers) > 0: + print( + '** The following layers are discarded ' + 'due to unmatched keys or layer size: {}'. + format(discarded_layers) + ) + + +########## +# Instantiation +########## +def osnet_x1_0(num_classes=1000, pretrained=True, loss='softmax', **kwargs): + # standard size (width x1.0) + model = OSNet( + num_classes, + blocks=[OSBlock, OSBlock, OSBlock], + layers=[2, 2, 2], + channels=[64, 256, 384, 512], + loss=loss, + **kwargs + ) + if pretrained: + init_pretrained_weights(model, key='osnet_x1_0') + return model + + +def osnet_x0_75(num_classes=1000, pretrained=True, loss='softmax', **kwargs): + # medium size (width x0.75) + model = OSNet( + num_classes, + blocks=[OSBlock, OSBlock, OSBlock], + layers=[2, 2, 2], + channels=[48, 192, 288, 384], + loss=loss, + **kwargs + ) + if pretrained: + init_pretrained_weights(model, key='osnet_x0_75') + return model + + +def osnet_x0_5(num_classes=1000, pretrained=True, loss='softmax', **kwargs): + # tiny size (width x0.5) + model = OSNet( + num_classes, + blocks=[OSBlock, OSBlock, OSBlock], + layers=[2, 2, 2], + channels=[32, 128, 192, 256], + loss=loss, + **kwargs + ) + if pretrained: + init_pretrained_weights(model, key='osnet_x0_5') + return model + + +def osnet_x0_25(num_classes=1000, pretrained=True, loss='softmax', **kwargs): + # very tiny size (width x0.25) + model = OSNet( + num_classes, + blocks=[OSBlock, OSBlock, OSBlock], + layers=[2, 2, 2], + channels=[16, 64, 96, 128], + loss=loss, + **kwargs + ) + if pretrained: + init_pretrained_weights(model, key='osnet_x0_25') + return model + + +def osnet_ibn_x1_0( + num_classes=1000, pretrained=True, loss='softmax', **kwargs +): + # standard size (width x1.0) + IBN layer + # Ref: Pan et al. Two at Once: Enhancing Learning and Generalization Capacities via IBN-Net. ECCV, 2018. + model = OSNet( + num_classes, + blocks=[OSBlock, OSBlock, OSBlock], + layers=[2, 2, 2], + channels=[64, 256, 384, 512], + loss=loss, + IN=True, + **kwargs + ) + if pretrained: + init_pretrained_weights(model, key='osnet_ibn_x1_0') + return model diff --git a/test/yolov7-tracker/tracker/trackers/reid_models/__init__.py b/test/yolov7-tracker/tracker/trackers/reid_models/__init__.py new file mode 100644 index 0000000..10c2fe3 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/reid_models/__init__.py @@ -0,0 +1,3 @@ +""" +file for reid_models folder +""" \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/reid_models/deepsort_reid.py b/test/yolov7-tracker/tracker/trackers/reid_models/deepsort_reid.py new file mode 100644 index 0000000..6571a28 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/reid_models/deepsort_reid.py @@ -0,0 +1,157 @@ +""" +file for DeepSORT Re-ID model +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import cv2 +import logging +import torchvision.transforms as transforms + + +class BasicBlock(nn.Module): + def __init__(self, c_in, c_out, is_downsample=False): + super(BasicBlock, self).__init__() + self.is_downsample = is_downsample + if is_downsample: + self.conv1 = nn.Conv2d( + c_in, c_out, 3, stride=2, padding=1, bias=False) + else: + self.conv1 = nn.Conv2d( + c_in, c_out, 3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(c_out) + self.relu = nn.ReLU(True) + self.conv2 = nn.Conv2d(c_out, c_out, 3, stride=1, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(c_out) + if is_downsample: + self.downsample = nn.Sequential( + nn.Conv2d(c_in, c_out, 1, stride=2, bias=False), + nn.BatchNorm2d(c_out) + ) + elif c_in != c_out: + self.downsample = nn.Sequential( + nn.Conv2d(c_in, c_out, 1, stride=1, bias=False), + nn.BatchNorm2d(c_out) + ) + self.is_downsample = True + + def forward(self, x): + y = self.conv1(x) + y = self.bn1(y) + y = self.relu(y) + y = self.conv2(y) + y = self.bn2(y) + if self.is_downsample: + x = self.downsample(x) + return F.relu(x.add(y), True) + + +def make_layers(c_in, c_out, repeat_times, is_downsample=False): + blocks = [] + for i in range(repeat_times): + if i == 0: + blocks += [BasicBlock(c_in, c_out, is_downsample=is_downsample), ] + else: + blocks += [BasicBlock(c_out, c_out), ] + return nn.Sequential(*blocks) + + +class Net(nn.Module): + def __init__(self, num_classes=751, reid=False): + super(Net, self).__init__() + # 3 128 64 + self.conv = nn.Sequential( + nn.Conv2d(3, 64, 3, stride=1, padding=1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True), + # nn.Conv2d(32,32,3,stride=1,padding=1), + # nn.BatchNorm2d(32), + # nn.ReLU(inplace=True), + nn.MaxPool2d(3, 2, padding=1), + ) + # 32 64 32 + self.layer1 = make_layers(64, 64, 2, False) + # 32 64 32 + self.layer2 = make_layers(64, 128, 2, True) + # 64 32 16 + self.layer3 = make_layers(128, 256, 2, True) + # 128 16 8 + self.layer4 = make_layers(256, 512, 2, True) + # 256 8 4 + self.avgpool = nn.AvgPool2d((8, 4), 1) + # 256 1 1 + self.reid = reid + self.classifier = nn.Sequential( + nn.Linear(512, 256), + nn.BatchNorm1d(256), + nn.ReLU(inplace=True), + nn.Dropout(), + nn.Linear(256, num_classes), + ) + + def forward(self, x): + x = self.conv(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.avgpool(x) + x = x.view(x.size(0), -1) + # B x 128 + if self.reid: + x = x.div(x.norm(p=2, dim=1, keepdim=True)) + return x + # classifier + x = self.classifier(x) + return x + + +class Extractor(object): + def __init__(self, model_path, use_cuda=True): + self.net = Net(reid=True) + self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu" + state_dict = torch.load(model_path, map_location=torch.device(self.device))[ + 'net_dict'] + self.net.load_state_dict(state_dict) + logger = logging.getLogger("root.tracker") + logger.info("Loading weights from {}... Done!".format(model_path)) + self.net.to(self.device) + self.size = (64, 128) + self.norm = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ]) + + def _preprocess(self, im_crops): + """ + TODO: + 1. to float with scale from 0 to 1 + 2. resize to (64, 128) as Market1501 dataset did + 3. concatenate to a numpy array + 3. to torch Tensor + 4. normalize + """ + def _resize(im, size): + try: + return cv2.resize(im.astype(np.float32)/255., size) + except: + print('Error: size in bbox exists zero, ', im.shape) + exit(0) + + im_batch = torch.cat([self.norm(_resize(im, self.size)).unsqueeze( + 0) for im in im_crops], dim=0).float() + return im_batch + + def __call__(self, im_crops): + if isinstance(im_crops, list): + im_batch = self._preprocess(im_crops) + else: + im_batch = im_crops + + with torch.no_grad(): + im_batch = im_batch.to(self.device) + features = self.net(im_batch) + return features diff --git a/test/yolov7-tracker/tracker/trackers/reid_models/load_model_tools.py b/test/yolov7-tracker/tracker/trackers/reid_models/load_model_tools.py new file mode 100644 index 0000000..49cb0fe --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/reid_models/load_model_tools.py @@ -0,0 +1,273 @@ +""" +load checkpoint file +copied from https://github.com/mikel-brostrom/Yolov5_StrongSORT_OSNet +""" +from __future__ import division, print_function, absolute_import +import pickle +import shutil +import os.path as osp +import warnings +from functools import partial +from collections import OrderedDict +import torch +import torch.nn as nn + + +__all__ = [ + 'save_checkpoint', 'load_checkpoint', 'resume_from_checkpoint', + 'open_all_layers', 'open_specified_layers', 'count_num_param', + 'load_pretrained_weights' +] + +def load_checkpoint(fpath): + r"""Loads checkpoint. + + ``UnicodeDecodeError`` can be well handled, which means + python2-saved files can be read from python3. + + Args: + fpath (str): path to checkpoint. + + Returns: + dict + + Examples:: + >>> from torchreid.utils import load_checkpoint + >>> fpath = 'log/my_model/model.pth.tar-10' + >>> checkpoint = load_checkpoint(fpath) + """ + if fpath is None: + raise ValueError('File path is None') + fpath = osp.abspath(osp.expanduser(fpath)) + if not osp.exists(fpath): + raise FileNotFoundError('File is not found at "{}"'.format(fpath)) + map_location = None if torch.cuda.is_available() else 'cpu' + try: + checkpoint = torch.load(fpath, map_location=map_location) + except UnicodeDecodeError: + pickle.load = partial(pickle.load, encoding="latin1") + pickle.Unpickler = partial(pickle.Unpickler, encoding="latin1") + checkpoint = torch.load( + fpath, pickle_module=pickle, map_location=map_location + ) + except Exception: + print('Unable to load checkpoint from "{}"'.format(fpath)) + raise + return checkpoint + + +def resume_from_checkpoint(fpath, model, optimizer=None, scheduler=None): + r"""Resumes training from a checkpoint. + + This will load (1) model weights and (2) ``state_dict`` + of optimizer if ``optimizer`` is not None. + + Args: + fpath (str): path to checkpoint. + model (nn.Module): model. + optimizer (Optimizer, optional): an Optimizer. + scheduler (LRScheduler, optional): an LRScheduler. + + Returns: + int: start_epoch. + + Examples:: + >>> from torchreid.utils import resume_from_checkpoint + >>> fpath = 'log/my_model/model.pth.tar-10' + >>> start_epoch = resume_from_checkpoint( + >>> fpath, model, optimizer, scheduler + >>> ) + """ + print('Loading checkpoint from "{}"'.format(fpath)) + checkpoint = load_checkpoint(fpath) + model.load_state_dict(checkpoint['state_dict']) + print('Loaded model weights') + if optimizer is not None and 'optimizer' in checkpoint.keys(): + optimizer.load_state_dict(checkpoint['optimizer']) + print('Loaded optimizer') + if scheduler is not None and 'scheduler' in checkpoint.keys(): + scheduler.load_state_dict(checkpoint['scheduler']) + print('Loaded scheduler') + start_epoch = checkpoint['epoch'] + print('Last epoch = {}'.format(start_epoch)) + if 'rank1' in checkpoint.keys(): + print('Last rank1 = {:.1%}'.format(checkpoint['rank1'])) + return start_epoch + + +def adjust_learning_rate( + optimizer, + base_lr, + epoch, + stepsize=20, + gamma=0.1, + linear_decay=False, + final_lr=0, + max_epoch=100 +): + r"""Adjusts learning rate. + + Deprecated. + """ + if linear_decay: + # linearly decay learning rate from base_lr to final_lr + frac_done = epoch / max_epoch + lr = frac_done*final_lr + (1.-frac_done) * base_lr + else: + # decay learning rate by gamma for every stepsize + lr = base_lr * (gamma**(epoch // stepsize)) + + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def set_bn_to_eval(m): + r"""Sets BatchNorm layers to eval mode.""" + # 1. no update for running mean and var + # 2. scale and shift parameters are still trainable + classname = m.__class__.__name__ + if classname.find('BatchNorm') != -1: + m.eval() + + +def open_all_layers(model): + r"""Opens all layers in model for training. + + Examples:: + >>> from torchreid.utils import open_all_layers + >>> open_all_layers(model) + """ + model.train() + for p in model.parameters(): + p.requires_grad = True + + +def open_specified_layers(model, open_layers): + r"""Opens specified layers in model for training while keeping + other layers frozen. + + Args: + model (nn.Module): neural net model. + open_layers (str or list): layers open for training. + + Examples:: + >>> from torchreid.utils import open_specified_layers + >>> # Only model.classifier will be updated. + >>> open_layers = 'classifier' + >>> open_specified_layers(model, open_layers) + >>> # Only model.fc and model.classifier will be updated. + >>> open_layers = ['fc', 'classifier'] + >>> open_specified_layers(model, open_layers) + """ + if isinstance(model, nn.DataParallel): + model = model.module + + if isinstance(open_layers, str): + open_layers = [open_layers] + + for layer in open_layers: + assert hasattr( + model, layer + ), '"{}" is not an attribute of the model, please provide the correct name'.format( + layer + ) + + for name, module in model.named_children(): + if name in open_layers: + module.train() + for p in module.parameters(): + p.requires_grad = True + else: + module.eval() + for p in module.parameters(): + p.requires_grad = False + + +def count_num_param(model): + r"""Counts number of parameters in a model while ignoring ``self.classifier``. + + Args: + model (nn.Module): network model. + + Examples:: + >>> from torchreid.utils import count_num_param + >>> model_size = count_num_param(model) + + .. warning:: + + This method is deprecated in favor of + ``torchreid.utils.compute_model_complexity``. + """ + warnings.warn( + 'This method is deprecated and will be removed in the future.' + ) + + num_param = sum(p.numel() for p in model.parameters()) + + if isinstance(model, nn.DataParallel): + model = model.module + + if hasattr(model, + 'classifier') and isinstance(model.classifier, nn.Module): + # we ignore the classifier because it is unused at test time + num_param -= sum(p.numel() for p in model.classifier.parameters()) + + return num_param + + +def load_pretrained_weights(model, weight_path): + r"""Loads pretrianed weights to model. + + Features:: + - Incompatible layers (unmatched in name or size) will be ignored. + - Can automatically deal with keys containing "module.". + + Args: + model (nn.Module): network model. + weight_path (str): path to pretrained weights. + + Examples:: + >>> from torchreid.utils import load_pretrained_weights + >>> weight_path = 'log/my_model/model-best.pth.tar' + >>> load_pretrained_weights(model, weight_path) + """ + checkpoint = load_checkpoint(weight_path) + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + model_dict = model.state_dict() + new_state_dict = OrderedDict() + matched_layers, discarded_layers = [], [] + + for k, v in state_dict.items(): + if k.startswith('module.'): + k = k[7:] # discard module. + + if k in model_dict and model_dict[k].size() == v.size(): + new_state_dict[k] = v + matched_layers.append(k) + else: + discarded_layers.append(k) + + model_dict.update(new_state_dict) + model.load_state_dict(model_dict) + + if len(matched_layers) == 0: + warnings.warn( + 'The pretrained weights "{}" cannot be loaded, ' + 'please check the key names manually ' + '(** ignored and continue **)'.format(weight_path) + ) + else: + print( + 'Successfully loaded pretrained weights from "{}"'. + format(weight_path) + ) + if len(discarded_layers) > 0: + print( + '** The following layers are discarded ' + 'due to unmatched keys or layer size: {}'. + format(discarded_layers) + ) diff --git a/test/yolov7-tracker/tracker/trackers/sort_tracker.py b/test/yolov7-tracker/tracker/trackers/sort_tracker.py new file mode 100644 index 0000000..3d40410 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/sort_tracker.py @@ -0,0 +1,169 @@ +""" +Sort +""" + +import numpy as np +from collections import deque +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet +from .matching import * + +class SortTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + + scores_keep = scores[remain_inds] + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with high score detection boxes''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + dists = iou_distance(tracklet_pool, detections) + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.9) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 3: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 4: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/sparse_tracker.py b/test/yolov7-tracker/tracker/trackers/sparse_tracker.py new file mode 100644 index 0000000..4a46e05 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/sparse_tracker.py @@ -0,0 +1,338 @@ +""" +Bot sort +""" + +import numpy as np +import torch +from torchvision.ops import nms + +import cv2 +import torchvision.transforms as T + +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_depth +from .matching import * + +from .reid_models.OSNet import * +from .reid_models.load_model_tools import load_pretrained_weights +from .reid_models.deepsort_reid import Extractor + +from .camera_motion_compensation import GMC + +REID_MODEL_DICT = { + 'osnet_x1_0': osnet_x1_0, + 'osnet_x0_75': osnet_x0_75, + 'osnet_x0_5': osnet_x0_5, + 'osnet_x0_25': osnet_x0_25, + 'deepsort': Extractor +} + + +def load_reid_model(reid_model, reid_model_path): + + if 'osnet' in reid_model: + func = REID_MODEL_DICT[reid_model] + model = func(num_classes=1, pretrained=False, ) + load_pretrained_weights(model, reid_model_path) + model.cuda().eval() + + elif 'deepsort' in reid_model: + model = REID_MODEL_DICT[reid_model](reid_model_path, use_cuda=True) + + else: + raise NotImplementedError + + return model + +class SparseTracker(object): + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + # camera motion compensation module + self.gmc = GMC(method='orb', downscale=2, verbose=None) + + def get_deep_range(self, obj, step): + col = [] + for t in obj: + lend = (t.deep_vec)[2] + col.append(lend) + max_len, mix_len = max(col), min(col) + if max_len != mix_len: + deep_range =np.arange(mix_len, max_len, (max_len - mix_len + 1) / step) + if deep_range[-1] < max_len: + deep_range = np.concatenate([deep_range, np.array([max_len],)]) + deep_range[0] = np.floor(deep_range[0]) + deep_range[-1] = np.ceil(deep_range[-1]) + else: + deep_range = [mix_len,] + mask = self.get_sub_mask(deep_range, col) + return mask + + def get_sub_mask(self, deep_range, col): + mix_len=deep_range[0] + max_len=deep_range[-1] + if max_len == mix_len: + lc = mix_len + mask = [] + for d in deep_range: + if d > deep_range[0] and d < deep_range[-1]: + mask.append((col >= lc) & (col < d)) + lc = d + elif d == deep_range[-1]: + mask.append((col >= lc) & (col <= d)) + lc = d + else: + lc = d + continue + return mask + + # core function + def DCM(self, detections, tracks, activated_tracklets, refind_tracklets, levels, thresh, is_fuse): + if len(detections) > 0: + det_mask = self.get_deep_range(detections, levels) + else: + det_mask = [] + + if len(tracks)!=0: + track_mask = self.get_deep_range(tracks, levels) + else: + track_mask = [] + + u_detection, u_tracks, res_det, res_track = [], [], [], [] + if len(track_mask) != 0: + if len(track_mask) < len(det_mask): + for i in range(len(det_mask) - len(track_mask)): + idx = np.argwhere(det_mask[len(track_mask) + i] == True) + for idd in idx: + res_det.append(detections[idd[0]]) + elif len(track_mask) > len(det_mask): + for i in range(len(track_mask) - len(det_mask)): + idx = np.argwhere(track_mask[len(det_mask) + i] == True) + for idd in idx: + res_track.append(tracks[idd[0]]) + + for dm, tm in zip(det_mask, track_mask): + det_idx = np.argwhere(dm == True) + trk_idx = np.argwhere(tm == True) + + # search det + det_ = [] + for idd in det_idx: + det_.append(detections[idd[0]]) + det_ = det_ + u_detection + # search trk + track_ = [] + for idt in trk_idx: + track_.append(tracks[idt[0]]) + # update trk + track_ = track_ + u_tracks + + dists = iou_distance(track_, det_) + + matches, u_track_, u_det_ = linear_assignment(dists, thresh) + for itracked, idet in matches: + track = track_[itracked] + det = det_[idet] + if track.state == TrackState.Tracked: + track.update(det_[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + u_tracks = [track_[t] for t in u_track_] + u_detection = [det_[t] for t in u_det_] + + u_tracks = u_tracks + res_track + u_detection = u_detection + res_det + + else: + u_detection = detections + + return activated_tracklets, refind_tracklets, u_tracks, u_detection + + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlwh format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + inds_low = scores > 0.1 + inds_high = scores < self.args.conf_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + cates_second = categories[inds_second] + + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + + if len(dets) > 0: + detections = [Tracklet_w_depth(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets, scores_keep, cates)] + else: + detections = [] + + ''' Step 1: Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with high score detection boxes, depth cascade mathcing''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + # Camera motion compensation + warp = self.gmc.apply(ori_img, dets) + self.gmc.multi_gmc(tracklet_pool, warp) + self.gmc.multi_gmc(unconfirmed, warp) + + # depth cascade matching + activated_tracklets, refind_tracklets, u_track, u_detection_high = self.DCM( + detections, + tracklet_pool, + activated_tracklets, + refind_tracklets, + levels=3, + thresh=0.75, + is_fuse=True) + + ''' Step 3: Second association, with low score detection boxes, depth cascade mathcing''' + if len(dets_second) > 0: + '''Detections''' + detections_second = [Tracklet_w_depth(tlwh, s, cate, motion=self.motion) for + (tlwh, s, cate) in zip(dets_second, scores_second, cates_second)] + else: + detections_second = [] + + r_tracked_tracklets = [t for t in u_track if t.state == TrackState.Tracked] + + activated_tracklets, refind_tracklets, u_track, u_detection_sec = self.DCM( + detections_second, + r_tracked_tracklets, + activated_tracklets, + refind_tracklets, + levels=3, + thresh=0.3, + is_fuse=False) + + for track in u_track: + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = u_detection_high + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/strongsort_tracker.py b/test/yolov7-tracker/tracker/trackers/strongsort_tracker.py new file mode 100644 index 0000000..3685630 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/strongsort_tracker.py @@ -0,0 +1,327 @@ +""" +Deep Sort +""" + +import numpy as np +import torch +from torchvision.ops import nms + +import cv2 +import torchvision.transforms as T + +from .basetrack import BaseTrack, TrackState +from .tracklet import Tracklet, Tracklet_w_reid +from .matching import * + +from .reid_models.OSNet import * +from .reid_models.load_model_tools import load_pretrained_weights +from .reid_models.deepsort_reid import Extractor + +REID_MODEL_DICT = { + 'osnet_x1_0': osnet_x1_0, + 'osnet_x0_75': osnet_x0_75, + 'osnet_x0_5': osnet_x0_5, + 'osnet_x0_25': osnet_x0_25, + 'deepsort': Extractor +} + + +def load_reid_model(reid_model, reid_model_path): + + if 'osnet' in reid_model: + func = REID_MODEL_DICT[reid_model] + model = func(num_classes=1, pretrained=False, ) + load_pretrained_weights(model, reid_model_path) + model.cuda().eval() + + elif 'deepsort' in reid_model: + model = REID_MODEL_DICT[reid_model](reid_model_path, use_cuda=True) + + else: + raise NotImplementedError + + return model + + +class StrongSortTracker(object): + + def __init__(self, args, frame_rate=30): + self.tracked_tracklets = [] # type: list[Tracklet] + self.lost_tracklets = [] # type: list[Tracklet] + self.removed_tracklets = [] # type: list[Tracklet] + + self.frame_id = 0 + self.args = args + + self.det_thresh = args.conf_thresh + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.max_time_lost = self.buffer_size + + self.motion = args.kalman_format + + self.with_reid = not args.discard_reid + + self.reid_model, self.crop_transforms = None, None + if self.with_reid: + self.reid_model = load_reid_model(args.reid_model, args.reid_model_path) + self.crop_transforms = T.Compose([ + # T.ToPILImage(), + # T.Resize(size=(256, 128)), + T.ToTensor(), # (c, 128, 256) + T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + ]) + + self.bbox_crop_size = (64, 128) if 'deepsort' in args.reid_model else (128, 128) + + self.lambda_ = 0.98 # the coef of cost mix in eq. 10 in paper + + + def reid_preprocess(self, obj_bbox): + """ + preprocess cropped object bboxes + + obj_bbox: np.ndarray, shape=(h_obj, w_obj, c) + + return: + torch.Tensor of shape (c, 128, 256) + """ + + obj_bbox = cv2.resize(obj_bbox.astype(np.float32) / 255.0, dsize=self.bbox_crop_size) # shape: (h, w, c) + + return self.crop_transforms(obj_bbox) + + def get_feature(self, tlwhs, ori_img): + """ + get apperance feature of an object + tlwhs: shape (num_of_objects, 4) + ori_img: original image, np.ndarray, shape(H, W, C) + """ + obj_bbox = [] + + for tlwh in tlwhs: + tlwh = list(map(int, tlwh)) + + # limit to the legal range + tlwh[0], tlwh[1] = max(tlwh[0], 0), max(tlwh[1], 0) + + tlbr_tensor = self.reid_preprocess(ori_img[tlwh[1]: tlwh[1] + tlwh[3], tlwh[0]: tlwh[0] + tlwh[2]]) + + obj_bbox.append(tlbr_tensor) + + if not obj_bbox: + return np.array([]) + + obj_bbox = torch.stack(obj_bbox, dim=0) + obj_bbox = obj_bbox.cuda() + + features = self.reid_model(obj_bbox) # shape: (num_of_objects, feature_dim) + return features.cpu().detach().numpy() + + def update(self, output_results, img, ori_img): + """ + output_results: processed detections (scale to original size) tlbr format + """ + + self.frame_id += 1 + activated_tracklets = [] + refind_tracklets = [] + lost_tracklets = [] + removed_tracklets = [] + + scores = output_results[:, 4] + bboxes = output_results[:, :4] + categories = output_results[:, -1] + + remain_inds = scores > self.args.conf_thresh + + dets = bboxes[remain_inds] + + cates = categories[remain_inds] + + scores_keep = scores[remain_inds] + + features_keep = self.get_feature(tlwhs=dets[:, :4], ori_img=ori_img) + + if len(dets) > 0: + '''Detections''' + detections = [Tracklet_w_reid(tlwh, s, cate, motion=self.motion, feat=feat) for + (tlwh, s, cate, feat) in zip(dets, scores_keep, cates, features_keep)] + else: + detections = [] + + ''' Add newly detected tracklets to tracked_tracklets''' + unconfirmed = [] + tracked_tracklets = [] # type: list[Tracklet] + for track in self.tracked_tracklets: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_tracklets.append(track) + + ''' Step 2: First association, with appearance''' + tracklet_pool = joint_tracklets(tracked_tracklets, self.lost_tracklets) + + # Predict the current location with Kalman + for tracklet in tracklet_pool: + tracklet.predict() + + # vallina matching + cost_matrix = self.gated_metric(tracklet_pool, detections) + matches, u_track, u_detection = linear_assignment(cost_matrix, thresh=0.9) + + for itracked, idet in matches: + track = tracklet_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(detections[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + '''Step 3: Second association, with iou''' + tracklet_for_iou = [tracklet_pool[i] for i in u_track if tracklet_pool[i].state == TrackState.Tracked] + detection_for_iou = [detections[i] for i in u_detection] + + dists = iou_distance(tracklet_for_iou, detection_for_iou) + + matches, u_track, u_detection = linear_assignment(dists, thresh=0.5) + + for itracked, idet in matches: + track = tracklet_for_iou[itracked] + det = detection_for_iou[idet] + if track.state == TrackState.Tracked: + track.update(detection_for_iou[idet], self.frame_id) + activated_tracklets.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_tracklets.append(track) + + for it in u_track: + track = tracklet_for_iou[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_tracklets.append(track) + + + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detection_for_iou[i] for i in u_detection] + dists = iou_distance(unconfirmed, detections) + + matches, u_unconfirmed, u_detection = linear_assignment(dists, thresh=0.7) + + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_tracklets.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_tracklets.append(track) + + """ Step 4: Init new tracklets""" + for inew in u_detection: + track = detections[inew] + if track.score < self.det_thresh: + continue + track.activate(self.frame_id) + activated_tracklets.append(track) + + """ Step 5: Update state""" + for track in self.lost_tracklets: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_tracklets.append(track) + + # print('Ramained match {} s'.format(t4-t3)) + + self.tracked_tracklets = [t for t in self.tracked_tracklets if t.state == TrackState.Tracked] + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, activated_tracklets) + self.tracked_tracklets = joint_tracklets(self.tracked_tracklets, refind_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.tracked_tracklets) + self.lost_tracklets.extend(lost_tracklets) + self.lost_tracklets = sub_tracklets(self.lost_tracklets, self.removed_tracklets) + self.removed_tracklets.extend(removed_tracklets) + self.tracked_tracklets, self.lost_tracklets = remove_duplicate_tracklets(self.tracked_tracklets, self.lost_tracklets) + # get scores of lost tracks + output_tracklets = [track for track in self.tracked_tracklets if track.is_activated] + + return output_tracklets + + def gated_metric(self, tracks, dets): + """ + get cost matrix, firstly calculate apperence cost, then filter by Kalman state. + + tracks: List[STrack] + dets: List[STrack] + """ + apperance_dist = embedding_distance(tracks=tracks, detections=dets, metric='cosine') + cost_matrix = self.gate_cost_matrix(apperance_dist, tracks, dets, ) + return cost_matrix + + def gate_cost_matrix(self, cost_matrix, tracks, dets, max_apperance_thresh=0.15, gated_cost=1e5, only_position=False): + """ + gate cost matrix by calculating the Kalman state distance and constrainted by + 0.95 confidence interval of x2 distribution + + cost_matrix: np.ndarray, shape (len(tracks), len(dets)) + tracks: List[STrack] + dets: List[STrack] + gated_cost: a very largt const to infeasible associations + only_position: use [xc, yc, a, h] as state vector or only use [xc, yc] + + return: + updated cost_matirx, np.ndarray + """ + gating_dim = 2 if only_position else 4 + gating_threshold = chi2inv95[gating_dim] + measurements = np.asarray([Tracklet.tlwh_to_xyah(det.tlwh) for det in dets]) # (len(dets), 4) + + cost_matrix[cost_matrix > max_apperance_thresh] = gated_cost + for row, track in enumerate(tracks): + gating_distance = track.kalman_filter.gating_distance(measurements, ) + cost_matrix[row, gating_distance > gating_threshold] = gated_cost + + cost_matrix[row] = self.lambda_ * cost_matrix[row] + (1 - self.lambda_) * gating_distance + return cost_matrix + + +def joint_tracklets(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + +def sub_tracklets(tlista, tlistb): + tracklets = {} + for t in tlista: + tracklets[t.track_id] = t + for t in tlistb: + tid = t.track_id + if tracklets.get(tid, 0): + del tracklets[tid] + return list(tracklets.values()) + + +def remove_duplicate_tracklets(trackletsa, trackletsb): + pdist = iou_distance(trackletsa, trackletsb) + pairs = np.where(pdist < 0.15) + dupa, dupb = list(), list() + for p, q in zip(*pairs): + timep = trackletsa[p].frame_id - trackletsa[p].start_frame + timeq = trackletsb[q].frame_id - trackletsb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(trackletsa) if not i in dupa] + resb = [t for i, t in enumerate(trackletsb) if not i in dupb] + return resa, resb \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackers/tracklet.py b/test/yolov7-tracker/tracker/trackers/tracklet.py new file mode 100644 index 0000000..626c18e --- /dev/null +++ b/test/yolov7-tracker/tracker/trackers/tracklet.py @@ -0,0 +1,366 @@ +""" +implements base elements of trajectory +""" + +import numpy as np +from collections import deque + +from .basetrack import BaseTrack, TrackState +from .kalman_filters.bytetrack_kalman import ByteKalman +from .kalman_filters.botsort_kalman import BotKalman +from .kalman_filters.ocsort_kalman import OCSORTKalman +from .kalman_filters.sort_kalman import SORTKalman +from .kalman_filters.strongsort_kalman import NSAKalman + +MOTION_MODEL_DICT = { + 'sort': SORTKalman, + 'byte': ByteKalman, + 'bot': BotKalman, + 'ocsort': OCSORTKalman, + 'strongsort': NSAKalman, +} + +STATE_CONVERT_DICT = { + 'sort': 'xysa', + 'byte': 'xyah', + 'bot': 'xywh', + 'ocsort': 'xysa', + 'strongsort': 'xyah' +} + +class Tracklet(BaseTrack): + def __init__(self, tlwh, score, category, motion='byte'): + + # initial position + self._tlwh = np.asarray(tlwh, dtype=np.float) + self.is_activated = False + + self.score = score + self.category = category + + # kalman + self.motion = motion + self.kalman_filter = MOTION_MODEL_DICT[motion]() + + self.convert_func = self.__getattribute__('tlwh_to_' + STATE_CONVERT_DICT[motion]) + + # init kalman + self.kalman_filter.initialize(self.convert_func(self._tlwh)) + + def predict(self): + self.kalman_filter.predict() + self.time_since_update += 1 + + def activate(self, frame_id): + self.track_id = self.next_id() + + self.state = TrackState.Tracked + if frame_id == 1: + self.is_activated = True + self.frame_id = frame_id + self.start_frame = frame_id + + + def re_activate(self, new_track, frame_id, new_id=False): + + # TODO different convert + self.kalman_filter.update(self.convert_func(new_track.tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + self.score = new_track.score + + def update(self, new_track, frame_id): + self.frame_id = frame_id + + new_tlwh = new_track.tlwh + self.score = new_track.score + + self.kalman_filter.update(self.convert_func(new_tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + + self.time_since_update = 0 + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + return self.__getattribute__(STATE_CONVERT_DICT[self.motion] + '_to_tlwh')() + + def xyah_to_tlwh(self, ): + x = self.kalman_filter.kf.x + ret = x[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:] / 2 + return ret + + def xywh_to_tlwh(self, ): + x = self.kalman_filter.kf.x + ret = x[:4].copy() + ret[:2] -= ret[2:] / 2 + return ret + + def xysa_to_tlwh(self, ): + x = self.kalman_filter.kf.x + ret = x[:4].copy() + ret[2] = np.sqrt(x[2] * x[3]) + ret[3] = x[2] / ret[2] + + ret[:2] -= ret[2:] / 2 + return ret + + +class Tracklet_w_reid(Tracklet): + """ + Tracklet class with reid features, for botsort, deepsort, etc. + """ + + def __init__(self, tlwh, score, category, motion='byte', + feat=None, feat_history=50): + super().__init__(tlwh, score, category, motion) + + self.smooth_feat = None # EMA feature + self.curr_feat = None # current feature + self.features = deque([], maxlen=feat_history) # all features + if feat is not None: + self.update_features(feat) + + self.alpha = 0.9 + + def update_features(self, feat): + feat /= np.linalg.norm(feat) + self.curr_feat = feat + if self.smooth_feat is None: + self.smooth_feat = feat + else: + self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat + self.features.append(feat) + self.smooth_feat /= np.linalg.norm(self.smooth_feat) + + def re_activate(self, new_track, frame_id, new_id=False): + + # TODO different convert + if isinstance(self.kalman_filter, NSAKalman): + self.kalman_filter.update(self.convert_func(new_track.tlwh), new_track.score) + else: + self.kalman_filter.update(self.convert_func(new_track.tlwh)) + + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + self.score = new_track.score + + def update(self, new_track, frame_id): + self.frame_id = frame_id + + new_tlwh = new_track.tlwh + self.score = new_track.score + + if isinstance(self.kalman_filter, NSAKalman): + self.kalman_filter.update(self.convert_func(new_tlwh), self.score) + else: + self.kalman_filter.update(self.convert_func(new_tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + + + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + + self.time_since_update = 0 + + +class Tracklet_w_velocity(Tracklet): + """ + Tracklet class with reid features, for ocsort. + """ + + def __init__(self, tlwh, score, category, motion='byte', delta_t=3): + super().__init__(tlwh, score, category, motion) + + self.last_observation = np.array([-1, -1, -1, -1, -1]) # placeholder + self.observations = dict() + self.history_observations = [] + self.velocity = None + self.delta_t = delta_t + + self.age = 0 # mark the age + + @staticmethod + def speed_direction(bbox1, bbox2): + cx1, cy1 = (bbox1[0] + bbox1[2]) / 2.0, (bbox1[1] + bbox1[3]) / 2.0 + cx2, cy2 = (bbox2[0] + bbox2[2]) / 2.0, (bbox2[1] + bbox2[3]) / 2.0 + speed = np.array([cy2 - cy1, cx2 - cx1]) + norm = np.sqrt((cy2 - cy1)**2 + (cx2 - cx1)**2) + 1e-6 + return speed / norm + + def predict(self): + self.kalman_filter.predict() + + self.age += 1 + self.time_since_update += 1 + + def update(self, new_track, frame_id): + self.frame_id = frame_id + + new_tlwh = new_track.tlwh + self.score = new_track.score + + self.kalman_filter.update(self.convert_func(new_tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + self.time_since_update = 0 + + # update velocity and history buffer + new_tlbr = Tracklet_w_bbox_buffer.tlwh_to_tlbr(new_tlwh) + + if self.last_observation.sum() >= 0: # no previous observation + previous_box = None + for i in range(self.delta_t): + dt = self.delta_t - i + if self.age - dt in self.observations: + previous_box = self.observations[self.age-dt] + break + if previous_box is None: + previous_box = self.last_observation + """ + Estimate the track speed direction with observations \Delta t steps away + """ + self.velocity = self.speed_direction(previous_box, new_tlbr) + + new_observation = np.r_[new_tlbr, new_track.score] + self.last_observation = new_observation + self.observations[self.age] = new_observation + self.history_observations.append(new_observation) + + + + +class Tracklet_w_bbox_buffer(Tracklet): + """ + Tracklet class with buffer of bbox, for C_BIoU track. + """ + def __init__(self, tlwh, score, category, motion='byte'): + super().__init__(tlwh, score, category, motion) + + # params in motion state + self.b1, self.b2, self.n = 0.3, 0.5, 5 + self.origin_bbox_buffer = deque() # a deque store the original bbox(tlwh) from t - self.n to t, where t is the last time detected + self.origin_bbox_buffer.append(self._tlwh) + # buffered bbox, two buffer sizes + self.buffer_bbox1 = self.get_buffer_bbox(level=1) + self.buffer_bbox2 = self.get_buffer_bbox(level=2) + # motion state, s^{t + \delta} = o^t + (\delta / n) * \sum_{i=t-n+1}^t(o^i - o^{i-1}) = o^t + (\delta / n) * (o^t - o^{t - n}) + self.motion_state1 = self.buffer_bbox1.copy() + self.motion_state2 = self.buffer_bbox2.copy() + + def get_buffer_bbox(self, level=1, bbox=None): + """ + get buffered bbox as: (top, left, w, h) -> (top - bw, y - bh, w + 2bw, h + 2bh) + level = 1: b = self.b1 level = 2: b = self.b2 + bbox: if not None, use bbox to calculate buffer_bbox, else use self._tlwh + """ + assert level in [1, 2], 'level must be 1 or 2' + + b = self.b1 if level == 1 else self.b2 + + if bbox is None: + buffer_bbox = self._tlwh + np.array([-b*self._tlwh[2], -b*self._tlwh[3], 2*b*self._tlwh[2], 2*b*self._tlwh[3]]) + else: + buffer_bbox = bbox + np.array([-b*bbox[2], -b*bbox[3], 2*b*bbox[2], 2*b*bbox[3]]) + return np.maximum(0.0, buffer_bbox) + + def re_activate(self, new_track, frame_id, new_id=False): + + # TODO different convert + self.kalman_filter.update(self.convert_func(new_track.tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + self.score = new_track.score + + self._tlwh = new_track._tlwh + # update stored bbox + if (len(self.origin_bbox_buffer) > self.n): + self.origin_bbox_buffer.popleft() + self.origin_bbox_buffer.append(self._tlwh) + else: + self.origin_bbox_buffer.append(self._tlwh) + + self.buffer_bbox1 = self.get_buffer_bbox(level=1) + self.buffer_bbox2 = self.get_buffer_bbox(level=2) + self.motion_state1 = self.buffer_bbox1.copy() + self.motion_state2 = self.buffer_bbox2.copy() + + def update(self, new_track, frame_id): + self.frame_id = frame_id + + new_tlwh = new_track.tlwh + self.score = new_track.score + + self.kalman_filter.update(self.convert_func(new_tlwh)) + + self.state = TrackState.Tracked + self.is_activated = True + + self.time_since_update = 0 + + # update stored bbox + if (len(self.origin_bbox_buffer) > self.n): + self.origin_bbox_buffer.popleft() + self.origin_bbox_buffer.append(new_tlwh) + else: + self.origin_bbox_buffer.append(new_tlwh) + + # update motion state + if self.time_since_update: # have some unmatched frames + if len(self.origin_bbox_buffer) < self.n: + self.motion_state1 = self.get_buffer_bbox(level=1, bbox=new_tlwh) + self.motion_state2 = self.get_buffer_bbox(level=2, bbox=new_tlwh) + else: # s^{t + \delta} = o^t + (\delta / n) * (o^t - o^{t - n}) + motion_state = self.origin_bbox_buffer[-1] + \ + (self.time_since_update / self.n) * (self.origin_bbox_buffer[-1] - self.origin_bbox_buffer[0]) + self.motion_state1 = self.get_buffer_bbox(level=1, bbox=motion_state) + self.motion_state2 = self.get_buffer_bbox(level=2, bbox=motion_state) + + else: # no unmatched frames, use current detection as motion state + self.motion_state1 = self.get_buffer_bbox(level=1, bbox=new_tlwh) + self.motion_state2 = self.get_buffer_bbox(level=2, bbox=new_tlwh) + + +class Tracklet_w_depth(Tracklet): + """ + tracklet with depth info (i.e., 2000 - y2), for SparseTrack + """ + + def __init__(self, tlwh, score, category, motion='byte'): + super().__init__(tlwh, score, category, motion) + + + @property + # @jit(nopython=True) + def deep_vec(self): + """Convert bounding box to format `((top left, bottom right)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + cx = ret[0] + 0.5 * ret[2] + y2 = ret[1] + ret[3] + lendth = 2000 - y2 + return np.asarray([cx, y2, lendth], dtype=np.float) \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/__init__.py b/test/yolov7-tracker/tracker/trackeval/__init__.py new file mode 100644 index 0000000..dce62da --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/__init__.py @@ -0,0 +1,5 @@ +from .eval import Evaluator +from . import datasets +from . import metrics +from . import plotting +from . import utils diff --git a/test/yolov7-tracker/tracker/trackeval/_timing.py b/test/yolov7-tracker/tracker/trackeval/_timing.py new file mode 100644 index 0000000..4614ba3 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/_timing.py @@ -0,0 +1,65 @@ +from functools import wraps +from time import perf_counter +import inspect + +DO_TIMING = False +DISPLAY_LESS_PROGRESS = False +timer_dict = {} +counter = 0 + + +def time(f): + @wraps(f) + def wrap(*args, **kw): + if DO_TIMING: + # Run function with timing + ts = perf_counter() + result = f(*args, **kw) + te = perf_counter() + tt = te-ts + + # Get function name + arg_names = inspect.getfullargspec(f)[0] + if arg_names[0] == 'self' and DISPLAY_LESS_PROGRESS: + return result + elif arg_names[0] == 'self': + method_name = type(args[0]).__name__ + '.' + f.__name__ + else: + method_name = f.__name__ + + # Record accumulative time in each function for analysis + if method_name in timer_dict.keys(): + timer_dict[method_name] += tt + else: + timer_dict[method_name] = tt + + # If code is finished, display timing summary + if method_name == "Evaluator.evaluate": + print("") + print("Timing analysis:") + for key, value in timer_dict.items(): + print('%-70s %2.4f sec' % (key, value)) + else: + # Get function argument values for printing special arguments of interest + arg_titles = ['tracker', 'seq', 'cls'] + arg_vals = [] + for i, a in enumerate(arg_names): + if a in arg_titles: + arg_vals.append(args[i]) + arg_text = '(' + ', '.join(arg_vals) + ')' + + # Display methods and functions with different indentation. + if arg_names[0] == 'self': + print('%-74s %2.4f sec' % (' '*4 + method_name + arg_text, tt)) + elif arg_names[0] == 'test': + pass + else: + global counter + counter += 1 + print('%i %-70s %2.4f sec' % (counter, method_name + arg_text, tt)) + + return result + else: + # If config["TIME_PROGRESS"] is false, or config["USE_PARALLEL"] is true, run functions normally without timing. + return f(*args, **kw) + return wrap diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/__init__.py b/test/yolov7-tracker/tracker/trackeval/baselines/__init__.py new file mode 100644 index 0000000..ddc9864 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/__init__.py @@ -0,0 +1,6 @@ +import baseline_utils +import stp +import non_overlap +import pascal_colormap +import thresholder +import vizualize \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/baseline_utils.py b/test/yolov7-tracker/tracker/trackeval/baselines/baseline_utils.py new file mode 100644 index 0000000..b6c88fd --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/baseline_utils.py @@ -0,0 +1,321 @@ + +import os +import csv +import numpy as np +from copy import deepcopy +from PIL import Image +from pycocotools import mask as mask_utils +from scipy.optimize import linear_sum_assignment +from trackeval.baselines.pascal_colormap import pascal_colormap + + +def load_seq(file_to_load): + """ Load input data from file in RobMOTS format (e.g. provided detections). + Returns: Data object with the following structure (see STP : + data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + """ + fp = open(file_to_load) + dialect = csv.Sniffer().sniff(fp.readline(), delimiters=' ') + dialect.skipinitialspace = True + fp.seek(0) + reader = csv.reader(fp, dialect) + read_data = {} + num_timesteps = 0 + for i, row in enumerate(reader): + if row[-1] in '': + row = row[:-1] + t = int(row[0]) + cid = row[1] + c = int(row[2]) + s = row[3] + h = row[4] + w = row[5] + rle = row[6] + + if t >= num_timesteps: + num_timesteps = t + 1 + + if c in read_data.keys(): + if t in read_data[c].keys(): + read_data[c][t]['ids'].append(cid) + read_data[c][t]['scores'].append(s) + read_data[c][t]['im_hs'].append(h) + read_data[c][t]['im_ws'].append(w) + read_data[c][t]['mask_rles'].append(rle) + else: + read_data[c][t] = {} + read_data[c][t]['ids'] = [cid] + read_data[c][t]['scores'] = [s] + read_data[c][t]['im_hs'] = [h] + read_data[c][t]['im_ws'] = [w] + read_data[c][t]['mask_rles'] = [rle] + else: + read_data[c] = {t: {}} + read_data[c][t]['ids'] = [cid] + read_data[c][t]['scores'] = [s] + read_data[c][t]['im_hs'] = [h] + read_data[c][t]['im_ws'] = [w] + read_data[c][t]['mask_rles'] = [rle] + fp.close() + + data = {} + for c in read_data.keys(): + data[c] = [{} for _ in range(num_timesteps)] + for t in range(num_timesteps): + if t in read_data[c].keys(): + data[c][t]['ids'] = np.atleast_1d(read_data[c][t]['ids']).astype(int) + data[c][t]['scores'] = np.atleast_1d(read_data[c][t]['scores']).astype(float) + data[c][t]['im_hs'] = np.atleast_1d(read_data[c][t]['im_hs']).astype(int) + data[c][t]['im_ws'] = np.atleast_1d(read_data[c][t]['im_ws']).astype(int) + data[c][t]['mask_rles'] = np.atleast_1d(read_data[c][t]['mask_rles']).astype(str) + else: + data[c][t]['ids'] = np.empty(0).astype(int) + data[c][t]['scores'] = np.empty(0).astype(float) + data[c][t]['im_hs'] = np.empty(0).astype(int) + data[c][t]['im_ws'] = np.empty(0).astype(int) + data[c][t]['mask_rles'] = np.empty(0).astype(str) + return data + + +def threshold(tdata, thresh): + """ Removes detections below a certian threshold ('thresh') score. """ + new_data = {} + to_keep = tdata['scores'] > thresh + for field in ['ids', 'scores', 'im_hs', 'im_ws', 'mask_rles']: + new_data[field] = tdata[field][to_keep] + return new_data + + +def create_coco_mask(mask_rles, im_hs, im_ws): + """ Converts mask as rle text (+ height and width) to encoded version used by pycocotools. """ + coco_masks = [{'size': [h, w], 'counts': m.encode(encoding='UTF-8')} + for h, w, m in zip(im_hs, im_ws, mask_rles)] + return coco_masks + + +def mask_iou(mask_rles1, mask_rles2, im_hs, im_ws, do_ioa=0): + """ Calculate mask IoU between two masks. + Further allows 'intersection over area' instead of IoU (over the area of mask_rle1). + Allows either to pass in 1 boolean for do_ioa for all mask_rles2 or also one for each mask_rles2. + It is recommended that mask_rles1 is a detection and mask_rles2 is a groundtruth. + """ + coco_masks1 = create_coco_mask(mask_rles1, im_hs, im_ws) + coco_masks2 = create_coco_mask(mask_rles2, im_hs, im_ws) + + if not hasattr(do_ioa, "__len__"): + do_ioa = [do_ioa]*len(coco_masks2) + assert(len(coco_masks2) == len(do_ioa)) + if len(coco_masks1) == 0 or len(coco_masks2) == 0: + iou = np.zeros(len(coco_masks1), len(coco_masks2)) + else: + iou = mask_utils.iou(coco_masks1, coco_masks2, do_ioa) + return iou + + +def sort_by_score(t_data): + """ Sorts data by score """ + sort_index = np.argsort(t_data['scores'])[::-1] + for k in t_data.keys(): + t_data[k] = t_data[k][sort_index] + return t_data + + +def mask_NMS(t_data, nms_threshold=0.5, already_sorted=False): + """ Remove redundant masks by performing non-maximum suppression (NMS) """ + + # Sort by score + if not already_sorted: + t_data = sort_by_score(t_data) + + # Calculate the mask IoU between all detections in the timestep. + mask_ious_all = mask_iou(t_data['mask_rles'], t_data['mask_rles'], t_data['im_hs'], t_data['im_ws']) + + # Determine which masks NMS should remove + # (those overlapping greater than nms_threshold with another mask that has a higher score) + num_dets = len(t_data['mask_rles']) + to_remove = [False for _ in range(num_dets)] + for i in range(num_dets): + if not to_remove[i]: + for j in range(i + 1, num_dets): + if mask_ious_all[i, j] > nms_threshold: + to_remove[j] = True + + # Remove detections which should be removed + to_keep = np.logical_not(to_remove) + for k in t_data.keys(): + t_data[k] = t_data[k][to_keep] + + return t_data + + +def non_overlap(t_data, already_sorted=False): + """ Enforces masks to be non-overlapping in an image, does this by putting masks 'on top of one another', + such that higher score masks 'occlude' and thus remove parts of lower scoring masks. + + Help wanted: if anyone knows a way to do this WITHOUT converting the RLE to the np.array let me know, because that + would be MUCH more efficient. (I have tried, but haven't yet had success). + """ + + # Sort by score + if not already_sorted: + t_data = sort_by_score(t_data) + + # Get coco masks + coco_masks = create_coco_mask(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws']) + + # Create a single np.array to hold all of the non-overlapping mask + masks_array = np.zeros((t_data['im_hs'][0], t_data['im_ws'][0]), 'uint8') + + # Decode each mask into a np.array, and place it into the overall array for the whole frame. + # Since masks with the lowest score are placed first, they are 'partially overridden' by masks with a higher score + # if they overlap. + for i, mask in enumerate(coco_masks[::-1]): + masks_array[mask_utils.decode(mask).astype('bool')] = i + 1 + + # Encode the resulting np.array back into a set of coco_masks which are now non-overlapping. + num_dets = len(coco_masks) + for i, j in enumerate(range(1, num_dets + 1)[::-1]): + coco_masks[i] = mask_utils.encode(np.asfortranarray(masks_array == j, dtype=np.uint8)) + + # Convert from coco_mask back into our mask_rle format. + t_data['mask_rles'] = [m['counts'].decode("utf-8") for m in coco_masks] + + return t_data + + +def masks2boxes(mask_rles, im_hs, im_ws): + """ Extracts bounding boxes which surround a set of masks. """ + coco_masks = create_coco_mask(mask_rles, im_hs, im_ws) + boxes = np.array([mask_utils.toBbox(x) for x in coco_masks]) + if len(boxes) == 0: + boxes = np.empty((0, 4)) + return boxes + + +def box_iou(bboxes1, bboxes2, box_format='xywh', do_ioa=False, do_giou=False): + """ Calculates the IOU (intersection over union) between two arrays of boxes. + Allows variable box formats ('xywh' and 'x0y0x1y1'). + If do_ioa (intersection over area), then calculates the intersection over the area of boxes1 - this is commonly + used to determine if detections are within crowd ignore region. + If do_giou (generalized intersection over union, then calculates giou. + """ + if len(bboxes1) == 0 or len(bboxes2) == 0: + ious = np.zeros((len(bboxes1), len(bboxes2))) + return ious + if box_format in 'xywh': + # layout: (x0, y0, w, h) + bboxes1 = deepcopy(bboxes1) + bboxes2 = deepcopy(bboxes2) + + bboxes1[:, 2] = bboxes1[:, 0] + bboxes1[:, 2] + bboxes1[:, 3] = bboxes1[:, 1] + bboxes1[:, 3] + bboxes2[:, 2] = bboxes2[:, 0] + bboxes2[:, 2] + bboxes2[:, 3] = bboxes2[:, 1] + bboxes2[:, 3] + elif box_format not in 'x0y0x1y1': + raise (Exception('box_format %s is not implemented' % box_format)) + + # layout: (x0, y0, x1, y1) + min_ = np.minimum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :]) + max_ = np.maximum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :]) + intersection = np.maximum(min_[..., 2] - max_[..., 0], 0) * np.maximum(min_[..., 3] - max_[..., 1], 0) + area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) + + if do_ioa: + ioas = np.zeros_like(intersection) + valid_mask = area1 > 0 + np.finfo('float').eps + ioas[valid_mask, :] = intersection[valid_mask, :] / area1[valid_mask][:, np.newaxis] + + return ioas + else: + area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) + union = area1[:, np.newaxis] + area2[np.newaxis, :] - intersection + intersection[area1 <= 0 + np.finfo('float').eps, :] = 0 + intersection[:, area2 <= 0 + np.finfo('float').eps] = 0 + intersection[union <= 0 + np.finfo('float').eps] = 0 + union[union <= 0 + np.finfo('float').eps] = 1 + ious = intersection / union + + if do_giou: + enclosing_area = np.maximum(max_[..., 2] - min_[..., 0], 0) * np.maximum(max_[..., 3] - min_[..., 1], 0) + eps = 1e-7 + # giou + ious = ious - ((enclosing_area - union) / (enclosing_area + eps)) + + return ious + + +def match(match_scores): + match_rows, match_cols = linear_sum_assignment(-match_scores) + return match_rows, match_cols + + +def write_seq(output_data, out_file): + out_loc = os.path.dirname(out_file) + if not os.path.exists(out_loc): + os.makedirs(out_loc, exist_ok=True) + fp = open(out_file, 'w', newline='') + writer = csv.writer(fp, delimiter=' ') + for row in output_data: + writer.writerow(row) + fp.close() + + +def combine_classes(data): + """ Converts data from a class-separated to a class-combined format. + Input format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + Output format: data[t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles', 'cls'} + """ + output_data = [{} for _ in list(data.values())[0]] + for cls, cls_data in data.items(): + for timestep, t_data in enumerate(cls_data): + for k in t_data.keys(): + if k in output_data[timestep].keys(): + output_data[timestep][k] += list(t_data[k]) + else: + output_data[timestep][k] = list(t_data[k]) + if 'cls' in output_data[timestep].keys(): + output_data[timestep]['cls'] += [cls]*len(output_data[timestep]['ids']) + else: + output_data[timestep]['cls'] = [cls]*len(output_data[timestep]['ids']) + + for timestep, t_data in enumerate(output_data): + for k in t_data.keys(): + output_data[timestep][k] = np.array(output_data[timestep][k]) + + return output_data + + +def save_as_png(t_data, out_file, im_h, im_w): + """ Save a set of segmentation masks into a PNG format, the same as used for the DAVIS dataset.""" + + if len(t_data['mask_rles']) > 0: + coco_masks = create_coco_mask(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws']) + + list_of_np_masks = [mask_utils.decode(mask) for mask in coco_masks] + + png = np.zeros((t_data['im_hs'][0], t_data['im_ws'][0])) + for mask, c_id in zip(list_of_np_masks, t_data['ids']): + png[mask.astype("bool")] = c_id + 1 + else: + png = np.zeros((im_h, im_w)) + + if not os.path.exists(os.path.dirname(out_file)): + os.makedirs(os.path.dirname(out_file)) + + colmap = (np.array(pascal_colormap) * 255).round().astype("uint8") + palimage = Image.new('P', (16, 16)) + palimage.putpalette(colmap) + im = Image.fromarray(np.squeeze(png.astype("uint8"))) + im2 = im.quantize(palette=palimage) + im2.save(out_file) + + +def get_frame_size(data): + """ Gets frame height and width from data. """ + for cls, cls_data in data.items(): + for timestep, t_data in enumerate(cls_data): + if len(t_data['im_hs'] > 0): + im_h = t_data['im_hs'][0] + im_w = t_data['im_ws'][0] + return im_h, im_w + return None diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/non_overlap.py b/test/yolov7-tracker/tracker/trackeval/baselines/non_overlap.py new file mode 100644 index 0000000..43b131d --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/non_overlap.py @@ -0,0 +1,92 @@ +""" +Non-Overlap: Code to take in a set of raw detections and produce a set of non-overlapping detections from it. + +Author: Jonathon Luiten +""" + +import os +import sys +from multiprocessing.pool import Pool +from multiprocessing import freeze_support + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +from trackeval.baselines import baseline_utils as butils +from trackeval.utils import get_code_path + +code_path = get_code_path() +config = { + 'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/raw_supplied/data/'), + 'OUTPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'), + 'SPLIT': 'train', # valid: 'train', 'val', 'test'. + 'Benchmarks': None, # If None, all benchmarks in SPLIT. + + 'Num_Parallel_Cores': None, # If None, run without parallel. + + 'THRESHOLD_NMS_MASK_IOU': 0.5, +} + + +def do_sequence(seq_file): + + # Load input data from file (e.g. provided detections) + # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + data = butils.load_seq(seq_file) + + # Converts data from a class-separated to a class-combined format. + # data[t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles', 'cls'} + data = butils.combine_classes(data) + + # Where to accumulate output data for writing out + output_data = [] + + # Run for each timestep. + for timestep, t_data in enumerate(data): + + # Remove redundant masks by performing non-maximum suppression (NMS) + t_data = butils.mask_NMS(t_data, nms_threshold=config['THRESHOLD_NMS_MASK_IOU']) + + # Perform non-overlap, to get non_overlapping masks. + t_data = butils.non_overlap(t_data, already_sorted=True) + + # Save result in output format to write to file later. + # Output Format = [timestep ID class score im_h im_w mask_RLE] + for i in range(len(t_data['ids'])): + row = [timestep, int(t_data['ids'][i]), t_data['cls'][i], t_data['scores'][i], t_data['im_hs'][i], + t_data['im_ws'][i], t_data['mask_rles'][i]] + output_data.append(row) + + # Write results to file + out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']), + config['OUTPUT_FOL'].format(split=config['SPLIT'])) + butils.write_seq(output_data, out_file) + + print('DONE:', seq_file) + + +if __name__ == '__main__': + + # Required to fix bug in multiprocessing on windows. + freeze_support() + + # Obtain list of sequences to run tracker for. + if config['Benchmarks']: + benchmarks = config['Benchmarks'] + else: + benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao'] + if config['SPLIT'] != 'train': + benchmarks += ['waymo', 'mots_challenge'] + seqs_todo = [] + for bench in benchmarks: + bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench) + seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)] + + # Run in parallel + if config['Num_Parallel_Cores']: + with Pool(config['Num_Parallel_Cores']) as pool: + results = pool.map(do_sequence, seqs_todo) + + # Run in series + else: + for seq_todo in seqs_todo: + do_sequence(seq_todo) + diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/pascal_colormap.py b/test/yolov7-tracker/tracker/trackeval/baselines/pascal_colormap.py new file mode 100644 index 0000000..b31f348 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/pascal_colormap.py @@ -0,0 +1,257 @@ +pascal_colormap = [ + 0 , 0, 0, + 0.5020, 0, 0, + 0, 0.5020, 0, + 0.5020, 0.5020, 0, + 0, 0, 0.5020, + 0.5020, 0, 0.5020, + 0, 0.5020, 0.5020, + 0.5020, 0.5020, 0.5020, + 0.2510, 0, 0, + 0.7529, 0, 0, + 0.2510, 0.5020, 0, + 0.7529, 0.5020, 0, + 0.2510, 0, 0.5020, + 0.7529, 0, 0.5020, + 0.2510, 0.5020, 0.5020, + 0.7529, 0.5020, 0.5020, + 0, 0.2510, 0, + 0.5020, 0.2510, 0, + 0, 0.7529, 0, + 0.5020, 0.7529, 0, + 0, 0.2510, 0.5020, + 0.5020, 0.2510, 0.5020, + 0, 0.7529, 0.5020, + 0.5020, 0.7529, 0.5020, + 0.2510, 0.2510, 0, + 0.7529, 0.2510, 0, + 0.2510, 0.7529, 0, + 0.7529, 0.7529, 0, + 0.2510, 0.2510, 0.5020, + 0.7529, 0.2510, 0.5020, + 0.2510, 0.7529, 0.5020, + 0.7529, 0.7529, 0.5020, + 0, 0, 0.2510, + 0.5020, 0, 0.2510, + 0, 0.5020, 0.2510, + 0.5020, 0.5020, 0.2510, + 0, 0, 0.7529, + 0.5020, 0, 0.7529, + 0, 0.5020, 0.7529, + 0.5020, 0.5020, 0.7529, + 0.2510, 0, 0.2510, + 0.7529, 0, 0.2510, + 0.2510, 0.5020, 0.2510, + 0.7529, 0.5020, 0.2510, + 0.2510, 0, 0.7529, + 0.7529, 0, 0.7529, + 0.2510, 0.5020, 0.7529, + 0.7529, 0.5020, 0.7529, + 0, 0.2510, 0.2510, + 0.5020, 0.2510, 0.2510, + 0, 0.7529, 0.2510, + 0.5020, 0.7529, 0.2510, + 0, 0.2510, 0.7529, + 0.5020, 0.2510, 0.7529, + 0, 0.7529, 0.7529, + 0.5020, 0.7529, 0.7529, + 0.2510, 0.2510, 0.2510, + 0.7529, 0.2510, 0.2510, + 0.2510, 0.7529, 0.2510, + 0.7529, 0.7529, 0.2510, + 0.2510, 0.2510, 0.7529, + 0.7529, 0.2510, 0.7529, + 0.2510, 0.7529, 0.7529, + 0.7529, 0.7529, 0.7529, + 0.1255, 0, 0, + 0.6275, 0, 0, + 0.1255, 0.5020, 0, + 0.6275, 0.5020, 0, + 0.1255, 0, 0.5020, + 0.6275, 0, 0.5020, + 0.1255, 0.5020, 0.5020, + 0.6275, 0.5020, 0.5020, + 0.3765, 0, 0, + 0.8784, 0, 0, + 0.3765, 0.5020, 0, + 0.8784, 0.5020, 0, + 0.3765, 0, 0.5020, + 0.8784, 0, 0.5020, + 0.3765, 0.5020, 0.5020, + 0.8784, 0.5020, 0.5020, + 0.1255, 0.2510, 0, + 0.6275, 0.2510, 0, + 0.1255, 0.7529, 0, + 0.6275, 0.7529, 0, + 0.1255, 0.2510, 0.5020, + 0.6275, 0.2510, 0.5020, + 0.1255, 0.7529, 0.5020, + 0.6275, 0.7529, 0.5020, + 0.3765, 0.2510, 0, + 0.8784, 0.2510, 0, + 0.3765, 0.7529, 0, + 0.8784, 0.7529, 0, + 0.3765, 0.2510, 0.5020, + 0.8784, 0.2510, 0.5020, + 0.3765, 0.7529, 0.5020, + 0.8784, 0.7529, 0.5020, + 0.1255, 0, 0.2510, + 0.6275, 0, 0.2510, + 0.1255, 0.5020, 0.2510, + 0.6275, 0.5020, 0.2510, + 0.1255, 0, 0.7529, + 0.6275, 0, 0.7529, + 0.1255, 0.5020, 0.7529, + 0.6275, 0.5020, 0.7529, + 0.3765, 0, 0.2510, + 0.8784, 0, 0.2510, + 0.3765, 0.5020, 0.2510, + 0.8784, 0.5020, 0.2510, + 0.3765, 0, 0.7529, + 0.8784, 0, 0.7529, + 0.3765, 0.5020, 0.7529, + 0.8784, 0.5020, 0.7529, + 0.1255, 0.2510, 0.2510, + 0.6275, 0.2510, 0.2510, + 0.1255, 0.7529, 0.2510, + 0.6275, 0.7529, 0.2510, + 0.1255, 0.2510, 0.7529, + 0.6275, 0.2510, 0.7529, + 0.1255, 0.7529, 0.7529, + 0.6275, 0.7529, 0.7529, + 0.3765, 0.2510, 0.2510, + 0.8784, 0.2510, 0.2510, + 0.3765, 0.7529, 0.2510, + 0.8784, 0.7529, 0.2510, + 0.3765, 0.2510, 0.7529, + 0.8784, 0.2510, 0.7529, + 0.3765, 0.7529, 0.7529, + 0.8784, 0.7529, 0.7529, + 0, 0.1255, 0, + 0.5020, 0.1255, 0, + 0, 0.6275, 0, + 0.5020, 0.6275, 0, + 0, 0.1255, 0.5020, + 0.5020, 0.1255, 0.5020, + 0, 0.6275, 0.5020, + 0.5020, 0.6275, 0.5020, + 0.2510, 0.1255, 0, + 0.7529, 0.1255, 0, + 0.2510, 0.6275, 0, + 0.7529, 0.6275, 0, + 0.2510, 0.1255, 0.5020, + 0.7529, 0.1255, 0.5020, + 0.2510, 0.6275, 0.5020, + 0.7529, 0.6275, 0.5020, + 0, 0.3765, 0, + 0.5020, 0.3765, 0, + 0, 0.8784, 0, + 0.5020, 0.8784, 0, + 0, 0.3765, 0.5020, + 0.5020, 0.3765, 0.5020, + 0, 0.8784, 0.5020, + 0.5020, 0.8784, 0.5020, + 0.2510, 0.3765, 0, + 0.7529, 0.3765, 0, + 0.2510, 0.8784, 0, + 0.7529, 0.8784, 0, + 0.2510, 0.3765, 0.5020, + 0.7529, 0.3765, 0.5020, + 0.2510, 0.8784, 0.5020, + 0.7529, 0.8784, 0.5020, + 0, 0.1255, 0.2510, + 0.5020, 0.1255, 0.2510, + 0, 0.6275, 0.2510, + 0.5020, 0.6275, 0.2510, + 0, 0.1255, 0.7529, + 0.5020, 0.1255, 0.7529, + 0, 0.6275, 0.7529, + 0.5020, 0.6275, 0.7529, + 0.2510, 0.1255, 0.2510, + 0.7529, 0.1255, 0.2510, + 0.2510, 0.6275, 0.2510, + 0.7529, 0.6275, 0.2510, + 0.2510, 0.1255, 0.7529, + 0.7529, 0.1255, 0.7529, + 0.2510, 0.6275, 0.7529, + 0.7529, 0.6275, 0.7529, + 0, 0.3765, 0.2510, + 0.5020, 0.3765, 0.2510, + 0, 0.8784, 0.2510, + 0.5020, 0.8784, 0.2510, + 0, 0.3765, 0.7529, + 0.5020, 0.3765, 0.7529, + 0, 0.8784, 0.7529, + 0.5020, 0.8784, 0.7529, + 0.2510, 0.3765, 0.2510, + 0.7529, 0.3765, 0.2510, + 0.2510, 0.8784, 0.2510, + 0.7529, 0.8784, 0.2510, + 0.2510, 0.3765, 0.7529, + 0.7529, 0.3765, 0.7529, + 0.2510, 0.8784, 0.7529, + 0.7529, 0.8784, 0.7529, + 0.1255, 0.1255, 0, + 0.6275, 0.1255, 0, + 0.1255, 0.6275, 0, + 0.6275, 0.6275, 0, + 0.1255, 0.1255, 0.5020, + 0.6275, 0.1255, 0.5020, + 0.1255, 0.6275, 0.5020, + 0.6275, 0.6275, 0.5020, + 0.3765, 0.1255, 0, + 0.8784, 0.1255, 0, + 0.3765, 0.6275, 0, + 0.8784, 0.6275, 0, + 0.3765, 0.1255, 0.5020, + 0.8784, 0.1255, 0.5020, + 0.3765, 0.6275, 0.5020, + 0.8784, 0.6275, 0.5020, + 0.1255, 0.3765, 0, + 0.6275, 0.3765, 0, + 0.1255, 0.8784, 0, + 0.6275, 0.8784, 0, + 0.1255, 0.3765, 0.5020, + 0.6275, 0.3765, 0.5020, + 0.1255, 0.8784, 0.5020, + 0.6275, 0.8784, 0.5020, + 0.3765, 0.3765, 0, + 0.8784, 0.3765, 0, + 0.3765, 0.8784, 0, + 0.8784, 0.8784, 0, + 0.3765, 0.3765, 0.5020, + 0.8784, 0.3765, 0.5020, + 0.3765, 0.8784, 0.5020, + 0.8784, 0.8784, 0.5020, + 0.1255, 0.1255, 0.2510, + 0.6275, 0.1255, 0.2510, + 0.1255, 0.6275, 0.2510, + 0.6275, 0.6275, 0.2510, + 0.1255, 0.1255, 0.7529, + 0.6275, 0.1255, 0.7529, + 0.1255, 0.6275, 0.7529, + 0.6275, 0.6275, 0.7529, + 0.3765, 0.1255, 0.2510, + 0.8784, 0.1255, 0.2510, + 0.3765, 0.6275, 0.2510, + 0.8784, 0.6275, 0.2510, + 0.3765, 0.1255, 0.7529, + 0.8784, 0.1255, 0.7529, + 0.3765, 0.6275, 0.7529, + 0.8784, 0.6275, 0.7529, + 0.1255, 0.3765, 0.2510, + 0.6275, 0.3765, 0.2510, + 0.1255, 0.8784, 0.2510, + 0.6275, 0.8784, 0.2510, + 0.1255, 0.3765, 0.7529, + 0.6275, 0.3765, 0.7529, + 0.1255, 0.8784, 0.7529, + 0.6275, 0.8784, 0.7529, + 0.3765, 0.3765, 0.2510, + 0.8784, 0.3765, 0.2510, + 0.3765, 0.8784, 0.2510, + 0.8784, 0.8784, 0.2510, + 0.3765, 0.3765, 0.7529, + 0.8784, 0.3765, 0.7529, + 0.3765, 0.8784, 0.7529, + 0.8784, 0.8784, 0.7529] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/stp.py b/test/yolov7-tracker/tracker/trackeval/baselines/stp.py new file mode 100644 index 0000000..c1c9d1e --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/stp.py @@ -0,0 +1,144 @@ +""" +STP: Simplest Tracker Possible + +Author: Jonathon Luiten + +This simple tracker, simply assigns track IDs which maximise the 'bounding box IoU' between previous tracks and current +detections. It is also able to match detections to tracks at more than one timestep previously. +""" + +import os +import sys +import numpy as np +from multiprocessing.pool import Pool +from multiprocessing import freeze_support + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +from trackeval.baselines import baseline_utils as butils +from trackeval.utils import get_code_path + +code_path = get_code_path() +config = { + 'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'), + 'OUTPUT_FOL': os.path.join(code_path, 'data/trackers/rob_mots/{split}/STP/data/'), + 'SPLIT': 'train', # valid: 'train', 'val', 'test'. + 'Benchmarks': None, # If None, all benchmarks in SPLIT. + + 'Num_Parallel_Cores': None, # If None, run without parallel. + + 'DETECTION_THRESHOLD': 0.5, + 'ASSOCIATION_THRESHOLD': 1e-10, + 'MAX_FRAMES_SKIP': 7 +} + + +def track_sequence(seq_file): + + # Load input data from file (e.g. provided detections) + # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + data = butils.load_seq(seq_file) + + # Where to accumulate output data for writing out + output_data = [] + + # To ensure IDs are unique per object across all classes. + curr_max_id = 0 + + # Run tracker for each class. + for cls, cls_data in data.items(): + + # Initialize container for holding previously tracked objects. + prev = {'boxes': np.empty((0, 4)), + 'ids': np.array([], np.int), + 'timesteps': np.array([])} + + # Run tracker for each timestep. + for timestep, t_data in enumerate(cls_data): + + # Threshold detections. + t_data = butils.threshold(t_data, config['DETECTION_THRESHOLD']) + + # Convert mask dets to bounding boxes. + boxes = butils.masks2boxes(t_data['mask_rles'], t_data['im_hs'], t_data['im_ws']) + + # Calculate IoU between previous and current frame dets. + ious = butils.box_iou(prev['boxes'], boxes) + + # Score which decreases quickly for previous dets depending on how many timesteps before they come from. + prev_timestep_scores = np.power(10, -1 * prev['timesteps']) + + # Matching score is such that it first tries to match 'most recent timesteps', + # and within each timestep maximised IoU. + match_scores = prev_timestep_scores[:, np.newaxis] * ious + + # Find best matching between current dets and previous tracks. + match_rows, match_cols = butils.match(match_scores) + + # Remove matches that have an IoU below a certain threshold. + actually_matched_mask = ious[match_rows, match_cols] > config['ASSOCIATION_THRESHOLD'] + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + # Assign the prev track ID to the current dets if they were matched. + ids = np.nan * np.ones((len(boxes),), np.int) + ids[match_cols] = prev['ids'][match_rows] + + # Create new track IDs for dets that were not matched to previous tracks. + num_not_matched = len(ids) - len(match_cols) + new_ids = np.arange(curr_max_id + 1, curr_max_id + num_not_matched + 1) + ids[np.isnan(ids)] = new_ids + + # Update maximum ID to ensure future added tracks have a unique ID value. + curr_max_id += num_not_matched + + # Drop tracks from 'previous tracks' if they have not been matched in the last MAX_FRAMES_SKIP frames. + unmatched_rows = [i for i in range(len(prev['ids'])) if + i not in match_rows and (prev['timesteps'][i] + 1 <= config['MAX_FRAMES_SKIP'])] + + # Update the set of previous tracking results to include the newly tracked detections. + prev['ids'] = np.concatenate((ids, prev['ids'][unmatched_rows]), axis=0) + prev['boxes'] = np.concatenate((np.atleast_2d(boxes), np.atleast_2d(prev['boxes'][unmatched_rows])), axis=0) + prev['timesteps'] = np.concatenate((np.zeros((len(ids),)), prev['timesteps'][unmatched_rows] + 1), axis=0) + + # Save result in output format to write to file later. + # Output Format = [timestep ID class score im_h im_w mask_RLE] + for i in range(len(t_data['ids'])): + row = [timestep, int(ids[i]), cls, t_data['scores'][i], t_data['im_hs'][i], t_data['im_ws'][i], + t_data['mask_rles'][i]] + output_data.append(row) + + # Write results to file + out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']), + config['OUTPUT_FOL'].format(split=config['SPLIT'])) + butils.write_seq(output_data, out_file) + + print('DONE:', seq_file) + + +if __name__ == '__main__': + + # Required to fix bug in multiprocessing on windows. + freeze_support() + + # Obtain list of sequences to run tracker for. + if config['Benchmarks']: + benchmarks = config['Benchmarks'] + else: + benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao'] + if config['SPLIT'] != 'train': + benchmarks += ['waymo', 'mots_challenge'] + seqs_todo = [] + for bench in benchmarks: + bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench) + seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)] + + # Run in parallel + if config['Num_Parallel_Cores']: + with Pool(config['Num_Parallel_Cores']) as pool: + results = pool.map(track_sequence, seqs_todo) + + # Run in series + else: + for seq_todo in seqs_todo: + track_sequence(seq_todo) + diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/thresholder.py b/test/yolov7-tracker/tracker/trackeval/baselines/thresholder.py new file mode 100644 index 0000000..c589e10 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/thresholder.py @@ -0,0 +1,92 @@ +""" +Thresholder + +Author: Jonathon Luiten + +Simply reads in a set of detection, thresholds them at a certain score threshold, and writes them out again. +""" + +import os +import sys +from multiprocessing.pool import Pool +from multiprocessing import freeze_support + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +from trackeval.baselines import baseline_utils as butils +from trackeval.utils import get_code_path + +THRESHOLD = 0.2 + +code_path = get_code_path() +config = { + 'INPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/non_overlap_supplied/data/'), + 'OUTPUT_FOL': os.path.join(code_path, 'data/detections/rob_mots/{split}/threshold_' + str(100*THRESHOLD) + '/data/'), + 'SPLIT': 'train', # valid: 'train', 'val', 'test'. + 'Benchmarks': None, # If None, all benchmarks in SPLIT. + + 'Num_Parallel_Cores': None, # If None, run without parallel. + + 'DETECTION_THRESHOLD': THRESHOLD, +} + + +def do_sequence(seq_file): + + # Load input data from file (e.g. provided detections) + # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + data = butils.load_seq(seq_file) + + # Where to accumulate output data for writing out + output_data = [] + + # Run for each class. + for cls, cls_data in data.items(): + + # Run for each timestep. + for timestep, t_data in enumerate(cls_data): + + # Threshold detections. + t_data = butils.threshold(t_data, config['DETECTION_THRESHOLD']) + + # Save result in output format to write to file later. + # Output Format = [timestep ID class score im_h im_w mask_RLE] + for i in range(len(t_data['ids'])): + row = [timestep, int(t_data['ids'][i]), cls, t_data['scores'][i], t_data['im_hs'][i], + t_data['im_ws'][i], t_data['mask_rles'][i]] + output_data.append(row) + + # Write results to file + out_file = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT']), + config['OUTPUT_FOL'].format(split=config['SPLIT'])) + butils.write_seq(output_data, out_file) + + print('DONE:', seq_todo) + + +if __name__ == '__main__': + + # Required to fix bug in multiprocessing on windows. + freeze_support() + + # Obtain list of sequences to run tracker for. + if config['Benchmarks']: + benchmarks = config['Benchmarks'] + else: + benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao'] + if config['SPLIT'] != 'train': + benchmarks += ['waymo', 'mots_challenge'] + seqs_todo = [] + for bench in benchmarks: + bench_fol = os.path.join(config['INPUT_FOL'].format(split=config['SPLIT']), bench) + seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)] + + # Run in parallel + if config['Num_Parallel_Cores']: + with Pool(config['Num_Parallel_Cores']) as pool: + results = pool.map(do_sequence, seqs_todo) + + # Run in series + else: + for seq_todo in seqs_todo: + do_sequence(seq_todo) + diff --git a/test/yolov7-tracker/tracker/trackeval/baselines/vizualize.py b/test/yolov7-tracker/tracker/trackeval/baselines/vizualize.py new file mode 100644 index 0000000..568a303 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/baselines/vizualize.py @@ -0,0 +1,94 @@ +""" +Vizualize: Code which converts .txt rle tracking results into a visual .png format. + +Author: Jonathon Luiten +""" + +import os +import sys +from multiprocessing.pool import Pool +from multiprocessing import freeze_support + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) +from trackeval.baselines import baseline_utils as butils +from trackeval.utils import get_code_path +from trackeval.datasets.rob_mots_classmap import cls_id_to_name + +code_path = get_code_path() +config = { + # Tracker format: + 'INPUT_FOL': os.path.join(code_path, 'data/trackers/rob_mots/{split}/STP/data/{bench}'), + 'OUTPUT_FOL': os.path.join(code_path, 'data/viz/rob_mots/{split}/STP/data/{bench}'), + # GT format: + # 'INPUT_FOL': os.path.join(code_path, 'data/gt/rob_mots/{split}/{bench}/data/'), + # 'OUTPUT_FOL': os.path.join(code_path, 'data/gt_viz/rob_mots/{split}/{bench}/'), + 'SPLIT': 'train', # valid: 'train', 'val', 'test'. + 'Benchmarks': None, # If None, all benchmarks in SPLIT. + 'Num_Parallel_Cores': None, # If None, run without parallel. +} + + +def do_sequence(seq_file): + # Folder to save resulting visualization in + out_fol = seq_file.replace(config['INPUT_FOL'].format(split=config['SPLIT'], bench=bench), + config['OUTPUT_FOL'].format(split=config['SPLIT'], bench=bench)).replace('.txt', '') + + # Load input data from file (e.g. provided detections) + # data format: data['cls'][t] = {'ids', 'scores', 'im_hs', 'im_ws', 'mask_rles'} + data = butils.load_seq(seq_file) + + # Get frame size for visualizing empty frames + im_h, im_w = butils.get_frame_size(data) + + # First run for each class. + for cls, cls_data in data.items(): + + if cls >= 100: + continue + + # Run for each timestep. + for timestep, t_data in enumerate(cls_data): + # Save out visualization + out_file = os.path.join(out_fol, cls_id_to_name[cls], str(timestep).zfill(5) + '.png') + butils.save_as_png(t_data, out_file, im_h, im_w) + + + # Then run for all classes combined + # Converts data from a class-separated to a class-combined format. + data = butils.combine_classes(data) + + # Run for each timestep. + for timestep, t_data in enumerate(data): + # Save out visualization + out_file = os.path.join(out_fol, 'all_classes', str(timestep).zfill(5) + '.png') + butils.save_as_png(t_data, out_file, im_h, im_w) + + print('DONE:', seq_file) + + +if __name__ == '__main__': + + # Required to fix bug in multiprocessing on windows. + freeze_support() + + # Obtain list of sequences to run tracker for. + if config['Benchmarks']: + benchmarks = config['Benchmarks'] + else: + benchmarks = ['davis_unsupervised', 'kitti_mots', 'youtube_vis', 'ovis', 'bdd_mots', 'tao'] + if config['SPLIT'] != 'train': + benchmarks += ['waymo', 'mots_challenge'] + seqs_todo = [] + for bench in benchmarks: + bench_fol = config['INPUT_FOL'].format(split=config['SPLIT'], bench=bench) + seqs_todo += [os.path.join(bench_fol, seq) for seq in os.listdir(bench_fol)] + + # Run in parallel + if config['Num_Parallel_Cores']: + with Pool(config['Num_Parallel_Cores']) as pool: + results = pool.map(do_sequence, seqs_todo) + + # Run in series + else: + for seq_todo in seqs_todo: + do_sequence(seq_todo) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/__init__.py b/test/yolov7-tracker/tracker/trackeval/datasets/__init__.py new file mode 100644 index 0000000..64e4f8d --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/__init__.py @@ -0,0 +1,15 @@ +from .kitti_2d_box import Kitti2DBox +from .kitti_mots import KittiMOTS +from .mot_challenge_2d_box import MotChallenge2DBox +from .mots_challenge import MOTSChallenge +from .bdd100k import BDD100K +from .davis import DAVIS +from .tao import TAO +from .tao_ow import TAO_OW +from .burst import BURST +from .burst_ow import BURST_OW +from .youtube_vis import YouTubeVIS +from .head_tracking_challenge import HeadTrackingChallenge +from .rob_mots import RobMOTS +from .person_path_22 import PersonPath22 +from .visdrone import VisDrone2DBox diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/_base_dataset.py b/test/yolov7-tracker/tracker/trackeval/datasets/_base_dataset.py new file mode 100644 index 0000000..64bf9fc --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/_base_dataset.py @@ -0,0 +1,326 @@ +import csv +import io +import zipfile +import os +import traceback +import numpy as np +from copy import deepcopy +from abc import ABC, abstractmethod +from .. import _timing +from ..utils import TrackEvalException + + +class _BaseDataset(ABC): + @abstractmethod + def __init__(self): + self.tracker_list = None + self.seq_list = None + self.class_list = None + self.output_fol = None + self.output_sub_fol = None + self.should_classes_combine = True + self.use_super_categories = False + + # Functions to implement: + + @staticmethod + @abstractmethod + def get_default_dataset_config(): + ... + + @abstractmethod + def _load_raw_file(self, tracker, seq, is_gt): + ... + + @_timing.time + @abstractmethod + def get_preprocessed_seq_data(self, raw_data, cls): + ... + + @abstractmethod + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + ... + + # Helper functions for all datasets: + + @classmethod + def get_class_name(cls): + return cls.__name__ + + def get_name(self): + return self.get_class_name() + + def get_output_fol(self, tracker): + return os.path.join(self.output_fol, tracker, self.output_sub_fol) + + def get_display_name(self, tracker): + """ Can be overwritten if the trackers name (in files) is different to how it should be displayed. + By default this method just returns the trackers name as is. + """ + return tracker + + def get_eval_info(self): + """Return info about the dataset needed for the Evaluator""" + return self.tracker_list, self.seq_list, self.class_list + + @_timing.time + def get_raw_seq_data(self, tracker, seq): + """ Loads raw data (tracker and ground-truth) for a single tracker on a single sequence. + Raw data includes all of the information needed for both preprocessing and evaluation, for all classes. + A later function (get_processed_seq_data) will perform such preprocessing and extract relevant information for + the evaluation of each class. + + This returns a dict which contains the fields: + [num_timesteps]: integer + [gt_ids, tracker_ids, gt_classes, tracker_classes, tracker_confidences]: + list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + [gt_extras]: dict (for each extra) of lists (for each timestep) of 1D NDArrays (for each det). + + gt_extras contains dataset specific information used for preprocessing such as occlusion and truncation levels. + + Note that similarities are extracted as part of the dataset and not the metric, because almost all metrics are + independent of the exact method of calculating the similarity. However datasets are not (e.g. segmentation + masks vs 2D boxes vs 3D boxes). + We calculate the similarity before preprocessing because often both preprocessing and evaluation require it and + we don't wish to calculate this twice. + We calculate similarity between all gt and tracker classes (not just each class individually) to allow for + calculation of metrics such as class confusion matrices. Typically the impact of this on performance is low. + """ + # Load raw data. + raw_gt_data = self._load_raw_file(tracker, seq, is_gt=True) + raw_tracker_data = self._load_raw_file(tracker, seq, is_gt=False) + raw_data = {**raw_tracker_data, **raw_gt_data} # Merges dictionaries + + # Calculate similarities for each timestep. + similarity_scores = [] + for t, (gt_dets_t, tracker_dets_t) in enumerate(zip(raw_data['gt_dets'], raw_data['tracker_dets'])): + ious = self._calculate_similarities(gt_dets_t, tracker_dets_t) + similarity_scores.append(ious) + raw_data['similarity_scores'] = similarity_scores + return raw_data + + @staticmethod + def _load_simple_text_file(file, time_col=0, id_col=None, remove_negative_ids=False, valid_filter=None, + crowd_ignore_filter=None, convert_filter=None, is_zipped=False, zip_file=None, + force_delimiters=None): + """ Function that loads data which is in a commonly used text file format. + Assumes each det is given by one row of a text file. + There is no limit to the number or meaning of each column, + however one column needs to give the timestep of each det (time_col) which is default col 0. + + The file dialect (deliminator, num cols, etc) is determined automatically. + This function automatically separates dets by timestep, + and is much faster than alternatives such as np.loadtext or pandas. + + If remove_negative_ids is True and id_col is not None, dets with negative values in id_col are excluded. + These are not excluded from ignore data. + + valid_filter can be used to only include certain classes. + It is a dict with ints as keys, and lists as values, + such that a row is included if "row[key].lower() is in value" for all key/value pairs in the dict. + If None, all classes are included. + + crowd_ignore_filter can be used to read crowd_ignore regions separately. It has the same format as valid filter. + + convert_filter can be used to convert value read to another format. + This is used most commonly to convert classes given as string to a class id. + This is a dict such that the key is the column to convert, and the value is another dict giving the mapping. + + Optionally, input files could be a zip of multiple text files for storage efficiency. + + Returns read_data and ignore_data. + Each is a dict (with keys as timesteps as strings) of lists (over dets) of lists (over column values). + Note that all data is returned as strings, and must be converted to float/int later if needed. + Note that timesteps will not be present in the returned dict keys if there are no dets for them + """ + + if remove_negative_ids and id_col is None: + raise TrackEvalException('remove_negative_ids is True, but id_col is not given.') + if crowd_ignore_filter is None: + crowd_ignore_filter = {} + if convert_filter is None: + convert_filter = {} + try: + if is_zipped: # Either open file directly or within a zip. + if zip_file is None: + raise TrackEvalException('is_zipped set to True, but no zip_file is given.') + archive = zipfile.ZipFile(os.path.join(zip_file), 'r') + fp = io.TextIOWrapper(archive.open(file, 'r')) + else: + fp = open(file) + read_data = {} + crowd_ignore_data = {} + fp.seek(0, os.SEEK_END) + # check if file is empty + if fp.tell(): + fp.seek(0) + dialect = csv.Sniffer().sniff(fp.readline(), delimiters=force_delimiters) # Auto determine structure. + dialect.skipinitialspace = True # Deal with extra spaces between columns + fp.seek(0) + reader = csv.reader(fp, dialect) + for row in reader: + try: + # Deal with extra trailing spaces at the end of rows + if row[-1] in '': + row = row[:-1] + timestep = str(int(float(row[time_col]))) + # Read ignore regions separately. + is_ignored = False + for ignore_key, ignore_value in crowd_ignore_filter.items(): + if row[ignore_key].lower() in ignore_value: + # Convert values in one column (e.g. string to id) + for convert_key, convert_value in convert_filter.items(): + row[convert_key] = convert_value[row[convert_key].lower()] + # Save data separated by timestep. + if timestep in crowd_ignore_data.keys(): + crowd_ignore_data[timestep].append(row) + else: + crowd_ignore_data[timestep] = [row] + is_ignored = True + if is_ignored: # if det is an ignore region, it cannot be a normal det. + continue + # Exclude some dets if not valid. + if valid_filter is not None: + for key, value in valid_filter.items(): + if row[key].lower() not in value: + continue + if remove_negative_ids: + if int(float(row[id_col])) < 0: + continue + # Convert values in one column (e.g. string to id) + for convert_key, convert_value in convert_filter.items(): + row[convert_key] = convert_value[row[convert_key].lower()] + # Save data separated by timestep. + if timestep in read_data.keys(): + read_data[timestep].append(row) + else: + read_data[timestep] = [row] + except Exception: + exc_str_init = 'In file %s the following line cannot be read correctly: \n' % os.path.basename( + file) + exc_str = ' '.join([exc_str_init]+row) + raise TrackEvalException(exc_str) + fp.close() + except Exception: + print('Error loading file: %s, printing traceback.' % file) + traceback.print_exc() + raise TrackEvalException( + 'File %s cannot be read because it is either not present or invalidly formatted' % os.path.basename( + file)) + return read_data, crowd_ignore_data + + @staticmethod + def _calculate_mask_ious(masks1, masks2, is_encoded=False, do_ioa=False): + """ Calculates the IOU (intersection over union) between two arrays of segmentation masks. + If is_encoded a run length encoding with pycocotools is assumed as input format, otherwise an input of numpy + arrays of the shape (num_masks, height, width) is assumed and the encoding is performed. + If do_ioa (intersection over area) , then calculates the intersection over the area of masks1 - this is commonly + used to determine if detections are within crowd ignore region. + :param masks1: first set of masks (numpy array of shape (num_masks, height, width) if not encoded, + else pycocotools rle encoded format) + :param masks2: second set of masks (numpy array of shape (num_masks, height, width) if not encoded, + else pycocotools rle encoded format) + :param is_encoded: whether the input is in pycocotools rle encoded format + :param do_ioa: whether to perform IoA computation + :return: the IoU/IoA scores + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + # use pycocotools for run length encoding of masks + if not is_encoded: + masks1 = mask_utils.encode(np.array(np.transpose(masks1, (1, 2, 0)), order='F')) + masks2 = mask_utils.encode(np.array(np.transpose(masks2, (1, 2, 0)), order='F')) + + # use pycocotools for iou computation of rle encoded masks + ious = mask_utils.iou(masks1, masks2, [do_ioa]*len(masks2)) + if len(masks1) == 0 or len(masks2) == 0: + ious = np.asarray(ious).reshape(len(masks1), len(masks2)) + assert (ious >= 0 - np.finfo('float').eps).all() + assert (ious <= 1 + np.finfo('float').eps).all() + + return ious + + @staticmethod + def _calculate_box_ious(bboxes1, bboxes2, box_format='xywh', do_ioa=False): + """ Calculates the IOU (intersection over union) between two arrays of boxes. + Allows variable box formats ('xywh' and 'x0y0x1y1'). + If do_ioa (intersection over area) , then calculates the intersection over the area of boxes1 - this is commonly + used to determine if detections are within crowd ignore region. + """ + if box_format in 'xywh': + # layout: (x0, y0, w, h) + bboxes1 = deepcopy(bboxes1) + bboxes2 = deepcopy(bboxes2) + + bboxes1[:, 2] = bboxes1[:, 0] + bboxes1[:, 2] + bboxes1[:, 3] = bboxes1[:, 1] + bboxes1[:, 3] + bboxes2[:, 2] = bboxes2[:, 0] + bboxes2[:, 2] + bboxes2[:, 3] = bboxes2[:, 1] + bboxes2[:, 3] + elif box_format not in 'x0y0x1y1': + raise (TrackEvalException('box_format %s is not implemented' % box_format)) + + # layout: (x0, y0, x1, y1) + min_ = np.minimum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :]) + max_ = np.maximum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :]) + intersection = np.maximum(min_[..., 2] - max_[..., 0], 0) * np.maximum(min_[..., 3] - max_[..., 1], 0) + area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (bboxes1[..., 3] - bboxes1[..., 1]) + + if do_ioa: + ioas = np.zeros_like(intersection) + valid_mask = area1 > 0 + np.finfo('float').eps + ioas[valid_mask, :] = intersection[valid_mask, :] / area1[valid_mask][:, np.newaxis] + + return ioas + else: + area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (bboxes2[..., 3] - bboxes2[..., 1]) + union = area1[:, np.newaxis] + area2[np.newaxis, :] - intersection + intersection[area1 <= 0 + np.finfo('float').eps, :] = 0 + intersection[:, area2 <= 0 + np.finfo('float').eps] = 0 + intersection[union <= 0 + np.finfo('float').eps] = 0 + union[union <= 0 + np.finfo('float').eps] = 1 + ious = intersection / union + return ious + + @staticmethod + def _calculate_euclidean_similarity(dets1, dets2, zero_distance=2.0): + """ Calculates the euclidean distance between two sets of detections, and then converts this into a similarity + measure with values between 0 and 1 using the following formula: sim = max(0, 1 - dist/zero_distance). + The default zero_distance of 2.0, corresponds to the default used in MOT15_3D, such that a 0.5 similarity + threshold corresponds to a 1m distance threshold for TPs. + """ + dist = np.linalg.norm(dets1[:, np.newaxis]-dets2[np.newaxis, :], axis=2) + sim = np.maximum(0, 1 - dist/zero_distance) + return sim + + @staticmethod + def _check_unique_ids(data, after_preproc=False): + """Check the requirement that the tracker_ids and gt_ids are unique per timestep""" + gt_ids = data['gt_ids'] + tracker_ids = data['tracker_ids'] + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(gt_ids, tracker_ids)): + if len(tracker_ids_t) > 0: + unique_ids, counts = np.unique(tracker_ids_t, return_counts=True) + if np.max(counts) != 1: + duplicate_ids = unique_ids[counts > 1] + exc_str_init = 'Tracker predicts the same ID more than once in a single timestep ' \ + '(seq: %s, frame: %i, ids:' % (data['seq'], t+1) + exc_str = ' '.join([exc_str_init] + [str(d) for d in duplicate_ids]) + ')' + if after_preproc: + exc_str_init += '\n Note that this error occurred after preprocessing (but not before), ' \ + 'so ids may not be as in file, and something seems wrong with preproc.' + raise TrackEvalException(exc_str) + if len(gt_ids_t) > 0: + unique_ids, counts = np.unique(gt_ids_t, return_counts=True) + if np.max(counts) != 1: + duplicate_ids = unique_ids[counts > 1] + exc_str_init = 'Ground-truth has the same ID more than once in a single timestep ' \ + '(seq: %s, frame: %i, ids:' % (data['seq'], t+1) + exc_str = ' '.join([exc_str_init] + [str(d) for d in duplicate_ids]) + ')' + if after_preproc: + exc_str_init += '\n Note that this error occurred after preprocessing (but not before), ' \ + 'so ids may not be as in file, and something seems wrong with preproc.' + raise TrackEvalException(exc_str) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/bdd100k.py b/test/yolov7-tracker/tracker/trackeval/datasets/bdd100k.py new file mode 100644 index 0000000..cc4fd06 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/bdd100k.py @@ -0,0 +1,302 @@ + +import os +import json +import numpy as np +from scipy.optimize import linear_sum_assignment +from ..utils import TrackEvalException +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing + + +class BDD100K(_BaseDataset): + """Dataset class for BDD100K tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/bdd100k/bdd100k_val'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/bdd100k/bdd100k_val'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle'], + # Valid: ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle'] + 'SPLIT_TO_EVAL': 'val', # Valid: 'training', 'val', + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = True + self.use_super_categories = True + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian', 'rider', 'car', 'bus', 'truck', 'train', 'motorcycle', 'bicycle'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes [pedestrian, rider, car, ' + 'bus, truck, train, motorcycle, bicycle] are valid.') + self.super_categories = {"HUMAN": [cls for cls in ["pedestrian", "rider"] if cls in self.class_list], + "VEHICLE": [cls for cls in ["car", "truck", "bus", "train"] if cls in self.class_list], + "BIKE": [cls for cls in ["motorcycle", "bicycle"] if cls in self.class_list]} + self.distractor_classes = ['other person', 'trailer', 'other vehicle'] + self.class_name_to_class_id = {'pedestrian': 1, 'rider': 2, 'other person': 3, 'car': 4, 'bus': 5, 'truck': 6, + 'train': 7, 'trailer': 8, 'other vehicle': 9, 'motorcycle': 10, 'bicycle': 11} + + # Get sequences to eval + self.seq_list = [] + self.seq_lengths = {} + + self.seq_list = [seq_file.replace('.json', '') for seq_file in os.listdir(self.gt_fol)] + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.json') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the BDD100K format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if is_gt: + file = os.path.join(self.gt_fol, seq + '.json') + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.json') + + with open(file) as f: + data = json.load(f) + + # sort data by frame index + data = sorted(data, key=lambda x: x['index']) + + # check sequence length + if is_gt: + self.seq_lengths[seq] = len(data) + num_timesteps = len(data) + else: + num_timesteps = self.seq_lengths[seq] + if num_timesteps != len(data): + raise TrackEvalException('Number of ground truth and tracker timesteps do not match for sequence %s' + % seq) + + # Convert data to required format + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for t in range(num_timesteps): + ig_ids = [] + keep_ids = [] + for i in range(len(data[t]['labels'])): + ann = data[t]['labels'][i] + if is_gt and (ann['category'] in self.distractor_classes or 'attributes' in ann.keys() + and ann['attributes']['Crowd']): + ig_ids.append(i) + else: + keep_ids.append(i) + + if keep_ids: + raw_data['dets'][t] = np.atleast_2d([[data[t]['labels'][i]['box2d']['x1'], + data[t]['labels'][i]['box2d']['y1'], + data[t]['labels'][i]['box2d']['x2'], + data[t]['labels'][i]['box2d']['y2'] + ] for i in keep_ids]).astype(float) + raw_data['ids'][t] = np.atleast_1d([data[t]['labels'][i]['id'] for i in keep_ids]).astype(int) + raw_data['classes'][t] = np.atleast_1d([self.class_name_to_class_id[data[t]['labels'][i]['category']] + for i in keep_ids]).astype(int) + else: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + + if is_gt: + if ig_ids: + raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d([[data[t]['labels'][i]['box2d']['x1'], + data[t]['labels'][i]['box2d']['y1'], + data[t]['labels'][i]['box2d']['x2'], + data[t]['labels'][i]['box2d']['y2'] + ] for i in ig_ids]).astype(float) + else: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)).astype(float) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + BDD100K: + In BDD100K, the 4 preproc steps are as follow: + 1) There are eight classes (pedestrian, rider, car, bus, truck, train, motorcycle, bicycle) + which are evaluated separately. + 2) For BDD100K there is no removal of matched tracker dets. + 3) Crowd ignore regions are used to remove unmatched detections. + 4) No removal of gt dets. + """ + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm) + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region. + unmatched_tracker_dets = tracker_dets[unmatched_indices, :] + crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t] + intersection_with_ignore_region = self._calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions, + box_format='x0y0x1y1', do_ioa=True) + is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, + axis=1) + + # Apply preprocessing to remove unwanted tracker dets. + to_remove_tracker = unmatched_indices[is_within_crowd_ignore_region] + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='x0y0x1y1') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst.py new file mode 100644 index 0000000..475c09e --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst.py @@ -0,0 +1,49 @@ +import os +from .burst_helpers.burst_base import BURSTBase +from .burst_helpers.format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter +from .. import utils + + +class BURST(BURSTBase): + """Dataset class for TAO tracking""" + + @staticmethod + def get_default_dataset_config(): + tao_config = BURSTBase.get_default_dataset_config() + code_path = utils.get_code_path() + + # e.g. 'data/gt/tsunami/exemplar_guided/' + tao_config['GT_FOLDER'] = os.path.join( + code_path, 'data/gt/burst/val/') # Location of GT data + # e.g. 'data/trackers/tsunami/exemplar_guided/mask_guided/validation/' + tao_config['TRACKERS_FOLDER'] = os.path.join( + code_path, 'data/trackers/burst/class-guided/') # Trackers location + # set to True or False + tao_config['EXEMPLAR_GUIDED'] = False + return tao_config + + def _iou_type(self): + return 'mask' + + def _box_or_mask_from_det(self, det): + return det['segmentation'] + + def _calculate_area_for_ann(self, ann): + import pycocotools.mask as cocomask + return cocomask.area(ann["segmentation"]) + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores + + def _is_exemplar_guided(self): + exemplar_guided = self.config['EXEMPLAR_GUIDED'] + return exemplar_guided + + def _postproc_ground_truth_data(self, data): + return GroundTruthBURSTFormatToTAOFormatConverter(data).convert() + + def _postproc_prediction_data(self, data): + return PredictionBURSTFormatToTAOFormatConverter( + self.gt_data, data, + exemplar_guided=self._is_exemplar_guided()).convert() diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/BURST_SPECIFIC_ISSUES.md b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/BURST_SPECIFIC_ISSUES.md new file mode 100644 index 0000000..184d53c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/BURST_SPECIFIC_ISSUES.md @@ -0,0 +1,7 @@ +The track ids in both ground truth and predictions are not globally unique, but +start from 1 for each video. At the moment when converting from Ali format to +TAO format, we remap the ids to be globally unique. It would be better to +directly have this in the data though. + + +Improve setting of EXEMPLAR_GUIDED flag, maybe this can be done automatically. diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/__init__.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_base.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_base.py new file mode 100644 index 0000000..394eda4 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_base.py @@ -0,0 +1,591 @@ +import os +import numpy as np +import json +import itertools +from collections import defaultdict +from scipy.optimize import linear_sum_assignment +from trackeval.utils import TrackEvalException +from trackeval.datasets._base_dataset import _BaseDataset +from trackeval import utils +from trackeval import _timing + + +class BURSTBase(_BaseDataset): + """Dataset class for TAO tracking""" + + def _postproc_ground_truth_data(self, data): + return data + + def _postproc_prediction_data(self, data): + return data + + def _iou_type(self): + return 'bbox' + + def _box_or_mask_from_det(self, det): + return np.atleast_1d(det['bbox']) + + def _calculate_area_for_ann(self, ann): + return ann["bbox"][2] * ann["bbox"][3] + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes) + 'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val' + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'MAX_DETECTIONS': 300, # Number of maximal allowed detections per image (0 for unlimited) + 'EXEMPLAR_GUIDED': False, + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = True + self.use_super_categories = False + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')] + if len(gt_dir_files) != 1: + raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.') + + with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f: + self.gt_data = self._postproc_ground_truth_data(json.load(f)) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks']) + + # Get sequences to eval and sequence information + self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']] + self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']} + # compute mappings from videos to annotation data + self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations']) + # compute sequence lengths + self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']} + for img in self.gt_data['images']: + self.seq_lengths[img['video_id']] += 1 + self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings() + self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track + in self.videos_to_gt_tracks[vid['id']]}), + 'neg_cat_ids': vid['neg_category_ids'], + 'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']} + for vid in self.gt_data['videos']} + + # Get classes to eval + considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list] + seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id + in self.seq_to_classes[vid_id]['pos_cat_ids']]) + # only classes with ground truth are evaluated in TAO, also we don't evaluate distactors. + distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567, + 569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883, + 912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220} + self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if (cls['id'] in seen_cats) and (cls['id'] not in distractors)] + cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']} + + if self.config['CLASSES_TO_EVAL']: + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(self.valid_classes) + + ' are valid (classes present in ground truth data).') + else: + self.class_list = [cls for cls in self.valid_classes] + self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list} + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + self.tracker_data = {tracker: dict() for tracker in self.tracker_list} + + for tracker in self.tracker_list: + tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)) + if file.endswith('.json')] + if len(tr_dir_files) != 1: + raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol) + + ' does not contain exactly one json file.') + with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f: + curr_data = self._postproc_prediction_data(json.load(f)) + + # limit detections if MAX_DETECTIONS > 0 + if self.config['MAX_DETECTIONS']: + curr_data = self._limit_dets_per_image(curr_data) + + # fill missing video ids + self._fill_video_ids_inplace(curr_data) + + # make track ids unique over whole evaluation set + self._make_track_ids_unique(curr_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(curr_data) + + # get tracker sequence information + curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data) + self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks + self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the TAO format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values + as keys and lists (for each track) as values + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values + as keys and lists as values + [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values + """ + seq_id = self.seq_name_to_seq_id[seq] + # File location + if is_gt: + imgs = self.videos_to_gt_images[seq_id] + else: + imgs = self.tracker_data[tracker]['vids_to_images'][seq_id] + + # Convert data to required format + num_timesteps = self.seq_lengths[seq_id] + img_to_timestep = self.seq_to_images_to_timestep[seq_id] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for img in imgs: + # some tracker data contains images without any ground truth information, these are ignored + try: + t = img_to_timestep[img['id']] + except KeyError: + continue + annotations = img['annotations'] + raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float) + raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int) + raw_data['classes'][t] = np.atleast_1d([ann['category_id'] for ann in annotations]).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float) + + for t, d in enumerate(raw_data['dets']): + if d is None: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.empty(0) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list] + if is_gt: + classes_to_consider = all_classes + all_tracks = self.videos_to_gt_tracks[seq_id] + else: + classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \ + + self.seq_to_classes[seq_id]['neg_cat_ids'] + all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id] + + classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls] + if cls in classes_to_consider else [] for cls in all_classes} + + # mapping from classes to track information + raw_data['classes_to_tracks'] = {cls: [{det['image_id']: self._box_or_mask_from_det(det) + for det in track['annotations']} for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks] + for cls, tracks in classes_to_tracks.items()} + + if not is_gt: + raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score']) + for x in track['annotations']]) + for track in tracks]) + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + key_map = {'classes_to_tracks': 'classes_to_gt_tracks', + 'classes_to_track_ids': 'classes_to_gt_track_ids', + 'classes_to_track_lengths': 'classes_to_gt_track_lengths', + 'classes_to_track_areas': 'classes_to_gt_track_areas'} + else: + key_map = {'classes_to_tracks': 'classes_to_dt_tracks', + 'classes_to_track_ids': 'classes_to_dt_track_ids', + 'classes_to_track_lengths': 'classes_to_dt_track_lengths', + 'classes_to_track_areas': 'classes_to_dt_track_areas'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids'] + raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids'] + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + TAO: + In TAO, the 4 preproc steps are as follow: + 1) All classes present in the ground truth data are evaluated separately. + 2) No matched tracker detections are removed. + 3) Unmatched tracker detections are removed if there is not ground truth data and the class does not + belong to the categories marked as negative for this sequence. Additionally, unmatched tracker + detections for classes which are marked as not exhaustively labeled are removed. + 4) No gt detections are removed. + Further, for TrackMAP computation track representations for the given class are accessed from a dictionary + and the tracks from the tracker data are sorted according to the tracker confidence. + """ + cls_id = self.class_name_to_class_id[cls] + is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls'] + is_neg_category = cls_id in raw_data['neg_cat_ids'] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + if not self.config['EXEMPLAR_GUIDED']: + # Match tracker and gt dets (with hungarian algorithm). + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + if gt_ids.shape[0] == 0 and not is_neg_category: + to_remove_tracker = unmatched_indices + elif is_not_exhaustively_labeled: + to_remove_tracker = unmatched_indices + else: + to_remove_tracker = np.array([], dtype=np.int) + + # remove all unwanted unmatched tracker detections + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + else: + data['tracker_ids'][t] = tracker_ids + data['tracker_dets'][t] = tracker_dets + data['tracker_confidences'][t] = tracker_confidences + + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # get track representations + data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id] + data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id] + data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id] + data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id] + data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id] + data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id] + data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id] + data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id] + data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id] + data['not_exhaustively_labeled'] = is_not_exhaustively_labeled + data['iou_type'] = self._iou_type() + + # sort tracker data tracks by tracker confidence scores + if data['dt_tracks']: + idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort") + data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx] + data['dt_tracks'] = [data['dt_tracks'][i] for i in idx] + data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx] + data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx] + data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx] + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t) + return similarity_scores + + def _merge_categories(self, annotations): + """ + Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset + :param annotations: the annotations in which the classes should be merged + :return: None + """ + merge_map = {} + for category in self.gt_data['categories']: + if 'merged' in category: + for to_merge in category['merged']: + merge_map[to_merge['id']] = category['id'] + + for ann in annotations: + ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id']) + + def _compute_vid_mappings(self, annotations): + """ + Computes mappings from Videos to corresponding tracks and images. + :param annotations: the annotations for which the mapping should be generated + :return: the video-to-track-mapping, the video-to-image-mapping + """ + vids_to_tracks = {} + vids_to_imgs = {} + vid_ids = [vid['id'] for vid in self.gt_data['videos']] + + # compute an mapping from image IDs to images + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + for ann in annotations: + ann["area"] = self._calculate_area_for_ann(ann) + + vid = ann["video_id"] + if ann["video_id"] not in vids_to_tracks.keys(): + vids_to_tracks[ann["video_id"]] = list() + if ann["video_id"] not in vids_to_imgs.keys(): + vids_to_imgs[ann["video_id"]] = list() + + # Fill in vids_to_tracks + tid = ann["track_id"] + exist_tids = [track["id"] for track in vids_to_tracks[vid]] + try: + index1 = exist_tids.index(tid) + except ValueError: + index1 = -1 + if tid not in exist_tids: + curr_track = {"id": tid, "category_id": ann["category_id"], + "video_id": vid, "annotations": [ann]} + vids_to_tracks[vid].append(curr_track) + else: + vids_to_tracks[vid][index1]["annotations"].append(ann) + + # Fill in vids_to_imgs + img_id = ann['image_id'] + exist_img_ids = [img["id"] for img in vids_to_imgs[vid]] + try: + index2 = exist_img_ids.index(img_id) + except ValueError: + index2 = -1 + if index2 == -1: + curr_img = {"id": img_id, "annotations": [ann]} + vids_to_imgs[vid].append(curr_img) + else: + vids_to_imgs[vid][index2]["annotations"].append(ann) + + # sort annotations by frame index and compute track area + for vid, tracks in vids_to_tracks.items(): + for track in tracks: + track["annotations"] = sorted( + track['annotations'], + key=lambda x: images[x['image_id']]['frame_index']) + # Computer average area + track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations'])) + + # Ensure all videos are present + for vid_id in vid_ids: + if vid_id not in vids_to_tracks.keys(): + vids_to_tracks[vid_id] = [] + if vid_id not in vids_to_imgs.keys(): + vids_to_imgs[vid_id] = [] + + return vids_to_tracks, vids_to_imgs + + def _compute_image_to_timestep_mappings(self): + """ + Computes a mapping from images to the corresponding timestep in the sequence. + :return: the image-to-timestep-mapping + """ + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']} + for vid in seq_to_imgs_to_timestep: + curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]] + curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index']) + seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))} + + return seq_to_imgs_to_timestep + + def _limit_dets_per_image(self, annotations): + """ + Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from + https://github.com/TAO-Dataset/ + :param annotations: the annotations in which the detections should be limited + :return: the annotations with limited detections + """ + max_dets = self.config['MAX_DETECTIONS'] + img_ann = defaultdict(list) + for ann in annotations: + img_ann[ann["image_id"]].append(ann) + + for img_id, _anns in img_ann.items(): + if len(_anns) <= max_dets: + continue + _anns = sorted(_anns, key=lambda x: x["score"], reverse=True) + img_ann[img_id] = _anns[:max_dets] + + return [ann for anns in img_ann.values() for ann in anns] + + def _fill_video_ids_inplace(self, annotations): + """ + Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotations for which the videos IDs should be filled inplace + :return: None + """ + missing_video_id = [x for x in annotations if 'video_id' not in x] + if missing_video_id: + image_id_to_video_id = { + x['id']: x['video_id'] for x in self.gt_data['images'] + } + for x in missing_video_id: + x['video_id'] = image_id_to_video_id[x['image_id']] + + @staticmethod + def _make_track_ids_unique(annotations): + """ + Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotation set + :return: the number of updated IDs + """ + track_id_videos = {} + track_ids_to_update = set() + max_track_id = 0 + for ann in annotations: + t = ann['track_id'] + if t not in track_id_videos: + track_id_videos[t] = ann['video_id'] + + if ann['video_id'] != track_id_videos[t]: + # Track id is assigned to multiple videos + track_ids_to_update.add(t) + max_track_id = max(max_track_id, t) + + if track_ids_to_update: + #print('true') + next_id = itertools.count(max_track_id + 1) + new_track_ids = defaultdict(lambda: next(next_id)) + for ann in annotations: + t = ann['track_id'] + v = ann['video_id'] + if t in track_ids_to_update: + ann['track_id'] = new_track_ids[t, v] + return len(track_ids_to_update) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_ow_base.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_ow_base.py new file mode 100644 index 0000000..bef14d2 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/burst_ow_base.py @@ -0,0 +1,675 @@ +import os +import numpy as np +import json +import itertools +from collections import defaultdict +from scipy.optimize import linear_sum_assignment +from trackeval.utils import TrackEvalException +from trackeval.datasets._base_dataset import _BaseDataset +from trackeval import utils +from trackeval import _timing + + +class BURST_OW_Base(_BaseDataset): + """Dataset class for TAO tracking""" + + def _postproc_ground_truth_data(self, data): + return data + + def _postproc_prediction_data(self, data): + return data + + def _iou_type(self): + return 'bbox' + + def _box_or_mask_from_det(self, det): + return np.atleast_1d(det['bbox']) + + def _calculate_area_for_ann(self, ann): + return ann["bbox"][2] * ann["bbox"][3] + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes) + 'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val' + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'MAX_DETECTIONS': 300, # Number of maximal allowed detections per image (0 for unlimited) + 'SUBSET': 'all' + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = True + self.use_super_categories = False + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')] + if len(gt_dir_files) != 1: + raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.') + + with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f: + self.gt_data = self._postproc_ground_truth_data(json.load(f)) + + self.subset = self.config['SUBSET'] + if self.subset != 'all': + # Split GT data into `known`, `unknown` or `distractor` + self._split_known_unknown_distractor() + self.gt_data = self._filter_gt_data(self.gt_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks']) + + # Get sequences to eval and sequence information + self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']] + self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']} + # compute mappings from videos to annotation data + self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations']) + # compute sequence lengths + self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']} + for img in self.gt_data['images']: + self.seq_lengths[img['video_id']] += 1 + self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings() + self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track + in self.videos_to_gt_tracks[vid['id']]}), + 'neg_cat_ids': vid['neg_category_ids'], + 'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']} + for vid in self.gt_data['videos']} + + # Get classes to eval + considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list] + seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id + in self.seq_to_classes[vid_id]['pos_cat_ids']]) + # only classes with ground truth are evaluated in TAO + self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats] + # cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']} + + if self.config['CLASSES_TO_EVAL']: + # self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + # for cls in self.config['CLASSES_TO_EVAL']] + self.class_list = ["object"] # class-agnostic + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(self.valid_classes) + + ' are valid (classes present in ground truth data).') + else: + # self.class_list = [cls for cls in self.valid_classes] + self.class_list = ["object"] # class-agnostic + # self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list} + self.class_name_to_class_id = {"object": 1} # class-agnostic + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + self.tracker_data = {tracker: dict() for tracker in self.tracker_list} + + for tracker in self.tracker_list: + tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)) + if file.endswith('.json')] + if len(tr_dir_files) != 1: + raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol) + + ' does not contain exactly one json file.') + with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f: + curr_data = self._postproc_prediction_data(json.load(f)) + + # limit detections if MAX_DETECTIONS > 0 + if self.config['MAX_DETECTIONS']: + curr_data = self._limit_dets_per_image(curr_data) + + # fill missing video ids + self._fill_video_ids_inplace(curr_data) + + # make track ids unique over whole evaluation set + self._make_track_ids_unique(curr_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(curr_data) + + # get tracker sequence information + curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data) + self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks + self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the TAO format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values + as keys and lists (for each track) as values + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values + as keys and lists as values + [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values + """ + seq_id = self.seq_name_to_seq_id[seq] + # File location + if is_gt: + imgs = self.videos_to_gt_images[seq_id] + else: + imgs = self.tracker_data[tracker]['vids_to_images'][seq_id] + + # Convert data to required format + num_timesteps = self.seq_lengths[seq_id] + img_to_timestep = self.seq_to_images_to_timestep[seq_id] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for img in imgs: + # some tracker data contains images without any ground truth information, these are ignored + try: + t = img_to_timestep[img['id']] + except KeyError: + continue + annotations = img['annotations'] + raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float) + raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int) + raw_data['classes'][t] = np.atleast_1d([1 for _ in annotations]).astype(int) # class-agnostic + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float) + + for t, d in enumerate(raw_data['dets']): + if d is None: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.empty(0) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + # all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list] + all_classes = [1] # class-agnostic + + if is_gt: + classes_to_consider = all_classes + all_tracks = self.videos_to_gt_tracks[seq_id] + else: + # classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \ + # + self.seq_to_classes[seq_id]['neg_cat_ids'] + classes_to_consider = all_classes # class-agnostic + all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id] + + # classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls] + # if cls in classes_to_consider else [] for cls in all_classes} + classes_to_tracks = {cls: [track for track in all_tracks] + if cls in classes_to_consider else [] for cls in all_classes} # class-agnostic + + # mapping from classes to track information + raw_data['classes_to_tracks'] = {cls: [{det['image_id']: self._box_or_mask_from_det(det) + for det in track['annotations']} for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks] + for cls, tracks in classes_to_tracks.items()} + + if not is_gt: + raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score']) + for x in track['annotations']]) + for track in tracks]) + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + key_map = {'classes_to_tracks': 'classes_to_gt_tracks', + 'classes_to_track_ids': 'classes_to_gt_track_ids', + 'classes_to_track_lengths': 'classes_to_gt_track_lengths', + 'classes_to_track_areas': 'classes_to_gt_track_areas'} + else: + key_map = {'classes_to_tracks': 'classes_to_dt_tracks', + 'classes_to_track_ids': 'classes_to_dt_track_ids', + 'classes_to_track_lengths': 'classes_to_dt_track_lengths', + 'classes_to_track_areas': 'classes_to_dt_track_areas'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids'] + raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids'] + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + TAO: + In TAO, the 4 preproc steps are as follow: + 1) All classes present in the ground truth data are evaluated separately. + 2) No matched tracker detections are removed. + 3) Unmatched tracker detections are removed if there is not ground truth data and the class does not + belong to the categories marked as negative for this sequence. Additionally, unmatched tracker + detections for classes which are marked as not exhaustively labeled are removed. + 4) No gt detections are removed. + Further, for TrackMAP computation track representations for the given class are accessed from a dictionary + and the tracks from the tracker data are sorted according to the tracker confidence. + """ + cls_id = self.class_name_to_class_id[cls] + is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls'] + is_neg_category = cls_id in raw_data['neg_cat_ids'] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm). + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + if gt_ids.shape[0] == 0 and not is_neg_category: + to_remove_tracker = unmatched_indices + elif is_not_exhaustively_labeled: + to_remove_tracker = unmatched_indices + else: + to_remove_tracker = np.array([], dtype=np.int) + + # remove all unwanted unmatched tracker detections + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # get track representations + data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id] + data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id] + data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id] + data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id] + data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id] + data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id] + data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id] + data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id] + data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id] + data['not_exhaustively_labeled'] = is_not_exhaustively_labeled + data['iou_type'] = self._iou_type() + + # sort tracker data tracks by tracker confidence scores + if data['dt_tracks']: + idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort") + data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx] + data['dt_tracks'] = [data['dt_tracks'][i] for i in idx] + data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx] + data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx] + data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx] + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t) + return similarity_scores + + def _merge_categories(self, annotations): + """ + Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset + :param annotations: the annotations in which the classes should be merged + :return: None + """ + merge_map = {} + for category in self.gt_data['categories']: + if 'merged' in category: + for to_merge in category['merged']: + merge_map[to_merge['id']] = category['id'] + + for ann in annotations: + ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id']) + + def _compute_vid_mappings(self, annotations): + """ + Computes mappings from Videos to corresponding tracks and images. + :param annotations: the annotations for which the mapping should be generated + :return: the video-to-track-mapping, the video-to-image-mapping + """ + vids_to_tracks = {} + vids_to_imgs = {} + vid_ids = [vid['id'] for vid in self.gt_data['videos']] + + # compute an mapping from image IDs to images + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + for ann in annotations: + ann["area"] = self._calculate_area_for_ann(ann) + + vid = ann["video_id"] + if ann["video_id"] not in vids_to_tracks.keys(): + vids_to_tracks[ann["video_id"]] = list() + if ann["video_id"] not in vids_to_imgs.keys(): + vids_to_imgs[ann["video_id"]] = list() + + # Fill in vids_to_tracks + tid = ann["track_id"] + exist_tids = [track["id"] for track in vids_to_tracks[vid]] + try: + index1 = exist_tids.index(tid) + except ValueError: + index1 = -1 + if tid not in exist_tids: + curr_track = {"id": tid, "category_id": ann["category_id"], + "video_id": vid, "annotations": [ann]} + vids_to_tracks[vid].append(curr_track) + else: + vids_to_tracks[vid][index1]["annotations"].append(ann) + + # Fill in vids_to_imgs + img_id = ann['image_id'] + exist_img_ids = [img["id"] for img in vids_to_imgs[vid]] + try: + index2 = exist_img_ids.index(img_id) + except ValueError: + index2 = -1 + if index2 == -1: + curr_img = {"id": img_id, "annotations": [ann]} + vids_to_imgs[vid].append(curr_img) + else: + vids_to_imgs[vid][index2]["annotations"].append(ann) + + # sort annotations by frame index and compute track area + for vid, tracks in vids_to_tracks.items(): + for track in tracks: + track["annotations"] = sorted( + track['annotations'], + key=lambda x: images[x['image_id']]['frame_index']) + # Computer average area + track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations'])) + + # Ensure all videos are present + for vid_id in vid_ids: + if vid_id not in vids_to_tracks.keys(): + vids_to_tracks[vid_id] = [] + if vid_id not in vids_to_imgs.keys(): + vids_to_imgs[vid_id] = [] + + return vids_to_tracks, vids_to_imgs + + def _compute_image_to_timestep_mappings(self): + """ + Computes a mapping from images to the corresponding timestep in the sequence. + :return: the image-to-timestep-mapping + """ + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']} + for vid in seq_to_imgs_to_timestep: + curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]] + curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index']) + seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))} + + return seq_to_imgs_to_timestep + + def _limit_dets_per_image(self, annotations): + """ + Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from + https://github.com/TAO-Dataset/ + :param annotations: the annotations in which the detections should be limited + :return: the annotations with limited detections + """ + max_dets = self.config['MAX_DETECTIONS'] + img_ann = defaultdict(list) + for ann in annotations: + img_ann[ann["image_id"]].append(ann) + + for img_id, _anns in img_ann.items(): + if len(_anns) <= max_dets: + continue + _anns = sorted(_anns, key=lambda x: x["score"], reverse=True) + img_ann[img_id] = _anns[:max_dets] + + return [ann for anns in img_ann.values() for ann in anns] + + def _fill_video_ids_inplace(self, annotations): + """ + Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotations for which the videos IDs should be filled inplace + :return: None + """ + missing_video_id = [x for x in annotations if 'video_id' not in x] + if missing_video_id: + image_id_to_video_id = { + x['id']: x['video_id'] for x in self.gt_data['images'] + } + for x in missing_video_id: + x['video_id'] = image_id_to_video_id[x['image_id']] + + @staticmethod + def _make_track_ids_unique(annotations): + """ + Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotation set + :return: the number of updated IDs + """ + track_id_videos = {} + track_ids_to_update = set() + max_track_id = 0 + for ann in annotations: + t = ann['track_id'] + if t not in track_id_videos: + track_id_videos[t] = ann['video_id'] + + if ann['video_id'] != track_id_videos[t]: + # Track id is assigned to multiple videos + track_ids_to_update.add(t) + max_track_id = max(max_track_id, t) + + if track_ids_to_update: + #print('true') + next_id = itertools.count(max_track_id + 1) + new_track_ids = defaultdict(lambda: next(next_id)) + for ann in annotations: + t = ann['track_id'] + v = ann['video_id'] + if t in track_ids_to_update: + ann['track_id'] = new_track_ids[t, v] + return len(track_ids_to_update) + + def _split_known_unknown_distractor(self): + all_ids = set([i for i in range(1, 2000)]) # 2000 is larger than the max category id in TAO-OW. + # `knowns` includes 78 TAO_category_ids that corresponds to 78 COCO classes. + # (The other 2 COCO classes do not have corresponding classes in TAO). + self.knowns = {4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115, + 1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185, + 699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347, + 371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018} + # `distractors` is defined as in the paper "Opening up Open-World Tracking" + self.distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567, + 569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883, + 912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220} + self.unknowns = all_ids.difference(self.knowns.union(self.distractors)) + + def _filter_gt_data(self, raw_gt_data): + """ + Filter out irrelevant data in the raw_gt_data + Args: + raw_gt_data: directly loaded from json. + + Returns: + filtered gt_data + """ + valid_cat_ids = list() + if self.subset == "known": + valid_cat_ids = self.knowns + elif self.subset == "distractor": + valid_cat_ids = self.distractors + elif self.subset == "unknown": + valid_cat_ids = self.unknowns + # elif self.subset == "test_only_unknowns": + # valid_cat_ids = test_only_unknowns + else: + raise Exception("The parameter `SUBSET` is incorrect") + + filtered = dict() + filtered["videos"] = raw_gt_data["videos"] + # filtered["videos"] = list() + unwanted_vid = set() + # for video in raw_gt_data["videos"]: + # datasrc = video["name"].split('/')[1] + # if datasrc in data_srcs: + # filtered["videos"].append(video) + # else: + # unwanted_vid.add(video["id"]) + + filtered["annotations"] = list() + for ann in raw_gt_data["annotations"]: + if (ann["video_id"] not in unwanted_vid) and (ann["category_id"] in valid_cat_ids): + filtered["annotations"].append(ann) + + filtered["tracks"] = list() + for track in raw_gt_data["tracks"]: + if (track["video_id"] not in unwanted_vid) and (track["category_id"] in valid_cat_ids): + filtered["tracks"].append(track) + + filtered["images"] = list() + for image in raw_gt_data["images"]: + if image["video_id"] not in unwanted_vid: + filtered["images"].append(image) + + filtered["categories"] = list() + for cat in raw_gt_data["categories"]: + if cat["id"] in valid_cat_ids: + filtered["categories"].append(cat) + + if "info" in raw_gt_data: + filtered["info"] = raw_gt_data["info"] + if "licenses" in raw_gt_data: + filtered["licenses"] = raw_gt_data["licenses"] + + if "track_id_offsets" in raw_gt_data: + filtered["track_id_offsets"] = raw_gt_data["track_id_offsets"] + + if "split" in raw_gt_data: + filtered["split"] = raw_gt_data["split"] + + return filtered diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/convert_burst_format_to_tao_format.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/convert_burst_format_to_tao_format.py new file mode 100644 index 0000000..129e42c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/convert_burst_format_to_tao_format.py @@ -0,0 +1,39 @@ +import json +import argparse +from .format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter + + +def main(args): + with open(args.gt_input_file) as f: + ali_format_gt = json.load(f) + tao_format_gt = GroundTruthBURSTFormatToTAOFormatConverter( + ali_format_gt, args.split).convert() + with open(args.gt_output_file, 'w') as f: + json.dump(tao_format_gt, f) + + if args.pred_input_file is None: + return + with open(args.pred_input_file) as f: + ali_format_pred = json.load(f) + tao_format_pred = PredictionBURSTFormatToTAOFormatConverter( + tao_format_gt, ali_format_pred, args.split, + args.exemplar_guided).convert() + with open(args.pred_output_file, 'w') as f: + json.dump(tao_format_pred, f) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--gt_input_file', type=str, + default='../data/gt/tsunami/exemplar_guided/validation_all_annotations.json') + parser.add_argument('--gt_output_file', type=str, + default='/tmp/val_gt.json') + parser.add_argument('--pred_input_file', type=str, + default='../data/trackers/tsunami/exemplar_guided/STCN_off_the_shelf/data/results.json') + parser.add_argument('--pred_output_file', type=str, + default='/tmp/pred.json') + parser.add_argument('--split', type=str, default='validation') + parser.add_argument('--exemplar_guided', type=bool, default=True) + args_ = parser.parse_args() + main(args_) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/format_converter.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/format_converter.py new file mode 100644 index 0000000..da43a9a --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/format_converter.py @@ -0,0 +1,259 @@ +import os +import json +import pycocotools.mask as cocomask +from tabulate import tabulate +from typing import Union + + +def _global_track_id(*, local_track_id: Union[str, int], + video_id: Union[str, int], + track_id_mapping) -> int: + # remap local track ids into globally unique ids + return track_id_mapping[str(video_id)][str(local_track_id)] + + +class GroundTruthBURSTFormatToTAOFormatConverter: + def __init__(self, ali_format): + self._ali_format = ali_format + self._split = ali_format['split'] + self._categories = self._make_categories() + self._videos = [] + self._annotations = [] + self._tracks = {} + self._images = [] + self._next_img_id = 0 + self._next_ann_id = 0 + + self._track_id_mapping = self._load_track_id_mapping() + + for seq in ali_format['sequences']: + self._visit_seq(seq) + + def _load_track_id_mapping(self): + id_map = {} + next_global_track_id = 1 + for seq in self._ali_format['sequences']: + seq_id = seq['id'] + seq_id_map = {} + id_map[str(seq_id)] = seq_id_map + for local_track_id in seq['track_category_ids']: + seq_id_map[str(local_track_id)] = next_global_track_id + next_global_track_id += 1 + return id_map + + def global_track_id(self, *, local_track_id: Union[str, int], + video_id: Union[str, int]) -> int: + return _global_track_id(local_track_id=local_track_id, + video_id=video_id, + track_id_mapping=self._track_id_mapping) + + def _visit_seq(self, seq): + self._make_video(seq) + imgs = self._make_images(seq) + self._make_annotations_and_tracks(seq, imgs) + + def _make_images(self, seq): + imgs = [] + for img_path in seq['annotated_image_paths']: + video = self._split + '/' + seq['dataset'] + '/' + seq['seq_name'] + file_name = video + '/' + img_path + + # TODO: once python 3.9 is more common, we can use this nicer and safer code + #stripped = img_path.removesuffix('.jpg').removesuffix('.png').removeprefix('frame') + stripped = img_path.replace('.jpg', '').replace('.png', '').replace('frame', '') + + last = stripped.split('_')[-1] + frame_idx = int(last) + + img = {'id': self._next_img_id, 'video': video, + 'width': seq['width'], 'height': seq['height'], + 'file_name': file_name, + 'frame_index': frame_idx, + 'video_id': seq['id']} + self._next_img_id += 1 + self._images.append(img) + imgs.append(img) + return imgs + + def _make_video(self, seq): + video_id = seq['id'] + dataset = seq['dataset'] + seq_name = seq['seq_name'] + name = f'{self._split}/' + dataset + '/' + seq_name + video = { + 'id': video_id, 'width': seq['width'], 'height': seq['height'], + 'neg_category_ids': seq['neg_category_ids'], + 'not_exhaustive_category_ids': seq['not_exhaustive_category_ids'], + 'name': name, 'metadata': {'dataset': dataset}} + self._videos.append(video) + + def _make_annotations_and_tracks(self, seq, imgs): + video_id = seq['id'] + segs = seq['segmentations'] + assert len(segs) == len(imgs), (len(segs), len(imgs)) + for frame_segs, img in zip(segs, imgs): + for local_track_id, seg in frame_segs.items(): + distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567, + 569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883, + 912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220} + global_track_id = self.global_track_id( + local_track_id=local_track_id, video_id=seq['id']) + rle = seg['rle'] + segmentation = {'counts': rle, + 'size': [img['height'], img['width']]} + image_id = img['id'] + category_id = int(seq['track_category_ids'][local_track_id]) + if category_id in distractors: + continue + coco_bbox = cocomask.toBbox(segmentation) + bbox = [int(x) for x in coco_bbox] + ann = {'segmentation': segmentation, 'id': self._next_ann_id, + 'image_id': image_id, 'category_id': category_id, + 'track_id': global_track_id, 'video_id': video_id, + 'bbox': bbox} + self._next_ann_id += 1 + self._annotations.append(ann) + + if global_track_id not in self._tracks: + track = {'id': global_track_id, 'category_id': category_id, + 'video_id': video_id} + self._tracks[global_track_id] = track + + def convert(self): + tracks = sorted(self._tracks.values(), key=lambda t: t['id']) + return {'videos': self._videos, 'annotations': self._annotations, + 'tracks': tracks, 'images': self._images, + 'categories': self._categories, + 'track_id_mapping': self._track_id_mapping, + 'split': self._split} + + def _make_categories(self): + tao_categories_path = os.path.join(os.path.dirname(__file__), 'tao_categories.json') + with open(tao_categories_path) as f: + return json.load(f) + + +class PredictionBURSTFormatToTAOFormatConverter: + def __init__(self, gt, ali_format, exemplar_guided): + self._gt = gt + self._ali_format = ali_format + if 'split' in ali_format: + self._split = ali_format['split'] + gt_split = self._gt['split'] + assert self._split == gt_split, (self._split, gt_split) + else: + self._split = self._gt['split'] + self._exemplar_guided = exemplar_guided + self._result = [] + self._next_det_id = 0 + + self._img_by_filename = {} + for img in self._gt['images']: + file_name = img['file_name'] + assert file_name not in self._img_by_filename + self._img_by_filename[file_name] = img + + self._gt_track_by_track_id = {} + for track in self._gt['tracks']: + self._gt_track_by_track_id[int(track['id'])] = track + + self._filtered_out_track_ids = set() + + for seq in ali_format['sequences']: + self._visit_seq(seq) + + if exemplar_guided and len(self._filtered_out_track_ids) > 0: + self.print_filter_out_debug_info(ali_format) + + def print_filter_out_debug_info(self, ali_format): + track_ids_in_pred = set() + a_dict_for_debugging = {} + for seq in ali_format['sequences']: + for local_track_id in seq['track_category_ids']: + global_track_id = _global_track_id( + local_track_id=local_track_id, video_id=seq['id'], + track_id_mapping=self._gt['track_id_mapping']) + track_ids_in_pred.add(global_track_id) + a_dict_for_debugging[global_track_id] = {'seq': seq, + 'local_track_id': local_track_id} + print('Number of Track ids in pred:', len(track_ids_in_pred)) + print('Exemplar Guided: Filtered out', + len(self._filtered_out_track_ids), + 'tracks which were not found in the ground truth.') + track_ids_after_filtering = set(d['track_id'] for d in self._result) + print('Number of tracks after filtering:', + len(track_ids_after_filtering)) + problem_tracks = list( + track_ids_in_pred - track_ids_after_filtering - self._filtered_out_track_ids) + if len(problem_tracks) > 0: + print("\nWARNING:", len(problem_tracks), + "object tracks are not present. There could be a number of reasons for this:\n" + "(1) If you are running evaluation for the box/point exemplar-guided task then this is to be expected" + " because your tracker probably didn't predict masks for every ground-truth object instance.\n" + "(2) If you are running evaluation for the mask exemplar-guided task, then this could indicate a " + "problem. Assume that you copied the given first-frame object mask to your predicted result, this " + "should not happen. It could be that your predictions are at the wrong frame-rate i.e. you have no " + "predicted masks for video frames which will be evaluated.\n") + + rows = [] + for xx in problem_tracks: + rows.append([a_dict_for_debugging[xx]['seq']['dataset'], + a_dict_for_debugging[xx]['seq']['seq_name'], + a_dict_for_debugging[xx]['local_track_id']]) + + print("For your reference, the sequence name and track IDs for these missing tracks are:") + print(tabulate(rows, ["Dataset", "Sequence Name", "Track ID"])) + + def _visit_seq(self, seq): + dataset = seq['dataset'] + seq_name = seq['seq_name'] + assert len(seq['segmentations']) == len(seq['annotated_image_paths']) + for frame_segs, img_path in zip(seq['segmentations'], + seq['annotated_image_paths']): + for local_track_id_str, track_det in frame_segs.items(): + rle = track_det['rle'] + + file_name = self._split + '/' + dataset + '/' + seq_name + '/' + img_path + # the result might have a higher frame rate than the ground truth + if file_name not in self._img_by_filename: + continue + + img = self._img_by_filename[file_name] + img_id = img['id'] + height = img['height'] + width = img['width'] + segmentation = {'counts': rle, 'size': [height, width]} + + local_track_id = int(local_track_id_str) + if self._exemplar_guided: + global_track_id = _global_track_id( + local_track_id=local_track_id, video_id=seq['id'], + track_id_mapping=self._gt['track_id_mapping']) + else: + global_track_id = local_track_id + coco_bbox = cocomask.toBbox(segmentation) + bbox = [int(x) for x in coco_bbox] + det = {'id': self._next_det_id, 'image_id': img_id, + 'track_id': global_track_id, 'bbox': bbox, + 'segmentation': segmentation} + if self._exemplar_guided: + if global_track_id not in self._gt_track_by_track_id: + self._filtered_out_track_ids.add(global_track_id) + continue + gt_track = self._gt_track_by_track_id[global_track_id] + category_id = gt_track['category_id'] + det['category_id'] = category_id + elif 'category_id' in track_det: + det['category_id'] = track_det['category_id'] + else: + category_id = seq['track_category_ids'][local_track_id_str] + det['category_id'] = category_id + self._next_det_id += 1 + if 'score' in track_det: + det['score'] = track_det['score'] + else: + det['score'] = 1.0 + self._result.append(det) + + def convert(self): + return self._result diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/tao_categories.json b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/tao_categories.json new file mode 100644 index 0000000..0368949 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_helpers/tao_categories.json @@ -0,0 +1 @@ +[{"id": 1, "synset": "acorn.n.01", "synonyms": ["acorn"], "def": "nut from an oak tree", "name": "acorn"}, {"id": 2, "synset": "aerosol.n.02", "synonyms": ["aerosol_can", "spray_can"], "def": "a dispenser that holds a substance under pressure", "name": "aerosol_can"}, {"id": 3, "synset": "air_conditioner.n.01", "synonyms": ["air_conditioner"], "def": "a machine that keeps air cool and dry", "name": "air_conditioner"}, {"id": 4, "synset": "airplane.n.01", "synonyms": ["airplane", "aeroplane"], "def": "an aircraft that has a fixed wing and is powered by propellers or jets", "name": "airplane"}, {"id": 5, "synset": "alarm_clock.n.01", "synonyms": ["alarm_clock"], "def": "a clock that wakes a sleeper at some preset time", "name": "alarm_clock"}, {"id": 6, "synset": "alcohol.n.01", "synonyms": ["alcohol", "alcoholic_beverage"], "def": "a liquor or brew containing alcohol as the active agent", "name": "alcohol"}, {"id": 7, "synset": "alligator.n.02", "synonyms": ["alligator", "gator"], "def": "amphibious reptiles related to crocodiles but with shorter broader snouts", "name": "alligator"}, {"id": 8, "synset": "almond.n.02", "synonyms": ["almond"], "def": "oval-shaped edible seed of the almond tree", "name": "almond"}, {"id": 9, "synset": "ambulance.n.01", "synonyms": ["ambulance"], "def": "a vehicle that takes people to and from hospitals", "name": "ambulance"}, {"id": 10, "synset": "amplifier.n.01", "synonyms": ["amplifier"], "def": "electronic equipment that increases strength of signals", "name": "amplifier"}, {"id": 11, "synset": "anklet.n.03", "synonyms": ["anklet", "ankle_bracelet"], "def": "an ornament worn around the ankle", "name": "anklet"}, {"id": 12, "synset": "antenna.n.01", "synonyms": ["antenna", "aerial", "transmitting_aerial"], "def": "an electrical device that sends or receives radio or television signals", "name": "antenna"}, {"id": 13, "synset": "apple.n.01", "synonyms": ["apple"], "def": "fruit with red or yellow or green skin and sweet to tart crisp whitish flesh", "name": "apple"}, {"id": 14, "synset": "apple_juice.n.01", "synonyms": ["apple_juice"], "def": "the juice of apples", "name": "apple_juice"}, {"id": 15, "synset": "applesauce.n.01", "synonyms": ["applesauce"], "def": "puree of stewed apples usually sweetened and spiced", "name": "applesauce"}, {"id": 16, "synset": "apricot.n.02", "synonyms": ["apricot"], "def": "downy yellow to rosy-colored fruit resembling a small peach", "name": "apricot"}, {"id": 17, "synset": "apron.n.01", "synonyms": ["apron"], "def": "a garment of cloth that is tied about the waist and worn to protect clothing", "name": "apron"}, {"id": 18, "synset": "aquarium.n.01", "synonyms": ["aquarium", "fish_tank"], "def": "a tank/pool/bowl filled with water for keeping live fish and underwater animals", "name": "aquarium"}, {"id": 19, "synset": "armband.n.02", "synonyms": ["armband"], "def": "a band worn around the upper arm", "name": "armband"}, {"id": 20, "synset": "armchair.n.01", "synonyms": ["armchair"], "def": "chair with a support on each side for arms", "name": "armchair"}, {"id": 21, "synset": "armoire.n.01", "synonyms": ["armoire"], "def": "a large wardrobe or cabinet", "name": "armoire"}, {"id": 22, "synset": "armor.n.01", "synonyms": ["armor", "armour"], "def": "protective covering made of metal and used in combat", "name": "armor"}, {"id": 23, "synset": "artichoke.n.02", "synonyms": ["artichoke"], "def": "a thistlelike flower head with edible fleshy leaves and heart", "name": "artichoke"}, {"id": 24, "synset": "ashcan.n.01", "synonyms": ["trash_can", "garbage_can", "wastebin", "dustbin", "trash_barrel", "trash_bin"], "def": "a bin that holds rubbish until it is collected", "name": "trash_can"}, {"id": 25, "synset": "ashtray.n.01", "synonyms": ["ashtray"], "def": "a receptacle for the ash from smokers' cigars or cigarettes", "name": "ashtray"}, {"id": 26, "synset": "asparagus.n.02", "synonyms": ["asparagus"], "def": "edible young shoots of the asparagus plant", "name": "asparagus"}, {"id": 27, "synset": "atomizer.n.01", "synonyms": ["atomizer", "atomiser", "spray", "sprayer", "nebulizer", "nebuliser"], "def": "a dispenser that turns a liquid (such as perfume) into a fine mist", "name": "atomizer"}, {"id": 28, "synset": "avocado.n.01", "synonyms": ["avocado"], "def": "a pear-shaped fruit with green or blackish skin and rich yellowish pulp enclosing a single large seed", "name": "avocado"}, {"id": 29, "synset": "award.n.02", "synonyms": ["award", "accolade"], "def": "a tangible symbol signifying approval or distinction", "name": "award"}, {"id": 30, "synset": "awning.n.01", "synonyms": ["awning"], "def": "a canopy made of canvas to shelter people or things from rain or sun", "name": "awning"}, {"id": 31, "synset": "ax.n.01", "synonyms": ["ax", "axe"], "def": "an edge tool with a heavy bladed head mounted across a handle", "name": "ax"}, {"id": 32, "synset": "baby_buggy.n.01", "synonyms": ["baby_buggy", "baby_carriage", "perambulator", "pram", "stroller"], "def": "a small vehicle with four wheels in which a baby or child is pushed around", "name": "baby_buggy"}, {"id": 33, "synset": "backboard.n.01", "synonyms": ["basketball_backboard"], "def": "a raised vertical board with basket attached; used to play basketball", "name": "basketball_backboard"}, {"id": 34, "synset": "backpack.n.01", "synonyms": ["backpack", "knapsack", "packsack", "rucksack", "haversack"], "def": "a bag carried by a strap on your back or shoulder", "name": "backpack"}, {"id": 35, "synset": "bag.n.04", "synonyms": ["handbag", "purse", "pocketbook"], "def": "a container used for carrying money and small personal items or accessories", "name": "handbag"}, {"id": 36, "synset": "bag.n.06", "synonyms": ["suitcase", "baggage", "luggage"], "def": "cases used to carry belongings when traveling", "name": "suitcase"}, {"id": 37, "synset": "bagel.n.01", "synonyms": ["bagel", "beigel"], "def": "glazed yeast-raised doughnut-shaped roll with hard crust", "name": "bagel"}, {"id": 38, "synset": "bagpipe.n.01", "synonyms": ["bagpipe"], "def": "a tubular wind instrument; the player blows air into a bag and squeezes it out", "name": "bagpipe"}, {"id": 39, "synset": "baguet.n.01", "synonyms": ["baguet", "baguette"], "def": "narrow French stick loaf", "name": "baguet"}, {"id": 40, "synset": "bait.n.02", "synonyms": ["bait", "lure"], "def": "something used to lure fish or other animals into danger so they can be trapped or killed", "name": "bait"}, {"id": 41, "synset": "ball.n.06", "synonyms": ["ball"], "def": "a spherical object used as a plaything", "name": "ball"}, {"id": 42, "synset": "ballet_skirt.n.01", "synonyms": ["ballet_skirt", "tutu"], "def": "very short skirt worn by ballerinas", "name": "ballet_skirt"}, {"id": 43, "synset": "balloon.n.01", "synonyms": ["balloon"], "def": "large tough nonrigid bag filled with gas or heated air", "name": "balloon"}, {"id": 44, "synset": "bamboo.n.02", "synonyms": ["bamboo"], "def": "woody tropical grass having hollow woody stems", "name": "bamboo"}, {"id": 45, "synset": "banana.n.02", "synonyms": ["banana"], "def": "elongated crescent-shaped yellow fruit with soft sweet flesh", "name": "banana"}, {"id": 46, "synset": "band_aid.n.01", "synonyms": ["Band_Aid"], "def": "trade name for an adhesive bandage to cover small cuts or blisters", "name": "Band_Aid"}, {"id": 47, "synset": "bandage.n.01", "synonyms": ["bandage"], "def": "a piece of soft material that covers and protects an injured part of the body", "name": "bandage"}, {"id": 48, "synset": "bandanna.n.01", "synonyms": ["bandanna", "bandana"], "def": "large and brightly colored handkerchief; often used as a neckerchief", "name": "bandanna"}, {"id": 49, "synset": "banjo.n.01", "synonyms": ["banjo"], "def": "a stringed instrument of the guitar family with a long neck and circular body", "name": "banjo"}, {"id": 50, "synset": "banner.n.01", "synonyms": ["banner", "streamer"], "def": "long strip of cloth or paper used for decoration or advertising", "name": "banner"}, {"id": 51, "synset": "barbell.n.01", "synonyms": ["barbell"], "def": "a bar to which heavy discs are attached at each end; used in weightlifting", "name": "barbell"}, {"id": 52, "synset": "barge.n.01", "synonyms": ["barge"], "def": "a flatbottom boat for carrying heavy loads (especially on canals)", "name": "barge"}, {"id": 53, "synset": "barrel.n.02", "synonyms": ["barrel", "cask"], "def": "a cylindrical container that holds liquids", "name": "barrel"}, {"id": 54, "synset": "barrette.n.01", "synonyms": ["barrette"], "def": "a pin for holding women's hair in place", "name": "barrette"}, {"id": 55, "synset": "barrow.n.03", "synonyms": ["barrow", "garden_cart", "lawn_cart", "wheelbarrow"], "def": "a cart for carrying small loads; has handles and one or more wheels", "name": "barrow"}, {"id": 56, "synset": "base.n.03", "synonyms": ["baseball_base"], "def": "a place that the runner must touch before scoring", "name": "baseball_base"}, {"id": 57, "synset": "baseball.n.02", "synonyms": ["baseball"], "def": "a ball used in playing baseball", "name": "baseball"}, {"id": 58, "synset": "baseball_bat.n.01", "synonyms": ["baseball_bat"], "def": "an implement used in baseball by the batter", "name": "baseball_bat"}, {"id": 59, "synset": "baseball_cap.n.01", "synonyms": ["baseball_cap", "jockey_cap", "golf_cap"], "def": "a cap with a bill", "name": "baseball_cap"}, {"id": 60, "synset": "baseball_glove.n.01", "synonyms": ["baseball_glove", "baseball_mitt"], "def": "the handwear used by fielders in playing baseball", "name": "baseball_glove"}, {"id": 61, "synset": "basket.n.01", "synonyms": ["basket", "handbasket"], "def": "a container that is usually woven and has handles", "name": "basket"}, {"id": 62, "synset": "basket.n.03", "synonyms": ["basketball_hoop"], "def": "metal hoop supporting a net through which players try to throw the basketball", "name": "basketball_hoop"}, {"id": 63, "synset": "basketball.n.02", "synonyms": ["basketball"], "def": "an inflated ball used in playing basketball", "name": "basketball"}, {"id": 64, "synset": "bass_horn.n.01", "synonyms": ["bass_horn", "sousaphone", "tuba"], "def": "the lowest brass wind instrument", "name": "bass_horn"}, {"id": 65, "synset": "bat.n.01", "synonyms": ["bat_(animal)"], "def": "nocturnal mouselike mammal with forelimbs modified to form membranous wings", "name": "bat_(animal)"}, {"id": 66, "synset": "bath_mat.n.01", "synonyms": ["bath_mat"], "def": "a heavy towel or mat to stand on while drying yourself after a bath", "name": "bath_mat"}, {"id": 67, "synset": "bath_towel.n.01", "synonyms": ["bath_towel"], "def": "a large towel; to dry yourself after a bath", "name": "bath_towel"}, {"id": 68, "synset": "bathrobe.n.01", "synonyms": ["bathrobe"], "def": "a loose-fitting robe of towelling; worn after a bath or swim", "name": "bathrobe"}, {"id": 69, "synset": "bathtub.n.01", "synonyms": ["bathtub", "bathing_tub"], "def": "a large open container that you fill with water and use to wash the body", "name": "bathtub"}, {"id": 70, "synset": "batter.n.02", "synonyms": ["batter_(food)"], "def": "a liquid or semiliquid mixture, as of flour, eggs, and milk, used in cooking", "name": "batter_(food)"}, {"id": 71, "synset": "battery.n.02", "synonyms": ["battery"], "def": "a portable device that produces electricity", "name": "battery"}, {"id": 72, "synset": "beach_ball.n.01", "synonyms": ["beachball"], "def": "large and light ball; for play at the seaside", "name": "beachball"}, {"id": 73, "synset": "bead.n.01", "synonyms": ["bead"], "def": "a small ball with a hole through the middle used for ornamentation, jewellery, etc.", "name": "bead"}, {"id": 74, "synset": "beaker.n.01", "synonyms": ["beaker"], "def": "a flatbottomed jar made of glass or plastic; used for chemistry", "name": "beaker"}, {"id": 75, "synset": "bean_curd.n.01", "synonyms": ["bean_curd", "tofu"], "def": "cheeselike food made of curdled soybean milk", "name": "bean_curd"}, {"id": 76, "synset": "beanbag.n.01", "synonyms": ["beanbag"], "def": "a bag filled with dried beans or similar items; used in games or to sit on", "name": "beanbag"}, {"id": 77, "synset": "beanie.n.01", "synonyms": ["beanie", "beany"], "def": "a small skullcap; formerly worn by schoolboys and college freshmen", "name": "beanie"}, {"id": 78, "synset": "bear.n.01", "synonyms": ["bear"], "def": "large carnivorous or omnivorous mammals with shaggy coats and claws", "name": "bear"}, {"id": 79, "synset": "bed.n.01", "synonyms": ["bed"], "def": "a piece of furniture that provides a place to sleep", "name": "bed"}, {"id": 80, "synset": "bedspread.n.01", "synonyms": ["bedspread", "bedcover", "bed_covering", "counterpane", "spread"], "def": "decorative cover for a bed", "name": "bedspread"}, {"id": 81, "synset": "beef.n.01", "synonyms": ["cow"], "def": "cattle that are reared for their meat", "name": "cow"}, {"id": 82, "synset": "beef.n.02", "synonyms": ["beef_(food)", "boeuf_(food)"], "def": "meat from an adult domestic bovine", "name": "beef_(food)"}, {"id": 83, "synset": "beeper.n.01", "synonyms": ["beeper", "pager"], "def": "an device that beeps when the person carrying it is being paged", "name": "beeper"}, {"id": 84, "synset": "beer_bottle.n.01", "synonyms": ["beer_bottle"], "def": "a bottle that holds beer", "name": "beer_bottle"}, {"id": 85, "synset": "beer_can.n.01", "synonyms": ["beer_can"], "def": "a can that holds beer", "name": "beer_can"}, {"id": 86, "synset": "beetle.n.01", "synonyms": ["beetle"], "def": "insect with hard wing covers", "name": "beetle"}, {"id": 87, "synset": "bell.n.01", "synonyms": ["bell"], "def": "a hollow device made of metal that makes a ringing sound when struck", "name": "bell"}, {"id": 88, "synset": "bell_pepper.n.02", "synonyms": ["bell_pepper", "capsicum"], "def": "large bell-shaped sweet pepper in green or red or yellow or orange or black varieties", "name": "bell_pepper"}, {"id": 89, "synset": "belt.n.02", "synonyms": ["belt"], "def": "a band to tie or buckle around the body (usually at the waist)", "name": "belt"}, {"id": 90, "synset": "belt_buckle.n.01", "synonyms": ["belt_buckle"], "def": "the buckle used to fasten a belt", "name": "belt_buckle"}, {"id": 91, "synset": "bench.n.01", "synonyms": ["bench"], "def": "a long seat for more than one person", "name": "bench"}, {"id": 92, "synset": "beret.n.01", "synonyms": ["beret"], "def": "a cap with no brim or bill; made of soft cloth", "name": "beret"}, {"id": 93, "synset": "bib.n.02", "synonyms": ["bib"], "def": "a napkin tied under the chin of a child while eating", "name": "bib"}, {"id": 94, "synset": "bible.n.01", "synonyms": ["Bible"], "def": "the sacred writings of the Christian religions", "name": "Bible"}, {"id": 95, "synset": "bicycle.n.01", "synonyms": ["bicycle", "bike_(bicycle)"], "def": "a wheeled vehicle that has two wheels and is moved by foot pedals", "name": "bicycle"}, {"id": 96, "synset": "bill.n.09", "synonyms": ["visor", "vizor"], "def": "a brim that projects to the front to shade the eyes", "name": "visor"}, {"id": 97, "synset": "binder.n.03", "synonyms": ["binder", "ring-binder"], "def": "holds loose papers or magazines", "name": "binder"}, {"id": 98, "synset": "binoculars.n.01", "synonyms": ["binoculars", "field_glasses", "opera_glasses"], "def": "an optical instrument designed for simultaneous use by both eyes", "name": "binoculars"}, {"id": 99, "synset": "bird.n.01", "synonyms": ["bird"], "def": "animal characterized by feathers and wings", "name": "bird"}, {"id": 100, "synset": "bird_feeder.n.01", "synonyms": ["birdfeeder"], "def": "an outdoor device that supplies food for wild birds", "name": "birdfeeder"}, {"id": 101, "synset": "birdbath.n.01", "synonyms": ["birdbath"], "def": "an ornamental basin (usually in a garden) for birds to bathe in", "name": "birdbath"}, {"id": 102, "synset": "birdcage.n.01", "synonyms": ["birdcage"], "def": "a cage in which a bird can be kept", "name": "birdcage"}, {"id": 103, "synset": "birdhouse.n.01", "synonyms": ["birdhouse"], "def": "a shelter for birds", "name": "birdhouse"}, {"id": 104, "synset": "birthday_cake.n.01", "synonyms": ["birthday_cake"], "def": "decorated cake served at a birthday party", "name": "birthday_cake"}, {"id": 105, "synset": "birthday_card.n.01", "synonyms": ["birthday_card"], "def": "a card expressing a birthday greeting", "name": "birthday_card"}, {"id": 106, "synset": "biscuit.n.01", "synonyms": ["biscuit_(bread)"], "def": "small round bread leavened with baking-powder or soda", "name": "biscuit_(bread)"}, {"id": 107, "synset": "black_flag.n.01", "synonyms": ["pirate_flag"], "def": "a flag usually bearing a white skull and crossbones on a black background", "name": "pirate_flag"}, {"id": 108, "synset": "black_sheep.n.02", "synonyms": ["black_sheep"], "def": "sheep with a black coat", "name": "black_sheep"}, {"id": 109, "synset": "blackboard.n.01", "synonyms": ["blackboard", "chalkboard"], "def": "sheet of slate; for writing with chalk", "name": "blackboard"}, {"id": 110, "synset": "blanket.n.01", "synonyms": ["blanket"], "def": "bedding that keeps a person warm in bed", "name": "blanket"}, {"id": 111, "synset": "blazer.n.01", "synonyms": ["blazer", "sport_jacket", "sport_coat", "sports_jacket", "sports_coat"], "def": "lightweight jacket; often striped in the colors of a club or school", "name": "blazer"}, {"id": 112, "synset": "blender.n.01", "synonyms": ["blender", "liquidizer", "liquidiser"], "def": "an electrically powered mixer that mix or chop or liquefy foods", "name": "blender"}, {"id": 113, "synset": "blimp.n.02", "synonyms": ["blimp"], "def": "a small nonrigid airship used for observation or as a barrage balloon", "name": "blimp"}, {"id": 114, "synset": "blinker.n.01", "synonyms": ["blinker", "flasher"], "def": "a light that flashes on and off; used as a signal or to send messages", "name": "blinker"}, {"id": 115, "synset": "blueberry.n.02", "synonyms": ["blueberry"], "def": "sweet edible dark-blue berries of blueberry plants", "name": "blueberry"}, {"id": 116, "synset": "boar.n.02", "synonyms": ["boar"], "def": "an uncastrated male hog", "name": "boar"}, {"id": 117, "synset": "board.n.09", "synonyms": ["gameboard"], "def": "a flat portable surface (usually rectangular) designed for board games", "name": "gameboard"}, {"id": 118, "synset": "boat.n.01", "synonyms": ["boat", "ship_(boat)"], "def": "a vessel for travel on water", "name": "boat"}, {"id": 119, "synset": "bobbin.n.01", "synonyms": ["bobbin", "spool", "reel"], "def": "a thing around which thread/tape/film or other flexible materials can be wound", "name": "bobbin"}, {"id": 120, "synset": "bobby_pin.n.01", "synonyms": ["bobby_pin", "hairgrip"], "def": "a flat wire hairpin used to hold bobbed hair in place", "name": "bobby_pin"}, {"id": 121, "synset": "boiled_egg.n.01", "synonyms": ["boiled_egg", "coddled_egg"], "def": "egg cooked briefly in the shell in gently boiling water", "name": "boiled_egg"}, {"id": 122, "synset": "bolo_tie.n.01", "synonyms": ["bolo_tie", "bolo", "bola_tie", "bola"], "def": "a cord fastened around the neck with an ornamental clasp and worn as a necktie", "name": "bolo_tie"}, {"id": 123, "synset": "bolt.n.03", "synonyms": ["deadbolt"], "def": "the part of a lock that is engaged or withdrawn with a key", "name": "deadbolt"}, {"id": 124, "synset": "bolt.n.06", "synonyms": ["bolt"], "def": "a screw that screws into a nut to form a fastener", "name": "bolt"}, {"id": 125, "synset": "bonnet.n.01", "synonyms": ["bonnet"], "def": "a hat tied under the chin", "name": "bonnet"}, {"id": 126, "synset": "book.n.01", "synonyms": ["book"], "def": "a written work or composition that has been published", "name": "book"}, {"id": 127, "synset": "book_bag.n.01", "synonyms": ["book_bag"], "def": "a bag in which students carry their books", "name": "book_bag"}, {"id": 128, "synset": "bookcase.n.01", "synonyms": ["bookcase"], "def": "a piece of furniture with shelves for storing books", "name": "bookcase"}, {"id": 129, "synset": "booklet.n.01", "synonyms": ["booklet", "brochure", "leaflet", "pamphlet"], "def": "a small book usually having a paper cover", "name": "booklet"}, {"id": 130, "synset": "bookmark.n.01", "synonyms": ["bookmark", "bookmarker"], "def": "a marker (a piece of paper or ribbon) placed between the pages of a book", "name": "bookmark"}, {"id": 131, "synset": "boom.n.04", "synonyms": ["boom_microphone", "microphone_boom"], "def": "a pole carrying an overhead microphone projected over a film or tv set", "name": "boom_microphone"}, {"id": 132, "synset": "boot.n.01", "synonyms": ["boot"], "def": "footwear that covers the whole foot and lower leg", "name": "boot"}, {"id": 133, "synset": "bottle.n.01", "synonyms": ["bottle"], "def": "a glass or plastic vessel used for storing drinks or other liquids", "name": "bottle"}, {"id": 134, "synset": "bottle_opener.n.01", "synonyms": ["bottle_opener"], "def": "an opener for removing caps or corks from bottles", "name": "bottle_opener"}, {"id": 135, "synset": "bouquet.n.01", "synonyms": ["bouquet"], "def": "an arrangement of flowers that is usually given as a present", "name": "bouquet"}, {"id": 136, "synset": "bow.n.04", "synonyms": ["bow_(weapon)"], "def": "a weapon for shooting arrows", "name": "bow_(weapon)"}, {"id": 137, "synset": "bow.n.08", "synonyms": ["bow_(decorative_ribbons)"], "def": "a decorative interlacing of ribbons", "name": "bow_(decorative_ribbons)"}, {"id": 138, "synset": "bow_tie.n.01", "synonyms": ["bow-tie", "bowtie"], "def": "a man's tie that ties in a bow", "name": "bow-tie"}, {"id": 139, "synset": "bowl.n.03", "synonyms": ["bowl"], "def": "a dish that is round and open at the top for serving foods", "name": "bowl"}, {"id": 140, "synset": "bowl.n.08", "synonyms": ["pipe_bowl"], "def": "a small round container that is open at the top for holding tobacco", "name": "pipe_bowl"}, {"id": 141, "synset": "bowler_hat.n.01", "synonyms": ["bowler_hat", "bowler", "derby_hat", "derby", "plug_hat"], "def": "a felt hat that is round and hard with a narrow brim", "name": "bowler_hat"}, {"id": 142, "synset": "bowling_ball.n.01", "synonyms": ["bowling_ball"], "def": "a large ball with finger holes used in the sport of bowling", "name": "bowling_ball"}, {"id": 143, "synset": "bowling_pin.n.01", "synonyms": ["bowling_pin"], "def": "a club-shaped wooden object used in bowling", "name": "bowling_pin"}, {"id": 144, "synset": "boxing_glove.n.01", "synonyms": ["boxing_glove"], "def": "large glove coverings the fists of a fighter worn for the sport of boxing", "name": "boxing_glove"}, {"id": 145, "synset": "brace.n.06", "synonyms": ["suspenders"], "def": "elastic straps that hold trousers up (usually used in the plural)", "name": "suspenders"}, {"id": 146, "synset": "bracelet.n.02", "synonyms": ["bracelet", "bangle"], "def": "jewelry worn around the wrist for decoration", "name": "bracelet"}, {"id": 147, "synset": "brass.n.07", "synonyms": ["brass_plaque"], "def": "a memorial made of brass", "name": "brass_plaque"}, {"id": 148, "synset": "brassiere.n.01", "synonyms": ["brassiere", "bra", "bandeau"], "def": "an undergarment worn by women to support their breasts", "name": "brassiere"}, {"id": 149, "synset": "bread-bin.n.01", "synonyms": ["bread-bin", "breadbox"], "def": "a container used to keep bread or cake in", "name": "bread-bin"}, {"id": 150, "synset": "breechcloth.n.01", "synonyms": ["breechcloth", "breechclout", "loincloth"], "def": "a garment that provides covering for the loins", "name": "breechcloth"}, {"id": 151, "synset": "bridal_gown.n.01", "synonyms": ["bridal_gown", "wedding_gown", "wedding_dress"], "def": "a gown worn by the bride at a wedding", "name": "bridal_gown"}, {"id": 152, "synset": "briefcase.n.01", "synonyms": ["briefcase"], "def": "a case with a handle; for carrying papers or files or books", "name": "briefcase"}, {"id": 153, "synset": "bristle_brush.n.01", "synonyms": ["bristle_brush"], "def": "a brush that is made with the short stiff hairs of an animal or plant", "name": "bristle_brush"}, {"id": 154, "synset": "broccoli.n.01", "synonyms": ["broccoli"], "def": "plant with dense clusters of tight green flower buds", "name": "broccoli"}, {"id": 155, "synset": "brooch.n.01", "synonyms": ["broach"], "def": "a decorative pin worn by women", "name": "broach"}, {"id": 156, "synset": "broom.n.01", "synonyms": ["broom"], "def": "bundle of straws or twigs attached to a long handle; used for cleaning", "name": "broom"}, {"id": 157, "synset": "brownie.n.03", "synonyms": ["brownie"], "def": "square or bar of very rich chocolate cake usually with nuts", "name": "brownie"}, {"id": 158, "synset": "brussels_sprouts.n.01", "synonyms": ["brussels_sprouts"], "def": "the small edible cabbage-like buds growing along a stalk", "name": "brussels_sprouts"}, {"id": 159, "synset": "bubble_gum.n.01", "synonyms": ["bubble_gum"], "def": "a kind of chewing gum that can be blown into bubbles", "name": "bubble_gum"}, {"id": 160, "synset": "bucket.n.01", "synonyms": ["bucket", "pail"], "def": "a roughly cylindrical vessel that is open at the top", "name": "bucket"}, {"id": 161, "synset": "buggy.n.01", "synonyms": ["horse_buggy"], "def": "a small lightweight carriage; drawn by a single horse", "name": "horse_buggy"}, {"id": 162, "synset": "bull.n.11", "synonyms": ["bull"], "def": "mature male cow", "name": "bull"}, {"id": 163, "synset": "bulldog.n.01", "synonyms": ["bulldog"], "def": "a thickset short-haired dog with a large head and strong undershot lower jaw", "name": "bulldog"}, {"id": 164, "synset": "bulldozer.n.01", "synonyms": ["bulldozer", "dozer"], "def": "large powerful tractor; a large blade in front flattens areas of ground", "name": "bulldozer"}, {"id": 165, "synset": "bullet_train.n.01", "synonyms": ["bullet_train"], "def": "a high-speed passenger train", "name": "bullet_train"}, {"id": 166, "synset": "bulletin_board.n.02", "synonyms": ["bulletin_board", "notice_board"], "def": "a board that hangs on a wall; displays announcements", "name": "bulletin_board"}, {"id": 167, "synset": "bulletproof_vest.n.01", "synonyms": ["bulletproof_vest"], "def": "a vest capable of resisting the impact of a bullet", "name": "bulletproof_vest"}, {"id": 168, "synset": "bullhorn.n.01", "synonyms": ["bullhorn", "megaphone"], "def": "a portable loudspeaker with built-in microphone and amplifier", "name": "bullhorn"}, {"id": 169, "synset": "bully_beef.n.01", "synonyms": ["corned_beef", "corn_beef"], "def": "beef cured or pickled in brine", "name": "corned_beef"}, {"id": 170, "synset": "bun.n.01", "synonyms": ["bun", "roll"], "def": "small rounded bread either plain or sweet", "name": "bun"}, {"id": 171, "synset": "bunk_bed.n.01", "synonyms": ["bunk_bed"], "def": "beds built one above the other", "name": "bunk_bed"}, {"id": 172, "synset": "buoy.n.01", "synonyms": ["buoy"], "def": "a float attached by rope to the seabed to mark channels in a harbor or underwater hazards", "name": "buoy"}, {"id": 173, "synset": "burrito.n.01", "synonyms": ["burrito"], "def": "a flour tortilla folded around a filling", "name": "burrito"}, {"id": 174, "synset": "bus.n.01", "synonyms": ["bus_(vehicle)", "autobus", "charabanc", "double-decker", "motorbus", "motorcoach"], "def": "a vehicle carrying many passengers; used for public transport", "name": "bus_(vehicle)"}, {"id": 175, "synset": "business_card.n.01", "synonyms": ["business_card"], "def": "a card on which are printed the person's name and business affiliation", "name": "business_card"}, {"id": 176, "synset": "butcher_knife.n.01", "synonyms": ["butcher_knife"], "def": "a large sharp knife for cutting or trimming meat", "name": "butcher_knife"}, {"id": 177, "synset": "butter.n.01", "synonyms": ["butter"], "def": "an edible emulsion of fat globules made by churning milk or cream; for cooking and table use", "name": "butter"}, {"id": 178, "synset": "butterfly.n.01", "synonyms": ["butterfly"], "def": "insect typically having a slender body with knobbed antennae and broad colorful wings", "name": "butterfly"}, {"id": 179, "synset": "button.n.01", "synonyms": ["button"], "def": "a round fastener sewn to shirts and coats etc to fit through buttonholes", "name": "button"}, {"id": 180, "synset": "cab.n.03", "synonyms": ["cab_(taxi)", "taxi", "taxicab"], "def": "a car that takes passengers where they want to go in exchange for money", "name": "cab_(taxi)"}, {"id": 181, "synset": "cabana.n.01", "synonyms": ["cabana"], "def": "a small tent used as a dressing room beside the sea or a swimming pool", "name": "cabana"}, {"id": 182, "synset": "cabin_car.n.01", "synonyms": ["cabin_car", "caboose"], "def": "a car on a freight train for use of the train crew; usually the last car on the train", "name": "cabin_car"}, {"id": 183, "synset": "cabinet.n.01", "synonyms": ["cabinet"], "def": "a piece of furniture resembling a cupboard with doors and shelves and drawers", "name": "cabinet"}, {"id": 184, "synset": "cabinet.n.03", "synonyms": ["locker", "storage_locker"], "def": "a storage compartment for clothes and valuables; usually it has a lock", "name": "locker"}, {"id": 185, "synset": "cake.n.03", "synonyms": ["cake"], "def": "baked goods made from or based on a mixture of flour, sugar, eggs, and fat", "name": "cake"}, {"id": 186, "synset": "calculator.n.02", "synonyms": ["calculator"], "def": "a small machine that is used for mathematical calculations", "name": "calculator"}, {"id": 187, "synset": "calendar.n.02", "synonyms": ["calendar"], "def": "a list or register of events (appointments/social events/court cases, etc)", "name": "calendar"}, {"id": 188, "synset": "calf.n.01", "synonyms": ["calf"], "def": "young of domestic cattle", "name": "calf"}, {"id": 189, "synset": "camcorder.n.01", "synonyms": ["camcorder"], "def": "a portable television camera and videocassette recorder", "name": "camcorder"}, {"id": 190, "synset": "camel.n.01", "synonyms": ["camel"], "def": "cud-chewing mammal used as a draft or saddle animal in desert regions", "name": "camel"}, {"id": 191, "synset": "camera.n.01", "synonyms": ["camera"], "def": "equipment for taking photographs", "name": "camera"}, {"id": 192, "synset": "camera_lens.n.01", "synonyms": ["camera_lens"], "def": "a lens that focuses the image in a camera", "name": "camera_lens"}, {"id": 193, "synset": "camper.n.02", "synonyms": ["camper_(vehicle)", "camping_bus", "motor_home"], "def": "a recreational vehicle equipped for camping out while traveling", "name": "camper_(vehicle)"}, {"id": 194, "synset": "can.n.01", "synonyms": ["can", "tin_can"], "def": "airtight sealed metal container for food or drink or paint etc.", "name": "can"}, {"id": 195, "synset": "can_opener.n.01", "synonyms": ["can_opener", "tin_opener"], "def": "a device for cutting cans open", "name": "can_opener"}, {"id": 196, "synset": "candelabrum.n.01", "synonyms": ["candelabrum", "candelabra"], "def": "branched candlestick; ornamental; has several lights", "name": "candelabrum"}, {"id": 197, "synset": "candle.n.01", "synonyms": ["candle", "candlestick"], "def": "stick of wax with a wick in the middle", "name": "candle"}, {"id": 198, "synset": "candlestick.n.01", "synonyms": ["candle_holder"], "def": "a holder with sockets for candles", "name": "candle_holder"}, {"id": 199, "synset": "candy_bar.n.01", "synonyms": ["candy_bar"], "def": "a candy shaped as a bar", "name": "candy_bar"}, {"id": 200, "synset": "candy_cane.n.01", "synonyms": ["candy_cane"], "def": "a hard candy in the shape of a rod (usually with stripes)", "name": "candy_cane"}, {"id": 201, "synset": "cane.n.01", "synonyms": ["walking_cane"], "def": "a stick that people can lean on to help them walk", "name": "walking_cane"}, {"id": 202, "synset": "canister.n.02", "synonyms": ["canister", "cannister"], "def": "metal container for storing dry foods such as tea or flour", "name": "canister"}, {"id": 203, "synset": "cannon.n.02", "synonyms": ["cannon"], "def": "heavy gun fired from a tank", "name": "cannon"}, {"id": 204, "synset": "canoe.n.01", "synonyms": ["canoe"], "def": "small and light boat; pointed at both ends; propelled with a paddle", "name": "canoe"}, {"id": 205, "synset": "cantaloup.n.02", "synonyms": ["cantaloup", "cantaloupe"], "def": "the fruit of a cantaloup vine; small to medium-sized melon with yellowish flesh", "name": "cantaloup"}, {"id": 206, "synset": "canteen.n.01", "synonyms": ["canteen"], "def": "a flask for carrying water; used by soldiers or travelers", "name": "canteen"}, {"id": 207, "synset": "cap.n.01", "synonyms": ["cap_(headwear)"], "def": "a tight-fitting headwear", "name": "cap_(headwear)"}, {"id": 208, "synset": "cap.n.02", "synonyms": ["bottle_cap", "cap_(container_lid)"], "def": "a top (as for a bottle)", "name": "bottle_cap"}, {"id": 209, "synset": "cape.n.02", "synonyms": ["cape"], "def": "a sleeveless garment like a cloak but shorter", "name": "cape"}, {"id": 210, "synset": "cappuccino.n.01", "synonyms": ["cappuccino", "coffee_cappuccino"], "def": "equal parts of espresso and steamed milk", "name": "cappuccino"}, {"id": 211, "synset": "car.n.01", "synonyms": ["car_(automobile)", "auto_(automobile)", "automobile"], "def": "a motor vehicle with four wheels", "name": "car_(automobile)"}, {"id": 212, "synset": "car.n.02", "synonyms": ["railcar_(part_of_a_train)", "railway_car_(part_of_a_train)", "railroad_car_(part_of_a_train)"], "def": "a wheeled vehicle adapted to the rails of railroad", "name": "railcar_(part_of_a_train)"}, {"id": 213, "synset": "car.n.04", "synonyms": ["elevator_car"], "def": "where passengers ride up and down", "name": "elevator_car"}, {"id": 214, "synset": "car_battery.n.01", "synonyms": ["car_battery", "automobile_battery"], "def": "a battery in a motor vehicle", "name": "car_battery"}, {"id": 215, "synset": "card.n.02", "synonyms": ["identity_card"], "def": "a card certifying the identity of the bearer", "name": "identity_card"}, {"id": 216, "synset": "card.n.03", "synonyms": ["card"], "def": "a rectangular piece of paper used to send messages (e.g. greetings or pictures)", "name": "card"}, {"id": 217, "synset": "cardigan.n.01", "synonyms": ["cardigan"], "def": "knitted jacket that is fastened up the front with buttons or a zipper", "name": "cardigan"}, {"id": 218, "synset": "cargo_ship.n.01", "synonyms": ["cargo_ship", "cargo_vessel"], "def": "a ship designed to carry cargo", "name": "cargo_ship"}, {"id": 219, "synset": "carnation.n.01", "synonyms": ["carnation"], "def": "plant with pink to purple-red spice-scented usually double flowers", "name": "carnation"}, {"id": 220, "synset": "carriage.n.02", "synonyms": ["horse_carriage"], "def": "a vehicle with wheels drawn by one or more horses", "name": "horse_carriage"}, {"id": 221, "synset": "carrot.n.01", "synonyms": ["carrot"], "def": "deep orange edible root of the cultivated carrot plant", "name": "carrot"}, {"id": 222, "synset": "carryall.n.01", "synonyms": ["tote_bag"], "def": "a capacious bag or basket", "name": "tote_bag"}, {"id": 223, "synset": "cart.n.01", "synonyms": ["cart"], "def": "a heavy open wagon usually having two wheels and drawn by an animal", "name": "cart"}, {"id": 224, "synset": "carton.n.02", "synonyms": ["carton"], "def": "a box made of cardboard; opens by flaps on top", "name": "carton"}, {"id": 225, "synset": "cash_register.n.01", "synonyms": ["cash_register", "register_(for_cash_transactions)"], "def": "a cashbox with an adding machine to register transactions", "name": "cash_register"}, {"id": 226, "synset": "casserole.n.01", "synonyms": ["casserole"], "def": "food cooked and served in a casserole", "name": "casserole"}, {"id": 227, "synset": "cassette.n.01", "synonyms": ["cassette"], "def": "a container that holds a magnetic tape used for recording or playing sound or video", "name": "cassette"}, {"id": 228, "synset": "cast.n.05", "synonyms": ["cast", "plaster_cast", "plaster_bandage"], "def": "bandage consisting of a firm covering that immobilizes broken bones while they heal", "name": "cast"}, {"id": 229, "synset": "cat.n.01", "synonyms": ["cat"], "def": "a domestic house cat", "name": "cat"}, {"id": 230, "synset": "cauliflower.n.02", "synonyms": ["cauliflower"], "def": "edible compact head of white undeveloped flowers", "name": "cauliflower"}, {"id": 231, "synset": "caviar.n.01", "synonyms": ["caviar", "caviare"], "def": "salted roe of sturgeon or other large fish; usually served as an hors d'oeuvre", "name": "caviar"}, {"id": 232, "synset": "cayenne.n.02", "synonyms": ["cayenne_(spice)", "cayenne_pepper_(spice)", "red_pepper_(spice)"], "def": "ground pods and seeds of pungent red peppers of the genus Capsicum", "name": "cayenne_(spice)"}, {"id": 233, "synset": "cd_player.n.01", "synonyms": ["CD_player"], "def": "electronic equipment for playing compact discs (CDs)", "name": "CD_player"}, {"id": 234, "synset": "celery.n.01", "synonyms": ["celery"], "def": "widely cultivated herb with aromatic leaf stalks that are eaten raw or cooked", "name": "celery"}, {"id": 235, "synset": "cellular_telephone.n.01", "synonyms": ["cellular_telephone", "cellular_phone", "cellphone", "mobile_phone", "smart_phone"], "def": "a hand-held mobile telephone", "name": "cellular_telephone"}, {"id": 236, "synset": "chain_mail.n.01", "synonyms": ["chain_mail", "ring_mail", "chain_armor", "chain_armour", "ring_armor", "ring_armour"], "def": "(Middle Ages) flexible armor made of interlinked metal rings", "name": "chain_mail"}, {"id": 237, "synset": "chair.n.01", "synonyms": ["chair"], "def": "a seat for one person, with a support for the back", "name": "chair"}, {"id": 238, "synset": "chaise_longue.n.01", "synonyms": ["chaise_longue", "chaise", "daybed"], "def": "a long chair; for reclining", "name": "chaise_longue"}, {"id": 239, "synset": "champagne.n.01", "synonyms": ["champagne"], "def": "a white sparkling wine produced in Champagne or resembling that produced there", "name": "champagne"}, {"id": 240, "synset": "chandelier.n.01", "synonyms": ["chandelier"], "def": "branched lighting fixture; often ornate; hangs from the ceiling", "name": "chandelier"}, {"id": 241, "synset": "chap.n.04", "synonyms": ["chap"], "def": "leather leggings without a seat; worn over trousers by cowboys to protect their legs", "name": "chap"}, {"id": 242, "synset": "checkbook.n.01", "synonyms": ["checkbook", "chequebook"], "def": "a book issued to holders of checking accounts", "name": "checkbook"}, {"id": 243, "synset": "checkerboard.n.01", "synonyms": ["checkerboard"], "def": "a board having 64 squares of two alternating colors", "name": "checkerboard"}, {"id": 244, "synset": "cherry.n.03", "synonyms": ["cherry"], "def": "a red fruit with a single hard stone", "name": "cherry"}, {"id": 245, "synset": "chessboard.n.01", "synonyms": ["chessboard"], "def": "a checkerboard used to play chess", "name": "chessboard"}, {"id": 246, "synset": "chest_of_drawers.n.01", "synonyms": ["chest_of_drawers_(furniture)", "bureau_(furniture)", "chest_(furniture)"], "def": "furniture with drawers for keeping clothes", "name": "chest_of_drawers_(furniture)"}, {"id": 247, "synset": "chicken.n.02", "synonyms": ["chicken_(animal)"], "def": "a domestic fowl bred for flesh or eggs", "name": "chicken_(animal)"}, {"id": 248, "synset": "chicken_wire.n.01", "synonyms": ["chicken_wire"], "def": "a galvanized wire network with a hexagonal mesh; used to build fences", "name": "chicken_wire"}, {"id": 249, "synset": "chickpea.n.01", "synonyms": ["chickpea", "garbanzo"], "def": "the seed of the chickpea plant; usually dried", "name": "chickpea"}, {"id": 250, "synset": "chihuahua.n.03", "synonyms": ["Chihuahua"], "def": "an old breed of tiny short-haired dog with protruding eyes from Mexico", "name": "Chihuahua"}, {"id": 251, "synset": "chili.n.02", "synonyms": ["chili_(vegetable)", "chili_pepper_(vegetable)", "chilli_(vegetable)", "chilly_(vegetable)", "chile_(vegetable)"], "def": "very hot and finely tapering pepper of special pungency", "name": "chili_(vegetable)"}, {"id": 252, "synset": "chime.n.01", "synonyms": ["chime", "gong"], "def": "an instrument consisting of a set of bells that are struck with a hammer", "name": "chime"}, {"id": 253, "synset": "chinaware.n.01", "synonyms": ["chinaware"], "def": "dishware made of high quality porcelain", "name": "chinaware"}, {"id": 254, "synset": "chip.n.04", "synonyms": ["crisp_(potato_chip)", "potato_chip"], "def": "a thin crisp slice of potato fried in deep fat", "name": "crisp_(potato_chip)"}, {"id": 255, "synset": "chip.n.06", "synonyms": ["poker_chip"], "def": "a small disk-shaped counter used to represent money when gambling", "name": "poker_chip"}, {"id": 256, "synset": "chocolate_bar.n.01", "synonyms": ["chocolate_bar"], "def": "a bar of chocolate candy", "name": "chocolate_bar"}, {"id": 257, "synset": "chocolate_cake.n.01", "synonyms": ["chocolate_cake"], "def": "cake containing chocolate", "name": "chocolate_cake"}, {"id": 258, "synset": "chocolate_milk.n.01", "synonyms": ["chocolate_milk"], "def": "milk flavored with chocolate syrup", "name": "chocolate_milk"}, {"id": 259, "synset": "chocolate_mousse.n.01", "synonyms": ["chocolate_mousse"], "def": "dessert mousse made with chocolate", "name": "chocolate_mousse"}, {"id": 260, "synset": "choker.n.03", "synonyms": ["choker", "collar", "neckband"], "def": "necklace that fits tightly around the neck", "name": "choker"}, {"id": 261, "synset": "chopping_board.n.01", "synonyms": ["chopping_board", "cutting_board", "chopping_block"], "def": "a wooden board where meats or vegetables can be cut", "name": "chopping_board"}, {"id": 262, "synset": "chopstick.n.01", "synonyms": ["chopstick"], "def": "one of a pair of slender sticks used as oriental tableware to eat food with", "name": "chopstick"}, {"id": 263, "synset": "christmas_tree.n.05", "synonyms": ["Christmas_tree"], "def": "an ornamented evergreen used as a Christmas decoration", "name": "Christmas_tree"}, {"id": 264, "synset": "chute.n.02", "synonyms": ["slide"], "def": "sloping channel through which things can descend", "name": "slide"}, {"id": 265, "synset": "cider.n.01", "synonyms": ["cider", "cyder"], "def": "a beverage made from juice pressed from apples", "name": "cider"}, {"id": 266, "synset": "cigar_box.n.01", "synonyms": ["cigar_box"], "def": "a box for holding cigars", "name": "cigar_box"}, {"id": 267, "synset": "cigarette.n.01", "synonyms": ["cigarette"], "def": "finely ground tobacco wrapped in paper; for smoking", "name": "cigarette"}, {"id": 268, "synset": "cigarette_case.n.01", "synonyms": ["cigarette_case", "cigarette_pack"], "def": "a small flat case for holding cigarettes", "name": "cigarette_case"}, {"id": 269, "synset": "cistern.n.02", "synonyms": ["cistern", "water_tank"], "def": "a tank that holds the water used to flush a toilet", "name": "cistern"}, {"id": 270, "synset": "clarinet.n.01", "synonyms": ["clarinet"], "def": "a single-reed instrument with a straight tube", "name": "clarinet"}, {"id": 271, "synset": "clasp.n.01", "synonyms": ["clasp"], "def": "a fastener (as a buckle or hook) that is used to hold two things together", "name": "clasp"}, {"id": 272, "synset": "cleansing_agent.n.01", "synonyms": ["cleansing_agent", "cleanser", "cleaner"], "def": "a preparation used in cleaning something", "name": "cleansing_agent"}, {"id": 273, "synset": "clementine.n.01", "synonyms": ["clementine"], "def": "a variety of mandarin orange", "name": "clementine"}, {"id": 274, "synset": "clip.n.03", "synonyms": ["clip"], "def": "any of various small fasteners used to hold loose articles together", "name": "clip"}, {"id": 275, "synset": "clipboard.n.01", "synonyms": ["clipboard"], "def": "a small writing board with a clip at the top for holding papers", "name": "clipboard"}, {"id": 276, "synset": "clock.n.01", "synonyms": ["clock", "timepiece", "timekeeper"], "def": "a timepiece that shows the time of day", "name": "clock"}, {"id": 277, "synset": "clock_tower.n.01", "synonyms": ["clock_tower"], "def": "a tower with a large clock visible high up on an outside face", "name": "clock_tower"}, {"id": 278, "synset": "clothes_hamper.n.01", "synonyms": ["clothes_hamper", "laundry_basket", "clothes_basket"], "def": "a hamper that holds dirty clothes to be washed or wet clothes to be dried", "name": "clothes_hamper"}, {"id": 279, "synset": "clothespin.n.01", "synonyms": ["clothespin", "clothes_peg"], "def": "wood or plastic fastener; for holding clothes on a clothesline", "name": "clothespin"}, {"id": 280, "synset": "clutch_bag.n.01", "synonyms": ["clutch_bag"], "def": "a woman's strapless purse that is carried in the hand", "name": "clutch_bag"}, {"id": 281, "synset": "coaster.n.03", "synonyms": ["coaster"], "def": "a covering (plate or mat) that protects the surface of a table", "name": "coaster"}, {"id": 282, "synset": "coat.n.01", "synonyms": ["coat"], "def": "an outer garment that has sleeves and covers the body from shoulder down", "name": "coat"}, {"id": 283, "synset": "coat_hanger.n.01", "synonyms": ["coat_hanger", "clothes_hanger", "dress_hanger"], "def": "a hanger that is shaped like a person's shoulders", "name": "coat_hanger"}, {"id": 284, "synset": "coatrack.n.01", "synonyms": ["coatrack", "hatrack"], "def": "a rack with hooks for temporarily holding coats and hats", "name": "coatrack"}, {"id": 285, "synset": "cock.n.04", "synonyms": ["cock", "rooster"], "def": "adult male chicken", "name": "cock"}, {"id": 286, "synset": "coconut.n.02", "synonyms": ["coconut", "cocoanut"], "def": "large hard-shelled brown oval nut with a fibrous husk", "name": "coconut"}, {"id": 287, "synset": "coffee_filter.n.01", "synonyms": ["coffee_filter"], "def": "filter (usually of paper) that passes the coffee and retains the coffee grounds", "name": "coffee_filter"}, {"id": 288, "synset": "coffee_maker.n.01", "synonyms": ["coffee_maker", "coffee_machine"], "def": "a kitchen appliance for brewing coffee automatically", "name": "coffee_maker"}, {"id": 289, "synset": "coffee_table.n.01", "synonyms": ["coffee_table", "cocktail_table"], "def": "low table where magazines can be placed and coffee or cocktails are served", "name": "coffee_table"}, {"id": 290, "synset": "coffeepot.n.01", "synonyms": ["coffeepot"], "def": "tall pot in which coffee is brewed", "name": "coffeepot"}, {"id": 291, "synset": "coil.n.05", "synonyms": ["coil"], "def": "tubing that is wound in a spiral", "name": "coil"}, {"id": 292, "synset": "coin.n.01", "synonyms": ["coin"], "def": "a flat metal piece (usually a disc) used as money", "name": "coin"}, {"id": 293, "synset": "colander.n.01", "synonyms": ["colander", "cullender"], "def": "bowl-shaped strainer; used to wash or drain foods", "name": "colander"}, {"id": 294, "synset": "coleslaw.n.01", "synonyms": ["coleslaw", "slaw"], "def": "basically shredded cabbage", "name": "coleslaw"}, {"id": 295, "synset": "coloring_material.n.01", "synonyms": ["coloring_material", "colouring_material"], "def": "any material used for its color", "name": "coloring_material"}, {"id": 296, "synset": "combination_lock.n.01", "synonyms": ["combination_lock"], "def": "lock that can be opened only by turning dials in a special sequence", "name": "combination_lock"}, {"id": 297, "synset": "comforter.n.04", "synonyms": ["pacifier", "teething_ring"], "def": "device used for an infant to suck or bite on", "name": "pacifier"}, {"id": 298, "synset": "comic_book.n.01", "synonyms": ["comic_book"], "def": "a magazine devoted to comic strips", "name": "comic_book"}, {"id": 299, "synset": "computer_keyboard.n.01", "synonyms": ["computer_keyboard", "keyboard_(computer)"], "def": "a keyboard that is a data input device for computers", "name": "computer_keyboard"}, {"id": 300, "synset": "concrete_mixer.n.01", "synonyms": ["concrete_mixer", "cement_mixer"], "def": "a machine with a large revolving drum in which cement/concrete is mixed", "name": "concrete_mixer"}, {"id": 301, "synset": "cone.n.01", "synonyms": ["cone", "traffic_cone"], "def": "a cone-shaped object used to direct traffic", "name": "cone"}, {"id": 302, "synset": "control.n.09", "synonyms": ["control", "controller"], "def": "a mechanism that controls the operation of a machine", "name": "control"}, {"id": 303, "synset": "convertible.n.01", "synonyms": ["convertible_(automobile)"], "def": "a car that has top that can be folded or removed", "name": "convertible_(automobile)"}, {"id": 304, "synset": "convertible.n.03", "synonyms": ["sofa_bed"], "def": "a sofa that can be converted into a bed", "name": "sofa_bed"}, {"id": 305, "synset": "cookie.n.01", "synonyms": ["cookie", "cooky", "biscuit_(cookie)"], "def": "any of various small flat sweet cakes (`biscuit' is the British term)", "name": "cookie"}, {"id": 306, "synset": "cookie_jar.n.01", "synonyms": ["cookie_jar", "cooky_jar"], "def": "a jar in which cookies are kept (and sometimes money is hidden)", "name": "cookie_jar"}, {"id": 307, "synset": "cooking_utensil.n.01", "synonyms": ["cooking_utensil"], "def": "a kitchen utensil made of material that does not melt easily; used for cooking", "name": "cooking_utensil"}, {"id": 308, "synset": "cooler.n.01", "synonyms": ["cooler_(for_food)", "ice_chest"], "def": "an insulated box for storing food often with ice", "name": "cooler_(for_food)"}, {"id": 309, "synset": "cork.n.04", "synonyms": ["cork_(bottle_plug)", "bottle_cork"], "def": "the plug in the mouth of a bottle (especially a wine bottle)", "name": "cork_(bottle_plug)"}, {"id": 310, "synset": "corkboard.n.01", "synonyms": ["corkboard"], "def": "a sheet consisting of cork granules", "name": "corkboard"}, {"id": 311, "synset": "corkscrew.n.01", "synonyms": ["corkscrew", "bottle_screw"], "def": "a bottle opener that pulls corks", "name": "corkscrew"}, {"id": 312, "synset": "corn.n.03", "synonyms": ["edible_corn", "corn", "maize"], "def": "ears of corn that can be prepared and served for human food", "name": "edible_corn"}, {"id": 313, "synset": "cornbread.n.01", "synonyms": ["cornbread"], "def": "bread made primarily of cornmeal", "name": "cornbread"}, {"id": 314, "synset": "cornet.n.01", "synonyms": ["cornet", "horn", "trumpet"], "def": "a brass musical instrument with a narrow tube and a flared bell and many valves", "name": "cornet"}, {"id": 315, "synset": "cornice.n.01", "synonyms": ["cornice", "valance", "valance_board", "pelmet"], "def": "a decorative framework to conceal curtain fixtures at the top of a window casing", "name": "cornice"}, {"id": 316, "synset": "cornmeal.n.01", "synonyms": ["cornmeal"], "def": "coarsely ground corn", "name": "cornmeal"}, {"id": 317, "synset": "corset.n.01", "synonyms": ["corset", "girdle"], "def": "a woman's close-fitting foundation garment", "name": "corset"}, {"id": 318, "synset": "cos.n.02", "synonyms": ["romaine_lettuce"], "def": "lettuce with long dark-green leaves in a loosely packed elongated head", "name": "romaine_lettuce"}, {"id": 319, "synset": "costume.n.04", "synonyms": ["costume"], "def": "the attire characteristic of a country or a time or a social class", "name": "costume"}, {"id": 320, "synset": "cougar.n.01", "synonyms": ["cougar", "puma", "catamount", "mountain_lion", "panther"], "def": "large American feline resembling a lion", "name": "cougar"}, {"id": 321, "synset": "coverall.n.01", "synonyms": ["coverall"], "def": "a loose-fitting protective garment that is worn over other clothing", "name": "coverall"}, {"id": 322, "synset": "cowbell.n.01", "synonyms": ["cowbell"], "def": "a bell hung around the neck of cow so that the cow can be easily located", "name": "cowbell"}, {"id": 323, "synset": "cowboy_hat.n.01", "synonyms": ["cowboy_hat", "ten-gallon_hat"], "def": "a hat with a wide brim and a soft crown; worn by American ranch hands", "name": "cowboy_hat"}, {"id": 324, "synset": "crab.n.01", "synonyms": ["crab_(animal)"], "def": "decapod having eyes on short stalks and a broad flattened shell and pincers", "name": "crab_(animal)"}, {"id": 325, "synset": "cracker.n.01", "synonyms": ["cracker"], "def": "a thin crisp wafer", "name": "cracker"}, {"id": 326, "synset": "crape.n.01", "synonyms": ["crape", "crepe", "French_pancake"], "def": "small very thin pancake", "name": "crape"}, {"id": 327, "synset": "crate.n.01", "synonyms": ["crate"], "def": "a rugged box (usually made of wood); used for shipping", "name": "crate"}, {"id": 328, "synset": "crayon.n.01", "synonyms": ["crayon", "wax_crayon"], "def": "writing or drawing implement made of a colored stick of composition wax", "name": "crayon"}, {"id": 329, "synset": "cream_pitcher.n.01", "synonyms": ["cream_pitcher"], "def": "a small pitcher for serving cream", "name": "cream_pitcher"}, {"id": 330, "synset": "credit_card.n.01", "synonyms": ["credit_card", "charge_card", "debit_card"], "def": "a card, usually plastic, used to pay for goods and services", "name": "credit_card"}, {"id": 331, "synset": "crescent_roll.n.01", "synonyms": ["crescent_roll", "croissant"], "def": "very rich flaky crescent-shaped roll", "name": "crescent_roll"}, {"id": 332, "synset": "crib.n.01", "synonyms": ["crib", "cot"], "def": "baby bed with high sides made of slats", "name": "crib"}, {"id": 333, "synset": "crock.n.03", "synonyms": ["crock_pot", "earthenware_jar"], "def": "an earthen jar (made of baked clay)", "name": "crock_pot"}, {"id": 334, "synset": "crossbar.n.01", "synonyms": ["crossbar"], "def": "a horizontal bar that goes across something", "name": "crossbar"}, {"id": 335, "synset": "crouton.n.01", "synonyms": ["crouton"], "def": "a small piece of toasted or fried bread; served in soup or salads", "name": "crouton"}, {"id": 336, "synset": "crow.n.01", "synonyms": ["crow"], "def": "black birds having a raucous call", "name": "crow"}, {"id": 337, "synset": "crown.n.04", "synonyms": ["crown"], "def": "an ornamental jeweled headdress signifying sovereignty", "name": "crown"}, {"id": 338, "synset": "crucifix.n.01", "synonyms": ["crucifix"], "def": "representation of the cross on which Jesus died", "name": "crucifix"}, {"id": 339, "synset": "cruise_ship.n.01", "synonyms": ["cruise_ship", "cruise_liner"], "def": "a passenger ship used commercially for pleasure cruises", "name": "cruise_ship"}, {"id": 340, "synset": "cruiser.n.01", "synonyms": ["police_cruiser", "patrol_car", "police_car", "squad_car"], "def": "a car in which policemen cruise the streets", "name": "police_cruiser"}, {"id": 341, "synset": "crumb.n.03", "synonyms": ["crumb"], "def": "small piece of e.g. bread or cake", "name": "crumb"}, {"id": 342, "synset": "crutch.n.01", "synonyms": ["crutch"], "def": "a wooden or metal staff that fits under the armpit and reaches to the ground", "name": "crutch"}, {"id": 343, "synset": "cub.n.03", "synonyms": ["cub_(animal)"], "def": "the young of certain carnivorous mammals such as the bear or wolf or lion", "name": "cub_(animal)"}, {"id": 344, "synset": "cube.n.05", "synonyms": ["cube", "square_block"], "def": "a block in the (approximate) shape of a cube", "name": "cube"}, {"id": 345, "synset": "cucumber.n.02", "synonyms": ["cucumber", "cuke"], "def": "cylindrical green fruit with thin green rind and white flesh eaten as a vegetable", "name": "cucumber"}, {"id": 346, "synset": "cufflink.n.01", "synonyms": ["cufflink"], "def": "jewelry consisting of linked buttons used to fasten the cuffs of a shirt", "name": "cufflink"}, {"id": 347, "synset": "cup.n.01", "synonyms": ["cup"], "def": "a small open container usually used for drinking; usually has a handle", "name": "cup", "merged": [{"frequency": "f", "id": 504, "synset": "glass.n.02", "image_count": 92, "instance_count": 313, "synonyms": ["glass_(drink_container)", "drinking_glass"], "def": "a container for holding liquids while drinking", "name": "glass_(drink_container)"}, {"frequency": "f", "id": 720, "synset": "mug.n.04", "image_count": 39, "instance_count": 93, "synonyms": ["mug"], "def": "with handle and usually cylindrical", "name": "mug"}]}, {"id": 348, "synset": "cup.n.08", "synonyms": ["trophy_cup"], "def": "a metal vessel with handles that is awarded as a trophy to a competition winner", "name": "trophy_cup"}, {"id": 349, "synset": "cupcake.n.01", "synonyms": ["cupcake"], "def": "small cake baked in a muffin tin", "name": "cupcake"}, {"id": 350, "synset": "curler.n.01", "synonyms": ["hair_curler", "hair_roller", "hair_crimper"], "def": "a cylindrical tube around which the hair is wound to curl it", "name": "hair_curler"}, {"id": 351, "synset": "curling_iron.n.01", "synonyms": ["curling_iron"], "def": "a cylindrical home appliance that heats hair that has been curled around it", "name": "curling_iron"}, {"id": 352, "synset": "curtain.n.01", "synonyms": ["curtain", "drapery"], "def": "hanging cloth used as a blind (especially for a window)", "name": "curtain"}, {"id": 353, "synset": "cushion.n.03", "synonyms": ["cushion"], "def": "a soft bag filled with air or padding such as feathers or foam rubber", "name": "cushion"}, {"id": 354, "synset": "custard.n.01", "synonyms": ["custard"], "def": "sweetened mixture of milk and eggs baked or boiled or frozen", "name": "custard"}, {"id": 355, "synset": "cutter.n.06", "synonyms": ["cutting_tool"], "def": "a cutting implement; a tool for cutting", "name": "cutting_tool"}, {"id": 356, "synset": "cylinder.n.04", "synonyms": ["cylinder"], "def": "a cylindrical container", "name": "cylinder"}, {"id": 357, "synset": "cymbal.n.01", "synonyms": ["cymbal"], "def": "a percussion instrument consisting of a concave brass disk", "name": "cymbal"}, {"id": 358, "synset": "dachshund.n.01", "synonyms": ["dachshund", "dachsie", "badger_dog"], "def": "small long-bodied short-legged breed of dog having a short sleek coat and long drooping ears", "name": "dachshund"}, {"id": 359, "synset": "dagger.n.01", "synonyms": ["dagger"], "def": "a short knife with a pointed blade used for piercing or stabbing", "name": "dagger"}, {"id": 360, "synset": "dartboard.n.01", "synonyms": ["dartboard"], "def": "a circular board of wood or cork used as the target in the game of darts", "name": "dartboard"}, {"id": 361, "synset": "date.n.08", "synonyms": ["date_(fruit)"], "def": "sweet edible fruit of the date palm with a single long woody seed", "name": "date_(fruit)"}, {"id": 362, "synset": "deck_chair.n.01", "synonyms": ["deck_chair", "beach_chair"], "def": "a folding chair for use outdoors; a wooden frame supports a length of canvas", "name": "deck_chair"}, {"id": 363, "synset": "deer.n.01", "synonyms": ["deer", "cervid"], "def": "distinguished from Bovidae by the male's having solid deciduous antlers", "name": "deer"}, {"id": 364, "synset": "dental_floss.n.01", "synonyms": ["dental_floss", "floss"], "def": "a soft thread for cleaning the spaces between the teeth", "name": "dental_floss"}, {"id": 365, "synset": "desk.n.01", "synonyms": ["desk"], "def": "a piece of furniture with a writing surface and usually drawers or other compartments", "name": "desk"}, {"id": 366, "synset": "detergent.n.01", "synonyms": ["detergent"], "def": "a surface-active chemical widely used in industry and laundering", "name": "detergent"}, {"id": 367, "synset": "diaper.n.01", "synonyms": ["diaper"], "def": "garment consisting of a folded cloth drawn up between the legs and fastened at the waist", "name": "diaper"}, {"id": 368, "synset": "diary.n.01", "synonyms": ["diary", "journal"], "def": "a daily written record of (usually personal) experiences and observations", "name": "diary"}, {"id": 369, "synset": "die.n.01", "synonyms": ["die", "dice"], "def": "a small cube with 1 to 6 spots on the six faces; used in gambling", "name": "die"}, {"id": 370, "synset": "dinghy.n.01", "synonyms": ["dinghy", "dory", "rowboat"], "def": "a small boat of shallow draft with seats and oars with which it is propelled", "name": "dinghy"}, {"id": 371, "synset": "dining_table.n.01", "synonyms": ["dining_table"], "def": "a table at which meals are served", "name": "dining_table"}, {"id": 372, "synset": "dinner_jacket.n.01", "synonyms": ["tux", "tuxedo"], "def": "semiformal evening dress for men", "name": "tux"}, {"id": 373, "synset": "dish.n.01", "synonyms": ["dish"], "def": "a piece of dishware normally used as a container for holding or serving food", "name": "dish"}, {"id": 374, "synset": "dish.n.05", "synonyms": ["dish_antenna"], "def": "directional antenna consisting of a parabolic reflector", "name": "dish_antenna"}, {"id": 375, "synset": "dishrag.n.01", "synonyms": ["dishrag", "dishcloth"], "def": "a cloth for washing dishes", "name": "dishrag"}, {"id": 376, "synset": "dishtowel.n.01", "synonyms": ["dishtowel", "tea_towel"], "def": "a towel for drying dishes", "name": "dishtowel"}, {"id": 377, "synset": "dishwasher.n.01", "synonyms": ["dishwasher", "dishwashing_machine"], "def": "a machine for washing dishes", "name": "dishwasher"}, {"id": 378, "synset": "dishwasher_detergent.n.01", "synonyms": ["dishwasher_detergent", "dishwashing_detergent", "dishwashing_liquid"], "def": "a low-sudsing detergent designed for use in dishwashers", "name": "dishwasher_detergent"}, {"id": 379, "synset": "diskette.n.01", "synonyms": ["diskette", "floppy", "floppy_disk"], "def": "a small plastic magnetic disk enclosed in a stiff envelope used to store data", "name": "diskette"}, {"id": 380, "synset": "dispenser.n.01", "synonyms": ["dispenser"], "def": "a container so designed that the contents can be used in prescribed amounts", "name": "dispenser"}, {"id": 381, "synset": "dixie_cup.n.01", "synonyms": ["Dixie_cup", "paper_cup"], "def": "a disposable cup made of paper; for holding drinks", "name": "Dixie_cup"}, {"id": 382, "synset": "dog.n.01", "synonyms": ["dog"], "def": "a common domesticated dog", "name": "dog"}, {"id": 383, "synset": "dog_collar.n.01", "synonyms": ["dog_collar"], "def": "a collar for a dog", "name": "dog_collar"}, {"id": 384, "synset": "doll.n.01", "synonyms": ["doll"], "def": "a toy replica of a HUMAN (NOT AN ANIMAL)", "name": "doll"}, {"id": 385, "synset": "dollar.n.02", "synonyms": ["dollar", "dollar_bill", "one_dollar_bill"], "def": "a piece of paper money worth one dollar", "name": "dollar"}, {"id": 386, "synset": "dolphin.n.02", "synonyms": ["dolphin"], "def": "any of various small toothed whales with a beaklike snout; larger than porpoises", "name": "dolphin"}, {"id": 387, "synset": "domestic_ass.n.01", "synonyms": ["domestic_ass", "donkey"], "def": "domestic beast of burden descended from the African wild ass; patient but stubborn", "name": "domestic_ass"}, {"id": 388, "synset": "domino.n.03", "synonyms": ["eye_mask"], "def": "a mask covering the upper part of the face but with holes for the eyes", "name": "eye_mask"}, {"id": 389, "synset": "doorbell.n.01", "synonyms": ["doorbell", "buzzer"], "def": "a button at an outer door that gives a ringing or buzzing signal when pushed", "name": "doorbell"}, {"id": 390, "synset": "doorknob.n.01", "synonyms": ["doorknob", "doorhandle"], "def": "a knob used to open a door (often called `doorhandle' in Great Britain)", "name": "doorknob"}, {"id": 391, "synset": "doormat.n.02", "synonyms": ["doormat", "welcome_mat"], "def": "a mat placed outside an exterior door for wiping the shoes before entering", "name": "doormat"}, {"id": 392, "synset": "doughnut.n.02", "synonyms": ["doughnut", "donut"], "def": "a small ring-shaped friedcake", "name": "doughnut"}, {"id": 393, "synset": "dove.n.01", "synonyms": ["dove"], "def": "any of numerous small pigeons", "name": "dove"}, {"id": 394, "synset": "dragonfly.n.01", "synonyms": ["dragonfly"], "def": "slender-bodied non-stinging insect having iridescent wings that are outspread at rest", "name": "dragonfly"}, {"id": 395, "synset": "drawer.n.01", "synonyms": ["drawer"], "def": "a boxlike container in a piece of furniture; made so as to slide in and out", "name": "drawer"}, {"id": 396, "synset": "drawers.n.01", "synonyms": ["underdrawers", "boxers", "boxershorts"], "def": "underpants worn by men", "name": "underdrawers"}, {"id": 397, "synset": "dress.n.01", "synonyms": ["dress", "frock"], "def": "a one-piece garment for a woman; has skirt and bodice", "name": "dress"}, {"id": 398, "synset": "dress_hat.n.01", "synonyms": ["dress_hat", "high_hat", "opera_hat", "silk_hat", "top_hat"], "def": "a man's hat with a tall crown; usually covered with silk or with beaver fur", "name": "dress_hat"}, {"id": 399, "synset": "dress_suit.n.01", "synonyms": ["dress_suit"], "def": "formalwear consisting of full evening dress for men", "name": "dress_suit"}, {"id": 400, "synset": "dresser.n.05", "synonyms": ["dresser"], "def": "a cabinet with shelves", "name": "dresser"}, {"id": 401, "synset": "drill.n.01", "synonyms": ["drill"], "def": "a tool with a sharp rotating point for making holes in hard materials", "name": "drill"}, {"id": 402, "synset": "drinking_fountain.n.01", "synonyms": ["drinking_fountain"], "def": "a public fountain to provide a jet of drinking water", "name": "drinking_fountain"}, {"id": 403, "synset": "drone.n.04", "synonyms": ["drone"], "def": "an aircraft without a pilot that is operated by remote control", "name": "drone"}, {"id": 404, "synset": "dropper.n.01", "synonyms": ["dropper", "eye_dropper"], "def": "pipet consisting of a small tube with a vacuum bulb at one end for drawing liquid in and releasing it a drop at a time", "name": "dropper"}, {"id": 405, "synset": "drum.n.01", "synonyms": ["drum_(musical_instrument)"], "def": "a musical percussion instrument; usually consists of a hollow cylinder with a membrane stretched across each end", "name": "drum_(musical_instrument)"}, {"id": 406, "synset": "drumstick.n.02", "synonyms": ["drumstick"], "def": "a stick used for playing a drum", "name": "drumstick"}, {"id": 407, "synset": "duck.n.01", "synonyms": ["duck"], "def": "small web-footed broad-billed swimming bird", "name": "duck"}, {"id": 408, "synset": "duckling.n.02", "synonyms": ["duckling"], "def": "young duck", "name": "duckling"}, {"id": 409, "synset": "duct_tape.n.01", "synonyms": ["duct_tape"], "def": "a wide silvery adhesive tape", "name": "duct_tape"}, {"id": 410, "synset": "duffel_bag.n.01", "synonyms": ["duffel_bag", "duffle_bag", "duffel", "duffle"], "def": "a large cylindrical bag of heavy cloth", "name": "duffel_bag"}, {"id": 411, "synset": "dumbbell.n.01", "synonyms": ["dumbbell"], "def": "an exercising weight with two ball-like ends connected by a short handle", "name": "dumbbell"}, {"id": 412, "synset": "dumpster.n.01", "synonyms": ["dumpster"], "def": "a container designed to receive and transport and dump waste", "name": "dumpster"}, {"id": 413, "synset": "dustpan.n.02", "synonyms": ["dustpan"], "def": "a short-handled receptacle into which dust can be swept", "name": "dustpan"}, {"id": 414, "synset": "dutch_oven.n.02", "synonyms": ["Dutch_oven"], "def": "iron or earthenware cooking pot; used for stews", "name": "Dutch_oven"}, {"id": 415, "synset": "eagle.n.01", "synonyms": ["eagle"], "def": "large birds of prey noted for their broad wings and strong soaring flight", "name": "eagle"}, {"id": 416, "synset": "earphone.n.01", "synonyms": ["earphone", "earpiece", "headphone"], "def": "device for listening to audio that is held over or inserted into the ear", "name": "earphone"}, {"id": 417, "synset": "earplug.n.01", "synonyms": ["earplug"], "def": "a soft plug that is inserted into the ear canal to block sound", "name": "earplug"}, {"id": 418, "synset": "earring.n.01", "synonyms": ["earring"], "def": "jewelry to ornament the ear", "name": "earring"}, {"id": 419, "synset": "easel.n.01", "synonyms": ["easel"], "def": "an upright tripod for displaying something (usually an artist's canvas)", "name": "easel"}, {"id": 420, "synset": "eclair.n.01", "synonyms": ["eclair"], "def": "oblong cream puff", "name": "eclair"}, {"id": 421, "synset": "eel.n.01", "synonyms": ["eel"], "def": "an elongate fish with fatty flesh", "name": "eel"}, {"id": 422, "synset": "egg.n.02", "synonyms": ["egg", "eggs"], "def": "oval reproductive body of a fowl (especially a hen) used as food", "name": "egg"}, {"id": 423, "synset": "egg_roll.n.01", "synonyms": ["egg_roll", "spring_roll"], "def": "minced vegetables and meat wrapped in a pancake and fried", "name": "egg_roll"}, {"id": 424, "synset": "egg_yolk.n.01", "synonyms": ["egg_yolk", "yolk_(egg)"], "def": "the yellow spherical part of an egg", "name": "egg_yolk"}, {"id": 425, "synset": "eggbeater.n.02", "synonyms": ["eggbeater", "eggwhisk"], "def": "a mixer for beating eggs or whipping cream", "name": "eggbeater"}, {"id": 426, "synset": "eggplant.n.01", "synonyms": ["eggplant", "aubergine"], "def": "egg-shaped vegetable having a shiny skin typically dark purple", "name": "eggplant"}, {"id": 427, "synset": "electric_chair.n.01", "synonyms": ["electric_chair"], "def": "a chair-shaped instrument of execution by electrocution", "name": "electric_chair"}, {"id": 428, "synset": "electric_refrigerator.n.01", "synonyms": ["refrigerator"], "def": "a refrigerator in which the coolant is pumped around by an electric motor", "name": "refrigerator"}, {"id": 429, "synset": "elephant.n.01", "synonyms": ["elephant"], "def": "a common elephant", "name": "elephant"}, {"id": 430, "synset": "elk.n.01", "synonyms": ["elk", "moose"], "def": "large northern deer with enormous flattened antlers in the male", "name": "elk"}, {"id": 431, "synset": "envelope.n.01", "synonyms": ["envelope"], "def": "a flat (usually rectangular) container for a letter, thin package, etc.", "name": "envelope"}, {"id": 432, "synset": "eraser.n.01", "synonyms": ["eraser"], "def": "an implement used to erase something", "name": "eraser"}, {"id": 433, "synset": "escargot.n.01", "synonyms": ["escargot"], "def": "edible snail usually served in the shell with a sauce of melted butter and garlic", "name": "escargot"}, {"id": 434, "synset": "eyepatch.n.01", "synonyms": ["eyepatch"], "def": "a protective cloth covering for an injured eye", "name": "eyepatch"}, {"id": 435, "synset": "falcon.n.01", "synonyms": ["falcon"], "def": "birds of prey having long pointed powerful wings adapted for swift flight", "name": "falcon"}, {"id": 436, "synset": "fan.n.01", "synonyms": ["fan"], "def": "a device for creating a current of air by movement of a surface or surfaces", "name": "fan"}, {"id": 437, "synset": "faucet.n.01", "synonyms": ["faucet", "spigot", "tap"], "def": "a regulator for controlling the flow of a liquid from a reservoir", "name": "faucet"}, {"id": 438, "synset": "fedora.n.01", "synonyms": ["fedora"], "def": "a hat made of felt with a creased crown", "name": "fedora"}, {"id": 439, "synset": "ferret.n.02", "synonyms": ["ferret"], "def": "domesticated albino variety of the European polecat bred for hunting rats and rabbits", "name": "ferret"}, {"id": 440, "synset": "ferris_wheel.n.01", "synonyms": ["Ferris_wheel"], "def": "a large wheel with suspended seats that remain upright as the wheel rotates", "name": "Ferris_wheel"}, {"id": 441, "synset": "ferry.n.01", "synonyms": ["ferry", "ferryboat"], "def": "a boat that transports people or vehicles across a body of water and operates on a regular schedule", "name": "ferry"}, {"id": 442, "synset": "fig.n.04", "synonyms": ["fig_(fruit)"], "def": "fleshy sweet pear-shaped yellowish or purple fruit eaten fresh or preserved or dried", "name": "fig_(fruit)"}, {"id": 443, "synset": "fighter.n.02", "synonyms": ["fighter_jet", "fighter_aircraft", "attack_aircraft"], "def": "a high-speed military or naval airplane designed to destroy enemy targets", "name": "fighter_jet"}, {"id": 444, "synset": "figurine.n.01", "synonyms": ["figurine"], "def": "a small carved or molded figure", "name": "figurine"}, {"id": 445, "synset": "file.n.03", "synonyms": ["file_cabinet", "filing_cabinet"], "def": "office furniture consisting of a container for keeping papers in order", "name": "file_cabinet"}, {"id": 446, "synset": "file.n.04", "synonyms": ["file_(tool)"], "def": "a steel hand tool with small sharp teeth on some or all of its surfaces; used for smoothing wood or metal", "name": "file_(tool)"}, {"id": 447, "synset": "fire_alarm.n.02", "synonyms": ["fire_alarm", "smoke_alarm"], "def": "an alarm that is tripped off by fire or smoke", "name": "fire_alarm"}, {"id": 448, "synset": "fire_engine.n.01", "synonyms": ["fire_engine", "fire_truck"], "def": "large trucks that carry firefighters and equipment to the site of a fire", "name": "fire_engine"}, {"id": 449, "synset": "fire_extinguisher.n.01", "synonyms": ["fire_extinguisher", "extinguisher"], "def": "a manually operated device for extinguishing small fires", "name": "fire_extinguisher"}, {"id": 450, "synset": "fire_hose.n.01", "synonyms": ["fire_hose"], "def": "a large hose that carries water from a fire hydrant to the site of the fire", "name": "fire_hose"}, {"id": 451, "synset": "fireplace.n.01", "synonyms": ["fireplace"], "def": "an open recess in a wall at the base of a chimney where a fire can be built", "name": "fireplace"}, {"id": 452, "synset": "fireplug.n.01", "synonyms": ["fireplug", "fire_hydrant", "hydrant"], "def": "an upright hydrant for drawing water to use in fighting a fire", "name": "fireplug"}, {"id": 453, "synset": "fish.n.01", "synonyms": ["fish"], "def": "any of various mostly cold-blooded aquatic vertebrates usually having scales and breathing through gills", "name": "fish"}, {"id": 454, "synset": "fish.n.02", "synonyms": ["fish_(food)"], "def": "the flesh of fish used as food", "name": "fish_(food)"}, {"id": 455, "synset": "fishbowl.n.02", "synonyms": ["fishbowl", "goldfish_bowl"], "def": "a transparent bowl in which small fish are kept", "name": "fishbowl"}, {"id": 456, "synset": "fishing_boat.n.01", "synonyms": ["fishing_boat", "fishing_vessel"], "def": "a vessel for fishing", "name": "fishing_boat"}, {"id": 457, "synset": "fishing_rod.n.01", "synonyms": ["fishing_rod", "fishing_pole"], "def": "a rod that is used in fishing to extend the fishing line", "name": "fishing_rod"}, {"id": 458, "synset": "flag.n.01", "synonyms": ["flag"], "def": "emblem usually consisting of a rectangular piece of cloth of distinctive design (do not include pole)", "name": "flag"}, {"id": 459, "synset": "flagpole.n.02", "synonyms": ["flagpole", "flagstaff"], "def": "a tall staff or pole on which a flag is raised", "name": "flagpole"}, {"id": 460, "synset": "flamingo.n.01", "synonyms": ["flamingo"], "def": "large pink web-footed bird with down-bent bill", "name": "flamingo"}, {"id": 461, "synset": "flannel.n.01", "synonyms": ["flannel"], "def": "a soft light woolen fabric; used for clothing", "name": "flannel"}, {"id": 462, "synset": "flash.n.10", "synonyms": ["flash", "flashbulb"], "def": "a lamp for providing momentary light to take a photograph", "name": "flash"}, {"id": 463, "synset": "flashlight.n.01", "synonyms": ["flashlight", "torch"], "def": "a small portable battery-powered electric lamp", "name": "flashlight"}, {"id": 464, "synset": "fleece.n.03", "synonyms": ["fleece"], "def": "a soft bulky fabric with deep pile; used chiefly for clothing", "name": "fleece"}, {"id": 465, "synset": "flip-flop.n.02", "synonyms": ["flip-flop_(sandal)"], "def": "a backless sandal held to the foot by a thong between two toes", "name": "flip-flop_(sandal)"}, {"id": 466, "synset": "flipper.n.01", "synonyms": ["flipper_(footwear)", "fin_(footwear)"], "def": "a shoe to aid a person in swimming", "name": "flipper_(footwear)"}, {"id": 467, "synset": "flower_arrangement.n.01", "synonyms": ["flower_arrangement", "floral_arrangement"], "def": "a decorative arrangement of flowers", "name": "flower_arrangement"}, {"id": 468, "synset": "flute.n.02", "synonyms": ["flute_glass", "champagne_flute"], "def": "a tall narrow wineglass", "name": "flute_glass"}, {"id": 469, "synset": "foal.n.01", "synonyms": ["foal"], "def": "a young horse", "name": "foal"}, {"id": 470, "synset": "folding_chair.n.01", "synonyms": ["folding_chair"], "def": "a chair that can be folded flat for storage", "name": "folding_chair"}, {"id": 471, "synset": "food_processor.n.01", "synonyms": ["food_processor"], "def": "a kitchen appliance for shredding, blending, chopping, or slicing food", "name": "food_processor"}, {"id": 472, "synset": "football.n.02", "synonyms": ["football_(American)"], "def": "the inflated oblong ball used in playing American football", "name": "football_(American)"}, {"id": 473, "synset": "football_helmet.n.01", "synonyms": ["football_helmet"], "def": "a padded helmet with a face mask to protect the head of football players", "name": "football_helmet"}, {"id": 474, "synset": "footstool.n.01", "synonyms": ["footstool", "footrest"], "def": "a low seat or a stool to rest the feet of a seated person", "name": "footstool"}, {"id": 475, "synset": "fork.n.01", "synonyms": ["fork"], "def": "cutlery used for serving and eating food", "name": "fork"}, {"id": 476, "synset": "forklift.n.01", "synonyms": ["forklift"], "def": "an industrial vehicle with a power operated fork in front that can be inserted under loads to lift and move them", "name": "forklift"}, {"id": 477, "synset": "freight_car.n.01", "synonyms": ["freight_car"], "def": "a railway car that carries freight", "name": "freight_car"}, {"id": 478, "synset": "french_toast.n.01", "synonyms": ["French_toast"], "def": "bread slice dipped in egg and milk and fried", "name": "French_toast"}, {"id": 479, "synset": "freshener.n.01", "synonyms": ["freshener", "air_freshener"], "def": "anything that freshens", "name": "freshener"}, {"id": 480, "synset": "frisbee.n.01", "synonyms": ["frisbee"], "def": "a light, plastic disk propelled with a flip of the wrist for recreation or competition", "name": "frisbee"}, {"id": 481, "synset": "frog.n.01", "synonyms": ["frog", "toad", "toad_frog"], "def": "a tailless stout-bodied amphibians with long hind limbs for leaping", "name": "frog"}, {"id": 482, "synset": "fruit_juice.n.01", "synonyms": ["fruit_juice"], "def": "drink produced by squeezing or crushing fruit", "name": "fruit_juice"}, {"id": 483, "synset": "fruit_salad.n.01", "synonyms": ["fruit_salad"], "def": "salad composed of fruits", "name": "fruit_salad"}, {"id": 484, "synset": "frying_pan.n.01", "synonyms": ["frying_pan", "frypan", "skillet"], "def": "a pan used for frying foods", "name": "frying_pan"}, {"id": 485, "synset": "fudge.n.01", "synonyms": ["fudge"], "def": "soft creamy candy", "name": "fudge"}, {"id": 486, "synset": "funnel.n.02", "synonyms": ["funnel"], "def": "a cone-shaped utensil used to channel a substance into a container with a small mouth", "name": "funnel"}, {"id": 487, "synset": "futon.n.01", "synonyms": ["futon"], "def": "a pad that is used for sleeping on the floor or on a raised frame", "name": "futon"}, {"id": 488, "synset": "gag.n.02", "synonyms": ["gag", "muzzle"], "def": "restraint put into a person's mouth to prevent speaking or shouting", "name": "gag"}, {"id": 489, "synset": "garbage.n.03", "synonyms": ["garbage"], "def": "a receptacle where waste can be discarded", "name": "garbage"}, {"id": 490, "synset": "garbage_truck.n.01", "synonyms": ["garbage_truck"], "def": "a truck for collecting domestic refuse", "name": "garbage_truck"}, {"id": 491, "synset": "garden_hose.n.01", "synonyms": ["garden_hose"], "def": "a hose used for watering a lawn or garden", "name": "garden_hose"}, {"id": 492, "synset": "gargle.n.01", "synonyms": ["gargle", "mouthwash"], "def": "a medicated solution used for gargling and rinsing the mouth", "name": "gargle"}, {"id": 493, "synset": "gargoyle.n.02", "synonyms": ["gargoyle"], "def": "an ornament consisting of a grotesquely carved figure of a person or animal", "name": "gargoyle"}, {"id": 494, "synset": "garlic.n.02", "synonyms": ["garlic", "ail"], "def": "aromatic bulb used as seasoning", "name": "garlic"}, {"id": 495, "synset": "gasmask.n.01", "synonyms": ["gasmask", "respirator", "gas_helmet"], "def": "a protective face mask with a filter", "name": "gasmask"}, {"id": 496, "synset": "gazelle.n.01", "synonyms": ["gazelle"], "def": "small swift graceful antelope of Africa and Asia having lustrous eyes", "name": "gazelle"}, {"id": 497, "synset": "gelatin.n.02", "synonyms": ["gelatin", "jelly"], "def": "an edible jelly made with gelatin and used as a dessert or salad base or a coating for foods", "name": "gelatin"}, {"id": 498, "synset": "gem.n.02", "synonyms": ["gemstone"], "def": "a crystalline rock that can be cut and polished for jewelry", "name": "gemstone"}, {"id": 499, "synset": "giant_panda.n.01", "synonyms": ["giant_panda", "panda", "panda_bear"], "def": "large black-and-white herbivorous mammal of bamboo forests of China and Tibet", "name": "giant_panda"}, {"id": 500, "synset": "gift_wrap.n.01", "synonyms": ["gift_wrap"], "def": "attractive wrapping paper suitable for wrapping gifts", "name": "gift_wrap"}, {"id": 501, "synset": "ginger.n.03", "synonyms": ["ginger", "gingerroot"], "def": "the root of the common ginger plant; used fresh as a seasoning", "name": "ginger"}, {"id": 502, "synset": "giraffe.n.01", "synonyms": ["giraffe"], "def": "tall animal having a spotted coat and small horns and very long neck and legs", "name": "giraffe"}, {"id": 503, "synset": "girdle.n.02", "synonyms": ["cincture", "sash", "waistband", "waistcloth"], "def": "a band of material around the waist that strengthens a skirt or trousers", "name": "cincture"}, {"id": 504, "synset": "glass.n.02", "synonyms": ["glass_(drink_container)", "drinking_glass"], "def": "a container for holding liquids while drinking", "name": "glass_(drink_container)"}, {"id": 505, "synset": "globe.n.03", "synonyms": ["globe"], "def": "a sphere on which a map (especially of the earth) is represented", "name": "globe"}, {"id": 506, "synset": "glove.n.02", "synonyms": ["glove"], "def": "handwear covering the hand", "name": "glove"}, {"id": 507, "synset": "goat.n.01", "synonyms": ["goat"], "def": "a common goat", "name": "goat"}, {"id": 508, "synset": "goggles.n.01", "synonyms": ["goggles"], "def": "tight-fitting spectacles worn to protect the eyes", "name": "goggles"}, {"id": 509, "synset": "goldfish.n.01", "synonyms": ["goldfish"], "def": "small golden or orange-red freshwater fishes used as pond or aquarium pets", "name": "goldfish"}, {"id": 510, "synset": "golf_club.n.02", "synonyms": ["golf_club", "golf-club"], "def": "golf equipment used by a golfer to hit a golf ball", "name": "golf_club"}, {"id": 511, "synset": "golfcart.n.01", "synonyms": ["golfcart"], "def": "a small motor vehicle in which golfers can ride between shots", "name": "golfcart"}, {"id": 512, "synset": "gondola.n.02", "synonyms": ["gondola_(boat)"], "def": "long narrow flat-bottomed boat propelled by sculling; traditionally used on canals of Venice", "name": "gondola_(boat)"}, {"id": 513, "synset": "goose.n.01", "synonyms": ["goose"], "def": "loud, web-footed long-necked aquatic birds usually larger than ducks", "name": "goose"}, {"id": 514, "synset": "gorilla.n.01", "synonyms": ["gorilla"], "def": "largest ape", "name": "gorilla"}, {"id": 515, "synset": "gourd.n.02", "synonyms": ["gourd"], "def": "any of numerous inedible fruits with hard rinds", "name": "gourd"}, {"id": 516, "synset": "gown.n.04", "synonyms": ["surgical_gown", "scrubs_(surgical_clothing)"], "def": "protective garment worn by surgeons during operations", "name": "surgical_gown"}, {"id": 517, "synset": "grape.n.01", "synonyms": ["grape"], "def": "any of various juicy fruit with green or purple skins; grow in clusters", "name": "grape"}, {"id": 518, "synset": "grasshopper.n.01", "synonyms": ["grasshopper"], "def": "plant-eating insect with hind legs adapted for leaping", "name": "grasshopper"}, {"id": 519, "synset": "grater.n.01", "synonyms": ["grater"], "def": "utensil with sharp perforations for shredding foods (as vegetables or cheese)", "name": "grater"}, {"id": 520, "synset": "gravestone.n.01", "synonyms": ["gravestone", "headstone", "tombstone"], "def": "a stone that is used to mark a grave", "name": "gravestone"}, {"id": 521, "synset": "gravy_boat.n.01", "synonyms": ["gravy_boat", "gravy_holder"], "def": "a dish (often boat-shaped) for serving gravy or sauce", "name": "gravy_boat"}, {"id": 522, "synset": "green_bean.n.02", "synonyms": ["green_bean"], "def": "a common bean plant cultivated for its slender green edible pods", "name": "green_bean"}, {"id": 523, "synset": "green_onion.n.01", "synonyms": ["green_onion", "spring_onion", "scallion"], "def": "a young onion before the bulb has enlarged", "name": "green_onion"}, {"id": 524, "synset": "griddle.n.01", "synonyms": ["griddle"], "def": "cooking utensil consisting of a flat heated surface on which food is cooked", "name": "griddle"}, {"id": 525, "synset": "grillroom.n.01", "synonyms": ["grillroom", "grill_(restaurant)"], "def": "a restaurant where food is cooked on a grill", "name": "grillroom"}, {"id": 526, "synset": "grinder.n.04", "synonyms": ["grinder_(tool)"], "def": "a machine tool that polishes metal", "name": "grinder_(tool)"}, {"id": 527, "synset": "grits.n.01", "synonyms": ["grits", "hominy_grits"], "def": "coarsely ground corn boiled as a breakfast dish", "name": "grits"}, {"id": 528, "synset": "grizzly.n.01", "synonyms": ["grizzly", "grizzly_bear"], "def": "powerful brownish-yellow bear of the uplands of western North America", "name": "grizzly"}, {"id": 529, "synset": "grocery_bag.n.01", "synonyms": ["grocery_bag"], "def": "a sack for holding customer's groceries", "name": "grocery_bag", "merged": [{"frequency": "f", "id": 912, "synset": "sack.n.01", "image_count": 37, "instance_count": 76, "synonyms": ["plastic_bag", "paper_bag"], "def": "a bag made of paper or plastic for holding customer's purchases", "name": "plastic_bag"}, {"frequency": "c", "id": 967, "synset": "shopping_bag.n.01", "image_count": 9, "instance_count": 18, "synonyms": ["shopping_bag"], "def": "a bag made of plastic or strong paper (often with handles); used to transport goods after shopping", "name": "shopping_bag"}]}, {"id": 530, "synset": "guacamole.n.01", "synonyms": ["guacamole"], "def": "a dip made of mashed avocado mixed with chopped onions and other seasonings", "name": "guacamole"}, {"id": 531, "synset": "guitar.n.01", "synonyms": ["guitar"], "def": "a stringed instrument usually having six strings; played by strumming or plucking", "name": "guitar"}, {"id": 532, "synset": "gull.n.02", "synonyms": ["gull", "seagull"], "def": "mostly white aquatic bird having long pointed wings and short legs", "name": "gull"}, {"id": 533, "synset": "gun.n.01", "synonyms": ["gun"], "def": "a weapon that discharges a bullet at high velocity from a metal tube", "name": "gun"}, {"id": 534, "synset": "hair_spray.n.01", "synonyms": ["hair_spray"], "def": "substance sprayed on the hair to hold it in place", "name": "hair_spray"}, {"id": 535, "synset": "hairbrush.n.01", "synonyms": ["hairbrush"], "def": "a brush used to groom a person's hair", "name": "hairbrush"}, {"id": 536, "synset": "hairnet.n.01", "synonyms": ["hairnet"], "def": "a small net that someone wears over their hair to keep it in place", "name": "hairnet"}, {"id": 537, "synset": "hairpin.n.01", "synonyms": ["hairpin"], "def": "a double pronged pin used to hold women's hair in place", "name": "hairpin"}, {"id": 538, "synset": "ham.n.01", "synonyms": ["ham", "jambon", "gammon"], "def": "meat cut from the thigh of a hog (usually smoked)", "name": "ham"}, {"id": 539, "synset": "hamburger.n.01", "synonyms": ["hamburger", "beefburger", "burger"], "def": "a sandwich consisting of a patty of minced beef served on a bun", "name": "hamburger"}, {"id": 540, "synset": "hammer.n.02", "synonyms": ["hammer"], "def": "a hand tool with a heavy head and a handle; used to deliver an impulsive force by striking", "name": "hammer"}, {"id": 541, "synset": "hammock.n.02", "synonyms": ["hammock"], "def": "a hanging bed of canvas or rope netting (usually suspended between two trees)", "name": "hammock"}, {"id": 542, "synset": "hamper.n.02", "synonyms": ["hamper"], "def": "a basket usually with a cover", "name": "hamper"}, {"id": 543, "synset": "hamster.n.01", "synonyms": ["hamster"], "def": "short-tailed burrowing rodent with large cheek pouches", "name": "hamster"}, {"id": 544, "synset": "hand_blower.n.01", "synonyms": ["hair_dryer"], "def": "a hand-held electric blower that can blow warm air onto the hair", "name": "hair_dryer"}, {"id": 545, "synset": "hand_glass.n.01", "synonyms": ["hand_glass", "hand_mirror"], "def": "a mirror intended to be held in the hand", "name": "hand_glass"}, {"id": 546, "synset": "hand_towel.n.01", "synonyms": ["hand_towel", "face_towel"], "def": "a small towel used to dry the hands or face", "name": "hand_towel"}, {"id": 547, "synset": "handcart.n.01", "synonyms": ["handcart", "pushcart", "hand_truck"], "def": "wheeled vehicle that can be pushed by a person", "name": "handcart"}, {"id": 548, "synset": "handcuff.n.01", "synonyms": ["handcuff"], "def": "shackle that consists of a metal loop that can be locked around the wrist", "name": "handcuff"}, {"id": 549, "synset": "handkerchief.n.01", "synonyms": ["handkerchief"], "def": "a square piece of cloth used for wiping the eyes or nose or as a costume accessory", "name": "handkerchief"}, {"id": 550, "synset": "handle.n.01", "synonyms": ["handle", "grip", "handgrip"], "def": "the appendage to an object that is designed to be held in order to use or move it", "name": "handle"}, {"id": 551, "synset": "handsaw.n.01", "synonyms": ["handsaw", "carpenter's_saw"], "def": "a saw used with one hand for cutting wood", "name": "handsaw"}, {"id": 552, "synset": "hardback.n.01", "synonyms": ["hardback_book", "hardcover_book"], "def": "a book with cardboard or cloth or leather covers", "name": "hardback_book"}, {"id": 553, "synset": "harmonium.n.01", "synonyms": ["harmonium", "organ_(musical_instrument)", "reed_organ_(musical_instrument)"], "def": "a free-reed instrument in which air is forced through the reeds by bellows", "name": "harmonium"}, {"id": 554, "synset": "hat.n.01", "synonyms": ["hat"], "def": "headwear that protects the head from bad weather, sun, or worn for fashion", "name": "hat", "merged": [{"frequency": "c", "id": 207, "synset": "cap.n.01", "image_count": 2, "instance_count": 5, "synonyms": ["cap_(headwear)"], "def": "a tight-fitting headwear", "name": "cap_(headwear)"}]}, {"id": 555, "synset": "hatbox.n.01", "synonyms": ["hatbox"], "def": "a round piece of luggage for carrying hats", "name": "hatbox"}, {"id": 556, "synset": "hatch.n.03", "synonyms": ["hatch"], "def": "a movable barrier covering a hatchway", "name": "hatch"}, {"id": 557, "synset": "head_covering.n.01", "synonyms": ["veil"], "def": "a garment that covers the head and face", "name": "veil"}, {"id": 558, "synset": "headband.n.01", "synonyms": ["headband"], "def": "a band worn around or over the head", "name": "headband"}, {"id": 559, "synset": "headboard.n.01", "synonyms": ["headboard"], "def": "a vertical board or panel forming the head of a bedstead", "name": "headboard"}, {"id": 560, "synset": "headlight.n.01", "synonyms": ["headlight", "headlamp"], "def": "a powerful light with reflector; attached to the front of an automobile or locomotive", "name": "headlight"}, {"id": 561, "synset": "headscarf.n.01", "synonyms": ["headscarf"], "def": "a kerchief worn over the head and tied under the chin", "name": "headscarf"}, {"id": 562, "synset": "headset.n.01", "synonyms": ["headset"], "def": "receiver consisting of a pair of headphones", "name": "headset"}, {"id": 563, "synset": "headstall.n.01", "synonyms": ["headstall_(for_horses)", "headpiece_(for_horses)"], "def": "the band that is the part of a bridle that fits around a horse's head", "name": "headstall_(for_horses)"}, {"id": 564, "synset": "hearing_aid.n.02", "synonyms": ["hearing_aid"], "def": "an acoustic device used to direct sound to the ear of a hearing-impaired person", "name": "hearing_aid"}, {"id": 565, "synset": "heart.n.02", "synonyms": ["heart"], "def": "a muscular organ; its contractions move the blood through the body", "name": "heart"}, {"id": 566, "synset": "heater.n.01", "synonyms": ["heater", "warmer"], "def": "device that heats water or supplies warmth to a room", "name": "heater"}, {"id": 567, "synset": "helicopter.n.01", "synonyms": ["helicopter"], "def": "an aircraft without wings that obtains its lift from the rotation of overhead blades", "name": "helicopter"}, {"id": 568, "synset": "helmet.n.02", "synonyms": ["helmet"], "def": "a protective headgear made of hard material to resist blows", "name": "helmet"}, {"id": 569, "synset": "heron.n.02", "synonyms": ["heron"], "def": "grey or white wading bird with long neck and long legs and (usually) long bill", "name": "heron"}, {"id": 570, "synset": "highchair.n.01", "synonyms": ["highchair", "feeding_chair"], "def": "a chair for feeding a very young child", "name": "highchair"}, {"id": 571, "synset": "hinge.n.01", "synonyms": ["hinge"], "def": "a joint that holds two parts together so that one can swing relative to the other", "name": "hinge"}, {"id": 572, "synset": "hippopotamus.n.01", "synonyms": ["hippopotamus"], "def": "massive thick-skinned animal living in or around rivers of tropical Africa", "name": "hippopotamus"}, {"id": 573, "synset": "hockey_stick.n.01", "synonyms": ["hockey_stick"], "def": "sports implement consisting of a stick used by hockey players to move the puck", "name": "hockey_stick"}, {"id": 574, "synset": "hog.n.03", "synonyms": ["hog", "pig"], "def": "domestic swine", "name": "hog"}, {"id": 575, "synset": "home_plate.n.01", "synonyms": ["home_plate_(baseball)", "home_base_(baseball)"], "def": "(baseball) a rubber slab where the batter stands; it must be touched by a base runner in order to score", "name": "home_plate_(baseball)"}, {"id": 576, "synset": "honey.n.01", "synonyms": ["honey"], "def": "a sweet yellow liquid produced by bees", "name": "honey"}, {"id": 577, "synset": "hood.n.06", "synonyms": ["fume_hood", "exhaust_hood"], "def": "metal covering leading to a vent that exhausts smoke or fumes", "name": "fume_hood"}, {"id": 578, "synset": "hook.n.05", "synonyms": ["hook"], "def": "a curved or bent implement for suspending or pulling something", "name": "hook"}, {"id": 579, "synset": "horse.n.01", "synonyms": ["horse"], "def": "a common horse", "name": "horse"}, {"id": 580, "synset": "hose.n.03", "synonyms": ["hose", "hosepipe"], "def": "a flexible pipe for conveying a liquid or gas", "name": "hose"}, {"id": 581, "synset": "hot-air_balloon.n.01", "synonyms": ["hot-air_balloon"], "def": "balloon for travel through the air in a basket suspended below a large bag of heated air", "name": "hot-air_balloon"}, {"id": 582, "synset": "hot_plate.n.01", "synonyms": ["hotplate"], "def": "a portable electric appliance for heating or cooking or keeping food warm", "name": "hotplate"}, {"id": 583, "synset": "hot_sauce.n.01", "synonyms": ["hot_sauce"], "def": "a pungent peppery sauce", "name": "hot_sauce"}, {"id": 584, "synset": "hourglass.n.01", "synonyms": ["hourglass"], "def": "a sandglass timer that runs for sixty minutes", "name": "hourglass"}, {"id": 585, "synset": "houseboat.n.01", "synonyms": ["houseboat"], "def": "a barge that is designed and equipped for use as a dwelling", "name": "houseboat"}, {"id": 586, "synset": "hummingbird.n.01", "synonyms": ["hummingbird"], "def": "tiny American bird having brilliant iridescent plumage and long slender bills", "name": "hummingbird"}, {"id": 587, "synset": "hummus.n.01", "synonyms": ["hummus", "humus", "hommos", "hoummos", "humous"], "def": "a thick spread made from mashed chickpeas", "name": "hummus"}, {"id": 588, "synset": "ice_bear.n.01", "synonyms": ["polar_bear"], "def": "white bear of Arctic regions", "name": "polar_bear"}, {"id": 589, "synset": "ice_cream.n.01", "synonyms": ["icecream"], "def": "frozen dessert containing cream and sugar and flavoring", "name": "icecream"}, {"id": 590, "synset": "ice_lolly.n.01", "synonyms": ["popsicle"], "def": "ice cream or water ice on a small wooden stick", "name": "popsicle"}, {"id": 591, "synset": "ice_maker.n.01", "synonyms": ["ice_maker"], "def": "an appliance included in some electric refrigerators for making ice cubes", "name": "ice_maker"}, {"id": 592, "synset": "ice_pack.n.01", "synonyms": ["ice_pack", "ice_bag"], "def": "a waterproof bag filled with ice: applied to the body (especially the head) to cool or reduce swelling", "name": "ice_pack"}, {"id": 593, "synset": "ice_skate.n.01", "synonyms": ["ice_skate"], "def": "skate consisting of a boot with a steel blade fitted to the sole", "name": "ice_skate"}, {"id": 594, "synset": "ice_tea.n.01", "synonyms": ["ice_tea", "iced_tea"], "def": "strong tea served over ice", "name": "ice_tea"}, {"id": 595, "synset": "igniter.n.01", "synonyms": ["igniter", "ignitor", "lighter"], "def": "a substance or device used to start a fire", "name": "igniter"}, {"id": 596, "synset": "incense.n.01", "synonyms": ["incense"], "def": "a substance that produces a fragrant odor when burned", "name": "incense"}, {"id": 597, "synset": "inhaler.n.01", "synonyms": ["inhaler", "inhalator"], "def": "a dispenser that produces a chemical vapor to be inhaled through mouth or nose", "name": "inhaler"}, {"id": 598, "synset": "ipod.n.01", "synonyms": ["iPod"], "def": "a pocket-sized device used to play music files", "name": "iPod"}, {"id": 599, "synset": "iron.n.04", "synonyms": ["iron_(for_clothing)", "smoothing_iron_(for_clothing)"], "def": "home appliance consisting of a flat metal base that is heated and used to smooth cloth", "name": "iron_(for_clothing)"}, {"id": 600, "synset": "ironing_board.n.01", "synonyms": ["ironing_board"], "def": "narrow padded board on collapsible supports; used for ironing clothes", "name": "ironing_board"}, {"id": 601, "synset": "jacket.n.01", "synonyms": ["jacket"], "def": "a waist-length coat", "name": "jacket"}, {"id": 602, "synset": "jam.n.01", "synonyms": ["jam"], "def": "preserve of crushed fruit", "name": "jam"}, {"id": 603, "synset": "jean.n.01", "synonyms": ["jean", "blue_jean", "denim"], "def": "(usually plural) close-fitting trousers of heavy denim for manual work or casual wear", "name": "jean"}, {"id": 604, "synset": "jeep.n.01", "synonyms": ["jeep", "landrover"], "def": "a car suitable for traveling over rough terrain", "name": "jeep"}, {"id": 605, "synset": "jelly_bean.n.01", "synonyms": ["jelly_bean", "jelly_egg"], "def": "sugar-glazed jellied candy", "name": "jelly_bean"}, {"id": 606, "synset": "jersey.n.03", "synonyms": ["jersey", "T-shirt", "tee_shirt"], "def": "a close-fitting pullover shirt", "name": "jersey"}, {"id": 607, "synset": "jet.n.01", "synonyms": ["jet_plane", "jet-propelled_plane"], "def": "an airplane powered by one or more jet engines", "name": "jet_plane"}, {"id": 608, "synset": "jewelry.n.01", "synonyms": ["jewelry", "jewellery"], "def": "an adornment (as a bracelet or ring or necklace) made of precious metals and set with gems (or imitation gems)", "name": "jewelry"}, {"id": 609, "synset": "joystick.n.02", "synonyms": ["joystick"], "def": "a control device for computers consisting of a vertical handle that can move freely in two directions", "name": "joystick"}, {"id": 610, "synset": "jump_suit.n.01", "synonyms": ["jumpsuit"], "def": "one-piece garment fashioned after a parachutist's uniform", "name": "jumpsuit"}, {"id": 611, "synset": "kayak.n.01", "synonyms": ["kayak"], "def": "a small canoe consisting of a light frame made watertight with animal skins", "name": "kayak"}, {"id": 612, "synset": "keg.n.02", "synonyms": ["keg"], "def": "small cask or barrel", "name": "keg"}, {"id": 613, "synset": "kennel.n.01", "synonyms": ["kennel", "doghouse"], "def": "outbuilding that serves as a shelter for a dog", "name": "kennel"}, {"id": 614, "synset": "kettle.n.01", "synonyms": ["kettle", "boiler"], "def": "a metal pot for stewing or boiling; usually has a lid", "name": "kettle"}, {"id": 615, "synset": "key.n.01", "synonyms": ["key"], "def": "metal instrument used to unlock a lock", "name": "key"}, {"id": 616, "synset": "keycard.n.01", "synonyms": ["keycard"], "def": "a plastic card used to gain access typically to a door", "name": "keycard"}, {"id": 617, "synset": "kilt.n.01", "synonyms": ["kilt"], "def": "a knee-length pleated tartan skirt worn by men as part of the traditional dress in the Highlands of northern Scotland", "name": "kilt"}, {"id": 618, "synset": "kimono.n.01", "synonyms": ["kimono"], "def": "a loose robe; imitated from robes originally worn by Japanese", "name": "kimono"}, {"id": 619, "synset": "kitchen_sink.n.01", "synonyms": ["kitchen_sink"], "def": "a sink in a kitchen", "name": "kitchen_sink"}, {"id": 620, "synset": "kitchen_table.n.01", "synonyms": ["kitchen_table"], "def": "a table in the kitchen", "name": "kitchen_table"}, {"id": 621, "synset": "kite.n.03", "synonyms": ["kite"], "def": "plaything consisting of a light frame covered with tissue paper; flown in wind at end of a string", "name": "kite"}, {"id": 622, "synset": "kitten.n.01", "synonyms": ["kitten", "kitty"], "def": "young domestic cat", "name": "kitten"}, {"id": 623, "synset": "kiwi.n.03", "synonyms": ["kiwi_fruit"], "def": "fuzzy brown egg-shaped fruit with slightly tart green flesh", "name": "kiwi_fruit"}, {"id": 624, "synset": "knee_pad.n.01", "synonyms": ["knee_pad"], "def": "protective garment consisting of a pad worn by football or baseball or hockey players", "name": "knee_pad"}, {"id": 625, "synset": "knife.n.01", "synonyms": ["knife"], "def": "tool with a blade and point used as a cutting instrument", "name": "knife"}, {"id": 626, "synset": "knight.n.02", "synonyms": ["knight_(chess_piece)", "horse_(chess_piece)"], "def": "a chess game piece shaped to resemble the head of a horse", "name": "knight_(chess_piece)"}, {"id": 627, "synset": "knitting_needle.n.01", "synonyms": ["knitting_needle"], "def": "needle consisting of a slender rod with pointed ends; usually used in pairs", "name": "knitting_needle"}, {"id": 628, "synset": "knob.n.02", "synonyms": ["knob"], "def": "a round handle often found on a door", "name": "knob"}, {"id": 629, "synset": "knocker.n.05", "synonyms": ["knocker_(on_a_door)", "doorknocker"], "def": "a device (usually metal and ornamental) attached by a hinge to a door", "name": "knocker_(on_a_door)"}, {"id": 630, "synset": "koala.n.01", "synonyms": ["koala", "koala_bear"], "def": "sluggish tailless Australian marsupial with grey furry ears and coat", "name": "koala"}, {"id": 631, "synset": "lab_coat.n.01", "synonyms": ["lab_coat", "laboratory_coat"], "def": "a light coat worn to protect clothing from substances used while working in a laboratory", "name": "lab_coat"}, {"id": 632, "synset": "ladder.n.01", "synonyms": ["ladder"], "def": "steps consisting of two parallel members connected by rungs", "name": "ladder"}, {"id": 633, "synset": "ladle.n.01", "synonyms": ["ladle"], "def": "a spoon-shaped vessel with a long handle frequently used to transfer liquids", "name": "ladle"}, {"id": 634, "synset": "ladybug.n.01", "synonyms": ["ladybug", "ladybeetle", "ladybird_beetle"], "def": "small round bright-colored and spotted beetle, typically red and black", "name": "ladybug"}, {"id": 635, "synset": "lamb.n.01", "synonyms": ["lamb_(animal)"], "def": "young sheep", "name": "lamb_(animal)"}, {"id": 636, "synset": "lamb_chop.n.01", "synonyms": ["lamb-chop", "lambchop"], "def": "chop cut from a lamb", "name": "lamb-chop"}, {"id": 637, "synset": "lamp.n.02", "synonyms": ["lamp"], "def": "a piece of furniture holding one or more electric light bulbs", "name": "lamp"}, {"id": 638, "synset": "lamppost.n.01", "synonyms": ["lamppost"], "def": "a metal post supporting an outdoor lamp (such as a streetlight)", "name": "lamppost"}, {"id": 639, "synset": "lampshade.n.01", "synonyms": ["lampshade"], "def": "a protective ornamental shade used to screen a light bulb from direct view", "name": "lampshade"}, {"id": 640, "synset": "lantern.n.01", "synonyms": ["lantern"], "def": "light in a transparent protective case", "name": "lantern"}, {"id": 641, "synset": "lanyard.n.02", "synonyms": ["lanyard", "laniard"], "def": "a cord worn around the neck to hold a knife or whistle, etc.", "name": "lanyard"}, {"id": 642, "synset": "laptop.n.01", "synonyms": ["laptop_computer", "notebook_computer"], "def": "a portable computer small enough to use in your lap", "name": "laptop_computer"}, {"id": 643, "synset": "lasagna.n.01", "synonyms": ["lasagna", "lasagne"], "def": "baked dish of layers of lasagna pasta with sauce and cheese and meat or vegetables", "name": "lasagna"}, {"id": 644, "synset": "latch.n.02", "synonyms": ["latch"], "def": "a bar that can be lowered or slid into a groove to fasten a door or gate", "name": "latch"}, {"id": 645, "synset": "lawn_mower.n.01", "synonyms": ["lawn_mower"], "def": "garden tool for mowing grass on lawns", "name": "lawn_mower"}, {"id": 646, "synset": "leather.n.01", "synonyms": ["leather"], "def": "an animal skin made smooth and flexible by removing the hair and then tanning", "name": "leather"}, {"id": 647, "synset": "legging.n.01", "synonyms": ["legging_(clothing)", "leging_(clothing)", "leg_covering"], "def": "a garment covering the leg (usually extending from the knee to the ankle)", "name": "legging_(clothing)"}, {"id": 648, "synset": "lego.n.01", "synonyms": ["Lego", "Lego_set"], "def": "a child's plastic construction set for making models from blocks", "name": "Lego"}, {"id": 649, "synset": "lemon.n.01", "synonyms": ["lemon"], "def": "yellow oval fruit with juicy acidic flesh", "name": "lemon"}, {"id": 650, "synset": "lemonade.n.01", "synonyms": ["lemonade"], "def": "sweetened beverage of diluted lemon juice", "name": "lemonade"}, {"id": 651, "synset": "lettuce.n.02", "synonyms": ["lettuce"], "def": "leafy plant commonly eaten in salad or on sandwiches", "name": "lettuce"}, {"id": 652, "synset": "license_plate.n.01", "synonyms": ["license_plate", "numberplate"], "def": "a plate mounted on the front and back of car and bearing the car's registration number", "name": "license_plate"}, {"id": 653, "synset": "life_buoy.n.01", "synonyms": ["life_buoy", "lifesaver", "life_belt", "life_ring"], "def": "a ring-shaped life preserver used to prevent drowning (NOT a life-jacket or vest)", "name": "life_buoy"}, {"id": 654, "synset": "life_jacket.n.01", "synonyms": ["life_jacket", "life_vest"], "def": "life preserver consisting of a sleeveless jacket of buoyant or inflatable design", "name": "life_jacket"}, {"id": 655, "synset": "light_bulb.n.01", "synonyms": ["lightbulb"], "def": "glass bulb or tube shaped electric device that emits light (DO NOT MARK LAMPS AS A WHOLE)", "name": "lightbulb"}, {"id": 656, "synset": "lightning_rod.n.02", "synonyms": ["lightning_rod", "lightning_conductor"], "def": "a metallic conductor that is attached to a high point and leads to the ground", "name": "lightning_rod"}, {"id": 657, "synset": "lime.n.06", "synonyms": ["lime"], "def": "the green acidic fruit of any of various lime trees", "name": "lime"}, {"id": 658, "synset": "limousine.n.01", "synonyms": ["limousine"], "def": "long luxurious car; usually driven by a chauffeur", "name": "limousine"}, {"id": 659, "synset": "linen.n.02", "synonyms": ["linen_paper"], "def": "a high-quality paper made of linen fibers or with a linen finish", "name": "linen_paper"}, {"id": 660, "synset": "lion.n.01", "synonyms": ["lion"], "def": "large gregarious predatory cat of Africa and India", "name": "lion"}, {"id": 661, "synset": "lip_balm.n.01", "synonyms": ["lip_balm"], "def": "a balm applied to the lips", "name": "lip_balm"}, {"id": 662, "synset": "lipstick.n.01", "synonyms": ["lipstick", "lip_rouge"], "def": "makeup that is used to color the lips", "name": "lipstick"}, {"id": 663, "synset": "liquor.n.01", "synonyms": ["liquor", "spirits", "hard_liquor", "liqueur", "cordial"], "def": "an alcoholic beverage that is distilled rather than fermented", "name": "liquor"}, {"id": 664, "synset": "lizard.n.01", "synonyms": ["lizard"], "def": "a reptile with usually two pairs of legs and a tapering tail", "name": "lizard"}, {"id": 665, "synset": "loafer.n.02", "synonyms": ["Loafer_(type_of_shoe)"], "def": "a low leather step-in shoe", "name": "Loafer_(type_of_shoe)"}, {"id": 666, "synset": "log.n.01", "synonyms": ["log"], "def": "a segment of the trunk of a tree when stripped of branches", "name": "log"}, {"id": 667, "synset": "lollipop.n.02", "synonyms": ["lollipop"], "def": "hard candy on a stick", "name": "lollipop"}, {"id": 668, "synset": "lotion.n.01", "synonyms": ["lotion"], "def": "any of various cosmetic preparations that are applied to the skin", "name": "lotion"}, {"id": 669, "synset": "loudspeaker.n.01", "synonyms": ["speaker_(stero_equipment)"], "def": "electronic device that produces sound often as part of a stereo system", "name": "speaker_(stero_equipment)"}, {"id": 670, "synset": "love_seat.n.01", "synonyms": ["loveseat"], "def": "small sofa that seats two people", "name": "loveseat"}, {"id": 671, "synset": "machine_gun.n.01", "synonyms": ["machine_gun"], "def": "a rapidly firing automatic gun", "name": "machine_gun"}, {"id": 672, "synset": "magazine.n.02", "synonyms": ["magazine"], "def": "a paperback periodic publication", "name": "magazine"}, {"id": 673, "synset": "magnet.n.01", "synonyms": ["magnet"], "def": "a device that attracts iron and produces a magnetic field", "name": "magnet"}, {"id": 674, "synset": "mail_slot.n.01", "synonyms": ["mail_slot"], "def": "a slot (usually in a door) through which mail can be delivered", "name": "mail_slot"}, {"id": 675, "synset": "mailbox.n.01", "synonyms": ["mailbox_(at_home)", "letter_box_(at_home)"], "def": "a private box for delivery of mail", "name": "mailbox_(at_home)"}, {"id": 676, "synset": "mallet.n.01", "synonyms": ["mallet"], "def": "a sports implement with a long handle and a hammer-like head used to hit a ball", "name": "mallet"}, {"id": 677, "synset": "mammoth.n.01", "synonyms": ["mammoth"], "def": "any of numerous extinct elephants widely distributed in the Pleistocene", "name": "mammoth"}, {"id": 678, "synset": "mandarin.n.05", "synonyms": ["mandarin_orange"], "def": "a somewhat flat reddish-orange loose skinned citrus of China", "name": "mandarin_orange"}, {"id": 679, "synset": "manger.n.01", "synonyms": ["manger", "trough"], "def": "a container (usually in a barn or stable) from which cattle or horses feed", "name": "manger"}, {"id": 680, "synset": "manhole.n.01", "synonyms": ["manhole"], "def": "a hole (usually with a flush cover) through which a person can gain access to an underground structure", "name": "manhole"}, {"id": 681, "synset": "map.n.01", "synonyms": ["map"], "def": "a diagrammatic representation of the earth's surface (or part of it)", "name": "map"}, {"id": 682, "synset": "marker.n.03", "synonyms": ["marker"], "def": "a writing implement for making a mark", "name": "marker"}, {"id": 683, "synset": "martini.n.01", "synonyms": ["martini"], "def": "a cocktail made of gin (or vodka) with dry vermouth", "name": "martini"}, {"id": 684, "synset": "mascot.n.01", "synonyms": ["mascot"], "def": "a person or animal that is adopted by a team or other group as a symbolic figure", "name": "mascot"}, {"id": 685, "synset": "mashed_potato.n.01", "synonyms": ["mashed_potato"], "def": "potato that has been peeled and boiled and then mashed", "name": "mashed_potato"}, {"id": 686, "synset": "masher.n.02", "synonyms": ["masher"], "def": "a kitchen utensil used for mashing (e.g. potatoes)", "name": "masher"}, {"id": 687, "synset": "mask.n.04", "synonyms": ["mask", "facemask"], "def": "a protective covering worn over the face", "name": "mask"}, {"id": 688, "synset": "mast.n.01", "synonyms": ["mast"], "def": "a vertical spar for supporting sails", "name": "mast"}, {"id": 689, "synset": "mat.n.03", "synonyms": ["mat_(gym_equipment)", "gym_mat"], "def": "sports equipment consisting of a piece of thick padding on the floor for gymnastics", "name": "mat_(gym_equipment)"}, {"id": 690, "synset": "matchbox.n.01", "synonyms": ["matchbox"], "def": "a box for holding matches", "name": "matchbox"}, {"id": 691, "synset": "mattress.n.01", "synonyms": ["mattress"], "def": "a thick pad filled with resilient material used as a bed or part of a bed", "name": "mattress"}, {"id": 692, "synset": "measuring_cup.n.01", "synonyms": ["measuring_cup"], "def": "graduated cup used to measure liquid or granular ingredients", "name": "measuring_cup"}, {"id": 693, "synset": "measuring_stick.n.01", "synonyms": ["measuring_stick", "ruler_(measuring_stick)", "measuring_rod"], "def": "measuring instrument having a sequence of marks at regular intervals", "name": "measuring_stick"}, {"id": 694, "synset": "meatball.n.01", "synonyms": ["meatball"], "def": "ground meat formed into a ball and fried or simmered in broth", "name": "meatball"}, {"id": 695, "synset": "medicine.n.02", "synonyms": ["medicine"], "def": "something that treats or prevents or alleviates the symptoms of disease", "name": "medicine"}, {"id": 696, "synset": "melon.n.01", "synonyms": ["melon"], "def": "fruit of the gourd family having a hard rind and sweet juicy flesh", "name": "melon"}, {"id": 697, "synset": "microphone.n.01", "synonyms": ["microphone"], "def": "device for converting sound waves into electrical energy", "name": "microphone"}, {"id": 698, "synset": "microscope.n.01", "synonyms": ["microscope"], "def": "magnifier of the image of small objects", "name": "microscope"}, {"id": 699, "synset": "microwave.n.02", "synonyms": ["microwave_oven"], "def": "kitchen appliance that cooks food by passing an electromagnetic wave through it", "name": "microwave_oven"}, {"id": 700, "synset": "milestone.n.01", "synonyms": ["milestone", "milepost"], "def": "stone post at side of a road to show distances", "name": "milestone"}, {"id": 701, "synset": "milk.n.01", "synonyms": ["milk"], "def": "a white nutritious liquid secreted by mammals and used as food by human beings", "name": "milk"}, {"id": 702, "synset": "minivan.n.01", "synonyms": ["minivan"], "def": "a small box-shaped passenger van", "name": "minivan"}, {"id": 703, "synset": "mint.n.05", "synonyms": ["mint_candy"], "def": "a candy that is flavored with a mint oil", "name": "mint_candy"}, {"id": 704, "synset": "mirror.n.01", "synonyms": ["mirror"], "def": "polished surface that forms images by reflecting light", "name": "mirror"}, {"id": 705, "synset": "mitten.n.01", "synonyms": ["mitten"], "def": "glove that encases the thumb separately and the other four fingers together", "name": "mitten"}, {"id": 706, "synset": "mixer.n.04", "synonyms": ["mixer_(kitchen_tool)", "stand_mixer"], "def": "a kitchen utensil that is used for mixing foods", "name": "mixer_(kitchen_tool)"}, {"id": 707, "synset": "money.n.03", "synonyms": ["money"], "def": "the official currency issued by a government or national bank", "name": "money"}, {"id": 708, "synset": "monitor.n.04", "synonyms": ["monitor_(computer_equipment) computer_monitor"], "def": "a computer monitor", "name": "monitor_(computer_equipment) computer_monitor"}, {"id": 709, "synset": "monkey.n.01", "synonyms": ["monkey"], "def": "any of various long-tailed primates", "name": "monkey"}, {"id": 710, "synset": "motor.n.01", "synonyms": ["motor"], "def": "machine that converts other forms of energy into mechanical energy and so imparts motion", "name": "motor"}, {"id": 711, "synset": "motor_scooter.n.01", "synonyms": ["motor_scooter", "scooter"], "def": "a wheeled vehicle with small wheels and a low-powered engine", "name": "motor_scooter"}, {"id": 712, "synset": "motor_vehicle.n.01", "synonyms": ["motor_vehicle", "automotive_vehicle"], "def": "a self-propelled wheeled vehicle that does not run on rails", "name": "motor_vehicle"}, {"id": 713, "synset": "motorboat.n.01", "synonyms": ["motorboat", "powerboat"], "def": "a boat propelled by an internal-combustion engine", "name": "motorboat"}, {"id": 714, "synset": "motorcycle.n.01", "synonyms": ["motorcycle"], "def": "a motor vehicle with two wheels and a strong frame", "name": "motorcycle"}, {"id": 715, "synset": "mound.n.01", "synonyms": ["mound_(baseball)", "pitcher's_mound"], "def": "(baseball) the slight elevation on which the pitcher stands", "name": "mound_(baseball)"}, {"id": 716, "synset": "mouse.n.01", "synonyms": ["mouse_(animal_rodent)"], "def": "a small rodent with pointed snouts and small ears on elongated bodies with slender usually hairless tails", "name": "mouse_(animal_rodent)"}, {"id": 717, "synset": "mouse.n.04", "synonyms": ["mouse_(computer_equipment)", "computer_mouse"], "def": "a computer input device that controls an on-screen pointer", "name": "mouse_(computer_equipment)"}, {"id": 718, "synset": "mousepad.n.01", "synonyms": ["mousepad"], "def": "a small portable pad that provides an operating surface for a computer mouse", "name": "mousepad"}, {"id": 719, "synset": "muffin.n.01", "synonyms": ["muffin"], "def": "a sweet quick bread baked in a cup-shaped pan", "name": "muffin"}, {"id": 720, "synset": "mug.n.04", "synonyms": ["mug"], "def": "with handle and usually cylindrical", "name": "mug"}, {"id": 721, "synset": "mushroom.n.02", "synonyms": ["mushroom"], "def": "a common mushroom", "name": "mushroom"}, {"id": 722, "synset": "music_stool.n.01", "synonyms": ["music_stool", "piano_stool"], "def": "a stool for piano players; usually adjustable in height", "name": "music_stool"}, {"id": 723, "synset": "musical_instrument.n.01", "synonyms": ["musical_instrument", "instrument_(musical)"], "def": "any of various devices or contrivances that can be used to produce musical tones or sounds", "name": "musical_instrument"}, {"id": 724, "synset": "nailfile.n.01", "synonyms": ["nailfile"], "def": "a small flat file for shaping the nails", "name": "nailfile"}, {"id": 725, "synset": "nameplate.n.01", "synonyms": ["nameplate"], "def": "a plate bearing a name", "name": "nameplate"}, {"id": 726, "synset": "napkin.n.01", "synonyms": ["napkin", "table_napkin", "serviette"], "def": "a small piece of table linen or paper that is used to wipe the mouth and to cover the lap in order to protect clothing", "name": "napkin"}, {"id": 727, "synset": "neckerchief.n.01", "synonyms": ["neckerchief"], "def": "a kerchief worn around the neck", "name": "neckerchief"}, {"id": 728, "synset": "necklace.n.01", "synonyms": ["necklace"], "def": "jewelry consisting of a cord or chain (often bearing gems) worn about the neck as an ornament", "name": "necklace"}, {"id": 729, "synset": "necktie.n.01", "synonyms": ["necktie", "tie_(necktie)"], "def": "neckwear consisting of a long narrow piece of material worn under a collar and tied in knot at the front", "name": "necktie"}, {"id": 730, "synset": "needle.n.03", "synonyms": ["needle"], "def": "a sharp pointed implement (usually metal)", "name": "needle"}, {"id": 731, "synset": "nest.n.01", "synonyms": ["nest"], "def": "a structure in which animals lay eggs or give birth to their young", "name": "nest"}, {"id": 732, "synset": "newsstand.n.01", "synonyms": ["newsstand"], "def": "a stall where newspapers and other periodicals are sold", "name": "newsstand"}, {"id": 733, "synset": "nightwear.n.01", "synonyms": ["nightshirt", "nightwear", "sleepwear", "nightclothes"], "def": "garments designed to be worn in bed", "name": "nightshirt"}, {"id": 734, "synset": "nosebag.n.01", "synonyms": ["nosebag_(for_animals)", "feedbag"], "def": "a canvas bag that is used to feed an animal (such as a horse); covers the muzzle and fastens at the top of the head", "name": "nosebag_(for_animals)"}, {"id": 735, "synset": "noseband.n.01", "synonyms": ["noseband_(for_animals)", "nosepiece_(for_animals)"], "def": "a strap that is the part of a bridle that goes over the animal's nose", "name": "noseband_(for_animals)"}, {"id": 736, "synset": "notebook.n.01", "synonyms": ["notebook"], "def": "a book with blank pages for recording notes or memoranda", "name": "notebook"}, {"id": 737, "synset": "notepad.n.01", "synonyms": ["notepad"], "def": "a pad of paper for keeping notes", "name": "notepad"}, {"id": 738, "synset": "nut.n.03", "synonyms": ["nut"], "def": "a small metal block (usually square or hexagonal) with internal screw thread to be fitted onto a bolt", "name": "nut"}, {"id": 739, "synset": "nutcracker.n.01", "synonyms": ["nutcracker"], "def": "a hand tool used to crack nuts open", "name": "nutcracker"}, {"id": 740, "synset": "oar.n.01", "synonyms": ["oar"], "def": "an implement used to propel or steer a boat", "name": "oar"}, {"id": 741, "synset": "octopus.n.01", "synonyms": ["octopus_(food)"], "def": "tentacles of octopus prepared as food", "name": "octopus_(food)"}, {"id": 742, "synset": "octopus.n.02", "synonyms": ["octopus_(animal)"], "def": "bottom-living cephalopod having a soft oval body with eight long tentacles", "name": "octopus_(animal)"}, {"id": 743, "synset": "oil_lamp.n.01", "synonyms": ["oil_lamp", "kerosene_lamp", "kerosine_lamp"], "def": "a lamp that burns oil (as kerosine) for light", "name": "oil_lamp"}, {"id": 744, "synset": "olive_oil.n.01", "synonyms": ["olive_oil"], "def": "oil from olives", "name": "olive_oil"}, {"id": 745, "synset": "omelet.n.01", "synonyms": ["omelet", "omelette"], "def": "beaten eggs cooked until just set; may be folded around e.g. ham or cheese or jelly", "name": "omelet"}, {"id": 746, "synset": "onion.n.01", "synonyms": ["onion"], "def": "the bulb of an onion plant", "name": "onion"}, {"id": 747, "synset": "orange.n.01", "synonyms": ["orange_(fruit)"], "def": "orange (FRUIT of an orange tree)", "name": "orange_(fruit)"}, {"id": 748, "synset": "orange_juice.n.01", "synonyms": ["orange_juice"], "def": "bottled or freshly squeezed juice of oranges", "name": "orange_juice"}, {"id": 749, "synset": "oregano.n.01", "synonyms": ["oregano", "marjoram"], "def": "aromatic Eurasian perennial herb used in cooking and baking", "name": "oregano"}, {"id": 750, "synset": "ostrich.n.02", "synonyms": ["ostrich"], "def": "fast-running African flightless bird with two-toed feet; largest living bird", "name": "ostrich"}, {"id": 751, "synset": "ottoman.n.03", "synonyms": ["ottoman", "pouf", "pouffe", "hassock"], "def": "thick cushion used as a seat", "name": "ottoman"}, {"id": 752, "synset": "overall.n.01", "synonyms": ["overalls_(clothing)"], "def": "work clothing consisting of denim trousers usually with a bib and shoulder straps", "name": "overalls_(clothing)"}, {"id": 753, "synset": "owl.n.01", "synonyms": ["owl"], "def": "nocturnal bird of prey with hawk-like beak and claws and large head with front-facing eyes", "name": "owl"}, {"id": 754, "synset": "packet.n.03", "synonyms": ["packet"], "def": "a small package or bundle", "name": "packet"}, {"id": 755, "synset": "pad.n.03", "synonyms": ["inkpad", "inking_pad", "stamp_pad"], "def": "absorbent material saturated with ink used to transfer ink evenly to a rubber stamp", "name": "inkpad"}, {"id": 756, "synset": "pad.n.04", "synonyms": ["pad"], "def": "a flat mass of soft material used for protection, stuffing, or comfort", "name": "pad"}, {"id": 757, "synset": "paddle.n.04", "synonyms": ["paddle", "boat_paddle"], "def": "a short light oar used without an oarlock to propel a canoe or small boat", "name": "paddle"}, {"id": 758, "synset": "padlock.n.01", "synonyms": ["padlock"], "def": "a detachable, portable lock", "name": "padlock"}, {"id": 759, "synset": "paintbox.n.01", "synonyms": ["paintbox"], "def": "a box containing a collection of cubes or tubes of artists' paint", "name": "paintbox"}, {"id": 760, "synset": "paintbrush.n.01", "synonyms": ["paintbrush"], "def": "a brush used as an applicator to apply paint", "name": "paintbrush"}, {"id": 761, "synset": "painting.n.01", "synonyms": ["painting"], "def": "graphic art consisting of an artistic composition made by applying paints to a surface", "name": "painting"}, {"id": 762, "synset": "pajama.n.02", "synonyms": ["pajamas", "pyjamas"], "def": "loose-fitting nightclothes worn for sleeping or lounging", "name": "pajamas"}, {"id": 763, "synset": "palette.n.02", "synonyms": ["palette", "pallet"], "def": "board that provides a flat surface on which artists mix paints and the range of colors used", "name": "palette"}, {"id": 764, "synset": "pan.n.01", "synonyms": ["pan_(for_cooking)", "cooking_pan"], "def": "cooking utensil consisting of a wide metal vessel", "name": "pan_(for_cooking)"}, {"id": 765, "synset": "pan.n.03", "synonyms": ["pan_(metal_container)"], "def": "shallow container made of metal", "name": "pan_(metal_container)"}, {"id": 766, "synset": "pancake.n.01", "synonyms": ["pancake"], "def": "a flat cake of thin batter fried on both sides on a griddle", "name": "pancake"}, {"id": 767, "synset": "pantyhose.n.01", "synonyms": ["pantyhose"], "def": "a woman's tights consisting of underpants and stockings", "name": "pantyhose"}, {"id": 768, "synset": "papaya.n.02", "synonyms": ["papaya"], "def": "large oval melon-like tropical fruit with yellowish flesh", "name": "papaya"}, {"id": 769, "synset": "paper_clip.n.01", "synonyms": ["paperclip"], "def": "a wire or plastic clip for holding sheets of paper together", "name": "paperclip"}, {"id": 770, "synset": "paper_plate.n.01", "synonyms": ["paper_plate"], "def": "a disposable plate made of cardboard", "name": "paper_plate"}, {"id": 771, "synset": "paper_towel.n.01", "synonyms": ["paper_towel"], "def": "a disposable towel made of absorbent paper", "name": "paper_towel"}, {"id": 772, "synset": "paperback_book.n.01", "synonyms": ["paperback_book", "paper-back_book", "softback_book", "soft-cover_book"], "def": "a book with paper covers", "name": "paperback_book"}, {"id": 773, "synset": "paperweight.n.01", "synonyms": ["paperweight"], "def": "a weight used to hold down a stack of papers", "name": "paperweight"}, {"id": 774, "synset": "parachute.n.01", "synonyms": ["parachute"], "def": "rescue equipment consisting of a device that fills with air and retards your fall", "name": "parachute"}, {"id": 775, "synset": "parakeet.n.01", "synonyms": ["parakeet", "parrakeet", "parroket", "paraquet", "paroquet", "parroquet"], "def": "any of numerous small slender long-tailed parrots", "name": "parakeet"}, {"id": 776, "synset": "parasail.n.01", "synonyms": ["parasail_(sports)"], "def": "parachute that will lift a person up into the air when it is towed by a motorboat or a car", "name": "parasail_(sports)"}, {"id": 777, "synset": "parchment.n.01", "synonyms": ["parchment"], "def": "a superior paper resembling sheepskin", "name": "parchment"}, {"id": 778, "synset": "parka.n.01", "synonyms": ["parka", "anorak"], "def": "a kind of heavy jacket (`windcheater' is a British term)", "name": "parka"}, {"id": 779, "synset": "parking_meter.n.01", "synonyms": ["parking_meter"], "def": "a coin-operated timer located next to a parking space", "name": "parking_meter"}, {"id": 780, "synset": "parrot.n.01", "synonyms": ["parrot"], "def": "usually brightly colored tropical birds with short hooked beaks and the ability to mimic sounds", "name": "parrot"}, {"id": 781, "synset": "passenger_car.n.01", "synonyms": ["passenger_car_(part_of_a_train)", "coach_(part_of_a_train)"], "def": "a railcar where passengers ride", "name": "passenger_car_(part_of_a_train)"}, {"id": 782, "synset": "passenger_ship.n.01", "synonyms": ["passenger_ship"], "def": "a ship built to carry passengers", "name": "passenger_ship"}, {"id": 783, "synset": "passport.n.02", "synonyms": ["passport"], "def": "a document issued by a country to a citizen allowing that person to travel abroad and re-enter the home country", "name": "passport"}, {"id": 784, "synset": "pastry.n.02", "synonyms": ["pastry"], "def": "any of various baked foods made of dough or batter", "name": "pastry"}, {"id": 785, "synset": "patty.n.01", "synonyms": ["patty_(food)"], "def": "small flat mass of chopped food", "name": "patty_(food)"}, {"id": 786, "synset": "pea.n.01", "synonyms": ["pea_(food)"], "def": "seed of a pea plant used for food", "name": "pea_(food)"}, {"id": 787, "synset": "peach.n.03", "synonyms": ["peach"], "def": "downy juicy fruit with sweet yellowish or whitish flesh", "name": "peach"}, {"id": 788, "synset": "peanut_butter.n.01", "synonyms": ["peanut_butter"], "def": "a spread made from ground peanuts", "name": "peanut_butter"}, {"id": 789, "synset": "pear.n.01", "synonyms": ["pear"], "def": "sweet juicy gritty-textured fruit available in many varieties", "name": "pear"}, {"id": 790, "synset": "peeler.n.03", "synonyms": ["peeler_(tool_for_fruit_and_vegetables)"], "def": "a device for peeling vegetables or fruits", "name": "peeler_(tool_for_fruit_and_vegetables)"}, {"id": 791, "synset": "pegboard.n.01", "synonyms": ["pegboard"], "def": "a board perforated with regularly spaced holes into which pegs can be fitted", "name": "pegboard"}, {"id": 792, "synset": "pelican.n.01", "synonyms": ["pelican"], "def": "large long-winged warm-water seabird having a large bill with a distensible pouch for fish", "name": "pelican"}, {"id": 793, "synset": "pen.n.01", "synonyms": ["pen"], "def": "a writing implement with a point from which ink flows", "name": "pen"}, {"id": 794, "synset": "pencil.n.01", "synonyms": ["pencil"], "def": "a thin cylindrical pointed writing implement made of wood and graphite", "name": "pencil"}, {"id": 795, "synset": "pencil_box.n.01", "synonyms": ["pencil_box", "pencil_case"], "def": "a box for holding pencils", "name": "pencil_box"}, {"id": 796, "synset": "pencil_sharpener.n.01", "synonyms": ["pencil_sharpener"], "def": "a rotary implement for sharpening the point on pencils", "name": "pencil_sharpener"}, {"id": 797, "synset": "pendulum.n.01", "synonyms": ["pendulum"], "def": "an apparatus consisting of an object mounted so that it swings freely under the influence of gravity", "name": "pendulum"}, {"id": 798, "synset": "penguin.n.01", "synonyms": ["penguin"], "def": "short-legged flightless birds of cold southern regions having webbed feet and wings modified as flippers", "name": "penguin"}, {"id": 799, "synset": "pennant.n.02", "synonyms": ["pennant"], "def": "a flag longer than it is wide (and often tapering)", "name": "pennant"}, {"id": 800, "synset": "penny.n.02", "synonyms": ["penny_(coin)"], "def": "a coin worth one-hundredth of the value of the basic unit", "name": "penny_(coin)"}, {"id": 801, "synset": "pepper.n.03", "synonyms": ["pepper", "peppercorn"], "def": "pungent seasoning from the berry of the common pepper plant; whole or ground", "name": "pepper"}, {"id": 802, "synset": "pepper_mill.n.01", "synonyms": ["pepper_mill", "pepper_grinder"], "def": "a mill for grinding pepper", "name": "pepper_mill"}, {"id": 803, "synset": "perfume.n.02", "synonyms": ["perfume"], "def": "a toiletry that emits and diffuses a fragrant odor", "name": "perfume"}, {"id": 804, "synset": "persimmon.n.02", "synonyms": ["persimmon"], "def": "orange fruit resembling a plum; edible when fully ripe", "name": "persimmon"}, {"id": 805, "synset": "person.n.01", "synonyms": ["baby", "child", "boy", "girl", "man", "woman", "person", "human"], "def": "a human being", "name": "baby"}, {"id": 806, "synset": "pet.n.01", "synonyms": ["pet"], "def": "a domesticated animal kept for companionship or amusement", "name": "pet"}, {"id": 807, "synset": "petfood.n.01", "synonyms": ["petfood", "pet-food"], "def": "food prepared for animal pets", "name": "petfood"}, {"id": 808, "synset": "pew.n.01", "synonyms": ["pew_(church_bench)", "church_bench"], "def": "long bench with backs; used in church by the congregation", "name": "pew_(church_bench)"}, {"id": 809, "synset": "phonebook.n.01", "synonyms": ["phonebook", "telephone_book", "telephone_directory"], "def": "a directory containing an alphabetical list of telephone subscribers and their telephone numbers", "name": "phonebook"}, {"id": 810, "synset": "phonograph_record.n.01", "synonyms": ["phonograph_record", "phonograph_recording", "record_(phonograph_recording)"], "def": "sound recording consisting of a typically black disk with a continuous groove", "name": "phonograph_record"}, {"id": 811, "synset": "piano.n.01", "synonyms": ["piano"], "def": "a keyboard instrument that is played by depressing keys that cause hammers to strike tuned strings and produce sounds", "name": "piano"}, {"id": 812, "synset": "pickle.n.01", "synonyms": ["pickle"], "def": "vegetables (especially cucumbers) preserved in brine or vinegar", "name": "pickle"}, {"id": 813, "synset": "pickup.n.01", "synonyms": ["pickup_truck"], "def": "a light truck with an open body and low sides and a tailboard", "name": "pickup_truck"}, {"id": 814, "synset": "pie.n.01", "synonyms": ["pie"], "def": "dish baked in pastry-lined pan often with a pastry top", "name": "pie"}, {"id": 815, "synset": "pigeon.n.01", "synonyms": ["pigeon"], "def": "wild and domesticated birds having a heavy body and short legs", "name": "pigeon"}, {"id": 816, "synset": "piggy_bank.n.01", "synonyms": ["piggy_bank", "penny_bank"], "def": "a child's coin bank (often shaped like a pig)", "name": "piggy_bank"}, {"id": 817, "synset": "pillow.n.01", "synonyms": ["pillow"], "def": "a cushion to support the head of a sleeping person", "name": "pillow"}, {"id": 818, "synset": "pin.n.09", "synonyms": ["pin_(non_jewelry)"], "def": "a small slender (often pointed) piece of wood or metal used to support or fasten or attach things", "name": "pin_(non_jewelry)"}, {"id": 819, "synset": "pineapple.n.02", "synonyms": ["pineapple"], "def": "large sweet fleshy tropical fruit with a tuft of stiff leaves", "name": "pineapple"}, {"id": 820, "synset": "pinecone.n.01", "synonyms": ["pinecone"], "def": "the seed-producing cone of a pine tree", "name": "pinecone"}, {"id": 821, "synset": "ping-pong_ball.n.01", "synonyms": ["ping-pong_ball"], "def": "light hollow ball used in playing table tennis", "name": "ping-pong_ball"}, {"id": 822, "synset": "pinwheel.n.03", "synonyms": ["pinwheel"], "def": "a toy consisting of vanes of colored paper or plastic that is pinned to a stick and spins when it is pointed into the wind", "name": "pinwheel"}, {"id": 823, "synset": "pipe.n.01", "synonyms": ["tobacco_pipe"], "def": "a tube with a small bowl at one end; used for smoking tobacco", "name": "tobacco_pipe"}, {"id": 824, "synset": "pipe.n.02", "synonyms": ["pipe", "piping"], "def": "a long tube made of metal or plastic that is used to carry water or oil or gas etc.", "name": "pipe"}, {"id": 825, "synset": "pistol.n.01", "synonyms": ["pistol", "handgun"], "def": "a firearm that is held and fired with one hand", "name": "pistol"}, {"id": 826, "synset": "pita.n.01", "synonyms": ["pita_(bread)", "pocket_bread"], "def": "usually small round bread that can open into a pocket for filling", "name": "pita_(bread)"}, {"id": 827, "synset": "pitcher.n.02", "synonyms": ["pitcher_(vessel_for_liquid)", "ewer"], "def": "an open vessel with a handle and a spout for pouring", "name": "pitcher_(vessel_for_liquid)"}, {"id": 828, "synset": "pitchfork.n.01", "synonyms": ["pitchfork"], "def": "a long-handled hand tool with sharp widely spaced prongs for lifting and pitching hay", "name": "pitchfork"}, {"id": 829, "synset": "pizza.n.01", "synonyms": ["pizza"], "def": "Italian open pie made of thin bread dough spread with a spiced mixture of e.g. tomato sauce and cheese", "name": "pizza"}, {"id": 830, "synset": "place_mat.n.01", "synonyms": ["place_mat"], "def": "a mat placed on a table for an individual place setting", "name": "place_mat"}, {"id": 831, "synset": "plate.n.04", "synonyms": ["plate"], "def": "dish on which food is served or from which food is eaten", "name": "plate"}, {"id": 832, "synset": "platter.n.01", "synonyms": ["platter"], "def": "a large shallow dish used for serving food", "name": "platter"}, {"id": 833, "synset": "playing_card.n.01", "synonyms": ["playing_card"], "def": "one of a pack of cards that are used to play card games", "name": "playing_card"}, {"id": 834, "synset": "playpen.n.01", "synonyms": ["playpen"], "def": "a portable enclosure in which babies may be left to play", "name": "playpen"}, {"id": 835, "synset": "pliers.n.01", "synonyms": ["pliers", "plyers"], "def": "a gripping hand tool with two hinged arms and (usually) serrated jaws", "name": "pliers"}, {"id": 836, "synset": "plow.n.01", "synonyms": ["plow_(farm_equipment)", "plough_(farm_equipment)"], "def": "a farm tool having one or more heavy blades to break the soil and cut a furrow prior to sowing", "name": "plow_(farm_equipment)"}, {"id": 837, "synset": "pocket_watch.n.01", "synonyms": ["pocket_watch"], "def": "a watch that is carried in a small watch pocket", "name": "pocket_watch"}, {"id": 838, "synset": "pocketknife.n.01", "synonyms": ["pocketknife"], "def": "a knife with a blade that folds into the handle; suitable for carrying in the pocket", "name": "pocketknife"}, {"id": 839, "synset": "poker.n.01", "synonyms": ["poker_(fire_stirring_tool)", "stove_poker", "fire_hook"], "def": "fire iron consisting of a metal rod with a handle; used to stir a fire", "name": "poker_(fire_stirring_tool)"}, {"id": 840, "synset": "pole.n.01", "synonyms": ["pole", "post"], "def": "a long (usually round) rod of wood or metal or plastic", "name": "pole"}, {"id": 841, "synset": "police_van.n.01", "synonyms": ["police_van", "police_wagon", "paddy_wagon", "patrol_wagon"], "def": "van used by police to transport prisoners", "name": "police_van"}, {"id": 842, "synset": "polo_shirt.n.01", "synonyms": ["polo_shirt", "sport_shirt"], "def": "a shirt with short sleeves designed for comfort and casual wear", "name": "polo_shirt"}, {"id": 843, "synset": "poncho.n.01", "synonyms": ["poncho"], "def": "a blanket-like cloak with a hole in the center for the head", "name": "poncho"}, {"id": 844, "synset": "pony.n.05", "synonyms": ["pony"], "def": "any of various breeds of small gentle horses usually less than five feet high at the shoulder", "name": "pony"}, {"id": 845, "synset": "pool_table.n.01", "synonyms": ["pool_table", "billiard_table", "snooker_table"], "def": "game equipment consisting of a heavy table on which pool is played", "name": "pool_table"}, {"id": 846, "synset": "pop.n.02", "synonyms": ["pop_(soda)", "soda_(pop)", "tonic", "soft_drink"], "def": "a sweet drink containing carbonated water and flavoring", "name": "pop_(soda)"}, {"id": 847, "synset": "portrait.n.02", "synonyms": ["portrait", "portrayal"], "def": "any likeness of a person, in any medium", "name": "portrait"}, {"id": 848, "synset": "postbox.n.01", "synonyms": ["postbox_(public)", "mailbox_(public)"], "def": "public box for deposit of mail", "name": "postbox_(public)"}, {"id": 849, "synset": "postcard.n.01", "synonyms": ["postcard", "postal_card", "mailing-card"], "def": "a card for sending messages by post without an envelope", "name": "postcard"}, {"id": 850, "synset": "poster.n.01", "synonyms": ["poster", "placard"], "def": "a sign posted in a public place as an advertisement", "name": "poster"}, {"id": 851, "synset": "pot.n.01", "synonyms": ["pot"], "def": "metal or earthenware cooking vessel that is usually round and deep; often has a handle and lid", "name": "pot"}, {"id": 852, "synset": "pot.n.04", "synonyms": ["flowerpot"], "def": "a container in which plants are cultivated", "name": "flowerpot"}, {"id": 853, "synset": "potato.n.01", "synonyms": ["potato"], "def": "an edible tuber native to South America", "name": "potato"}, {"id": 854, "synset": "potholder.n.01", "synonyms": ["potholder"], "def": "an insulated pad for holding hot pots", "name": "potholder"}, {"id": 855, "synset": "pottery.n.01", "synonyms": ["pottery", "clayware"], "def": "ceramic ware made from clay and baked in a kiln", "name": "pottery"}, {"id": 856, "synset": "pouch.n.01", "synonyms": ["pouch"], "def": "a small or medium size container for holding or carrying things", "name": "pouch"}, {"id": 857, "synset": "power_shovel.n.01", "synonyms": ["power_shovel", "excavator", "digger"], "def": "a machine for excavating", "name": "power_shovel"}, {"id": 858, "synset": "prawn.n.01", "synonyms": ["prawn", "shrimp"], "def": "any of various edible decapod crustaceans", "name": "prawn"}, {"id": 859, "synset": "printer.n.03", "synonyms": ["printer", "printing_machine"], "def": "a machine that prints", "name": "printer"}, {"id": 860, "synset": "projectile.n.01", "synonyms": ["projectile_(weapon)", "missile"], "def": "a weapon that is forcibly thrown or projected at a targets", "name": "projectile_(weapon)"}, {"id": 861, "synset": "projector.n.02", "synonyms": ["projector"], "def": "an optical instrument that projects an enlarged image onto a screen", "name": "projector"}, {"id": 862, "synset": "propeller.n.01", "synonyms": ["propeller", "propellor"], "def": "a mechanical device that rotates to push against air or water", "name": "propeller"}, {"id": 863, "synset": "prune.n.01", "synonyms": ["prune"], "def": "dried plum", "name": "prune"}, {"id": 864, "synset": "pudding.n.01", "synonyms": ["pudding"], "def": "any of various soft thick unsweetened baked dishes", "name": "pudding"}, {"id": 865, "synset": "puffer.n.02", "synonyms": ["puffer_(fish)", "pufferfish", "blowfish", "globefish"], "def": "fishes whose elongated spiny body can inflate itself with water or air to form a globe", "name": "puffer_(fish)"}, {"id": 866, "synset": "puffin.n.01", "synonyms": ["puffin"], "def": "seabirds having short necks and brightly colored compressed bills", "name": "puffin"}, {"id": 867, "synset": "pug.n.01", "synonyms": ["pug-dog"], "def": "small compact smooth-coated breed of Asiatic origin having a tightly curled tail and broad flat wrinkled muzzle", "name": "pug-dog"}, {"id": 868, "synset": "pumpkin.n.02", "synonyms": ["pumpkin"], "def": "usually large pulpy deep-yellow round fruit of the squash family maturing in late summer or early autumn", "name": "pumpkin"}, {"id": 869, "synset": "punch.n.03", "synonyms": ["puncher"], "def": "a tool for making holes or indentations", "name": "puncher"}, {"id": 870, "synset": "puppet.n.01", "synonyms": ["puppet", "marionette"], "def": "a small figure of a person operated from above with strings by a puppeteer", "name": "puppet"}, {"id": 871, "synset": "puppy.n.01", "synonyms": ["puppy"], "def": "a young dog", "name": "puppy"}, {"id": 872, "synset": "quesadilla.n.01", "synonyms": ["quesadilla"], "def": "a tortilla that is filled with cheese and heated", "name": "quesadilla"}, {"id": 873, "synset": "quiche.n.02", "synonyms": ["quiche"], "def": "a tart filled with rich unsweetened custard; often contains other ingredients (as cheese or ham or seafood or vegetables)", "name": "quiche"}, {"id": 874, "synset": "quilt.n.01", "synonyms": ["quilt", "comforter"], "def": "bedding made of two layers of cloth filled with stuffing and stitched together", "name": "quilt"}, {"id": 875, "synset": "rabbit.n.01", "synonyms": ["rabbit"], "def": "any of various burrowing animals of the family Leporidae having long ears and short tails", "name": "rabbit"}, {"id": 876, "synset": "racer.n.02", "synonyms": ["race_car", "racing_car"], "def": "a fast car that competes in races", "name": "race_car"}, {"id": 877, "synset": "racket.n.04", "synonyms": ["racket", "racquet"], "def": "a sports implement used to strike a ball in various games", "name": "racket"}, {"id": 878, "synset": "radar.n.01", "synonyms": ["radar"], "def": "measuring instrument in which the echo of a pulse of microwave radiation is used to detect and locate distant objects", "name": "radar"}, {"id": 879, "synset": "radiator.n.03", "synonyms": ["radiator"], "def": "a mechanism consisting of a metal honeycomb through which hot fluids circulate", "name": "radiator"}, {"id": 880, "synset": "radio_receiver.n.01", "synonyms": ["radio_receiver", "radio_set", "radio", "tuner_(radio)"], "def": "an electronic receiver that detects and demodulates and amplifies transmitted radio signals", "name": "radio_receiver"}, {"id": 881, "synset": "radish.n.03", "synonyms": ["radish", "daikon"], "def": "pungent edible root of any of various cultivated radish plants", "name": "radish"}, {"id": 882, "synset": "raft.n.01", "synonyms": ["raft"], "def": "a flat float (usually made of logs or planks) that can be used for transport or as a platform for swimmers", "name": "raft"}, {"id": 883, "synset": "rag_doll.n.01", "synonyms": ["rag_doll"], "def": "a cloth doll that is stuffed and (usually) painted", "name": "rag_doll"}, {"id": 884, "synset": "raincoat.n.01", "synonyms": ["raincoat", "waterproof_jacket"], "def": "a water-resistant coat", "name": "raincoat"}, {"id": 885, "synset": "ram.n.05", "synonyms": ["ram_(animal)"], "def": "uncastrated adult male sheep", "name": "ram_(animal)"}, {"id": 886, "synset": "raspberry.n.02", "synonyms": ["raspberry"], "def": "red or black edible aggregate berries usually smaller than the related blackberries", "name": "raspberry"}, {"id": 887, "synset": "rat.n.01", "synonyms": ["rat"], "def": "any of various long-tailed rodents similar to but larger than a mouse", "name": "rat"}, {"id": 888, "synset": "razorblade.n.01", "synonyms": ["razorblade"], "def": "a blade that has very sharp edge", "name": "razorblade"}, {"id": 889, "synset": "reamer.n.01", "synonyms": ["reamer_(juicer)", "juicer", "juice_reamer"], "def": "a squeezer with a conical ridged center that is used for squeezing juice from citrus fruit", "name": "reamer_(juicer)"}, {"id": 890, "synset": "rearview_mirror.n.01", "synonyms": ["rearview_mirror"], "def": "car mirror that reflects the view out of the rear window", "name": "rearview_mirror"}, {"id": 891, "synset": "receipt.n.02", "synonyms": ["receipt"], "def": "an acknowledgment (usually tangible) that payment has been made", "name": "receipt"}, {"id": 892, "synset": "recliner.n.01", "synonyms": ["recliner", "reclining_chair", "lounger_(chair)"], "def": "an armchair whose back can be lowered and foot can be raised to allow the sitter to recline in it", "name": "recliner"}, {"id": 893, "synset": "record_player.n.01", "synonyms": ["record_player", "phonograph_(record_player)", "turntable"], "def": "machine in which rotating records cause a stylus to vibrate and the vibrations are amplified acoustically or electronically", "name": "record_player"}, {"id": 894, "synset": "red_cabbage.n.02", "synonyms": ["red_cabbage"], "def": "compact head of purplish-red leaves", "name": "red_cabbage"}, {"id": 895, "synset": "reflector.n.01", "synonyms": ["reflector"], "def": "device that reflects light, radiation, etc.", "name": "reflector"}, {"id": 896, "synset": "remote_control.n.01", "synonyms": ["remote_control"], "def": "a device that can be used to control a machine or apparatus from a distance", "name": "remote_control"}, {"id": 897, "synset": "rhinoceros.n.01", "synonyms": ["rhinoceros"], "def": "massive powerful herbivorous odd-toed ungulate of southeast Asia and Africa having very thick skin and one or two horns on the snout", "name": "rhinoceros"}, {"id": 898, "synset": "rib.n.03", "synonyms": ["rib_(food)"], "def": "cut of meat including one or more ribs", "name": "rib_(food)"}, {"id": 899, "synset": "rifle.n.01", "synonyms": ["rifle"], "def": "a shoulder firearm with a long barrel", "name": "rifle"}, {"id": 900, "synset": "ring.n.08", "synonyms": ["ring"], "def": "jewelry consisting of a circlet of precious metal (often set with jewels) worn on the finger", "name": "ring"}, {"id": 901, "synset": "river_boat.n.01", "synonyms": ["river_boat"], "def": "a boat used on rivers or to ply a river", "name": "river_boat"}, {"id": 902, "synset": "road_map.n.02", "synonyms": ["road_map"], "def": "(NOT A ROAD) a MAP showing roads (for automobile travel)", "name": "road_map"}, {"id": 903, "synset": "robe.n.01", "synonyms": ["robe"], "def": "any loose flowing garment", "name": "robe"}, {"id": 904, "synset": "rocking_chair.n.01", "synonyms": ["rocking_chair"], "def": "a chair mounted on rockers", "name": "rocking_chair"}, {"id": 905, "synset": "roller_skate.n.01", "synonyms": ["roller_skate"], "def": "a shoe with pairs of rollers (small hard wheels) fixed to the sole", "name": "roller_skate"}, {"id": 906, "synset": "rollerblade.n.01", "synonyms": ["Rollerblade"], "def": "an in-line variant of a roller skate", "name": "Rollerblade"}, {"id": 907, "synset": "rolling_pin.n.01", "synonyms": ["rolling_pin"], "def": "utensil consisting of a cylinder (usually of wood) with a handle at each end; used to roll out dough", "name": "rolling_pin"}, {"id": 908, "synset": "root_beer.n.01", "synonyms": ["root_beer"], "def": "carbonated drink containing extracts of roots and herbs", "name": "root_beer"}, {"id": 909, "synset": "router.n.02", "synonyms": ["router_(computer_equipment)"], "def": "a device that forwards data packets between computer networks", "name": "router_(computer_equipment)"}, {"id": 910, "synset": "rubber_band.n.01", "synonyms": ["rubber_band", "elastic_band"], "def": "a narrow band of elastic rubber used to hold things (such as papers) together", "name": "rubber_band"}, {"id": 911, "synset": "runner.n.08", "synonyms": ["runner_(carpet)"], "def": "a long narrow carpet", "name": "runner_(carpet)"}, {"id": 912, "synset": "sack.n.01", "synonyms": ["plastic_bag", "paper_bag"], "def": "a bag made of paper or plastic for holding customer's purchases", "name": "plastic_bag"}, {"id": 913, "synset": "saddle.n.01", "synonyms": ["saddle_(on_an_animal)"], "def": "a seat for the rider of a horse or camel", "name": "saddle_(on_an_animal)"}, {"id": 914, "synset": "saddle_blanket.n.01", "synonyms": ["saddle_blanket", "saddlecloth", "horse_blanket"], "def": "stable gear consisting of a blanket placed under the saddle", "name": "saddle_blanket"}, {"id": 915, "synset": "saddlebag.n.01", "synonyms": ["saddlebag"], "def": "a large bag (or pair of bags) hung over a saddle", "name": "saddlebag"}, {"id": 916, "synset": "safety_pin.n.01", "synonyms": ["safety_pin"], "def": "a pin in the form of a clasp; has a guard so the point of the pin will not stick the user", "name": "safety_pin"}, {"id": 917, "synset": "sail.n.01", "synonyms": ["sail"], "def": "a large piece of fabric by means of which wind is used to propel a sailing vessel", "name": "sail"}, {"id": 918, "synset": "salad.n.01", "synonyms": ["salad"], "def": "food mixtures either arranged on a plate or tossed and served with a moist dressing; usually consisting of or including greens", "name": "salad"}, {"id": 919, "synset": "salad_plate.n.01", "synonyms": ["salad_plate", "salad_bowl"], "def": "a plate or bowl for individual servings of salad", "name": "salad_plate"}, {"id": 920, "synset": "salami.n.01", "synonyms": ["salami"], "def": "highly seasoned fatty sausage of pork and beef usually dried", "name": "salami"}, {"id": 921, "synset": "salmon.n.01", "synonyms": ["salmon_(fish)"], "def": "any of various large food and game fishes of northern waters", "name": "salmon_(fish)"}, {"id": 922, "synset": "salmon.n.03", "synonyms": ["salmon_(food)"], "def": "flesh of any of various marine or freshwater fish of the family Salmonidae", "name": "salmon_(food)"}, {"id": 923, "synset": "salsa.n.01", "synonyms": ["salsa"], "def": "spicy sauce of tomatoes and onions and chili peppers to accompany Mexican foods", "name": "salsa"}, {"id": 924, "synset": "saltshaker.n.01", "synonyms": ["saltshaker"], "def": "a shaker with a perforated top for sprinkling salt", "name": "saltshaker"}, {"id": 925, "synset": "sandal.n.01", "synonyms": ["sandal_(type_of_shoe)"], "def": "a shoe consisting of a sole fastened by straps to the foot", "name": "sandal_(type_of_shoe)"}, {"id": 926, "synset": "sandwich.n.01", "synonyms": ["sandwich"], "def": "two (or more) slices of bread with a filling between them", "name": "sandwich"}, {"id": 927, "synset": "satchel.n.01", "synonyms": ["satchel"], "def": "luggage consisting of a small case with a flat bottom and (usually) a shoulder strap", "name": "satchel"}, {"id": 928, "synset": "saucepan.n.01", "synonyms": ["saucepan"], "def": "a deep pan with a handle; used for stewing or boiling", "name": "saucepan"}, {"id": 929, "synset": "saucer.n.02", "synonyms": ["saucer"], "def": "a small shallow dish for holding a cup at the table", "name": "saucer"}, {"id": 930, "synset": "sausage.n.01", "synonyms": ["sausage"], "def": "highly seasoned minced meat stuffed in casings", "name": "sausage"}, {"id": 931, "synset": "sawhorse.n.01", "synonyms": ["sawhorse", "sawbuck"], "def": "a framework for holding wood that is being sawed", "name": "sawhorse"}, {"id": 932, "synset": "sax.n.02", "synonyms": ["saxophone"], "def": "a wind instrument with a `J'-shaped form typically made of brass", "name": "saxophone"}, {"id": 933, "synset": "scale.n.07", "synonyms": ["scale_(measuring_instrument)"], "def": "a measuring instrument for weighing; shows amount of mass", "name": "scale_(measuring_instrument)"}, {"id": 934, "synset": "scarecrow.n.01", "synonyms": ["scarecrow", "strawman"], "def": "an effigy in the shape of a man to frighten birds away from seeds", "name": "scarecrow"}, {"id": 935, "synset": "scarf.n.01", "synonyms": ["scarf"], "def": "a garment worn around the head or neck or shoulders for warmth or decoration", "name": "scarf"}, {"id": 936, "synset": "school_bus.n.01", "synonyms": ["school_bus"], "def": "a bus used to transport children to or from school", "name": "school_bus"}, {"id": 937, "synset": "scissors.n.01", "synonyms": ["scissors"], "def": "a tool having two crossed pivoting blades with looped handles", "name": "scissors"}, {"id": 938, "synset": "scoreboard.n.01", "synonyms": ["scoreboard"], "def": "a large board for displaying the score of a contest (and some other information)", "name": "scoreboard"}, {"id": 939, "synset": "scrambled_eggs.n.01", "synonyms": ["scrambled_eggs"], "def": "eggs beaten and cooked to a soft firm consistency while stirring", "name": "scrambled_eggs"}, {"id": 940, "synset": "scraper.n.01", "synonyms": ["scraper"], "def": "any of various hand tools for scraping", "name": "scraper"}, {"id": 941, "synset": "scratcher.n.03", "synonyms": ["scratcher"], "def": "a device used for scratching", "name": "scratcher"}, {"id": 942, "synset": "screwdriver.n.01", "synonyms": ["screwdriver"], "def": "a hand tool for driving screws; has a tip that fits into the head of a screw", "name": "screwdriver"}, {"id": 943, "synset": "scrub_brush.n.01", "synonyms": ["scrubbing_brush"], "def": "a brush with short stiff bristles for heavy cleaning", "name": "scrubbing_brush", "merged": [{"frequency": "c", "id": 153, "synset": "bristle_brush.n.01", "image_count": 3, "instance_count": 3, "synonyms": ["bristle_brush"], "def": "a brush that is made with the short stiff hairs of an animal or plant", "name": "bristle_brush"}]}, {"id": 944, "synset": "sculpture.n.01", "synonyms": ["sculpture"], "def": "a three-dimensional work of art", "name": "sculpture"}, {"id": 945, "synset": "seabird.n.01", "synonyms": ["seabird", "seafowl"], "def": "a bird that frequents coastal waters and the open ocean: gulls; pelicans; gannets; cormorants; albatrosses; petrels; etc.", "name": "seabird"}, {"id": 946, "synset": "seahorse.n.02", "synonyms": ["seahorse"], "def": "small fish with horse-like heads bent sharply downward and curled tails", "name": "seahorse"}, {"id": 947, "synset": "seaplane.n.01", "synonyms": ["seaplane", "hydroplane"], "def": "an airplane that can land on or take off from water", "name": "seaplane"}, {"id": 948, "synset": "seashell.n.01", "synonyms": ["seashell"], "def": "the shell of a marine organism", "name": "seashell"}, {"id": 949, "synset": "seedling.n.01", "synonyms": ["seedling"], "def": "young plant or tree grown from a seed", "name": "seedling"}, {"id": 950, "synset": "serving_dish.n.01", "synonyms": ["serving_dish"], "def": "a dish used for serving food", "name": "serving_dish"}, {"id": 951, "synset": "sewing_machine.n.01", "synonyms": ["sewing_machine"], "def": "a textile machine used as a home appliance for sewing", "name": "sewing_machine"}, {"id": 952, "synset": "shaker.n.03", "synonyms": ["shaker"], "def": "a container in which something can be shaken", "name": "shaker"}, {"id": 953, "synset": "shampoo.n.01", "synonyms": ["shampoo"], "def": "cleansing agent consisting of soaps or detergents used for washing the hair", "name": "shampoo"}, {"id": 954, "synset": "shark.n.01", "synonyms": ["shark"], "def": "typically large carnivorous fishes with sharpe teeth", "name": "shark"}, {"id": 955, "synset": "sharpener.n.01", "synonyms": ["sharpener"], "def": "any implement that is used to make something (an edge or a point) sharper", "name": "sharpener"}, {"id": 956, "synset": "sharpie.n.03", "synonyms": ["Sharpie"], "def": "a pen with indelible ink that will write on any surface", "name": "Sharpie"}, {"id": 957, "synset": "shaver.n.03", "synonyms": ["shaver_(electric)", "electric_shaver", "electric_razor"], "def": "a razor powered by an electric motor", "name": "shaver_(electric)"}, {"id": 958, "synset": "shaving_cream.n.01", "synonyms": ["shaving_cream", "shaving_soap"], "def": "toiletry consisting that forms a rich lather for softening the beard before shaving", "name": "shaving_cream"}, {"id": 959, "synset": "shawl.n.01", "synonyms": ["shawl"], "def": "cloak consisting of an oblong piece of cloth used to cover the head and shoulders", "name": "shawl"}, {"id": 960, "synset": "shears.n.01", "synonyms": ["shears"], "def": "large scissors with strong blades", "name": "shears"}, {"id": 961, "synset": "sheep.n.01", "synonyms": ["sheep"], "def": "woolly usually horned ruminant mammal related to the goat", "name": "sheep"}, {"id": 962, "synset": "shepherd_dog.n.01", "synonyms": ["shepherd_dog", "sheepdog"], "def": "any of various usually long-haired breeds of dog reared to herd and guard sheep", "name": "shepherd_dog"}, {"id": 963, "synset": "sherbert.n.01", "synonyms": ["sherbert", "sherbet"], "def": "a frozen dessert made primarily of fruit juice and sugar", "name": "sherbert"}, {"id": 964, "synset": "shield.n.02", "synonyms": ["shield"], "def": "armor carried on the arm to intercept blows", "name": "shield"}, {"id": 965, "synset": "shirt.n.01", "synonyms": ["shirt"], "def": "a garment worn on the upper half of the body", "name": "shirt"}, {"id": 966, "synset": "shoe.n.01", "synonyms": ["shoe", "sneaker_(type_of_shoe)", "tennis_shoe"], "def": "common footwear covering the foot", "name": "shoe"}, {"id": 967, "synset": "shopping_bag.n.01", "synonyms": ["shopping_bag"], "def": "a bag made of plastic or strong paper (often with handles); used to transport goods after shopping", "name": "shopping_bag"}, {"id": 968, "synset": "shopping_cart.n.01", "synonyms": ["shopping_cart"], "def": "a handcart that holds groceries or other goods while shopping", "name": "shopping_cart"}, {"id": 969, "synset": "short_pants.n.01", "synonyms": ["short_pants", "shorts_(clothing)", "trunks_(clothing)"], "def": "trousers that end at or above the knee", "name": "short_pants"}, {"id": 970, "synset": "shot_glass.n.01", "synonyms": ["shot_glass"], "def": "a small glass adequate to hold a single swallow of whiskey", "name": "shot_glass"}, {"id": 971, "synset": "shoulder_bag.n.01", "synonyms": ["shoulder_bag"], "def": "a large handbag that can be carried by a strap looped over the shoulder", "name": "shoulder_bag"}, {"id": 972, "synset": "shovel.n.01", "synonyms": ["shovel"], "def": "a hand tool for lifting loose material such as snow, dirt, etc.", "name": "shovel"}, {"id": 973, "synset": "shower.n.01", "synonyms": ["shower_head"], "def": "a plumbing fixture that sprays water over you", "name": "shower_head"}, {"id": 974, "synset": "shower_curtain.n.01", "synonyms": ["shower_curtain"], "def": "a curtain that keeps water from splashing out of the shower area", "name": "shower_curtain"}, {"id": 975, "synset": "shredder.n.01", "synonyms": ["shredder_(for_paper)"], "def": "a device that shreds documents", "name": "shredder_(for_paper)"}, {"id": 976, "synset": "sieve.n.01", "synonyms": ["sieve", "screen_(sieve)"], "def": "a strainer for separating lumps from powdered material or grading particles", "name": "sieve"}, {"id": 977, "synset": "signboard.n.01", "synonyms": ["signboard"], "def": "structure displaying a board on which advertisements can be posted", "name": "signboard"}, {"id": 978, "synset": "silo.n.01", "synonyms": ["silo"], "def": "a cylindrical tower used for storing goods", "name": "silo"}, {"id": 979, "synset": "sink.n.01", "synonyms": ["sink"], "def": "plumbing fixture consisting of a water basin fixed to a wall or floor and having a drainpipe", "name": "sink"}, {"id": 980, "synset": "skateboard.n.01", "synonyms": ["skateboard"], "def": "a board with wheels that is ridden in a standing or crouching position and propelled by foot", "name": "skateboard"}, {"id": 981, "synset": "skewer.n.01", "synonyms": ["skewer"], "def": "a long pin for holding meat in position while it is being roasted", "name": "skewer"}, {"id": 982, "synset": "ski.n.01", "synonyms": ["ski"], "def": "sports equipment for skiing on snow", "name": "ski"}, {"id": 983, "synset": "ski_boot.n.01", "synonyms": ["ski_boot"], "def": "a stiff boot that is fastened to a ski with a ski binding", "name": "ski_boot"}, {"id": 984, "synset": "ski_parka.n.01", "synonyms": ["ski_parka", "ski_jacket"], "def": "a parka to be worn while skiing", "name": "ski_parka"}, {"id": 985, "synset": "ski_pole.n.01", "synonyms": ["ski_pole"], "def": "a pole with metal points used as an aid in skiing", "name": "ski_pole"}, {"id": 986, "synset": "skirt.n.02", "synonyms": ["skirt"], "def": "a garment hanging from the waist; worn mainly by girls and women", "name": "skirt"}, {"id": 987, "synset": "sled.n.01", "synonyms": ["sled", "sledge", "sleigh"], "def": "a vehicle or flat object for transportation over snow by sliding or pulled by dogs, etc.", "name": "sled"}, {"id": 988, "synset": "sleeping_bag.n.01", "synonyms": ["sleeping_bag"], "def": "large padded bag designed to be slept in outdoors", "name": "sleeping_bag"}, {"id": 989, "synset": "sling.n.05", "synonyms": ["sling_(bandage)", "triangular_bandage"], "def": "bandage to support an injured forearm; slung over the shoulder or neck", "name": "sling_(bandage)"}, {"id": 990, "synset": "slipper.n.01", "synonyms": ["slipper_(footwear)", "carpet_slipper_(footwear)"], "def": "low footwear that can be slipped on and off easily; usually worn indoors", "name": "slipper_(footwear)"}, {"id": 991, "synset": "smoothie.n.02", "synonyms": ["smoothie"], "def": "a thick smooth drink consisting of fresh fruit pureed with ice cream or yoghurt or milk", "name": "smoothie"}, {"id": 992, "synset": "snake.n.01", "synonyms": ["snake", "serpent"], "def": "limbless scaly elongate reptile; some are venomous", "name": "snake"}, {"id": 993, "synset": "snowboard.n.01", "synonyms": ["snowboard"], "def": "a board that resembles a broad ski or a small surfboard; used in a standing position to slide down snow-covered slopes", "name": "snowboard"}, {"id": 994, "synset": "snowman.n.01", "synonyms": ["snowman"], "def": "a figure of a person made of packed snow", "name": "snowman"}, {"id": 995, "synset": "snowmobile.n.01", "synonyms": ["snowmobile"], "def": "tracked vehicle for travel on snow having skis in front", "name": "snowmobile"}, {"id": 996, "synset": "soap.n.01", "synonyms": ["soap"], "def": "a cleansing agent made from the salts of vegetable or animal fats", "name": "soap"}, {"id": 997, "synset": "soccer_ball.n.01", "synonyms": ["soccer_ball"], "def": "an inflated ball used in playing soccer (called `football' outside of the United States)", "name": "soccer_ball"}, {"id": 998, "synset": "sock.n.01", "synonyms": ["sock"], "def": "cloth covering for the foot; worn inside the shoe; reaches to between the ankle and the knee", "name": "sock"}, {"id": 999, "synset": "soda_fountain.n.02", "synonyms": ["soda_fountain"], "def": "an apparatus for dispensing soda water", "name": "soda_fountain"}, {"id": 1000, "synset": "soda_water.n.01", "synonyms": ["carbonated_water", "club_soda", "seltzer", "sparkling_water"], "def": "effervescent beverage artificially charged with carbon dioxide", "name": "carbonated_water"}, {"id": 1001, "synset": "sofa.n.01", "synonyms": ["sofa", "couch", "lounge"], "def": "an upholstered seat for more than one person", "name": "sofa"}, {"id": 1002, "synset": "softball.n.01", "synonyms": ["softball"], "def": "ball used in playing softball", "name": "softball"}, {"id": 1003, "synset": "solar_array.n.01", "synonyms": ["solar_array", "solar_battery", "solar_panel"], "def": "electrical device consisting of a large array of connected solar cells", "name": "solar_array"}, {"id": 1004, "synset": "sombrero.n.02", "synonyms": ["sombrero"], "def": "a straw hat with a tall crown and broad brim; worn in American southwest and in Mexico", "name": "sombrero"}, {"id": 1005, "synset": "soup.n.01", "synonyms": ["soup"], "def": "liquid food especially of meat or fish or vegetable stock often containing pieces of solid food", "name": "soup"}, {"id": 1006, "synset": "soup_bowl.n.01", "synonyms": ["soup_bowl"], "def": "a bowl for serving soup", "name": "soup_bowl"}, {"id": 1007, "synset": "soupspoon.n.01", "synonyms": ["soupspoon"], "def": "a spoon with a rounded bowl for eating soup", "name": "soupspoon"}, {"id": 1008, "synset": "sour_cream.n.01", "synonyms": ["sour_cream", "soured_cream"], "def": "soured light cream", "name": "sour_cream"}, {"id": 1009, "synset": "soya_milk.n.01", "synonyms": ["soya_milk", "soybean_milk", "soymilk"], "def": "a milk substitute containing soybean flour and water; used in some infant formulas and in making tofu", "name": "soya_milk"}, {"id": 1010, "synset": "space_shuttle.n.01", "synonyms": ["space_shuttle"], "def": "a reusable spacecraft with wings for a controlled descent through the Earth's atmosphere", "name": "space_shuttle"}, {"id": 1011, "synset": "sparkler.n.02", "synonyms": ["sparkler_(fireworks)"], "def": "a firework that burns slowly and throws out a shower of sparks", "name": "sparkler_(fireworks)"}, {"id": 1012, "synset": "spatula.n.02", "synonyms": ["spatula"], "def": "a hand tool with a thin flexible blade used to mix or spread soft substances", "name": "spatula"}, {"id": 1013, "synset": "spear.n.01", "synonyms": ["spear", "lance"], "def": "a long pointed rod used as a tool or weapon", "name": "spear"}, {"id": 1014, "synset": "spectacles.n.01", "synonyms": ["spectacles", "specs", "eyeglasses", "glasses"], "def": "optical instrument consisting of a frame that holds a pair of lenses for correcting defective vision", "name": "spectacles"}, {"id": 1015, "synset": "spice_rack.n.01", "synonyms": ["spice_rack"], "def": "a rack for displaying containers filled with spices", "name": "spice_rack"}, {"id": 1016, "synset": "spider.n.01", "synonyms": ["spider"], "def": "predatory arachnid with eight legs, two poison fangs, two feelers, and usually two silk-spinning organs at the back end of the body", "name": "spider"}, {"id": 1017, "synset": "sponge.n.01", "synonyms": ["sponge"], "def": "a porous mass usable to absorb water typically used for cleaning", "name": "sponge"}, {"id": 1018, "synset": "spoon.n.01", "synonyms": ["spoon"], "def": "a piece of cutlery with a shallow bowl-shaped container and a handle", "name": "spoon"}, {"id": 1019, "synset": "sportswear.n.01", "synonyms": ["sportswear", "athletic_wear", "activewear"], "def": "attire worn for sport or for casual wear", "name": "sportswear"}, {"id": 1020, "synset": "spotlight.n.02", "synonyms": ["spotlight"], "def": "a lamp that produces a strong beam of light to illuminate a restricted area; used to focus attention of a stage performer", "name": "spotlight"}, {"id": 1021, "synset": "squirrel.n.01", "synonyms": ["squirrel"], "def": "a kind of arboreal rodent having a long bushy tail", "name": "squirrel"}, {"id": 1022, "synset": "stapler.n.01", "synonyms": ["stapler_(stapling_machine)"], "def": "a machine that inserts staples into sheets of paper in order to fasten them together", "name": "stapler_(stapling_machine)"}, {"id": 1023, "synset": "starfish.n.01", "synonyms": ["starfish", "sea_star"], "def": "echinoderms characterized by five arms extending from a central disk", "name": "starfish"}, {"id": 1024, "synset": "statue.n.01", "synonyms": ["statue_(sculpture)"], "def": "a sculpture representing a human or animal", "name": "statue_(sculpture)"}, {"id": 1025, "synset": "steak.n.01", "synonyms": ["steak_(food)"], "def": "a slice of meat cut from the fleshy part of an animal or large fish", "name": "steak_(food)"}, {"id": 1026, "synset": "steak_knife.n.01", "synonyms": ["steak_knife"], "def": "a sharp table knife used in eating steak", "name": "steak_knife"}, {"id": 1027, "synset": "steamer.n.02", "synonyms": ["steamer_(kitchen_appliance)"], "def": "a cooking utensil that can be used to cook food by steaming it", "name": "steamer_(kitchen_appliance)"}, {"id": 1028, "synset": "steering_wheel.n.01", "synonyms": ["steering_wheel"], "def": "a handwheel that is used for steering", "name": "steering_wheel"}, {"id": 1029, "synset": "stencil.n.01", "synonyms": ["stencil"], "def": "a sheet of material (metal, plastic, etc.) that has been perforated with a pattern; ink or paint can pass through the perforations to create the printed pattern on the surface below", "name": "stencil"}, {"id": 1030, "synset": "step_ladder.n.01", "synonyms": ["stepladder"], "def": "a folding portable ladder hinged at the top", "name": "stepladder"}, {"id": 1031, "synset": "step_stool.n.01", "synonyms": ["step_stool"], "def": "a stool that has one or two steps that fold under the seat", "name": "step_stool"}, {"id": 1032, "synset": "stereo.n.01", "synonyms": ["stereo_(sound_system)"], "def": "electronic device for playing audio", "name": "stereo_(sound_system)"}, {"id": 1033, "synset": "stew.n.02", "synonyms": ["stew"], "def": "food prepared by stewing especially meat or fish with vegetables", "name": "stew"}, {"id": 1034, "synset": "stirrer.n.02", "synonyms": ["stirrer"], "def": "an implement used for stirring", "name": "stirrer"}, {"id": 1035, "synset": "stirrup.n.01", "synonyms": ["stirrup"], "def": "support consisting of metal loops into which rider's feet go", "name": "stirrup"}, {"id": 1036, "synset": "stocking.n.01", "synonyms": ["stockings_(leg_wear)"], "def": "close-fitting hosiery to cover the foot and leg; come in matched pairs", "name": "stockings_(leg_wear)"}, {"id": 1037, "synset": "stool.n.01", "synonyms": ["stool"], "def": "a simple seat without a back or arms", "name": "stool"}, {"id": 1038, "synset": "stop_sign.n.01", "synonyms": ["stop_sign"], "def": "a traffic sign to notify drivers that they must come to a complete stop", "name": "stop_sign"}, {"id": 1039, "synset": "stoplight.n.01", "synonyms": ["brake_light"], "def": "a red light on the rear of a motor vehicle that signals when the brakes are applied", "name": "brake_light"}, {"id": 1040, "synset": "stove.n.01", "synonyms": ["stove", "kitchen_stove", "range_(kitchen_appliance)", "kitchen_range", "cooking_stove"], "def": "a kitchen appliance used for cooking food", "name": "stove"}, {"id": 1041, "synset": "strainer.n.01", "synonyms": ["strainer"], "def": "a filter to retain larger pieces while smaller pieces and liquids pass through", "name": "strainer"}, {"id": 1042, "synset": "strap.n.01", "synonyms": ["strap"], "def": "an elongated strip of material for binding things together or holding", "name": "strap"}, {"id": 1043, "synset": "straw.n.04", "synonyms": ["straw_(for_drinking)", "drinking_straw"], "def": "a thin paper or plastic tube used to suck liquids into the mouth", "name": "straw_(for_drinking)"}, {"id": 1044, "synset": "strawberry.n.01", "synonyms": ["strawberry"], "def": "sweet fleshy red fruit", "name": "strawberry"}, {"id": 1045, "synset": "street_sign.n.01", "synonyms": ["street_sign"], "def": "a sign visible from the street", "name": "street_sign"}, {"id": 1046, "synset": "streetlight.n.01", "synonyms": ["streetlight", "street_lamp"], "def": "a lamp supported on a lamppost; for illuminating a street", "name": "streetlight"}, {"id": 1047, "synset": "string_cheese.n.01", "synonyms": ["string_cheese"], "def": "cheese formed in long strings twisted together", "name": "string_cheese"}, {"id": 1048, "synset": "stylus.n.02", "synonyms": ["stylus"], "def": "a pointed tool for writing or drawing or engraving", "name": "stylus"}, {"id": 1049, "synset": "subwoofer.n.01", "synonyms": ["subwoofer"], "def": "a loudspeaker that is designed to reproduce very low bass frequencies", "name": "subwoofer"}, {"id": 1050, "synset": "sugar_bowl.n.01", "synonyms": ["sugar_bowl"], "def": "a dish in which sugar is served", "name": "sugar_bowl"}, {"id": 1051, "synset": "sugarcane.n.01", "synonyms": ["sugarcane_(plant)"], "def": "juicy canes whose sap is a source of molasses and commercial sugar; fresh canes are sometimes chewed for the juice", "name": "sugarcane_(plant)"}, {"id": 1052, "synset": "suit.n.01", "synonyms": ["suit_(clothing)"], "def": "a set of garments (usually including a jacket and trousers or skirt) for outerwear all of the same fabric and color", "name": "suit_(clothing)"}, {"id": 1053, "synset": "sunflower.n.01", "synonyms": ["sunflower"], "def": "any plant of the genus Helianthus having large flower heads with dark disk florets and showy yellow rays", "name": "sunflower"}, {"id": 1054, "synset": "sunglasses.n.01", "synonyms": ["sunglasses"], "def": "spectacles that are darkened or polarized to protect the eyes from the glare of the sun", "name": "sunglasses"}, {"id": 1055, "synset": "sunhat.n.01", "synonyms": ["sunhat"], "def": "a hat with a broad brim that protects the face from direct exposure to the sun", "name": "sunhat"}, {"id": 1056, "synset": "sunscreen.n.01", "synonyms": ["sunscreen", "sunblock"], "def": "a cream spread on the skin; contains a chemical to filter out ultraviolet light and so protect from sunburn", "name": "sunscreen"}, {"id": 1057, "synset": "surfboard.n.01", "synonyms": ["surfboard"], "def": "a narrow buoyant board for riding surf", "name": "surfboard"}, {"id": 1058, "synset": "sushi.n.01", "synonyms": ["sushi"], "def": "rice (with raw fish) wrapped in seaweed", "name": "sushi"}, {"id": 1059, "synset": "swab.n.02", "synonyms": ["mop"], "def": "cleaning implement consisting of absorbent material fastened to a handle; for cleaning floors", "name": "mop"}, {"id": 1060, "synset": "sweat_pants.n.01", "synonyms": ["sweat_pants"], "def": "loose-fitting trousers with elastic cuffs; worn by athletes", "name": "sweat_pants"}, {"id": 1061, "synset": "sweatband.n.02", "synonyms": ["sweatband"], "def": "a band of material tied around the forehead or wrist to absorb sweat", "name": "sweatband"}, {"id": 1062, "synset": "sweater.n.01", "synonyms": ["sweater"], "def": "a crocheted or knitted garment covering the upper part of the body", "name": "sweater"}, {"id": 1063, "synset": "sweatshirt.n.01", "synonyms": ["sweatshirt"], "def": "cotton knit pullover with long sleeves worn during athletic activity", "name": "sweatshirt"}, {"id": 1064, "synset": "sweet_potato.n.02", "synonyms": ["sweet_potato"], "def": "the edible tuberous root of the sweet potato vine", "name": "sweet_potato"}, {"id": 1065, "synset": "swimsuit.n.01", "synonyms": ["swimsuit", "swimwear", "bathing_suit", "swimming_costume", "bathing_costume", "swimming_trunks", "bathing_trunks"], "def": "garment worn for swimming", "name": "swimsuit"}, {"id": 1066, "synset": "sword.n.01", "synonyms": ["sword"], "def": "a cutting or thrusting weapon that has a long metal blade", "name": "sword"}, {"id": 1067, "synset": "syringe.n.01", "synonyms": ["syringe"], "def": "a medical instrument used to inject or withdraw fluids", "name": "syringe"}, {"id": 1068, "synset": "tabasco.n.02", "synonyms": ["Tabasco_sauce"], "def": "very spicy sauce (trade name Tabasco) made from fully-aged red peppers", "name": "Tabasco_sauce"}, {"id": 1069, "synset": "table-tennis_table.n.01", "synonyms": ["table-tennis_table", "ping-pong_table"], "def": "a table used for playing table tennis", "name": "table-tennis_table"}, {"id": 1070, "synset": "table.n.02", "synonyms": ["table"], "def": "a piece of furniture having a smooth flat top that is usually supported by one or more vertical legs", "name": "table"}, {"id": 1071, "synset": "table_lamp.n.01", "synonyms": ["table_lamp"], "def": "a lamp that sits on a table", "name": "table_lamp"}, {"id": 1072, "synset": "tablecloth.n.01", "synonyms": ["tablecloth"], "def": "a covering spread over a dining table", "name": "tablecloth"}, {"id": 1073, "synset": "tachometer.n.01", "synonyms": ["tachometer"], "def": "measuring instrument for indicating speed of rotation", "name": "tachometer"}, {"id": 1074, "synset": "taco.n.02", "synonyms": ["taco"], "def": "a small tortilla cupped around a filling", "name": "taco"}, {"id": 1075, "synset": "tag.n.02", "synonyms": ["tag"], "def": "a label associated with something for the purpose of identification or information", "name": "tag"}, {"id": 1076, "synset": "taillight.n.01", "synonyms": ["taillight", "rear_light"], "def": "lamp (usually red) mounted at the rear of a motor vehicle", "name": "taillight"}, {"id": 1077, "synset": "tambourine.n.01", "synonyms": ["tambourine"], "def": "a shallow drum with a single drumhead and with metallic disks in the sides", "name": "tambourine"}, {"id": 1078, "synset": "tank.n.01", "synonyms": ["army_tank", "armored_combat_vehicle", "armoured_combat_vehicle"], "def": "an enclosed armored military vehicle; has a cannon and moves on caterpillar treads", "name": "army_tank"}, {"id": 1079, "synset": "tank.n.02", "synonyms": ["tank_(storage_vessel)", "storage_tank"], "def": "a large (usually metallic) vessel for holding gases or liquids", "name": "tank_(storage_vessel)"}, {"id": 1080, "synset": "tank_top.n.01", "synonyms": ["tank_top_(clothing)"], "def": "a tight-fitting sleeveless shirt with wide shoulder straps and low neck and no front opening", "name": "tank_top_(clothing)"}, {"id": 1081, "synset": "tape.n.01", "synonyms": ["tape_(sticky_cloth_or_paper)"], "def": "a long thin piece of cloth or paper as used for binding or fastening", "name": "tape_(sticky_cloth_or_paper)"}, {"id": 1082, "synset": "tape.n.04", "synonyms": ["tape_measure", "measuring_tape"], "def": "measuring instrument consisting of a narrow strip (cloth or metal) marked in inches or centimeters and used for measuring lengths", "name": "tape_measure"}, {"id": 1083, "synset": "tapestry.n.02", "synonyms": ["tapestry"], "def": "a heavy textile with a woven design; used for curtains and upholstery", "name": "tapestry"}, {"id": 1084, "synset": "tarpaulin.n.01", "synonyms": ["tarp"], "def": "waterproofed canvas", "name": "tarp"}, {"id": 1085, "synset": "tartan.n.01", "synonyms": ["tartan", "plaid"], "def": "a cloth having a crisscross design", "name": "tartan"}, {"id": 1086, "synset": "tassel.n.01", "synonyms": ["tassel"], "def": "adornment consisting of a bunch of cords fastened at one end", "name": "tassel"}, {"id": 1087, "synset": "tea_bag.n.01", "synonyms": ["tea_bag"], "def": "a measured amount of tea in a bag for an individual serving of tea", "name": "tea_bag"}, {"id": 1088, "synset": "teacup.n.02", "synonyms": ["teacup"], "def": "a cup from which tea is drunk", "name": "teacup"}, {"id": 1089, "synset": "teakettle.n.01", "synonyms": ["teakettle"], "def": "kettle for boiling water to make tea", "name": "teakettle"}, {"id": 1090, "synset": "teapot.n.01", "synonyms": ["teapot"], "def": "pot for brewing tea; usually has a spout and handle", "name": "teapot"}, {"id": 1091, "synset": "teddy.n.01", "synonyms": ["teddy_bear"], "def": "plaything consisting of a child's toy bear (usually plush and stuffed with soft materials)", "name": "teddy_bear"}, {"id": 1092, "synset": "telephone.n.01", "synonyms": ["telephone", "phone", "telephone_set"], "def": "electronic device for communicating by voice over long distances", "name": "telephone"}, {"id": 1093, "synset": "telephone_booth.n.01", "synonyms": ["telephone_booth", "phone_booth", "call_box", "telephone_box", "telephone_kiosk"], "def": "booth for using a telephone", "name": "telephone_booth"}, {"id": 1094, "synset": "telephone_pole.n.01", "synonyms": ["telephone_pole", "telegraph_pole", "telegraph_post"], "def": "tall pole supporting telephone wires", "name": "telephone_pole"}, {"id": 1095, "synset": "telephoto_lens.n.01", "synonyms": ["telephoto_lens", "zoom_lens"], "def": "a camera lens that magnifies the image", "name": "telephoto_lens"}, {"id": 1096, "synset": "television_camera.n.01", "synonyms": ["television_camera", "tv_camera"], "def": "television equipment for capturing and recording video", "name": "television_camera"}, {"id": 1097, "synset": "television_receiver.n.01", "synonyms": ["television_set", "tv", "tv_set"], "def": "an electronic device that receives television signals and displays them on a screen", "name": "television_set"}, {"id": 1098, "synset": "tennis_ball.n.01", "synonyms": ["tennis_ball"], "def": "ball about the size of a fist used in playing tennis", "name": "tennis_ball"}, {"id": 1099, "synset": "tennis_racket.n.01", "synonyms": ["tennis_racket"], "def": "a racket used to play tennis", "name": "tennis_racket"}, {"id": 1100, "synset": "tequila.n.01", "synonyms": ["tequila"], "def": "Mexican liquor made from fermented juices of an agave plant", "name": "tequila"}, {"id": 1101, "synset": "thermometer.n.01", "synonyms": ["thermometer"], "def": "measuring instrument for measuring temperature", "name": "thermometer"}, {"id": 1102, "synset": "thermos.n.01", "synonyms": ["thermos_bottle"], "def": "vacuum flask that preserves temperature of hot or cold drinks", "name": "thermos_bottle"}, {"id": 1103, "synset": "thermostat.n.01", "synonyms": ["thermostat"], "def": "a regulator for automatically regulating temperature by starting or stopping the supply of heat", "name": "thermostat"}, {"id": 1104, "synset": "thimble.n.02", "synonyms": ["thimble"], "def": "a small metal cap to protect the finger while sewing; can be used as a small container", "name": "thimble"}, {"id": 1105, "synset": "thread.n.01", "synonyms": ["thread", "yarn"], "def": "a fine cord of twisted fibers (of cotton or silk or wool or nylon etc.) used in sewing and weaving", "name": "thread"}, {"id": 1106, "synset": "thumbtack.n.01", "synonyms": ["thumbtack", "drawing_pin", "pushpin"], "def": "a tack for attaching papers to a bulletin board or drawing board", "name": "thumbtack"}, {"id": 1107, "synset": "tiara.n.01", "synonyms": ["tiara"], "def": "a jeweled headdress worn by women on formal occasions", "name": "tiara"}, {"id": 1108, "synset": "tiger.n.02", "synonyms": ["tiger"], "def": "large feline of forests in most of Asia having a tawny coat with black stripes", "name": "tiger"}, {"id": 1109, "synset": "tights.n.01", "synonyms": ["tights_(clothing)", "leotards"], "def": "skintight knit hose covering the body from the waist to the feet worn by acrobats and dancers and as stockings by women and girls", "name": "tights_(clothing)"}, {"id": 1110, "synset": "timer.n.01", "synonyms": ["timer", "stopwatch"], "def": "a timepiece that measures a time interval and signals its end", "name": "timer"}, {"id": 1111, "synset": "tinfoil.n.01", "synonyms": ["tinfoil"], "def": "foil made of tin or an alloy of tin and lead", "name": "tinfoil"}, {"id": 1112, "synset": "tinsel.n.01", "synonyms": ["tinsel"], "def": "a showy decoration that is basically valueless", "name": "tinsel"}, {"id": 1113, "synset": "tissue.n.02", "synonyms": ["tissue_paper"], "def": "a soft thin (usually translucent) paper", "name": "tissue_paper"}, {"id": 1114, "synset": "toast.n.01", "synonyms": ["toast_(food)"], "def": "slice of bread that has been toasted", "name": "toast_(food)"}, {"id": 1115, "synset": "toaster.n.02", "synonyms": ["toaster"], "def": "a kitchen appliance (usually electric) for toasting bread", "name": "toaster"}, {"id": 1116, "synset": "toaster_oven.n.01", "synonyms": ["toaster_oven"], "def": "kitchen appliance consisting of a small electric oven for toasting or warming food", "name": "toaster_oven"}, {"id": 1117, "synset": "toilet.n.02", "synonyms": ["toilet"], "def": "a plumbing fixture for defecation and urination", "name": "toilet"}, {"id": 1118, "synset": "toilet_tissue.n.01", "synonyms": ["toilet_tissue", "toilet_paper", "bathroom_tissue"], "def": "a soft thin absorbent paper for use in toilets", "name": "toilet_tissue"}, {"id": 1119, "synset": "tomato.n.01", "synonyms": ["tomato"], "def": "mildly acid red or yellow pulpy fruit eaten as a vegetable", "name": "tomato"}, {"id": 1120, "synset": "tongs.n.01", "synonyms": ["tongs"], "def": "any of various devices for taking hold of objects; usually have two hinged legs with handles above and pointed hooks below", "name": "tongs"}, {"id": 1121, "synset": "toolbox.n.01", "synonyms": ["toolbox"], "def": "a box or chest or cabinet for holding hand tools", "name": "toolbox"}, {"id": 1122, "synset": "toothbrush.n.01", "synonyms": ["toothbrush"], "def": "small brush; has long handle; used to clean teeth", "name": "toothbrush"}, {"id": 1123, "synset": "toothpaste.n.01", "synonyms": ["toothpaste"], "def": "a dentifrice in the form of a paste", "name": "toothpaste"}, {"id": 1124, "synset": "toothpick.n.01", "synonyms": ["toothpick"], "def": "pick consisting of a small strip of wood or plastic; used to pick food from between the teeth", "name": "toothpick"}, {"id": 1125, "synset": "top.n.09", "synonyms": ["cover"], "def": "covering for a hole (especially a hole in the top of a container)", "name": "cover"}, {"id": 1126, "synset": "tortilla.n.01", "synonyms": ["tortilla"], "def": "thin unleavened pancake made from cornmeal or wheat flour", "name": "tortilla"}, {"id": 1127, "synset": "tow_truck.n.01", "synonyms": ["tow_truck"], "def": "a truck equipped to hoist and pull wrecked cars (or to remove cars from no-parking zones)", "name": "tow_truck"}, {"id": 1128, "synset": "towel.n.01", "synonyms": ["towel"], "def": "a rectangular piece of absorbent cloth (or paper) for drying or wiping", "name": "towel"}, {"id": 1129, "synset": "towel_rack.n.01", "synonyms": ["towel_rack", "towel_rail", "towel_bar"], "def": "a rack consisting of one or more bars on which towels can be hung", "name": "towel_rack"}, {"id": 1130, "synset": "toy.n.03", "synonyms": ["toy"], "def": "a device regarded as providing amusement", "name": "toy"}, {"id": 1131, "synset": "tractor.n.01", "synonyms": ["tractor_(farm_equipment)"], "def": "a wheeled vehicle with large wheels; used in farming and other applications", "name": "tractor_(farm_equipment)"}, {"id": 1132, "synset": "traffic_light.n.01", "synonyms": ["traffic_light"], "def": "a device to control vehicle traffic often consisting of three or more lights", "name": "traffic_light"}, {"id": 1133, "synset": "trail_bike.n.01", "synonyms": ["dirt_bike"], "def": "a lightweight motorcycle equipped with rugged tires and suspension for off-road use", "name": "dirt_bike"}, {"id": 1134, "synset": "trailer_truck.n.01", "synonyms": ["trailer_truck", "tractor_trailer", "trucking_rig", "articulated_lorry", "semi_truck"], "def": "a truck consisting of a tractor and trailer together", "name": "trailer_truck"}, {"id": 1135, "synset": "train.n.01", "synonyms": ["train_(railroad_vehicle)", "railroad_train"], "def": "public or private transport provided by a line of railway cars coupled together and drawn by a locomotive", "name": "train_(railroad_vehicle)"}, {"id": 1136, "synset": "trampoline.n.01", "synonyms": ["trampoline"], "def": "gymnastic apparatus consisting of a strong canvas sheet attached with springs to a metal frame", "name": "trampoline"}, {"id": 1137, "synset": "tray.n.01", "synonyms": ["tray"], "def": "an open receptacle for holding or displaying or serving articles or food", "name": "tray"}, {"id": 1138, "synset": "tree_house.n.01", "synonyms": ["tree_house"], "def": "(NOT A TREE) a PLAYHOUSE built in the branches of a tree", "name": "tree_house"}, {"id": 1139, "synset": "trench_coat.n.01", "synonyms": ["trench_coat"], "def": "a military style raincoat; belted with deep pockets", "name": "trench_coat"}, {"id": 1140, "synset": "triangle.n.05", "synonyms": ["triangle_(musical_instrument)"], "def": "a percussion instrument consisting of a metal bar bent in the shape of an open triangle", "name": "triangle_(musical_instrument)"}, {"id": 1141, "synset": "tricycle.n.01", "synonyms": ["tricycle"], "def": "a vehicle with three wheels that is moved by foot pedals", "name": "tricycle"}, {"id": 1142, "synset": "tripod.n.01", "synonyms": ["tripod"], "def": "a three-legged rack used for support", "name": "tripod"}, {"id": 1143, "synset": "trouser.n.01", "synonyms": ["trousers", "pants_(clothing)"], "def": "a garment extending from the waist to the knee or ankle, covering each leg separately", "name": "trousers"}, {"id": 1144, "synset": "truck.n.01", "synonyms": ["truck"], "def": "an automotive vehicle suitable for hauling", "name": "truck"}, {"id": 1145, "synset": "truffle.n.03", "synonyms": ["truffle_(chocolate)", "chocolate_truffle"], "def": "creamy chocolate candy", "name": "truffle_(chocolate)"}, {"id": 1146, "synset": "trunk.n.02", "synonyms": ["trunk"], "def": "luggage consisting of a large strong case used when traveling or for storage", "name": "trunk"}, {"id": 1147, "synset": "tub.n.02", "synonyms": ["vat"], "def": "a large open vessel for holding or storing liquids", "name": "vat"}, {"id": 1148, "synset": "turban.n.01", "synonyms": ["turban"], "def": "a traditional headdress consisting of a long scarf wrapped around the head", "name": "turban"}, {"id": 1149, "synset": "turkey.n.01", "synonyms": ["turkey_(bird)"], "def": "large gallinaceous bird with fan-shaped tail; widely domesticated for food", "name": "turkey_(bird)"}, {"id": 1150, "synset": "turkey.n.04", "synonyms": ["turkey_(food)"], "def": "flesh of large domesticated fowl usually roasted", "name": "turkey_(food)"}, {"id": 1151, "synset": "turnip.n.01", "synonyms": ["turnip"], "def": "widely cultivated plant having a large fleshy edible white or yellow root", "name": "turnip"}, {"id": 1152, "synset": "turtle.n.02", "synonyms": ["turtle"], "def": "any of various aquatic and land reptiles having a bony shell and flipper-like limbs for swimming", "name": "turtle"}, {"id": 1153, "synset": "turtleneck.n.01", "synonyms": ["turtleneck_(clothing)", "polo-neck"], "def": "a sweater or jersey with a high close-fitting collar", "name": "turtleneck_(clothing)"}, {"id": 1154, "synset": "typewriter.n.01", "synonyms": ["typewriter"], "def": "hand-operated character printer for printing written messages one character at a time", "name": "typewriter"}, {"id": 1155, "synset": "umbrella.n.01", "synonyms": ["umbrella"], "def": "a lightweight handheld collapsible canopy", "name": "umbrella"}, {"id": 1156, "synset": "underwear.n.01", "synonyms": ["underwear", "underclothes", "underclothing", "underpants"], "def": "undergarment worn next to the skin and under the outer garments", "name": "underwear"}, {"id": 1157, "synset": "unicycle.n.01", "synonyms": ["unicycle"], "def": "a vehicle with a single wheel that is driven by pedals", "name": "unicycle"}, {"id": 1158, "synset": "urinal.n.01", "synonyms": ["urinal"], "def": "a plumbing fixture (usually attached to the wall) used by men to urinate", "name": "urinal"}, {"id": 1159, "synset": "urn.n.01", "synonyms": ["urn"], "def": "a large vase that usually has a pedestal or feet", "name": "urn"}, {"id": 1160, "synset": "vacuum.n.04", "synonyms": ["vacuum_cleaner"], "def": "an electrical home appliance that cleans by suction", "name": "vacuum_cleaner"}, {"id": 1161, "synset": "valve.n.03", "synonyms": ["valve"], "def": "control consisting of a mechanical device for controlling the flow of a fluid", "name": "valve"}, {"id": 1162, "synset": "vase.n.01", "synonyms": ["vase"], "def": "an open jar of glass or porcelain used as an ornament or to hold flowers", "name": "vase"}, {"id": 1163, "synset": "vending_machine.n.01", "synonyms": ["vending_machine"], "def": "a slot machine for selling goods", "name": "vending_machine"}, {"id": 1164, "synset": "vent.n.01", "synonyms": ["vent", "blowhole", "air_vent"], "def": "a hole for the escape of gas or air", "name": "vent"}, {"id": 1165, "synset": "videotape.n.01", "synonyms": ["videotape"], "def": "a video recording made on magnetic tape", "name": "videotape"}, {"id": 1166, "synset": "vinegar.n.01", "synonyms": ["vinegar"], "def": "sour-tasting liquid produced usually by oxidation of the alcohol in wine or cider and used as a condiment or food preservative", "name": "vinegar"}, {"id": 1167, "synset": "violin.n.01", "synonyms": ["violin", "fiddle"], "def": "bowed stringed instrument that is the highest member of the violin family", "name": "violin"}, {"id": 1168, "synset": "vodka.n.01", "synonyms": ["vodka"], "def": "unaged colorless liquor originating in Russia", "name": "vodka"}, {"id": 1169, "synset": "volleyball.n.02", "synonyms": ["volleyball"], "def": "an inflated ball used in playing volleyball", "name": "volleyball"}, {"id": 1170, "synset": "vulture.n.01", "synonyms": ["vulture"], "def": "any of various large birds of prey having naked heads and weak claws and feeding chiefly on carrion", "name": "vulture"}, {"id": 1171, "synset": "waffle.n.01", "synonyms": ["waffle"], "def": "pancake batter baked in a waffle iron", "name": "waffle"}, {"id": 1172, "synset": "waffle_iron.n.01", "synonyms": ["waffle_iron"], "def": "a kitchen appliance for baking waffles", "name": "waffle_iron"}, {"id": 1173, "synset": "wagon.n.01", "synonyms": ["wagon"], "def": "any of various kinds of wheeled vehicles drawn by an animal or a tractor", "name": "wagon"}, {"id": 1174, "synset": "wagon_wheel.n.01", "synonyms": ["wagon_wheel"], "def": "a wheel of a wagon", "name": "wagon_wheel"}, {"id": 1175, "synset": "walking_stick.n.01", "synonyms": ["walking_stick"], "def": "a stick carried in the hand for support in walking", "name": "walking_stick", "merged": [{"frequency": "c", "id": 201, "synset": "cane.n.01", "image_count": 5, "instance_count": 5, "synonyms": ["walking_cane"], "def": "a stick that people can lean on to help them walk", "name": "walking_cane"}]}, {"id": 1176, "synset": "wall_clock.n.01", "synonyms": ["wall_clock"], "def": "a clock mounted on a wall", "name": "wall_clock"}, {"id": 1177, "synset": "wall_socket.n.01", "synonyms": ["wall_socket", "wall_plug", "electric_outlet", "electrical_outlet", "outlet", "electric_receptacle"], "def": "receptacle providing a place in a wiring system where current can be taken to run electrical devices", "name": "wall_socket"}, {"id": 1178, "synset": "wallet.n.01", "synonyms": ["wallet", "billfold"], "def": "a pocket-size case for holding papers and paper money", "name": "wallet"}, {"id": 1179, "synset": "walrus.n.01", "synonyms": ["walrus"], "def": "either of two large northern marine mammals having ivory tusks and tough hide over thick blubber", "name": "walrus"}, {"id": 1180, "synset": "wardrobe.n.01", "synonyms": ["wardrobe"], "def": "a tall piece of furniture that provides storage space for clothes; has a door and rails or hooks for hanging clothes", "name": "wardrobe"}, {"id": 1181, "synset": "wasabi.n.02", "synonyms": ["wasabi"], "def": "the thick green root of the wasabi plant that the Japanese use in cooking and that tastes like strong horseradish", "name": "wasabi"}, {"id": 1182, "synset": "washer.n.03", "synonyms": ["automatic_washer", "washing_machine"], "def": "a home appliance for washing clothes and linens automatically", "name": "automatic_washer"}, {"id": 1183, "synset": "watch.n.01", "synonyms": ["watch", "wristwatch"], "def": "a small, portable timepiece", "name": "watch"}, {"id": 1184, "synset": "water_bottle.n.01", "synonyms": ["water_bottle"], "def": "a bottle for holding water", "name": "water_bottle"}, {"id": 1185, "synset": "water_cooler.n.01", "synonyms": ["water_cooler"], "def": "a device for cooling and dispensing drinking water", "name": "water_cooler"}, {"id": 1186, "synset": "water_faucet.n.01", "synonyms": ["water_faucet", "water_tap", "tap_(water_faucet)"], "def": "a faucet for drawing water from a pipe or cask", "name": "water_faucet"}, {"id": 1187, "synset": "water_filter.n.01", "synonyms": ["water_filter"], "def": "a filter to remove impurities from the water supply", "name": "water_filter"}, {"id": 1188, "synset": "water_heater.n.01", "synonyms": ["water_heater", "hot-water_heater"], "def": "a heater and storage tank to supply heated water", "name": "water_heater"}, {"id": 1189, "synset": "water_jug.n.01", "synonyms": ["water_jug"], "def": "a jug that holds water", "name": "water_jug"}, {"id": 1190, "synset": "water_pistol.n.01", "synonyms": ["water_gun", "squirt_gun"], "def": "plaything consisting of a toy pistol that squirts water", "name": "water_gun"}, {"id": 1191, "synset": "water_scooter.n.01", "synonyms": ["water_scooter", "sea_scooter", "jet_ski"], "def": "a motorboat resembling a motor scooter (NOT A SURFBOARD OR WATER SKI)", "name": "water_scooter"}, {"id": 1192, "synset": "water_ski.n.01", "synonyms": ["water_ski"], "def": "broad ski for skimming over water towed by a speedboat (DO NOT MARK WATER)", "name": "water_ski"}, {"id": 1193, "synset": "water_tower.n.01", "synonyms": ["water_tower"], "def": "a large reservoir for water", "name": "water_tower"}, {"id": 1194, "synset": "watering_can.n.01", "synonyms": ["watering_can"], "def": "a container with a handle and a spout with a perforated nozzle; used to sprinkle water over plants", "name": "watering_can"}, {"id": 1195, "synset": "watermelon.n.02", "synonyms": ["watermelon"], "def": "large oblong or roundish melon with a hard green rind and sweet watery red or occasionally yellowish pulp", "name": "watermelon"}, {"id": 1196, "synset": "weathervane.n.01", "synonyms": ["weathervane", "vane_(weathervane)", "wind_vane"], "def": "mechanical device attached to an elevated structure; rotates freely to show the direction of the wind", "name": "weathervane"}, {"id": 1197, "synset": "webcam.n.01", "synonyms": ["webcam"], "def": "a digital camera designed to take digital photographs and transmit them over the internet", "name": "webcam"}, {"id": 1198, "synset": "wedding_cake.n.01", "synonyms": ["wedding_cake", "bridecake"], "def": "a rich cake with two or more tiers and covered with frosting and decorations; served at a wedding reception", "name": "wedding_cake"}, {"id": 1199, "synset": "wedding_ring.n.01", "synonyms": ["wedding_ring", "wedding_band"], "def": "a ring given to the bride and/or groom at the wedding", "name": "wedding_ring"}, {"id": 1200, "synset": "wet_suit.n.01", "synonyms": ["wet_suit"], "def": "a close-fitting garment made of a permeable material; worn in cold water to retain body heat", "name": "wet_suit"}, {"id": 1201, "synset": "wheel.n.01", "synonyms": ["wheel"], "def": "a circular frame with spokes (or a solid disc) that can rotate on a shaft or axle", "name": "wheel"}, {"id": 1202, "synset": "wheelchair.n.01", "synonyms": ["wheelchair"], "def": "a movable chair mounted on large wheels", "name": "wheelchair"}, {"id": 1203, "synset": "whipped_cream.n.01", "synonyms": ["whipped_cream"], "def": "cream that has been beaten until light and fluffy", "name": "whipped_cream"}, {"id": 1204, "synset": "whiskey.n.01", "synonyms": ["whiskey"], "def": "a liquor made from fermented mash of grain", "name": "whiskey"}, {"id": 1205, "synset": "whistle.n.03", "synonyms": ["whistle"], "def": "a small wind instrument that produces a whistling sound by blowing into it", "name": "whistle"}, {"id": 1206, "synset": "wick.n.02", "synonyms": ["wick"], "def": "a loosely woven cord in a candle or oil lamp that is lit on fire", "name": "wick"}, {"id": 1207, "synset": "wig.n.01", "synonyms": ["wig"], "def": "hairpiece covering the head and made of real or synthetic hair", "name": "wig"}, {"id": 1208, "synset": "wind_chime.n.01", "synonyms": ["wind_chime"], "def": "a decorative arrangement of pieces of metal or glass or pottery that hang together loosely so the wind can cause them to tinkle", "name": "wind_chime"}, {"id": 1209, "synset": "windmill.n.01", "synonyms": ["windmill"], "def": "a mill that is powered by the wind", "name": "windmill"}, {"id": 1210, "synset": "window_box.n.01", "synonyms": ["window_box_(for_plants)"], "def": "a container for growing plants on a windowsill", "name": "window_box_(for_plants)"}, {"id": 1211, "synset": "windshield_wiper.n.01", "synonyms": ["windshield_wiper", "windscreen_wiper", "wiper_(for_windshield/screen)"], "def": "a mechanical device that cleans the windshield", "name": "windshield_wiper"}, {"id": 1212, "synset": "windsock.n.01", "synonyms": ["windsock", "air_sock", "air-sleeve", "wind_sleeve", "wind_cone"], "def": "a truncated cloth cone mounted on a mast/pole; shows wind direction", "name": "windsock"}, {"id": 1213, "synset": "wine_bottle.n.01", "synonyms": ["wine_bottle"], "def": "a bottle for holding wine", "name": "wine_bottle"}, {"id": 1214, "synset": "wine_bucket.n.01", "synonyms": ["wine_bucket", "wine_cooler"], "def": "a bucket of ice used to chill a bottle of wine", "name": "wine_bucket"}, {"id": 1215, "synset": "wineglass.n.01", "synonyms": ["wineglass"], "def": "a glass that has a stem and in which wine is served", "name": "wineglass"}, {"id": 1216, "synset": "wing_chair.n.01", "synonyms": ["wing_chair"], "def": "easy chair having wings on each side of a high back", "name": "wing_chair"}, {"id": 1217, "synset": "winker.n.02", "synonyms": ["blinder_(for_horses)"], "def": "blinds that prevent a horse from seeing something on either side", "name": "blinder_(for_horses)"}, {"id": 1218, "synset": "wok.n.01", "synonyms": ["wok"], "def": "pan with a convex bottom; used for frying in Chinese cooking", "name": "wok"}, {"id": 1219, "synset": "wolf.n.01", "synonyms": ["wolf"], "def": "a wild carnivorous mammal of the dog family, living and hunting in packs", "name": "wolf"}, {"id": 1220, "synset": "wooden_spoon.n.02", "synonyms": ["wooden_spoon"], "def": "a spoon made of wood", "name": "wooden_spoon"}, {"id": 1221, "synset": "wreath.n.01", "synonyms": ["wreath"], "def": "an arrangement of flowers, leaves, or stems fastened in a ring", "name": "wreath"}, {"id": 1222, "synset": "wrench.n.03", "synonyms": ["wrench", "spanner"], "def": "a hand tool that is used to hold or twist a nut or bolt", "name": "wrench"}, {"id": 1223, "synset": "wristband.n.01", "synonyms": ["wristband"], "def": "band consisting of a part of a sleeve that covers the wrist", "name": "wristband"}, {"id": 1224, "synset": "wristlet.n.01", "synonyms": ["wristlet", "wrist_band"], "def": "a band or bracelet worn around the wrist", "name": "wristlet"}, {"id": 1225, "synset": "yacht.n.01", "synonyms": ["yacht"], "def": "an expensive vessel propelled by sail or power and used for cruising or racing", "name": "yacht"}, {"id": 1226, "synset": "yak.n.02", "synonyms": ["yak"], "def": "large long-haired wild ox of Tibet often domesticated", "name": "yak"}, {"id": 1227, "synset": "yogurt.n.01", "synonyms": ["yogurt", "yoghurt", "yoghourt"], "def": "a custard-like food made from curdled milk", "name": "yogurt"}, {"id": 1228, "synset": "yoke.n.07", "synonyms": ["yoke_(animal_equipment)"], "def": "gear joining two animals at the neck; NOT egg yolk", "name": "yoke_(animal_equipment)"}, {"id": 1229, "synset": "zebra.n.01", "synonyms": ["zebra"], "def": "any of several fleet black-and-white striped African equines", "name": "zebra"}, {"id": 1230, "synset": "zucchini.n.02", "synonyms": ["zucchini", "courgette"], "def": "small cucumber-shaped vegetable marrow; typically dark green", "name": "zucchini"}] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/burst_ow.py b/test/yolov7-tracker/tracker/trackeval/datasets/burst_ow.py new file mode 100644 index 0000000..da77545 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/burst_ow.py @@ -0,0 +1,91 @@ +import json +import os +from .burst_helpers.burst_ow_base import BURST_OW_Base +from .burst_helpers.format_converter import GroundTruthBURSTFormatToTAOFormatConverter, PredictionBURSTFormatToTAOFormatConverter +from .. import utils + + +class BURST_OW(BURST_OW_Base): + """Dataset class for TAO tracking""" + + @staticmethod + def get_default_dataset_config(): + tao_config = BURST_OW_Base.get_default_dataset_config() + code_path = utils.get_code_path() + tao_config['GT_FOLDER'] = os.path.join( + code_path, 'data/gt/burst/all_classes/val/') # Location of GT data + tao_config['TRACKERS_FOLDER'] = os.path.join( + code_path, 'data/trackers/burst/open-world/val/') # Trackers location + return tao_config + + def _iou_type(self): + return 'mask' + + def _box_or_mask_from_det(self, det): + if "segmentation" in det: + return det["segmentation"] + else: + return det["mask"] + + def _calculate_area_for_ann(self, ann): + import pycocotools.mask as cocomask + seg = self._box_or_mask_from_det(ann) + return cocomask.area(seg) + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores + + def _postproc_ground_truth_data(self, data): + return GroundTruthBURSTFormatToTAOFormatConverter(data).convert() + + def _postproc_prediction_data(self, data): + # if it's a list, it's already in TAO format and not in Ali format + # however the image ids do not match and need to be remapped + if isinstance(data, list): + _remap_image_ids(data, self.gt_data) + return data + + return PredictionBURSTFormatToTAOFormatConverter( + self.gt_data, data, + exemplar_guided=False).convert() + + +def _remap_image_ids(pred_data, ali_gt_data): + code_path = utils.get_code_path() + if 'split' in ali_gt_data: + split = ali_gt_data['split'] + else: + split = 'val' + + if split in ('val', 'validation'): + tao_gt_path = os.path.join( + code_path, 'data/gt/tao/tao_validation/gt.json') + else: + tao_gt_path = os.path.join( + code_path, 'data/gt/tao/tao_test/test_without_annotations.json') + + with open(tao_gt_path) as f: + tao_gt = json.load(f) + + tao_img_by_id = {} + for img in tao_gt['images']: + img_id = img['id'] + tao_img_by_id[img_id] = img + + ali_img_id_by_filename = {} + for ali_img in ali_gt_data['images']: + ali_img_id = ali_img['id'] + file_name = ali_img['file_name'].replace("validation", "val") + ali_img_id_by_filename[file_name] = ali_img_id + + ali_img_id_by_tao_img_id = {} + for tao_img_id, tao_img in tao_img_by_id.items(): + file_name = tao_img['file_name'] + ali_img_id = ali_img_id_by_filename[file_name] + ali_img_id_by_tao_img_id[tao_img_id] = ali_img_id + + for det in pred_data: + tao_img_id = det['image_id'] + ali_img_id = ali_img_id_by_tao_img_id[tao_img_id] + det['image_id'] = ali_img_id diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/davis.py b/test/yolov7-tracker/tracker/trackeval/datasets/davis.py new file mode 100644 index 0000000..9db25e9 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/davis.py @@ -0,0 +1,276 @@ +import os +import csv +import numpy as np +from ._base_dataset import _BaseDataset +from ..utils import TrackEvalException +from .. import utils +from .. import _timing + + +class DAVIS(_BaseDataset): + """Dataset class for DAVIS tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/davis/davis_unsupervised_val/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/davis/davis_unsupervised_val/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'SPLIT_TO_EVAL': 'val', # Valid: 'val', 'train' + 'CLASSES_TO_EVAL': ['general'], + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FILE': None, # Specify seqmap file + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + # '{gt_folder}/Annotations_unsupervised/480p/{seq}' + 'MAX_DETECTIONS': 0 # Maximum number of allowed detections per sequence (0 for no threshold) + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + # defining a default class since there are no classes in DAVIS + self.should_classes_combine = False + self.use_super_categories = False + + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.config['TRACKERS_FOLDER'] + + self.max_det = self.config['MAX_DETECTIONS'] + + # Get classes to eval + self.valid_classes = ['general'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only general class is valid.') + + # Get sequences to eval + if self.config["SEQ_INFO"]: + self.seq_list = list(self.config["SEQ_INFO"].keys()) + self.seq_lengths = self.config["SEQ_INFO"] + elif self.config["SEQMAP_FILE"]: + self.seq_list = [] + seqmap_file = self.config["SEQMAP_FILE"] + if not os.path.isfile(seqmap_file): + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if row[0] == '': + continue + seq = row[0] + self.seq_list.append(seq) + else: + self.seq_list = os.listdir(self.gt_fol) + + self.seq_lengths = {seq: len(os.listdir(os.path.join(self.gt_fol, seq))) for seq in self.seq_list} + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + for tracker in self.tracker_list: + for seq in self.seq_list: + curr_dir = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq) + if not os.path.isdir(curr_dir): + print('Tracker directory not found: ' + curr_dir) + raise TrackEvalException('Tracker directory not found: ' + + os.path.join(tracker, self.tracker_sub_fol, seq)) + tr_timesteps = len(os.listdir(curr_dir)) + if self.seq_lengths[seq] != tr_timesteps: + raise TrackEvalException('GT folder and tracker folder have a different number' + 'timesteps for tracker %s and sequence %s' % (tracker, seq)) + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the DAVIS format + + If is_gt, this returns a dict which contains the fields: + [gt_ids] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [masks_void]: list of masks with void pixels (pixels to be ignored during evaluation) + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + from PIL import Image + + # File location + if is_gt: + seq_dir = os.path.join(self.gt_fol, seq) + else: + seq_dir = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq) + + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'dets', 'masks_void'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # read frames + frames = [os.path.join(seq_dir, im_name) for im_name in sorted(os.listdir(seq_dir))] + + id_list = [] + for t in range(num_timesteps): + frame = np.array(Image.open(frames[t])) + if is_gt: + void = frame == 255 + frame[void] = 0 + raw_data['masks_void'][t] = mask_utils.encode(np.asfortranarray(void.astype(np.uint8))) + id_values = np.unique(frame) + id_values = id_values[id_values != 0] + id_list += list(id_values) + tmp = np.ones((len(id_values), *frame.shape)) + tmp = tmp * id_values[:, None, None] + masks = np.array(tmp == frame[None, ...]).astype(np.uint8) + raw_data['dets'][t] = mask_utils.encode(np.array(np.transpose(masks, (1, 2, 0)), order='F')) + raw_data['ids'][t] = id_values.astype(int) + num_objects = len(np.unique(id_list)) + + if not is_gt and num_objects > self.max_det > 0: + raise Exception('Number of proposals (%i) for sequence %s exceeds number of maximum allowed proposals (%i).' + % (num_objects, seq, self.max_det)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data["num_timesteps"] = num_timesteps + raw_data['mask_shape'] = np.array(Image.open(frames[0])).shape + if is_gt: + raw_data['num_gt_ids'] = num_objects + else: + raw_data['num_tracker_ids'] = num_objects + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + DAVIS: + In DAVIS, the 4 preproc steps are as follow: + 1) There are no classes, all detections are evaluated jointly + 2) No matched tracker detections are removed. + 3) No unmatched tracker detections are removed. + 4) There are no ground truth detections (e.g. those of distractor classes) to be removed. + Preprocessing special to DAVIS: Pixels which are marked as void in the ground truth are set to zero in the + tracker detections since they are not considered during evaluation. + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + num_gt_dets = 0 + num_tracker_dets = 0 + unique_gt_ids = [] + unique_tracker_ids = [] + num_timesteps = raw_data['num_timesteps'] + + # count detections + for t in range(num_timesteps): + num_gt_dets += len(raw_data['gt_dets'][t]) + num_tracker_dets += len(raw_data['tracker_dets'][t]) + unique_gt_ids += list(np.unique(raw_data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(raw_data['tracker_ids'][t])) + + data['gt_ids'] = raw_data['gt_ids'] + data['gt_dets'] = raw_data['gt_dets'] + data['similarity_scores'] = raw_data['similarity_scores'] + data['tracker_ids'] = raw_data['tracker_ids'] + + # set void pixels in tracker detections to zero + for t in range(num_timesteps): + void_mask = raw_data['masks_void'][t] + if mask_utils.area(void_mask) > 0: + void_mask_ious = np.atleast_1d(mask_utils.iou(raw_data['tracker_dets'][t], [void_mask], [False])) + if void_mask_ious.any(): + rows, columns = np.where(void_mask_ious > 0) + for r in rows: + det = mask_utils.decode(raw_data['tracker_dets'][t][r]) + void = mask_utils.decode(void_mask).astype(np.bool) + det[void] = 0 + det = mask_utils.encode(np.array(det, order='F').astype(np.uint8)) + raw_data['tracker_dets'][t][r] = det + data['tracker_dets'] = raw_data['tracker_dets'] + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = raw_data['num_tracker_ids'] + data['num_gt_ids'] = raw_data['num_gt_ids'] + data['mask_shape'] = raw_data['mask_shape'] + data['num_timesteps'] = num_timesteps + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/head_tracking_challenge.py b/test/yolov7-tracker/tracker/trackeval/datasets/head_tracking_challenge.py new file mode 100644 index 0000000..469e9a3 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/head_tracking_challenge.py @@ -0,0 +1,459 @@ +import os +import csv +import configparser +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + + +class HeadTrackingChallenge(_BaseDataset): + """Dataset class for Head Tracking Challenge - 2D bounding box tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrian'], # Valid: ['pedestrian'] + 'BENCHMARK': 'HT', # Valid: 'HT'. Refers to "Head Tracking or the dataset CroHD" + 'SPLIT_TO_EVAL': 'train', # Valid: 'train', 'test', 'all' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'DO_PREPROC': True, # Whether to perform preprocessing (never done for MOT15) + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt', # '{gt_folder}/{seq}/gt/gt.txt' + 'SKIP_SPLIT_FOL': False, # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in + # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/ + # If True, then the middle 'benchmark-split' folder is skipped for both. + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + + self.benchmark = self.config['BENCHMARK'] + gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL'] + self.gt_set = gt_set + if not self.config['SKIP_SPLIT_FOL']: + split_fol = gt_set + else: + split_fol = '' + self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol) + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol) + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + self.do_preproc = self.config['DO_PREPROC'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.') + self.class_name_to_class_id = {'pedestrian': 1, 'static': 2, 'ignore': 3, 'person_on_vehicle': 4} + self.valid_class_numbers = list(self.class_name_to_class_id.values()) + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + + # If sequence length is 'None' tries to read sequence length from .ini files. + for seq, seq_length in seq_lengths.items(): + if seq_length is None: + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if i == 0 or row[0] == '': + continue + seq = row[0] + seq_list.append(seq) + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the MOT Challenge 2D box format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det). + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file) + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions', 'gt_extras'] + else: + data_keys += ['tracker_confidences'] + + if self.benchmark == 'HT': + data_keys += ['visibility'] + data_keys += ['gt_conf'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str( t+ 1) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t+1) + if time_key in read_data.keys(): + try: + time_data = np.asarray(read_data[time_key], dtype=np.float) + except ValueError: + if is_gt: + raise TrackEvalException( + 'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % ( + tracker, seq)) + try: + raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6]) + raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int) + except IndexError: + if is_gt: + err = 'Cannot load gt data from sequence %s, because there is not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + if time_data.shape[1] >= 8: + raw_data['gt_conf'][t] = np.atleast_1d(time_data[:, 6]).astype(float) + raw_data['visibility'][t] = np.atleast_1d(time_data[:, 8]).astype(float) + raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int) + else: + if not is_gt: + raw_data['classes'][t] = np.ones_like(raw_data['ids'][t]) + else: + raise TrackEvalException( + 'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % ( + seq, t)) + if is_gt: + gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6]) + else: + raw_data['dets'][t] = np.empty((0, 4)) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + gt_extras_dict = {'zero_marked': np.empty(0)} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.empty(0) + if is_gt: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + MOT Challenge: + In MOT Challenge, the 4 preproc steps are as follow: + 1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc. + 2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor + objects are removed. + 3) There is no crowd ignore regions. + 4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + # 'static': 2, 'ignore': 3, 'person_on_vehicle': + + distractor_class_names = ['static', 'ignore', 'person_on_vehicle'] + + distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names] + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', + 'similarity_scores', 'gt_visibility'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Get all data + gt_ids = raw_data['gt_ids'][t] + gt_dets = raw_data['gt_dets'][t] + gt_classes = raw_data['gt_classes'][t] + gt_visibility = raw_data['visibility'][t] + gt_conf = raw_data['gt_conf'][t] + + gt_zero_marked = raw_data['gt_extras'][t]['zero_marked'] + + tracker_ids = raw_data['tracker_ids'][t] + tracker_dets = raw_data['tracker_dets'][t] + tracker_classes = raw_data['tracker_classes'][t] + tracker_confidences = raw_data['tracker_confidences'][t] + similarity_scores = raw_data['similarity_scores'][t] + + # Evaluation is ONLY valid for pedestrian class + if len(tracker_classes) > 0 and np.max(tracker_classes) > 1: + raise TrackEvalException( + 'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at ' + 'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t)) + + # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets + # which are labeled as belonging to a distractor class. + to_remove_tracker = np.array([], np.int) + if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + + # Check all classes are valid: + invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers) + if len(invalid_classes) > 0: + print(' '.join([str(x) for x in invalid_classes])) + raise(TrackEvalException('Attempting to evaluate using invalid gt classes. ' + 'This warning only triggers if preprocessing is performed, ' + 'e.g. not for MOT15 or where prepropressing is explicitly disabled. ' + 'Please either check your gt data, or disable preprocessing. ' + 'The following invalid classes were found in timestep ' + str(t) + ': ' + + ' '.join([str(x) for x in invalid_classes]))) + + matching_scores = similarity_scores.copy() + + matching_scores[matching_scores < 0.4 - np.finfo('float').eps] = 0 + + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + is_distractor_class = np.logical_not(np.isin(gt_classes[match_rows], cls_id)) + if self.benchmark == 'HT': + is_invisible_class = gt_visibility[match_rows] < np.finfo('float').eps + low_conf_class = gt_conf[match_rows] < np.finfo('float').eps + are_distractors = np.logical_or(is_invisible_class, is_distractor_class, low_conf_class) + to_remove_tracker = match_cols[are_distractors] + else: + to_remove_tracker = match_cols[is_distractor_class] + + # Apply preprocessing to remove all unwanted tracker dets. + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian + if self.do_preproc and self.benchmark == 'HT': + gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \ + (np.equal(gt_classes, cls_id)) & \ + (gt_visibility > 0.) & \ + (gt_conf > 0.) + + else: + # There are no classes for MOT15 + gt_to_keep_mask = np.not_equal(gt_zero_marked, 0) + data['gt_ids'][t] = gt_ids[gt_to_keep_mask] + data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :] + data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask] + data['gt_visibility'][t] = gt_visibility # No mask! + + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/kitti_2d_box.py b/test/yolov7-tracker/tracker/trackeval/datasets/kitti_2d_box.py new file mode 100644 index 0000000..c582c43 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/kitti_2d_box.py @@ -0,0 +1,389 @@ + +import os +import csv +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from ..utils import TrackEvalException +from .. import _timing + + +class Kitti2DBox(_BaseDataset): + """Dataset class for KITTI 2D bounding box tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/kitti/kitti_2d_box_train'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/kitti/kitti_2d_box_train/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['car', 'pedestrian'], # Valid: ['car', 'pedestrian'] + 'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val', 'training_minus_val', 'test' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + self.max_occlusion = 2 + self.max_truncation = 0 + self.min_height = 25 + + # Get classes to eval + self.valid_classes = ['car', 'pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes [car, pedestrian] are valid.') + self.class_name_to_class_id = {'car': 1, 'van': 2, 'truck': 3, 'pedestrian': 4, 'person': 5, # person sitting + 'cyclist': 6, 'tram': 7, 'misc': 8, 'dontcare': 9, 'car_2': 1} + + # Get sequences to eval and check gt files exist + self.seq_list = [] + self.seq_lengths = {} + seqmap_name = 'evaluate_tracking.seqmap.' + self.config['SPLIT_TO_EVAL'] + seqmap_file = os.path.join(self.gt_fol, seqmap_name) + if not os.path.isfile(seqmap_file): + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + dialect = csv.Sniffer().sniff(fp.read(1024)) + fp.seek(0) + reader = csv.reader(fp, dialect) + for row in reader: + if len(row) >= 4: + seq = row[0] + self.seq_list.append(seq) + self.seq_lengths[seq] = int(row[3]) + if not self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'label_02', seq + '.txt') + if not os.path.isfile(curr_file): + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the kitti 2D box format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det). + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = os.path.join(self.gt_fol, 'label_02', seq + '.txt') + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Ignore regions + if is_gt: + crowd_ignore_filter = {2: ['dontcare']} + else: + crowd_ignore_filter = None + + # Valid classes + valid_filter = {2: [x for x in self.class_list]} + if is_gt: + if 'car' in self.class_list: + valid_filter[2].append('van') + if 'pedestrian' in self.class_list: + valid_filter[2] += ['person'] + + # Convert kitti class strings to class ids + convert_filter = {2: self.class_name_to_class_id} + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, time_col=0, id_col=1, remove_negative_ids=True, + valid_filter=valid_filter, + crowd_ignore_filter=crowd_ignore_filter, + convert_filter=convert_filter, + is_zipped=self.data_is_zipped, zip_file=zip_file) + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions', 'gt_extras'] + else: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str(t) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t) + if time_key in read_data.keys(): + time_data = np.asarray(read_data[time_key], dtype=np.float) + raw_data['dets'][t] = np.atleast_2d(time_data[:, 6:10]) + raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int) + raw_data['classes'][t] = np.atleast_1d(time_data[:, 2]).astype(int) + if is_gt: + gt_extras_dict = {'truncation': np.atleast_1d(time_data[:, 3].astype(int)), + 'occlusion': np.atleast_1d(time_data[:, 4].astype(int))} + raw_data['gt_extras'][t] = gt_extras_dict + else: + if time_data.shape[1] > 17: + raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 17]) + else: + raw_data['tracker_confidences'][t] = np.ones(time_data.shape[0]) + else: + raw_data['dets'][t] = np.empty((0, 4)) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + gt_extras_dict = {'truncation': np.empty(0), + 'occlusion': np.empty(0)} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.empty(0) + if is_gt: + if time_key in ignore_data.keys(): + time_ignore = np.asarray(ignore_data[time_key], dtype=np.float) + raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d(time_ignore[:, 6:10]) + else: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + KITTI: + In KITTI, the 4 preproc steps are as follow: + 1) There are two classes (pedestrian and car) which are evaluated separately. + 2) For the pedestrian class, the 'person' class is distractor objects (people sitting). + For the car class, the 'van' class are distractor objects. + GT boxes marked as having occlusion level > 2 or truncation level > 0 are also treated as + distractors. + 3) Crowd ignore regions are used to remove unmatched detections. Also unmatched detections with + height <= 25 pixels are removed. + 4) Distractor gt dets (including truncated and occluded) are removed. + """ + if cls == 'pedestrian': + distractor_classes = [self.class_name_to_class_id['person']] + elif cls == 'car': + distractor_classes = [self.class_name_to_class_id['van']] + else: + raise (TrackEvalException('Class %s is not evaluatable' % cls)) + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls + distractor classes) + gt_class_mask = np.sum([raw_data['gt_classes'][t] == c for c in [cls_id] + distractor_classes], axis=0) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + gt_classes = raw_data['gt_classes'][t][gt_class_mask] + gt_occlusion = raw_data['gt_extras'][t]['occlusion'][gt_class_mask] + gt_truncation = raw_data['gt_extras'][t]['truncation'][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets + # which are labeled as truncated, occluded, or belonging to a distractor class. + to_remove_matched = np.array([], np.int) + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes) + is_occluded_or_truncated = np.logical_or( + gt_occlusion[match_rows] > self.max_occlusion + np.finfo('float').eps, + gt_truncation[match_rows] > self.max_truncation + np.finfo('float').eps) + to_remove_matched = np.logical_or(is_distractor_class, is_occluded_or_truncated) + to_remove_matched = match_cols[to_remove_matched] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + # For unmatched tracker dets, also remove those smaller than a minimum height. + unmatched_tracker_dets = tracker_dets[unmatched_indices, :] + unmatched_heights = unmatched_tracker_dets[:, 3] - unmatched_tracker_dets[:, 1] + is_too_small = unmatched_heights <= self.min_height + np.finfo('float').eps + + # For unmatched tracker dets, also remove those that are greater than 50% within a crowd ignore region. + crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t] + intersection_with_ignore_region = self._calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions, + box_format='x0y0x1y1', do_ioa=True) + is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1) + + # Apply preprocessing to remove all unwanted tracker dets. + to_remove_unmatched = unmatched_indices[np.logical_or(is_too_small, is_within_crowd_ignore_region)] + to_remove_tracker = np.concatenate((to_remove_matched, to_remove_unmatched), axis=0) + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Also remove gt dets that were only useful for preprocessing and are not needed for evaluation. + # These are those that are occluded, truncated and from distractor objects. + gt_to_keep_mask = (np.less_equal(gt_occlusion, self.max_occlusion)) & \ + (np.less_equal(gt_truncation, self.max_truncation)) & \ + (np.equal(gt_classes, cls_id)) + data['gt_ids'][t] = gt_ids[gt_to_keep_mask] + data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :] + data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask] + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='x0y0x1y1') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/kitti_mots.py b/test/yolov7-tracker/tracker/trackeval/datasets/kitti_mots.py new file mode 100644 index 0000000..9e04d3c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/kitti_mots.py @@ -0,0 +1,426 @@ +import os +import csv +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + + +class KittiMOTS(_BaseDataset): + """Dataset class for KITTI MOTS tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/kitti/kitti_mots_val'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/kitti/kitti_mots_val'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['car', 'pedestrian'], # Valid: ['car', 'pedestrian'] + 'SPLIT_TO_EVAL': 'val', # Valid: 'training', 'val' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/split_to_eval.seqmap) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/label_02/{seq}.txt', # format of gt localization + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.split_to_eval = self.config['SPLIT_TO_EVAL'] + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['car', 'pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. ' + 'Only classes [car, pedestrian] are valid.') + self.class_name_to_class_id = {'car': '1', 'pedestrian': '2', 'ignore': '10'} + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + seqmap_name = 'evaluate_mots.seqmap.' + self.config['SPLIT_TO_EVAL'] + + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], seqmap_name) + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], seqmap_name) + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, _ in enumerate(reader): + dialect = csv.Sniffer().sniff(fp.read(1024)) + fp.seek(0) + reader = csv.reader(fp, dialect) + for row in reader: + if len(row) >= 4: + seq = "%04d" % int(row[0]) + seq_list.append(seq) + seq_lengths[seq] = int(row[3]) + 1 + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the KITTI MOTS format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [gt_ignore_region]: list (for each timestep) of masks for the ignore regions + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Ignore regions + if is_gt: + crowd_ignore_filter = {2: ['10']} + else: + crowd_ignore_filter = None + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, crowd_ignore_filter=crowd_ignore_filter, + is_zipped=self.data_is_zipped, zip_file=zip_file, + force_delimiters=' ') + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_ignore_region'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str(t) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t) + # list to collect all masks of a timestep to check for overlapping areas + all_masks = [] + if time_key in read_data.keys(): + try: + raw_data['dets'][t] = [{'size': [int(region[3]), int(region[4])], + 'counts': region[5].encode(encoding='UTF-8')} + for region in read_data[time_key]] + raw_data['ids'][t] = np.atleast_1d([region[1] for region in read_data[time_key]]).astype(int) + raw_data['classes'][t] = np.atleast_1d([region[2] for region in read_data[time_key]]).astype(int) + all_masks += raw_data['dets'][t] + except IndexError: + self._raise_index_error(is_gt, tracker, seq) + except ValueError: + self._raise_value_error(is_gt, tracker, seq) + else: + raw_data['dets'][t] = [] + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + if time_key in ignore_data.keys(): + try: + time_ignore = [{'size': [int(region[3]), int(region[4])], + 'counts': region[5].encode(encoding='UTF-8')} + for region in ignore_data[time_key]] + raw_data['gt_ignore_region'][t] = mask_utils.merge([mask for mask in time_ignore], + intersect=False) + all_masks += [raw_data['gt_ignore_region'][t]] + except IndexError: + self._raise_index_error(is_gt, tracker, seq) + except ValueError: + self._raise_value_error(is_gt, tracker, seq) + else: + raw_data['gt_ignore_region'][t] = mask_utils.merge([], intersect=False) + + # check for overlapping masks + if all_masks: + masks_merged = all_masks[0] + for mask in all_masks[1:]: + if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0: + raise TrackEvalException( + 'Tracker has overlapping masks. Tracker: ' + tracker + ' Seq: ' + seq + ' Timestep: ' + str( + t)) + masks_merged = mask_utils.merge([masks_merged, mask], intersect=False) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data["num_timesteps"] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + KITTI MOTS: + In KITTI MOTS, the 4 preproc steps are as follow: + 1) There are two classes (car and pedestrian) which are evaluated separately. + 2) There are no ground truth detections marked as to be removed/distractor classes. + Therefore also no matched tracker detections are removed. + 3) Ignore regions are used to remove unmatched detections (at least 50% overlap with ignore region). + 4) There are no ground truth detections (e.g. those of distractor classes) to be removed. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + cls_id = int(self.class_name_to_class_id[cls]) + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if + tracker_class_mask[ind]] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm) + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = -10000 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region. + unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if i in unmatched_indices] + ignore_region = raw_data['gt_ignore_region'][t] + intersection_with_ignore_region = self._calculate_mask_ious(unmatched_tracker_dets, [ignore_region], + is_encoded=True, do_ioa=True) + is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1) + + # Apply preprocessing to remove unwanted tracker dets. + to_remove_tracker = unmatched_indices[is_within_ignore_region] + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Keep all ground truth detections + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + data['cls'] = cls + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores + + @staticmethod + def _raise_index_error(is_gt, tracker, seq): + """ + Auxiliary method to raise an evaluation error in case of an index error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + err = 'Cannot load gt data from sequence %s, because there are not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there are not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + + @staticmethod + def _raise_value_error(is_gt, tracker, seq): + """ + Auxiliary method to raise an evaluation error in case of an value error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + raise TrackEvalException( + 'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Tracking data from tracker %s, sequence %s cannot be converted to the right format. ' + 'Is data corrupted?' % (tracker, seq)) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/mot_challenge_2d_box.py b/test/yolov7-tracker/tracker/trackeval/datasets/mot_challenge_2d_box.py new file mode 100644 index 0000000..68aac51 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/mot_challenge_2d_box.py @@ -0,0 +1,437 @@ +import os +import csv +import configparser +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + + +class MotChallenge2DBox(_BaseDataset): + """Dataset class for MOT Challenge 2D bounding box tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrian'], # Valid: ['pedestrian'] + 'BENCHMARK': 'MOT17', # Valid: 'MOT17', 'MOT16', 'MOT20', 'MOT15' + 'SPLIT_TO_EVAL': 'train', # Valid: 'train', 'test', 'all' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'DO_PREPROC': True, # Whether to perform preprocessing (never done for MOT15) + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt', # '{gt_folder}/{seq}/gt/gt.txt' + 'SKIP_SPLIT_FOL': False, # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in + # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/ + # If True, then the middle 'benchmark-split' folder is skipped for both. + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + + self.benchmark = self.config['BENCHMARK'] + gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL'] + self.gt_set = gt_set + if not self.config['SKIP_SPLIT_FOL']: + split_fol = gt_set + else: + split_fol = '' + self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol) + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol) + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + self.do_preproc = self.config['DO_PREPROC'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.') + self.class_name_to_class_id = {'pedestrian': 1, 'person_on_vehicle': 2, 'car': 3, 'bicycle': 4, 'motorbike': 5, + 'non_mot_vehicle': 6, 'static_person': 7, 'distractor': 8, 'occluder': 9, + 'occluder_on_ground': 10, 'occluder_full': 11, 'reflection': 12, 'crowd': 13} + self.valid_class_numbers = list(self.class_name_to_class_id.values()) + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + + # If sequence length is 'None' tries to read sequence length from .ini files. + for seq, seq_length in seq_lengths.items(): + if seq_length is None: + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if i == 0 or row[0] == '': + continue + seq = row[0] + seq_list.append(seq) + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the MOT Challenge 2D box format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det). + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file) + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions', 'gt_extras'] + else: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str( t+ 1) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t+1) + if time_key in read_data.keys(): + try: + time_data = np.asarray(read_data[time_key], dtype=np.float) + except ValueError: + if is_gt: + raise TrackEvalException( + 'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % ( + tracker, seq)) + try: + raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6]) + raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int) + except IndexError: + if is_gt: + err = 'Cannot load gt data from sequence %s, because there is not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + if time_data.shape[1] >= 8: + raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int) + else: + if not is_gt: + raw_data['classes'][t] = np.ones_like(raw_data['ids'][t]) + else: + raise TrackEvalException( + 'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % ( + seq, t)) + if is_gt: + gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6]) + else: + raw_data['dets'][t] = np.empty((0, 4)) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + gt_extras_dict = {'zero_marked': np.empty(0)} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.empty(0) + if is_gt: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + MOT Challenge: + In MOT Challenge, the 4 preproc steps are as follow: + 1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc. + 2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor + objects are removed. + 3) There is no crowd ignore regions. + 4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection'] + if self.benchmark == 'MOT20': + distractor_class_names.append('non_mot_vehicle') + distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names] + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Get all data + gt_ids = raw_data['gt_ids'][t] + gt_dets = raw_data['gt_dets'][t] + gt_classes = raw_data['gt_classes'][t] + gt_zero_marked = raw_data['gt_extras'][t]['zero_marked'] + + tracker_ids = raw_data['tracker_ids'][t] + tracker_dets = raw_data['tracker_dets'][t] + tracker_classes = raw_data['tracker_classes'][t] + tracker_confidences = raw_data['tracker_confidences'][t] + similarity_scores = raw_data['similarity_scores'][t] + + # Evaluation is ONLY valid for pedestrian class + if len(tracker_classes) > 0 and np.max(tracker_classes) > 1: + raise TrackEvalException( + 'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at ' + 'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t)) + + # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets + # which are labeled as belonging to a distractor class. + to_remove_tracker = np.array([], np.int) + if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + + # Check all classes are valid: + invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers) + if len(invalid_classes) > 0: + print(' '.join([str(x) for x in invalid_classes])) + raise(TrackEvalException('Attempting to evaluate using invalid gt classes. ' + 'This warning only triggers if preprocessing is performed, ' + 'e.g. not for MOT15 or where prepropressing is explicitly disabled. ' + 'Please either check your gt data, or disable preprocessing. ' + 'The following invalid classes were found in timestep ' + str(t) + ': ' + + ' '.join([str(x) for x in invalid_classes]))) + + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes) + to_remove_tracker = match_cols[is_distractor_class] + + # Apply preprocessing to remove all unwanted tracker dets. + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian + # class (not applicable for MOT15) + if self.do_preproc and self.benchmark != 'MOT15': + gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \ + (np.equal(gt_classes, cls_id)) + else: + # There are no classes for MOT15 + gt_to_keep_mask = np.not_equal(gt_zero_marked, 0) + data['gt_ids'][t] = gt_ids[gt_to_keep_mask] + data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :] + data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask] + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/mots_challenge.py b/test/yolov7-tracker/tracker/trackeval/datasets/mots_challenge.py new file mode 100644 index 0000000..191b438 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/mots_challenge.py @@ -0,0 +1,446 @@ +import os +import csv +import configparser +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + + +class MOTSChallenge(_BaseDataset): + """Dataset class for MOTS Challenge tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrian'], # Valid: ['pedestrian'] + 'SPLIT_TO_EVAL': 'train', # Valid: 'train', 'test' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/MOTS-split_to_eval) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt', # '{gt_folder}/{seq}/gt/gt.txt' + 'SKIP_SPLIT_FOL': False, # If False, data is in GT_FOLDER/MOTS-SPLIT_TO_EVAL/ and in + # TRACKERS_FOLDER/MOTS-SPLIT_TO_EVAL/tracker/ + # If True, then the middle 'MOTS-split' folder is skipped for both. + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + + self.benchmark = 'MOTS' + self.gt_set = self.benchmark + '-' + self.config['SPLIT_TO_EVAL'] + if not self.config['SKIP_SPLIT_FOL']: + split_fol = self.gt_set + else: + split_fol = '' + self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol) + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol) + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.') + self.class_name_to_class_id = {'pedestrian': '2', 'ignore': '10'} + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + + # If sequence length is 'None' tries to read sequence length from .ini files. + for seq, seq_length in seq_lengths.items(): + if seq_length is None: + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if i == 0 or row[0] == '': + continue + seq = row[0] + seq_list.append(seq) + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the MOTS Challenge format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [gt_ignore_region]: list (for each timestep) of masks for the ignore regions + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Ignore regions + if is_gt: + crowd_ignore_filter = {2: ['10']} + else: + crowd_ignore_filter = None + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, crowd_ignore_filter=crowd_ignore_filter, + is_zipped=self.data_is_zipped, zip_file=zip_file, + force_delimiters=' ') + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_ignore_region'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str(t + 1) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t+1) + # list to collect all masks of a timestep to check for overlapping areas + all_masks = [] + if time_key in read_data.keys(): + try: + raw_data['dets'][t] = [{'size': [int(region[3]), int(region[4])], + 'counts': region[5].encode(encoding='UTF-8')} + for region in read_data[time_key]] + raw_data['ids'][t] = np.atleast_1d([region[1] for region in read_data[time_key]]).astype(int) + raw_data['classes'][t] = np.atleast_1d([region[2] for region in read_data[time_key]]).astype(int) + all_masks += raw_data['dets'][t] + except IndexError: + self._raise_index_error(is_gt, tracker, seq) + except ValueError: + self._raise_value_error(is_gt, tracker, seq) + else: + raw_data['dets'][t] = [] + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + if time_key in ignore_data.keys(): + try: + time_ignore = [{'size': [int(region[3]), int(region[4])], + 'counts': region[5].encode(encoding='UTF-8')} + for region in ignore_data[time_key]] + raw_data['gt_ignore_region'][t] = mask_utils.merge([mask for mask in time_ignore], + intersect=False) + all_masks += [raw_data['gt_ignore_region'][t]] + except IndexError: + self._raise_index_error(is_gt, tracker, seq) + except ValueError: + self._raise_value_error(is_gt, tracker, seq) + else: + raw_data['gt_ignore_region'][t] = mask_utils.merge([], intersect=False) + + # check for overlapping masks + if all_masks: + masks_merged = all_masks[0] + for mask in all_masks[1:]: + if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0: + raise TrackEvalException( + 'Tracker has overlapping masks. Tracker: ' + tracker + ' Seq: ' + seq + ' Timestep: ' + str( + t)) + masks_merged = mask_utils.merge([masks_merged, mask], intersect=False) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detection masks. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + MOTS Challenge: + In MOTS Challenge, the 4 preproc steps are as follow: + 1) There is only one class (pedestrians) to be evaluated. + 2) There are no ground truth detections marked as to be removed/distractor classes. + Therefore also no matched tracker detections are removed. + 3) Ignore regions are used to remove unmatched detections (at least 50% overlap with ignore region). + 4) There are no ground truth detections (e.g. those of distractor classes) to be removed. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + cls_id = int(self.class_name_to_class_id[cls]) + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if + tracker_class_mask[ind]] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm) + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = -10000 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + # For unmatched tracker dets, remove those that are greater than 50% within a crowd ignore region. + unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if i in unmatched_indices] + ignore_region = raw_data['gt_ignore_region'][t] + intersection_with_ignore_region = self._calculate_mask_ious(unmatched_tracker_dets, [ignore_region], + is_encoded=True, do_ioa=True) + is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1) + + # Apply preprocessing to remove unwanted tracker dets. + to_remove_tracker = unmatched_indices[is_within_ignore_region] + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Keep all ground truth detections + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores + + @staticmethod + def _raise_index_error(is_gt, tracker, seq): + """ + Auxiliary method to raise an evaluation error in case of an index error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + err = 'Cannot load gt data from sequence %s, because there are not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there are not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + + @staticmethod + def _raise_value_error(is_gt, tracker, seq): + """ + Auxiliary method to raise an evaluation error in case of an value error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + raise TrackEvalException( + 'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Tracking data from tracker %s, sequence %s cannot be converted to the right format. ' + 'Is data corrupted?' % (tracker, seq)) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/person_path_22.py b/test/yolov7-tracker/tracker/trackeval/datasets/person_path_22.py new file mode 100644 index 0000000..177954a --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/person_path_22.py @@ -0,0 +1,452 @@ +import os +import csv +import configparser +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + +class PersonPath22(_BaseDataset): + """Dataset class for MOT Challenge 2D bounding box tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/person_path_22/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/person_path_22/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrian'], # Valid: ['pedestrian'] + 'BENCHMARK': 'person_path_22', # Valid: 'person_path_22' + 'SPLIT_TO_EVAL': 'test', # Valid: 'train', 'test', 'all' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'DO_PREPROC': True, # Whether to perform preprocessing + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt', # '{gt_folder}/{seq}/gt/gt.txt' + 'SKIP_SPLIT_FOL': False, # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in + # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/ + # If True, then the middle 'benchmark-split' folder is skipped for both. + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + + self.benchmark = self.config['BENCHMARK'] + gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL'] + self.gt_set = gt_set + if not self.config['SKIP_SPLIT_FOL']: + split_fol = gt_set + else: + split_fol = '' + self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol) + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol) + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + self.do_preproc = self.config['DO_PREPROC'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.') + self.class_name_to_class_id = {'pedestrian': 1, 'person_on_vehicle': 2, 'car': 3, 'bicycle': 4, 'motorbike': 5, + 'non_mot_vehicle': 6, 'static_person': 7, 'distractor': 8, 'occluder': 9, + 'occluder_on_ground': 10, 'occluder_full': 11, 'reflection': 12, 'crowd': 13} + self.valid_class_numbers = list(self.class_name_to_class_id.values()) + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + + # If sequence length is 'None' tries to read sequence length from .ini files. + for seq, seq_length in seq_lengths.items(): + if seq_length is None: + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if i == 0 or row[0] == '': + continue + seq = row[0] + seq_list.append(seq) + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the MOT Challenge 2D box format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det). + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Ignore regions + if is_gt: + crowd_ignore_filter = {7: ['13']} + else: + crowd_ignore_filter = None + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file, crowd_ignore_filter=crowd_ignore_filter) + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions', 'gt_extras'] + else: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str( t+ 1) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t+1) + if time_key in read_data.keys(): + try: + time_data = np.asarray(read_data[time_key], dtype=np.float) + except ValueError: + if is_gt: + raise TrackEvalException( + 'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % ( + tracker, seq)) + try: + raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6]) + raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int) + except IndexError: + if is_gt: + err = 'Cannot load gt data from sequence %s, because there is not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + if time_data.shape[1] >= 8: + raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int) + else: + if not is_gt: + raw_data['classes'][t] = np.ones_like(raw_data['ids'][t]) + else: + raise TrackEvalException( + 'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % ( + seq, t)) + if is_gt: + gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6]) + else: + raw_data['dets'][t] = np.empty((0, 4)) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + gt_extras_dict = {'zero_marked': np.empty(0)} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.empty(0) + if is_gt: + if time_key in ignore_data.keys(): + time_ignore = np.asarray(ignore_data[time_key], dtype=np.float) + raw_data['gt_crowd_ignore_regions'][t] = np.atleast_2d(time_ignore[:, 2:6]) + else: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + MOT Challenge: + In MOT Challenge, the 4 preproc steps are as follow: + 1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc. + 2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor + objects are removed. + 3) There is no crowd ignore regions. + 4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection'] + if self.benchmark == 'MOT20': + distractor_class_names.append('non_mot_vehicle') + distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names] + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Get all data + gt_ids = raw_data['gt_ids'][t] + gt_dets = raw_data['gt_dets'][t] + gt_classes = raw_data['gt_classes'][t] + gt_zero_marked = raw_data['gt_extras'][t]['zero_marked'] + + tracker_ids = raw_data['tracker_ids'][t] + tracker_dets = raw_data['tracker_dets'][t] + tracker_classes = raw_data['tracker_classes'][t] + tracker_confidences = raw_data['tracker_confidences'][t] + similarity_scores = raw_data['similarity_scores'][t] + crowd_ignore_regions = raw_data['gt_crowd_ignore_regions'][t] + + # Evaluation is ONLY valid for pedestrian class + if len(tracker_classes) > 0 and np.max(tracker_classes) > 1: + raise TrackEvalException( + 'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at ' + 'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t)) + + # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets + # which are labeled as belonging to a distractor class. + to_remove_tracker = np.array([], np.int) + if self.do_preproc and self.benchmark != 'MOT15' and (gt_ids.shape[0] > 0 or len(crowd_ignore_regions) > 0) and tracker_ids.shape[0] > 0: + + # Check all classes are valid: + invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers) + if len(invalid_classes) > 0: + print(' '.join([str(x) for x in invalid_classes])) + raise(TrackEvalException('Attempting to evaluate using invalid gt classes. ' + 'This warning only triggers if preprocessing is performed, ' + 'e.g. not for MOT15 or where prepropressing is explicitly disabled. ' + 'Please either check your gt data, or disable preprocessing. ' + 'The following invalid classes were found in timestep ' + str(t) + ': ' + + ' '.join([str(x) for x in invalid_classes]))) + + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes) + to_remove_tracker = match_cols[is_distractor_class] + + # remove bounding boxes that overlap with crowd ignore region. + intersection_with_ignore_region = self._calculate_box_ious(tracker_dets, crowd_ignore_regions, box_format='xywh', do_ioa=True) + is_within_crowd_ignore_region = np.any(intersection_with_ignore_region > 0.95 + np.finfo('float').eps, axis=1) + to_remove_tracker = np.unique(np.concatenate([to_remove_tracker, np.where(is_within_crowd_ignore_region)[0]])) + + # Apply preprocessing to remove all unwanted tracker dets. + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian + # class (not applicable for MOT15) + if self.do_preproc and self.benchmark != 'MOT15': + gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \ + (np.equal(gt_classes, cls_id)) + else: + # There are no classes for MOT15 + gt_to_keep_mask = np.not_equal(gt_zero_marked, 0) + data['gt_ids'][t] = gt_ids[gt_to_keep_mask] + data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :] + data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask] + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots.py b/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots.py new file mode 100644 index 0000000..d6a6d1e --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots.py @@ -0,0 +1,508 @@ + +import os +import csv +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from ..utils import TrackEvalException +from .. import _timing +from ..datasets.rob_mots_classmap import cls_id_to_name + + +class RobMOTS(_BaseDataset): + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/rob_mots'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/rob_mots'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'SUB_BENCHMARK': None, # REQUIRED. Sub-benchmark to eval. If None, then error. + # ['mots_challenge', 'kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'waymo', 'tao'] + 'CLASSES_TO_EVAL': None, # List of classes to eval. If None, then it does all COCO classes. + 'SPLIT_TO_EVAL': 'train', # valid: ['train', 'val', 'test'] + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'OUTPUT_SUB_FOLDER': 'results', # Output files are saved in OUTPUT_FOLDER/DATA_LOC_FORMAT/OUTPUT_SUB_FOLDER + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/DATA_LOC_FORMAT/TRACKER_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/dataset_subfolder/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use SEQMAP_FOLDER/BENCHMARK_SPLIT_TO_EVAL) + 'CLSMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/dataset_subfolder/clsmaps) + 'CLSMAP_FILE': None, # Directly specify seqmap file (if none use CLSMAP_FOLDER/BENCHMARK_SPLIT_TO_EVAL) + } + return default_config + + def __init__(self, config=None): + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config()) + + self.split = self.config['SPLIT_TO_EVAL'] + valid_benchmarks = ['mots_challenge', 'kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'waymo', 'tao'] + self.box_gt_benchmarks = ['waymo', 'tao'] + + self.sub_benchmark = self.config['SUB_BENCHMARK'] + if not self.sub_benchmark: + raise TrackEvalException('SUB_BENCHMARK config input is required (there is no default value)' + + ', '.join(valid_benchmarks) + ' are valid.') + if self.sub_benchmark not in valid_benchmarks: + raise TrackEvalException('Attempted to evaluate an invalid benchmark: ' + self.sub_benchmark + '. Only benchmarks ' + + ', '.join(valid_benchmarks) + ' are valid.') + + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], self.config['SPLIT_TO_EVAL']) + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = os.path.join(self.config['OUTPUT_SUB_FOLDER'], self.sub_benchmark) + + # Loops through all sub-benchmarks, and reads in seqmaps to info on all sequences to eval. + self._get_seq_info() + + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + valid_class_ids = np.atleast_1d(np.genfromtxt(os.path.join(self.gt_fol, self.split, self.sub_benchmark, + 'clsmap.txt'))) + valid_classes = [cls_id_to_name[int(x)] for x in valid_class_ids] + ['all'] + self.valid_class_ids = valid_class_ids + self.class_name_to_class_id = {cls_name: cls_id for cls_id, cls_name in cls_id_to_name.items()} + self.class_name_to_class_id['all'] = -1 + if not self.config['CLASSES_TO_EVAL']: + self.class_list = valid_classes + else: + self.class_list = [cls if cls in valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(valid_classes) + ' are valid.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data', seq + '.txt') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data.zip') + if not os.path.isfile(curr_file): + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, 'data.zip') + if not os.path.isfile(curr_file): + raise TrackEvalException('Tracker file not found: ' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, self.sub_benchmark, seq + + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + self.sub_benchmark + '/' + os.path.basename(curr_file)) + + def get_name(self): + return self.get_class_name() + '.' + self.sub_benchmark + + def _get_seq_info(self): + self.seq_list = [] + self.seq_lengths = {} + self.seq_sizes = {} + self.seq_ignore_class_ids = {} + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'seqmap.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.split + '.seqmap') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + dialect = csv.Sniffer().sniff(fp.readline(), delimiters=' ') + fp.seek(0) + reader = csv.reader(fp, dialect) + for i, row in enumerate(reader): + if len(row) >= 4: + # first col: sequence, second col: sequence length, third and fourth col: sequence height/width + # The rest of the columns list the 'sequence ignore class ids' which are classes not penalized as + # FPs for this sequence. + seq = row[0] + self.seq_list.append(seq) + self.seq_lengths[seq] = int(row[1]) + self.seq_sizes[seq] = (int(row[2]), int(row[3])) + self.seq_ignore_class_ids[seq] = [int(row[x]) for x in range(4, len(row))] + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the unified RobMOTS format. + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # import to reduce minimum requirements + from pycocotools import mask as mask_utils + + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, 'data.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = os.path.join(self.gt_fol, self.split, self.sub_benchmark, 'data', seq + '.txt') + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, self.sub_benchmark, seq + '.txt') + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file, + force_delimiters=' ') + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for t in range(num_timesteps): + time_key = str(t) + # list to collect all masks of a timestep to check for overlapping areas (for segmentation datasets) + all_valid_masks = [] + if time_key in read_data.keys(): + try: + raw_data['ids'][t] = np.atleast_1d([det[1] for det in read_data[time_key]]).astype(int) + raw_data['classes'][t] = np.atleast_1d([det[2] for det in read_data[time_key]]).astype(int) + if (not is_gt) or (self.sub_benchmark not in self.box_gt_benchmarks): + raw_data['dets'][t] = [{'size': [int(region[4]), int(region[5])], + 'counts': region[6].encode(encoding='UTF-8')} + for region in read_data[time_key]] + all_valid_masks += [mask for mask, cls in zip(raw_data['dets'][t], raw_data['classes'][t]) if + cls < 100] + else: + raw_data['dets'][t] = np.atleast_2d([det[4:8] for det in read_data[time_key]]).astype(float) + + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([det[3] for det + in read_data[time_key]]).astype(float) + except IndexError: + self._raise_index_error(is_gt, self.sub_benchmark, seq) + except ValueError: + self._raise_value_error(is_gt, self.sub_benchmark, seq) + # no detection in this timestep + else: + if (not is_gt) or (self.sub_benchmark not in self.box_gt_benchmarks): + raw_data['dets'][t] = [] + else: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.empty(0).astype(float) + + # check for overlapping masks + if all_valid_masks: + masks_merged = all_valid_masks[0] + for mask in all_valid_masks[1:]: + if mask_utils.area(mask_utils.merge([masks_merged, mask], intersect=True)) != 0.0: + err = 'Overlapping masks in frame %d' % t + raise TrackEvalException(err) + masks_merged = mask_utils.merge([masks_merged, mask], intersect=False) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['frame_size'] = self.seq_sizes[seq] + raw_data['seq'] = seq + return raw_data + + @staticmethod + def _raise_index_error(is_gt, sub_benchmark, seq): + """ + Auxiliary method to raise an evaluation error in case of an index error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + err = 'Cannot load gt data from sequence %s, because there are not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from benchmark %s, sequence %s, because there are not enough ' \ + 'columns in the data.' % (sub_benchmark, seq) + raise TrackEvalException(err) + + @staticmethod + def _raise_value_error(is_gt, sub_benchmark, seq): + """ + Auxiliary method to raise an evaluation error in case of an value error while reading files. + :param is_gt: whether gt or tracker data is read + :param tracker: the name of the tracker + :param seq: the name of the seq + :return: None + """ + if is_gt: + raise TrackEvalException( + 'GT data for sequence %s cannot be converted to the right format. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Tracking data from benchmark %s, sequence %s cannot be converted to the right format. ' + 'Is data corrupted?' % (sub_benchmark, seq)) + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + Preprocessing (preproc) occurs in 3 steps. + 1) Extract only detections relevant for the class to be evaluated. + 2) Match gt dets and tracker dets. Tracker dets that are to a gt det (TPs) are marked as not to be + removed. + 3) Remove unmatched tracker dets if they fall within an ignore region or are too small, or if that class + is marked as an ignore class for that sequence. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + Note that there is a special 'all' class, which evaluates all of the COCO classes together in a + 'class agnostic' fashion. + """ + # import to reduce minimum requirements + from pycocotools import mask as mask_utils + + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + cls_id = self.class_name_to_class_id[cls] + ignore_class_id = cls_id+100 + seq = raw_data['seq'] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class + if cls == 'all': + gt_class_mask = raw_data['gt_classes'][t] < 100 + # For waymo, combine predictions for [car, truck, bus, motorcycle] into car, because they are all annotated + # together as one 'vehicle' class. + elif self.sub_benchmark == 'waymo' and cls == 'car': + waymo_vehicle_classes = np.array([3, 4, 6, 8]) + gt_class_mask = np.isin(raw_data['gt_classes'][t], waymo_vehicle_classes) + else: + gt_class_mask = raw_data['gt_classes'][t] == cls_id + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + if cls == 'all': + ignore_regions_mask = raw_data['gt_classes'][t] >= 100 + else: + ignore_regions_mask = raw_data['gt_classes'][t] == ignore_class_id + ignore_regions_mask = np.logical_or(ignore_regions_mask, raw_data['gt_classes'][t] == 100) + if self.sub_benchmark in self.box_gt_benchmarks: + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + ignore_regions_box = raw_data['gt_dets'][t][ignore_regions_mask] + if len(ignore_regions_box) > 0: + ignore_regions_box[:, 2] = ignore_regions_box[:, 2] - ignore_regions_box[:, 0] + ignore_regions_box[:, 3] = ignore_regions_box[:, 3] - ignore_regions_box[:, 1] + ignore_regions = mask_utils.frPyObjects(ignore_regions_box, self.seq_sizes[seq][0], self.seq_sizes[seq][1]) + else: + ignore_regions = [] + else: + gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]] + ignore_regions = [raw_data['gt_dets'][t][ind] for ind in range(len(ignore_regions_mask)) if + ignore_regions_mask[ind]] + + if cls == 'all': + tracker_class_mask = np.ones_like(raw_data['tracker_classes'][t]) + else: + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if + tracker_class_mask[ind]] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + tracker_classes = raw_data['tracker_classes'][t][tracker_class_mask] + + # Only do preproc if there are ignore regions defined to remove + if tracker_ids.shape[0] > 0: + + # Match tracker and gt dets (with hungarian algorithm) + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + # match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + # For unmatched tracker dets remove those that are greater than 50% within an ignore region. + # unmatched_tracker_dets = tracker_dets[unmatched_indices, :] + # crowd_ignore_regions = raw_data['gt_ignore_regions'][t] + # intersection_with_ignore_region = self. \ + # _calculate_box_ious(unmatched_tracker_dets, crowd_ignore_regions, box_format='x0y0x1y1', + # do_ioa=True) + + + if cls_id in self.seq_ignore_class_ids[seq]: + # Remove unmatched detections for classes that are marked as 'ignore' for the whole sequence. + to_remove_tracker = unmatched_indices + else: + unmatched_tracker_dets = [tracker_dets[i] for i in range(len(tracker_dets)) if + i in unmatched_indices] + + # For unmatched tracker dets remove those that are too small. + tracker_boxes_t = mask_utils.toBbox(unmatched_tracker_dets) + unmatched_widths = tracker_boxes_t[:, 2] + unmatched_heights = tracker_boxes_t[:, 3] + unmatched_size = np.maximum(unmatched_heights, unmatched_widths) + min_size = np.min(self.seq_sizes[seq])/8 + is_too_small = unmatched_size <= min_size + np.finfo('float').eps + + # For unmatched tracker dets remove those that are greater than 50% within an ignore region. + if ignore_regions: + ignore_region_merged = ignore_regions[0] + for mask in ignore_regions[1:]: + ignore_region_merged = mask_utils.merge([ignore_region_merged, mask], intersect=False) + intersection_with_ignore_region = self. \ + _calculate_mask_ious(unmatched_tracker_dets, [ignore_region_merged], is_encoded=True, do_ioa=True) + is_within_ignore_region = np.any(intersection_with_ignore_region > 0.5 + np.finfo('float').eps, axis=1) + to_remove_tracker = unmatched_indices[np.logical_or(is_too_small, is_within_ignore_region)] + else: + to_remove_tracker = unmatched_indices[is_too_small] + + # For the special 'all' class, you need to remove unmatched detections from all ignore classes and + # non-evaluated classes. + if cls == 'all': + unmatched_tracker_classes = [tracker_classes[i] for i in range(len(tracker_classes)) if + i in unmatched_indices] + is_ignore_class = np.isin(unmatched_tracker_classes, self.seq_ignore_class_ids[seq]) + is_not_evaled_class = np.logical_not(np.isin(unmatched_tracker_classes, self.valid_class_ids)) + to_remove_all = unmatched_indices[np.logical_or(is_ignore_class, is_not_evaled_class)] + to_remove_tracker = np.concatenate([to_remove_tracker, to_remove_all], axis=0) + else: + to_remove_tracker = np.array([], dtype=np.int) + + # remove all unwanted tracker detections + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # keep all ground truth detections + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + data['frame_size'] = raw_data['frame_size'] + + # Ensure that ids are unique per timestep. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + if self.sub_benchmark in self.box_gt_benchmarks: + # Convert tracker masks to bboxes (for benchmarks with only bbox ground-truth), + # and then convert to x0y0x1y1 format. + tracker_boxes_t = mask_utils.toBbox(tracker_dets_t) + tracker_boxes_t[:, 2] = tracker_boxes_t[:, 0] + tracker_boxes_t[:, 2] + tracker_boxes_t[:, 3] = tracker_boxes_t[:, 1] + tracker_boxes_t[:, 3] + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_boxes_t, box_format='x0y0x1y1') + else: + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots_classmap.py b/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots_classmap.py new file mode 100644 index 0000000..1b3644d --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/rob_mots_classmap.py @@ -0,0 +1,81 @@ +cls_id_to_name = { + 1: 'person', + 2: 'bicycle', + 3: 'car', + 4: 'motorcycle', + 5: 'airplane', + 6: 'bus', + 7: 'train', + 8: 'truck', + 9: 'boat', + 10: 'traffic light', + 11: 'fire hydrant', + 12: 'stop sign', + 13: 'parking meter', + 14: 'bench', + 15: 'bird', + 16: 'cat', + 17: 'dog', + 18: 'horse', + 19: 'sheep', + 20: 'cow', + 21: 'elephant', + 22: 'bear', + 23: 'zebra', + 24: 'giraffe', + 25: 'backpack', + 26: 'umbrella', + 27: 'handbag', + 28: 'tie', + 29: 'suitcase', + 30: 'frisbee', + 31: 'skis', + 32: 'snowboard', + 33: 'sports ball', + 34: 'kite', + 35: 'baseball bat', + 36: 'baseball glove', + 37: 'skateboard', + 38: 'surfboard', + 39: 'tennis racket', + 40: 'bottle', + 41: 'wine glass', + 42: 'cup', + 43: 'fork', + 44: 'knife', + 45: 'spoon', + 46: 'bowl', + 47: 'banana', + 48: 'apple', + 49: 'sandwich', + 50: 'orange', + 51: 'broccoli', + 52: 'carrot', + 53: 'hot dog', + 54: 'pizza', + 55: 'donut', + 56: 'cake', + 57: 'chair', + 58: 'couch', + 59: 'potted plant', + 60: 'bed', + 61: 'dining table', + 62: 'toilet', + 63: 'tv', + 64: 'laptop', + 65: 'mouse', + 66: 'remote', + 67: 'keyboard', + 68: 'cell phone', + 69: 'microwave', + 70: 'oven', + 71: 'toaster', + 72: 'sink', + 73: 'refrigerator', + 74: 'book', + 75: 'clock', + 76: 'vase', + 77: 'scissors', + 78: 'teddy bear', + 79: 'hair drier', + 80: 'toothbrush'} \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/run_rob_mots.py b/test/yolov7-tracker/tracker/trackeval/datasets/run_rob_mots.py new file mode 100644 index 0000000..87c1412 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/run_rob_mots.py @@ -0,0 +1,113 @@ + +# python3 scripts\run_rob_mots.py --ROBMOTS_SPLIT val --TRACKERS_TO_EVAL tracker_name (e.g. STP) --USE_PARALLEL True --NUM_PARALLEL_CORES 4 + +import sys +import os +import csv +import numpy as np +from multiprocessing import freeze_support + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import trackeval # noqa: E402 +from trackeval import utils +code_path = utils.get_code_path() + +if __name__ == '__main__': + freeze_support() + + script_config = { + 'ROBMOTS_SPLIT': 'train', # 'train', # valid: 'train', 'val', 'test', 'test_live', 'test_post', 'test_all' + 'BENCHMARKS': ['kitti_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao'], # 'bdd_mots' coming soon + 'GT_FOLDER': os.path.join(code_path, 'data/gt/rob_mots'), + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/rob_mots'), + } + + default_eval_config = trackeval.Evaluator.get_default_eval_config() + default_eval_config['PRINT_ONLY_COMBINED'] = True + default_eval_config['DISPLAY_LESS_PROGRESS'] = True + default_dataset_config = trackeval.datasets.RobMOTS.get_default_dataset_config() + config = {**default_eval_config, **default_dataset_config, **script_config} + + # Command line interface: + config = utils.update_config(config) + + if config['ROBMOTS_SPLIT'] == 'val': + config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', + 'tao', 'mots_challenge'] + config['SPLIT_TO_EVAL'] = 'val' + elif config['ROBMOTS_SPLIT'] == 'test' or config['SPLIT_TO_EVAL'] == 'test_live': + config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao'] + config['SPLIT_TO_EVAL'] = 'test' + elif config['ROBMOTS_SPLIT'] == 'test_post': + config['BENCHMARKS'] = ['mots_challenge', 'waymo'] + config['SPLIT_TO_EVAL'] = 'test' + elif config['ROBMOTS_SPLIT'] == 'test_all': + config['BENCHMARKS'] = ['kitti_mots', 'bdd_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', + 'tao', 'mots_challenge', 'waymo'] + config['SPLIT_TO_EVAL'] = 'test' + elif config['ROBMOTS_SPLIT'] == 'train': + config['BENCHMARKS'] = ['kitti_mots', 'davis_unsupervised', 'youtube_vis', 'ovis', 'tao'] # 'bdd_mots' coming soon + config['SPLIT_TO_EVAL'] = 'train' + + metrics_config = {'METRICS': ['HOTA']} + # metrics_config = {'METRICS': ['HOTA', 'CLEAR', 'Identity']} + eval_config = {k: v for k, v in config.items() if k in config.keys()} + dataset_config = {k: v for k, v in config.items() if k in config.keys()} + + # Run code + dataset_list = [] + for bench in config['BENCHMARKS']: + dataset_config['SUB_BENCHMARK'] = bench + dataset_list.append(trackeval.datasets.RobMOTS(dataset_config)) + evaluator = trackeval.Evaluator(eval_config) + metrics_list = [] + for metric in [trackeval.metrics.HOTA, trackeval.metrics.CLEAR, trackeval.metrics.Identity]: + if metric.get_name() in metrics_config['METRICS']: + metrics_list.append(metric()) + if len(metrics_list) == 0: + raise Exception('No metrics selected for evaluation') + output_res, output_msg = evaluator.evaluate(dataset_list, metrics_list) + + + # For each benchmark, combine the 'all' score with the 'cls_averaged' using geometric mean. + metrics_to_calc = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA'] + trackers = list(output_res['RobMOTS.' + config['BENCHMARKS'][0]].keys()) + for tracker in trackers: + # final_results[benchmark][result_type][metric] + final_results = {} + res = {bench: output_res['RobMOTS.' + bench][tracker]['COMBINED_SEQ'] for bench in config['BENCHMARKS']} + for bench in config['BENCHMARKS']: + final_results[bench] = {'cls_av': {}, 'det_av': {}, 'final': {}} + for metric in metrics_to_calc: + final_results[bench]['cls_av'][metric] = np.mean(res[bench]['cls_comb_cls_av']['HOTA'][metric]) + final_results[bench]['det_av'][metric] = np.mean(res[bench]['all']['HOTA'][metric]) + final_results[bench]['final'][metric] = \ + np.sqrt(final_results[bench]['cls_av'][metric] * final_results[bench]['det_av'][metric]) + + # Take the arithmetic mean over all the benchmarks + final_results['overall'] = {'cls_av': {}, 'det_av': {}, 'final': {}} + for metric in metrics_to_calc: + final_results['overall']['cls_av'][metric] = \ + np.mean([final_results[bench]['cls_av'][metric] for bench in config['BENCHMARKS']]) + final_results['overall']['det_av'][metric] = \ + np.mean([final_results[bench]['det_av'][metric] for bench in config['BENCHMARKS']]) + final_results['overall']['final'][metric] = \ + np.mean([final_results[bench]['final'][metric] for bench in config['BENCHMARKS']]) + + # Save out result + headers = [config['SPLIT_TO_EVAL']] + [x + '___' + metric for x in ['f', 'c', 'd'] for metric in metrics_to_calc] + + def rowify(d): + return [d[x][metric] for x in ['final', 'cls_av', 'det_av'] for metric in metrics_to_calc] + + out_file = os.path.join(script_config['TRACKERS_FOLDER'], script_config['ROBMOTS_SPLIT'], tracker, + 'final_results.csv') + + with open(out_file, 'w', newline='') as f: + writer = csv.writer(f, delimiter=',') + writer.writerow(headers) + writer.writerow(['overall'] + rowify(final_results['overall'])) + for bench in config['BENCHMARKS']: + if bench == 'overall': + continue + writer.writerow([bench] + rowify(final_results[bench])) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/tao.py b/test/yolov7-tracker/tracker/trackeval/datasets/tao.py new file mode 100644 index 0000000..e846167 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/tao.py @@ -0,0 +1,566 @@ +import os +import numpy as np +import json +import itertools +from collections import defaultdict +from scipy.optimize import linear_sum_assignment +from ..utils import TrackEvalException +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing + + +class TAO(_BaseDataset): + """Dataset class for TAO tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes) + 'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val' + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'MAX_DETECTIONS': 300, # Number of maximal allowed detections per image (0 for unlimited) + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = True + self.use_super_categories = False + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')] + if len(gt_dir_files) != 1: + raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.') + + with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f: + self.gt_data = json.load(f) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks']) + + # Get sequences to eval and sequence information + self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']] + self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']} + # compute mappings from videos to annotation data + self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations']) + # compute sequence lengths + self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']} + for img in self.gt_data['images']: + self.seq_lengths[img['video_id']] += 1 + self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings() + self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track + in self.videos_to_gt_tracks[vid['id']]}), + 'neg_cat_ids': vid['neg_category_ids'], + 'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']} + for vid in self.gt_data['videos']} + + # Get classes to eval + considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list] + seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id + in self.seq_to_classes[vid_id]['pos_cat_ids']]) + # only classes with ground truth are evaluated in TAO + self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats] + cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']} + + if self.config['CLASSES_TO_EVAL']: + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(self.valid_classes) + + ' are valid (classes present in ground truth data).') + else: + self.class_list = [cls for cls in self.valid_classes] + self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list} + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + self.tracker_data = {tracker: dict() for tracker in self.tracker_list} + + for tracker in self.tracker_list: + tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)) + if file.endswith('.json')] + if len(tr_dir_files) != 1: + raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol) + + ' does not contain exactly one json file.') + with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f: + curr_data = json.load(f) + + # limit detections if MAX_DETECTIONS > 0 + if self.config['MAX_DETECTIONS']: + curr_data = self._limit_dets_per_image(curr_data) + + # fill missing video ids + self._fill_video_ids_inplace(curr_data) + + # make track ids unique over whole evaluation set + self._make_track_ids_unique(curr_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(curr_data) + + # get tracker sequence information + curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data) + self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks + self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the TAO format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values + as keys and lists (for each track) as values + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values + as keys and lists as values + [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values + """ + seq_id = self.seq_name_to_seq_id[seq] + # File location + if is_gt: + imgs = self.videos_to_gt_images[seq_id] + else: + imgs = self.tracker_data[tracker]['vids_to_images'][seq_id] + + # Convert data to required format + num_timesteps = self.seq_lengths[seq_id] + img_to_timestep = self.seq_to_images_to_timestep[seq_id] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for img in imgs: + # some tracker data contains images without any ground truth information, these are ignored + try: + t = img_to_timestep[img['id']] + except KeyError: + continue + annotations = img['annotations'] + raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float) + raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int) + raw_data['classes'][t] = np.atleast_1d([ann['category_id'] for ann in annotations]).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float) + + for t, d in enumerate(raw_data['dets']): + if d is None: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.empty(0) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list] + if is_gt: + classes_to_consider = all_classes + all_tracks = self.videos_to_gt_tracks[seq_id] + else: + classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \ + + self.seq_to_classes[seq_id]['neg_cat_ids'] + all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id] + + classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls] + if cls in classes_to_consider else [] for cls in all_classes} + + # mapping from classes to track information + raw_data['classes_to_tracks'] = {cls: [{det['image_id']: np.atleast_1d(det['bbox']) + for det in track['annotations']} for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks] + for cls, tracks in classes_to_tracks.items()} + + if not is_gt: + raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score']) + for x in track['annotations']]) + for track in tracks]) + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + key_map = {'classes_to_tracks': 'classes_to_gt_tracks', + 'classes_to_track_ids': 'classes_to_gt_track_ids', + 'classes_to_track_lengths': 'classes_to_gt_track_lengths', + 'classes_to_track_areas': 'classes_to_gt_track_areas'} + else: + key_map = {'classes_to_tracks': 'classes_to_dt_tracks', + 'classes_to_track_ids': 'classes_to_dt_track_ids', + 'classes_to_track_lengths': 'classes_to_dt_track_lengths', + 'classes_to_track_areas': 'classes_to_dt_track_areas'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids'] + raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids'] + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + TAO: + In TAO, the 4 preproc steps are as follow: + 1) All classes present in the ground truth data are evaluated separately. + 2) No matched tracker detections are removed. + 3) Unmatched tracker detections are removed if there is not ground truth data and the class does not + belong to the categories marked as negative for this sequence. Additionally, unmatched tracker + detections for classes which are marked as not exhaustively labeled are removed. + 4) No gt detections are removed. + Further, for TrackMAP computation track representations for the given class are accessed from a dictionary + and the tracks from the tracker data are sorted according to the tracker confidence. + """ + cls_id = self.class_name_to_class_id[cls] + is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls'] + is_neg_category = cls_id in raw_data['neg_cat_ids'] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm). + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + if gt_ids.shape[0] == 0 and not is_neg_category: + to_remove_tracker = unmatched_indices + elif is_not_exhaustively_labeled: + to_remove_tracker = unmatched_indices + else: + to_remove_tracker = np.array([], dtype=np.int) + + # remove all unwanted unmatched tracker detections + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # get track representations + data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id] + data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id] + data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id] + data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id] + data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id] + data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id] + data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id] + data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id] + data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id] + data['not_exhaustively_labeled'] = is_not_exhaustively_labeled + data['iou_type'] = 'bbox' + + # sort tracker data tracks by tracker confidence scores + if data['dt_tracks']: + idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort") + data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx] + data['dt_tracks'] = [data['dt_tracks'][i] for i in idx] + data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx] + data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx] + data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx] + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t) + return similarity_scores + + def _merge_categories(self, annotations): + """ + Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset + :param annotations: the annotations in which the classes should be merged + :return: None + """ + merge_map = {} + for category in self.gt_data['categories']: + if 'merged' in category: + for to_merge in category['merged']: + merge_map[to_merge['id']] = category['id'] + + for ann in annotations: + ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id']) + + def _compute_vid_mappings(self, annotations): + """ + Computes mappings from Videos to corresponding tracks and images. + :param annotations: the annotations for which the mapping should be generated + :return: the video-to-track-mapping, the video-to-image-mapping + """ + vids_to_tracks = {} + vids_to_imgs = {} + vid_ids = [vid['id'] for vid in self.gt_data['videos']] + + # compute an mapping from image IDs to images + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + for ann in annotations: + ann["area"] = ann["bbox"][2] * ann["bbox"][3] + + vid = ann["video_id"] + if ann["video_id"] not in vids_to_tracks.keys(): + vids_to_tracks[ann["video_id"]] = list() + if ann["video_id"] not in vids_to_imgs.keys(): + vids_to_imgs[ann["video_id"]] = list() + + # Fill in vids_to_tracks + tid = ann["track_id"] + exist_tids = [track["id"] for track in vids_to_tracks[vid]] + try: + index1 = exist_tids.index(tid) + except ValueError: + index1 = -1 + if tid not in exist_tids: + curr_track = {"id": tid, "category_id": ann['category_id'], + "video_id": vid, "annotations": [ann]} + vids_to_tracks[vid].append(curr_track) + else: + vids_to_tracks[vid][index1]["annotations"].append(ann) + + # Fill in vids_to_imgs + img_id = ann['image_id'] + exist_img_ids = [img["id"] for img in vids_to_imgs[vid]] + try: + index2 = exist_img_ids.index(img_id) + except ValueError: + index2 = -1 + if index2 == -1: + curr_img = {"id": img_id, "annotations": [ann]} + vids_to_imgs[vid].append(curr_img) + else: + vids_to_imgs[vid][index2]["annotations"].append(ann) + + # sort annotations by frame index and compute track area + for vid, tracks in vids_to_tracks.items(): + for track in tracks: + track["annotations"] = sorted( + track['annotations'], + key=lambda x: images[x['image_id']]['frame_index']) + # Computer average area + track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations'])) + + # Ensure all videos are present + for vid_id in vid_ids: + if vid_id not in vids_to_tracks.keys(): + vids_to_tracks[vid_id] = [] + if vid_id not in vids_to_imgs.keys(): + vids_to_imgs[vid_id] = [] + + return vids_to_tracks, vids_to_imgs + + def _compute_image_to_timestep_mappings(self): + """ + Computes a mapping from images to the corresponding timestep in the sequence. + :return: the image-to-timestep-mapping + """ + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']} + for vid in seq_to_imgs_to_timestep: + curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]] + curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index']) + seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))} + + return seq_to_imgs_to_timestep + + def _limit_dets_per_image(self, annotations): + """ + Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from + https://github.com/TAO-Dataset/ + :param annotations: the annotations in which the detections should be limited + :return: the annotations with limited detections + """ + max_dets = self.config['MAX_DETECTIONS'] + img_ann = defaultdict(list) + for ann in annotations: + img_ann[ann["image_id"]].append(ann) + + for img_id, _anns in img_ann.items(): + if len(_anns) <= max_dets: + continue + _anns = sorted(_anns, key=lambda x: x["score"], reverse=True) + img_ann[img_id] = _anns[:max_dets] + + return [ann for anns in img_ann.values() for ann in anns] + + def _fill_video_ids_inplace(self, annotations): + """ + Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotations for which the videos IDs should be filled inplace + :return: None + """ + missing_video_id = [x for x in annotations if 'video_id' not in x] + if missing_video_id: + image_id_to_video_id = { + x['id']: x['video_id'] for x in self.gt_data['images'] + } + for x in missing_video_id: + x['video_id'] = image_id_to_video_id[x['image_id']] + + @staticmethod + def _make_track_ids_unique(annotations): + """ + Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotation set + :return: the number of updated IDs + """ + track_id_videos = {} + track_ids_to_update = set() + max_track_id = 0 + for ann in annotations: + t = ann['track_id'] + if t not in track_id_videos: + track_id_videos[t] = ann['video_id'] + + if ann['video_id'] != track_id_videos[t]: + # Track id is assigned to multiple videos + track_ids_to_update.add(t) + max_track_id = max(max_track_id, t) + + if track_ids_to_update: + print('true') + next_id = itertools.count(max_track_id + 1) + new_track_ids = defaultdict(lambda: next(next_id)) + for ann in annotations: + t = ann['track_id'] + v = ann['video_id'] + if t in track_ids_to_update: + ann['track_id'] = new_track_ids[t, v] + return len(track_ids_to_update) diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/tao_ow.py b/test/yolov7-tracker/tracker/trackeval/datasets/tao_ow.py new file mode 100644 index 0000000..40f80d7 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/tao_ow.py @@ -0,0 +1,652 @@ +import os +import numpy as np +import json +import itertools +from collections import defaultdict +from scipy.optimize import linear_sum_assignment +from ..utils import TrackEvalException +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing + + +class TAO_OW(_BaseDataset): + """Dataset class for TAO tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/tao/tao_training'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/tao/tao_training'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes) + 'SPLIT_TO_EVAL': 'training', # Valid: 'training', 'val' + 'PRINT_CONFIG': True, # Whether to print current config + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'MAX_DETECTIONS': 300, # Number of maximal allowed detections per image (0 for unlimited) + 'SUBSET': 'all' + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + self.should_classes_combine = True + self.use_super_categories = False + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')] + if len(gt_dir_files) != 1: + raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.') + + with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f: + self.gt_data = json.load(f) + + self.subset = self.config['SUBSET'] + if self.subset != 'all': + # Split GT data into `known`, `unknown` or `distractor` + self._split_known_unknown_distractor() + self.gt_data = self._filter_gt_data(self.gt_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(self.gt_data['annotations'] + self.gt_data['tracks']) + + # Get sequences to eval and sequence information + self.seq_list = [vid['name'].replace('/', '-') for vid in self.gt_data['videos']] + self.seq_name_to_seq_id = {vid['name'].replace('/', '-'): vid['id'] for vid in self.gt_data['videos']} + # compute mappings from videos to annotation data + self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(self.gt_data['annotations']) + # compute sequence lengths + self.seq_lengths = {vid['id']: 0 for vid in self.gt_data['videos']} + for img in self.gt_data['images']: + self.seq_lengths[img['video_id']] += 1 + self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings() + self.seq_to_classes = {vid['id']: {'pos_cat_ids': list({track['category_id'] for track + in self.videos_to_gt_tracks[vid['id']]}), + 'neg_cat_ids': vid['neg_category_ids'], + 'not_exhaustively_labeled_cat_ids': vid['not_exhaustive_category_ids']} + for vid in self.gt_data['videos']} + + # Get classes to eval + considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list] + seen_cats = set([cat_id for vid_id in considered_vid_ids for cat_id + in self.seq_to_classes[vid_id]['pos_cat_ids']]) + # only classes with ground truth are evaluated in TAO + self.valid_classes = [cls['name'] for cls in self.gt_data['categories'] if cls['id'] in seen_cats] + # cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']} + + if self.config['CLASSES_TO_EVAL']: + # self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + # for cls in self.config['CLASSES_TO_EVAL']] + self.class_list = ["object"] # class-agnostic + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(self.valid_classes) + + ' are valid (classes present in ground truth data).') + else: + # self.class_list = [cls for cls in self.valid_classes] + self.class_list = ["object"] # class-agnostic + # self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list} + self.class_name_to_class_id = {"object": 1} # class-agnostic + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + self.tracker_data = {tracker: dict() for tracker in self.tracker_list} + + for tracker in self.tracker_list: + tr_dir_files = [file for file in os.listdir(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)) + if file.endswith('.json')] + if len(tr_dir_files) != 1: + raise TrackEvalException(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol) + + ' does not contain exactly one json file.') + with open(os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0])) as f: + curr_data = json.load(f) + + # limit detections if MAX_DETECTIONS > 0 + if self.config['MAX_DETECTIONS']: + curr_data = self._limit_dets_per_image(curr_data) + + # fill missing video ids + self._fill_video_ids_inplace(curr_data) + + # make track ids unique over whole evaluation set + self._make_track_ids_unique(curr_data) + + # merge categories marked with a merged tag in TAO dataset + self._merge_categories(curr_data) + + # get tracker sequence information + curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = self._compute_vid_mappings(curr_data) + self.tracker_data[tracker]['vids_to_tracks'] = curr_videos_to_tracker_tracks + self.tracker_data[tracker]['vids_to_images'] = curr_videos_to_tracker_images + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the TAO format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values + as keys and lists (for each track) as values + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values + as keys and lists as values + [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values + """ + seq_id = self.seq_name_to_seq_id[seq] + # File location + if is_gt: + imgs = self.videos_to_gt_images[seq_id] + else: + imgs = self.tracker_data[tracker]['vids_to_images'][seq_id] + + # Convert data to required format + num_timesteps = self.seq_lengths[seq_id] + img_to_timestep = self.seq_to_images_to_timestep[seq_id] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for img in imgs: + # some tracker data contains images without any ground truth information, these are ignored + try: + t = img_to_timestep[img['id']] + except KeyError: + continue + annotations = img['annotations'] + raw_data['dets'][t] = np.atleast_2d([ann['bbox'] for ann in annotations]).astype(float) + raw_data['ids'][t] = np.atleast_1d([ann['track_id'] for ann in annotations]).astype(int) + raw_data['classes'][t] = np.atleast_1d([1 for _ in annotations]).astype(int) # class-agnostic + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([ann['score'] for ann in annotations]).astype(float) + + for t, d in enumerate(raw_data['dets']): + if d is None: + raw_data['dets'][t] = np.empty((0, 4)).astype(float) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.empty(0) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + # all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list] + all_classes = [1] # class-agnostic + + if is_gt: + classes_to_consider = all_classes + all_tracks = self.videos_to_gt_tracks[seq_id] + else: + # classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \ + # + self.seq_to_classes[seq_id]['neg_cat_ids'] + classes_to_consider = all_classes # class-agnostic + all_tracks = self.tracker_data[tracker]['vids_to_tracks'][seq_id] + + # classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls] + # if cls in classes_to_consider else [] for cls in all_classes} + classes_to_tracks = {cls: [track for track in all_tracks] + if cls in classes_to_consider else [] for cls in all_classes} # class-agnostic + + # mapping from classes to track information + raw_data['classes_to_tracks'] = {cls: [{det['image_id']: np.atleast_1d(det['bbox']) + for det in track['annotations']} for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_lengths'] = {cls: [len(track['annotations']) for track in tracks] + for cls, tracks in classes_to_tracks.items()} + + if not is_gt: + raw_data['classes_to_dt_track_scores'] = {cls: np.array([np.mean([float(x['score']) + for x in track['annotations']]) + for track in tracks]) + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + key_map = {'classes_to_tracks': 'classes_to_gt_tracks', + 'classes_to_track_ids': 'classes_to_gt_track_ids', + 'classes_to_track_lengths': 'classes_to_gt_track_lengths', + 'classes_to_track_areas': 'classes_to_gt_track_areas'} + else: + key_map = {'classes_to_tracks': 'classes_to_dt_tracks', + 'classes_to_track_ids': 'classes_to_dt_track_ids', + 'classes_to_track_lengths': 'classes_to_dt_track_lengths', + 'classes_to_track_areas': 'classes_to_dt_track_areas'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['neg_cat_ids'] = self.seq_to_classes[seq_id]['neg_cat_ids'] + raw_data['not_exhaustively_labeled_cls'] = self.seq_to_classes[seq_id]['not_exhaustively_labeled_cat_ids'] + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + TAO: + In TAO, the 4 preproc steps are as follow: + 1) All classes present in the ground truth data are evaluated separately. + 2) No matched tracker detections are removed. + 3) Unmatched tracker detections are removed if there is not ground truth data and the class does not + belong to the categories marked as negative for this sequence. Additionally, unmatched tracker + detections for classes which are marked as not exhaustively labeled are removed. + 4) No gt detections are removed. + Further, for TrackMAP computation track representations for the given class are accessed from a dictionary + and the tracks from the tracker data are sorted according to the tracker confidence. + """ + cls_id = self.class_name_to_class_id[cls] + is_not_exhaustively_labeled = cls_id in raw_data['not_exhaustively_labeled_cls'] + is_neg_category = cls_id in raw_data['neg_cat_ids'] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for preproc and eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = raw_data['gt_dets'][t][gt_class_mask] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = raw_data['tracker_dets'][t][tracker_class_mask] + tracker_confidences = raw_data['tracker_confidences'][t][tracker_class_mask] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + # Match tracker and gt dets (with hungarian algorithm). + unmatched_indices = np.arange(tracker_ids.shape[0]) + if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_cols = match_cols[actually_matched_mask] + unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0) + + if gt_ids.shape[0] == 0 and not is_neg_category: + to_remove_tracker = unmatched_indices + elif is_not_exhaustively_labeled: + to_remove_tracker = unmatched_indices + else: + to_remove_tracker = np.array([], dtype=np.int) + + # remove all unwanted unmatched tracker detections + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # get track representations + data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id] + data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id] + data['gt_track_lengths'] = raw_data['classes_to_gt_track_lengths'][cls_id] + data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id] + data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id] + data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id] + data['dt_track_lengths'] = raw_data['classes_to_dt_track_lengths'][cls_id] + data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id] + data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id] + data['not_exhaustively_labeled'] = is_not_exhaustively_labeled + data['iou_type'] = 'bbox' + + # sort tracker data tracks by tracker confidence scores + if data['dt_tracks']: + idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort") + data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx] + data['dt_tracks'] = [data['dt_tracks'][i] for i in idx] + data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx] + data['dt_track_lengths'] = [data['dt_track_lengths'][i] for i in idx] + data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx] + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t) + return similarity_scores + + def _merge_categories(self, annotations): + """ + Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset + :param annotations: the annotations in which the classes should be merged + :return: None + """ + merge_map = {} + for category in self.gt_data['categories']: + if 'merged' in category: + for to_merge in category['merged']: + merge_map[to_merge['id']] = category['id'] + + for ann in annotations: + ann['category_id'] = merge_map.get(ann['category_id'], ann['category_id']) + + def _compute_vid_mappings(self, annotations): + """ + Computes mappings from Videos to corresponding tracks and images. + :param annotations: the annotations for which the mapping should be generated + :return: the video-to-track-mapping, the video-to-image-mapping + """ + vids_to_tracks = {} + vids_to_imgs = {} + vid_ids = [vid['id'] for vid in self.gt_data['videos']] + + # compute an mapping from image IDs to images + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + for ann in annotations: + ann["area"] = ann["bbox"][2] * ann["bbox"][3] + + vid = ann["video_id"] + if ann["video_id"] not in vids_to_tracks.keys(): + vids_to_tracks[ann["video_id"]] = list() + if ann["video_id"] not in vids_to_imgs.keys(): + vids_to_imgs[ann["video_id"]] = list() + + # Fill in vids_to_tracks + tid = ann["track_id"] + exist_tids = [track["id"] for track in vids_to_tracks[vid]] + try: + index1 = exist_tids.index(tid) + except ValueError: + index1 = -1 + if tid not in exist_tids: + curr_track = {"id": tid, "category_id": ann['category_id'], + "video_id": vid, "annotations": [ann]} + vids_to_tracks[vid].append(curr_track) + else: + vids_to_tracks[vid][index1]["annotations"].append(ann) + + # Fill in vids_to_imgs + img_id = ann['image_id'] + exist_img_ids = [img["id"] for img in vids_to_imgs[vid]] + try: + index2 = exist_img_ids.index(img_id) + except ValueError: + index2 = -1 + if index2 == -1: + curr_img = {"id": img_id, "annotations": [ann]} + vids_to_imgs[vid].append(curr_img) + else: + vids_to_imgs[vid][index2]["annotations"].append(ann) + + # sort annotations by frame index and compute track area + for vid, tracks in vids_to_tracks.items(): + for track in tracks: + track["annotations"] = sorted( + track['annotations'], + key=lambda x: images[x['image_id']]['frame_index']) + # Computer average area + track["area"] = (sum(x['area'] for x in track['annotations']) / len(track['annotations'])) + + # Ensure all videos are present + for vid_id in vid_ids: + if vid_id not in vids_to_tracks.keys(): + vids_to_tracks[vid_id] = [] + if vid_id not in vids_to_imgs.keys(): + vids_to_imgs[vid_id] = [] + + return vids_to_tracks, vids_to_imgs + + def _compute_image_to_timestep_mappings(self): + """ + Computes a mapping from images to the corresponding timestep in the sequence. + :return: the image-to-timestep-mapping + """ + images = {} + for image in self.gt_data['images']: + images[image['id']] = image + + seq_to_imgs_to_timestep = {vid['id']: dict() for vid in self.gt_data['videos']} + for vid in seq_to_imgs_to_timestep: + curr_imgs = [img['id'] for img in self.videos_to_gt_images[vid]] + curr_imgs = sorted(curr_imgs, key=lambda x: images[x]['frame_index']) + seq_to_imgs_to_timestep[vid] = {curr_imgs[i]: i for i in range(len(curr_imgs))} + + return seq_to_imgs_to_timestep + + def _limit_dets_per_image(self, annotations): + """ + Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from + https://github.com/TAO-Dataset/ + :param annotations: the annotations in which the detections should be limited + :return: the annotations with limited detections + """ + max_dets = self.config['MAX_DETECTIONS'] + img_ann = defaultdict(list) + for ann in annotations: + img_ann[ann["image_id"]].append(ann) + + for img_id, _anns in img_ann.items(): + if len(_anns) <= max_dets: + continue + _anns = sorted(_anns, key=lambda x: x["score"], reverse=True) + img_ann[img_id] = _anns[:max_dets] + + return [ann for anns in img_ann.values() for ann in anns] + + def _fill_video_ids_inplace(self, annotations): + """ + Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotations for which the videos IDs should be filled inplace + :return: None + """ + missing_video_id = [x for x in annotations if 'video_id' not in x] + if missing_video_id: + image_id_to_video_id = { + x['id']: x['video_id'] for x in self.gt_data['images'] + } + for x in missing_video_id: + x['video_id'] = image_id_to_video_id[x['image_id']] + + @staticmethod + def _make_track_ids_unique(annotations): + """ + Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/ + :param annotations: the annotation set + :return: the number of updated IDs + """ + track_id_videos = {} + track_ids_to_update = set() + max_track_id = 0 + for ann in annotations: + t = ann['track_id'] + if t not in track_id_videos: + track_id_videos[t] = ann['video_id'] + + if ann['video_id'] != track_id_videos[t]: + # Track id is assigned to multiple videos + track_ids_to_update.add(t) + max_track_id = max(max_track_id, t) + + if track_ids_to_update: + print('true') + next_id = itertools.count(max_track_id + 1) + new_track_ids = defaultdict(lambda: next(next_id)) + for ann in annotations: + t = ann['track_id'] + v = ann['video_id'] + if t in track_ids_to_update: + ann['track_id'] = new_track_ids[t, v] + return len(track_ids_to_update) + + def _split_known_unknown_distractor(self): + all_ids = set([i for i in range(1, 2000)]) # 2000 is larger than the max category id in TAO-OW. + # `knowns` includes 78 TAO_category_ids that corresponds to 78 COCO classes. + # (The other 2 COCO classes do not have corresponding classes in TAO). + self.knowns = {4, 13, 1038, 544, 1057, 34, 35, 36, 41, 45, 58, 60, 579, 1091, 1097, 1099, 78, 79, 81, 91, 1115, + 1117, 95, 1122, 99, 1132, 621, 1135, 625, 118, 1144, 126, 642, 1155, 133, 1162, 139, 154, 174, 185, + 699, 1215, 714, 717, 1229, 211, 729, 221, 229, 747, 235, 237, 779, 276, 805, 299, 829, 852, 347, + 371, 382, 896, 392, 926, 937, 428, 429, 961, 452, 979, 980, 982, 475, 480, 993, 1001, 502, 1018} + # `distractors` is defined as in the paper "Opening up Open-World Tracking" + self.distractors = {20, 63, 108, 180, 188, 204, 212, 247, 303, 403, 407, 415, 490, 504, 507, 513, 529, 567, + 569, 588, 672, 691, 702, 708, 711, 720, 736, 737, 798, 813, 815, 827, 831, 851, 877, 883, + 912, 971, 976, 1130, 1133, 1134, 1169, 1184, 1220} + self.unknowns = all_ids.difference(self.knowns.union(self.distractors)) + + def _filter_gt_data(self, raw_gt_data): + """ + Filter out irrelevant data in the raw_gt_data + Args: + raw_gt_data: directly loaded from json. + + Returns: + filtered gt_data + """ + valid_cat_ids = list() + if self.subset == "known": + valid_cat_ids = self.knowns + elif self.subset == "distractor": + valid_cat_ids = self.distractors + elif self.subset == "unknown": + valid_cat_ids = self.unknowns + # elif self.subset == "test_only_unknowns": + # valid_cat_ids = test_only_unknowns + else: + raise Exception("The parameter `SUBSET` is incorrect") + + filtered = dict() + filtered["videos"] = raw_gt_data["videos"] + # filtered["videos"] = list() + unwanted_vid = set() + # for video in raw_gt_data["videos"]: + # datasrc = video["name"].split('/')[1] + # if datasrc in data_srcs: + # filtered["videos"].append(video) + # else: + # unwanted_vid.add(video["id"]) + + filtered["annotations"] = list() + for ann in raw_gt_data["annotations"]: + if (ann["video_id"] not in unwanted_vid) and (ann["category_id"] in valid_cat_ids): + filtered["annotations"].append(ann) + + filtered["tracks"] = list() + for track in raw_gt_data["tracks"]: + if (track["video_id"] not in unwanted_vid) and (track["category_id"] in valid_cat_ids): + filtered["tracks"].append(track) + + filtered["images"] = list() + for image in raw_gt_data["images"]: + if image["video_id"] not in unwanted_vid: + filtered["images"].append(image) + + filtered["categories"] = list() + for cat in raw_gt_data["categories"]: + if cat["id"] in valid_cat_ids: + filtered["categories"].append(cat) + + filtered["info"] = raw_gt_data["info"] + filtered["licenses"] = raw_gt_data["licenses"] + + return filtered diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/visdrone.py b/test/yolov7-tracker/tracker/trackeval/datasets/visdrone.py new file mode 100644 index 0000000..4dfcb7d --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/visdrone.py @@ -0,0 +1,438 @@ +import os +import csv +import configparser +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_dataset import _BaseDataset +from .. import utils +from .. import _timing +from ..utils import TrackEvalException + + +class VisDrone2DBox(_BaseDataset): + """Dataset class for MOT Challenge 2D bounding box tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/mot_challenge/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/mot_challenge/'), # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': ['pedestrain', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor'], # Valid: ['pedestrian'] + 'BENCHMARK': 'MOT17', # Valid: 'MOT17', 'MOT16', 'MOT20', 'MOT15' + 'SPLIT_TO_EVAL': 'train', # Valid: 'train', 'test', 'all' + 'INPUT_AS_ZIP': False, # Whether tracker input files are zipped + 'PRINT_CONFIG': True, # Whether to print current config + 'DO_PREPROC': True, # Whether to perform preprocessing (never done for MOT15) + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + 'SEQMAP_FOLDER': None, # Where seqmaps are found (if None, GT_FOLDER/seqmaps) + 'SEQMAP_FILE': None, # Directly specify seqmap file (if none use seqmap_folder/benchmark-split_to_eval) + 'SEQ_INFO': None, # If not None, directly specify sequences to eval and their number of timesteps + 'GT_LOC_FORMAT': '{gt_folder}/{seq}/gt/gt.txt', # '{gt_folder}/{seq}/gt/gt.txt' + 'SKIP_SPLIT_FOL': False, # If False, data is in GT_FOLDER/BENCHMARK-SPLIT_TO_EVAL/ and in + # TRACKERS_FOLDER/BENCHMARK-SPLIT_TO_EVAL/tracker/ + # If True, then the middle 'benchmark-split' folder is skipped for both. + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + + self.benchmark = self.config['BENCHMARK'] + gt_set = self.config['BENCHMARK'] + '-' + self.config['SPLIT_TO_EVAL'] + self.gt_set = gt_set + if not self.config['SKIP_SPLIT_FOL']: + split_fol = gt_set + else: + split_fol = '' + self.gt_fol = os.path.join(self.config['GT_FOLDER'], split_fol) + self.tracker_fol = os.path.join(self.config['TRACKERS_FOLDER'], split_fol) + self.should_classes_combine = False + self.use_super_categories = False + self.data_is_zipped = self.config['INPUT_AS_ZIP'] + self.do_preproc = self.config['DO_PREPROC'] + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + + # Get classes to eval + self.valid_classes = ['pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning-tricycle', 'bus', 'motor'] + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only pedestrian class is valid.') + self.class_name_to_class_id = {'ignored': 0, 'pedestrian': 1, 'people': 2, 'bicycle': 3, 'car': 4, 'van': 5, + 'truck': 6, 'tricycle': 7, 'awning-tricycle': 8, 'bus': 9, + 'motor': 10, 'other': 11} + self.valid_class_numbers = list(self.class_name_to_class_id.values()) + + # Get sequences to eval and check gt files exist + self.seq_list, self.seq_lengths = self._get_seq_info() + if len(self.seq_list) < 1: + raise TrackEvalException('No sequences are selected to be evaluated.') + + # Check gt files exist + for seq in self.seq_list: + if not self.data_is_zipped: + curr_file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found for sequence: ' + seq) + if self.data_is_zipped: + curr_file = os.path.join(self.gt_fol, 'data.zip') + if not os.path.isfile(curr_file): + print('GT file not found ' + curr_file) + raise TrackEvalException('GT file not found: ' + os.path.basename(curr_file)) + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + for tracker in self.tracker_list: + if self.data_is_zipped: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException('Tracker file not found: ' + tracker + '/' + os.path.basename(curr_file)) + else: + for seq in self.seq_list: + curr_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + if not os.path.isfile(curr_file): + print('Tracker file not found: ' + curr_file) + raise TrackEvalException( + 'Tracker file not found: ' + tracker + '/' + self.tracker_sub_fol + '/' + os.path.basename( + curr_file)) + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _get_seq_info(self): + seq_list = [] + seq_lengths = {} + if self.config["SEQ_INFO"]: + seq_list = list(self.config["SEQ_INFO"].keys()) + seq_lengths = self.config["SEQ_INFO"] + + # If sequence length is 'None' tries to read sequence length from .ini files. + for seq, seq_length in seq_lengths.items(): + if seq_length is None: + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + + else: + if self.config["SEQMAP_FILE"]: + seqmap_file = self.config["SEQMAP_FILE"] + else: + if self.config["SEQMAP_FOLDER"] is None: + seqmap_file = os.path.join(self.config['GT_FOLDER'], 'seqmaps', self.gt_set + '.txt') + else: + seqmap_file = os.path.join(self.config["SEQMAP_FOLDER"], self.gt_set + '.txt') + if not os.path.isfile(seqmap_file): + print('no seqmap found: ' + seqmap_file) + raise TrackEvalException('no seqmap found: ' + os.path.basename(seqmap_file)) + with open(seqmap_file) as fp: + reader = csv.reader(fp) + for i, row in enumerate(reader): + if i == 0 or row[0] == '': + continue + seq = row[0] + seq_list.append(seq) + ini_file = os.path.join(self.gt_fol, seq, 'seqinfo.ini') + if not os.path.isfile(ini_file): + raise TrackEvalException('ini file does not exist: ' + seq + '/' + os.path.basename(ini_file)) + ini_data = configparser.ConfigParser() + ini_data.read(ini_file) + seq_lengths[seq] = int(ini_data['Sequence']['seqLength']) + return seq_list, seq_lengths + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the MOT Challenge 2D box format + + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections. + [gt_extras] : list (for each timestep) of dicts (for each extra) of 1D NDArrays (for each det). + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + """ + # File location + if self.data_is_zipped: + if is_gt: + zip_file = os.path.join(self.gt_fol, 'data.zip') + else: + zip_file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol + '.zip') + file = seq + '.txt' + else: + zip_file = None + if is_gt: + file = self.config["GT_LOC_FORMAT"].format(gt_folder=self.gt_fol, seq=seq) + else: + file = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol, seq + '.txt') + + # Load raw data from text file + read_data, ignore_data = self._load_simple_text_file(file, is_zipped=self.data_is_zipped, zip_file=zip_file) + + # Convert data to required format + num_timesteps = self.seq_lengths[seq] + data_keys = ['ids', 'classes', 'dets'] + if is_gt: + data_keys += ['gt_crowd_ignore_regions', 'gt_extras'] + else: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + + # Check for any extra time keys + current_time_keys = [str( t+ 1) for t in range(num_timesteps)] + extra_time_keys = [x for x in read_data.keys() if x not in current_time_keys] + if len(extra_time_keys) > 0: + if is_gt: + text = 'Ground-truth' + else: + text = 'Tracking' + raise TrackEvalException( + text + ' data contains the following invalid timesteps in seq %s: ' % seq + ', '.join( + [str(x) + ', ' for x in extra_time_keys])) + + for t in range(num_timesteps): + time_key = str(t+1) + if time_key in read_data.keys(): + try: + time_data = np.asarray(read_data[time_key], dtype=np.float) + except ValueError: + if is_gt: + raise TrackEvalException( + 'Cannot convert gt data for sequence %s to float. Is data corrupted?' % seq) + else: + raise TrackEvalException( + 'Cannot convert tracking data from tracker %s, sequence %s to float. Is data corrupted?' % ( + tracker, seq)) + try: + raw_data['dets'][t] = np.atleast_2d(time_data[:, 2:6]) + raw_data['ids'][t] = np.atleast_1d(time_data[:, 1]).astype(int) + except IndexError: + if is_gt: + err = 'Cannot load gt data from sequence %s, because there is not enough ' \ + 'columns in the data.' % seq + raise TrackEvalException(err) + else: + err = 'Cannot load tracker data from tracker %s, sequence %s, because there is not enough ' \ + 'columns in the data.' % (tracker, seq) + raise TrackEvalException(err) + if time_data.shape[1] >= 8: + raw_data['classes'][t] = np.atleast_1d(time_data[:, 7]).astype(int) + else: + if not is_gt: + raw_data['classes'][t] = np.ones_like(raw_data['ids'][t]) + else: + raise TrackEvalException( + 'GT data is not in a valid format, there is not enough rows in seq %s, timestep %i.' % ( + seq, t)) + if is_gt: + gt_extras_dict = {'zero_marked': np.atleast_1d(time_data[:, 6].astype(int))} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.atleast_1d(time_data[:, 6]) + else: + raw_data['dets'][t] = np.empty((0, 4)) + raw_data['ids'][t] = np.empty(0).astype(int) + raw_data['classes'][t] = np.empty(0).astype(int) + if is_gt: + gt_extras_dict = {'zero_marked': np.empty(0)} + raw_data['gt_extras'][t] = gt_extras_dict + else: + raw_data['tracker_confidences'][t] = np.empty(0) + if is_gt: + raw_data['gt_crowd_ignore_regions'][t] = np.empty((0, 4)) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + + MOT Challenge: + In MOT Challenge, the 4 preproc steps are as follow: + 1) There is only one class (pedestrian) to be evaluated, but all other classes are used for preproc. + 2) Predictions are matched against all gt boxes (regardless of class), those matching with distractor + objects are removed. + 3) There is no crowd ignore regions. + 4) All gt dets except pedestrian are removed, also removes pedestrian gt dets marked with zero_marked. + """ + # Check that input data has unique ids + self._check_unique_ids(raw_data) + + # distractor_class_names = ['person_on_vehicle', 'static_person', 'distractor', 'reflection'] + distractor_class_names = ['ignored', 'other'] + if self.benchmark == 'MOT20': + distractor_class_names.append('non_mot_vehicle') + distractor_classes = [self.class_name_to_class_id[x] for x in distractor_class_names] + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'tracker_confidences', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + for t in range(raw_data['num_timesteps']): + + # Get all data + gt_ids = raw_data['gt_ids'][t] + gt_dets = raw_data['gt_dets'][t] + gt_classes = raw_data['gt_classes'][t] + gt_zero_marked = raw_data['gt_extras'][t]['zero_marked'] + + tracker_ids = raw_data['tracker_ids'][t] + tracker_dets = raw_data['tracker_dets'][t] + tracker_classes = raw_data['tracker_classes'][t] + tracker_confidences = raw_data['tracker_confidences'][t] + similarity_scores = raw_data['similarity_scores'][t] + + # Evaluation is ONLY valid for pedestrian class + if len(tracker_classes) > 0 and np.max(tracker_classes) > 1: + raise TrackEvalException( + 'Evaluation is only valid for pedestrian class. Non pedestrian class (%i) found in sequence %s at ' + 'timestep %i.' % (np.max(tracker_classes), raw_data['seq'], t)) + + # Match tracker and gt dets (with hungarian algorithm) and remove tracker dets which match with gt dets + # which are labeled as belonging to a distractor class. + to_remove_tracker = np.array([], np.int) + if self.do_preproc and self.benchmark != 'MOT15' and gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0: + + # Check all classes are valid: + invalid_classes = np.setdiff1d(np.unique(gt_classes), self.valid_class_numbers) + if len(invalid_classes) > 0: + print(' '.join([str(x) for x in invalid_classes])) + raise(TrackEvalException('Attempting to evaluate using invalid gt classes. ' + 'This warning only triggers if preprocessing is performed, ' + 'e.g. not for MOT15 or where prepropressing is explicitly disabled. ' + 'Please either check your gt data, or disable preprocessing. ' + 'The following invalid classes were found in timestep ' + str(t) + ': ' + + ' '.join([str(x) for x in invalid_classes]))) + + matching_scores = similarity_scores.copy() + matching_scores[matching_scores < 0.5 - np.finfo('float').eps] = 0 + match_rows, match_cols = linear_sum_assignment(-matching_scores) + actually_matched_mask = matching_scores[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + is_distractor_class = np.isin(gt_classes[match_rows], distractor_classes) + to_remove_tracker = match_cols[is_distractor_class] + + # Apply preprocessing to remove all unwanted tracker dets. + data['tracker_ids'][t] = np.delete(tracker_ids, to_remove_tracker, axis=0) + data['tracker_dets'][t] = np.delete(tracker_dets, to_remove_tracker, axis=0) + data['tracker_confidences'][t] = np.delete(tracker_confidences, to_remove_tracker, axis=0) + similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1) + + # Remove gt detections marked as to remove (zero marked), and also remove gt detections not in pedestrian + # class (not applicable for MOT15) + if self.do_preproc and self.benchmark != 'MOT15': + gt_to_keep_mask = (np.not_equal(gt_zero_marked, 0)) & \ + (np.equal(gt_classes, cls_id)) + else: + # There are no classes for MOT15 + gt_to_keep_mask = np.not_equal(gt_zero_marked, 0) + data['gt_ids'][t] = gt_ids[gt_to_keep_mask] + data['gt_dets'][t] = gt_dets[gt_to_keep_mask, :] + data['similarity_scores'][t] = similarity_scores[gt_to_keep_mask] + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # Ensure again that ids are unique per timestep after preproc. + self._check_unique_ids(data, after_preproc=True) + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t, box_format='xywh') + return similarity_scores diff --git a/test/yolov7-tracker/tracker/trackeval/datasets/youtube_vis.py b/test/yolov7-tracker/tracker/trackeval/datasets/youtube_vis.py new file mode 100644 index 0000000..6d5b54c --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/datasets/youtube_vis.py @@ -0,0 +1,364 @@ +import os +import numpy as np +import json +from ._base_dataset import _BaseDataset +from ..utils import TrackEvalException +from .. import utils +from .. import _timing + + +class YouTubeVIS(_BaseDataset): + """Dataset class for YouTubeVIS tracking""" + + @staticmethod + def get_default_dataset_config(): + """Default class config values""" + code_path = utils.get_code_path() + default_config = { + 'GT_FOLDER': os.path.join(code_path, 'data/gt/youtube_vis/'), # Location of GT data + 'TRACKERS_FOLDER': os.path.join(code_path, 'data/trackers/youtube_vis/'), + # Trackers location + 'OUTPUT_FOLDER': None, # Where to save eval results (if None, same as TRACKERS_FOLDER) + 'TRACKERS_TO_EVAL': None, # Filenames of trackers to eval (if None, all in folder) + 'CLASSES_TO_EVAL': None, # Classes to eval (if None, all classes) + 'SPLIT_TO_EVAL': 'train_sub_split', # Valid: 'train', 'val', 'train_sub_split' + 'PRINT_CONFIG': True, # Whether to print current config + 'OUTPUT_SUB_FOLDER': '', # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER + 'TRACKER_SUB_FOLDER': 'data', # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER + 'TRACKER_DISPLAY_NAMES': None, # Names of trackers to display, if None: TRACKERS_TO_EVAL + } + return default_config + + def __init__(self, config=None): + """Initialise dataset, checking that all required files are present""" + super().__init__() + # Fill non-given config values with defaults + self.config = utils.init_config(config, self.get_default_dataset_config(), self.get_name()) + self.gt_fol = self.config['GT_FOLDER'] + 'youtube_vis_' + self.config['SPLIT_TO_EVAL'] + self.tracker_fol = self.config['TRACKERS_FOLDER'] + 'youtube_vis_' + self.config['SPLIT_TO_EVAL'] + self.use_super_categories = False + self.should_classes_combine = True + + self.output_fol = self.config['OUTPUT_FOLDER'] + if self.output_fol is None: + self.output_fol = self.tracker_fol + self.output_sub_fol = self.config['OUTPUT_SUB_FOLDER'] + self.tracker_sub_fol = self.config['TRACKER_SUB_FOLDER'] + + if not os.path.exists(self.gt_fol): + print("GT folder not found: " + self.gt_fol) + raise TrackEvalException("GT folder not found: " + os.path.basename(self.gt_fol)) + gt_dir_files = [file for file in os.listdir(self.gt_fol) if file.endswith('.json')] + if len(gt_dir_files) != 1: + raise TrackEvalException(self.gt_fol + ' does not contain exactly one json file.') + + with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f: + self.gt_data = json.load(f) + + # Get classes to eval + self.valid_classes = [cls['name'] for cls in self.gt_data['categories']] + cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']} + + if self.config['CLASSES_TO_EVAL']: + self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None + for cls in self.config['CLASSES_TO_EVAL']] + if not all(self.class_list): + raise TrackEvalException('Attempted to evaluate an invalid class. Only classes ' + + ', '.join(self.valid_classes) + ' are valid.') + else: + self.class_list = [cls['name'] for cls in self.gt_data['categories']] + self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list} + + # Get sequences to eval and check gt files exist + self.seq_list = [vid['file_names'][0].split('/')[0] for vid in self.gt_data['videos']] + self.seq_name_to_seq_id = {vid['file_names'][0].split('/')[0]: vid['id'] for vid in self.gt_data['videos']} + self.seq_lengths = {vid['id']: len(vid['file_names']) for vid in self.gt_data['videos']} + + # encode masks and compute track areas + self._prepare_gt_annotations() + + # Get trackers to eval + if self.config['TRACKERS_TO_EVAL'] is None: + self.tracker_list = os.listdir(self.tracker_fol) + else: + self.tracker_list = self.config['TRACKERS_TO_EVAL'] + + if self.config['TRACKER_DISPLAY_NAMES'] is None: + self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list)) + elif (self.config['TRACKERS_TO_EVAL'] is not None) and ( + len(self.config['TRACKER_DISPLAY_NAMES']) == len(self.tracker_list)): + self.tracker_to_disp = dict(zip(self.tracker_list, self.config['TRACKER_DISPLAY_NAMES'])) + else: + raise TrackEvalException('List of tracker files and tracker display names do not match.') + + # counter for globally unique track IDs + self.global_tid_counter = 0 + + self.tracker_data = dict() + for tracker in self.tracker_list: + tracker_dir_path = os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol) + tr_dir_files = [file for file in os.listdir(tracker_dir_path) if file.endswith('.json')] + if len(tr_dir_files) != 1: + raise TrackEvalException(tracker_dir_path + ' does not contain exactly one json file.') + + with open(os.path.join(tracker_dir_path, tr_dir_files[0])) as f: + curr_data = json.load(f) + + self.tracker_data[tracker] = curr_data + + def get_display_name(self, tracker): + return self.tracker_to_disp[tracker] + + def _load_raw_file(self, tracker, seq, is_gt): + """Load a file (gt or tracker) in the YouTubeVIS format + If is_gt, this returns a dict which contains the fields: + [gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det). + [gt_dets]: list (for each timestep) of lists of detections. + [classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_iscrowd]: dictionary with class values + as keys and lists (for each track) as values + + if not is_gt, this returns a dict which contains the fields: + [tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det). + [tracker_dets]: list (for each timestep) of lists of detections. + [classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as + keys and corresponding segmentations as values) for each track + [classes_to_dt_track_ids, classes_to_dt_track_areas]: dictionary with class values as keys and lists as values + [classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values + """ + # select sequence tracks + seq_id = self.seq_name_to_seq_id[seq] + if is_gt: + tracks = [ann for ann in self.gt_data['annotations'] if ann['video_id'] == seq_id] + else: + tracks = self._get_tracker_seq_tracks(tracker, seq_id) + + # Convert data to required format + num_timesteps = self.seq_lengths[seq_id] + data_keys = ['ids', 'classes', 'dets'] + if not is_gt: + data_keys += ['tracker_confidences'] + raw_data = {key: [None] * num_timesteps for key in data_keys} + for t in range(num_timesteps): + raw_data['dets'][t] = [track['segmentations'][t] for track in tracks if track['segmentations'][t]] + raw_data['ids'][t] = np.atleast_1d([track['id'] for track in tracks + if track['segmentations'][t]]).astype(int) + raw_data['classes'][t] = np.atleast_1d([track['category_id'] for track in tracks + if track['segmentations'][t]]).astype(int) + if not is_gt: + raw_data['tracker_confidences'][t] = np.atleast_1d([track['score'] for track in tracks + if track['segmentations'][t]]).astype(float) + + if is_gt: + key_map = {'ids': 'gt_ids', + 'classes': 'gt_classes', + 'dets': 'gt_dets'} + else: + key_map = {'ids': 'tracker_ids', + 'classes': 'tracker_classes', + 'dets': 'tracker_dets'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + all_cls_ids = {self.class_name_to_class_id[cls] for cls in self.class_list} + classes_to_tracks = {cls: [track for track in tracks if track['category_id'] == cls] for cls in all_cls_ids} + + # mapping from classes to track representations and track information + raw_data['classes_to_tracks'] = {cls: [{i: track['segmentations'][i] + for i in range(len(track['segmentations']))} for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_ids'] = {cls: [track['id'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + raw_data['classes_to_track_areas'] = {cls: [track['area'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + raw_data['classes_to_gt_track_iscrowd'] = {cls: [track['iscrowd'] for track in tracks] + for cls, tracks in classes_to_tracks.items()} + else: + raw_data['classes_to_dt_track_scores'] = {cls: np.array([track['score'] for track in tracks]) + for cls, tracks in classes_to_tracks.items()} + + if is_gt: + key_map = {'classes_to_tracks': 'classes_to_gt_tracks', + 'classes_to_track_ids': 'classes_to_gt_track_ids', + 'classes_to_track_areas': 'classes_to_gt_track_areas'} + else: + key_map = {'classes_to_tracks': 'classes_to_dt_tracks', + 'classes_to_track_ids': 'classes_to_dt_track_ids', + 'classes_to_track_areas': 'classes_to_dt_track_areas'} + for k, v in key_map.items(): + raw_data[v] = raw_data.pop(k) + + raw_data['num_timesteps'] = num_timesteps + raw_data['seq'] = seq + return raw_data + + @_timing.time + def get_preprocessed_seq_data(self, raw_data, cls): + """ Preprocess data for a single sequence for a single class ready for evaluation. + Inputs: + - raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data(). + - cls is the class to be evaluated. + Outputs: + - data is a dict containing all of the information that metrics need to perform evaluation. + It contains the following fields: + [num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers. + [gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det). + [gt_dets, tracker_dets]: list (for each timestep) of lists of detections. + [similarity_scores]: list (for each timestep) of 2D NDArrays. + Notes: + General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps. + 1) Extract only detections relevant for the class to be evaluated (including distractor detections). + 2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a + distractor class, or otherwise marked as to be removed. + 3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain + other criteria (e.g. are too small). + 4) Remove gt dets that were only useful for preprocessing and not for actual evaluation. + After the above preprocessing steps, this function also calculates the number of gt and tracker detections + and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are + unique within each timestep. + YouTubeVIS: + In YouTubeVIS, the 4 preproc steps are as follow: + 1) There are 40 classes which are evaluated separately. + 2) No matched tracker dets are removed. + 3) No unmatched tracker dets are removed. + 4) No gt dets are removed. + Further, for TrackMAP computation track representations for the given class are accessed from a dictionary + and the tracks from the tracker data are sorted according to the tracker confidence. + """ + cls_id = self.class_name_to_class_id[cls] + + data_keys = ['gt_ids', 'tracker_ids', 'gt_dets', 'tracker_dets', 'similarity_scores'] + data = {key: [None] * raw_data['num_timesteps'] for key in data_keys} + unique_gt_ids = [] + unique_tracker_ids = [] + num_gt_dets = 0 + num_tracker_dets = 0 + + for t in range(raw_data['num_timesteps']): + + # Only extract relevant dets for this class for eval (cls) + gt_class_mask = np.atleast_1d(raw_data['gt_classes'][t] == cls_id) + gt_class_mask = gt_class_mask.astype(np.bool) + gt_ids = raw_data['gt_ids'][t][gt_class_mask] + gt_dets = [raw_data['gt_dets'][t][ind] for ind in range(len(gt_class_mask)) if gt_class_mask[ind]] + + tracker_class_mask = np.atleast_1d(raw_data['tracker_classes'][t] == cls_id) + tracker_class_mask = tracker_class_mask.astype(np.bool) + tracker_ids = raw_data['tracker_ids'][t][tracker_class_mask] + tracker_dets = [raw_data['tracker_dets'][t][ind] for ind in range(len(tracker_class_mask)) if + tracker_class_mask[ind]] + similarity_scores = raw_data['similarity_scores'][t][gt_class_mask, :][:, tracker_class_mask] + + data['tracker_ids'][t] = tracker_ids + data['tracker_dets'][t] = tracker_dets + data['gt_ids'][t] = gt_ids + data['gt_dets'][t] = gt_dets + data['similarity_scores'][t] = similarity_scores + + unique_gt_ids += list(np.unique(data['gt_ids'][t])) + unique_tracker_ids += list(np.unique(data['tracker_ids'][t])) + num_tracker_dets += len(data['tracker_ids'][t]) + num_gt_dets += len(data['gt_ids'][t]) + + # Re-label IDs such that there are no empty IDs + if len(unique_gt_ids) > 0: + unique_gt_ids = np.unique(unique_gt_ids) + gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1)) + gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['gt_ids'][t]) > 0: + data['gt_ids'][t] = gt_id_map[data['gt_ids'][t]].astype(np.int) + if len(unique_tracker_ids) > 0: + unique_tracker_ids = np.unique(unique_tracker_ids) + tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1)) + tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids)) + for t in range(raw_data['num_timesteps']): + if len(data['tracker_ids'][t]) > 0: + data['tracker_ids'][t] = tracker_id_map[data['tracker_ids'][t]].astype(np.int) + + # Ensure that ids are unique per timestep. + self._check_unique_ids(data) + + # Record overview statistics. + data['num_tracker_dets'] = num_tracker_dets + data['num_gt_dets'] = num_gt_dets + data['num_tracker_ids'] = len(unique_tracker_ids) + data['num_gt_ids'] = len(unique_gt_ids) + data['num_timesteps'] = raw_data['num_timesteps'] + data['seq'] = raw_data['seq'] + + # get track representations + data['gt_tracks'] = raw_data['classes_to_gt_tracks'][cls_id] + data['gt_track_ids'] = raw_data['classes_to_gt_track_ids'][cls_id] + data['gt_track_areas'] = raw_data['classes_to_gt_track_areas'][cls_id] + data['gt_track_iscrowd'] = raw_data['classes_to_gt_track_iscrowd'][cls_id] + data['dt_tracks'] = raw_data['classes_to_dt_tracks'][cls_id] + data['dt_track_ids'] = raw_data['classes_to_dt_track_ids'][cls_id] + data['dt_track_areas'] = raw_data['classes_to_dt_track_areas'][cls_id] + data['dt_track_scores'] = raw_data['classes_to_dt_track_scores'][cls_id] + data['iou_type'] = 'mask' + + # sort tracker data tracks by tracker confidence scores + if data['dt_tracks']: + idx = np.argsort([-score for score in data['dt_track_scores']], kind="mergesort") + data['dt_track_scores'] = [data['dt_track_scores'][i] for i in idx] + data['dt_tracks'] = [data['dt_tracks'][i] for i in idx] + data['dt_track_ids'] = [data['dt_track_ids'][i] for i in idx] + data['dt_track_areas'] = [data['dt_track_areas'][i] for i in idx] + + return data + + def _calculate_similarities(self, gt_dets_t, tracker_dets_t): + similarity_scores = self._calculate_mask_ious(gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False) + return similarity_scores + + def _prepare_gt_annotations(self): + """ + Prepares GT data by rle encoding segmentations and computing the average track area. + :return: None + """ + # only loaded when needed to reduce minimum requirements + from pycocotools import mask as mask_utils + + for track in self.gt_data['annotations']: + h = track['height'] + w = track['width'] + for i, seg in enumerate(track['segmentations']): + if seg: + track['segmentations'][i] = mask_utils.frPyObjects(seg, h, w) + areas = [a for a in track['areas'] if a] + if len(areas) == 0: + track['area'] = 0 + else: + track['area'] = np.array(areas).mean() + + def _get_tracker_seq_tracks(self, tracker, seq_id): + """ + Prepares tracker data for a given sequence. Extracts all annotations for given sequence ID, computes + average track area and assigns a track ID. + :param tracker: the given tracker + :param seq_id: the sequence ID + :return: the extracted tracks + """ + # only loaded when needed to reduce minimum requirements + from pycocotools import mask as mask_utils + + tracks = [ann for ann in self.tracker_data[tracker] if ann['video_id'] == seq_id] + for track in tracks: + track['areas'] = [] + for seg in track['segmentations']: + if seg: + track['areas'].append(mask_utils.area(seg)) + else: + track['areas'].append(None) + areas = [a for a in track['areas'] if a] + if len(areas) == 0: + track['area'] = 0 + else: + track['area'] = np.array(areas).mean() + track['id'] = self.global_tid_counter + self.global_tid_counter += 1 + return tracks diff --git a/test/yolov7-tracker/tracker/trackeval/eval.py b/test/yolov7-tracker/tracker/trackeval/eval.py new file mode 100644 index 0000000..82d62a0 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/eval.py @@ -0,0 +1,225 @@ +import time +import traceback +from multiprocessing.pool import Pool +from functools import partial +import os +from . import utils +from .utils import TrackEvalException +from . import _timing +from .metrics import Count + +try: + import tqdm + TQDM_IMPORTED = True +except ImportError as _: + TQDM_IMPORTED = False + + +class Evaluator: + """Evaluator class for evaluating different metrics for different datasets""" + + @staticmethod + def get_default_eval_config(): + """Returns the default config values for evaluation""" + code_path = utils.get_code_path() + default_config = { + 'USE_PARALLEL': False, + 'NUM_PARALLEL_CORES': 8, + 'BREAK_ON_ERROR': True, # Raises exception and exits with error + 'RETURN_ON_ERROR': False, # if not BREAK_ON_ERROR, then returns from function on error + 'LOG_ON_ERROR': os.path.join(code_path, 'error_log.txt'), # if not None, save any errors into a log file. + + 'PRINT_RESULTS': True, + 'PRINT_ONLY_COMBINED': False, + 'PRINT_CONFIG': True, + 'TIME_PROGRESS': True, + 'DISPLAY_LESS_PROGRESS': True, + + 'OUTPUT_SUMMARY': True, + 'OUTPUT_EMPTY_CLASSES': True, # If False, summary files are not output for classes with no detections + 'OUTPUT_DETAILED': True, + 'PLOT_CURVES': True, + } + return default_config + + def __init__(self, config=None): + """Initialise the evaluator with a config file""" + self.config = utils.init_config(config, self.get_default_eval_config(), 'Eval') + # Only run timing analysis if not run in parallel. + if self.config['TIME_PROGRESS'] and not self.config['USE_PARALLEL']: + _timing.DO_TIMING = True + if self.config['DISPLAY_LESS_PROGRESS']: + _timing.DISPLAY_LESS_PROGRESS = True + + @_timing.time + def evaluate(self, dataset_list, metrics_list, show_progressbar=False): + """Evaluate a set of metrics on a set of datasets""" + config = self.config + metrics_list = metrics_list + [Count()] # Count metrics are always run + metric_names = utils.validate_metrics_list(metrics_list) + dataset_names = [dataset.get_name() for dataset in dataset_list] + output_res = {} + output_msg = {} + + for dataset, dataset_name in zip(dataset_list, dataset_names): + # Get dataset info about what to evaluate + output_res[dataset_name] = {} + output_msg[dataset_name] = {} + tracker_list, seq_list, class_list = dataset.get_eval_info() + print('\nEvaluating %i tracker(s) on %i sequence(s) for %i class(es) on %s dataset using the following ' + 'metrics: %s\n' % (len(tracker_list), len(seq_list), len(class_list), dataset_name, + ', '.join(metric_names))) + + # Evaluate each tracker + for tracker in tracker_list: + # if not config['BREAK_ON_ERROR'] then go to next tracker without breaking + try: + # Evaluate each sequence in parallel or in series. + # returns a nested dict (res), indexed like: res[seq][class][metric_name][sub_metric field] + # e.g. res[seq_0001][pedestrian][hota][DetA] + print('\nEvaluating %s\n' % tracker) + time_start = time.time() + if config['USE_PARALLEL']: + if show_progressbar and TQDM_IMPORTED: + seq_list_sorted = sorted(seq_list) + + with Pool(config['NUM_PARALLEL_CORES']) as pool, tqdm.tqdm(total=len(seq_list)) as pbar: + _eval_sequence = partial(eval_sequence, dataset=dataset, tracker=tracker, + class_list=class_list, metrics_list=metrics_list, + metric_names=metric_names) + results = [] + for r in pool.imap(_eval_sequence, seq_list_sorted, + chunksize=20): + results.append(r) + pbar.update() + res = dict(zip(seq_list_sorted, results)) + + else: + with Pool(config['NUM_PARALLEL_CORES']) as pool: + _eval_sequence = partial(eval_sequence, dataset=dataset, tracker=tracker, + class_list=class_list, metrics_list=metrics_list, + metric_names=metric_names) + results = pool.map(_eval_sequence, seq_list) + res = dict(zip(seq_list, results)) + else: + res = {} + if show_progressbar and TQDM_IMPORTED: + seq_list_sorted = sorted(seq_list) + for curr_seq in tqdm.tqdm(seq_list_sorted): + res[curr_seq] = eval_sequence(curr_seq, dataset, tracker, class_list, metrics_list, + metric_names) + else: + for curr_seq in sorted(seq_list): + res[curr_seq] = eval_sequence(curr_seq, dataset, tracker, class_list, metrics_list, + metric_names) + + # Combine results over all sequences and then over all classes + + # collecting combined cls keys (cls averaged, det averaged, super classes) + combined_cls_keys = [] + res['COMBINED_SEQ'] = {} + # combine sequences for each class + for c_cls in class_list: + res['COMBINED_SEQ'][c_cls] = {} + for metric, metric_name in zip(metrics_list, metric_names): + curr_res = {seq_key: seq_value[c_cls][metric_name] for seq_key, seq_value in res.items() if + seq_key != 'COMBINED_SEQ'} + res['COMBINED_SEQ'][c_cls][metric_name] = metric.combine_sequences(curr_res) + # combine classes + if dataset.should_classes_combine: + combined_cls_keys += ['cls_comb_cls_av', 'cls_comb_det_av', 'all'] + res['COMBINED_SEQ']['cls_comb_cls_av'] = {} + res['COMBINED_SEQ']['cls_comb_det_av'] = {} + for metric, metric_name in zip(metrics_list, metric_names): + cls_res = {cls_key: cls_value[metric_name] for cls_key, cls_value in + res['COMBINED_SEQ'].items() if cls_key not in combined_cls_keys} + res['COMBINED_SEQ']['cls_comb_cls_av'][metric_name] = \ + metric.combine_classes_class_averaged(cls_res) + res['COMBINED_SEQ']['cls_comb_det_av'][metric_name] = \ + metric.combine_classes_det_averaged(cls_res) + # combine classes to super classes + if dataset.use_super_categories: + for cat, sub_cats in dataset.super_categories.items(): + combined_cls_keys.append(cat) + res['COMBINED_SEQ'][cat] = {} + for metric, metric_name in zip(metrics_list, metric_names): + cat_res = {cls_key: cls_value[metric_name] for cls_key, cls_value in + res['COMBINED_SEQ'].items() if cls_key in sub_cats} + res['COMBINED_SEQ'][cat][metric_name] = metric.combine_classes_det_averaged(cat_res) + + # Print and output results in various formats + if config['TIME_PROGRESS']: + print('\nAll sequences for %s finished in %.2f seconds' % (tracker, time.time() - time_start)) + output_fol = dataset.get_output_fol(tracker) + tracker_display_name = dataset.get_display_name(tracker) + for c_cls in res['COMBINED_SEQ'].keys(): # class_list + combined classes if calculated + summaries = [] + details = [] + num_dets = res['COMBINED_SEQ'][c_cls]['Count']['Dets'] + if config['OUTPUT_EMPTY_CLASSES'] or num_dets > 0: + for metric, metric_name in zip(metrics_list, metric_names): + # for combined classes there is no per sequence evaluation + if c_cls in combined_cls_keys: + table_res = {'COMBINED_SEQ': res['COMBINED_SEQ'][c_cls][metric_name]} + else: + table_res = {seq_key: seq_value[c_cls][metric_name] for seq_key, seq_value + in res.items()} + + if config['PRINT_RESULTS'] and config['PRINT_ONLY_COMBINED']: + dont_print = dataset.should_classes_combine and c_cls not in combined_cls_keys + if not dont_print: + metric.print_table({'COMBINED_SEQ': table_res['COMBINED_SEQ']}, + tracker_display_name, c_cls) + elif config['PRINT_RESULTS']: + metric.print_table(table_res, tracker_display_name, c_cls) + if config['OUTPUT_SUMMARY']: + summaries.append(metric.summary_results(table_res)) + if config['OUTPUT_DETAILED']: + details.append(metric.detailed_results(table_res)) + if config['PLOT_CURVES']: + metric.plot_single_tracker_results(table_res, tracker_display_name, c_cls, + output_fol) + if config['OUTPUT_SUMMARY']: + utils.write_summary_results(summaries, c_cls, output_fol) + if config['OUTPUT_DETAILED']: + utils.write_detailed_results(details, c_cls, output_fol) + + # Output for returning from function + output_res[dataset_name][tracker] = res + output_msg[dataset_name][tracker] = 'Success' + + except Exception as err: + output_res[dataset_name][tracker] = None + if type(err) == TrackEvalException: + output_msg[dataset_name][tracker] = str(err) + else: + output_msg[dataset_name][tracker] = 'Unknown error occurred.' + print('Tracker %s was unable to be evaluated.' % tracker) + print(err) + traceback.print_exc() + if config['LOG_ON_ERROR'] is not None: + with open(config['LOG_ON_ERROR'], 'a') as f: + print(dataset_name, file=f) + print(tracker, file=f) + print(traceback.format_exc(), file=f) + print('\n\n\n', file=f) + if config['BREAK_ON_ERROR']: + raise err + elif config['RETURN_ON_ERROR']: + return output_res, output_msg + + return output_res, output_msg + + +@_timing.time +def eval_sequence(seq, dataset, tracker, class_list, metrics_list, metric_names): + """Function for evaluating a single sequence""" + + raw_data = dataset.get_raw_seq_data(tracker, seq) + seq_res = {} + for cls in class_list: + seq_res[cls] = {} + data = dataset.get_preprocessed_seq_data(raw_data, cls) + for metric, met_name in zip(metrics_list, metric_names): + seq_res[cls][met_name] = metric.eval_sequence(data) + return seq_res diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/__init__.py b/test/yolov7-tracker/tracker/trackeval/metrics/__init__.py new file mode 100644 index 0000000..1f84774 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/__init__.py @@ -0,0 +1,8 @@ +from .hota import HOTA +from .clear import CLEAR +from .identity import Identity +from .count import Count +from .j_and_f import JAndF +from .track_map import TrackMAP +from .vace import VACE +from .ideucl import IDEucl \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/_base_metric.py b/test/yolov7-tracker/tracker/trackeval/metrics/_base_metric.py new file mode 100644 index 0000000..ea48885 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/_base_metric.py @@ -0,0 +1,133 @@ + +import numpy as np +from abc import ABC, abstractmethod +from .. import _timing +from ..utils import TrackEvalException + + +class _BaseMetric(ABC): + @abstractmethod + def __init__(self): + self.plottable = False + self.integer_fields = [] + self.float_fields = [] + self.array_labels = [] + self.integer_array_fields = [] + self.float_array_fields = [] + self.fields = [] + self.summary_fields = [] + self.registered = False + + ##################################################################### + # Abstract functions for subclasses to implement + + @_timing.time + @abstractmethod + def eval_sequence(self, data): + ... + + @abstractmethod + def combine_sequences(self, all_res): + ... + + @abstractmethod + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + ... + + @ abstractmethod + def combine_classes_det_averaged(self, all_res): + ... + + def plot_single_tracker_results(self, all_res, tracker, output_folder, cls): + """Plot results of metrics, only valid for metrics with self.plottable""" + if self.plottable: + raise NotImplementedError('plot_results is not implemented for metric %s' % self.get_name()) + else: + pass + + ##################################################################### + # Helper functions which are useful for all metrics: + + @classmethod + def get_name(cls): + return cls.__name__ + + @staticmethod + def _combine_sum(all_res, field): + """Combine sequence results via sum""" + return sum([all_res[k][field] for k in all_res.keys()]) + + @staticmethod + def _combine_weighted_av(all_res, field, comb_res, weight_field): + """Combine sequence results via weighted average""" + return sum([all_res[k][field] * all_res[k][weight_field] for k in all_res.keys()]) / np.maximum(1.0, comb_res[ + weight_field]) + + def print_table(self, table_res, tracker, cls): + """Prints table of results for all sequences""" + print('') + metric_name = self.get_name() + self._row_print([metric_name + ': ' + tracker + '-' + cls] + self.summary_fields) + for seq, results in sorted(table_res.items()): + if seq == 'COMBINED_SEQ': + continue + summary_res = self._summary_row(results) + self._row_print([seq] + summary_res) + summary_res = self._summary_row(table_res['COMBINED_SEQ']) + self._row_print(['COMBINED'] + summary_res) + + def _summary_row(self, results_): + vals = [] + for h in self.summary_fields: + if h in self.float_array_fields: + vals.append("{0:1.5g}".format(100 * np.mean(results_[h]))) + elif h in self.float_fields: + vals.append("{0:1.5g}".format(100 * float(results_[h]))) + elif h in self.integer_fields: + vals.append("{0:d}".format(int(results_[h]))) + else: + raise NotImplementedError("Summary function not implemented for this field type.") + return vals + + @staticmethod + def _row_print(*argv): + """Prints results in an evenly spaced rows, with more space in first row""" + if len(argv) == 1: + argv = argv[0] + to_print = '%-35s' % argv[0] + for v in argv[1:]: + to_print += '%-10s' % str(v) + print(to_print) + + def summary_results(self, table_res): + """Returns a simple summary of final results for a tracker""" + return dict(zip(self.summary_fields, self._summary_row(table_res['COMBINED_SEQ']))) + + def detailed_results(self, table_res): + """Returns detailed final results for a tracker""" + # Get detailed field information + detailed_fields = self.float_fields + self.integer_fields + for h in self.float_array_fields + self.integer_array_fields: + for alpha in [int(100*x) for x in self.array_labels]: + detailed_fields.append(h + '___' + str(alpha)) + detailed_fields.append(h + '___AUC') + + # Get detailed results + detailed_results = {} + for seq, res in table_res.items(): + detailed_row = self._detailed_row(res) + if len(detailed_row) != len(detailed_fields): + raise TrackEvalException( + 'Field names and data have different sizes (%i and %i)' % (len(detailed_row), len(detailed_fields))) + detailed_results[seq] = dict(zip(detailed_fields, detailed_row)) + return detailed_results + + def _detailed_row(self, res): + detailed_row = [] + for h in self.float_fields + self.integer_fields: + detailed_row.append(res[h]) + for h in self.float_array_fields + self.integer_array_fields: + for i, alpha in enumerate([int(100 * x) for x in self.array_labels]): + detailed_row.append(res[h][i]) + detailed_row.append(np.mean(res[h])) + return detailed_row diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/clear.py b/test/yolov7-tracker/tracker/trackeval/metrics/clear.py new file mode 100644 index 0000000..8b5e291 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/clear.py @@ -0,0 +1,186 @@ + +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_metric import _BaseMetric +from .. import _timing +from .. import utils + +class CLEAR(_BaseMetric): + """Class which implements the CLEAR metrics""" + + @staticmethod + def get_default_config(): + """Default class config values""" + default_config = { + 'THRESHOLD': 0.5, # Similarity score threshold required for a TP match. Default 0.5. + 'PRINT_CONFIG': True, # Whether to print the config information on init. Default: False. + } + return default_config + + def __init__(self, config=None): + super().__init__() + main_integer_fields = ['CLR_TP', 'CLR_FN', 'CLR_FP', 'IDSW', 'MT', 'PT', 'ML', 'Frag'] + extra_integer_fields = ['CLR_Frames'] + self.integer_fields = main_integer_fields + extra_integer_fields + main_float_fields = ['MOTA', 'MOTP', 'MODA', 'CLR_Re', 'CLR_Pr', 'MTR', 'PTR', 'MLR', 'sMOTA'] + extra_float_fields = ['CLR_F1', 'FP_per_frame', 'MOTAL', 'MOTP_sum'] + self.float_fields = main_float_fields + extra_float_fields + self.fields = self.float_fields + self.integer_fields + self.summed_fields = self.integer_fields + ['MOTP_sum'] + self.summary_fields = main_float_fields + main_integer_fields + + # Configuration options: + self.config = utils.init_config(config, self.get_default_config(), self.get_name()) + self.threshold = float(self.config['THRESHOLD']) + + + @_timing.time + def eval_sequence(self, data): + """Calculates CLEAR metrics for one sequence""" + # Initialise results + res = {} + for field in self.fields: + res[field] = 0 + + # Return result quickly if tracker or gt sequence is empty + if data['num_tracker_dets'] == 0: + res['CLR_FN'] = data['num_gt_dets'] + res['ML'] = data['num_gt_ids'] + res['MLR'] = 1.0 + return res + if data['num_gt_dets'] == 0: + res['CLR_FP'] = data['num_tracker_dets'] + res['MLR'] = 1.0 + return res + + # Variables counting global association + num_gt_ids = data['num_gt_ids'] + gt_id_count = np.zeros(num_gt_ids) # For MT/ML/PT + gt_matched_count = np.zeros(num_gt_ids) # For MT/ML/PT + gt_frag_count = np.zeros(num_gt_ids) # For Frag + + # Note that IDSWs are counted based on the last time each gt_id was present (any number of frames previously), + # but are only used in matching to continue current tracks based on the gt_id in the single previous timestep. + prev_tracker_id = np.nan * np.zeros(num_gt_ids) # For scoring IDSW + prev_timestep_tracker_id = np.nan * np.zeros(num_gt_ids) # For matching IDSW + + # Calculate scores for each timestep + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + # Deal with the case that there are no gt_det/tracker_det in a timestep. + if len(gt_ids_t) == 0: + res['CLR_FP'] += len(tracker_ids_t) + continue + if len(tracker_ids_t) == 0: + res['CLR_FN'] += len(gt_ids_t) + gt_id_count[gt_ids_t] += 1 + continue + + # Calc score matrix to first minimise IDSWs from previous frame, and then maximise MOTP secondarily + similarity = data['similarity_scores'][t] + score_mat = (tracker_ids_t[np.newaxis, :] == prev_timestep_tracker_id[gt_ids_t[:, np.newaxis]]) + score_mat = 1000 * score_mat + similarity + score_mat[similarity < self.threshold - np.finfo('float').eps] = 0 + + # Hungarian algorithm to find best matches + match_rows, match_cols = linear_sum_assignment(-score_mat) + actually_matched_mask = score_mat[match_rows, match_cols] > 0 + np.finfo('float').eps + match_rows = match_rows[actually_matched_mask] + match_cols = match_cols[actually_matched_mask] + + matched_gt_ids = gt_ids_t[match_rows] + matched_tracker_ids = tracker_ids_t[match_cols] + + # Calc IDSW for MOTA + prev_matched_tracker_ids = prev_tracker_id[matched_gt_ids] + is_idsw = (np.logical_not(np.isnan(prev_matched_tracker_ids))) & ( + np.not_equal(matched_tracker_ids, prev_matched_tracker_ids)) + res['IDSW'] += np.sum(is_idsw) + + # Update counters for MT/ML/PT/Frag and record for IDSW/Frag for next timestep + gt_id_count[gt_ids_t] += 1 + gt_matched_count[matched_gt_ids] += 1 + not_previously_tracked = np.isnan(prev_timestep_tracker_id) + prev_tracker_id[matched_gt_ids] = matched_tracker_ids + prev_timestep_tracker_id[:] = np.nan + prev_timestep_tracker_id[matched_gt_ids] = matched_tracker_ids + currently_tracked = np.logical_not(np.isnan(prev_timestep_tracker_id)) + gt_frag_count += np.logical_and(not_previously_tracked, currently_tracked) + + # Calculate and accumulate basic statistics + num_matches = len(matched_gt_ids) + res['CLR_TP'] += num_matches + res['CLR_FN'] += len(gt_ids_t) - num_matches + res['CLR_FP'] += len(tracker_ids_t) - num_matches + if num_matches > 0: + res['MOTP_sum'] += sum(similarity[match_rows, match_cols]) + + # Calculate MT/ML/PT/Frag/MOTP + tracked_ratio = gt_matched_count[gt_id_count > 0] / gt_id_count[gt_id_count > 0] + res['MT'] = np.sum(np.greater(tracked_ratio, 0.8)) + res['PT'] = np.sum(np.greater_equal(tracked_ratio, 0.2)) - res['MT'] + res['ML'] = num_gt_ids - res['MT'] - res['PT'] + res['Frag'] = np.sum(np.subtract(gt_frag_count[gt_frag_count > 0], 1)) + res['MOTP'] = res['MOTP_sum'] / np.maximum(1.0, res['CLR_TP']) + + res['CLR_Frames'] = data['num_timesteps'] + + # Calculate final CLEAR scores + res = self._compute_final_fields(res) + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for field in self.summed_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self.summed_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + """Combines metrics across all classes by averaging over the class values. + If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection. + """ + res = {} + for field in self.integer_fields: + if ignore_empty_classes: + res[field] = self._combine_sum( + {k: v for k, v in all_res.items() if v['CLR_TP'] + v['CLR_FN'] + v['CLR_FP'] > 0}, field) + else: + res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field) + for field in self.float_fields: + if ignore_empty_classes: + res[field] = np.mean( + [v[field] for v in all_res.values() if v['CLR_TP'] + v['CLR_FN'] + v['CLR_FP'] > 0], axis=0) + else: + res[field] = np.mean([v[field] for v in all_res.values()], axis=0) + return res + + @staticmethod + def _compute_final_fields(res): + """Calculate sub-metric ('field') values which only depend on other sub-metric values. + This function is used both for both per-sequence calculation, and in combining values across sequences. + """ + num_gt_ids = res['MT'] + res['ML'] + res['PT'] + res['MTR'] = res['MT'] / np.maximum(1.0, num_gt_ids) + res['MLR'] = res['ML'] / np.maximum(1.0, num_gt_ids) + res['PTR'] = res['PT'] / np.maximum(1.0, num_gt_ids) + res['CLR_Re'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN']) + res['CLR_Pr'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + res['CLR_FP']) + res['MODA'] = (res['CLR_TP'] - res['CLR_FP']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN']) + res['MOTA'] = (res['CLR_TP'] - res['CLR_FP'] - res['IDSW']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN']) + res['MOTP'] = res['MOTP_sum'] / np.maximum(1.0, res['CLR_TP']) + res['sMOTA'] = (res['MOTP_sum'] - res['CLR_FP'] - res['IDSW']) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN']) + + res['CLR_F1'] = res['CLR_TP'] / np.maximum(1.0, res['CLR_TP'] + 0.5*res['CLR_FN'] + 0.5*res['CLR_FP']) + res['FP_per_frame'] = res['CLR_FP'] / np.maximum(1.0, res['CLR_Frames']) + safe_log_idsw = np.log10(res['IDSW']) if res['IDSW'] > 0 else res['IDSW'] + res['MOTAL'] = (res['CLR_TP'] - res['CLR_FP'] - safe_log_idsw) / np.maximum(1.0, res['CLR_TP'] + res['CLR_FN']) + return res diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/count.py b/test/yolov7-tracker/tracker/trackeval/metrics/count.py new file mode 100644 index 0000000..49049b1 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/count.py @@ -0,0 +1,44 @@ + +from ._base_metric import _BaseMetric +from .. import _timing + + +class Count(_BaseMetric): + """Class which simply counts the number of tracker and gt detections and ids.""" + def __init__(self, config=None): + super().__init__() + self.integer_fields = ['Dets', 'GT_Dets', 'IDs', 'GT_IDs'] + self.fields = self.integer_fields + self.summary_fields = self.fields + + @_timing.time + def eval_sequence(self, data): + """Returns counts for one sequence""" + # Get results + res = {'Dets': data['num_tracker_dets'], + 'GT_Dets': data['num_gt_dets'], + 'IDs': data['num_tracker_ids'], + 'GT_IDs': data['num_gt_ids'], + 'Frames': data['num_timesteps']} + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for field in self.integer_fields: + res[field] = self._combine_sum(all_res, field) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=None): + """Combines metrics across all classes by averaging over the class values""" + res = {} + for field in self.integer_fields: + res[field] = self._combine_sum(all_res, field) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self.integer_fields: + res[field] = self._combine_sum(all_res, field) + return res diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/hota.py b/test/yolov7-tracker/tracker/trackeval/metrics/hota.py new file mode 100644 index 0000000..f551b76 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/hota.py @@ -0,0 +1,203 @@ + +import os +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_metric import _BaseMetric +from .. import _timing + + +class HOTA(_BaseMetric): + """Class which implements the HOTA metrics. + See: https://link.springer.com/article/10.1007/s11263-020-01375-2 + """ + + def __init__(self, config=None): + super().__init__() + self.plottable = True + self.array_labels = np.arange(0.05, 0.99, 0.05) + self.integer_array_fields = ['HOTA_TP', 'HOTA_FN', 'HOTA_FP'] + self.float_array_fields = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA', 'OWTA'] + self.float_fields = ['HOTA(0)', 'LocA(0)', 'HOTALocA(0)'] + self.fields = self.float_array_fields + self.integer_array_fields + self.float_fields + self.summary_fields = self.float_array_fields + self.float_fields + + @_timing.time + def eval_sequence(self, data): + """Calculates the HOTA metrics for one sequence""" + + # Initialise results + res = {} + for field in self.float_array_fields + self.integer_array_fields: + res[field] = np.zeros((len(self.array_labels)), dtype=np.float) + for field in self.float_fields: + res[field] = 0 + + # Return result quickly if tracker or gt sequence is empty + if data['num_tracker_dets'] == 0: + res['HOTA_FN'] = data['num_gt_dets'] * np.ones((len(self.array_labels)), dtype=np.float) + res['LocA'] = np.ones((len(self.array_labels)), dtype=np.float) + res['LocA(0)'] = 1.0 + return res + if data['num_gt_dets'] == 0: + res['HOTA_FP'] = data['num_tracker_dets'] * np.ones((len(self.array_labels)), dtype=np.float) + res['LocA'] = np.ones((len(self.array_labels)), dtype=np.float) + res['LocA(0)'] = 1.0 + return res + + # Variables counting global association + potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids'])) + gt_id_count = np.zeros((data['num_gt_ids'], 1)) + tracker_id_count = np.zeros((1, data['num_tracker_ids'])) + + # First loop through each timestep and accumulate global track information. + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + # Count the potential matches between ids in each timestep + # These are normalised, weighted by the match similarity. + similarity = data['similarity_scores'][t] + sim_iou_denom = similarity.sum(0)[np.newaxis, :] + similarity.sum(1)[:, np.newaxis] - similarity + sim_iou = np.zeros_like(similarity) + sim_iou_mask = sim_iou_denom > 0 + np.finfo('float').eps + sim_iou[sim_iou_mask] = similarity[sim_iou_mask] / sim_iou_denom[sim_iou_mask] + potential_matches_count[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] += sim_iou + + # Calculate the total number of dets for each gt_id and tracker_id. + gt_id_count[gt_ids_t] += 1 + tracker_id_count[0, tracker_ids_t] += 1 + + # Calculate overall jaccard alignment score (before unique matching) between IDs + global_alignment_score = potential_matches_count / (gt_id_count + tracker_id_count - potential_matches_count) + matches_counts = [np.zeros_like(potential_matches_count) for _ in self.array_labels] + + # Calculate scores for each timestep + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + # Deal with the case that there are no gt_det/tracker_det in a timestep. + if len(gt_ids_t) == 0: + for a, alpha in enumerate(self.array_labels): + res['HOTA_FP'][a] += len(tracker_ids_t) + continue + if len(tracker_ids_t) == 0: + for a, alpha in enumerate(self.array_labels): + res['HOTA_FN'][a] += len(gt_ids_t) + continue + + # Get matching scores between pairs of dets for optimizing HOTA + similarity = data['similarity_scores'][t] + score_mat = global_alignment_score[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] * similarity + + # Hungarian algorithm to find best matches + match_rows, match_cols = linear_sum_assignment(-score_mat) + + # Calculate and accumulate basic statistics + for a, alpha in enumerate(self.array_labels): + actually_matched_mask = similarity[match_rows, match_cols] >= alpha - np.finfo('float').eps + alpha_match_rows = match_rows[actually_matched_mask] + alpha_match_cols = match_cols[actually_matched_mask] + num_matches = len(alpha_match_rows) + res['HOTA_TP'][a] += num_matches + res['HOTA_FN'][a] += len(gt_ids_t) - num_matches + res['HOTA_FP'][a] += len(tracker_ids_t) - num_matches + if num_matches > 0: + res['LocA'][a] += sum(similarity[alpha_match_rows, alpha_match_cols]) + matches_counts[a][gt_ids_t[alpha_match_rows], tracker_ids_t[alpha_match_cols]] += 1 + + # Calculate association scores (AssA, AssRe, AssPr) for the alpha value. + # First calculate scores per gt_id/tracker_id combo and then average over the number of detections. + for a, alpha in enumerate(self.array_labels): + matches_count = matches_counts[a] + ass_a = matches_count / np.maximum(1, gt_id_count + tracker_id_count - matches_count) + res['AssA'][a] = np.sum(matches_count * ass_a) / np.maximum(1, res['HOTA_TP'][a]) + ass_re = matches_count / np.maximum(1, gt_id_count) + res['AssRe'][a] = np.sum(matches_count * ass_re) / np.maximum(1, res['HOTA_TP'][a]) + ass_pr = matches_count / np.maximum(1, tracker_id_count) + res['AssPr'][a] = np.sum(matches_count * ass_pr) / np.maximum(1, res['HOTA_TP'][a]) + + # Calculate final scores + res['LocA'] = np.maximum(1e-10, res['LocA']) / np.maximum(1e-10, res['HOTA_TP']) + res = self._compute_final_fields(res) + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for field in self.integer_array_fields: + res[field] = self._combine_sum(all_res, field) + for field in ['AssRe', 'AssPr', 'AssA']: + res[field] = self._combine_weighted_av(all_res, field, res, weight_field='HOTA_TP') + loca_weighted_sum = sum([all_res[k]['LocA'] * all_res[k]['HOTA_TP'] for k in all_res.keys()]) + res['LocA'] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(1e-10, res['HOTA_TP']) + res = self._compute_final_fields(res) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + """Combines metrics across all classes by averaging over the class values. + If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection. + """ + res = {} + for field in self.integer_array_fields: + if ignore_empty_classes: + res[field] = self._combine_sum( + {k: v for k, v in all_res.items() + if (v['HOTA_TP'] + v['HOTA_FN'] + v['HOTA_FP'] > 0 + np.finfo('float').eps).any()}, field) + else: + res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field) + + for field in self.float_fields + self.float_array_fields: + if ignore_empty_classes: + res[field] = np.mean([v[field] for v in all_res.values() if + (v['HOTA_TP'] + v['HOTA_FN'] + v['HOTA_FP'] > 0 + np.finfo('float').eps).any()], + axis=0) + else: + res[field] = np.mean([v[field] for v in all_res.values()], axis=0) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self.integer_array_fields: + res[field] = self._combine_sum(all_res, field) + for field in ['AssRe', 'AssPr', 'AssA']: + res[field] = self._combine_weighted_av(all_res, field, res, weight_field='HOTA_TP') + loca_weighted_sum = sum([all_res[k]['LocA'] * all_res[k]['HOTA_TP'] for k in all_res.keys()]) + res['LocA'] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(1e-10, res['HOTA_TP']) + res = self._compute_final_fields(res) + return res + + @staticmethod + def _compute_final_fields(res): + """Calculate sub-metric ('field') values which only depend on other sub-metric values. + This function is used both for both per-sequence calculation, and in combining values across sequences. + """ + res['DetRe'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FN']) + res['DetPr'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FP']) + res['DetA'] = res['HOTA_TP'] / np.maximum(1, res['HOTA_TP'] + res['HOTA_FN'] + res['HOTA_FP']) + res['HOTA'] = np.sqrt(res['DetA'] * res['AssA']) + res['OWTA'] = np.sqrt(res['DetRe'] * res['AssA']) + + res['HOTA(0)'] = res['HOTA'][0] + res['LocA(0)'] = res['LocA'][0] + res['HOTALocA(0)'] = res['HOTA(0)']*res['LocA(0)'] + return res + + def plot_single_tracker_results(self, table_res, tracker, cls, output_folder): + """Create plot of results""" + + # Only loaded when run to reduce minimum requirements + from matplotlib import pyplot as plt + + res = table_res['COMBINED_SEQ'] + styles_to_plot = ['r', 'b', 'g', 'b--', 'b:', 'g--', 'g:', 'm'] + for name, style in zip(self.float_array_fields, styles_to_plot): + plt.plot(self.array_labels, res[name], style) + plt.xlabel('alpha') + plt.ylabel('score') + plt.title(tracker + ' - ' + cls) + plt.axis([0, 1, 0, 1]) + legend = [] + for name in self.float_array_fields: + legend += [name + ' (' + str(np.round(np.mean(res[name]), 2)) + ')'] + plt.legend(legend, loc='lower left') + out_file = os.path.join(output_folder, cls + '_plot.pdf') + os.makedirs(os.path.dirname(out_file), exist_ok=True) + plt.savefig(out_file) + plt.savefig(out_file.replace('.pdf', '.png')) + plt.clf() diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/identity.py b/test/yolov7-tracker/tracker/trackeval/metrics/identity.py new file mode 100644 index 0000000..c8c6c80 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/identity.py @@ -0,0 +1,135 @@ +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_metric import _BaseMetric +from .. import _timing +from .. import utils + + +class Identity(_BaseMetric): + """Class which implements the ID metrics""" + + @staticmethod + def get_default_config(): + """Default class config values""" + default_config = { + 'THRESHOLD': 0.5, # Similarity score threshold required for a IDTP match. Default 0.5. + 'PRINT_CONFIG': True, # Whether to print the config information on init. Default: False. + } + return default_config + + def __init__(self, config=None): + super().__init__() + self.integer_fields = ['IDTP', 'IDFN', 'IDFP'] + self.float_fields = ['IDF1', 'IDR', 'IDP'] + self.fields = self.float_fields + self.integer_fields + self.summary_fields = self.fields + + # Configuration options: + self.config = utils.init_config(config, self.get_default_config(), self.get_name()) + self.threshold = float(self.config['THRESHOLD']) + + @_timing.time + def eval_sequence(self, data): + """Calculates ID metrics for one sequence""" + # Initialise results + res = {} + for field in self.fields: + res[field] = 0 + + # Return result quickly if tracker or gt sequence is empty + if data['num_tracker_dets'] == 0: + res['IDFN'] = data['num_gt_dets'] + return res + if data['num_gt_dets'] == 0: + res['IDFP'] = data['num_tracker_dets'] + return res + + # Variables counting global association + potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids'])) + gt_id_count = np.zeros(data['num_gt_ids']) + tracker_id_count = np.zeros(data['num_tracker_ids']) + + # First loop through each timestep and accumulate global track information. + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + # Count the potential matches between ids in each timestep + matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold) + match_idx_gt, match_idx_tracker = np.nonzero(matches_mask) + potential_matches_count[gt_ids_t[match_idx_gt], tracker_ids_t[match_idx_tracker]] += 1 + + # Calculate the total number of dets for each gt_id and tracker_id. + gt_id_count[gt_ids_t] += 1 + tracker_id_count[tracker_ids_t] += 1 + + # Calculate optimal assignment cost matrix for ID metrics + num_gt_ids = data['num_gt_ids'] + num_tracker_ids = data['num_tracker_ids'] + fp_mat = np.zeros((num_gt_ids + num_tracker_ids, num_gt_ids + num_tracker_ids)) + fn_mat = np.zeros((num_gt_ids + num_tracker_ids, num_gt_ids + num_tracker_ids)) + fp_mat[num_gt_ids:, :num_tracker_ids] = 1e10 + fn_mat[:num_gt_ids, num_tracker_ids:] = 1e10 + for gt_id in range(num_gt_ids): + fn_mat[gt_id, :num_tracker_ids] = gt_id_count[gt_id] + fn_mat[gt_id, num_tracker_ids + gt_id] = gt_id_count[gt_id] + for tracker_id in range(num_tracker_ids): + fp_mat[:num_gt_ids, tracker_id] = tracker_id_count[tracker_id] + fp_mat[tracker_id + num_gt_ids, tracker_id] = tracker_id_count[tracker_id] + fn_mat[:num_gt_ids, :num_tracker_ids] -= potential_matches_count + fp_mat[:num_gt_ids, :num_tracker_ids] -= potential_matches_count + + # Hungarian algorithm + match_rows, match_cols = linear_sum_assignment(fn_mat + fp_mat) + + # Accumulate basic statistics + res['IDFN'] = fn_mat[match_rows, match_cols].sum().astype(np.int) + res['IDFP'] = fp_mat[match_rows, match_cols].sum().astype(np.int) + res['IDTP'] = (gt_id_count.sum() - res['IDFN']).astype(np.int) + + # Calculate final ID scores + res = self._compute_final_fields(res) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + """Combines metrics across all classes by averaging over the class values. + If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection. + """ + res = {} + for field in self.integer_fields: + if ignore_empty_classes: + res[field] = self._combine_sum({k: v for k, v in all_res.items() + if v['IDTP'] + v['IDFN'] + v['IDFP'] > 0 + np.finfo('float').eps}, + field) + else: + res[field] = self._combine_sum({k: v for k, v in all_res.items()}, field) + for field in self.float_fields: + if ignore_empty_classes: + res[field] = np.mean([v[field] for v in all_res.values() + if v['IDTP'] + v['IDFN'] + v['IDFP'] > 0 + np.finfo('float').eps], axis=0) + else: + res[field] = np.mean([v[field] for v in all_res.values()], axis=0) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self.integer_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res) + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for field in self.integer_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res) + return res + + @staticmethod + def _compute_final_fields(res): + """Calculate sub-metric ('field') values which only depend on other sub-metric values. + This function is used both for both per-sequence calculation, and in combining values across sequences. + """ + res['IDR'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + res['IDFN']) + res['IDP'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + res['IDFP']) + res['IDF1'] = res['IDTP'] / np.maximum(1.0, res['IDTP'] + 0.5 * res['IDFP'] + 0.5 * res['IDFN']) + return res diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/ideucl.py b/test/yolov7-tracker/tracker/trackeval/metrics/ideucl.py new file mode 100644 index 0000000..db9b57b --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/ideucl.py @@ -0,0 +1,135 @@ +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_metric import _BaseMetric +from .. import _timing +from collections import defaultdict +from .. import utils + + +class IDEucl(_BaseMetric): + """Class which implements the ID metrics""" + + @staticmethod + def get_default_config(): + """Default class config values""" + default_config = { + 'THRESHOLD': 0.4, # Similarity score threshold required for a IDTP match. 0.4 for IDEucl. + 'PRINT_CONFIG': True, # Whether to print the config information on init. Default: False. + } + return default_config + + def __init__(self, config=None): + super().__init__() + self.fields = ['IDEucl'] + self.float_fields = self.fields + self.summary_fields = self.fields + + # Configuration options: + self.config = utils.init_config(config, self.get_default_config(), self.get_name()) + self.threshold = float(self.config['THRESHOLD']) + + + @_timing.time + def eval_sequence(self, data): + """Calculates IDEucl metrics for all frames""" + # Initialise results + res = {'IDEucl' : 0} + + # Return result quickly if tracker or gt sequence is empty + if data['num_tracker_dets'] == 0 or data['num_gt_dets'] == 0.: + return res + + data['centroid'] = [] + for t, gt_det in enumerate(data['gt_dets']): + # import pdb;pdb.set_trace() + data['centroid'].append(self._compute_centroid(gt_det)) + + oid_hid_cent = defaultdict(list) + oid_cent = defaultdict(list) + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold) + + # I hope the orders of ids and boxes are maintained in `data` + for ind, gid in enumerate(gt_ids_t): + oid_cent[gid].append(data['centroid'][t][ind]) + + match_idx_gt, match_idx_tracker = np.nonzero(matches_mask) + for m_gid, m_tid in zip(match_idx_gt, match_idx_tracker): + oid_hid_cent[gt_ids_t[m_gid], tracker_ids_t[m_tid]].append(data['centroid'][t][m_gid]) + + oid_hid_dist = {k : np.sum(np.linalg.norm(np.diff(np.array(v), axis=0), axis=1)) for k, v in oid_hid_cent.items()} + oid_dist = {int(k) : np.sum(np.linalg.norm(np.diff(np.array(v), axis=0), axis=1)) for k, v in oid_cent.items()} + + unique_oid = np.unique([i[0] for i in oid_hid_dist.keys()]).tolist() + unique_hid = np.unique([i[1] for i in oid_hid_dist.keys()]).tolist() + o_len = len(unique_oid) + h_len = len(unique_hid) + dist_matrix = np.zeros((o_len, h_len)) + for ((oid, hid), dist) in oid_hid_dist.items(): + oid_ind = unique_oid.index(oid) + hid_ind = unique_hid.index(hid) + dist_matrix[oid_ind, hid_ind] = dist + + # opt_hyp_dist contains GT ID : max dist covered by track + opt_hyp_dist = dict.fromkeys(oid_dist.keys(), 0.) + cost_matrix = np.max(dist_matrix) - dist_matrix + rows, cols = linear_sum_assignment(cost_matrix) + for (row, col) in zip(rows, cols): + value = dist_matrix[row, col] + opt_hyp_dist[int(unique_oid[row])] = value + + assert len(opt_hyp_dist.keys()) == len(oid_dist.keys()) + hyp_length = np.sum(list(opt_hyp_dist.values())) + gt_length = np.sum(list(oid_dist.values())) + id_eucl =np.mean([np.divide(a, b, out=np.zeros_like(a), where=b!=0) for a, b in zip(opt_hyp_dist.values(), oid_dist.values())]) + res['IDEucl'] = np.divide(hyp_length, gt_length, out=np.zeros_like(hyp_length), where=gt_length!=0) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + """Combines metrics across all classes by averaging over the class values. + If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection. + """ + res = {} + + for field in self.float_fields: + if ignore_empty_classes: + res[field] = np.mean([v[field] for v in all_res.values() + if v['IDEucl'] > 0 + np.finfo('float').eps], axis=0) + else: + res[field] = np.mean([v[field] for v in all_res.values()], axis=0) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self.float_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res, len(all_res)) + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for field in self.float_fields: + res[field] = self._combine_sum(all_res, field) + res = self._compute_final_fields(res, len(all_res)) + return res + + + @staticmethod + def _compute_centroid(box): + box = np.array(box) + if len(box.shape) == 1: + centroid = (box[0:2] + box[2:4])/2 + else: + centroid = (box[:, 0:2] + box[:, 2:4])/2 + return np.flip(centroid, axis=1) + + + @staticmethod + def _compute_final_fields(res, res_len): + """ + Exists only to match signature with the original Identiy class. + + """ + return {k:v/res_len for k,v in res.items()} diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/j_and_f.py b/test/yolov7-tracker/tracker/trackeval/metrics/j_and_f.py new file mode 100644 index 0000000..1b18f04 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/j_and_f.py @@ -0,0 +1,310 @@ + +import numpy as np +import math +from scipy.optimize import linear_sum_assignment +from ..utils import TrackEvalException +from ._base_metric import _BaseMetric +from .. import _timing + + +class JAndF(_BaseMetric): + """Class which implements the J&F metrics""" + def __init__(self, config=None): + super().__init__() + self.integer_fields = ['num_gt_tracks'] + self.float_fields = ['J-Mean', 'J-Recall', 'J-Decay', 'F-Mean', 'F-Recall', 'F-Decay', 'J&F'] + self.fields = self.float_fields + self.integer_fields + self.summary_fields = self.float_fields + self.optim_type = 'J' # possible values J, J&F + + @_timing.time + def eval_sequence(self, data): + """Returns J&F metrics for one sequence""" + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + num_timesteps = data['num_timesteps'] + num_tracker_ids = data['num_tracker_ids'] + num_gt_ids = data['num_gt_ids'] + gt_dets = data['gt_dets'] + tracker_dets = data['tracker_dets'] + gt_ids = data['gt_ids'] + tracker_ids = data['tracker_ids'] + + # get shape of frames + frame_shape = None + if num_gt_ids > 0: + for t in range(num_timesteps): + if len(gt_ids[t]) > 0: + frame_shape = gt_dets[t][0]['size'] + break + elif num_tracker_ids > 0: + for t in range(num_timesteps): + if len(tracker_ids[t]) > 0: + frame_shape = tracker_dets[t][0]['size'] + break + + if frame_shape: + # append all zero masks for timesteps in which tracks do not have a detection + zero_padding = np.zeros((frame_shape), order= 'F').astype(np.uint8) + padding_mask = mask_utils.encode(zero_padding) + for t in range(num_timesteps): + gt_id_det_mapping = {gt_ids[t][i]: gt_dets[t][i] for i in range(len(gt_ids[t]))} + gt_dets[t] = [gt_id_det_mapping[index] if index in gt_ids[t] else padding_mask for index + in range(num_gt_ids)] + tracker_id_det_mapping = {tracker_ids[t][i]: tracker_dets[t][i] for i in range(len(tracker_ids[t]))} + tracker_dets[t] = [tracker_id_det_mapping[index] if index in tracker_ids[t] else padding_mask for index + in range(num_tracker_ids)] + # also perform zero padding if number of tracker IDs < number of ground truth IDs + if num_tracker_ids < num_gt_ids: + diff = num_gt_ids - num_tracker_ids + for t in range(num_timesteps): + tracker_dets[t] = tracker_dets[t] + [padding_mask for _ in range(diff)] + num_tracker_ids += diff + + j = self._compute_j(gt_dets, tracker_dets, num_gt_ids, num_tracker_ids, num_timesteps) + + # boundary threshold for F computation + bound_th = 0.008 + + # perform matching + if self.optim_type == 'J&F': + f = np.zeros_like(j) + for k in range(num_tracker_ids): + for i in range(num_gt_ids): + f[k, i, :] = self._compute_f(gt_dets, tracker_dets, k, i, bound_th) + optim_metrics = (np.mean(j, axis=2) + np.mean(f, axis=2)) / 2 + row_ind, col_ind = linear_sum_assignment(- optim_metrics) + j_m = j[row_ind, col_ind, :] + f_m = f[row_ind, col_ind, :] + elif self.optim_type == 'J': + optim_metrics = np.mean(j, axis=2) + row_ind, col_ind = linear_sum_assignment(- optim_metrics) + j_m = j[row_ind, col_ind, :] + f_m = np.zeros_like(j_m) + for i, (tr_ind, gt_ind) in enumerate(zip(row_ind, col_ind)): + f_m[i] = self._compute_f(gt_dets, tracker_dets, tr_ind, gt_ind, bound_th) + else: + raise TrackEvalException('Unsupported optimization type %s for J&F metric.' % self.optim_type) + + # append zeros for false negatives + if j_m.shape[0] < data['num_gt_ids']: + diff = data['num_gt_ids'] - j_m.shape[0] + j_m = np.concatenate((j_m, np.zeros((diff, j_m.shape[1]))), axis=0) + f_m = np.concatenate((f_m, np.zeros((diff, f_m.shape[1]))), axis=0) + + # compute the metrics for each ground truth track + res = { + 'J-Mean': [np.nanmean(j_m[i, :]) for i in range(j_m.shape[0])], + 'J-Recall': [np.nanmean(j_m[i, :] > 0.5 + np.finfo('float').eps) for i in range(j_m.shape[0])], + 'F-Mean': [np.nanmean(f_m[i, :]) for i in range(f_m.shape[0])], + 'F-Recall': [np.nanmean(f_m[i, :] > 0.5 + np.finfo('float').eps) for i in range(f_m.shape[0])], + 'J-Decay': [], + 'F-Decay': [] + } + n_bins = 4 + ids = np.round(np.linspace(1, data['num_timesteps'], n_bins + 1) + 1e-10) - 1 + ids = ids.astype(np.uint8) + + for k in range(j_m.shape[0]): + d_bins_j = [j_m[k][ids[i]:ids[i + 1] + 1] for i in range(0, n_bins)] + res['J-Decay'].append(np.nanmean(d_bins_j[0]) - np.nanmean(d_bins_j[3])) + for k in range(f_m.shape[0]): + d_bins_f = [f_m[k][ids[i]:ids[i + 1] + 1] for i in range(0, n_bins)] + res['F-Decay'].append(np.nanmean(d_bins_f[0]) - np.nanmean(d_bins_f[3])) + + # count number of tracks for weighting of the result + res['num_gt_tracks'] = len(res['J-Mean']) + for field in ['J-Mean', 'J-Recall', 'J-Decay', 'F-Mean', 'F-Recall', 'F-Decay']: + res[field] = np.mean(res[field]) + res['J&F'] = (res['J-Mean'] + res['F-Mean']) / 2 + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')} + for field in self.summary_fields: + res[field] = self._combine_weighted_av(all_res, field, res, weight_field='num_gt_tracks') + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): + """Combines metrics across all classes by averaging over the class values + 'ignore empty classes' is not yet implemented here. + """ + res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')} + for field in self.float_fields: + res[field] = np.mean([v[field] for v in all_res.values()]) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {'num_gt_tracks': self._combine_sum(all_res, 'num_gt_tracks')} + for field in self.float_fields: + res[field] = np.mean([v[field] for v in all_res.values()]) + return res + + @staticmethod + def _seg2bmap(seg, width=None, height=None): + """ + From a segmentation, compute a binary boundary map with 1 pixel wide + boundaries. The boundary pixels are offset by 1/2 pixel towards the + origin from the actual segment boundary. + Arguments: + seg : Segments labeled from 1..k. + width : Width of desired bmap <= seg.shape[1] + height : Height of desired bmap <= seg.shape[0] + Returns: + bmap (ndarray): Binary boundary map. + David Martin + January 2003 + """ + + seg = seg.astype(np.bool) + seg[seg > 0] = 1 + + assert np.atleast_3d(seg).shape[2] == 1 + + width = seg.shape[1] if width is None else width + height = seg.shape[0] if height is None else height + + h, w = seg.shape[:2] + + ar1 = float(width) / float(height) + ar2 = float(w) / float(h) + + assert not ( + width > w | height > h | abs(ar1 - ar2) > 0.01 + ), "Can" "t convert %dx%d seg to %dx%d bmap." % (w, h, width, height) + + e = np.zeros_like(seg) + s = np.zeros_like(seg) + se = np.zeros_like(seg) + + e[:, :-1] = seg[:, 1:] + s[:-1, :] = seg[1:, :] + se[:-1, :-1] = seg[1:, 1:] + + b = seg ^ e | seg ^ s | seg ^ se + b[-1, :] = seg[-1, :] ^ e[-1, :] + b[:, -1] = seg[:, -1] ^ s[:, -1] + b[-1, -1] = 0 + + if w == width and h == height: + bmap = b + else: + bmap = np.zeros((height, width)) + for x in range(w): + for y in range(h): + if b[y, x]: + j = 1 + math.floor((y - 1) + height / h) + i = 1 + math.floor((x - 1) + width / h) + bmap[j, i] = 1 + + return bmap + + @staticmethod + def _compute_f(gt_data, tracker_data, tracker_data_id, gt_id, bound_th): + """ + Perform F computation for a given gt and a given tracker ID. Adapted from + https://github.com/davisvideochallenge/davis2017-evaluation + :param gt_data: the encoded gt masks + :param tracker_data: the encoded tracker masks + :param tracker_data_id: the tracker ID + :param gt_id: the ground truth ID + :param bound_th: boundary threshold parameter + :return: the F value for the given tracker and gt ID + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + from skimage.morphology import disk + import cv2 + + f = np.zeros(len(gt_data)) + + for t, (gt_masks, tracker_masks) in enumerate(zip(gt_data, tracker_data)): + curr_tracker_mask = mask_utils.decode(tracker_masks[tracker_data_id]) + curr_gt_mask = mask_utils.decode(gt_masks[gt_id]) + + bound_pix = bound_th if bound_th >= 1 - np.finfo('float').eps else \ + np.ceil(bound_th * np.linalg.norm(curr_tracker_mask.shape)) + + # Get the pixel boundaries of both masks + fg_boundary = JAndF._seg2bmap(curr_tracker_mask) + gt_boundary = JAndF._seg2bmap(curr_gt_mask) + + # fg_dil = binary_dilation(fg_boundary, disk(bound_pix)) + fg_dil = cv2.dilate(fg_boundary.astype(np.uint8), disk(bound_pix).astype(np.uint8)) + # gt_dil = binary_dilation(gt_boundary, disk(bound_pix)) + gt_dil = cv2.dilate(gt_boundary.astype(np.uint8), disk(bound_pix).astype(np.uint8)) + + # Get the intersection + gt_match = gt_boundary * fg_dil + fg_match = fg_boundary * gt_dil + + # Area of the intersection + n_fg = np.sum(fg_boundary) + n_gt = np.sum(gt_boundary) + + # % Compute precision and recall + if n_fg == 0 and n_gt > 0: + precision = 1 + recall = 0 + elif n_fg > 0 and n_gt == 0: + precision = 0 + recall = 1 + elif n_fg == 0 and n_gt == 0: + precision = 1 + recall = 1 + else: + precision = np.sum(fg_match) / float(n_fg) + recall = np.sum(gt_match) / float(n_gt) + + # Compute F measure + if precision + recall == 0: + f_val = 0 + else: + f_val = 2 * precision * recall / (precision + recall) + + f[t] = f_val + + return f + + @staticmethod + def _compute_j(gt_data, tracker_data, num_gt_ids, num_tracker_ids, num_timesteps): + """ + Computation of J value for all ground truth IDs and all tracker IDs in the given sequence. Adapted from + https://github.com/davisvideochallenge/davis2017-evaluation + :param gt_data: the ground truth masks + :param tracker_data: the tracker masks + :param num_gt_ids: the number of ground truth IDs + :param num_tracker_ids: the number of tracker IDs + :param num_timesteps: the number of timesteps + :return: the J values + """ + + # Only loaded when run to reduce minimum requirements + from pycocotools import mask as mask_utils + + j = np.zeros((num_tracker_ids, num_gt_ids, num_timesteps)) + + for t, (time_gt, time_data) in enumerate(zip(gt_data, tracker_data)): + # run length encoded masks with pycocotools + area_gt = mask_utils.area(time_gt) + time_data = list(time_data) + area_tr = mask_utils.area(time_data) + + area_tr = np.repeat(area_tr[:, np.newaxis], len(area_gt), axis=1) + area_gt = np.repeat(area_gt[np.newaxis, :], len(area_tr), axis=0) + + # mask iou computation with pycocotools + ious = np.atleast_2d(mask_utils.iou(time_data, time_gt, [0]*len(time_gt))) + # set iou to 1 if both masks are close to 0 (no ground truth and no predicted mask in timestep) + ious[np.isclose(area_tr, 0) & np.isclose(area_gt, 0)] = 1 + assert (ious >= 0 - np.finfo('float').eps).all() + assert (ious <= 1 + np.finfo('float').eps).all() + + j[..., t] = ious + + return j diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/track_map.py b/test/yolov7-tracker/tracker/trackeval/metrics/track_map.py new file mode 100644 index 0000000..039f890 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/track_map.py @@ -0,0 +1,462 @@ +import numpy as np +from ._base_metric import _BaseMetric +from .. import _timing +from functools import partial +from .. import utils +from ..utils import TrackEvalException + + +class TrackMAP(_BaseMetric): + """Class which implements the TrackMAP metrics""" + + @staticmethod + def get_default_metric_config(): + """Default class config values""" + default_config = { + 'USE_AREA_RANGES': True, # whether to evaluate for certain area ranges + 'AREA_RANGES': [[0 ** 2, 32 ** 2], # additional area range sets for which TrackMAP is evaluated + [32 ** 2, 96 ** 2], # (all area range always included), default values for TAO + [96 ** 2, 1e5 ** 2]], # evaluation + 'AREA_RANGE_LABELS': ["area_s", "area_m", "area_l"], # the labels for the area ranges + 'USE_TIME_RANGES': True, # whether to evaluate for certain time ranges (length of tracks) + 'TIME_RANGES': [[0, 3], [3, 10], [10, 1e5]], # additional time range sets for which TrackMAP is evaluated + # (all time range always included) , default values for TAO evaluation + 'TIME_RANGE_LABELS': ["time_s", "time_m", "time_l"], # the labels for the time ranges + 'IOU_THRESHOLDS': np.arange(0.5, 0.96, 0.05), # the IoU thresholds + 'RECALL_THRESHOLDS': np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01) + 1), endpoint=True), + # recall thresholds at which precision is evaluated + 'MAX_DETECTIONS': 0, # limit the maximum number of considered tracks per sequence (0 for unlimited) + 'PRINT_CONFIG': True + } + return default_config + + def __init__(self, config=None): + super().__init__() + self.config = utils.init_config(config, self.get_default_metric_config(), self.get_name()) + + self.num_ig_masks = 1 + self.lbls = ['all'] + self.use_area_rngs = self.config['USE_AREA_RANGES'] + if self.use_area_rngs: + self.area_rngs = self.config['AREA_RANGES'] + self.area_rng_lbls = self.config['AREA_RANGE_LABELS'] + self.num_ig_masks += len(self.area_rng_lbls) + self.lbls += self.area_rng_lbls + + self.use_time_rngs = self.config['USE_TIME_RANGES'] + if self.use_time_rngs: + self.time_rngs = self.config['TIME_RANGES'] + self.time_rng_lbls = self.config['TIME_RANGE_LABELS'] + self.num_ig_masks += len(self.time_rng_lbls) + self.lbls += self.time_rng_lbls + + self.array_labels = self.config['IOU_THRESHOLDS'] + self.rec_thrs = self.config['RECALL_THRESHOLDS'] + + self.maxDet = self.config['MAX_DETECTIONS'] + self.float_array_fields = ['AP_' + lbl for lbl in self.lbls] + ['AR_' + lbl for lbl in self.lbls] + self.fields = self.float_array_fields + self.summary_fields = self.float_array_fields + + @_timing.time + def eval_sequence(self, data): + """Calculates GT and Tracker matches for one sequence for TrackMAP metrics. Adapted from + https://github.com/TAO-Dataset/""" + + # Initialise results to zero for each sequence as the fields are only defined over the set of all sequences + res = {} + for field in self.fields: + res[field] = [0 for _ in self.array_labels] + + gt_ids, dt_ids = data['gt_track_ids'], data['dt_track_ids'] + + if len(gt_ids) == 0 and len(dt_ids) == 0: + for idx in range(self.num_ig_masks): + res[idx] = None + return res + + # get track data + gt_tr_areas = data.get('gt_track_areas', None) if self.use_area_rngs else None + gt_tr_lengths = data.get('gt_track_lengths', None) if self.use_time_rngs else None + gt_tr_iscrowd = data.get('gt_track_iscrowd', None) + dt_tr_areas = data.get('dt_track_areas', None) if self.use_area_rngs else None + dt_tr_lengths = data.get('dt_track_lengths', None) if self.use_time_rngs else None + is_nel = data.get('not_exhaustively_labeled', False) + + # compute ignore masks for different track sets to eval + gt_ig_masks = self._compute_track_ig_masks(len(gt_ids), track_lengths=gt_tr_lengths, track_areas=gt_tr_areas, + iscrowd=gt_tr_iscrowd) + dt_ig_masks = self._compute_track_ig_masks(len(dt_ids), track_lengths=dt_tr_lengths, track_areas=dt_tr_areas, + is_not_exhaustively_labeled=is_nel, is_gt=False) + + boxformat = data.get('boxformat', 'xywh') + ious = self._compute_track_ious(data['dt_tracks'], data['gt_tracks'], iou_function=data['iou_type'], + boxformat=boxformat) + + for mask_idx in range(self.num_ig_masks): + gt_ig_mask = gt_ig_masks[mask_idx] + + # Sort gt ignore last + gt_idx = np.argsort([g for g in gt_ig_mask], kind="mergesort") + gt_ids = [gt_ids[i] for i in gt_idx] + + ious_sorted = ious[:, gt_idx] if len(ious) > 0 else ious + + num_thrs = len(self.array_labels) + num_gt = len(gt_ids) + num_dt = len(dt_ids) + + # Array to store the "id" of the matched dt/gt + gt_m = np.zeros((num_thrs, num_gt)) - 1 + dt_m = np.zeros((num_thrs, num_dt)) - 1 + + gt_ig = np.array([gt_ig_mask[idx] for idx in gt_idx]) + dt_ig = np.zeros((num_thrs, num_dt)) + + for iou_thr_idx, iou_thr in enumerate(self.array_labels): + if len(ious_sorted) == 0: + break + + for dt_idx, _dt in enumerate(dt_ids): + iou = min([iou_thr, 1 - 1e-10]) + # information about best match so far (m=-1 -> unmatched) + # store the gt_idx which matched for _dt + m = -1 + for gt_idx, _ in enumerate(gt_ids): + # if this gt already matched continue + if gt_m[iou_thr_idx, gt_idx] > 0: + continue + # if _dt matched to reg gt, and on ignore gt, stop + if m > -1 and gt_ig[m] == 0 and gt_ig[gt_idx] == 1: + break + # continue to next gt unless better match made + if ious_sorted[dt_idx, gt_idx] < iou - np.finfo('float').eps: + continue + # if match successful and best so far, store appropriately + iou = ious_sorted[dt_idx, gt_idx] + m = gt_idx + + # No match found for _dt, go to next _dt + if m == -1: + continue + + # if gt to ignore for some reason update dt_ig. + # Should not be used in evaluation. + dt_ig[iou_thr_idx, dt_idx] = gt_ig[m] + # _dt match found, update gt_m, and dt_m with "id" + dt_m[iou_thr_idx, dt_idx] = gt_ids[m] + gt_m[iou_thr_idx, m] = _dt + + dt_ig_mask = dt_ig_masks[mask_idx] + + dt_ig_mask = np.array(dt_ig_mask).reshape((1, num_dt)) # 1 X num_dt + dt_ig_mask = np.repeat(dt_ig_mask, num_thrs, 0) # num_thrs X num_dt + + # Based on dt_ig_mask ignore any unmatched detection by updating dt_ig + dt_ig = np.logical_or(dt_ig, np.logical_and(dt_m == -1, dt_ig_mask)) + # store results for given video and category + res[mask_idx] = { + "dt_ids": dt_ids, + "gt_ids": gt_ids, + "dt_matches": dt_m, + "gt_matches": gt_m, + "dt_scores": data['dt_track_scores'], + "gt_ignore": gt_ig, + "dt_ignore": dt_ig, + } + + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences. Computes precision and recall values based on track matches. + Adapted from https://github.com/TAO-Dataset/ + """ + num_thrs = len(self.array_labels) + num_recalls = len(self.rec_thrs) + + # -1 for absent categories + precision = -np.ones( + (num_thrs, num_recalls, self.num_ig_masks) + ) + recall = -np.ones((num_thrs, self.num_ig_masks)) + + for ig_idx in range(self.num_ig_masks): + ig_idx_results = [res[ig_idx] for res in all_res.values() if res[ig_idx] is not None] + + # Remove elements which are None + if len(ig_idx_results) == 0: + continue + + # Append all scores: shape (N,) + # limit considered tracks for each sequence if maxDet > 0 + if self.maxDet == 0: + dt_scores = np.concatenate([res["dt_scores"] for res in ig_idx_results], axis=0) + + dt_idx = np.argsort(-dt_scores, kind="mergesort") + + dt_m = np.concatenate([e["dt_matches"] for e in ig_idx_results], + axis=1)[:, dt_idx] + dt_ig = np.concatenate([e["dt_ignore"] for e in ig_idx_results], + axis=1)[:, dt_idx] + elif self.maxDet > 0: + dt_scores = np.concatenate([res["dt_scores"][0:self.maxDet] for res in ig_idx_results], axis=0) + + dt_idx = np.argsort(-dt_scores, kind="mergesort") + + dt_m = np.concatenate([e["dt_matches"][:, 0:self.maxDet] for e in ig_idx_results], + axis=1)[:, dt_idx] + dt_ig = np.concatenate([e["dt_ignore"][:, 0:self.maxDet] for e in ig_idx_results], + axis=1)[:, dt_idx] + else: + raise Exception("Number of maximum detections must be >= 0, but is set to %i" % self.maxDet) + + gt_ig = np.concatenate([res["gt_ignore"] for res in ig_idx_results]) + # num gt anns to consider + num_gt = np.count_nonzero(gt_ig == 0) + + if num_gt == 0: + continue + + tps = np.logical_and(dt_m != -1, np.logical_not(dt_ig)) + fps = np.logical_and(dt_m == -1, np.logical_not(dt_ig)) + + tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) + + for iou_thr_idx, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + num_tp = len(tp) + rc = tp / num_gt + if num_tp: + recall[iou_thr_idx, ig_idx] = rc[-1] + else: + recall[iou_thr_idx, ig_idx] = 0 + + # np.spacing(1) ~= eps + pr = tp / (fp + tp + np.spacing(1)) + pr = pr.tolist() + + # Ensure precision values are monotonically decreasing + for i in range(num_tp - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + # find indices at the predefined recall values + rec_thrs_insert_idx = np.searchsorted(rc, self.rec_thrs, side="left") + + pr_at_recall = [0.0] * num_recalls + + try: + for _idx, pr_idx in enumerate(rec_thrs_insert_idx): + pr_at_recall[_idx] = pr[pr_idx] + except IndexError: + pass + + precision[iou_thr_idx, :, ig_idx] = (np.array(pr_at_recall)) + + res = {'precision': precision, 'recall': recall} + + # compute the precision and recall averages for the respective alpha thresholds and ignore masks + for lbl in self.lbls: + res['AP_' + lbl] = np.zeros((len(self.array_labels)), dtype=np.float) + res['AR_' + lbl] = np.zeros((len(self.array_labels)), dtype=np.float) + + for a_id, alpha in enumerate(self.array_labels): + for lbl_idx, lbl in enumerate(self.lbls): + p = precision[a_id, :, lbl_idx] + if len(p[p > -1]) == 0: + mean_p = -1 + else: + mean_p = np.mean(p[p > -1]) + res['AP_' + lbl][a_id] = mean_p + res['AR_' + lbl][a_id] = recall[a_id, lbl_idx] + + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=True): + """Combines metrics across all classes by averaging over the class values + Note mAP is not well defined for 'empty classes' so 'ignore empty classes' is always true here. + """ + res = {} + for field in self.fields: + res[field] = np.zeros((len(self.array_labels)), dtype=np.float) + field_stacked = np.array([res[field] for res in all_res.values()]) + + for a_id, alpha in enumerate(self.array_labels): + values = field_stacked[:, a_id] + if len(values[values > -1]) == 0: + mean = -1 + else: + mean = np.mean(values[values > -1]) + res[field][a_id] = mean + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + + res = {} + for field in self.fields: + res[field] = np.zeros((len(self.array_labels)), dtype=np.float) + field_stacked = np.array([res[field] for res in all_res.values()]) + + for a_id, alpha in enumerate(self.array_labels): + values = field_stacked[:, a_id] + if len(values[values > -1]) == 0: + mean = -1 + else: + mean = np.mean(values[values > -1]) + res[field][a_id] = mean + return res + + def _compute_track_ig_masks(self, num_ids, track_lengths=None, track_areas=None, iscrowd=None, + is_not_exhaustively_labeled=False, is_gt=True): + """ + Computes ignore masks for different track sets to evaluate + :param num_ids: the number of track IDs + :param track_lengths: the lengths of the tracks (number of timesteps) + :param track_areas: the average area of a track + :param iscrowd: whether a track is marked as crowd + :param is_not_exhaustively_labeled: whether the track category is not exhaustively labeled + :param is_gt: whether it is gt + :return: the track ignore masks + """ + # for TAO tracks for classes which are not exhaustively labeled are not evaluated + if not is_gt and is_not_exhaustively_labeled: + track_ig_masks = [[1 for _ in range(num_ids)] for i in range(self.num_ig_masks)] + else: + # consider all tracks + track_ig_masks = [[0 for _ in range(num_ids)]] + + # consider tracks with certain area + if self.use_area_rngs: + for rng in self.area_rngs: + track_ig_masks.append([0 if rng[0] - np.finfo('float').eps <= area <= rng[1] + np.finfo('float').eps + else 1 for area in track_areas]) + + # consider tracks with certain duration + if self.use_time_rngs: + for rng in self.time_rngs: + track_ig_masks.append([0 if rng[0] - np.finfo('float').eps <= length + <= rng[1] + np.finfo('float').eps else 1 for length in track_lengths]) + + # for YouTubeVIS evaluation tracks with crowd tag are not evaluated + if is_gt and iscrowd: + track_ig_masks = [np.logical_or(mask, iscrowd) for mask in track_ig_masks] + + return track_ig_masks + + @staticmethod + def _compute_bb_track_iou(dt_track, gt_track, boxformat='xywh'): + """ + Calculates the track IoU for one detected track and one ground truth track for bounding boxes + :param dt_track: the detected track (format: dictionary with frame index as keys and + numpy arrays as values) + :param gt_track: the ground truth track (format: dictionary with frame index as keys and + numpy array as values) + :param boxformat: the format of the boxes + :return: the track IoU + """ + intersect = 0 + union = 0 + image_ids = set(gt_track.keys()) | set(dt_track.keys()) + for image in image_ids: + g = gt_track.get(image, None) + d = dt_track.get(image, None) + if boxformat == 'xywh': + if d is not None and g is not None: + dx, dy, dw, dh = d + gx, gy, gw, gh = g + w = max(min(dx + dw, gx + gw) - max(dx, gx), 0) + h = max(min(dy + dh, gy + gh) - max(dy, gy), 0) + i = w * h + u = dw * dh + gw * gh - i + intersect += i + union += u + elif d is None and g is not None: + union += g[2] * g[3] + elif d is not None and g is None: + union += d[2] * d[3] + elif boxformat == 'x0y0x1y1': + if d is not None and g is not None: + dx0, dy0, dx1, dy1 = d + gx0, gy0, gx1, gy1 = g + w = max(min(dx1, gx1) - max(dx0, gx0), 0) + h = max(min(dy1, gy1) - max(dy0, gy0), 0) + i = w * h + u = (dx1 - dx0) * (dy1 - dy0) + (gx1 - gx0) * (gy1 - gy0) - i + intersect += i + union += u + elif d is None and g is not None: + union += (g[2] - g[0]) * (g[3] - g[1]) + elif d is not None and g is None: + union += (d[2] - d[0]) * (d[3] - d[1]) + else: + raise TrackEvalException('BoxFormat not implemented') + if intersect > union: + raise TrackEvalException("Intersection value > union value. Are the box values corrupted?") + return intersect / union if union > 0 else 0 + + @staticmethod + def _compute_mask_track_iou(dt_track, gt_track): + """ + Calculates the track IoU for one detected track and one ground truth track for segmentation masks + :param dt_track: the detected track (format: dictionary with frame index as keys and + pycocotools rle encoded masks as values) + :param gt_track: the ground truth track (format: dictionary with frame index as keys and + pycocotools rle encoded masks as values) + :return: the track IoU + """ + # only loaded when needed to reduce minimum requirements + from pycocotools import mask as mask_utils + + intersect = .0 + union = .0 + image_ids = set(gt_track.keys()) | set(dt_track.keys()) + for image in image_ids: + g = gt_track.get(image, None) + d = dt_track.get(image, None) + if d and g: + intersect += mask_utils.area(mask_utils.merge([d, g], True)) + union += mask_utils.area(mask_utils.merge([d, g], False)) + elif not d and g: + union += mask_utils.area(g) + elif d and not g: + union += mask_utils.area(d) + if union < 0.0 - np.finfo('float').eps: + raise TrackEvalException("Union value < 0. Are the segmentaions corrupted?") + if intersect > union: + raise TrackEvalException("Intersection value > union value. Are the segmentations corrupted?") + iou = intersect / union if union > 0.0 + np.finfo('float').eps else 0.0 + return iou + + @staticmethod + def _compute_track_ious(dt, gt, iou_function='bbox', boxformat='xywh'): + """ + Calculate track IoUs for a set of ground truth tracks and a set of detected tracks + """ + + if len(gt) == 0 and len(dt) == 0: + return [] + + if iou_function == 'bbox': + track_iou_function = partial(TrackMAP._compute_bb_track_iou, boxformat=boxformat) + elif iou_function == 'mask': + track_iou_function = partial(TrackMAP._compute_mask_track_iou) + else: + raise Exception('IoU function not implemented') + + ious = np.zeros([len(dt), len(gt)]) + for i, j in np.ndindex(ious.shape): + ious[i, j] = track_iou_function(dt[i], gt[j]) + return ious + + @staticmethod + def _row_print(*argv): + """Prints results in an evenly spaced rows, with more space in first row""" + if len(argv) == 1: + argv = argv[0] + to_print = '%-40s' % argv[0] + for v in argv[1:]: + to_print += '%-12s' % str(v) + print(to_print) diff --git a/test/yolov7-tracker/tracker/trackeval/metrics/vace.py b/test/yolov7-tracker/tracker/trackeval/metrics/vace.py new file mode 100644 index 0000000..81858d4 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/metrics/vace.py @@ -0,0 +1,131 @@ +import numpy as np +from scipy.optimize import linear_sum_assignment +from ._base_metric import _BaseMetric +from .. import _timing + + +class VACE(_BaseMetric): + """Class which implements the VACE metrics. + + The metrics are described in: + Manohar et al. (2006) "Performance Evaluation of Object Detection and Tracking in Video" + https://link.springer.com/chapter/10.1007/11612704_16 + + This implementation uses the "relaxed" variant of the metrics, + where an overlap threshold is applied in each frame. + """ + + def __init__(self, config=None): + super().__init__() + self.integer_fields = ['VACE_IDs', 'VACE_GT_IDs', 'num_non_empty_timesteps'] + self.float_fields = ['STDA', 'ATA', 'FDA', 'SFDA'] + self.fields = self.integer_fields + self.float_fields + self.summary_fields = ['SFDA', 'ATA'] + + # Fields that are accumulated over multiple videos. + self._additive_fields = self.integer_fields + ['STDA', 'FDA'] + + self.threshold = 0.5 + + @_timing.time + def eval_sequence(self, data): + """Calculates VACE metrics for one sequence. + + Depends on the fields: + data['num_gt_ids'] + data['num_tracker_ids'] + data['gt_ids'] + data['tracker_ids'] + data['similarity_scores'] + """ + res = {} + + # Obtain Average Tracking Accuracy (ATA) using track correspondence. + # Obtain counts necessary to compute temporal IOU. + # Assume that integer counts can be represented exactly as floats. + potential_matches_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids'])) + gt_id_count = np.zeros(data['num_gt_ids']) + tracker_id_count = np.zeros(data['num_tracker_ids']) + both_present_count = np.zeros((data['num_gt_ids'], data['num_tracker_ids'])) + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + # Count the number of frames in which two tracks satisfy the overlap criterion. + matches_mask = np.greater_equal(data['similarity_scores'][t], self.threshold) + match_idx_gt, match_idx_tracker = np.nonzero(matches_mask) + potential_matches_count[gt_ids_t[match_idx_gt], tracker_ids_t[match_idx_tracker]] += 1 + # Count the number of frames in which the tracks are present. + gt_id_count[gt_ids_t] += 1 + tracker_id_count[tracker_ids_t] += 1 + both_present_count[gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]] += 1 + # Number of frames in which either track is present (union of the two sets of frames). + union_count = (gt_id_count[:, np.newaxis] + + tracker_id_count[np.newaxis, :] + - both_present_count) + # The denominator should always be non-zero if all tracks are non-empty. + with np.errstate(divide='raise', invalid='raise'): + temporal_iou = potential_matches_count / union_count + # Find assignment that maximizes temporal IOU. + match_rows, match_cols = linear_sum_assignment(-temporal_iou) + res['STDA'] = temporal_iou[match_rows, match_cols].sum() + res['VACE_IDs'] = data['num_tracker_ids'] + res['VACE_GT_IDs'] = data['num_gt_ids'] + + # Obtain Frame Detection Accuracy (FDA) using per-frame correspondence. + non_empty_count = 0 + fda = 0 + for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(data['gt_ids'], data['tracker_ids'])): + n_g = len(gt_ids_t) + n_d = len(tracker_ids_t) + if not (n_g or n_d): + continue + # n_g > 0 or n_d > 0 + non_empty_count += 1 + if not (n_g and n_d): + continue + # n_g > 0 and n_d > 0 + spatial_overlap = data['similarity_scores'][t] + match_rows, match_cols = linear_sum_assignment(-spatial_overlap) + overlap_ratio = spatial_overlap[match_rows, match_cols].sum() + fda += overlap_ratio / (0.5 * (n_g + n_d)) + res['FDA'] = fda + res['num_non_empty_timesteps'] = non_empty_count + + res.update(self._compute_final_fields(res)) + return res + + def combine_classes_class_averaged(self, all_res, ignore_empty_classes=True): + """Combines metrics across all classes by averaging over the class values. + If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection. + """ + res = {} + for field in self.fields: + if ignore_empty_classes: + res[field] = np.mean([v[field] for v in all_res.values() + if v['VACE_GT_IDs'] > 0 or v['VACE_IDs'] > 0], axis=0) + else: + res[field] = np.mean([v[field] for v in all_res.values()], axis=0) + return res + + def combine_classes_det_averaged(self, all_res): + """Combines metrics across all classes by averaging over the detection values""" + res = {} + for field in self._additive_fields: + res[field] = _BaseMetric._combine_sum(all_res, field) + res = self._compute_final_fields(res) + return res + + def combine_sequences(self, all_res): + """Combines metrics across all sequences""" + res = {} + for header in self._additive_fields: + res[header] = _BaseMetric._combine_sum(all_res, header) + res.update(self._compute_final_fields(res)) + return res + + @staticmethod + def _compute_final_fields(additive): + final = {} + with np.errstate(invalid='ignore'): # Permit nan results. + final['ATA'] = (additive['STDA'] / + (0.5 * (additive['VACE_IDs'] + additive['VACE_GT_IDs']))) + final['SFDA'] = additive['FDA'] / additive['num_non_empty_timesteps'] + return final diff --git a/test/yolov7-tracker/tracker/trackeval/plotting.py b/test/yolov7-tracker/tracker/trackeval/plotting.py new file mode 100644 index 0000000..e76fd08 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/plotting.py @@ -0,0 +1,230 @@ + +import os +import numpy as np +from .utils import TrackEvalException + + +def plot_compare_trackers(tracker_folder, tracker_list, cls, output_folder, plots_list=None): + """Create plots which compare metrics across different trackers.""" + # Define what to plot + if plots_list is None: + plots_list = get_default_plots_list() + + # Load data + data = load_multiple_tracker_summaries(tracker_folder, tracker_list, cls) + out_loc = os.path.join(output_folder, cls) + + # Plot + for args in plots_list: + create_comparison_plot(data, out_loc, *args) + + +def get_default_plots_list(): + # y_label, x_label, sort_label, bg_label, bg_function + plots_list = [ + ['AssA', 'DetA', 'HOTA', 'HOTA', 'geometric_mean'], + ['AssPr', 'AssRe', 'HOTA', 'AssA', 'jaccard'], + ['DetPr', 'DetRe', 'HOTA', 'DetA', 'jaccard'], + ['HOTA(0)', 'LocA(0)', 'HOTA', 'HOTALocA(0)', 'multiplication'], + ['HOTA', 'LocA', 'HOTA', None, None], + + ['HOTA', 'MOTA', 'HOTA', None, None], + ['HOTA', 'IDF1', 'HOTA', None, None], + ['IDF1', 'MOTA', 'HOTA', None, None], + ] + return plots_list + + +def load_multiple_tracker_summaries(tracker_folder, tracker_list, cls): + """Loads summary data for multiple trackers.""" + data = {} + for tracker in tracker_list: + with open(os.path.join(tracker_folder, tracker, cls + '_summary.txt')) as f: + keys = next(f).split(' ') + done = False + while not done: + values = next(f).split(' ') + if len(values) == len(keys): + done = True + data[tracker] = dict(zip(keys, map(float, values))) + return data + + +def create_comparison_plot(data, out_loc, y_label, x_label, sort_label, bg_label=None, bg_function=None, settings=None): + """ Creates a scatter plot comparing multiple trackers between two metric fields, with one on the x-axis and the + other on the y axis. Adds pareto optical lines and (optionally) a background contour. + + Inputs: + data: dict of dicts such that data[tracker_name][metric_field_name] = float + y_label: the metric_field_name to be plotted on the y-axis + x_label: the metric_field_name to be plotted on the x-axis + sort_label: the metric_field_name by which trackers are ordered and ranked + bg_label: the metric_field_name by which (optional) background contours are plotted + bg_function: the (optional) function bg_function(x,y) which converts the x_label / y_label values into bg_label. + settings: dict of plot settings with keys: + 'gap_val': gap between axis ticks and bg curves. + 'num_to_plot': maximum number of trackers to plot + """ + + # Only loaded when run to reduce minimum requirements + from matplotlib import pyplot as plt + + # Get plot settings + if settings is None: + gap_val = 2 + num_to_plot = 20 + else: + gap_val = settings['gap_val'] + num_to_plot = settings['num_to_plot'] + + if (bg_label is None) != (bg_function is None): + raise TrackEvalException('bg_function and bg_label must either be both given or neither given.') + + # Extract data + tracker_names = np.array(list(data.keys())) + sort_index = np.array([data[t][sort_label] for t in tracker_names]).argsort()[::-1] + x_values = np.array([data[t][x_label] for t in tracker_names])[sort_index][:num_to_plot] + y_values = np.array([data[t][y_label] for t in tracker_names])[sort_index][:num_to_plot] + + # Print info on what is being plotted + tracker_names = tracker_names[sort_index][:num_to_plot] + print('\nPlotting %s vs %s, for the following (ordered) trackers:' % (y_label, x_label)) + for i, name in enumerate(tracker_names): + print('%i: %s' % (i+1, name)) + + # Find best fitting boundaries for data + boundaries = _get_boundaries(x_values, y_values, round_val=gap_val/2) + + fig = plt.figure() + + # Plot background contour + if bg_function is not None: + _plot_bg_contour(bg_function, boundaries, gap_val) + + # Plot pareto optimal lines + _plot_pareto_optimal_lines(x_values, y_values) + + # Plot data points with number labels + labels = np.arange(len(y_values)) + 1 + plt.plot(x_values, y_values, 'b.', markersize=15) + for xx, yy, l in zip(x_values, y_values, labels): + plt.text(xx, yy, str(l), color="red", fontsize=15) + + # Add extra explanatory text to plots + plt.text(0, -0.11, 'label order:\nHOTA', horizontalalignment='left', verticalalignment='center', + transform=fig.axes[0].transAxes, color="red", fontsize=12) + if bg_label is not None: + plt.text(1, -0.11, 'curve values:\n' + bg_label, horizontalalignment='right', verticalalignment='center', + transform=fig.axes[0].transAxes, color="grey", fontsize=12) + + plt.xlabel(x_label, fontsize=15) + plt.ylabel(y_label, fontsize=15) + title = y_label + ' vs ' + x_label + if bg_label is not None: + title += ' (' + bg_label + ')' + plt.title(title, fontsize=17) + plt.xticks(np.arange(0, 100, gap_val)) + plt.yticks(np.arange(0, 100, gap_val)) + min_x, max_x, min_y, max_y = boundaries + plt.xlim(min_x, max_x) + plt.ylim(min_y, max_y) + plt.gca().set_aspect('equal', adjustable='box') + plt.tight_layout() + + os.makedirs(out_loc, exist_ok=True) + filename = os.path.join(out_loc, title.replace(' ', '_')) + plt.savefig(filename + '.pdf', bbox_inches='tight', pad_inches=0.05) + plt.savefig(filename + '.png', bbox_inches='tight', pad_inches=0.05) + + +def _get_boundaries(x_values, y_values, round_val): + x1 = np.min(np.floor((x_values - 0.5) / round_val) * round_val) + x2 = np.max(np.ceil((x_values + 0.5) / round_val) * round_val) + y1 = np.min(np.floor((y_values - 0.5) / round_val) * round_val) + y2 = np.max(np.ceil((y_values + 0.5) / round_val) * round_val) + x_range = x2 - x1 + y_range = y2 - y1 + max_range = max(x_range, y_range) + x_center = (x1 + x2) / 2 + y_center = (y1 + y2) / 2 + min_x = max(x_center - max_range / 2, 0) + max_x = min(x_center + max_range / 2, 100) + min_y = max(y_center - max_range / 2, 0) + max_y = min(y_center + max_range / 2, 100) + return min_x, max_x, min_y, max_y + + +def geometric_mean(x, y): + return np.sqrt(x * y) + + +def jaccard(x, y): + x = x / 100 + y = y / 100 + return 100 * (x * y) / (x + y - x * y) + + +def multiplication(x, y): + return x * y / 100 + + +bg_function_dict = { + "geometric_mean": geometric_mean, + "jaccard": jaccard, + "multiplication": multiplication, + } + + +def _plot_bg_contour(bg_function, plot_boundaries, gap_val): + """ Plot background contour. """ + + # Only loaded when run to reduce minimum requirements + from matplotlib import pyplot as plt + + # Plot background contour + min_x, max_x, min_y, max_y = plot_boundaries + x = np.arange(min_x, max_x, 0.1) + y = np.arange(min_y, max_y, 0.1) + x_grid, y_grid = np.meshgrid(x, y) + if bg_function in bg_function_dict.keys(): + z_grid = bg_function_dict[bg_function](x_grid, y_grid) + else: + raise TrackEvalException("background plotting function '%s' is not defined." % bg_function) + levels = np.arange(0, 100, gap_val) + con = plt.contour(x_grid, y_grid, z_grid, levels, colors='grey') + + def bg_format(val): + s = '{:1f}'.format(val) + return '{:.0f}'.format(val) if s[-1] == '0' else s + + con.levels = [bg_format(val) for val in con.levels] + plt.clabel(con, con.levels, inline=True, fmt='%r', fontsize=8) + + +def _plot_pareto_optimal_lines(x_values, y_values): + """ Plot pareto optimal lines """ + + # Only loaded when run to reduce minimum requirements + from matplotlib import pyplot as plt + + # Plot pareto optimal lines + cxs = x_values + cys = y_values + best_y = np.argmax(cys) + x_pareto = [0, cxs[best_y]] + y_pareto = [cys[best_y], cys[best_y]] + t = 2 + remaining = cxs > x_pareto[t - 1] + cys = cys[remaining] + cxs = cxs[remaining] + while len(cxs) > 0 and len(cys) > 0: + best_y = np.argmax(cys) + x_pareto += [x_pareto[t - 1], cxs[best_y]] + y_pareto += [cys[best_y], cys[best_y]] + t += 2 + remaining = cxs > x_pareto[t - 1] + cys = cys[remaining] + cxs = cxs[remaining] + x_pareto.append(x_pareto[t - 1]) + y_pareto.append(0) + plt.plot(np.array(x_pareto), np.array(y_pareto), '--r') diff --git a/test/yolov7-tracker/tracker/trackeval/utils.py b/test/yolov7-tracker/tracker/trackeval/utils.py new file mode 100644 index 0000000..8c7c916 --- /dev/null +++ b/test/yolov7-tracker/tracker/trackeval/utils.py @@ -0,0 +1,146 @@ + +import os +import csv +import argparse +from collections import OrderedDict + + +def init_config(config, default_config, name=None): + """Initialise non-given config values with defaults""" + if config is None: + config = default_config + else: + for k in default_config.keys(): + if k not in config.keys(): + config[k] = default_config[k] + if name and config['PRINT_CONFIG']: + print('\n%s Config:' % name) + for c in config.keys(): + print('%-20s : %-30s' % (c, config[c])) + return config + + +def update_config(config): + """ + Parse the arguments of a script and updates the config values for a given value if specified in the arguments. + :param config: the config to update + :return: the updated config + """ + parser = argparse.ArgumentParser() + for setting in config.keys(): + if type(config[setting]) == list or type(config[setting]) == type(None): + parser.add_argument("--" + setting, nargs='+') + else: + parser.add_argument("--" + setting) + args = parser.parse_args().__dict__ + for setting in args.keys(): + if args[setting] is not None: + if type(config[setting]) == type(True): + if args[setting] == 'True': + x = True + elif args[setting] == 'False': + x = False + else: + raise Exception('Command line parameter ' + setting + 'must be True or False') + elif type(config[setting]) == type(1): + x = int(args[setting]) + elif type(args[setting]) == type(None): + x = None + else: + x = args[setting] + config[setting] = x + return config + + +def get_code_path(): + """Get base path where code is""" + return os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + + +def validate_metrics_list(metrics_list): + """Get names of metric class and ensures they are unique, further checks that the fields within each metric class + do not have overlapping names. + """ + metric_names = [metric.get_name() for metric in metrics_list] + # check metric names are unique + if len(metric_names) != len(set(metric_names)): + raise TrackEvalException('Code being run with multiple metrics of the same name') + fields = [] + for m in metrics_list: + fields += m.fields + # check metric fields are unique + if len(fields) != len(set(fields)): + raise TrackEvalException('Code being run with multiple metrics with fields of the same name') + return metric_names + + +def write_summary_results(summaries, cls, output_folder): + """Write summary results to file""" + + fields = sum([list(s.keys()) for s in summaries], []) + values = sum([list(s.values()) for s in summaries], []) + + # In order to remain consistent upon new fields being adding, for each of the following fields if they are present + # they will be output in the summary first in the order below. Any further fields will be output in the order each + # metric family is called, and within each family either in the order they were added to the dict (python >= 3.6) or + # randomly (python < 3.6). + default_order = ['HOTA', 'DetA', 'AssA', 'DetRe', 'DetPr', 'AssRe', 'AssPr', 'LocA', 'OWTA', 'HOTA(0)', 'LocA(0)', + 'HOTALocA(0)', 'MOTA', 'MOTP', 'MODA', 'CLR_Re', 'CLR_Pr', 'MTR', 'PTR', 'MLR', 'CLR_TP', 'CLR_FN', + 'CLR_FP', 'IDSW', 'MT', 'PT', 'ML', 'Frag', 'sMOTA', 'IDF1', 'IDR', 'IDP', 'IDTP', 'IDFN', 'IDFP', + 'Dets', 'GT_Dets', 'IDs', 'GT_IDs'] + default_ordered_dict = OrderedDict(zip(default_order, [None for _ in default_order])) + for f, v in zip(fields, values): + default_ordered_dict[f] = v + for df in default_order: + if default_ordered_dict[df] is None: + del default_ordered_dict[df] + fields = list(default_ordered_dict.keys()) + values = list(default_ordered_dict.values()) + + out_file = os.path.join(output_folder, cls + '_summary.txt') + os.makedirs(os.path.dirname(out_file), exist_ok=True) + with open(out_file, 'w', newline='') as f: + writer = csv.writer(f, delimiter=' ') + writer.writerow(fields) + writer.writerow(values) + + +def write_detailed_results(details, cls, output_folder): + """Write detailed results to file""" + sequences = details[0].keys() + fields = ['seq'] + sum([list(s['COMBINED_SEQ'].keys()) for s in details], []) + out_file = os.path.join(output_folder, cls + '_detailed.csv') + os.makedirs(os.path.dirname(out_file), exist_ok=True) + with open(out_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(fields) + for seq in sorted(sequences): + if seq == 'COMBINED_SEQ': + continue + writer.writerow([seq] + sum([list(s[seq].values()) for s in details], [])) + writer.writerow(['COMBINED'] + sum([list(s['COMBINED_SEQ'].values()) for s in details], [])) + + +def load_detail(file): + """Loads detailed data for a tracker.""" + data = {} + with open(file) as f: + for i, row_text in enumerate(f): + row = row_text.replace('\r', '').replace('\n', '').split(',') + if i == 0: + keys = row[1:] + continue + current_values = row[1:] + seq = row[0] + if seq == 'COMBINED': + seq = 'COMBINED_SEQ' + if (len(current_values) == len(keys)) and seq != '': + data[seq] = {} + for key, value in zip(keys, current_values): + data[seq][key] = float(value) + return data + + +class TrackEvalException(Exception): + """Custom exception for catching expected errors.""" + ... diff --git a/test/yolov7-tracker/tracker/tracking_utils/envs.py b/test/yolov7-tracker/tracker/tracking_utils/envs.py new file mode 100644 index 0000000..b3b39e9 --- /dev/null +++ b/test/yolov7-tracker/tracker/tracking_utils/envs.py @@ -0,0 +1,36 @@ +""" +set gpus and ramdom seeds +""" + +import os +import random +import numpy as np +from loguru import logger + +import torch +import torch.backends.cudnn as cudnn + +def select_device(device): + """ set device + Args: + device: str, 'cpu' or '0' or '1,2,3'-like + + Return: + torch.device + + """ + + if device == 'cpu': + logger.info('Use CPU for training') + + elif ',' in device: # multi-gpu + logger.error('Multi-GPU currently not supported') + + else: + logger.info(f'set gpu {device}') + os.environ['CUDA_VISIBLE_DEVICES'] = device + assert torch.cuda.is_available() + + cuda = device != 'cpu' and torch.cuda.is_available() + device = torch.device('cuda:0' if cuda else 'cpu') + return device \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/tracking_utils/tools.py b/test/yolov7-tracker/tracker/tracking_utils/tools.py new file mode 100644 index 0000000..9c42d46 --- /dev/null +++ b/test/yolov7-tracker/tracker/tracking_utils/tools.py @@ -0,0 +1,26 @@ +import numpy as np +import cv2 +import os + +def save_results(folder_name, seq_name, results, data_type='default'): + """ + write results to txt file + + results: list row format: frame id, target id, box coordinate, class(optional) + to_file: file path(optional) + data_type: 'default' | 'mot_challenge', write data format, default or MOT submission + """ + assert len(results) + + if not os.path.exists(f'./track_results/{folder_name}'): + + os.makedirs(f'./track_results/{folder_name}') + + with open(os.path.join('./track_results', folder_name, seq_name + '.txt'), 'w') as f: + for frame_id, target_ids, tlwhs, clses, scores in results: + for id, tlwh, score in zip(target_ids, tlwhs, scores): + f.write(f'{frame_id},{id},{tlwh[0]:.2f},{tlwh[1]:.2f},{tlwh[2]:.2f},{tlwh[3]:.2f},{score:.2f},-1,-1,-1\n') + + f.close() + + return folder_name diff --git a/test/yolov7-tracker/tracker/tracking_utils/visualization.py b/test/yolov7-tracker/tracker/tracking_utils/visualization.py new file mode 100644 index 0000000..1b3dab9 --- /dev/null +++ b/test/yolov7-tracker/tracker/tracking_utils/visualization.py @@ -0,0 +1,64 @@ +import cv2 +import os +import numpy as np +from PIL import Image + +def plot_img(img, frame_id, results, save_dir): + """ + img: np.ndarray: (H, W, C) + frame_id: int + results: [tlwhs, ids, clses] + save_dir: sr + + plot images with bboxes of a seq + """ + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + assert img is not None + + if len(img.shape) > 3: + img = img.squeeze(0) + + img_ = np.ascontiguousarray(np.copy(img)) + + tlwhs, ids, clses = results[0], results[1], results[2] + for tlwh, id, cls in zip(tlwhs, ids, clses): + + # convert tlwh to tlbr + tlbr = tuple([int(tlwh[0]), int(tlwh[1]), int(tlwh[0] + tlwh[2]), int(tlwh[1] + tlwh[3])]) + # draw a rect + cv2.rectangle(img_, tlbr[:2], tlbr[2:], get_color(id), thickness=3, ) + # note the id and cls + text = f'{int(cls)}_{id}' + cv2.putText(img_, text, (tlbr[0], tlbr[1]), fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=1, + color=(255, 164, 0), thickness=2) + + cv2.imwrite(filename=os.path.join(save_dir, f'{frame_id:05d}.jpg'), img=img_) + +def get_color(idx): + """ + aux func for plot_seq + get a unique color for each id + """ + idx = idx * 3 + color = ((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255) + + return color + +def save_video(images_path): + """ + save images (frames) to a video + """ + + images_list = sorted(os.listdir(images_path)) + save_video_path = os.path.join(images_path, images_path.split('/')[-1] + '.mp4') + + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + + img0 = Image.open(os.path.join(images_path, images_list[0])) + vw = cv2.VideoWriter(save_video_path, fourcc, 15, img0.size) + + for image_name in images_list: + image = cv2.imread(filename=os.path.join(images_path, image_name)) + vw.write(image) diff --git a/test/yolov7-tracker/tracker/yolov7_utils/postprocess.py b/test/yolov7-tracker/tracker/yolov7_utils/postprocess.py new file mode 100644 index 0000000..fdaf4cc --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov7_utils/postprocess.py @@ -0,0 +1,16 @@ +from utils.general import non_max_suppression, scale_coords + +def postprocess(out, conf_thresh, nms_thresh, img_size, ori_img_size): + """ + Args: + out: out from v7 model + det_config: configs + """ + + out = out[0] + out = non_max_suppression(out, conf_thresh, nms_thresh, )[0] + out[:, :4] = scale_coords(img_size, out[:, :4], ori_img_size, ratio_pad=None).round() + + # out: tlbr, conf, cls + + return out \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/airmot.yaml b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/airmot.yaml new file mode 100644 index 0000000..7df3c0b --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/airmot.yaml @@ -0,0 +1,7 @@ +train: /data/wujiapeng/codes/DroneGraphTracker/airmot/train.txt +val: /data/wujiapeng/codes/DroneGraphTracker/airmot/test.txt +test: /data/wujiapeng/codes/DroneGraphTracker/airmot/test.txt + +nc: 2 + +names: ['plane', 'ship'] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/uavdt.yaml b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/uavdt.yaml new file mode 100644 index 0000000..d019ae4 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/uavdt.yaml @@ -0,0 +1,7 @@ +train: /data/wujiapeng/codes/DroneGraphTracker/uavdt/train.txt +val: /data/wujiapeng/codes/DroneGraphTracker/uavdt/test.txt +test: /data/wujiapeng/codes/DroneGraphTracker/uavdt/test.txt + +nc: 1 + +names: ['car'] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone.yaml b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone.yaml new file mode 100644 index 0000000..f1653e1 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone.yaml @@ -0,0 +1,7 @@ +train: /data/wujiapeng/codes/DroneGraphTracker/visdrone/train.txt +val: /data/wujiapeng/codes/DroneGraphTracker/visdrone/test.txt +test: /data/wujiapeng/codes/DroneGraphTracker/visdrone/test.txt + +nc: 5 + +names: ['pedestrain', 'car', 'van', 'truck', 'bus'] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone_det.yaml b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone_det.yaml new file mode 100644 index 0000000..945acd0 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/data_cfgs/visdrone_det.yaml @@ -0,0 +1,7 @@ +train: /data/wujiapeng/codes/DroneGraphTracker/visdrone_det/train.txt +val: /data/wujiapeng/codes/DroneGraphTracker/visdrone_det/test.txt +test: /data/wujiapeng/codes/DroneGraphTracker/visdrone_det/test.txt + +nc: 5 + +names: ['pedestrain', 'car', 'van', 'truck', 'bus'] \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/postprocess.py b/test/yolov7-tracker/tracker/yolov8_utils/postprocess.py new file mode 100644 index 0000000..c3f99cf --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/postprocess.py @@ -0,0 +1,6 @@ +from ultralytics import YOLO + +def postprocess(out): + + out = out[0].boxes + return out.data \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolov8_utils/train_yolov8.py b/test/yolov7-tracker/tracker/yolov8_utils/train_yolov8.py new file mode 100644 index 0000000..c1665d1 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolov8_utils/train_yolov8.py @@ -0,0 +1,36 @@ +import torch +from ultralytics import YOLO +import numpy as np + +import argparse + +def main(args): + """ main func + + """ + + model = YOLO(model=args.model_weight) + model.train( + data=args.data_cfg, + epochs=args.epochs, + batch=args.batch_size, + imgsz=args.img_sz, + patience=50, # epochs to wait for no observable improvement for early stopping of training + device=args.device, + ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser("YOLO v8 train parser") + + parser.add_argument('--model', type=str, default='yolov8s.yaml', help='yaml or pt file') + parser.add_argument('--model_weight', type=str, default='yolov8s.pt', help='') + parser.add_argument('--data_cfg', type=str, default='yolov8_utils/data_cfgs/visdrone.yaml', help='') + parser.add_argument('--epochs', type=int, default=30, help='') + parser.add_argument('--batch_size', type=int, default=8, help='') + parser.add_argument('--img_sz', type=int, default=1280, help='') + parser.add_argument('--device', type=str, default='0', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + + args = parser.parse_args() + + main(args) \ No newline at end of file diff --git a/test/yolov7-tracker/tracker/yolox_utils/mot_dataset.py b/test/yolov7-tracker/tracker/yolox_utils/mot_dataset.py new file mode 100644 index 0000000..5a293b5 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolox_utils/mot_dataset.py @@ -0,0 +1,155 @@ +import cv2 +import numpy as np +from pycocotools.coco import COCO + +import os + +from yolox.data.datasets import Dataset + + +class MOTDataset(Dataset): + """ + COCO dataset class. + """ + + def __init__( + self, + data_dir=None, + json_file="train_half.json", + name="train", + img_size=(608, 1088), + preproc=None, + ): + """ + COCO dataset initialization. Annotation data are read into memory by COCO API. + Args: + data_dir (str): dataset root directory + json_file (str): COCO json file name + name (str): COCO data name (e.g. 'train2017' or 'val2017') + img_size (int): target image size after pre-processing + preproc: data augmentation strategy + """ + super().__init__(img_size) + + self.data_dir = data_dir + self.json_file = json_file + + self.coco = COCO(os.path.join(self.data_dir, "annotations", self.json_file)) + self.ids = self.coco.getImgIds() + self.class_ids = sorted(self.coco.getCatIds()) + cats = self.coco.loadCats(self.coco.getCatIds()) + self._classes = tuple([c["name"] for c in cats]) + self.annotations = self._load_coco_annotations() + self.name = name + self.img_size = img_size + self.preproc = preproc + + def __len__(self): + return len(self.ids) + + def _load_coco_annotations(self): + return [self.load_anno_from_ids(_ids) for _ids in self.ids] + + def load_anno_from_ids(self, id_): + im_ann = self.coco.loadImgs(id_)[0] + width = im_ann["width"] + height = im_ann["height"] + frame_id = im_ann["frame_id"] + video_id = im_ann["video_id"] + anno_ids = self.coco.getAnnIds(imgIds=[int(id_)], iscrowd=False) + annotations = self.coco.loadAnns(anno_ids) + objs = [] + for obj in annotations: + x1 = obj["bbox"][0] + y1 = obj["bbox"][1] + x2 = x1 + obj["bbox"][2] + y2 = y1 + obj["bbox"][3] + if obj["area"] > 0 and x2 >= x1 and y2 >= y1: + obj["clean_bbox"] = [x1, y1, x2, y2] + objs.append(obj) + + num_objs = len(objs) + + res = np.zeros((num_objs, 6)) + + for ix, obj in enumerate(objs): + cls = self.class_ids.index(obj["category_id"]) + res[ix, 0:4] = obj["clean_bbox"] + res[ix, 4] = cls + res[ix, 5] = obj["track_id"] + + file_name = im_ann["file_name"] if "file_name" in im_ann else "{:012}".format(id_) + ".jpg" + img_info = (height, width, frame_id, video_id, file_name) + + del im_ann, annotations + + return (res, img_info, file_name) + + def load_anno(self, index): + return self.annotations[index][0] + + def pull_item(self, index): + id_ = self.ids[index] + + res, img_info, file_name = self.annotations[index] + # load image and preprocess + img_file = os.path.join( + self.data_dir, 'images', self.name, file_name + ) + # for debug + # print(f"************{img_file}************") + # exit() + img = cv2.imread(img_file) + assert img is not None + + return img, res.copy(), img_info, np.array([id_]) + + @Dataset.resize_getitem + def __getitem__(self, index): + """ + One image / label pair for the given index is picked up and pre-processed. + + Args: + index (int): data index + + Returns: + img (numpy.ndarray): pre-processed image + padded_labels (torch.Tensor): pre-processed label data. + The shape is :math:`[max_labels, 5]`. + each label consists of [class, xc, yc, w, h]: + class (float): class index. + xc, yc (float) : center of bbox whose values range from 0 to 1. + w, h (float) : size of bbox whose values range from 0 to 1. + info_img : tuple of h, w, nh, nw, dx, dy. + h, w (int): original shape of the image + nh, nw (int): shape of the resized image without padding + dx, dy (int): pad size + img_id (int): same as the input index. Used for evaluation. + """ + img, target, img_info, img_id = self.pull_item(index) + + if self.preproc is not None: + img, target = self.preproc(img, target, self.input_dim) + return img, target, img_info, img_id + +class VisDroneDataset(MOTDataset): + def __init__(self, data_dir=None, json_file="train_half.json", name="train", img_size=(608, 1088), preproc=None): + super().__init__(data_dir, json_file, name, img_size, preproc) + self.DATA_ROOT = '/data/wujiapeng/datasets/VisDrone2019/VisDrone2019' + self.VisD_dict = {'train':'VisDrone2019-MOT-train', + 'test':'VisDrone2019-MOT-test-dev'} + def pull_item(self, index): + id_ = self.ids[index] + + res, img_info, file_name = self.annotations[index] + # load image and preprocess + # img_file = os.path.join( + # self.data_dir, self.name, file_name + # ) + img_file = os.path.join( + self.DATA_ROOT, self.VisD_dict[self.name], 'sequences', file_name + ) + img = cv2.imread(img_file) + assert img is not None + + return img, res.copy(), img_info, np.array([id_]) diff --git a/test/yolov7-tracker/tracker/yolox_utils/postprocess.py b/test/yolov7-tracker/tracker/yolox_utils/postprocess.py new file mode 100644 index 0000000..2a73a12 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolox_utils/postprocess.py @@ -0,0 +1,29 @@ +import torch + +from yolox.utils import postprocess + +def postprocess_yolox(out, num_classes, conf_thresh, img, ori_img): + """ + convert out to -> (tlbr, conf, cls) + """ + + out = postprocess(out, num_classes, conf_thresh, )[0] # (tlbr, obj_conf, cls_conf, cls) + + if out is None: return out + + # merge conf + out[:, 4] *= out[:, 5] + out[:, 5] = out[:, -1] + out = out[:, :-1] + + # scale to origin size + + img_size = [img.shape[-2], img.shape[-1]] # h, w + ori_img_size = [ori_img.shape[0], ori_img.shape[1]] # h0, w0 + img_h, img_w = img_size[0], img_size[1] + + scale = min(float(img_h) / ori_img_size[0], float(img_w) / ori_img_size[1]) + + out[:, :4] /= scale + + return out diff --git a/test/yolov7-tracker/tracker/yolox_utils/train_yolox.py b/test/yolov7-tracker/tracker/yolox_utils/train_yolox.py new file mode 100644 index 0000000..d053609 --- /dev/null +++ b/test/yolov7-tracker/tracker/yolox_utils/train_yolox.py @@ -0,0 +1,122 @@ +from loguru import logger + +import torch +import torch.backends.cudnn as cudnn + +from yolox.core import Trainer, launch +from yolox.exp import get_exp + +import argparse +import random +import warnings + + +def make_parser(): + parser = argparse.ArgumentParser("YOLOX train parser") + parser.add_argument("-expn", "--experiment-name", type=str, default=None) + parser.add_argument("-n", "--name", type=str, default=None, help="model name") + + # distributed + parser.add_argument( + "--dist-backend", default="nccl", type=str, help="distributed backend" + ) + parser.add_argument( + "--dist-url", + default=None, + type=str, + help="url used to set up distributed training", + ) + parser.add_argument("-b", "--batch-size", type=int, default=64, help="batch size") + parser.add_argument( + "-d", "--devices", default=None, type=int, help="device for training" + ) + parser.add_argument( + "--local_rank", default=0, type=int, help="local rank for dist training" + ) + parser.add_argument( + "-f", + "--exp_file", + default=None, + type=str, + help="plz input your expriment description file", + ) + parser.add_argument( + "--resume", default=False, action="store_true", help="resume training" + ) + parser.add_argument("-c", "--ckpt", default=None, type=str, help="checkpoint file") + parser.add_argument( + "-e", + "--start_epoch", + default=None, + type=int, + help="resume training start epoch", + ) + parser.add_argument( + "--num_machines", default=1, type=int, help="num of node for training" + ) + parser.add_argument( + "--machine_rank", default=0, type=int, help="node rank for multi-node training" + ) + parser.add_argument( + "--fp16", + dest="fp16", + default=True, + action="store_true", + help="Adopting mix precision training.", + ) + parser.add_argument( + "-o", + "--occupy", + dest="occupy", + default=False, + action="store_true", + help="occupy GPU memory first for training.", + ) + parser.add_argument( + "opts", + help="Modify config options using the command-line", + default=None, + nargs=argparse.REMAINDER, + ) + return parser + + +@logger.catch +def main(exp, args): + if exp.seed is not None: + random.seed(exp.seed) + torch.manual_seed(exp.seed) + cudnn.deterministic = True + warnings.warn( + "You have chosen to seed training. This will turn on the CUDNN deterministic setting, " + "which can slow down your training considerably! You may see unexpected behavior " + "when restarting from checkpoints." + ) + + # set environment variables for distributed training + cudnn.benchmark = True + + trainer = Trainer(exp, args) + trainer.train() + + +if __name__ == "__main__": + args = make_parser().parse_args() + exp = get_exp(args.exp_file, args.name) + exp.merge(args.opts) + + if not args.experiment_name: + args.experiment_name = exp.exp_name + + num_gpu = torch.cuda.device_count() if args.devices is None else args.devices + assert num_gpu <= torch.cuda.device_count() + + launch( + main, + num_gpu, + args.num_machines, + args.machine_rank, + backend=args.dist_backend, + dist_url=args.dist_url, + args=(exp, args), + ) diff --git a/test/yolov7-tracker/tracker/yolox_utils/yolox_m.py b/test/yolov7-tracker/tracker/yolox_utils/yolox_m.py new file mode 100644 index 0000000..79bd66a --- /dev/null +++ b/test/yolov7-tracker/tracker/yolox_utils/yolox_m.py @@ -0,0 +1,144 @@ +# encoding: utf-8 +import os +import random +import torch +import torch.nn as nn +import torch.distributed as dist + +from yolox.exp import Exp as MyExp +from yolox.data import get_yolox_datadir + +class Exp(MyExp): + def __init__(self): + super(Exp, self).__init__() + self.num_classes = 1 # 1 for uavdt mot17 + self.depth = 0.67 + self.width = 0.75 + self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0] + self.train_ann = "train.json" + self.val_ann = "test.json" + self.input_size = (800, 1440) + self.test_size = (800, 1440) + self.random_size = (18, 32) + self.max_epoch = 80 + self.print_interval = 20 + self.eval_interval = 5 + self.test_conf = 0.001 + self.nmsthre = 0.7 + self.no_aug_epochs = 10 + self.basic_lr_per_img = 0.001 / 64.0 + self.warmup_epochs = 1 + + def get_data_loader(self, batch_size, is_distributed, no_aug=False): + from yolox.data import ( + TrainTransform, + YoloBatchSampler, + DataLoader, + InfiniteSampler, + MosaicDetection, + ) + + from mot_dataset import MOTDataset + + dataset = MOTDataset( + # data_dir=os.path.join(get_yolox_datadir(), "mot"), + # data_dir='/data/wujiapeng/datasets/UAVDT', + data_dir='/data/wujiapeng/datasets/VisDrone2019/VisDrone2019', + json_file=self.train_ann, + name='train', + img_size=self.input_size, + preproc=TrainTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + max_labels=500, + ), + ) + + dataset = MosaicDetection( + dataset, + mosaic=not no_aug, + img_size=self.input_size, + preproc=TrainTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + max_labels=1000, + ), + degrees=self.degrees, + translate=self.translate, + scale=self.scale, + shear=self.shear, + perspective=self.perspective, + enable_mixup=self.enable_mixup, + ) + + self.dataset = dataset + + if is_distributed: + batch_size = batch_size // dist.get_world_size() + + sampler = InfiniteSampler( + len(self.dataset), seed=self.seed if self.seed else 0 + ) + + batch_sampler = YoloBatchSampler( + sampler=sampler, + batch_size=batch_size, + drop_last=False, + input_dimension=self.input_size, + mosaic=not no_aug, + ) + + dataloader_kwargs = {"num_workers": self.data_num_workers, "pin_memory": True} + dataloader_kwargs["batch_sampler"] = batch_sampler + train_loader = DataLoader(self.dataset, **dataloader_kwargs) + + return train_loader + + def get_eval_loader(self, batch_size, is_distributed, testdev=False): + from yolox.data import ValTransform + from mot_dataset import MOTDataset + + valdataset = MOTDataset( + # data_dir=os.path.join(get_yolox_datadir(), "mot"), + # data_dir='/data/wujiapeng/datasets/UAVDT', + data_dir='/data/wujiapeng/datasets/VisDrone2019/VisDrone2019', + json_file=self.val_ann, + img_size=self.test_size, + name='test', + preproc=ValTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + ), + ) + + if is_distributed: + batch_size = batch_size // dist.get_world_size() + sampler = torch.utils.data.distributed.DistributedSampler( + valdataset, shuffle=False + ) + else: + sampler = torch.utils.data.SequentialSampler(valdataset) + + dataloader_kwargs = { + "num_workers": self.data_num_workers, + "pin_memory": True, + "sampler": sampler, + } + dataloader_kwargs["batch_size"] = batch_size + val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs) + + return val_loader + + def get_evaluator(self, batch_size, is_distributed, testdev=False): + from yolox.evaluators import COCOEvaluator + + val_loader = self.get_eval_loader(batch_size, is_distributed, testdev=testdev) + evaluator = COCOEvaluator( + dataloader=val_loader, + img_size=self.test_size, + confthre=self.test_conf, + nmsthre=self.nmsthre, + num_classes=self.num_classes, + testdev=testdev, + ) + return evaluator diff --git a/test/yolov7-tracker/tracker/yolox_utils/yolox_x.py b/test/yolov7-tracker/tracker/yolox_utils/yolox_x.py new file mode 100644 index 0000000..feeb26b --- /dev/null +++ b/test/yolov7-tracker/tracker/yolox_utils/yolox_x.py @@ -0,0 +1,142 @@ +# encoding: utf-8 +import os +import random +import torch +import torch.nn as nn +import torch.distributed as dist + +from yolox.exp import Exp as MyExp +from yolox.data import get_yolox_datadir + +class Exp(MyExp): + def __init__(self): + super(Exp, self).__init__() + self.num_classes = 1 + self.depth = 1.33 + self.width = 1.25 + self.exp_name = os.path.split(os.path.realpath(__file__))[1].split(".")[0] + self.train_ann = "train.json" + self.val_ann = "test.json" + self.input_size = (800, 1440) + self.test_size = (800, 1440) + self.random_size = (18, 32) + self.max_epoch = 80 + self.print_interval = 20 + self.eval_interval = 5 + self.test_conf = 0.001 + self.nmsthre = 0.7 + self.no_aug_epochs = 10 + self.basic_lr_per_img = 0.001 / 64.0 + self.warmup_epochs = 1 + + def get_data_loader(self, batch_size, is_distributed, no_aug=False): + from yolox.data import ( + TrainTransform, + YoloBatchSampler, + DataLoader, + InfiniteSampler, + MosaicDetection, + ) + + from mot_dataset import MOTDataset + + dataset = MOTDataset( + # data_dir=os.path.join(get_yolox_datadir(), "mot"), + data_dir='/data/wujiapeng/datasets/UAVDT', + json_file=self.train_ann, + name='train', + img_size=self.input_size, + preproc=TrainTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + max_labels=500, + ), + ) + + dataset = MosaicDetection( + dataset, + mosaic=not no_aug, + img_size=self.input_size, + preproc=TrainTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + max_labels=1000, + ), + degrees=self.degrees, + translate=self.translate, + scale=self.scale, + shear=self.shear, + perspective=self.perspective, + enable_mixup=self.enable_mixup, + ) + + self.dataset = dataset + + if is_distributed: + batch_size = batch_size // dist.get_world_size() + + sampler = InfiniteSampler( + len(self.dataset), seed=self.seed if self.seed else 0 + ) + + batch_sampler = YoloBatchSampler( + sampler=sampler, + batch_size=batch_size, + drop_last=False, + input_dimension=self.input_size, + mosaic=not no_aug, + ) + + dataloader_kwargs = {"num_workers": self.data_num_workers, "pin_memory": True} + dataloader_kwargs["batch_sampler"] = batch_sampler + train_loader = DataLoader(self.dataset, **dataloader_kwargs) + + return train_loader + + def get_eval_loader(self, batch_size, is_distributed, testdev=False): + from yolox.data import ValTransform + from mot_dataset import MOTDataset + + valdataset = MOTDataset( + # data_dir=os.path.join(get_yolox_datadir(), "mot"), + data_dir='/data/wujiapeng/datasets/UAVDT', + json_file=self.val_ann, + img_size=self.test_size, + name='test', + preproc=ValTransform( + rgb_means=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + ), + ) + + if is_distributed: + batch_size = batch_size // dist.get_world_size() + sampler = torch.utils.data.distributed.DistributedSampler( + valdataset, shuffle=False + ) + else: + sampler = torch.utils.data.SequentialSampler(valdataset) + + dataloader_kwargs = { + "num_workers": self.data_num_workers, + "pin_memory": True, + "sampler": sampler, + } + dataloader_kwargs["batch_size"] = batch_size + val_loader = torch.utils.data.DataLoader(valdataset, **dataloader_kwargs) + + return val_loader + + def get_evaluator(self, batch_size, is_distributed, testdev=False): + from yolox.evaluators import COCOEvaluator + + val_loader = self.get_eval_loader(batch_size, is_distributed, testdev=testdev) + evaluator = COCOEvaluator( + dataloader=val_loader, + img_size=self.test_size, + confthre=self.test_conf, + nmsthre=self.nmsthre, + num_classes=self.num_classes, + testdev=testdev, + ) + return evaluator diff --git a/test/yolov7-tracker/train.py b/test/yolov7-tracker/train.py new file mode 100644 index 0000000..9352aef --- /dev/null +++ b/test/yolov7-tracker/train.py @@ -0,0 +1,694 @@ +import argparse +import logging +import math +import os +import random +import time +from copy import deepcopy +from pathlib import Path +from threading import Thread + +import numpy as np +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.optim.lr_scheduler as lr_scheduler +import torch.utils.data +import yaml +from torch.cuda import amp +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm + +import test # import test.py to get mAP after each epoch +from models.experimental import attempt_load +from models.yolo import Model +from utils.autoanchor import check_anchors +from utils.datasets import create_dataloader +from utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \ + fitness, strip_optimizer, get_latest_run, check_dataset, check_file, check_git_status, check_img_size, \ + check_requirements, print_mutation, set_logging, one_cycle, colorstr +from utils.google_utils import attempt_download +from utils.loss import ComputeLoss, ComputeLossOTA +from utils.plots import plot_images, plot_labels, plot_results, plot_evolution +from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first, is_parallel +from utils.wandb_logging.wandb_utils import WandbLogger, check_wandb_resume + +logger = logging.getLogger(__name__) + + +def train(hyp, opt, device, tb_writer=None): + logger.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items())) + save_dir, epochs, batch_size, total_batch_size, weights, rank = \ + Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank + + # Directories + wdir = save_dir / 'weights' + wdir.mkdir(parents=True, exist_ok=True) # make dir + last = wdir / 'last.pt' + best = wdir / 'best.pt' + results_file = save_dir / 'results.txt' + + # Save run settings + with open(save_dir / 'hyp.yaml', 'w') as f: + yaml.dump(hyp, f, sort_keys=False) + with open(save_dir / 'opt.yaml', 'w') as f: + yaml.dump(vars(opt), f, sort_keys=False) + + # Configure + plots = not opt.evolve # create plots + cuda = device.type != 'cpu' + init_seeds(2 + rank) + with open(opt.data) as f: + data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict data/coco.yaml + is_coco = opt.data.endswith('coco.yaml') + + # Logging- Doing this before checking the dataset. Might update data_dict + loggers = {'wandb': None} # loggers dict + if rank in [-1, 0]: + opt.hyp = hyp # add hyperparameters + run_id = torch.load(weights).get('wandb_id') if weights.endswith('.pt') and os.path.isfile(weights) else None + wandb_logger = WandbLogger(opt, Path(opt.save_dir).stem, run_id, data_dict) + loggers['wandb'] = wandb_logger.wandb + data_dict = wandb_logger.data_dict + if wandb_logger.wandb: + weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp # WandbLogger might update weights, epochs if resuming + + nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes + names = ['item'] if opt.single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names + assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check + + # Model + pretrained = weights.endswith('.pt') + if pretrained: + with torch_distributed_zero_first(rank): + attempt_download(weights) # download if not found locally + ckpt = torch.load(weights, map_location=device) # load checkpoint + model = Model(opt.cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + exclude = ['anchor'] if (opt.cfg or hyp.get('anchors')) and not opt.resume else [] # exclude keys + state_dict = ckpt['model'].float().state_dict() # to FP32 + state_dict = intersect_dicts(state_dict, model.state_dict(), exclude=exclude) # intersect + model.load_state_dict(state_dict, strict=False) # load + logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report + else: + model = Model(opt.cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + with torch_distributed_zero_first(rank): + check_dataset(data_dict) # check + train_path = data_dict['train'] # ./coco/train2017.txt + test_path = data_dict['val'] # ./coco/val2017.txt + + # Freeze + freeze = [] # parameter names to freeze (full or partial) + for k, v in model.named_parameters(): + v.requires_grad = True # train all layers + if any(x in k for x in freeze): + print('freezing %s' % k) + v.requires_grad = False + + # Optimizer + nbs = 64 # nominal batch size + accumulate = max(round(nbs / total_batch_size), 1) # accumulate loss before optimizing + hyp['weight_decay'] *= total_batch_size * accumulate / nbs # scale weight_decay + logger.info(f"Scaled weight_decay = {hyp['weight_decay']}") + + pg0, pg1, pg2 = [], [], [] # optimizer parameter groups + for k, v in model.named_modules(): + if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter): + pg2.append(v.bias) # biases + if isinstance(v, nn.BatchNorm2d): + pg0.append(v.weight) # no decay + elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter): + pg1.append(v.weight) # apply decay + if hasattr(v, 'im'): + if hasattr(v.im, 'implicit'): + pg0.append(v.im.implicit) + else: + for iv in v.im: + pg0.append(iv.implicit) + if hasattr(v, 'imc'): + if hasattr(v.imc, 'implicit'): + pg0.append(v.imc.implicit) + else: + for iv in v.imc: + pg0.append(iv.implicit) + if hasattr(v, 'imb'): + if hasattr(v.imb, 'implicit'): + pg0.append(v.imb.implicit) + else: + for iv in v.imb: + pg0.append(iv.implicit) + if hasattr(v, 'imo'): + if hasattr(v.imo, 'implicit'): + pg0.append(v.imo.implicit) + else: + for iv in v.imo: + pg0.append(iv.implicit) + if hasattr(v, 'ia'): + if hasattr(v.ia, 'implicit'): + pg0.append(v.ia.implicit) + else: + for iv in v.ia: + pg0.append(iv.implicit) + if hasattr(v, 'attn'): + if hasattr(v.attn, 'logit_scale'): + pg0.append(v.attn.logit_scale) + if hasattr(v.attn, 'q_bias'): + pg0.append(v.attn.q_bias) + if hasattr(v.attn, 'v_bias'): + pg0.append(v.attn.v_bias) + if hasattr(v.attn, 'relative_position_bias_table'): + pg0.append(v.attn.relative_position_bias_table) + if hasattr(v, 'rbr_dense'): + if hasattr(v.rbr_dense, 'weight_rbr_origin'): + pg0.append(v.rbr_dense.weight_rbr_origin) + if hasattr(v.rbr_dense, 'weight_rbr_avg_conv'): + pg0.append(v.rbr_dense.weight_rbr_avg_conv) + if hasattr(v.rbr_dense, 'weight_rbr_pfir_conv'): + pg0.append(v.rbr_dense.weight_rbr_pfir_conv) + if hasattr(v.rbr_dense, 'weight_rbr_1x1_kxk_idconv1'): + pg0.append(v.rbr_dense.weight_rbr_1x1_kxk_idconv1) + if hasattr(v.rbr_dense, 'weight_rbr_1x1_kxk_conv2'): + pg0.append(v.rbr_dense.weight_rbr_1x1_kxk_conv2) + if hasattr(v.rbr_dense, 'weight_rbr_gconv_dw'): + pg0.append(v.rbr_dense.weight_rbr_gconv_dw) + if hasattr(v.rbr_dense, 'weight_rbr_gconv_pw'): + pg0.append(v.rbr_dense.weight_rbr_gconv_pw) + if hasattr(v.rbr_dense, 'vector'): + pg0.append(v.rbr_dense.vector) + + if opt.adam: + optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum + else: + optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True) + + optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay + optimizer.add_param_group({'params': pg2}) # add pg2 (biases) + logger.info('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0))) + del pg0, pg1, pg2 + + # Scheduler https://arxiv.org/pdf/1812.01187.pdf + # https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#OneCycleLR + if opt.linear_lr: + lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear + else: + lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf'] + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) + # plot_lr_scheduler(optimizer, scheduler, epochs) + + # EMA + ema = ModelEMA(model) if rank in [-1, 0] else None + + # Resume + start_epoch, best_fitness = 0, 0.0 + if pretrained: + # Optimizer + if ckpt['optimizer'] is not None: + optimizer.load_state_dict(ckpt['optimizer']) + best_fitness = ckpt['best_fitness'] + + # EMA + if ema and ckpt.get('ema'): + ema.ema.load_state_dict(ckpt['ema'].float().state_dict()) + ema.updates = ckpt['updates'] + + # Results + if ckpt.get('training_results') is not None: + results_file.write_text(ckpt['training_results']) # write results.txt + + # Epochs + start_epoch = ckpt['epoch'] + 1 + if opt.resume: + assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (weights, epochs) + if epochs < start_epoch: + logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' % + (weights, ckpt['epoch'], epochs)) + epochs += ckpt['epoch'] # finetune additional epochs + + del ckpt, state_dict + + # Image sizes + gs = max(int(model.stride.max()), 32) # grid size (max stride) + nl = model.model[-1].nl # number of detection layers (used for scaling hyp['obj']) + imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples + + # DP mode + if cuda and rank == -1 and torch.cuda.device_count() > 1: + model = torch.nn.DataParallel(model) + + # SyncBatchNorm + if opt.sync_bn and cuda and rank != -1: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device) + logger.info('Using SyncBatchNorm()') + + # Trainloader + # train_path: ./coco/train2017.txt + dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt, + hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect, rank=rank, + world_size=opt.world_size, workers=opt.workers, + image_weights=opt.image_weights, quad=opt.quad, prefix=colorstr('train: ')) + mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class + nb = len(dataloader) # number of batches + assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1) + + # Process 0 + if rank in [-1, 0]: + testloader = create_dataloader(test_path, imgsz_test, batch_size * 2, gs, opt, # testloader + hyp=hyp, cache=opt.cache_images and not opt.notest, rect=True, rank=-1, + world_size=opt.world_size, workers=opt.workers, + pad=0.5, prefix=colorstr('val: '))[0] + + if not opt.resume: + labels = np.concatenate(dataset.labels, 0) + c = torch.tensor(labels[:, 0]) # classes + # cf = torch.bincount(c.long(), minlength=nc) + 1. # frequency + # model._initialize_biases(cf.to(device)) + if plots: + #plot_labels(labels, names, save_dir, loggers) + if tb_writer: + tb_writer.add_histogram('classes', c, 0) + + # Anchors + if not opt.noautoanchor: + check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz) + model.half().float() # pre-reduce anchor precision + + # DDP mode + if cuda and rank != -1: + model = DDP(model, device_ids=[opt.local_rank], output_device=opt.local_rank, + # nn.MultiheadAttention incompatibility with DDP https://github.com/pytorch/pytorch/issues/26698 + find_unused_parameters=any(isinstance(layer, nn.MultiheadAttention) for layer in model.modules())) + + # Model parameters + hyp['box'] *= 3. / nl # scale to layers + hyp['cls'] *= nc / 80. * 3. / nl # scale to classes and layers + hyp['obj'] *= (imgsz / 640) ** 2 * 3. / nl # scale to image size and layers + hyp['label_smoothing'] = opt.label_smoothing + model.nc = nc # attach number of classes to model + model.hyp = hyp # attach hyperparameters to model + model.gr = 1.0 # iou loss ratio (obj_loss = 1.0 or iou) + model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights + model.names = names + + # Start training + t0 = time.time() + nw = max(round(hyp['warmup_epochs'] * nb), 1000) # number of warmup iterations, max(3 epochs, 1k iterations) + # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training + maps = np.zeros(nc) # mAP per class + results = (0, 0, 0, 0, 0, 0, 0) # P, R, mAP@.5, mAP@.5-.95, val_loss(box, obj, cls) + scheduler.last_epoch = start_epoch - 1 # do not move + scaler = amp.GradScaler(enabled=cuda) + compute_loss_ota = ComputeLossOTA(model) # init loss class + compute_loss = ComputeLoss(model) # init loss class + logger.info(f'Image sizes {imgsz} train, {imgsz_test} test\n' + f'Using {dataloader.num_workers} dataloader workers\n' + f'Logging results to {save_dir}\n' + f'Starting training for {epochs} epochs...') + torch.save(model, wdir / 'init.pt') + for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------ + model.train() + + # Update image weights (optional) + if opt.image_weights: + # Generate indices + if rank in [-1, 0]: + cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights + iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights + dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx + # Broadcast if DDP + if rank != -1: + indices = (torch.tensor(dataset.indices) if rank == 0 else torch.zeros(dataset.n)).int() + dist.broadcast(indices, 0) + if rank != 0: + dataset.indices = indices.cpu().numpy() + + # Update mosaic border + # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs) + # dataset.mosaic_border = [b - imgsz, -b] # height, width borders + + mloss = torch.zeros(4, device=device) # mean losses + if rank != -1: + dataloader.sampler.set_epoch(epoch) + pbar = enumerate(dataloader) + logger.info(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'total', 'labels', 'img_size')) + if rank in [-1, 0]: + pbar = tqdm(pbar, total=nb) # progress bar + optimizer.zero_grad() + for i, (imgs, targets, paths, _) in pbar: # batch ------------------------------------------------------------- + ni = i + nb * epoch # number integrated batches (since train start) + imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0 + + # Warmup + if ni <= nw: + xi = [0, nw] # x interp + # model.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou) + accumulate = max(1, np.interp(ni, xi, [1, nbs / total_batch_size]).round()) + for j, x in enumerate(optimizer.param_groups): + # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)]) + if 'momentum' in x: + x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]) + + # Multi-scale + if opt.multi_scale: + sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple) + imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) + + # Forward + with amp.autocast(enabled=cuda): + pred = model(imgs) # forward + loss, loss_items = compute_loss_ota(pred, targets.to(device), imgs) # loss scaled by batch_size + if rank != -1: + loss *= opt.world_size # gradient averaged between devices in DDP mode + if opt.quad: + loss *= 4. + + # Backward + scaler.scale(loss).backward() + + # Optimize + if ni % accumulate == 0: + scaler.step(optimizer) # optimizer.step + scaler.update() + optimizer.zero_grad() + if ema: + ema.update(model) + + # Print + if rank in [-1, 0]: + mloss = (mloss * i + loss_items) / (i + 1) # update mean losses + mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB) + s = ('%10s' * 2 + '%10.4g' * 6) % ( + '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1]) + pbar.set_description(s) + + # Plot + if plots and ni < 10: + f = save_dir / f'train_batch{ni}.jpg' # filename + Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start() + # if tb_writer: + # tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch) + # tb_writer.add_graph(torch.jit.trace(model, imgs, strict=False), []) # add model graph + elif plots and ni == 10 and wandb_logger.wandb: + wandb_logger.log({"Mosaics": [wandb_logger.wandb.Image(str(x), caption=x.name) for x in + save_dir.glob('train*.jpg') if x.exists()]}) + + # end batch ------------------------------------------------------------------------------------------------ + # end epoch ---------------------------------------------------------------------------------------------------- + + # Scheduler + lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard + scheduler.step() + + # DDP process 0 or single-GPU + if rank in [-1, 0]: + # mAP + ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride', 'class_weights']) + final_epoch = epoch + 1 == epochs + if not opt.notest or final_epoch: # Calculate mAP + wandb_logger.current_epoch = epoch + 1 + results, maps, times = test.test(data_dict, + batch_size=batch_size * 2, + imgsz=imgsz_test, + model=ema.ema, + single_cls=opt.single_cls, + dataloader=testloader, + save_dir=save_dir, + verbose=nc < 50 and final_epoch, + plots=plots and final_epoch, + wandb_logger=wandb_logger, + compute_loss=compute_loss, + is_coco=is_coco) + + # Write + with open(results_file, 'a') as f: + f.write(s + '%10.4g' * 7 % results + '\n') # append metrics, val_loss + if len(opt.name) and opt.bucket: + os.system('gsutil cp %s gs://%s/results/results%s.txt' % (results_file, opt.bucket, opt.name)) + + # Log + tags = ['train/box_loss', 'train/obj_loss', 'train/cls_loss', # train loss + 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', + 'val/box_loss', 'val/obj_loss', 'val/cls_loss', # val loss + 'x/lr0', 'x/lr1', 'x/lr2'] # params + for x, tag in zip(list(mloss[:-1]) + list(results) + lr, tags): + if tb_writer: + tb_writer.add_scalar(tag, x, epoch) # tensorboard + if wandb_logger.wandb: + wandb_logger.log({tag: x}) # W&B + + # Update best mAP + fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] + if fi > best_fitness: + best_fitness = fi + wandb_logger.end_epoch(best_result=best_fitness == fi) + + # Save model + if (not opt.nosave) or (final_epoch and not opt.evolve): # if save + ckpt = {'epoch': epoch, + 'best_fitness': best_fitness, + 'training_results': results_file.read_text(), + 'model': deepcopy(model.module if is_parallel(model) else model).half(), + 'ema': deepcopy(ema.ema).half(), + 'updates': ema.updates, + 'optimizer': optimizer.state_dict(), + 'wandb_id': wandb_logger.wandb_run.id if wandb_logger.wandb else None} + + # Save last, best and delete + torch.save(ckpt, last) + if best_fitness == fi: + torch.save(ckpt, best) + if (best_fitness == fi) and (epoch >= 200): + torch.save(ckpt, wdir / 'best_{:03d}.pt'.format(epoch)) + if epoch == 0: + torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + elif ((epoch+1) % 25) == 0: + torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + elif epoch >= (epochs-5): + torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + if wandb_logger.wandb: + if ((epoch + 1) % opt.save_period == 0 and not final_epoch) and opt.save_period != -1: + wandb_logger.log_model( + last.parent, opt, epoch, fi, best_model=best_fitness == fi) + del ckpt + + # end epoch ---------------------------------------------------------------------------------------------------- + # end training + if rank in [-1, 0]: + # Plots + if plots: + plot_results(save_dir=save_dir) # save as results.png + if wandb_logger.wandb: + files = ['results.png', 'confusion_matrix.png', *[f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R')]] + wandb_logger.log({"Results": [wandb_logger.wandb.Image(str(save_dir / f), caption=f) for f in files + if (save_dir / f).exists()]}) + # Test best.pt + logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) + if opt.data.endswith('coco.yaml') and nc == 80: # if COCO + for m in (last, best) if best.exists() else (last): # speed, mAP tests + results, _, _ = test.test(opt.data, + batch_size=batch_size * 2, + imgsz=imgsz_test, + conf_thres=0.001, + iou_thres=0.7, + model=attempt_load(m, device).half(), + single_cls=opt.single_cls, + dataloader=testloader, + save_dir=save_dir, + save_json=True, + plots=False, + is_coco=is_coco) + + # Strip optimizers + final = best if best.exists() else last # final model + for f in last, best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if opt.bucket: + os.system(f'gsutil cp {final} gs://{opt.bucket}/weights') # upload + if wandb_logger.wandb and not opt.evolve: # Log the stripped model + wandb_logger.wandb.log_artifact(str(final), type='model', + name='run_' + wandb_logger.wandb_run.id + '_model', + aliases=['last', 'best', 'stripped']) + wandb_logger.finish_run() + else: + dist.destroy_process_group() + torch.cuda.empty_cache() + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='COCO', help='dataset name') + + parser.add_argument('--weights', type=str, default='yolo7.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default='data/coco.yaml', help='data.yaml path') + parser.add_argument('--hyp', type=str, default='data/hyp.scratch.p5.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=300) + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') + parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='[train, test] image sizes') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--notest', action='store_true', help='only test final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check') + parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache-images', action='store_true', help='cache images for faster training') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') + parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') + parser.add_argument('--project', default='runs/train', help='save to project/name') + parser.add_argument('--entity', default=None, help='W&B entity') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--linear-lr', action='store_true', help='linear LR') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--upload_dataset', action='store_true', help='Upload dataset as W&B artifact table') + parser.add_argument('--bbox_interval', type=int, default=-1, help='Set bounding-box image logging interval for W&B') + parser.add_argument('--save_period', type=int, default=-1, help='Log model after every "save_period" epoch') + parser.add_argument('--artifact_alias', type=str, default="latest", help='version of dataset artifact to be used') + opt = parser.parse_args() + + # Set DDP variables + opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1 + opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1 + set_logging(opt.global_rank) + #if opt.global_rank in [-1, 0]: + # check_git_status() + # check_requirements() + + # Resume + wandb_run = check_wandb_resume(opt) + if opt.resume and not wandb_run: # resume an interrupted run + ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path + assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' + apriori = opt.global_rank, opt.local_rank + with open(Path(ckpt).parent.parent / 'opt.yaml') as f: + opt = argparse.Namespace(**yaml.load(f, Loader=yaml.SafeLoader)) # replace + opt.cfg, opt.weights, opt.resume, opt.batch_size, opt.global_rank, opt.local_rank = '', ckpt, True, opt.total_batch_size, *apriori # reinstate + logger.info('Resuming training from %s' % ckpt) + else: + # opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml') + opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files + assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' + opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test) + opt.name = 'evolve' if opt.evolve else opt.name + opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok | opt.evolve) # increment run + + # DDP mode + opt.total_batch_size = opt.batch_size + device = select_device(opt.device, batch_size=opt.batch_size) + if opt.local_rank != -1: + assert torch.cuda.device_count() > opt.local_rank + torch.cuda.set_device(opt.local_rank) + device = torch.device('cuda', opt.local_rank) + dist.init_process_group(backend='nccl', init_method='env://') # distributed backend + assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count' + opt.batch_size = opt.total_batch_size // opt.world_size + + # Hyperparameters + with open(opt.hyp) as f: + hyp = yaml.load(f, Loader=yaml.SafeLoader) # load hyps + + # Train + logger.info(opt) + if not opt.evolve: + tb_writer = None # init loggers + if opt.global_rank in [-1, 0]: + prefix = colorstr('tensorboard: ') + logger.info(f"{prefix}Start with 'tensorboard --logdir {opt.project}', view at http://localhost:6006/") + tb_writer = SummaryWriter(opt.save_dir) # Tensorboard + train(hyp, opt, device, tb_writer) + + # Evolve hyperparameters (optional) + else: + # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) + meta = {'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) + 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + 'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1 + 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay + 'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok) + 'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum + 'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr + 'box': (1, 0.02, 0.2), # box loss gain + 'cls': (1, 0.2, 4.0), # cls loss gain + 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight + 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) + 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight + 'iou_t': (0, 0.1, 0.7), # IoU training threshold + 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold + 'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore) + 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) + 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) + 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) + 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) + 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) + 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) + 'scale': (1, 0.0, 0.9), # image scale (+/- gain) + 'shear': (1, 0.0, 10.0), # image shear (+/- deg) + 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) + 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) + 'mosaic': (1, 0.0, 1.0), # image mixup (probability) + 'mixup': (1, 0.0, 1.0)} # image mixup (probability) + + assert opt.local_rank == -1, 'DDP mode not implemented for --evolve' + opt.notest, opt.nosave = True, True # only test/save final epoch + # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices + yaml_file = Path(opt.save_dir) / 'hyp_evolved.yaml' # save best result here + if opt.bucket: + os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket) # download evolve.txt if exists + + for _ in range(300): # generations to evolve + if Path('evolve.txt').exists(): # if evolve.txt exists: select best hyps and mutate + # Select parent(s) + parent = 'single' # parent selection method: 'single' or 'weighted' + x = np.loadtxt('evolve.txt', ndmin=2) + n = min(5, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness(x))][:n] # top n mutations + w = fitness(x) - fitness(x).min() # weights + if parent == 'single' or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == 'weighted': + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + mp, s = 0.8, 0.2 # mutation probability, sigma + npr = np.random + npr.seed(int(time.time())) + g = np.array([x[0] for x in meta.values()]) # gains 0-1 + ng = len(meta) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) + for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) + hyp[k] = float(x[i + 7] * v[i]) # mutate + + # Constrain to limits + for k, v in meta.items(): + hyp[k] = max(hyp[k], v[1]) # lower limit + hyp[k] = min(hyp[k], v[2]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + # Train mutation + results = train(hyp.copy(), opt, device) + + # Write mutation results + print_mutation(hyp.copy(), results, yaml_file, opt.bucket) + + # Plot results + plot_evolution(yaml_file) + print(f'Hyperparameter evolution complete. Best results saved as: {yaml_file}\n' + f'Command to train a new model with these hyperparameters: $ python train.py --hyp {yaml_file}') diff --git a/test/yolov7-tracker/train_aux.py b/test/yolov7-tracker/train_aux.py new file mode 100644 index 0000000..35811c6 --- /dev/null +++ b/test/yolov7-tracker/train_aux.py @@ -0,0 +1,693 @@ +import argparse +import logging +import math +import os +import random +import time +from copy import deepcopy +from pathlib import Path +from threading import Thread + +import numpy as np +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torch.optim.lr_scheduler as lr_scheduler +import torch.utils.data +import yaml +from torch.cuda import amp +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.tensorboard import SummaryWriter +from tqdm import tqdm + +import test # import test.py to get mAP after each epoch +from models.experimental import attempt_load +from models.yolo import Model +from utils.autoanchor import check_anchors +from utils.datasets import create_dataloader +from utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \ + fitness, strip_optimizer, get_latest_run, check_dataset, check_file, check_git_status, check_img_size, \ + check_requirements, print_mutation, set_logging, one_cycle, colorstr +from utils.google_utils import attempt_download +from utils.loss import ComputeLoss, ComputeLossAuxOTA +from utils.plots import plot_images, plot_labels, plot_results, plot_evolution +from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first, is_parallel +from utils.wandb_logging.wandb_utils import WandbLogger, check_wandb_resume + +logger = logging.getLogger(__name__) + + +def train(hyp, opt, device, tb_writer=None): + logger.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items())) + save_dir, epochs, batch_size, total_batch_size, weights, rank = \ + Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank + + # Directories + wdir = save_dir / 'weights' + wdir.mkdir(parents=True, exist_ok=True) # make dir + last = wdir / 'last.pt' + best = wdir / 'best.pt' + results_file = save_dir / 'results.txt' + + # Save run settings + with open(save_dir / 'hyp.yaml', 'w') as f: + yaml.dump(hyp, f, sort_keys=False) + with open(save_dir / 'opt.yaml', 'w') as f: + yaml.dump(vars(opt), f, sort_keys=False) + + # Configure + plots = not opt.evolve # create plots + cuda = device.type != 'cpu' + init_seeds(2 + rank) + with open(opt.data) as f: + data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict + is_coco = opt.data.endswith('coco.yaml') + + # Logging- Doing this before checking the dataset. Might update data_dict + loggers = {'wandb': None} # loggers dict + if rank in [-1, 0]: + opt.hyp = hyp # add hyperparameters + run_id = torch.load(weights).get('wandb_id') if weights.endswith('.pt') and os.path.isfile(weights) else None + wandb_logger = WandbLogger(opt, Path(opt.save_dir).stem, run_id, data_dict) + loggers['wandb'] = wandb_logger.wandb + data_dict = wandb_logger.data_dict + if wandb_logger.wandb: + weights, epochs, hyp = opt.weights, opt.epochs, opt.hyp # WandbLogger might update weights, epochs if resuming + + nc = 1 if opt.single_cls else int(data_dict['nc']) # number of classes + names = ['item'] if opt.single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names + assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check + + # Model + pretrained = weights.endswith('.pt') + if pretrained: + with torch_distributed_zero_first(rank): + attempt_download(weights) # download if not found locally + ckpt = torch.load(weights, map_location=device) # load checkpoint + model = Model(opt.cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + exclude = ['anchor'] if (opt.cfg or hyp.get('anchors')) and not opt.resume else [] # exclude keys + state_dict = ckpt['model'].float().state_dict() # to FP32 + state_dict = intersect_dicts(state_dict, model.state_dict(), exclude=exclude) # intersect + model.load_state_dict(state_dict, strict=False) # load + logger.info('Transferred %g/%g items from %s' % (len(state_dict), len(model.state_dict()), weights)) # report + else: + model = Model(opt.cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + with torch_distributed_zero_first(rank): + check_dataset(data_dict) # check + train_path = data_dict['train'] + test_path = data_dict['val'] + + # Freeze + freeze = [] # parameter names to freeze (full or partial) + for k, v in model.named_parameters(): + v.requires_grad = True # train all layers + if any(x in k for x in freeze): + print('freezing %s' % k) + v.requires_grad = False + + # Optimizer + nbs = 64 # nominal batch size + accumulate = max(round(nbs / total_batch_size), 1) # accumulate loss before optimizing + hyp['weight_decay'] *= total_batch_size * accumulate / nbs # scale weight_decay + logger.info(f"Scaled weight_decay = {hyp['weight_decay']}") + + pg0, pg1, pg2 = [], [], [] # optimizer parameter groups + for k, v in model.named_modules(): + if hasattr(v, 'bias') and isinstance(v.bias, nn.Parameter): + pg2.append(v.bias) # biases + if isinstance(v, nn.BatchNorm2d): + pg0.append(v.weight) # no decay + elif hasattr(v, 'weight') and isinstance(v.weight, nn.Parameter): + pg1.append(v.weight) # apply decay + if hasattr(v, 'im'): + if hasattr(v.im, 'implicit'): + pg0.append(v.im.implicit) + else: + for iv in v.im: + pg0.append(iv.implicit) + if hasattr(v, 'imc'): + if hasattr(v.imc, 'implicit'): + pg0.append(v.imc.implicit) + else: + for iv in v.imc: + pg0.append(iv.implicit) + if hasattr(v, 'imb'): + if hasattr(v.imb, 'implicit'): + pg0.append(v.imb.implicit) + else: + for iv in v.imb: + pg0.append(iv.implicit) + if hasattr(v, 'imo'): + if hasattr(v.imo, 'implicit'): + pg0.append(v.imo.implicit) + else: + for iv in v.imo: + pg0.append(iv.implicit) + if hasattr(v, 'ia'): + if hasattr(v.ia, 'implicit'): + pg0.append(v.ia.implicit) + else: + for iv in v.ia: + pg0.append(iv.implicit) + if hasattr(v, 'attn'): + if hasattr(v.attn, 'logit_scale'): + pg0.append(v.attn.logit_scale) + if hasattr(v.attn, 'q_bias'): + pg0.append(v.attn.q_bias) + if hasattr(v.attn, 'v_bias'): + pg0.append(v.attn.v_bias) + if hasattr(v.attn, 'relative_position_bias_table'): + pg0.append(v.attn.relative_position_bias_table) + if hasattr(v, 'rbr_dense'): + if hasattr(v.rbr_dense, 'weight_rbr_origin'): + pg0.append(v.rbr_dense.weight_rbr_origin) + if hasattr(v.rbr_dense, 'weight_rbr_avg_conv'): + pg0.append(v.rbr_dense.weight_rbr_avg_conv) + if hasattr(v.rbr_dense, 'weight_rbr_pfir_conv'): + pg0.append(v.rbr_dense.weight_rbr_pfir_conv) + if hasattr(v.rbr_dense, 'weight_rbr_1x1_kxk_idconv1'): + pg0.append(v.rbr_dense.weight_rbr_1x1_kxk_idconv1) + if hasattr(v.rbr_dense, 'weight_rbr_1x1_kxk_conv2'): + pg0.append(v.rbr_dense.weight_rbr_1x1_kxk_conv2) + if hasattr(v.rbr_dense, 'weight_rbr_gconv_dw'): + pg0.append(v.rbr_dense.weight_rbr_gconv_dw) + if hasattr(v.rbr_dense, 'weight_rbr_gconv_pw'): + pg0.append(v.rbr_dense.weight_rbr_gconv_pw) + if hasattr(v.rbr_dense, 'vector'): + pg0.append(v.rbr_dense.vector) + + if opt.adam: + optimizer = optim.Adam(pg0, lr=hyp['lr0'], betas=(hyp['momentum'], 0.999)) # adjust beta1 to momentum + else: + optimizer = optim.SGD(pg0, lr=hyp['lr0'], momentum=hyp['momentum'], nesterov=True) + + optimizer.add_param_group({'params': pg1, 'weight_decay': hyp['weight_decay']}) # add pg1 with weight_decay + optimizer.add_param_group({'params': pg2}) # add pg2 (biases) + logger.info('Optimizer groups: %g .bias, %g conv.weight, %g other' % (len(pg2), len(pg1), len(pg0))) + del pg0, pg1, pg2 + + # Scheduler https://arxiv.org/pdf/1812.01187.pdf + # https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html#OneCycleLR + if opt.linear_lr: + lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear + else: + lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf'] + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) + # plot_lr_scheduler(optimizer, scheduler, epochs) + + # EMA + ema = ModelEMA(model) if rank in [-1, 0] else None + + # Resume + start_epoch, best_fitness = 0, 0.0 + if pretrained: + # Optimizer + if ckpt['optimizer'] is not None: + optimizer.load_state_dict(ckpt['optimizer']) + best_fitness = ckpt['best_fitness'] + + # EMA + if ema and ckpt.get('ema'): + ema.ema.load_state_dict(ckpt['ema'].float().state_dict()) + ema.updates = ckpt['updates'] + + # Results + if ckpt.get('training_results') is not None: + results_file.write_text(ckpt['training_results']) # write results.txt + + # Epochs + start_epoch = ckpt['epoch'] + 1 + if opt.resume: + assert start_epoch > 0, '%s training to %g epochs is finished, nothing to resume.' % (weights, epochs) + if epochs < start_epoch: + logger.info('%s has been trained for %g epochs. Fine-tuning for %g additional epochs.' % + (weights, ckpt['epoch'], epochs)) + epochs += ckpt['epoch'] # finetune additional epochs + + del ckpt, state_dict + + # Image sizes + gs = max(int(model.stride.max()), 32) # grid size (max stride) + nl = model.model[-1].nl # number of detection layers (used for scaling hyp['obj']) + imgsz, imgsz_test = [check_img_size(x, gs) for x in opt.img_size] # verify imgsz are gs-multiples + + # DP mode + if cuda and rank == -1 and torch.cuda.device_count() > 1: + model = torch.nn.DataParallel(model) + + # SyncBatchNorm + if opt.sync_bn and cuda and rank != -1: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device) + logger.info('Using SyncBatchNorm()') + + # Trainloader + dataloader, dataset = create_dataloader(train_path, imgsz, batch_size, gs, opt, + hyp=hyp, augment=True, cache=opt.cache_images, rect=opt.rect, rank=rank, + world_size=opt.world_size, workers=opt.workers, + image_weights=opt.image_weights, quad=opt.quad, prefix=colorstr('train: ')) + mlc = np.concatenate(dataset.labels, 0)[:, 0].max() # max label class + nb = len(dataloader) # number of batches + assert mlc < nc, 'Label class %g exceeds nc=%g in %s. Possible class labels are 0-%g' % (mlc, nc, opt.data, nc - 1) + + # Process 0 + if rank in [-1, 0]: + testloader = create_dataloader(test_path, imgsz_test, batch_size * 2, gs, opt, # testloader + hyp=hyp, cache=opt.cache_images and not opt.notest, rect=True, rank=-1, + world_size=opt.world_size, workers=opt.workers, + pad=0.5, prefix=colorstr('val: '))[0] + + if not opt.resume: + labels = np.concatenate(dataset.labels, 0) + c = torch.tensor(labels[:, 0]) # classes + # cf = torch.bincount(c.long(), minlength=nc) + 1. # frequency + # model._initialize_biases(cf.to(device)) + if plots: + #plot_labels(labels, names, save_dir, loggers) + if tb_writer: + tb_writer.add_histogram('classes', c, 0) + + # Anchors + if not opt.noautoanchor: + check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz) + model.half().float() # pre-reduce anchor precision + + # DDP mode + if cuda and rank != -1: + model = DDP(model, device_ids=[opt.local_rank], output_device=opt.local_rank, + # nn.MultiheadAttention incompatibility with DDP https://github.com/pytorch/pytorch/issues/26698 + find_unused_parameters=any(isinstance(layer, nn.MultiheadAttention) for layer in model.modules())) + + # Model parameters + hyp['box'] *= 3. / nl # scale to layers + hyp['cls'] *= nc / 80. * 3. / nl # scale to classes and layers + hyp['obj'] *= (imgsz / 640) ** 2 * 3. / nl # scale to image size and layers + hyp['label_smoothing'] = opt.label_smoothing + model.nc = nc # attach number of classes to model + model.hyp = hyp # attach hyperparameters to model + model.gr = 1.0 # iou loss ratio (obj_loss = 1.0 or iou) + model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights + model.names = names + + # Start training + t0 = time.time() + nw = max(round(hyp['warmup_epochs'] * nb), 1000) # number of warmup iterations, max(3 epochs, 1k iterations) + # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training + maps = np.zeros(nc) # mAP per class + results = (0, 0, 0, 0, 0, 0, 0) # P, R, mAP@.5, mAP@.5-.95, val_loss(box, obj, cls) + scheduler.last_epoch = start_epoch - 1 # do not move + scaler = amp.GradScaler(enabled=cuda) + compute_loss_ota = ComputeLossAuxOTA(model) # init loss class + compute_loss = ComputeLoss(model) # init loss class + logger.info(f'Image sizes {imgsz} train, {imgsz_test} test\n' + f'Using {dataloader.num_workers} dataloader workers\n' + f'Logging results to {save_dir}\n' + f'Starting training for {epochs} epochs...') + torch.save(model, wdir / 'init.pt') + for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------ + model.train() + + # Update image weights (optional) + if opt.image_weights: + # Generate indices + if rank in [-1, 0]: + cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights + iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights + dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx + # Broadcast if DDP + if rank != -1: + indices = (torch.tensor(dataset.indices) if rank == 0 else torch.zeros(dataset.n)).int() + dist.broadcast(indices, 0) + if rank != 0: + dataset.indices = indices.cpu().numpy() + + # Update mosaic border + # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs) + # dataset.mosaic_border = [b - imgsz, -b] # height, width borders + + mloss = torch.zeros(4, device=device) # mean losses + if rank != -1: + dataloader.sampler.set_epoch(epoch) + pbar = enumerate(dataloader) + logger.info(('\n' + '%10s' * 8) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'total', 'labels', 'img_size')) + if rank in [-1, 0]: + pbar = tqdm(pbar, total=nb) # progress bar + optimizer.zero_grad() + for i, (imgs, targets, paths, _) in pbar: # batch ------------------------------------------------------------- + ni = i + nb * epoch # number integrated batches (since train start) + imgs = imgs.to(device, non_blocking=True).float() / 255.0 # uint8 to float32, 0-255 to 0.0-1.0 + + # Warmup + if ni <= nw: + xi = [0, nw] # x interp + # model.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou) + accumulate = max(1, np.interp(ni, xi, [1, nbs / total_batch_size]).round()) + for j, x in enumerate(optimizer.param_groups): + # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)]) + if 'momentum' in x: + x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]) + + # Multi-scale + if opt.multi_scale: + sz = random.randrange(imgsz * 0.5, imgsz * 1.5 + gs) // gs * gs # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple) + imgs = F.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) + + # Forward + with amp.autocast(enabled=cuda): + pred = model(imgs) # forward + loss, loss_items = compute_loss_ota(pred, targets.to(device), imgs) # loss scaled by batch_size + if rank != -1: + loss *= opt.world_size # gradient averaged between devices in DDP mode + if opt.quad: + loss *= 4. + + # Backward + scaler.scale(loss).backward() + + # Optimize + if ni % accumulate == 0: + scaler.step(optimizer) # optimizer.step + scaler.update() + optimizer.zero_grad() + if ema: + ema.update(model) + + # Print + if rank in [-1, 0]: + mloss = (mloss * i + loss_items) / (i + 1) # update mean losses + mem = '%.3gG' % (torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0) # (GB) + s = ('%10s' * 2 + '%10.4g' * 6) % ( + '%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1]) + pbar.set_description(s) + + # Plot + if plots and ni < 10: + f = save_dir / f'train_batch{ni}.jpg' # filename + Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start() + # if tb_writer: + # tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch) + # tb_writer.add_graph(torch.jit.trace(model, imgs, strict=False), []) # add model graph + elif plots and ni == 10 and wandb_logger.wandb: + wandb_logger.log({"Mosaics": [wandb_logger.wandb.Image(str(x), caption=x.name) for x in + save_dir.glob('train*.jpg') if x.exists()]}) + + # end batch ------------------------------------------------------------------------------------------------ + # end epoch ---------------------------------------------------------------------------------------------------- + + # Scheduler + lr = [x['lr'] for x in optimizer.param_groups] # for tensorboard + scheduler.step() + + # DDP process 0 or single-GPU + if rank in [-1, 0]: + # mAP + ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'gr', 'names', 'stride', 'class_weights']) + final_epoch = epoch + 1 == epochs + if not opt.notest or final_epoch: # Calculate mAP + wandb_logger.current_epoch = epoch + 1 + results, maps, times = test.test(data_dict, + batch_size=batch_size * 2, + imgsz=imgsz_test, + model=ema.ema, + single_cls=opt.single_cls, + dataloader=testloader, + save_dir=save_dir, + verbose=nc < 50 and final_epoch, + plots=plots and final_epoch, + wandb_logger=wandb_logger, + compute_loss=compute_loss, + is_coco=is_coco) + + # Write + with open(results_file, 'a') as f: + f.write(s + '%10.4g' * 7 % results + '\n') # append metrics, val_loss + if len(opt.name) and opt.bucket: + os.system('gsutil cp %s gs://%s/results/results%s.txt' % (results_file, opt.bucket, opt.name)) + + # Log + tags = ['train/box_loss', 'train/obj_loss', 'train/cls_loss', # train loss + 'metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', + 'val/box_loss', 'val/obj_loss', 'val/cls_loss', # val loss + 'x/lr0', 'x/lr1', 'x/lr2'] # params + for x, tag in zip(list(mloss[:-1]) + list(results) + lr, tags): + if tb_writer: + tb_writer.add_scalar(tag, x, epoch) # tensorboard + if wandb_logger.wandb: + wandb_logger.log({tag: x}) # W&B + + # Update best mAP + fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] + if fi > best_fitness: + best_fitness = fi + wandb_logger.end_epoch(best_result=best_fitness == fi) + + # Save model + if (not opt.nosave) or (final_epoch and not opt.evolve): # if save + ckpt = {'epoch': epoch, + 'best_fitness': best_fitness, + 'training_results': results_file.read_text(), + 'model': deepcopy(model.module if is_parallel(model) else model).half(), + 'ema': deepcopy(ema.ema).half(), + 'updates': ema.updates, + 'optimizer': optimizer.state_dict(), + 'wandb_id': wandb_logger.wandb_run.id if wandb_logger.wandb else None} + + # Save last, best and delete + torch.save(ckpt, last) + if best_fitness == fi: # best 衡量的标准是0.1*mAP@0.5 + 0.9*mAP@0.5:0.95 + torch.save(ckpt, best) + if (best_fitness == fi) and (epoch >= 200): + torch.save(ckpt, wdir / 'best_{:03d}.pt'.format(epoch)) + # if epoch == 0: + # torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + # elif ((epoch+1) % 25) == 0: + # torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + # elif epoch >= (epochs-5): + # torch.save(ckpt, wdir / 'epoch_{:03d}.pt'.format(epoch)) + if wandb_logger.wandb: + if ((epoch + 1) % opt.save_period == 0 and not final_epoch) and opt.save_period != -1: + wandb_logger.log_model( + last.parent, opt, epoch, fi, best_model=best_fitness == fi) + del ckpt + + # end epoch ---------------------------------------------------------------------------------------------------- + # end training + if rank in [-1, 0]: + # Plots + if plots: + plot_results(save_dir=save_dir) # save as results.png + if wandb_logger.wandb: + files = ['results.png', 'confusion_matrix.png', *[f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R')]] + wandb_logger.log({"Results": [wandb_logger.wandb.Image(str(save_dir / f), caption=f) for f in files + if (save_dir / f).exists()]}) + # Test best.pt + logger.info('%g epochs completed in %.3f hours.\n' % (epoch - start_epoch + 1, (time.time() - t0) / 3600)) + if opt.data.endswith('coco.yaml') and nc == 80: # if COCO + for m in (last, best) if best.exists() else (last): # speed, mAP tests + results, _, _ = test.test(opt.data, + batch_size=batch_size * 2, + imgsz=imgsz_test, + conf_thres=0.001, + iou_thres=0.7, + model=attempt_load(m, device).half(), + single_cls=opt.single_cls, + dataloader=testloader, + save_dir=save_dir, + save_json=True, + plots=False, + is_coco=is_coco) + + # Strip optimizers + final = best if best.exists() else last # final model + for f in last, best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if opt.bucket: + os.system(f'gsutil cp {final} gs://{opt.bucket}/weights') # upload + if wandb_logger.wandb and not opt.evolve: # Log the stripped model + wandb_logger.wandb.log_artifact(str(final), type='model', + name='run_' + wandb_logger.wandb_run.id + '_model', + aliases=['last', 'best', 'stripped']) + wandb_logger.finish_run() + else: + dist.destroy_process_group() + torch.cuda.empty_cache() + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='COCO', help='dataset name') + + parser.add_argument('--weights', type=str, default='yolo7.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default='data/coco.yaml', help='data.yaml path') + parser.add_argument('--hyp', type=str, default='data/hyp.scratch.p5.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=30) + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs') + parser.add_argument('--img-size', nargs='+', type=int, default=[640, 640], help='[train, test] image sizes') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--notest', action='store_true', help='only test final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable autoanchor check') + parser.add_argument('--evolve', action='store_true', help='evolve hyperparameters') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache-images', action='store_true', help='cache images for faster training') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--adam', action='store_true', help='use torch.optim.Adam() optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--local_rank', type=int, default=-1, help='DDP parameter, do not modify') + parser.add_argument('--workers', type=int, default=8, help='maximum number of dataloader workers') + parser.add_argument('--project', default='runs/train', help='save to project/name') + parser.add_argument('--entity', default=None, help='W&B entity') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--linear-lr', action='store_true', help='linear LR') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--upload_dataset', action='store_true', help='Upload dataset as W&B artifact table') + parser.add_argument('--bbox_interval', type=int, default=-1, help='Set bounding-box image logging interval for W&B') + parser.add_argument('--save_period', type=int, default=-1, help='Log model after every "save_period" epoch') + parser.add_argument('--artifact_alias', type=str, default="latest", help='version of dataset artifact to be used') + opt = parser.parse_args() + + # Set DDP variables + opt.world_size = int(os.environ['WORLD_SIZE']) if 'WORLD_SIZE' in os.environ else 1 + opt.global_rank = int(os.environ['RANK']) if 'RANK' in os.environ else -1 + set_logging(opt.global_rank) + #if opt.global_rank in [-1, 0]: + # check_git_status() + # check_requirements() + + # Resume + wandb_run = check_wandb_resume(opt) + if opt.resume and not wandb_run: # resume an interrupted run + ckpt = opt.resume if isinstance(opt.resume, str) else get_latest_run() # specified or most recent path + assert os.path.isfile(ckpt), 'ERROR: --resume checkpoint does not exist' + apriori = opt.global_rank, opt.local_rank + with open(Path(ckpt).parent.parent / 'opt.yaml') as f: + opt = argparse.Namespace(**yaml.load(f, Loader=yaml.SafeLoader)) # replace + opt.cfg, opt.weights, opt.resume, opt.batch_size, opt.global_rank, opt.local_rank = '', ckpt, True, opt.total_batch_size, *apriori # reinstate + logger.info('Resuming training from %s' % ckpt) + else: + # opt.hyp = opt.hyp or ('hyp.finetune.yaml' if opt.weights else 'hyp.scratch.yaml') + opt.data, opt.cfg, opt.hyp = check_file(opt.data), check_file(opt.cfg), check_file(opt.hyp) # check files + assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' + opt.img_size.extend([opt.img_size[-1]] * (2 - len(opt.img_size))) # extend to 2 sizes (train, test) + opt.name = 'evolve' if opt.evolve else opt.name + opt.save_dir = increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok | opt.evolve) # increment run + + # DDP mode + opt.total_batch_size = opt.batch_size + device = select_device(opt.device, batch_size=opt.batch_size) + if opt.local_rank != -1: + assert torch.cuda.device_count() > opt.local_rank + torch.cuda.set_device(opt.local_rank) + device = torch.device('cuda', opt.local_rank) + dist.init_process_group(backend='nccl', init_method='env://') # distributed backend + assert opt.batch_size % opt.world_size == 0, '--batch-size must be multiple of CUDA device count' + opt.batch_size = opt.total_batch_size // opt.world_size + + # Hyperparameters + with open(opt.hyp) as f: + hyp = yaml.load(f, Loader=yaml.SafeLoader) # load hyps + + # Train + logger.info(opt) + if not opt.evolve: + tb_writer = None # init loggers + if opt.global_rank in [-1, 0]: + prefix = colorstr('tensorboard: ') + logger.info(f"{prefix}Start with 'tensorboard --logdir {opt.project}', view at http://localhost:8090/") + tb_writer = SummaryWriter(opt.save_dir) # Tensorboard + train(hyp, opt, device, tb_writer) + + # Evolve hyperparameters (optional) + else: + # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) + meta = {'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) + 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + 'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1 + 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay + 'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok) + 'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum + 'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr + 'box': (1, 0.02, 0.2), # box loss gain + 'cls': (1, 0.2, 4.0), # cls loss gain + 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight + 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) + 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight + 'iou_t': (0, 0.1, 0.7), # IoU training threshold + 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold + 'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore) + 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) + 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) + 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) + 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) + 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) + 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) + 'scale': (1, 0.0, 0.9), # image scale (+/- gain) + 'shear': (1, 0.0, 10.0), # image shear (+/- deg) + 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) + 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) + 'mosaic': (1, 0.0, 1.0), # image mixup (probability) + 'mixup': (1, 0.0, 1.0)} # image mixup (probability) + + assert opt.local_rank == -1, 'DDP mode not implemented for --evolve' + opt.notest, opt.nosave = True, True # only test/save final epoch + # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices + yaml_file = Path(opt.save_dir) / 'hyp_evolved.yaml' # save best result here + if opt.bucket: + os.system('gsutil cp gs://%s/evolve.txt .' % opt.bucket) # download evolve.txt if exists + + for _ in range(300): # generations to evolve + if Path('evolve.txt').exists(): # if evolve.txt exists: select best hyps and mutate + # Select parent(s) + parent = 'single' # parent selection method: 'single' or 'weighted' + x = np.loadtxt('evolve.txt', ndmin=2) + n = min(5, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness(x))][:n] # top n mutations + w = fitness(x) - fitness(x).min() # weights + if parent == 'single' or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == 'weighted': + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + mp, s = 0.8, 0.2 # mutation probability, sigma + npr = np.random + npr.seed(int(time.time())) + g = np.array([x[0] for x in meta.values()]) # gains 0-1 + ng = len(meta) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) + for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) + hyp[k] = float(x[i + 7] * v[i]) # mutate + + # Constrain to limits + for k, v in meta.items(): + hyp[k] = max(hyp[k], v[1]) # lower limit + hyp[k] = min(hyp[k], v[2]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + # Train mutation + results = train(hyp.copy(), opt, device) + + # Write mutation results + print_mutation(hyp.copy(), results, yaml_file, opt.bucket) + + # Plot results + plot_evolution(yaml_file) + print(f'Hyperparameter evolution complete. Best results saved as: {yaml_file}\n' + f'Command to train a new model with these hyperparameters: $ python train.py --hyp {yaml_file}') diff --git a/test/yolov7-tracker/utils/__init__.py b/test/yolov7-tracker/utils/__init__.py new file mode 100644 index 0000000..84952a8 --- /dev/null +++ b/test/yolov7-tracker/utils/__init__.py @@ -0,0 +1 @@ +# init \ No newline at end of file diff --git a/test/yolov7-tracker/utils/activations.py b/test/yolov7-tracker/utils/activations.py new file mode 100644 index 0000000..aa3ddf0 --- /dev/null +++ b/test/yolov7-tracker/utils/activations.py @@ -0,0 +1,72 @@ +# Activation functions + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +# SiLU https://arxiv.org/pdf/1606.08415.pdf ---------------------------------------------------------------------------- +class SiLU(nn.Module): # export-friendly version of nn.SiLU() + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +class Hardswish(nn.Module): # export-friendly version of nn.Hardswish() + @staticmethod + def forward(x): + # return x * F.hardsigmoid(x) # for torchscript and CoreML + return x * F.hardtanh(x + 3, 0., 6.) / 6. # for torchscript, CoreML and ONNX + + +class MemoryEfficientSwish(nn.Module): + class F(torch.autograd.Function): + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return x * torch.sigmoid(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + sx = torch.sigmoid(x) + return grad_output * (sx * (1 + x * (1 - sx))) + + def forward(self, x): + return self.F.apply(x) + + +# Mish https://github.com/digantamisra98/Mish -------------------------------------------------------------------------- +class Mish(nn.Module): + @staticmethod + def forward(x): + return x * F.softplus(x).tanh() + + +class MemoryEfficientMish(nn.Module): + class F(torch.autograd.Function): + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x))) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + sx = torch.sigmoid(x) + fx = F.softplus(x).tanh() + return grad_output * (fx + x * sx * (1 - fx * fx)) + + def forward(self, x): + return self.F.apply(x) + + +# FReLU https://arxiv.org/abs/2007.11824 ------------------------------------------------------------------------------- +class FReLU(nn.Module): + def __init__(self, c1, k=3): # ch_in, kernel + super().__init__() + self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False) + self.bn = nn.BatchNorm2d(c1) + + def forward(self, x): + return torch.max(x, self.bn(self.conv(x))) diff --git a/test/yolov7-tracker/utils/autoanchor.py b/test/yolov7-tracker/utils/autoanchor.py new file mode 100644 index 0000000..bec9017 --- /dev/null +++ b/test/yolov7-tracker/utils/autoanchor.py @@ -0,0 +1,160 @@ +# Auto-anchor utils + +import numpy as np +import torch +import yaml +from scipy.cluster.vq import kmeans +from tqdm import tqdm + +from utils.general import colorstr + + +def check_anchor_order(m): + # Check anchor order against stride order for YOLO Detect() module m, and correct if necessary + a = m.anchor_grid.prod(-1).view(-1) # anchor area + da = a[-1] - a[0] # delta a + ds = m.stride[-1] - m.stride[0] # delta s + if da.sign() != ds.sign(): # same order + print('Reversing anchor order') + m.anchors[:] = m.anchors.flip(0) + m.anchor_grid[:] = m.anchor_grid.flip(0) + + +def check_anchors(dataset, model, thr=4.0, imgsz=640): + # Check anchor fit to data, recompute if necessary + prefix = colorstr('autoanchor: ') + print(f'\n{prefix}Analyzing anchors... ', end='') + m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect() + shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True) + scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale + wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh + + def metric(k): # compute metric + r = wh[:, None] / k[None] + x = torch.min(r, 1. / r).min(2)[0] # ratio metric + best = x.max(1)[0] # best_x + aat = (x > 1. / thr).float().sum(1).mean() # anchors above threshold + bpr = (best > 1. / thr).float().mean() # best possible recall + return bpr, aat + + anchors = m.anchor_grid.clone().cpu().view(-1, 2) # current anchors + bpr, aat = metric(anchors) + print(f'anchors/target = {aat:.2f}, Best Possible Recall (BPR) = {bpr:.4f}', end='') + if bpr < 0.98: # threshold to recompute + print('. Attempting to improve anchors, please wait...') + na = m.anchor_grid.numel() // 2 # number of anchors + try: + anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False) + except Exception as e: + print(f'{prefix}ERROR: {e}') + new_bpr = metric(anchors)[0] + if new_bpr > bpr: # replace anchors + anchors = torch.tensor(anchors, device=m.anchors.device).type_as(m.anchors) + m.anchor_grid[:] = anchors.clone().view_as(m.anchor_grid) # for inference + m.anchors[:] = anchors.clone().view_as(m.anchors) / m.stride.to(m.anchors.device).view(-1, 1, 1) # loss + check_anchor_order(m) + print(f'{prefix}New anchors saved to model. Update model *.yaml to use these anchors in the future.') + else: + print(f'{prefix}Original anchors better than new anchors. Proceeding with original anchors.') + print('') # newline + + +def kmean_anchors(path='./data/coco.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True): + """ Creates kmeans-evolved anchors from training dataset + + Arguments: + path: path to dataset *.yaml, or a loaded dataset + n: number of anchors + img_size: image size used for training + thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0 + gen: generations to evolve anchors using genetic algorithm + verbose: print all results + + Return: + k: kmeans evolved anchors + + Usage: + from utils.autoanchor import *; _ = kmean_anchors() + """ + thr = 1. / thr + prefix = colorstr('autoanchor: ') + + def metric(k, wh): # compute metrics + r = wh[:, None] / k[None] + x = torch.min(r, 1. / r).min(2)[0] # ratio metric + # x = wh_iou(wh, torch.tensor(k)) # iou metric + return x, x.max(1)[0] # x, best_x + + def anchor_fitness(k): # mutation fitness + _, best = metric(torch.tensor(k, dtype=torch.float32), wh) + return (best * (best > thr).float()).mean() # fitness + + def print_results(k): + k = k[np.argsort(k.prod(1))] # sort small to large + x, best = metric(k, wh0) + bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr + print(f'{prefix}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr') + print(f'{prefix}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, ' + f'past_thr={x[x > thr].mean():.3f}-mean: ', end='') + for i, x in enumerate(k): + print('%i,%i' % (round(x[0]), round(x[1])), end=', ' if i < len(k) - 1 else '\n') # use in *.cfg + return k + + if isinstance(path, str): # *.yaml file + with open(path) as f: + data_dict = yaml.load(f, Loader=yaml.SafeLoader) # model dict + from utils.datasets import LoadImagesAndLabels + dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True) + else: + dataset = path # dataset + + # Get label wh + shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True) + wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh + + # Filter + i = (wh0 < 3.0).any(1).sum() + if i: + print(f'{prefix}WARNING: Extremely small objects found. {i} of {len(wh0)} labels are < 3 pixels in size.') + wh = wh0[(wh0 >= 2.0).any(1)] # filter > 2 pixels + # wh = wh * (np.random.rand(wh.shape[0], 1) * 0.9 + 0.1) # multiply by random scale 0-1 + + # Kmeans calculation + print(f'{prefix}Running kmeans for {n} anchors on {len(wh)} points...') + s = wh.std(0) # sigmas for whitening + k, dist = kmeans(wh / s, n, iter=30) # points, mean distance + assert len(k) == n, print(f'{prefix}ERROR: scipy.cluster.vq.kmeans requested {n} points but returned only {len(k)}') + k *= s + wh = torch.tensor(wh, dtype=torch.float32) # filtered + wh0 = torch.tensor(wh0, dtype=torch.float32) # unfiltered + k = print_results(k) + + # Plot + # k, d = [None] * 20, [None] * 20 + # for i in tqdm(range(1, 21)): + # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance + # fig, ax = plt.subplots(1, 2, figsize=(14, 7), tight_layout=True) + # ax = ax.ravel() + # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.') + # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh + # ax[0].hist(wh[wh[:, 0]<100, 0],400) + # ax[1].hist(wh[wh[:, 1]<100, 1],400) + # fig.savefig('wh.png', dpi=200) + + # Evolve + npr = np.random + f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma + pbar = tqdm(range(gen), desc=f'{prefix}Evolving anchors with Genetic Algorithm:') # progress bar + for _ in pbar: + v = np.ones(sh) + while (v == 1).all(): # mutate until a change occurs (prevent duplicates) + v = ((npr.random(sh) < mp) * npr.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0) + kg = (k.copy() * v).clip(min=2.0) + fg = anchor_fitness(kg) + if fg > f: + f, k = fg, kg.copy() + pbar.desc = f'{prefix}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}' + if verbose: + print_results(k) + + return print_results(k) diff --git a/test/yolov7-tracker/utils/aws/__init__.py b/test/yolov7-tracker/utils/aws/__init__.py new file mode 100644 index 0000000..e9691f2 --- /dev/null +++ b/test/yolov7-tracker/utils/aws/__init__.py @@ -0,0 +1 @@ +#init \ No newline at end of file diff --git a/test/yolov7-tracker/utils/aws/mime.sh b/test/yolov7-tracker/utils/aws/mime.sh new file mode 100644 index 0000000..c319a83 --- /dev/null +++ b/test/yolov7-tracker/utils/aws/mime.sh @@ -0,0 +1,26 @@ +# AWS EC2 instance startup 'MIME' script https://aws.amazon.com/premiumsupport/knowledge-center/execute-user-data-ec2/ +# This script will run on every instance restart, not only on first start +# --- DO NOT COPY ABOVE COMMENTS WHEN PASTING INTO USERDATA --- + +Content-Type: multipart/mixed; boundary="//" +MIME-Version: 1.0 + +--// +Content-Type: text/cloud-config; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cloud-config.txt" + +#cloud-config +cloud_final_modules: +- [scripts-user, always] + +--// +Content-Type: text/x-shellscript; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="userdata.txt" + +#!/bin/bash +# --- paste contents of userdata.sh here --- +--// diff --git a/test/yolov7-tracker/utils/aws/resume.py b/test/yolov7-tracker/utils/aws/resume.py new file mode 100644 index 0000000..338685b --- /dev/null +++ b/test/yolov7-tracker/utils/aws/resume.py @@ -0,0 +1,37 @@ +# Resume all interrupted trainings in yolor/ dir including DDP trainings +# Usage: $ python utils/aws/resume.py + +import os +import sys +from pathlib import Path + +import torch +import yaml + +sys.path.append('./') # to run '$ python *.py' files in subdirectories + +port = 0 # --master_port +path = Path('').resolve() +for last in path.rglob('*/**/last.pt'): + ckpt = torch.load(last) + if ckpt['optimizer'] is None: + continue + + # Load opt.yaml + with open(last.parent.parent / 'opt.yaml') as f: + opt = yaml.load(f, Loader=yaml.SafeLoader) + + # Get device count + d = opt['device'].split(',') # devices + nd = len(d) # number of devices + ddp = nd > 1 or (nd == 0 and torch.cuda.device_count() > 1) # distributed data parallel + + if ddp: # multi-GPU + port += 1 + cmd = f'python -m torch.distributed.launch --nproc_per_node {nd} --master_port {port} train.py --resume {last}' + else: # single-GPU + cmd = f'python train.py --resume {last}' + + cmd += ' > /dev/null 2>&1 &' # redirect output to dev/null and run in daemon thread + print(cmd) + os.system(cmd) diff --git a/test/yolov7-tracker/utils/aws/userdata.sh b/test/yolov7-tracker/utils/aws/userdata.sh new file mode 100644 index 0000000..5762ae5 --- /dev/null +++ b/test/yolov7-tracker/utils/aws/userdata.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# AWS EC2 instance startup script https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html +# This script will run only once on first instance start (for a re-start script see mime.sh) +# /home/ubuntu (ubuntu) or /home/ec2-user (amazon-linux) is working dir +# Use >300 GB SSD + +cd home/ubuntu +if [ ! -d yolor ]; then + echo "Running first-time script." # install dependencies, download COCO, pull Docker + git clone -b paper https://github.com/WongKinYiu/yolor && sudo chmod -R 777 yolor + cd yolor + bash data/scripts/get_coco.sh && echo "Data done." & + sudo docker pull nvcr.io/nvidia/pytorch:21.08-py3 && echo "Docker done." & + python -m pip install --upgrade pip && pip install -r requirements.txt && python detect.py && echo "Requirements done." & + wait && echo "All tasks done." # finish background tasks +else + echo "Running re-start script." # resume interrupted runs + i=0 + list=$(sudo docker ps -qa) # container list i.e. $'one\ntwo\nthree\nfour' + while IFS= read -r id; do + ((i++)) + echo "restarting container $i: $id" + sudo docker start $id + # sudo docker exec -it $id python train.py --resume # single-GPU + sudo docker exec -d $id python utils/aws/resume.py # multi-scenario + done <<<"$list" +fi diff --git a/test/yolov7-tracker/utils/datasets.py b/test/yolov7-tracker/utils/datasets.py new file mode 100644 index 0000000..98886ff --- /dev/null +++ b/test/yolov7-tracker/utils/datasets.py @@ -0,0 +1,1451 @@ +# Dataset utils and dataloaders + +import glob +import logging +import math +import os +import random +import shutil +import time +from itertools import repeat +from multiprocessing.pool import ThreadPool +from pathlib import Path +from threading import Thread + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image, ExifTags +from torch.utils.data import Dataset +from tqdm import tqdm + +import pickle +from copy import deepcopy +#from pycocotools import mask as maskUtils +from torchvision.utils import save_image +from torchvision.ops import roi_pool, roi_align, ps_roi_pool, ps_roi_align + +from utils.general import check_requirements, xyxy2xywh, xywh2xyxy, xywhn2xyxy, xyn2xy, segment2box, segments2boxes, \ + resample_segments, clean_str +from utils.torch_utils import torch_distributed_zero_first + +# Parameters +help_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data' +img_formats = ['bmp', 'jpg', 'jpeg', 'png', 'tif', 'tiff', 'dng', 'webp', 'mpo'] # acceptable image suffixes +vid_formats = ['mov', 'avi', 'mp4', 'mpg', 'mpeg', 'm4v', 'wmv', 'mkv'] # acceptable video suffixes +logger = logging.getLogger(__name__) + +# Get orientation exif tag +for orientation in ExifTags.TAGS.keys(): + if ExifTags.TAGS[orientation] == 'Orientation': + break + + +def get_hash(files): + # Returns a single hash value of a list of files + return sum(os.path.getsize(f) for f in files if os.path.isfile(f)) + + +def exif_size(img): + # Returns exif-corrected PIL size + s = img.size # (width, height) + try: + rotation = dict(img._getexif().items())[orientation] + if rotation == 6: # rotation 270 + s = (s[1], s[0]) + elif rotation == 8: # rotation 90 + s = (s[1], s[0]) + except: + pass + + return s + + +def create_dataloader(path, imgsz, batch_size, stride, opt, hyp=None, augment=False, cache=False, pad=0.0, rect=False, + rank=-1, world_size=1, workers=8, image_weights=False, quad=False, prefix=''): + # Make sure only the first process in DDP process the dataset first, and the following others can use the cache + with torch_distributed_zero_first(rank): + if opt.dataset == 'COCO': + dataset = LoadImagesAndLabels(path, imgsz, batch_size, + augment=augment, # augment images + hyp=hyp, # augmentation hyperparameters + rect=rect, # rectangular training + cache_images=cache, + single_cls=opt.single_cls, + stride=int(stride), + pad=pad, + image_weights=image_weights, + prefix=prefix) + else: + dataset = LoadImagesAndLabelsCustom(path, imgsz, batch_size, + augment=augment, # augment images + hyp=hyp, # augmentation hyperparameters + rect=rect, # rectangular training + cache_images=cache, + single_cls=opt.single_cls, + stride=int(stride), + pad=pad, + image_weights=image_weights, + prefix=prefix) + + batch_size = min(batch_size, len(dataset)) + nw = min([os.cpu_count() // world_size, batch_size if batch_size > 1 else 0, workers]) # number of workers + sampler = torch.utils.data.distributed.DistributedSampler(dataset) if rank != -1 else None + loader = torch.utils.data.DataLoader if image_weights else InfiniteDataLoader + # Use torch.utils.data.DataLoader() if dataset.properties will update during training else InfiniteDataLoader() + dataloader = loader(dataset, + batch_size=batch_size, + num_workers=nw, + sampler=sampler, + pin_memory=True, + collate_fn=LoadImagesAndLabels.collate_fn4 if quad else LoadImagesAndLabels.collate_fn) + return dataloader, dataset + + +class InfiniteDataLoader(torch.utils.data.dataloader.DataLoader): + """ Dataloader that reuses workers + + Uses same syntax as vanilla DataLoader + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler)) + self.iterator = super().__iter__() + + def __len__(self): + return len(self.batch_sampler.sampler) + + def __iter__(self): + for i in range(len(self)): + yield next(self.iterator) + + +class _RepeatSampler(object): + """ Sampler that repeats forever + + Args: + sampler (Sampler) + """ + + def __init__(self, sampler): + self.sampler = sampler + + def __iter__(self): + while True: + yield from iter(self.sampler) + + +class LoadImages: # for inference + def __init__(self, path, img_size=640, stride=32): + p = str(Path(path).absolute()) # os-agnostic absolute path + if '*' in p: + files = sorted(glob.glob(p, recursive=True)) # glob + elif os.path.isdir(p): + files = sorted(glob.glob(os.path.join(p, '*.*'))) # dir + elif os.path.isfile(p): + files = [p] # files + else: + raise Exception(f'ERROR: {p} does not exist') + + images = [x for x in files if x.split('.')[-1].lower() in img_formats] + videos = [x for x in files if x.split('.')[-1].lower() in vid_formats] + ni, nv = len(images), len(videos) + + self.img_size = img_size + self.stride = stride + self.files = images + videos + self.nf = ni + nv # number of files + self.video_flag = [False] * ni + [True] * nv + self.mode = 'image' + if any(videos): + self.new_video(videos[0]) # new video + else: + self.cap = None + assert self.nf > 0, f'No images or videos found in {p}. ' \ + f'Supported formats are:\nimages: {img_formats}\nvideos: {vid_formats}' + + def __iter__(self): + self.count = 0 + return self + + def __next__(self): + if self.count == self.nf: + raise StopIteration + path = self.files[self.count] + + if self.video_flag[self.count]: + # Read video + self.mode = 'video' + ret_val, img0 = self.cap.read() + if not ret_val: + self.count += 1 + self.cap.release() + if self.count == self.nf: # last video + raise StopIteration + else: + path = self.files[self.count] + self.new_video(path) + ret_val, img0 = self.cap.read() + + self.frame += 1 + print(f'video {self.count + 1}/{self.nf} ({self.frame}/{self.nframes}) {path}: ', end='') + + else: + # Read image + self.count += 1 + img0 = cv2.imread(path) # BGR + assert img0 is not None, 'Image Not Found ' + path + #print(f'image {self.count}/{self.nf} {path}: ', end='') + + # Padded resize + img = letterbox(img0, self.img_size, stride=self.stride)[0] + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + return path, img, img0, self.cap + + def new_video(self, path): + self.frame = 0 + self.cap = cv2.VideoCapture(path) + self.nframes = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + def __len__(self): + return self.nf # number of files + + +class LoadWebcam: # for inference + def __init__(self, pipe='0', img_size=640, stride=32): + self.img_size = img_size + self.stride = stride + + if pipe.isnumeric(): + pipe = eval(pipe) # local camera + # pipe = 'rtsp://192.168.1.64/1' # IP camera + # pipe = 'rtsp://username:password@192.168.1.64/1' # IP camera with login + # pipe = 'http://wmccpinetop.axiscam.net/mjpg/video.mjpg' # IP golf camera + + self.pipe = pipe + self.cap = cv2.VideoCapture(pipe) # video capture object + self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 3) # set buffer size + + def __iter__(self): + self.count = -1 + return self + + def __next__(self): + self.count += 1 + if cv2.waitKey(1) == ord('q'): # q to quit + self.cap.release() + cv2.destroyAllWindows() + raise StopIteration + + # Read frame + if self.pipe == 0: # local camera + ret_val, img0 = self.cap.read() + img0 = cv2.flip(img0, 1) # flip left-right + else: # IP camera + n = 0 + while True: + n += 1 + self.cap.grab() + if n % 30 == 0: # skip frames + ret_val, img0 = self.cap.retrieve() + if ret_val: + break + + # Print + assert ret_val, f'Camera Error {self.pipe}' + img_path = 'webcam.jpg' + print(f'webcam {self.count}: ', end='') + + # Padded resize + img = letterbox(img0, self.img_size, stride=self.stride)[0] + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + return img_path, img, img0, None + + def __len__(self): + return 0 + + +class LoadStreams: # multiple IP or RTSP cameras + def __init__(self, sources='streams.txt', img_size=640, stride=32): + self.mode = 'stream' + self.img_size = img_size + self.stride = stride + + if os.path.isfile(sources): + with open(sources, 'r') as f: + sources = [x.strip() for x in f.read().strip().splitlines() if len(x.strip())] + else: + sources = [sources] + + n = len(sources) + self.imgs = [None] * n + self.sources = [clean_str(x) for x in sources] # clean source names for later + for i, s in enumerate(sources): + # Start the thread to read frames from the video stream + print(f'{i + 1}/{n}: {s}... ', end='') + url = eval(s) if s.isnumeric() else s + if 'youtube.com/' in url or 'youtu.be/' in url: # if source is YouTube video + check_requirements(('pafy', 'youtube_dl')) + import pafy + url = pafy.new(url).getbest(preftype="mp4").url + cap = cv2.VideoCapture(url) + assert cap.isOpened(), f'Failed to open {s}' + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + self.fps = cap.get(cv2.CAP_PROP_FPS) % 100 + + _, self.imgs[i] = cap.read() # guarantee first frame + thread = Thread(target=self.update, args=([i, cap]), daemon=True) + print(f' success ({w}x{h} at {self.fps:.2f} FPS).') + thread.start() + print('') # newline + + # check for common shapes + s = np.stack([letterbox(x, self.img_size, stride=self.stride)[0].shape for x in self.imgs], 0) # shapes + self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal + if not self.rect: + print('WARNING: Different stream shapes detected. For optimal performance supply similarly-shaped streams.') + + def update(self, index, cap): + # Read next stream frame in a daemon thread + n = 0 + while cap.isOpened(): + n += 1 + # _, self.imgs[index] = cap.read() + cap.grab() + if n == 4: # read every 4th frame + success, im = cap.retrieve() + self.imgs[index] = im if success else self.imgs[index] * 0 + n = 0 + time.sleep(1 / self.fps) # wait time + + def __iter__(self): + self.count = -1 + return self + + def __next__(self): + self.count += 1 + img0 = self.imgs.copy() + if cv2.waitKey(1) == ord('q'): # q to quit + cv2.destroyAllWindows() + raise StopIteration + + # Letterbox + img = [letterbox(x, self.img_size, auto=self.rect, stride=self.stride)[0] for x in img0] + + # Stack + img = np.stack(img, 0) + + # Convert + img = img[:, :, :, ::-1].transpose(0, 3, 1, 2) # BGR to RGB, to bsx3x416x416 + img = np.ascontiguousarray(img) + + return self.sources, img, img0, None + + def __len__(self): + return 0 # 1E12 frames = 32 streams at 30 FPS for 30 years + + +def img2label_paths(img_paths): + # Define label paths as a function of image paths + sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings + return ['txt'.join(x.replace(sa, sb, 1).rsplit(x.split('.')[-1], 1)) for x in img_paths] + + +class LoadImagesAndLabels(Dataset): # for training/testing + def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False, + cache_images=False, single_cls=False, stride=32, pad=0.0, prefix=''): + self.img_size = img_size + self.augment = augment + self.hyp = hyp + self.image_weights = image_weights + self.rect = False if image_weights else rect + self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training) + self.mosaic_border = [-img_size // 2, -img_size // 2] + self.stride = stride + self.path = path + #self.albumentations = Albumentations() if augment else None + + try: + f = [] # image files + for p in path if isinstance(path, list) else [path]: + p = Path(p) # os-agnostic + if p.is_dir(): # dir + f += glob.glob(str(p / '**' / '*.*'), recursive=True) + # f = list(p.rglob('**/*.*')) # pathlib + elif p.is_file(): # file + with open(p, 'r') as t: + t = t.read().strip().splitlines() # 数据集中每个图片的路径 + parent = str(p.parent) + os.sep # 代码文件夹的上一级 + # 遍历t 找到图片路径 注意这里默认数据集和代码在一个文件夹下 + f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path 变成全局路径 + # f += [p.parent / x.lstrip(os.sep) for x in t] # local to global path (pathlib) + else: + raise Exception(f'{prefix}{p} does not exist') + + self.img_files = sorted([x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in img_formats]) + # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in img_formats]) # pathlib + assert self.img_files, f'{prefix}No images found' + except Exception as e: + raise Exception(f'{prefix}Error loading data from {path}: {e}\nSee {help_url}') + + # Check cache + self.label_files = img2label_paths(self.img_files) # labels 得到真值txt文件路径 + cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache') # cached labels + if cache_path.is_file(): + cache, exists = torch.load(cache_path), True # load + #if cache['hash'] != get_hash(self.label_files + self.img_files) or 'version' not in cache: # changed + # cache, exists = self.cache_labels(cache_path, prefix), False # re-cache + else: + cache, exists = self.cache_labels(cache_path, prefix), False # cache + + # Display cache + nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupted, total + if exists: + d = f"Scanning '{cache_path}' images and labels... {nf} found, {nm} missing, {ne} empty, {nc} corrupted" + tqdm(None, desc=prefix + d, total=n, initial=n) # display cache results + assert nf > 0 or not augment, f'{prefix}No labels in {cache_path}. Can not train without labels. See {help_url}' + + # Read cache + cache.pop('hash') # remove hash + cache.pop('version') # remove version + labels, shapes, self.segments = zip(*cache.values()) + self.labels = list(labels) + self.shapes = np.array(shapes, dtype=np.float64) + self.img_files = list(cache.keys()) # update + self.label_files = img2label_paths(cache.keys()) # update + if single_cls: + for x in self.labels: + x[:, 0] = 0 + + n = len(shapes) # number of images + bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index + nb = bi[-1] + 1 # number of batches + self.batch = bi # batch index of image + self.n = n + self.indices = range(n) + + # Rectangular Training + if self.rect: + # Sort by aspect ratio + s = self.shapes # wh + ar = s[:, 1] / s[:, 0] # aspect ratio + irect = ar.argsort() + self.img_files = [self.img_files[i] for i in irect] + self.label_files = [self.label_files[i] for i in irect] + self.labels = [self.labels[i] for i in irect] + self.shapes = s[irect] # wh + ar = ar[irect] + + # Set training image shapes + shapes = [[1, 1]] * nb + for i in range(nb): + ari = ar[bi == i] + mini, maxi = ari.min(), ari.max() + if maxi < 1: + shapes[i] = [maxi, 1] + elif mini > 1: + shapes[i] = [1, 1 / mini] + + self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride + + # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM) + self.imgs = [None] * n + if cache_images: + if cache_images == 'disk': + self.im_cache_dir = Path(Path(self.img_files[0]).parent.as_posix() + '_npy') + self.img_npy = [self.im_cache_dir / Path(f).with_suffix('.npy').name for f in self.img_files] + self.im_cache_dir.mkdir(parents=True, exist_ok=True) + gb = 0 # Gigabytes of cached images + self.img_hw0, self.img_hw = [None] * n, [None] * n + results = ThreadPool(8).imap(lambda x: load_image(*x), zip(repeat(self), range(n))) + pbar = tqdm(enumerate(results), total=n) + for i, x in pbar: + if cache_images == 'disk': + if not self.img_npy[i].exists(): + np.save(self.img_npy[i].as_posix(), x[0]) + gb += self.img_npy[i].stat().st_size + else: + self.imgs[i], self.img_hw0[i], self.img_hw[i] = x + gb += self.imgs[i].nbytes + pbar.desc = f'{prefix}Caching images ({gb / 1E9:.1f}GB)' + pbar.close() + + def cache_labels(self, path=Path('./labels.cache'), prefix=''): + # Cache dataset labels, check images and read shapes + x = {} # dict + nm, nf, ne, nc = 0, 0, 0, 0 # number missing, found, empty, duplicate + pbar = tqdm(zip(self.img_files, self.label_files), desc='Scanning images', total=len(self.img_files)) + for i, (im_file, lb_file) in enumerate(pbar): + try: + # verify images + im = Image.open(im_file) + im.verify() # PIL verify + shape = exif_size(im) # image size + segments = [] # instance segments + assert (shape[0] > 9) & (shape[1] > 9), f'image size {shape} <10 pixels' + assert im.format.lower() in img_formats, f'invalid image format {im.format}' + + # verify labels + if os.path.isfile(lb_file): + nf += 1 # label found + with open(lb_file, 'r') as f: + l = [x.split() for x in f.read().strip().splitlines()] + if any([len(x) > 8 for x in l]): # is segment + classes = np.array([x[0] for x in l], dtype=np.float32) + segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in l] # (cls, xy1...) + l = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh) + l = np.array(l, dtype=np.float32) + if len(l): + assert l.shape[1] == 5, 'labels require 5 columns each' + assert (l >= 0).all(), 'negative labels' + assert (l[:, 1:] <= 1).all(), 'non-normalized or out of bounds coordinate labels' + assert np.unique(l, axis=0).shape[0] == l.shape[0], 'duplicate labels' + else: + ne += 1 # label empty + l = np.zeros((0, 5), dtype=np.float32) + else: + nm += 1 # label missing + l = np.zeros((0, 5), dtype=np.float32) + x[im_file] = [l, shape, segments] + except Exception as e: + nc += 1 + print(f'{prefix}WARNING: Ignoring corrupted image and/or label {im_file}: {e}') + + pbar.desc = f"{prefix}Scanning '{path.parent / path.stem}' images and labels... " \ + f"{nf} found, {nm} missing, {ne} empty, {nc} corrupted" + pbar.close() + + if nf == 0: + print(f'{prefix}WARNING: No labels found in {path}. See {help_url}') + + x['hash'] = get_hash(self.label_files + self.img_files) + x['results'] = nf, nm, ne, nc, i + 1 + x['version'] = 0.1 # cache version + torch.save(x, path) # save for next time + logging.info(f'{prefix}New cache created: {path}') + return x + + def __len__(self): + return len(self.img_files) + + # def __iter__(self): + # self.count = -1 + # print('ran dataset iter') + # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF) + # return self + + def __getitem__(self, index): + index = self.indices[index] # linear, shuffled, or image_weights + + hyp = self.hyp + mosaic = self.mosaic and random.random() < hyp['mosaic'] + if mosaic: + # Load mosaic + if random.random() < 0.8: + img, labels = load_mosaic(self, index) + else: + img, labels = load_mosaic9(self, index) + shapes = None + + # MixUp https://arxiv.org/pdf/1710.09412.pdf + if random.random() < hyp['mixup']: + if random.random() < 0.8: + img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1)) + else: + img2, labels2 = load_mosaic9(self, random.randint(0, len(self.labels) - 1)) + r = np.random.beta(8.0, 8.0) # mixup ratio, alpha=beta=8.0 + img = (img * r + img2 * (1 - r)).astype(np.uint8) + labels = np.concatenate((labels, labels2), 0) + + else: + # Load image + img, (h0, w0), (h, w) = load_image(self, index) + + # Letterbox + shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape + img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment) + shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling + + labels = self.labels[index].copy() + if labels.size: # normalized xywh to pixel xyxy format + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1]) + + if self.augment: + # Augment imagespace + if not mosaic: + img, labels = random_perspective(img, labels, + degrees=hyp['degrees'], + translate=hyp['translate'], + scale=hyp['scale'], + shear=hyp['shear'], + perspective=hyp['perspective']) + + + #img, labels = self.albumentations(img, labels) + + # Augment colorspace + augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v']) + + # Apply cutouts + # if random.random() < 0.9: + # labels = cutout(img, labels) + + if random.random() < hyp['paste_in']: + sample_labels, sample_images, sample_masks = [], [], [] + while len(sample_labels) < 30: + sample_labels_, sample_images_, sample_masks_ = load_samples(self, random.randint(0, len(self.labels) - 1)) + sample_labels += sample_labels_ + sample_images += sample_images_ + sample_masks += sample_masks_ + #print(len(sample_labels)) + if len(sample_labels) == 0: + break + labels = pastein(img, labels, sample_labels, sample_images, sample_masks) + + nL = len(labels) # number of labels + if nL: + labels[:, 1:5] = xyxy2xywh(labels[:, 1:5]) # convert xyxy to xywh + labels[:, [2, 4]] /= img.shape[0] # normalized height 0-1 + labels[:, [1, 3]] /= img.shape[1] # normalized width 0-1 + + if self.augment: + # flip up-down + if random.random() < hyp['flipud']: + img = np.flipud(img) + if nL: + labels[:, 2] = 1 - labels[:, 2] + + # flip left-right + if random.random() < hyp['fliplr']: + img = np.fliplr(img) + if nL: + labels[:, 1] = 1 - labels[:, 1] + + labels_out = torch.zeros((nL, 6)) + if nL: + labels_out[:, 1:] = torch.from_numpy(labels) + + # Convert + img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + img = np.ascontiguousarray(img) + + return torch.from_numpy(img), labels_out, self.img_files[index], shapes + + @staticmethod + def collate_fn(batch): + img, label, path, shapes = zip(*batch) # transposed + for i, l in enumerate(label): + l[:, 0] = i # add target image index for build_targets() + return torch.stack(img, 0), torch.cat(label, 0), path, shapes + + @staticmethod + def collate_fn4(batch): + img, label, path, shapes = zip(*batch) # transposed + n = len(shapes) // 4 + img4, label4, path4, shapes4 = [], [], path[:n], shapes[:n] + + ho = torch.tensor([[0., 0, 0, 1, 0, 0]]) + wo = torch.tensor([[0., 0, 1, 0, 0, 0]]) + s = torch.tensor([[1, 1, .5, .5, .5, .5]]) # scale + for i in range(n): # zidane torch.zeros(16,3,720,1280) # BCHW + i *= 4 + if random.random() < 0.5: + im = F.interpolate(img[i].unsqueeze(0).float(), scale_factor=2., mode='bilinear', align_corners=False)[ + 0].type(img[i].type()) + l = label[i] + else: + im = torch.cat((torch.cat((img[i], img[i + 1]), 1), torch.cat((img[i + 2], img[i + 3]), 1)), 2) + l = torch.cat((label[i], label[i + 1] + ho, label[i + 2] + wo, label[i + 3] + ho + wo), 0) * s + img4.append(im) + label4.append(l) + + for i, l in enumerate(label4): + l[:, 0] = i # add target image index for build_targets() + + return torch.stack(img4, 0), torch.cat(label4, 0), path4, shapes4 + + +def img2label_paths_VisDrone(img_paths): + """ + img_paths: List[str] + """ + # 默认标签和图片文件夹在同一目录 + return [p.replace('images', 'labels').replace('jpg', 'txt') for p in img_paths] + + + +class LoadImagesAndLabelsCustom(LoadImagesAndLabels): + def __init__(self, path, img_size=640, batch_size=16, augment=False, + hyp=None, rect=False, image_weights=False, cache_images=False, single_cls=False, stride=32, pad=0, prefix=''): + # super().__init__(path, img_size, batch_size, augment, hyp, rect, image_weights, cache_images, single_cls, stride, pad, prefix) + self.img_size = img_size + self.augment = augment + self.hyp = hyp + self.image_weights = image_weights + self.rect = False if image_weights else rect + self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training) + self.mosaic_border = [-img_size // 2, -img_size // 2] + self.stride = stride + self.path = path + + PREFIX = '/data/wujiapeng/datasets/' + path = Path(path) + assert path.is_file(), 'wrong format for VisDrone' + + image_files = [] # 提取txt中的内容 + + with open(path, 'r') as t: + lines = t.read().strip().splitlines() + + image_files += [os.path.join(PREFIX, line) for line in lines] + + t.close() + self.img_files = image_files + self.label_files = img2label_paths_VisDrone(self.img_files) # 真值txt的路径 + cache_path = path.with_suffix('.cache') # 创建.cache + + if cache_path.is_file(): + cache, exists = torch.load(cache_path), True # load + else: + cache, exists = self.cache_labels(cache_path, prefix), False # cache + + # Display cache + nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupted, total + if exists: + d = f"Scanning '{cache_path}' images and labels... {nf} found, {nm} missing, {ne} empty, {nc} corrupted" + tqdm(None, desc=prefix + d, total=n, initial=n) # display cache results + assert nf > 0 or not augment, f'{prefix}No labels in {cache_path}. Can not train without labels. See {help_url}' + + # Read cache + cache.pop('hash') # remove hash + cache.pop('version') # remove version + labels, shapes, self.segments = zip(*cache.values()) + self.labels = list(labels) + self.shapes = np.array(shapes, dtype=np.float64) + self.img_files = list(cache.keys()) # update + self.label_files = img2label_paths_VisDrone(cache.keys()) # update + + if single_cls: + for x in self.labels: + x[:, 0] = 0 + + n = len(shapes) # number of images + bi = np.floor(np.arange(n) / batch_size).astype(np.int) # batch index + nb = bi[-1] + 1 # number of batches + self.batch = bi # batch index of image + self.n = n + self.indices = range(n) + + # Rectangular Training + if self.rect: + # Sort by aspect ratio + s = self.shapes # wh + ar = s[:, 1] / s[:, 0] # aspect ratio + irect = ar.argsort() + self.img_files = [self.img_files[i] for i in irect] + self.label_files = [self.label_files[i] for i in irect] + self.labels = [self.labels[i] for i in irect] + self.shapes = s[irect] # wh + ar = ar[irect] + + # Set training image shapes + shapes = [[1, 1]] * nb + for i in range(nb): + ari = ar[bi == i] + mini, maxi = ari.min(), ari.max() + if maxi < 1: + shapes[i] = [maxi, 1] + elif mini > 1: + shapes[i] = [1, 1 / mini] + + self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(np.int) * stride + + # Cache images into memory for faster training (WARNING: large datasets may exceed system RAM) + self.imgs = [None] * n + if cache_images: + if cache_images == 'disk': + self.im_cache_dir = Path(Path(self.img_files[0]).parent.as_posix() + '_npy') + self.img_npy = [self.im_cache_dir / Path(f).with_suffix('.npy').name for f in self.img_files] + self.im_cache_dir.mkdir(parents=True, exist_ok=True) + gb = 0 # Gigabytes of cached images + self.img_hw0, self.img_hw = [None] * n, [None] * n + results = ThreadPool(8).imap(lambda x: load_image(*x), zip(repeat(self), range(n))) + pbar = tqdm(enumerate(results), total=n) + for i, x in pbar: + if cache_images == 'disk': + if not self.img_npy[i].exists(): + np.save(self.img_npy[i].as_posix(), x[0]) + gb += self.img_npy[i].stat().st_size + else: + self.imgs[i], self.img_hw0[i], self.img_hw[i] = x + gb += self.imgs[i].nbytes + pbar.desc = f'{prefix}Caching images ({gb / 1E9:.1f}GB)' + pbar.close() + +# Ancillary functions -------------------------------------------------------------------------------------------------- +def load_image(self, index): + # loads 1 image from dataset, returns img, original hw, resized hw + img = self.imgs[index] + if img is None: # not cached + path = self.img_files[index] + img = cv2.imread(path) # BGR + assert img is not None, 'Image Not Found ' + path + h0, w0 = img.shape[:2] # orig hw + r = self.img_size / max(h0, w0) # resize image to img_size + if r != 1: # always resize down, only resize up if training with augmentation + interp = cv2.INTER_AREA if r < 1 and not self.augment else cv2.INTER_LINEAR + img = cv2.resize(img, (int(w0 * r), int(h0 * r)), interpolation=interp) + return img, (h0, w0), img.shape[:2] # img, hw_original, hw_resized + else: + return self.imgs[index], self.img_hw0[index], self.img_hw[index] # img, hw_original, hw_resized + + +def augment_hsv(img, hgain=0.5, sgain=0.5, vgain=0.5): + r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains + hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV)) + dtype = img.dtype # uint8 + + x = np.arange(0, 256, dtype=np.int16) + lut_hue = ((x * r[0]) % 180).astype(dtype) + lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) + lut_val = np.clip(x * r[2], 0, 255).astype(dtype) + + img_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))).astype(dtype) + cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed + + +def hist_equalize(img, clahe=True, bgr=False): + # Equalize histogram on BGR image 'img' with img.shape(n,m,3) and range 0-255 + yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV) + if clahe: + c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + yuv[:, :, 0] = c.apply(yuv[:, :, 0]) + else: + yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram + return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB + + +def load_mosaic(self, index): + # loads images in a 4-mosaic + + labels4, segments4 = [], [] + s = self.img_size + yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y + indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = load_image(self, index) + + # place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + # Labels + labels, segments = self.labels[index].copy(), self.segments[index].copy() + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padw, padh) for x in segments] + labels4.append(labels) + segments4.extend(segments) + + # Concat/clip labels + labels4 = np.concatenate(labels4, 0) + for x in (labels4[:, 1:], *segments4): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img4, labels4 = replicate(img4, labels4) # replicate + + # Augment + #img4, labels4, segments4 = remove_background(img4, labels4, segments4) + #sample_segments(img4, labels4, segments4, probability=self.hyp['copy_paste']) + img4, labels4, segments4 = copy_paste(img4, labels4, segments4, probability=self.hyp['copy_paste']) + img4, labels4 = random_perspective(img4, labels4, segments4, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + + return img4, labels4 + + +def load_mosaic9(self, index): + # loads images in a 9-mosaic + + labels9, segments9 = [], [] + s = self.img_size + indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = load_image(self, index) + + # place img in img9 + if i == 0: # center + img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + h0, w0 = h, w + c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates + elif i == 1: # top + c = s, s - h, s + w, s + elif i == 2: # top right + c = s + wp, s - h, s + wp + w, s + elif i == 3: # right + c = s + w0, s, s + w0 + w, s + h + elif i == 4: # bottom right + c = s + w0, s + hp, s + w0 + w, s + hp + h + elif i == 5: # bottom + c = s + w0 - w, s + h0, s + w0, s + h0 + h + elif i == 6: # bottom left + c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h + elif i == 7: # left + c = s - w, s + h0 - h, s, s + h0 + elif i == 8: # top left + c = s - w, s + h0 - hp - h, s, s + h0 - hp + + padx, pady = c[:2] + x1, y1, x2, y2 = [max(x, 0) for x in c] # allocate coords + + # Labels + labels, segments = self.labels[index].copy(), self.segments[index].copy() + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padx, pady) for x in segments] + labels9.append(labels) + segments9.extend(segments) + + # Image + img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax] + hp, wp = h, w # height, width previous + + # Offset + yc, xc = [int(random.uniform(0, s)) for _ in self.mosaic_border] # mosaic center x, y + img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s] + + # Concat/clip labels + labels9 = np.concatenate(labels9, 0) + labels9[:, [1, 3]] -= xc + labels9[:, [2, 4]] -= yc + c = np.array([xc, yc]) # centers + segments9 = [x - c for x in segments9] + + for x in (labels9[:, 1:], *segments9): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img9, labels9 = replicate(img9, labels9) # replicate + + # Augment + #img9, labels9, segments9 = remove_background(img9, labels9, segments9) + img9, labels9, segments9 = copy_paste(img9, labels9, segments9, probability=self.hyp['copy_paste']) + img9, labels9 = random_perspective(img9, labels9, segments9, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + + return img9, labels9 + + +def load_samples(self, index): + # loads images in a 4-mosaic + + labels4, segments4 = [], [] + s = self.img_size + yc, xc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border] # mosaic center x, y + indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = load_image(self, index) + + # place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + # Labels + labels, segments = self.labels[index].copy(), self.segments[index].copy() + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padw, padh) for x in segments] + labels4.append(labels) + segments4.extend(segments) + + # Concat/clip labels + labels4 = np.concatenate(labels4, 0) + for x in (labels4[:, 1:], *segments4): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img4, labels4 = replicate(img4, labels4) # replicate + + # Augment + #img4, labels4, segments4 = remove_background(img4, labels4, segments4) + sample_labels, sample_images, sample_masks = sample_segments(img4, labels4, segments4, probability=0.5) + + return sample_labels, sample_images, sample_masks + + +def copy_paste(img, labels, segments, probability=0.5): + # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy) + n = len(segments) + if probability and n: + h, w, c = img.shape # height, width, channels + im_new = np.zeros(img.shape, np.uint8) + for j in random.sample(range(n), k=round(probability * n)): + l, s = labels[j], segments[j] + box = w - l[3], l[2], w - l[1], l[4] + ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area + if (ioa < 0.30).all(): # allow 30% obscuration of existing labels + labels = np.concatenate((labels, [[l[0], *box]]), 0) + segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1)) + cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED) + + result = cv2.bitwise_and(src1=img, src2=im_new) + result = cv2.flip(result, 1) # augment segments (flip left-right) + i = result > 0 # pixels to replace + # i[:, :] = result.max(2).reshape(h, w, 1) # act over ch + img[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug + + return img, labels, segments + + +def remove_background(img, labels, segments): + # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy) + n = len(segments) + h, w, c = img.shape # height, width, channels + im_new = np.zeros(img.shape, np.uint8) + img_new = np.ones(img.shape, np.uint8) * 114 + for j in range(n): + cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED) + + result = cv2.bitwise_and(src1=img, src2=im_new) + + i = result > 0 # pixels to replace + img_new[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug + + return img_new, labels, segments + + +def sample_segments(img, labels, segments, probability=0.5): + # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy) + n = len(segments) + sample_labels = [] + sample_images = [] + sample_masks = [] + if probability and n: + h, w, c = img.shape # height, width, channels + for j in random.sample(range(n), k=round(probability * n)): + l, s = labels[j], segments[j] + box = l[1].astype(int).clip(0,w-1), l[2].astype(int).clip(0,h-1), l[3].astype(int).clip(0,w-1), l[4].astype(int).clip(0,h-1) + + #print(box) + if (box[2] <= box[0]) or (box[3] <= box[1]): + continue + + sample_labels.append(l[0]) + + mask = np.zeros(img.shape, np.uint8) + + cv2.drawContours(mask, [segments[j].astype(np.int32)], -1, (255, 255, 255), cv2.FILLED) + sample_masks.append(mask[box[1]:box[3],box[0]:box[2],:]) + + result = cv2.bitwise_and(src1=img, src2=mask) + i = result > 0 # pixels to replace + mask[i] = result[i] # cv2.imwrite('debug.jpg', img) # debug + #print(box) + sample_images.append(mask[box[1]:box[3],box[0]:box[2],:]) + + return sample_labels, sample_images, sample_masks + + +def replicate(img, labels): + # Replicate labels + h, w = img.shape[:2] + boxes = labels[:, 1:].astype(int) + x1, y1, x2, y2 = boxes.T + s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels) + for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices + x1b, y1b, x2b, y2b = boxes[i] + bh, bw = y2b - y1b, x2b - x1b + yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y + x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh] + img[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0) + + return img, labels + + +def letterbox(img, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32): + # Resize and pad image while meeting stride-multiple constraints + shape = img.shape[:2] # current shape [height, width] + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + if not scaleup: # only scale down, do not scale up (for better test mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding + if auto: # minimum rectangle + dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding + elif scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios + + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) + top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) + img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border + return img, ratio, (dw, dh) + + +def random_perspective(img, targets=(), segments=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, + border=(0, 0)): + # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10)) + # targets = [cls, xyxy] + + height = img.shape[0] + border[0] * 2 # shape(h,w,c) + width = img.shape[1] + border[1] * 2 + + # Center + C = np.eye(3) + C[0, 2] = -img.shape[1] / 2 # x translation (pixels) + C[1, 2] = -img.shape[0] / 2 # y translation (pixels) + + # Perspective + P = np.eye(3) + P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y) + P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x) + + # Rotation and Scale + R = np.eye(3) + a = random.uniform(-degrees, degrees) + # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations + s = random.uniform(1 - scale, 1.1 + scale) + # s = 2 ** random.uniform(-scale, scale) + R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) + + # Shear + S = np.eye(3) + S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg) + S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg) + + # Translation + T = np.eye(3) + T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels) + T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels) + + # Combined rotation matrix + M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT + if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed + if perspective: + img = cv2.warpPerspective(img, M, dsize=(width, height), borderValue=(114, 114, 114)) + else: # affine + img = cv2.warpAffine(img, M[:2], dsize=(width, height), borderValue=(114, 114, 114)) + + # Visualize + # import matplotlib.pyplot as plt + # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel() + # ax[0].imshow(img[:, :, ::-1]) # base + # ax[1].imshow(img2[:, :, ::-1]) # warped + + # Transform label coordinates + n = len(targets) + if n: + use_segments = any(x.any() for x in segments) + new = np.zeros((n, 4)) + if use_segments: # warp segments + segments = resample_segments(segments) # upsample + for i, segment in enumerate(segments): + xy = np.ones((len(segment), 3)) + xy[:, :2] = segment + xy = xy @ M.T # transform + xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine + + # clip + new[i] = segment2box(xy, width, height) + + else: # warp boxes + xy = np.ones((n * 4, 3)) + xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 + xy = xy @ M.T # transform + xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine + + # create new boxes + x = xy[:, [0, 2, 4, 6]] + y = xy[:, [1, 3, 5, 7]] + new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T + + # clip + new[:, [0, 2]] = new[:, [0, 2]].clip(0, width) + new[:, [1, 3]] = new[:, [1, 3]].clip(0, height) + + # filter candidates + i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10) + targets = targets[i] + targets[:, 1:5] = new[i] + + return img, targets + + +def box_candidates(box1, box2, wh_thr=2, ar_thr=20, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n) + # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio + w1, h1 = box1[2] - box1[0], box1[3] - box1[1] + w2, h2 = box2[2] - box2[0], box2[3] - box2[1] + ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio + return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates + + +def bbox_ioa(box1, box2): + # Returns the intersection over box2 area given box1, box2. box1 is 4, box2 is nx4. boxes are x1y1x2y2 + box2 = box2.transpose() + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] + b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] + + # Intersection area + inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \ + (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0) + + # box2 area + box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + 1e-16 + + # Intersection over box2 area + return inter_area / box2_area + + +def cutout(image, labels): + # Applies image cutout augmentation https://arxiv.org/abs/1708.04552 + h, w = image.shape[:2] + + # create random masks + scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction + for s in scales: + mask_h = random.randint(1, int(h * s)) + mask_w = random.randint(1, int(w * s)) + + # box + xmin = max(0, random.randint(0, w) - mask_w // 2) + ymin = max(0, random.randint(0, h) - mask_h // 2) + xmax = min(w, xmin + mask_w) + ymax = min(h, ymin + mask_h) + + # apply random color mask + image[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)] + + # return unobscured labels + if len(labels) and s > 0.03: + box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32) + ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area + labels = labels[ioa < 0.60] # remove >60% obscured labels + + return labels + + +def pastein(image, labels, sample_labels, sample_images, sample_masks): + # Applies image cutout augmentation https://arxiv.org/abs/1708.04552 + h, w = image.shape[:2] + + # create random masks + scales = [0.75] * 2 + [0.5] * 4 + [0.25] * 4 + [0.125] * 4 + [0.0625] * 6 # image size fraction + for s in scales: + if random.random() < 0.2: + continue + mask_h = random.randint(1, int(h * s)) + mask_w = random.randint(1, int(w * s)) + + # box + xmin = max(0, random.randint(0, w) - mask_w // 2) + ymin = max(0, random.randint(0, h) - mask_h // 2) + xmax = min(w, xmin + mask_w) + ymax = min(h, ymin + mask_h) + + box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32) + if len(labels): + ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area + else: + ioa = np.zeros(1) + + if (ioa < 0.30).all() and len(sample_labels) and (xmax > xmin+20) and (ymax > ymin+20): # allow 30% obscuration of existing labels + sel_ind = random.randint(0, len(sample_labels)-1) + #print(len(sample_labels)) + #print(sel_ind) + #print((xmax-xmin, ymax-ymin)) + #print(image[ymin:ymax, xmin:xmax].shape) + #print([[sample_labels[sel_ind], *box]]) + #print(labels.shape) + hs, ws, cs = sample_images[sel_ind].shape + r_scale = min((ymax-ymin)/hs, (xmax-xmin)/ws) + r_w = int(ws*r_scale) + r_h = int(hs*r_scale) + + if (r_w > 10) and (r_h > 10): + r_mask = cv2.resize(sample_masks[sel_ind], (r_w, r_h)) + r_image = cv2.resize(sample_images[sel_ind], (r_w, r_h)) + temp_crop = image[ymin:ymin+r_h, xmin:xmin+r_w] + m_ind = r_mask > 0 + if m_ind.astype(np.int).sum() > 60: + temp_crop[m_ind] = r_image[m_ind] + #print(sample_labels[sel_ind]) + #print(sample_images[sel_ind].shape) + #print(temp_crop.shape) + box = np.array([xmin, ymin, xmin+r_w, ymin+r_h], dtype=np.float32) + if len(labels): + labels = np.concatenate((labels, [[sample_labels[sel_ind], *box]]), 0) + else: + labels = np.array([[sample_labels[sel_ind], *box]]) + + image[ymin:ymin+r_h, xmin:xmin+r_w] = temp_crop + + return labels + +class Albumentations: + # YOLOv5 Albumentations class (optional, only used if package is installed) + def __init__(self): + self.transform = None + import albumentations as A + + self.transform = A.Compose([ + A.CLAHE(p=0.01), + A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.01), + A.RandomGamma(gamma_limit=[80, 120], p=0.01), + A.Blur(p=0.01), + A.MedianBlur(p=0.01), + A.ToGray(p=0.01), + A.ImageCompression(quality_lower=75, p=0.01),], + bbox_params=A.BboxParams(format='pascal_voc', label_fields=['class_labels'])) + + #logging.info(colorstr('albumentations: ') + ', '.join(f'{x}' for x in self.transform.transforms if x.p)) + + def __call__(self, im, labels, p=1.0): + if self.transform and random.random() < p: + new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed + im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])]) + return im, labels + + +def create_folder(path='./new'): + # Create folder + if os.path.exists(path): + shutil.rmtree(path) # delete output folder + os.makedirs(path) # make new output folder + + +def flatten_recursive(path='../coco'): + # Flatten a recursive directory by bringing all files to top level + new_path = Path(path + '_flat') + create_folder(new_path) + for file in tqdm(glob.glob(str(Path(path)) + '/**/*.*', recursive=True)): + shutil.copyfile(file, new_path / Path(file).name) + + +def extract_boxes(path='../coco/'): # from utils.datasets import *; extract_boxes('../coco128') + # Convert detection dataset into classification dataset, with one directory per class + + path = Path(path) # images dir + shutil.rmtree(path / 'classifier') if (path / 'classifier').is_dir() else None # remove existing + files = list(path.rglob('*.*')) + n = len(files) # number of files + for im_file in tqdm(files, total=n): + if im_file.suffix[1:] in img_formats: + # image + im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB + h, w = im.shape[:2] + + # labels + lb_file = Path(img2label_paths([str(im_file)])[0]) + if Path(lb_file).exists(): + with open(lb_file, 'r') as f: + lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels + + for j, x in enumerate(lb): + c = int(x[0]) # class + f = (path / 'classifier') / f'{c}' / f'{path.stem}_{im_file.stem}_{j}.jpg' # new filename + if not f.parent.is_dir(): + f.parent.mkdir(parents=True) + + b = x[1:] * [w, h, w, h] # box + # b[2:] = b[2:].max() # rectangle to square + b[2:] = b[2:] * 1.2 + 3 # pad + b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(np.int) + + b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image + b[[1, 3]] = np.clip(b[[1, 3]], 0, h) + assert cv2.imwrite(str(f), im[b[1]:b[3], b[0]:b[2]]), f'box failure in {f}' + + +def autosplit(path='../coco', weights=(0.9, 0.1, 0.0), annotated_only=False): + """ Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files + Usage: from utils.datasets import *; autosplit('../coco') + Arguments + path: Path to images directory + weights: Train, val, test weights (list) + annotated_only: Only use images with an annotated txt file + """ + path = Path(path) # images dir + files = sum([list(path.rglob(f"*.{img_ext}")) for img_ext in img_formats], []) # image files only + n = len(files) # number of files + indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split + + txt = ['autosplit_train.txt', 'autosplit_val.txt', 'autosplit_test.txt'] # 3 txt files + [(path / x).unlink() for x in txt if (path / x).exists()] # remove existing + + print(f'Autosplitting images from {path}' + ', using *.txt labeled images only' * annotated_only) + for i, img in tqdm(zip(indices, files), total=n): + if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label + with open(path / txt[i], 'a') as f: + f.write(str(img) + '\n') # add image to txt file + + +def load_segmentations(self, index): + key = '/work/handsomejw66/coco17/' + self.img_files[index] + #print(key) + # /work/handsomejw66/coco17/ + return self.segs[key] diff --git a/test/yolov7-tracker/utils/general.py b/test/yolov7-tracker/utils/general.py new file mode 100644 index 0000000..b00dc27 --- /dev/null +++ b/test/yolov7-tracker/utils/general.py @@ -0,0 +1,790 @@ +# YOLOR general utils + +import glob +import logging +import math +import os +import platform +import random +import re +import subprocess +import time +from pathlib import Path + +import cv2 +import numpy as np +import pandas as pd +import torch +import torchvision +import yaml + +from utils.google_utils import gsutil_getsize +from utils.metrics import fitness +from utils.torch_utils import init_torch_seeds + +# Settings +torch.set_printoptions(linewidth=320, precision=5, profile='long') +np.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5 +pd.options.display.max_columns = 10 +cv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader) +os.environ['NUMEXPR_MAX_THREADS'] = str(min(os.cpu_count(), 8)) # NumExpr max threads + + +def set_logging(rank=-1): + logging.basicConfig( + format="%(message)s", + level=logging.INFO if rank in [-1, 0] else logging.WARN) + + +def init_seeds(seed=0): + # Initialize random number generator (RNG) seeds + random.seed(seed) + np.random.seed(seed) + init_torch_seeds(seed) + + +def get_latest_run(search_dir='.'): + # Return path to most recent 'last.pt' in /runs (i.e. to --resume from) + last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True) + return max(last_list, key=os.path.getctime) if last_list else '' + + +def isdocker(): + # Is environment a Docker container + return Path('/workspace').exists() # or Path('/.dockerenv').exists() + + +def emojis(str=''): + # Return platform-dependent emoji-safe version of string + return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str + + +def check_online(): + # Check internet connectivity + import socket + try: + socket.create_connection(("1.1.1.1", 443), 5) # check host accesability + return True + except OSError: + return False + + +def check_git_status(): + # Recommend 'git pull' if code is out of date + print(colorstr('github: '), end='') + try: + assert Path('.git').exists(), 'skipping check (not a git repository)' + assert not isdocker(), 'skipping check (Docker image)' + assert check_online(), 'skipping check (offline)' + + cmd = 'git fetch && git config --get remote.origin.url' + url = subprocess.check_output(cmd, shell=True).decode().strip().rstrip('.git') # github repo url + branch = subprocess.check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode().strip() # checked out + n = int(subprocess.check_output(f'git rev-list {branch}..origin/master --count', shell=True)) # commits behind + if n > 0: + s = f"⚠️ WARNING: code is out of date by {n} commit{'s' * (n > 1)}. " \ + f"Use 'git pull' to update or 'git clone {url}' to download latest." + else: + s = f'up to date with {url} ✅' + print(emojis(s)) # emoji-safe + except Exception as e: + print(e) + + +def check_requirements(requirements='requirements.txt', exclude=()): + # Check installed dependencies meet requirements (pass *.txt file or list of packages) + import pkg_resources as pkg + prefix = colorstr('red', 'bold', 'requirements:') + if isinstance(requirements, (str, Path)): # requirements.txt file + file = Path(requirements) + if not file.exists(): + print(f"{prefix} {file.resolve()} not found, check failed.") + return + requirements = [f'{x.name}{x.specifier}' for x in pkg.parse_requirements(file.open()) if x.name not in exclude] + else: # list or tuple of packages + requirements = [x for x in requirements if x not in exclude] + + n = 0 # number of packages updates + for r in requirements: + try: + pkg.require(r) + except Exception as e: # DistributionNotFound or VersionConflict if requirements not met + n += 1 + print(f"{prefix} {e.req} not found and is required by YOLOR, attempting auto-update...") + print(subprocess.check_output(f"pip install '{e.req}'", shell=True).decode()) + + if n: # if packages updated + source = file.resolve() if 'file' in locals() else requirements + s = f"{prefix} {n} package{'s' * (n > 1)} updated per {source}\n" \ + f"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\n" + print(emojis(s)) # emoji-safe + + +def check_img_size(img_size, s=32): + # Verify img_size is a multiple of stride s + new_size = make_divisible(img_size, int(s)) # ceil gs-multiple + if new_size != img_size: + print('WARNING: --img-size %g must be multiple of max stride %g, updating to %g' % (img_size, s, new_size)) + return new_size + + +def check_imshow(): + # Check if environment supports image displays + try: + assert not isdocker(), 'cv2.imshow() is disabled in Docker environments' + cv2.imshow('test', np.zeros((1, 1, 3))) + cv2.waitKey(1) + cv2.destroyAllWindows() + cv2.waitKey(1) + return True + except Exception as e: + print(f'WARNING: Environment does not support cv2.imshow() or PIL Image.show() image displays\n{e}') + return False + + +def check_file(file): + # Search for file if not found + if Path(file).is_file() or file == '': + return file + else: + files = glob.glob('./**/' + file, recursive=True) # find file + assert len(files), f'File Not Found: {file}' # assert file was found + assert len(files) == 1, f"Multiple files match '{file}', specify exact path: {files}" # assert unique + return files[0] # return file + + +def check_dataset(dict): + # Download dataset if not found locally + val, s = dict.get('val'), dict.get('download') + if val and len(val): + val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path + if not all(x.exists() for x in val): + print('\nWARNING: Dataset not found, nonexistent paths: %s' % [str(x) for x in val if not x.exists()]) + if s and len(s): # download script + print('Downloading %s ...' % s) + if s.startswith('http') and s.endswith('.zip'): # URL + f = Path(s).name # filename + torch.hub.download_url_to_file(s, f) + r = os.system('unzip -q %s -d ../ && rm %s' % (f, f)) # unzip + else: # bash script + r = os.system(s) + print('Dataset autodownload %s\n' % ('success' if r == 0 else 'failure')) # analyze return value + else: + raise Exception('Dataset not found.') + + +def make_divisible(x, divisor): + # Returns x evenly divisible by divisor + return math.ceil(x / divisor) * divisor + + +def clean_str(s): + # Cleans a string by replacing special characters with underscore _ + return re.sub(pattern="[|@#!¡·$€%&()=?¿^*;:,¨´><+]", repl="_", string=s) + + +def one_cycle(y1=0.0, y2=1.0, steps=100): + # lambda function for sinusoidal ramp from y1 to y2 + return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1 + + +def colorstr(*input): + # Colors a string https://en.wikipedia.org/wiki/ANSI_escape_code, i.e. colorstr('blue', 'hello world') + *args, string = input if len(input) > 1 else ('blue', 'bold', input[0]) # color arguments, string + colors = {'black': '\033[30m', # basic colors + 'red': '\033[31m', + 'green': '\033[32m', + 'yellow': '\033[33m', + 'blue': '\033[34m', + 'magenta': '\033[35m', + 'cyan': '\033[36m', + 'white': '\033[37m', + 'bright_black': '\033[90m', # bright colors + 'bright_red': '\033[91m', + 'bright_green': '\033[92m', + 'bright_yellow': '\033[93m', + 'bright_blue': '\033[94m', + 'bright_magenta': '\033[95m', + 'bright_cyan': '\033[96m', + 'bright_white': '\033[97m', + 'end': '\033[0m', # misc + 'bold': '\033[1m', + 'underline': '\033[4m'} + return ''.join(colors[x] for x in args) + f'{string}' + colors['end'] + + +def labels_to_class_weights(labels, nc=80): + # Get class weights (inverse frequency) from training labels + if labels[0] is None: # no labels loaded + return torch.Tensor() + + labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO + classes = labels[:, 0].astype(np.int) # labels = [class xywh] + weights = np.bincount(classes, minlength=nc) # occurrences per class + + # Prepend gridpoint count (for uCE training) + # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image + # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start + + weights[weights == 0] = 1 # replace empty bins with 1 + weights = 1 / weights # number of targets per class + weights /= weights.sum() # normalize + return torch.from_numpy(weights) + + +def labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)): + # Produces image weights based on class_weights and image contents + class_counts = np.array([np.bincount(x[:, 0].astype(np.int), minlength=nc) for x in labels]) + image_weights = (class_weights.reshape(1, nc) * class_counts).sum(1) + # index = random.choices(range(n), weights=image_weights, k=1) # weight image sample + return image_weights + + +def coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper) + # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ + # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') + # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') + # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco + # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet + x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90] + return x + + +def xyxy2xywh(x): + # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[:, 0] = (x[:, 0] + x[:, 2]) / 2 # x center + y[:, 1] = (x[:, 1] + x[:, 3]) / 2 # y center + y[:, 2] = x[:, 2] - x[:, 0] # width + y[:, 3] = x[:, 3] - x[:, 1] # height + return y + + +def xywh2xyxy(x): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x + y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y + y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x + y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y + return y + + +def xywhn2xyxy(x, w=640, h=640, padw=0, padh=0): + # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[:, 0] = w * (x[:, 0] - x[:, 2] / 2) + padw # top left x + y[:, 1] = h * (x[:, 1] - x[:, 3] / 2) + padh # top left y + y[:, 2] = w * (x[:, 0] + x[:, 2] / 2) + padw # bottom right x + y[:, 3] = h * (x[:, 1] + x[:, 3] / 2) + padh # bottom right y + return y + + +def xyn2xy(x, w=640, h=640, padw=0, padh=0): + # Convert normalized segments into pixel segments, shape (n,2) + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[:, 0] = w * x[:, 0] + padw # top left x + y[:, 1] = h * x[:, 1] + padh # top left y + return y + + +def segment2box(segment, width=640, height=640): + # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy) + x, y = segment.T # segment xy + inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height) + x, y, = x[inside], y[inside] + return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy + + +def segments2boxes(segments): + # Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh) + boxes = [] + for s in segments: + x, y = s.T # segment xy + boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy + return xyxy2xywh(np.array(boxes)) # cls, xywh + + +def resample_segments(segments, n=1000): + # Up-sample an (n,2) segment + for i, s in enumerate(segments): + x = np.linspace(0, len(s) - 1, n) + xp = np.arange(len(s)) + segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy + return segments + + +def scale_coords(img1_shape, coords, img0_shape, ratio_pad=None): + # Rescale coords (xyxy) from img1_shape to img0_shape + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + coords[:, [0, 2]] -= pad[0] # x padding + coords[:, [1, 3]] -= pad[1] # y padding + coords[:, :4] /= gain + clip_coords(coords, img0_shape) + return coords + + +def clip_coords(boxes, img_shape): + # Clip bounding xyxy bounding boxes to image shape (height, width) + boxes[:, 0].clamp_(0, img_shape[1]) # x1 + boxes[:, 1].clamp_(0, img_shape[0]) # y1 + boxes[:, 2].clamp_(0, img_shape[1]) # x2 + boxes[:, 3].clamp_(0, img_shape[0]) # y2 + + +def bbox_iou(box1, box2, x1y1x2y2=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7): + # Returns the IoU of box1 to box2. box1 is 4, box2 is nx4 + box2 = box2.T + + # Get the coordinates of bounding boxes + if x1y1x2y2: # x1, y1, x2, y2 = box1 + b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] + b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] + else: # transform from xywh to xyxy + b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2 + b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2 + b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2 + b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2 + + # Intersection area + inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \ + (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0) + + # Union Area + w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps + w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps + union = w1 * h1 + w2 * h2 - inter + eps + + iou = inter / union + + if GIoU or DIoU or CIoU: + cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width + ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared + rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center distance squared + if DIoU: + return iou - rho2 / c2 # DIoU + elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2) + with torch.no_grad(): + alpha = v / (v - iou + (1 + eps)) + return iou - (rho2 / c2 + v * alpha) # CIoU + else: # GIoU https://arxiv.org/pdf/1902.09630.pdf + c_area = cw * ch + eps # convex area + return iou - (c_area - union) / c_area # GIoU + else: + return iou # IoU + + + + +def bbox_alpha_iou(box1, box2, x1y1x2y2=False, GIoU=False, DIoU=False, CIoU=False, alpha=2, eps=1e-9): + # Returns tsqrt_he IoU of box1 to box2. box1 is 4, box2 is nx4 + box2 = box2.T + + # Get the coordinates of bounding boxes + if x1y1x2y2: # x1, y1, x2, y2 = box1 + b1_x1, b1_y1, b1_x2, b1_y2 = box1[0], box1[1], box1[2], box1[3] + b2_x1, b2_y1, b2_x2, b2_y2 = box2[0], box2[1], box2[2], box2[3] + else: # transform from xywh to xyxy + b1_x1, b1_x2 = box1[0] - box1[2] / 2, box1[0] + box1[2] / 2 + b1_y1, b1_y2 = box1[1] - box1[3] / 2, box1[1] + box1[3] / 2 + b2_x1, b2_x2 = box2[0] - box2[2] / 2, box2[0] + box2[2] / 2 + b2_y1, b2_y2 = box2[1] - box2[3] / 2, box2[1] + box2[3] / 2 + + # Intersection area + inter = (torch.min(b1_x2, b2_x2) - torch.max(b1_x1, b2_x1)).clamp(0) * \ + (torch.min(b1_y2, b2_y2) - torch.max(b1_y1, b2_y1)).clamp(0) + + # Union Area + w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps + w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps + union = w1 * h1 + w2 * h2 - inter + eps + + # change iou into pow(iou+eps) + # iou = inter / union + iou = torch.pow(inter/union + eps, alpha) + # beta = 2 * alpha + if GIoU or DIoU or CIoU: + cw = torch.max(b1_x2, b2_x2) - torch.min(b1_x1, b2_x1) # convex (smallest enclosing box) width + ch = torch.max(b1_y2, b2_y2) - torch.min(b1_y1, b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = (cw ** 2 + ch ** 2) ** alpha + eps # convex diagonal + rho_x = torch.abs(b2_x1 + b2_x2 - b1_x1 - b1_x2) + rho_y = torch.abs(b2_y1 + b2_y2 - b1_y1 - b1_y2) + rho2 = ((rho_x ** 2 + rho_y ** 2) / 4) ** alpha # center distance + if DIoU: + return iou - rho2 / c2 # DIoU + elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi ** 2) * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2) + with torch.no_grad(): + alpha_ciou = v / ((1 + eps) - inter / union + v) + # return iou - (rho2 / c2 + v * alpha_ciou) # CIoU + return iou - (rho2 / c2 + torch.pow(v * alpha_ciou + eps, alpha)) # CIoU + else: # GIoU https://arxiv.org/pdf/1902.09630.pdf + # c_area = cw * ch + eps # convex area + # return iou - (c_area - union) / c_area # GIoU + c_area = torch.max(cw * ch + eps, union) # convex area + return iou - torch.pow((c_area - union) / c_area + eps, alpha) # GIoU + else: + return iou # torch.log(iou+eps) or iou + + +def box_iou(box1, box2): + # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + box1 (Tensor[N, 4]) + box2 (Tensor[M, 4]) + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2) + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + return inter / (area1[:, None] + area2 - inter) # iou = inter / (area1 + area2 - inter) + + +def wh_iou(wh1, wh2): + # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2 + wh1 = wh1[:, None] # [N,1,2] + wh2 = wh2[None] # [1,M,2] + inter = torch.min(wh1, wh2).prod(2) # [N,M] + return inter / (wh1.prod(2) + wh2.prod(2) - inter) # iou = inter / (area1 + area2 - inter) + + +def box_giou(box1, box2): + """ + Return generalized intersection-over-union (Jaccard index) between two sets of boxes. + Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with + ``0 <= x1 < x2`` and ``0 <= y1 < y2``. + Args: + boxes1 (Tensor[N, 4]): first set of boxes + boxes2 (Tensor[M, 4]): second set of boxes + Returns: + Tensor[N, M]: the NxM matrix containing the pairwise generalized IoU values + for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + union = (area1[:, None] + area2 - inter) + + iou = inter / union + + lti = torch.min(box1[:, None, :2], box2[:, :2]) + rbi = torch.max(box1[:, None, 2:], box2[:, 2:]) + + whi = (rbi - lti).clamp(min=0) # [N,M,2] + areai = whi[:, :, 0] * whi[:, :, 1] + + return iou - (areai - union) / areai + + +def box_ciou(box1, box2, eps: float = 1e-7): + """ + Return complete intersection-over-union (Jaccard index) between two sets of boxes. + Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with + ``0 <= x1 < x2`` and ``0 <= y1 < y2``. + Args: + boxes1 (Tensor[N, 4]): first set of boxes + boxes2 (Tensor[M, 4]): second set of boxes + eps (float, optional): small number to prevent division by zero. Default: 1e-7 + Returns: + Tensor[N, M]: the NxM matrix containing the pairwise complete IoU values + for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + union = (area1[:, None] + area2 - inter) + + iou = inter / union + + lti = torch.min(box1[:, None, :2], box2[:, :2]) + rbi = torch.max(box1[:, None, 2:], box2[:, 2:]) + + whi = (rbi - lti).clamp(min=0) # [N,M,2] + diagonal_distance_squared = (whi[:, :, 0] ** 2) + (whi[:, :, 1] ** 2) + eps + + # centers of boxes + x_p = (box1[:, None, 0] + box1[:, None, 2]) / 2 + y_p = (box1[:, None, 1] + box1[:, None, 3]) / 2 + x_g = (box2[:, 0] + box2[:, 2]) / 2 + y_g = (box2[:, 1] + box2[:, 3]) / 2 + # The distance between boxes' centers squared. + centers_distance_squared = (x_p - x_g) ** 2 + (y_p - y_g) ** 2 + + w_pred = box1[:, None, 2] - box1[:, None, 0] + h_pred = box1[:, None, 3] - box1[:, None, 1] + + w_gt = box2[:, 2] - box2[:, 0] + h_gt = box2[:, 3] - box2[:, 1] + + v = (4 / (torch.pi ** 2)) * torch.pow((torch.atan(w_gt / h_gt) - torch.atan(w_pred / h_pred)), 2) + with torch.no_grad(): + alpha = v / (1 - iou + v + eps) + return iou - (centers_distance_squared / diagonal_distance_squared) - alpha * v + + +def box_diou(box1, box2, eps: float = 1e-7): + """ + Return distance intersection-over-union (Jaccard index) between two sets of boxes. + Both sets of boxes are expected to be in ``(x1, y1, x2, y2)`` format with + ``0 <= x1 < x2`` and ``0 <= y1 < y2``. + Args: + boxes1 (Tensor[N, 4]): first set of boxes + boxes2 (Tensor[M, 4]): second set of boxes + eps (float, optional): small number to prevent division by zero. Default: 1e-7 + Returns: + Tensor[N, M]: the NxM matrix containing the pairwise distance IoU values + for every element in boxes1 and boxes2 + """ + + def box_area(box): + # box = 4xn + return (box[2] - box[0]) * (box[3] - box[1]) + + area1 = box_area(box1.T) + area2 = box_area(box2.T) + + inter = (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2])).clamp(0).prod(2) + union = (area1[:, None] + area2 - inter) + + iou = inter / union + + lti = torch.min(box1[:, None, :2], box2[:, :2]) + rbi = torch.max(box1[:, None, 2:], box2[:, 2:]) + + whi = (rbi - lti).clamp(min=0) # [N,M,2] + diagonal_distance_squared = (whi[:, :, 0] ** 2) + (whi[:, :, 1] ** 2) + eps + + # centers of boxes + x_p = (box1[:, None, 0] + box1[:, None, 2]) / 2 + y_p = (box1[:, None, 1] + box1[:, None, 3]) / 2 + x_g = (box2[:, 0] + box2[:, 2]) / 2 + y_g = (box2[:, 1] + box2[:, 3]) / 2 + # The distance between boxes' centers squared. + centers_distance_squared = (x_p - x_g) ** 2 + (y_p - y_g) ** 2 + + # The distance IoU is the IoU penalized by a normalized + # distance between boxes' centers squared. + return iou - (centers_distance_squared / diagonal_distance_squared) + + +def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False, + labels=()): + """Runs Non-Maximum Suppression (NMS) on inference results + + Returns: + list of detections, on (n,6) tensor per image [xyxy, conf, cls] + """ + + nc = prediction.shape[2] - 5 # number of classes + xc = prediction[..., 4] > conf_thres # candidates + + # Settings + min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height + max_det = 300 # maximum number of detections per image + max_nms = 30000 # maximum number of boxes into torchvision.ops.nms() + time_limit = 10.0 # seconds to quit after + redundant = True # require redundant detections + multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img) + merge = False # use merge-NMS + + t = time.time() + output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0] + for xi, x in enumerate(prediction): # image index, image inference + # Apply constraints + # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height + x = x[xc[xi]] # confidence + + # Cat apriori labels if autolabelling + if labels and len(labels[xi]): + l = labels[xi] + v = torch.zeros((len(l), nc + 5), device=x.device) + v[:, :4] = l[:, 1:5] # box + v[:, 4] = 1.0 # conf + v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls + x = torch.cat((x, v), 0) + + # If none remain process next image + if not x.shape[0]: + continue + + # Compute conf + x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf + + # Box (center x, center y, width, height) to (x1, y1, x2, y2) + box = xywh2xyxy(x[:, :4]) + + # Detections matrix nx6 (xyxy, conf, cls) + if multi_label: + i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T + x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1) + else: # best class only + conf, j = x[:, 5:].max(1, keepdim=True) + x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres] + + # Filter by class + if classes is not None: + x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] + + # Apply finite constraint + # if not torch.isfinite(x).all(): + # x = x[torch.isfinite(x).all(1)] + + # Check shape + n = x.shape[0] # number of boxes + if not n: # no boxes + continue + elif n > max_nms: # excess boxes + x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence + + # Batched NMS + c = x[:, 5:6] * (0 if agnostic else max_wh) # classes + boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores + i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS + if i.shape[0] > max_det: # limit detections + i = i[:max_det] + if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) + # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) + iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix + weights = iou * scores[None] # box weights + x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes + if redundant: + i = i[iou.sum(1) > 1] # require redundancy + + output[xi] = x[i] + if (time.time() - t) > time_limit: + print(f'WARNING: NMS time limit {time_limit}s exceeded') + break # time limit exceeded + + return output + + +def strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_optimizer() + # Strip optimizer from 'f' to finalize training, optionally save as 's' + x = torch.load(f, map_location=torch.device('cpu')) + if x.get('ema'): + x['model'] = x['ema'] # replace model with ema + for k in 'optimizer', 'training_results', 'wandb_id', 'ema', 'updates': # keys + x[k] = None + x['epoch'] = -1 + x['model'].half() # to FP16 + for p in x['model'].parameters(): + p.requires_grad = False + torch.save(x, s or f) + mb = os.path.getsize(s or f) / 1E6 # filesize + print(f"Optimizer stripped from {f},{(' saved as %s,' % s) if s else ''} {mb:.1f}MB") + + +def print_mutation(hyp, results, yaml_file='hyp_evolved.yaml', bucket=''): + # Print mutation results to evolve.txt (for use with train.py --evolve) + a = '%10s' * len(hyp) % tuple(hyp.keys()) # hyperparam keys + b = '%10.3g' * len(hyp) % tuple(hyp.values()) # hyperparam values + c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) + print('\n%s\n%s\nEvolved fitness: %s\n' % (a, b, c)) + + if bucket: + url = 'gs://%s/evolve.txt' % bucket + if gsutil_getsize(url) > (os.path.getsize('evolve.txt') if os.path.exists('evolve.txt') else 0): + os.system('gsutil cp %s .' % url) # download evolve.txt if larger than local + + with open('evolve.txt', 'a') as f: # append result + f.write(c + b + '\n') + x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0) # load unique rows + x = x[np.argsort(-fitness(x))] # sort + np.savetxt('evolve.txt', x, '%10.3g') # save sort by fitness + + # Save yaml + for i, k in enumerate(hyp.keys()): + hyp[k] = float(x[0, i + 7]) + with open(yaml_file, 'w') as f: + results = tuple(x[0, :7]) + c = '%10.4g' * len(results) % results # results (P, R, mAP@0.5, mAP@0.5:0.95, val_losses x 3) + f.write('# Hyperparameter Evolution Results\n# Generations: %g\n# Metrics: ' % len(x) + c + '\n\n') + yaml.dump(hyp, f, sort_keys=False) + + if bucket: + os.system('gsutil cp evolve.txt %s gs://%s' % (yaml_file, bucket)) # upload + + +def apply_classifier(x, model, img, im0): + # applies a second stage classifier to yolo outputs + im0 = [im0] if isinstance(im0, np.ndarray) else im0 + for i, d in enumerate(x): # per image + if d is not None and len(d): + d = d.clone() + + # Reshape and pad cutouts + b = xyxy2xywh(d[:, :4]) # boxes + b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square + b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad + d[:, :4] = xywh2xyxy(b).long() + + # Rescale boxes from img_size to im0 size + scale_coords(img.shape[2:], d[:, :4], im0[i].shape) + + # Classes + pred_cls1 = d[:, 5].long() + ims = [] + for j, a in enumerate(d): # per item + cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])] + im = cv2.resize(cutout, (224, 224)) # BGR + # cv2.imwrite('test%i.jpg' % j, cutout) + + im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32 + im /= 255.0 # 0 - 255 to 0.0 - 1.0 + ims.append(im) + + pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction + x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections + + return x + + +def increment_path(path, exist_ok=True, sep=''): + # Increment path, i.e. runs/exp --> runs/exp{sep}0, runs/exp{sep}1 etc. + path = Path(path) # os-agnostic + if (path.exists() and exist_ok) or (not path.exists()): + return str(path) + else: + dirs = glob.glob(f"{path}{sep}*") # similar paths + matches = [re.search(rf"%s{sep}(\d+)" % path.stem, d) for d in dirs] + i = [int(m.groups()[0]) for m in matches if m] # indices + n = max(i) + 1 if i else 2 # increment number + return f"{path}{sep}{n}" # update path diff --git a/test/yolov7-tracker/utils/google_app_engine/Dockerfile b/test/yolov7-tracker/utils/google_app_engine/Dockerfile new file mode 100644 index 0000000..0155618 --- /dev/null +++ b/test/yolov7-tracker/utils/google_app_engine/Dockerfile @@ -0,0 +1,25 @@ +FROM gcr.io/google-appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +# Use -p python3 or -p python3.7 to select python version. Default is version 2. +RUN virtualenv /env -p python3 + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +RUN apt-get update && apt-get install -y python-opencv + +# Copy the application's requirements.txt and run pip to install all +# dependencies into the virtualenv. +ADD requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt + +# Add the application source code. +ADD . /app + +# Run a WSGI server to serve the application. gunicorn must be declared as +# a dependency in requirements.txt. +CMD gunicorn -b :$PORT main:app diff --git a/test/yolov7-tracker/utils/google_app_engine/additional_requirements.txt b/test/yolov7-tracker/utils/google_app_engine/additional_requirements.txt new file mode 100644 index 0000000..5fcc305 --- /dev/null +++ b/test/yolov7-tracker/utils/google_app_engine/additional_requirements.txt @@ -0,0 +1,4 @@ +# add these requirements in your app on top of the existing ones +pip==18.1 +Flask==1.0.2 +gunicorn==19.9.0 diff --git a/test/yolov7-tracker/utils/google_app_engine/app.yaml b/test/yolov7-tracker/utils/google_app_engine/app.yaml new file mode 100644 index 0000000..69b8f68 --- /dev/null +++ b/test/yolov7-tracker/utils/google_app_engine/app.yaml @@ -0,0 +1,14 @@ +runtime: custom +env: flex + +service: yolorapp + +liveness_check: + initial_delay_sec: 600 + +manual_scaling: + instances: 1 +resources: + cpu: 1 + memory_gb: 4 + disk_size_gb: 20 \ No newline at end of file diff --git a/test/yolov7-tracker/utils/google_utils.py b/test/yolov7-tracker/utils/google_utils.py new file mode 100644 index 0000000..c311fd5 --- /dev/null +++ b/test/yolov7-tracker/utils/google_utils.py @@ -0,0 +1,122 @@ +# Google utils: https://cloud.google.com/storage/docs/reference/libraries + +import os +import platform +import subprocess +import time +from pathlib import Path + +import requests +import torch + + +def gsutil_getsize(url=''): + # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du + s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8') + return eval(s.split(' ')[0]) if len(s) else 0 # bytes + + +def attempt_download(file, repo='WongKinYiu/yolov7'): + # Attempt file download if does not exist + file = Path(str(file).strip().replace("'", '').lower()) + + if not file.exists(): + try: + response = requests.get(f'https://api.github.com/repos/{repo}/releases/latest').json() # github api + assets = [x['name'] for x in response['assets']] # release assets + tag = response['tag_name'] # i.e. 'v1.0' + except: # fallback plan + assets = ['yolov7.pt'] + tag = subprocess.check_output('git tag', shell=True).decode().split()[-1] + + name = file.name + if name in assets: + msg = f'{file} missing, try downloading from https://github.com/{repo}/releases/' + redundant = False # second download option + try: # GitHub + url = f'https://github.com/{repo}/releases/download/{tag}/{name}' + print(f'Downloading {url} to {file}...') + torch.hub.download_url_to_file(url, file) + assert file.exists() and file.stat().st_size > 1E6 # check + except Exception as e: # GCP + print(f'Download error: {e}') + assert redundant, 'No secondary mirror' + url = f'https://storage.googleapis.com/{repo}/ckpt/{name}' + print(f'Downloading {url} to {file}...') + os.system(f'curl -L {url} -o {file}') # torch.hub.download_url_to_file(url, weights) + finally: + if not file.exists() or file.stat().st_size < 1E6: # check + file.unlink(missing_ok=True) # remove partial downloads + print(f'ERROR: Download failure: {msg}') + print('') + return + + +def gdrive_download(id='', file='tmp.zip'): + # Downloads a file from Google Drive. from yolov7.utils.google_utils import *; gdrive_download() + t = time.time() + file = Path(file) + cookie = Path('cookie') # gdrive cookie + print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='') + file.unlink(missing_ok=True) # remove existing file + cookie.unlink(missing_ok=True) # remove existing cookie + + # Attempt file download + out = "NUL" if platform.system() == "Windows" else "/dev/null" + os.system(f'curl -c ./cookie -s -L "drive.google.com/uc?export=download&id={id}" > {out}') + if os.path.exists('cookie'): # large file + s = f'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm={get_token()}&id={id}" -o {file}' + else: # small file + s = f'curl -s -L -o {file} "drive.google.com/uc?export=download&id={id}"' + r = os.system(s) # execute, capture return + cookie.unlink(missing_ok=True) # remove existing cookie + + # Error check + if r != 0: + file.unlink(missing_ok=True) # remove partial + print('Download error ') # raise Exception('Download error') + return r + + # Unzip if archive + if file.suffix == '.zip': + print('unzipping... ', end='') + os.system(f'unzip -q {file}') # unzip + file.unlink() # remove zip to free space + + print(f'Done ({time.time() - t:.1f}s)') + return r + + +def get_token(cookie="./cookie"): + with open(cookie) as f: + for line in f: + if "download" in line: + return line.split()[-1] + return "" + +# def upload_blob(bucket_name, source_file_name, destination_blob_name): +# # Uploads a file to a bucket +# # https://cloud.google.com/storage/docs/uploading-objects#storage-upload-object-python +# +# storage_client = storage.Client() +# bucket = storage_client.get_bucket(bucket_name) +# blob = bucket.blob(destination_blob_name) +# +# blob.upload_from_filename(source_file_name) +# +# print('File {} uploaded to {}.'.format( +# source_file_name, +# destination_blob_name)) +# +# +# def download_blob(bucket_name, source_blob_name, destination_file_name): +# # Uploads a blob from a bucket +# storage_client = storage.Client() +# bucket = storage_client.get_bucket(bucket_name) +# blob = bucket.blob(source_blob_name) +# +# blob.download_to_filename(destination_file_name) +# +# print('Blob {} downloaded to {}.'.format( +# source_blob_name, +# destination_file_name)) diff --git a/test/yolov7-tracker/utils/loss.py b/test/yolov7-tracker/utils/loss.py new file mode 100644 index 0000000..2d6c6af --- /dev/null +++ b/test/yolov7-tracker/utils/loss.py @@ -0,0 +1,1697 @@ +# Loss functions + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from utils.general import bbox_iou, bbox_alpha_iou, box_iou, box_giou, box_diou, box_ciou, xywh2xyxy +from utils.torch_utils import is_parallel + + +def smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441 + # return positive, negative label smoothing BCE targets + return 1.0 - 0.5 * eps, 0.5 * eps + + +class BCEBlurWithLogitsLoss(nn.Module): + # BCEwithLogitLoss() with reduced missing label effects. + def __init__(self, alpha=0.05): + super(BCEBlurWithLogitsLoss, self).__init__() + self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss() + self.alpha = alpha + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + pred = torch.sigmoid(pred) # prob from logits + dx = pred - true # reduce only missing label effects + # dx = (pred - true).abs() # reduce missing label and false label effects + alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4)) + loss *= alpha_factor + return loss.mean() + + +class SigmoidBin(nn.Module): + stride = None # strides computed during build + export = False # onnx export + + def __init__(self, bin_count=10, min=0.0, max=1.0, reg_scale = 2.0, use_loss_regression=True, use_fw_regression=True, BCE_weight=1.0, smooth_eps=0.0): + super(SigmoidBin, self).__init__() + + self.bin_count = bin_count + self.length = bin_count + 1 + self.min = min + self.max = max + self.scale = float(max - min) + self.shift = self.scale / 2.0 + + self.use_loss_regression = use_loss_regression + self.use_fw_regression = use_fw_regression + self.reg_scale = reg_scale + self.BCE_weight = BCE_weight + + start = min + (self.scale/2.0) / self.bin_count + end = max - (self.scale/2.0) / self.bin_count + step = self.scale / self.bin_count + self.step = step + #print(f" start = {start}, end = {end}, step = {step} ") + + bins = torch.range(start, end + 0.0001, step).float() + self.register_buffer('bins', bins) + + + self.cp = 1.0 - 0.5 * smooth_eps + self.cn = 0.5 * smooth_eps + + self.BCEbins = nn.BCEWithLogitsLoss(pos_weight=torch.Tensor([BCE_weight])) + self.MSELoss = nn.MSELoss() + + def get_length(self): + return self.length + + def forward(self, pred): + assert pred.shape[-1] == self.length, 'pred.shape[-1]=%d is not equal to self.length=%d' % (pred.shape[-1], self.length) + + pred_reg = (pred[..., 0] * self.reg_scale - self.reg_scale/2.0) * self.step + pred_bin = pred[..., 1:(1+self.bin_count)] + + _, bin_idx = torch.max(pred_bin, dim=-1) + bin_bias = self.bins[bin_idx] + + if self.use_fw_regression: + result = pred_reg + bin_bias + else: + result = bin_bias + result = result.clamp(min=self.min, max=self.max) + + return result + + + def training_loss(self, pred, target): + assert pred.shape[-1] == self.length, 'pred.shape[-1]=%d is not equal to self.length=%d' % (pred.shape[-1], self.length) + assert pred.shape[0] == target.shape[0], 'pred.shape=%d is not equal to the target.shape=%d' % (pred.shape[0], target.shape[0]) + device = pred.device + + pred_reg = (pred[..., 0].sigmoid() * self.reg_scale - self.reg_scale/2.0) * self.step + pred_bin = pred[..., 1:(1+self.bin_count)] + + diff_bin_target = torch.abs(target[..., None] - self.bins) + _, bin_idx = torch.min(diff_bin_target, dim=-1) + + bin_bias = self.bins[bin_idx] + bin_bias.requires_grad = False + result = pred_reg + bin_bias + + target_bins = torch.full_like(pred_bin, self.cn, device=device) # targets + n = pred.shape[0] + target_bins[range(n), bin_idx] = self.cp + + loss_bin = self.BCEbins(pred_bin, target_bins) # BCE + + if self.use_loss_regression: + loss_regression = self.MSELoss(result, target) # MSE + loss = loss_bin + loss_regression + else: + loss = loss_bin + + out_result = result.clamp(min=self.min, max=self.max) + + return loss, out_result + + +class FocalLoss(nn.Module): + # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5) + def __init__(self, loss_fcn, gamma=1.5, alpha=0.25): + super(FocalLoss, self).__init__() + self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss() + self.gamma = gamma + self.alpha = alpha + self.reduction = loss_fcn.reduction + self.loss_fcn.reduction = 'none' # required to apply FL to each element + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + # p_t = torch.exp(-loss) + # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability + + # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py + pred_prob = torch.sigmoid(pred) # prob from logits + p_t = true * pred_prob + (1 - true) * (1 - pred_prob) + alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha) + modulating_factor = (1.0 - p_t) ** self.gamma + loss *= alpha_factor * modulating_factor + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + + +class QFocalLoss(nn.Module): + # Wraps Quality focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5) + def __init__(self, loss_fcn, gamma=1.5, alpha=0.25): + super(QFocalLoss, self).__init__() + self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss() + self.gamma = gamma + self.alpha = alpha + self.reduction = loss_fcn.reduction + self.loss_fcn.reduction = 'none' # required to apply FL to each element + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + + pred_prob = torch.sigmoid(pred) # prob from logits + alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha) + modulating_factor = torch.abs(true - pred_prob) ** self.gamma + loss *= alpha_factor * modulating_factor + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + +class RankSort(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, targets, delta_RS=0.50, eps=1e-10): + + classification_grads=torch.zeros(logits.shape).cuda() + + #Filter fg logits + fg_labels = (targets > 0.) + fg_logits = logits[fg_labels] + fg_targets = targets[fg_labels] + fg_num = len(fg_logits) + + #Do not use bg with scores less than minimum fg logit + #since changing its score does not have an effect on precision + threshold_logit = torch.min(fg_logits)-delta_RS + relevant_bg_labels=((targets==0) & (logits>=threshold_logit)) + + relevant_bg_logits = logits[relevant_bg_labels] + relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda() + sorting_error=torch.zeros(fg_num).cuda() + ranking_error=torch.zeros(fg_num).cuda() + fg_grad=torch.zeros(fg_num).cuda() + + #sort the fg logits + order=torch.argsort(fg_logits) + #Loops over each positive following the order + for ii in order: + # Difference Transforms (x_ij) + fg_relations=fg_logits-fg_logits[ii] + bg_relations=relevant_bg_logits-fg_logits[ii] + + if delta_RS > 0: + fg_relations=torch.clamp(fg_relations/(2*delta_RS)+0.5,min=0,max=1) + bg_relations=torch.clamp(bg_relations/(2*delta_RS)+0.5,min=0,max=1) + else: + fg_relations = (fg_relations >= 0).float() + bg_relations = (bg_relations >= 0).float() + + # Rank of ii among pos and false positive number (bg with larger scores) + rank_pos=torch.sum(fg_relations) + FP_num=torch.sum(bg_relations) + + # Rank of ii among all examples + rank=rank_pos+FP_num + + # Ranking error of example ii. target_ranking_error is always 0. (Eq. 7) + ranking_error[ii]=FP_num/rank + + # Current sorting error of example ii. (Eq. 7) + current_sorting_error = torch.sum(fg_relations*(1-fg_targets))/rank_pos + + #Find examples in the target sorted order for example ii + iou_relations = (fg_targets >= fg_targets[ii]) + target_sorted_order = iou_relations * fg_relations + + #The rank of ii among positives in sorted order + rank_pos_target = torch.sum(target_sorted_order) + + #Compute target sorting error. (Eq. 8) + #Since target ranking error is 0, this is also total target error + target_sorting_error= torch.sum(target_sorted_order*(1-fg_targets))/rank_pos_target + + #Compute sorting error on example ii + sorting_error[ii] = current_sorting_error - target_sorting_error + + #Identity Update for Ranking Error + if FP_num > eps: + #For ii the update is the ranking error + fg_grad[ii] -= ranking_error[ii] + #For negatives, distribute error via ranking pmf (i.e. bg_relations/FP_num) + relevant_bg_grad += (bg_relations*(ranking_error[ii]/FP_num)) + + #Find the positives that are misranked (the cause of the error) + #These are the ones with smaller IoU but larger logits + missorted_examples = (~ iou_relations) * fg_relations + + #Denominotor of sorting pmf + sorting_pmf_denom = torch.sum(missorted_examples) + + #Identity Update for Sorting Error + if sorting_pmf_denom > eps: + #For ii the update is the sorting error + fg_grad[ii] -= sorting_error[ii] + #For positives, distribute error via sorting pmf (i.e. missorted_examples/sorting_pmf_denom) + fg_grad += (missorted_examples*(sorting_error[ii]/sorting_pmf_denom)) + + #Normalize gradients by number of positives + classification_grads[fg_labels]= (fg_grad/fg_num) + classification_grads[relevant_bg_labels]= (relevant_bg_grad/fg_num) + + ctx.save_for_backward(classification_grads) + + return ranking_error.mean(), sorting_error.mean() + + @staticmethod + def backward(ctx, out_grad1, out_grad2): + g1, =ctx.saved_tensors + return g1*out_grad1, None, None, None + +class aLRPLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, targets, regression_losses, delta=1., eps=1e-5): + classification_grads=torch.zeros(logits.shape).cuda() + + #Filter fg logits + fg_labels = (targets == 1) + fg_logits = logits[fg_labels] + fg_num = len(fg_logits) + + #Do not use bg with scores less than minimum fg logit + #since changing its score does not have an effect on precision + threshold_logit = torch.min(fg_logits)-delta + + #Get valid bg logits + relevant_bg_labels=((targets==0)&(logits>=threshold_logit)) + relevant_bg_logits=logits[relevant_bg_labels] + relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda() + rank=torch.zeros(fg_num).cuda() + prec=torch.zeros(fg_num).cuda() + fg_grad=torch.zeros(fg_num).cuda() + + max_prec=0 + #sort the fg logits + order=torch.argsort(fg_logits) + #Loops over each positive following the order + for ii in order: + #x_ij s as score differences with fgs + fg_relations=fg_logits-fg_logits[ii] + #Apply piecewise linear function and determine relations with fgs + fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1) + #Discard i=j in the summation in rank_pos + fg_relations[ii]=0 + + #x_ij s as score differences with bgs + bg_relations=relevant_bg_logits-fg_logits[ii] + #Apply piecewise linear function and determine relations with bgs + bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1) + + #Compute the rank of the example within fgs and number of bgs with larger scores + rank_pos=1+torch.sum(fg_relations) + FP_num=torch.sum(bg_relations) + #Store the total since it is normalizer also for aLRP Regression error + rank[ii]=rank_pos+FP_num + + #Compute precision for this example to compute classification loss + prec[ii]=rank_pos/rank[ii] + #For stability, set eps to a infinitesmall value (e.g. 1e-6), then compute grads + if FP_num > eps: + fg_grad[ii] = -(torch.sum(fg_relations*regression_losses)+FP_num)/rank[ii] + relevant_bg_grad += (bg_relations*(-fg_grad[ii]/FP_num)) + + #aLRP with grad formulation fg gradient + classification_grads[fg_labels]= fg_grad + #aLRP with grad formulation bg gradient + classification_grads[relevant_bg_labels]= relevant_bg_grad + + classification_grads /= (fg_num) + + cls_loss=1-prec.mean() + ctx.save_for_backward(classification_grads) + + return cls_loss, rank, order + + @staticmethod + def backward(ctx, out_grad1, out_grad2, out_grad3): + g1, =ctx.saved_tensors + return g1*out_grad1, None, None, None, None + + +class APLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, targets, delta=1.): + classification_grads=torch.zeros(logits.shape).cuda() + + #Filter fg logits + fg_labels = (targets == 1) + fg_logits = logits[fg_labels] + fg_num = len(fg_logits) + + #Do not use bg with scores less than minimum fg logit + #since changing its score does not have an effect on precision + threshold_logit = torch.min(fg_logits)-delta + + #Get valid bg logits + relevant_bg_labels=((targets==0)&(logits>=threshold_logit)) + relevant_bg_logits=logits[relevant_bg_labels] + relevant_bg_grad=torch.zeros(len(relevant_bg_logits)).cuda() + rank=torch.zeros(fg_num).cuda() + prec=torch.zeros(fg_num).cuda() + fg_grad=torch.zeros(fg_num).cuda() + + max_prec=0 + #sort the fg logits + order=torch.argsort(fg_logits) + #Loops over each positive following the order + for ii in order: + #x_ij s as score differences with fgs + fg_relations=fg_logits-fg_logits[ii] + #Apply piecewise linear function and determine relations with fgs + fg_relations=torch.clamp(fg_relations/(2*delta)+0.5,min=0,max=1) + #Discard i=j in the summation in rank_pos + fg_relations[ii]=0 + + #x_ij s as score differences with bgs + bg_relations=relevant_bg_logits-fg_logits[ii] + #Apply piecewise linear function and determine relations with bgs + bg_relations=torch.clamp(bg_relations/(2*delta)+0.5,min=0,max=1) + + #Compute the rank of the example within fgs and number of bgs with larger scores + rank_pos=1+torch.sum(fg_relations) + FP_num=torch.sum(bg_relations) + #Store the total since it is normalizer also for aLRP Regression error + rank[ii]=rank_pos+FP_num + + #Compute precision for this example + current_prec=rank_pos/rank[ii] + + #Compute interpolated AP and store gradients for relevant bg examples + if (max_prec<=current_prec): + max_prec=current_prec + relevant_bg_grad += (bg_relations/rank[ii]) + else: + relevant_bg_grad += (bg_relations/rank[ii])*(((1-max_prec)/(1-current_prec))) + + #Store fg gradients + fg_grad[ii]=-(1-max_prec) + prec[ii]=max_prec + + #aLRP with grad formulation fg gradient + classification_grads[fg_labels]= fg_grad + #aLRP with grad formulation bg gradient + classification_grads[relevant_bg_labels]= relevant_bg_grad + + classification_grads /= fg_num + + cls_loss=1-prec.mean() + ctx.save_for_backward(classification_grads) + + return cls_loss + + @staticmethod + def backward(ctx, out_grad1): + g1, =ctx.saved_tensors + return g1*out_grad1, None, None + + +class ComputeLoss: + # Compute losses + def __init__(self, model, autobalance=False): + super(ComputeLoss, self).__init__() + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7 + #self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.1, .05]) # P3-P7 + #self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.5, 0.4, .1]) # P3-P7 + self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance + for k in 'na', 'nc', 'nl', 'anchors': + setattr(self, k, getattr(det, k)) + + def __call__(self, p, targets): # predictions, targets, model + device = targets.device + lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device) + tcls, tbox, indices, anchors = self.build_targets(p, targets) # targets + + # Losses + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = indices[i] # image, anchor, gridy, gridx + tobj = torch.zeros_like(pi[..., 0], device=device) # target obj + + n = b.shape[0] # number of targets + if n: + ps = pi[b, a, gj, gi] # prediction subset corresponding to targets + + # Regression + pxy = ps[:, :2].sigmoid() * 2. - 0.5 + pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1) # predicted box + iou = bbox_iou(pbox.T, tbox[i], x1y1x2y2=False, CIoU=True) # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio + + # Classification + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets + t[range(n), tcls[i]] = self.cp + #t[t==self.cp] = iou.detach().clamp(0).type(t.dtype) + lcls += self.BCEcls(ps[:, 5:], t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + obji = self.BCEobj(pi[..., 4], tobj) + lobj += obji * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + bs = tobj.shape[0] # batch size + + loss = lbox + lobj + lcls + return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach() + + def build_targets(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + tcls, tbox, indices, anch = [], [], [], [] + gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(self.nl): + anchors = self.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices + tbox.append(torch.cat((gxy - gij, gwh), 1)) # box + anch.append(anchors[a]) # anchors + tcls.append(c) # class + + return tcls, tbox, indices, anch + + +class ComputeLossOTA: + # Compute losses + def __init__(self, model, autobalance=False): + super(ComputeLossOTA, self).__init__() + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7 + self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance + for k in 'na', 'nc', 'nl', 'anchors', 'stride': + setattr(self, k, getattr(det, k)) + + def __call__(self, p, targets, imgs): # predictions, targets, model + device = targets.device + lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device) + bs, as_, gjs, gis, targets, anchors = self.build_targets(p, targets, imgs) + pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p] + + + # Losses + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx + tobj = torch.zeros_like(pi[..., 0], device=device) # target obj + + n = b.shape[0] # number of targets + if n: + ps = pi[b, a, gj, gi] # prediction subset corresponding to targets + + # Regression + grid = torch.stack([gi, gj], dim=1) + pxy = ps[:, :2].sigmoid() * 2. - 0.5 + #pxy = ps[:, :2].sigmoid() * 3. - 1. + pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1) # predicted box + selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i] + selected_tbox[:, :2] -= grid + iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio + + # Classification + selected_tcls = targets[i][:, 1].long() + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets + t[range(n), selected_tcls] = self.cp + lcls += self.BCEcls(ps[:, 5:], t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + obji = self.BCEobj(pi[..., 4], tobj) + lobj += obji * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + bs = tobj.shape[0] # batch size + + loss = lbox + lobj + lcls + return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach() + + def build_targets(self, p, targets, imgs): + + #indices, anch = self.find_positive(p, targets) + indices, anch = self.find_3_positive(p, targets) + #indices, anch = self.find_4_positive(p, targets) + #indices, anch = self.find_5_positive(p, targets) + #indices, anch = self.find_9_positive(p, targets) + + matching_bs = [[] for pp in p] + matching_as = [[] for pp in p] + matching_gjs = [[] for pp in p] + matching_gis = [[] for pp in p] + matching_targets = [[] for pp in p] + matching_anchs = [[] for pp in p] + + nl = len(p) + + for batch_idx in range(p[0].shape[0]): + + b_idx = targets[:, 0]==batch_idx + this_target = targets[b_idx] + if this_target.shape[0] == 0: + continue + + txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1] + txyxy = xywh2xyxy(txywh) + + pxyxys = [] + p_cls = [] + p_obj = [] + from_which_layer = [] + all_b = [] + all_a = [] + all_gj = [] + all_gi = [] + all_anch = [] + + for i, pi in enumerate(p): + + b, a, gj, gi = indices[i] + idx = (b == batch_idx) + b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] + all_b.append(b) + all_a.append(a) + all_gj.append(gj) + all_gi.append(gi) + all_anch.append(anch[i][idx]) + from_which_layer.append(torch.ones(size=(len(b),)) * i) + + fg_pred = pi[b, a, gj, gi] + p_obj.append(fg_pred[:, 4:5]) + p_cls.append(fg_pred[:, 5:]) + + grid = torch.stack([gi, gj], dim=1) + pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8. + #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i] + pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8. + pxywh = torch.cat([pxy, pwh], dim=-1) + pxyxy = xywh2xyxy(pxywh) + pxyxys.append(pxyxy) + + pxyxys = torch.cat(pxyxys, dim=0) + if pxyxys.shape[0] == 0: + continue + p_obj = torch.cat(p_obj, dim=0) + p_cls = torch.cat(p_cls, dim=0) + from_which_layer = torch.cat(from_which_layer, dim=0) + all_b = torch.cat(all_b, dim=0) + all_a = torch.cat(all_a, dim=0) + all_gj = torch.cat(all_gj, dim=0) + all_gi = torch.cat(all_gi, dim=0) + all_anch = torch.cat(all_anch, dim=0) + + pair_wise_iou = box_iou(txyxy, pxyxys) + + pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8) + + top_k, _ = torch.topk(pair_wise_iou, min(10, pair_wise_iou.shape[1]), dim=1) + dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1) + + gt_cls_per_image = ( + F.one_hot(this_target[:, 1].to(torch.int64), self.nc) + .float() + .unsqueeze(1) + .repeat(1, pxyxys.shape[0], 1) + ) + + num_gt = this_target.shape[0] + cls_preds_ = ( + p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + + y = cls_preds_.sqrt_() + pair_wise_cls_loss = F.binary_cross_entropy_with_logits( + torch.log(y/(1-y)) , gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_iou_loss + ) + + matching_matrix = torch.zeros_like(cost) + + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1.0 + + del top_k, dynamic_ks + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0.0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0 + fg_mask_inboxes = matching_matrix.sum(0) > 0.0 + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + + from_which_layer = from_which_layer[fg_mask_inboxes] + all_b = all_b[fg_mask_inboxes] + all_a = all_a[fg_mask_inboxes] + all_gj = all_gj[fg_mask_inboxes] + all_gi = all_gi[fg_mask_inboxes] + all_anch = all_anch[fg_mask_inboxes] + + this_target = this_target[matched_gt_inds] + + for i in range(nl): + layer_idx = from_which_layer == i + matching_bs[i].append(all_b[layer_idx]) + matching_as[i].append(all_a[layer_idx]) + matching_gjs[i].append(all_gj[layer_idx]) + matching_gis[i].append(all_gi[layer_idx]) + matching_targets[i].append(this_target[layer_idx]) + matching_anchs[i].append(all_anch[layer_idx]) + + for i in range(nl): + if matching_targets[i] != []: + matching_bs[i] = torch.cat(matching_bs[i], dim=0) + matching_as[i] = torch.cat(matching_as[i], dim=0) + matching_gjs[i] = torch.cat(matching_gjs[i], dim=0) + matching_gis[i] = torch.cat(matching_gis[i], dim=0) + matching_targets[i] = torch.cat(matching_targets[i], dim=0) + matching_anchs[i] = torch.cat(matching_anchs[i], dim=0) + else: + matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + + return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs + + def find_3_positive(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + indices, anch = [], [] + gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(self.nl): + anchors = self.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices + anch.append(anchors[a]) # anchors + + return indices, anch + + +class ComputeLossBinOTA: + # Compute losses + def __init__(self, model, autobalance=False): + super(ComputeLossBinOTA, self).__init__() + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + #MSEangle = nn.MSELoss().to(device) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7 + self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance + for k in 'na', 'nc', 'nl', 'anchors', 'stride', 'bin_count': + setattr(self, k, getattr(det, k)) + + #xy_bin_sigmoid = SigmoidBin(bin_count=11, min=-0.5, max=1.5, use_loss_regression=False).to(device) + wh_bin_sigmoid = SigmoidBin(bin_count=self.bin_count, min=0.0, max=4.0, use_loss_regression=False).to(device) + #angle_bin_sigmoid = SigmoidBin(bin_count=31, min=-1.1, max=1.1, use_loss_regression=False).to(device) + self.wh_bin_sigmoid = wh_bin_sigmoid + + def __call__(self, p, targets, imgs): # predictions, targets, model + device = targets.device + lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device) + bs, as_, gjs, gis, targets, anchors = self.build_targets(p, targets, imgs) + pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p] + + + # Losses + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx + tobj = torch.zeros_like(pi[..., 0], device=device) # target obj + + obj_idx = self.wh_bin_sigmoid.get_length()*2 + 2 # x,y, w-bce, h-bce # xy_bin_sigmoid.get_length()*2 + + n = b.shape[0] # number of targets + if n: + ps = pi[b, a, gj, gi] # prediction subset corresponding to targets + + # Regression + grid = torch.stack([gi, gj], dim=1) + selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i] + selected_tbox[:, :2] -= grid + + #pxy = ps[:, :2].sigmoid() * 2. - 0.5 + ##pxy = ps[:, :2].sigmoid() * 3. - 1. + #pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i] + #pbox = torch.cat((pxy, pwh), 1) # predicted box + + #x_loss, px = xy_bin_sigmoid.training_loss(ps[..., 0:12], tbox[i][..., 0]) + #y_loss, py = xy_bin_sigmoid.training_loss(ps[..., 12:24], tbox[i][..., 1]) + w_loss, pw = self.wh_bin_sigmoid.training_loss(ps[..., 2:(3+self.bin_count)], selected_tbox[..., 2] / anchors[i][..., 0]) + h_loss, ph = self.wh_bin_sigmoid.training_loss(ps[..., (3+self.bin_count):obj_idx], selected_tbox[..., 3] / anchors[i][..., 1]) + + pw *= anchors[i][..., 0] + ph *= anchors[i][..., 1] + + px = ps[:, 0].sigmoid() * 2. - 0.5 + py = ps[:, 1].sigmoid() * 2. - 0.5 + + lbox += w_loss + h_loss # + x_loss + y_loss + + #print(f"\n px = {px.shape}, py = {py.shape}, pw = {pw.shape}, ph = {ph.shape} \n") + + pbox = torch.cat((px.unsqueeze(1), py.unsqueeze(1), pw.unsqueeze(1), ph.unsqueeze(1)), 1).to(device) # predicted box + + + + + iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio + + # Classification + selected_tcls = targets[i][:, 1].long() + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(ps[:, (1+obj_idx):], self.cn, device=device) # targets + t[range(n), selected_tcls] = self.cp + lcls += self.BCEcls(ps[:, (1+obj_idx):], t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + obji = self.BCEobj(pi[..., obj_idx], tobj) + lobj += obji * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + bs = tobj.shape[0] # batch size + + loss = lbox + lobj + lcls + return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach() + + def build_targets(self, p, targets, imgs): + + #indices, anch = self.find_positive(p, targets) + indices, anch = self.find_3_positive(p, targets) + #indices, anch = self.find_4_positive(p, targets) + #indices, anch = self.find_5_positive(p, targets) + #indices, anch = self.find_9_positive(p, targets) + + matching_bs = [[] for pp in p] + matching_as = [[] for pp in p] + matching_gjs = [[] for pp in p] + matching_gis = [[] for pp in p] + matching_targets = [[] for pp in p] + matching_anchs = [[] for pp in p] + + nl = len(p) + + for batch_idx in range(p[0].shape[0]): + + b_idx = targets[:, 0]==batch_idx + this_target = targets[b_idx] + if this_target.shape[0] == 0: + continue + + txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1] + txyxy = xywh2xyxy(txywh) + + pxyxys = [] + p_cls = [] + p_obj = [] + from_which_layer = [] + all_b = [] + all_a = [] + all_gj = [] + all_gi = [] + all_anch = [] + + for i, pi in enumerate(p): + + obj_idx = self.wh_bin_sigmoid.get_length()*2 + 2 + + b, a, gj, gi = indices[i] + idx = (b == batch_idx) + b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] + all_b.append(b) + all_a.append(a) + all_gj.append(gj) + all_gi.append(gi) + all_anch.append(anch[i][idx]) + from_which_layer.append(torch.ones(size=(len(b),)) * i) + + fg_pred = pi[b, a, gj, gi] + p_obj.append(fg_pred[:, obj_idx:(obj_idx+1)]) + p_cls.append(fg_pred[:, (obj_idx+1):]) + + grid = torch.stack([gi, gj], dim=1) + pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8. + #pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8. + pw = self.wh_bin_sigmoid.forward(fg_pred[..., 2:(3+self.bin_count)].sigmoid()) * anch[i][idx][:, 0] * self.stride[i] + ph = self.wh_bin_sigmoid.forward(fg_pred[..., (3+self.bin_count):obj_idx].sigmoid()) * anch[i][idx][:, 1] * self.stride[i] + + pxywh = torch.cat([pxy, pw.unsqueeze(1), ph.unsqueeze(1)], dim=-1) + pxyxy = xywh2xyxy(pxywh) + pxyxys.append(pxyxy) + + pxyxys = torch.cat(pxyxys, dim=0) + if pxyxys.shape[0] == 0: + continue + p_obj = torch.cat(p_obj, dim=0) + p_cls = torch.cat(p_cls, dim=0) + from_which_layer = torch.cat(from_which_layer, dim=0) + all_b = torch.cat(all_b, dim=0) + all_a = torch.cat(all_a, dim=0) + all_gj = torch.cat(all_gj, dim=0) + all_gi = torch.cat(all_gi, dim=0) + all_anch = torch.cat(all_anch, dim=0) + + pair_wise_iou = box_iou(txyxy, pxyxys) + + pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8) + + top_k, _ = torch.topk(pair_wise_iou, min(10, pair_wise_iou.shape[1]), dim=1) + dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1) + + gt_cls_per_image = ( + F.one_hot(this_target[:, 1].to(torch.int64), self.nc) + .float() + .unsqueeze(1) + .repeat(1, pxyxys.shape[0], 1) + ) + + num_gt = this_target.shape[0] + cls_preds_ = ( + p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + + y = cls_preds_.sqrt_() + pair_wise_cls_loss = F.binary_cross_entropy_with_logits( + torch.log(y/(1-y)) , gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_iou_loss + ) + + matching_matrix = torch.zeros_like(cost) + + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1.0 + + del top_k, dynamic_ks + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0.0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0 + fg_mask_inboxes = matching_matrix.sum(0) > 0.0 + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + + from_which_layer = from_which_layer[fg_mask_inboxes] + all_b = all_b[fg_mask_inboxes] + all_a = all_a[fg_mask_inboxes] + all_gj = all_gj[fg_mask_inboxes] + all_gi = all_gi[fg_mask_inboxes] + all_anch = all_anch[fg_mask_inboxes] + + this_target = this_target[matched_gt_inds] + + for i in range(nl): + layer_idx = from_which_layer == i + matching_bs[i].append(all_b[layer_idx]) + matching_as[i].append(all_a[layer_idx]) + matching_gjs[i].append(all_gj[layer_idx]) + matching_gis[i].append(all_gi[layer_idx]) + matching_targets[i].append(this_target[layer_idx]) + matching_anchs[i].append(all_anch[layer_idx]) + + for i in range(nl): + if matching_targets[i] != []: + matching_bs[i] = torch.cat(matching_bs[i], dim=0) + matching_as[i] = torch.cat(matching_as[i], dim=0) + matching_gjs[i] = torch.cat(matching_gjs[i], dim=0) + matching_gis[i] = torch.cat(matching_gis[i], dim=0) + matching_targets[i] = torch.cat(matching_targets[i], dim=0) + matching_anchs[i] = torch.cat(matching_anchs[i], dim=0) + else: + matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + + return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs + + def find_3_positive(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + indices, anch = [], [] + gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(self.nl): + anchors = self.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices + anch.append(anchors[a]) # anchors + + return indices, anch + + +class ComputeLossAuxOTA: + # Compute losses + def __init__(self, model, autobalance=False): + super(ComputeLossAuxOTA, self).__init__() + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + det = model.module.model[-1] if is_parallel(model) else model.model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(det.nl, [4.0, 1.0, 0.25, 0.06, .02]) # P3-P7 + self.ssi = list(det.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, model.gr, h, autobalance + for k in 'na', 'nc', 'nl', 'anchors', 'stride': + setattr(self, k, getattr(det, k)) + + def __call__(self, p, targets, imgs): # predictions, targets, model + device = targets.device + lcls, lbox, lobj = torch.zeros(1, device=device), torch.zeros(1, device=device), torch.zeros(1, device=device) + bs_aux, as_aux_, gjs_aux, gis_aux, targets_aux, anchors_aux = self.build_targets2(p[:self.nl], targets, imgs) + bs, as_, gjs, gis, targets, anchors = self.build_targets(p[:self.nl], targets, imgs) + pre_gen_gains_aux = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p[:self.nl]] + pre_gen_gains = [torch.tensor(pp.shape, device=device)[[3, 2, 3, 2]] for pp in p[:self.nl]] + + + # Losses + for i in range(self.nl): # layer index, layer predictions + pi = p[i] + pi_aux = p[i+self.nl] + b, a, gj, gi = bs[i], as_[i], gjs[i], gis[i] # image, anchor, gridy, gridx + b_aux, a_aux, gj_aux, gi_aux = bs_aux[i], as_aux_[i], gjs_aux[i], gis_aux[i] # image, anchor, gridy, gridx + tobj = torch.zeros_like(pi[..., 0], device=device) # target obj + tobj_aux = torch.zeros_like(pi_aux[..., 0], device=device) # target obj + + n = b.shape[0] # number of targets + if n: + ps = pi[b, a, gj, gi] # prediction subset corresponding to targets + + # Regression + grid = torch.stack([gi, gj], dim=1) + pxy = ps[:, :2].sigmoid() * 2. - 0.5 + pwh = (ps[:, 2:4].sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1) # predicted box + selected_tbox = targets[i][:, 2:6] * pre_gen_gains[i] + selected_tbox[:, :2] -= grid + iou = bbox_iou(pbox.T, selected_tbox, x1y1x2y2=False, CIoU=True) # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + tobj[b, a, gj, gi] = (1.0 - self.gr) + self.gr * iou.detach().clamp(0).type(tobj.dtype) # iou ratio + + # Classification + selected_tcls = targets[i][:, 1].long() + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(ps[:, 5:], self.cn, device=device) # targets + t[range(n), selected_tcls] = self.cp + lcls += self.BCEcls(ps[:, 5:], t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + n_aux = b_aux.shape[0] # number of targets + if n_aux: + ps_aux = pi_aux[b_aux, a_aux, gj_aux, gi_aux] # prediction subset corresponding to targets + grid_aux = torch.stack([gi_aux, gj_aux], dim=1) + pxy_aux = ps_aux[:, :2].sigmoid() * 2. - 0.5 + #pxy_aux = ps_aux[:, :2].sigmoid() * 3. - 1. + pwh_aux = (ps_aux[:, 2:4].sigmoid() * 2) ** 2 * anchors_aux[i] + pbox_aux = torch.cat((pxy_aux, pwh_aux), 1) # predicted box + selected_tbox_aux = targets_aux[i][:, 2:6] * pre_gen_gains_aux[i] + selected_tbox_aux[:, :2] -= grid_aux + iou_aux = bbox_iou(pbox_aux.T, selected_tbox_aux, x1y1x2y2=False, CIoU=True) # iou(prediction, target) + lbox += 0.25 * (1.0 - iou_aux).mean() # iou loss + + # Objectness + tobj_aux[b_aux, a_aux, gj_aux, gi_aux] = (1.0 - self.gr) + self.gr * iou_aux.detach().clamp(0).type(tobj_aux.dtype) # iou ratio + + # Classification + selected_tcls_aux = targets_aux[i][:, 1].long() + if self.nc > 1: # cls loss (only if multiple classes) + t_aux = torch.full_like(ps_aux[:, 5:], self.cn, device=device) # targets + t_aux[range(n_aux), selected_tcls_aux] = self.cp + lcls += 0.25 * self.BCEcls(ps_aux[:, 5:], t_aux) # BCE + + obji = self.BCEobj(pi[..., 4], tobj) + obji_aux = self.BCEobj(pi_aux[..., 4], tobj_aux) + lobj += obji * self.balance[i] + 0.25 * obji_aux * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + bs = tobj.shape[0] # batch size + + loss = lbox + lobj + lcls + return loss * bs, torch.cat((lbox, lobj, lcls, loss)).detach() + + def build_targets(self, p, targets, imgs): + + indices, anch = self.find_3_positive(p, targets) + + matching_bs = [[] for pp in p] + matching_as = [[] for pp in p] + matching_gjs = [[] for pp in p] + matching_gis = [[] for pp in p] + matching_targets = [[] for pp in p] + matching_anchs = [[] for pp in p] + + nl = len(p) + + for batch_idx in range(p[0].shape[0]): + + b_idx = targets[:, 0]==batch_idx + this_target = targets[b_idx] + if this_target.shape[0] == 0: + continue + + txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1] + txyxy = xywh2xyxy(txywh) + + pxyxys = [] + p_cls = [] + p_obj = [] + from_which_layer = [] + all_b = [] + all_a = [] + all_gj = [] + all_gi = [] + all_anch = [] + + for i, pi in enumerate(p): + + b, a, gj, gi = indices[i] + idx = (b == batch_idx) + b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] + all_b.append(b) + all_a.append(a) + all_gj.append(gj) + all_gi.append(gi) + all_anch.append(anch[i][idx]) + from_which_layer.append(torch.ones(size=(len(b),)) * i) + + fg_pred = pi[b, a, gj, gi] + p_obj.append(fg_pred[:, 4:5]) + p_cls.append(fg_pred[:, 5:]) + + grid = torch.stack([gi, gj], dim=1) + pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8. + #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i] + pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8. + pxywh = torch.cat([pxy, pwh], dim=-1) + pxyxy = xywh2xyxy(pxywh) + pxyxys.append(pxyxy) + + pxyxys = torch.cat(pxyxys, dim=0) + if pxyxys.shape[0] == 0: + continue + p_obj = torch.cat(p_obj, dim=0) + p_cls = torch.cat(p_cls, dim=0) + from_which_layer = torch.cat(from_which_layer, dim=0) + all_b = torch.cat(all_b, dim=0) + all_a = torch.cat(all_a, dim=0) + all_gj = torch.cat(all_gj, dim=0) + all_gi = torch.cat(all_gi, dim=0) + all_anch = torch.cat(all_anch, dim=0) + + pair_wise_iou = box_iou(txyxy, pxyxys) + + pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8) + + top_k, _ = torch.topk(pair_wise_iou, min(20, pair_wise_iou.shape[1]), dim=1) + dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1) + + gt_cls_per_image = ( + F.one_hot(this_target[:, 1].to(torch.int64), self.nc) + .float() + .unsqueeze(1) + .repeat(1, pxyxys.shape[0], 1) + ) + + num_gt = this_target.shape[0] + cls_preds_ = ( + p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + + y = cls_preds_.sqrt_() + pair_wise_cls_loss = F.binary_cross_entropy_with_logits( + torch.log(y/(1-y)) , gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_iou_loss + ) + + matching_matrix = torch.zeros_like(cost) + + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1.0 + + del top_k, dynamic_ks + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0.0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0 + fg_mask_inboxes = matching_matrix.sum(0) > 0.0 + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + + from_which_layer = from_which_layer[fg_mask_inboxes] + all_b = all_b[fg_mask_inboxes] + all_a = all_a[fg_mask_inboxes] + all_gj = all_gj[fg_mask_inboxes] + all_gi = all_gi[fg_mask_inboxes] + all_anch = all_anch[fg_mask_inboxes] + + this_target = this_target[matched_gt_inds] + + for i in range(nl): + layer_idx = from_which_layer == i + matching_bs[i].append(all_b[layer_idx]) + matching_as[i].append(all_a[layer_idx]) + matching_gjs[i].append(all_gj[layer_idx]) + matching_gis[i].append(all_gi[layer_idx]) + matching_targets[i].append(this_target[layer_idx]) + matching_anchs[i].append(all_anch[layer_idx]) + + for i in range(nl): + if matching_targets[i] != []: + matching_bs[i] = torch.cat(matching_bs[i], dim=0) + matching_as[i] = torch.cat(matching_as[i], dim=0) + matching_gjs[i] = torch.cat(matching_gjs[i], dim=0) + matching_gis[i] = torch.cat(matching_gis[i], dim=0) + matching_targets[i] = torch.cat(matching_targets[i], dim=0) + matching_anchs[i] = torch.cat(matching_anchs[i], dim=0) + else: + matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + + return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs + + def build_targets2(self, p, targets, imgs): + + indices, anch = self.find_5_positive(p, targets) + + matching_bs = [[] for pp in p] + matching_as = [[] for pp in p] + matching_gjs = [[] for pp in p] + matching_gis = [[] for pp in p] + matching_targets = [[] for pp in p] + matching_anchs = [[] for pp in p] + + nl = len(p) + + for batch_idx in range(p[0].shape[0]): + + b_idx = targets[:, 0]==batch_idx + this_target = targets[b_idx] + if this_target.shape[0] == 0: + continue + + txywh = this_target[:, 2:6] * imgs[batch_idx].shape[1] + txyxy = xywh2xyxy(txywh) + + pxyxys = [] + p_cls = [] + p_obj = [] + from_which_layer = [] + all_b = [] + all_a = [] + all_gj = [] + all_gi = [] + all_anch = [] + + for i, pi in enumerate(p): + + b, a, gj, gi = indices[i] + idx = (b == batch_idx) + b, a, gj, gi = b[idx], a[idx], gj[idx], gi[idx] + all_b.append(b) + all_a.append(a) + all_gj.append(gj) + all_gi.append(gi) + all_anch.append(anch[i][idx]) + from_which_layer.append(torch.ones(size=(len(b),)) * i) + + fg_pred = pi[b, a, gj, gi] + p_obj.append(fg_pred[:, 4:5]) + p_cls.append(fg_pred[:, 5:]) + + grid = torch.stack([gi, gj], dim=1) + pxy = (fg_pred[:, :2].sigmoid() * 2. - 0.5 + grid) * self.stride[i] #/ 8. + #pxy = (fg_pred[:, :2].sigmoid() * 3. - 1. + grid) * self.stride[i] + pwh = (fg_pred[:, 2:4].sigmoid() * 2) ** 2 * anch[i][idx] * self.stride[i] #/ 8. + pxywh = torch.cat([pxy, pwh], dim=-1) + pxyxy = xywh2xyxy(pxywh) + pxyxys.append(pxyxy) + + pxyxys = torch.cat(pxyxys, dim=0) + if pxyxys.shape[0] == 0: + continue + p_obj = torch.cat(p_obj, dim=0) + p_cls = torch.cat(p_cls, dim=0) + from_which_layer = torch.cat(from_which_layer, dim=0) + all_b = torch.cat(all_b, dim=0) + all_a = torch.cat(all_a, dim=0) + all_gj = torch.cat(all_gj, dim=0) + all_gi = torch.cat(all_gi, dim=0) + all_anch = torch.cat(all_anch, dim=0) + + pair_wise_iou = box_iou(txyxy, pxyxys) + + pair_wise_iou_loss = -torch.log(pair_wise_iou + 1e-8) + + top_k, _ = torch.topk(pair_wise_iou, min(20, pair_wise_iou.shape[1]), dim=1) + dynamic_ks = torch.clamp(top_k.sum(1).int(), min=1) + + gt_cls_per_image = ( + F.one_hot(this_target[:, 1].to(torch.int64), self.nc) + .float() + .unsqueeze(1) + .repeat(1, pxyxys.shape[0], 1) + ) + + num_gt = this_target.shape[0] + cls_preds_ = ( + p_cls.float().unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + * p_obj.unsqueeze(0).repeat(num_gt, 1, 1).sigmoid_() + ) + + y = cls_preds_.sqrt_() + pair_wise_cls_loss = F.binary_cross_entropy_with_logits( + torch.log(y/(1-y)) , gt_cls_per_image, reduction="none" + ).sum(-1) + del cls_preds_ + + cost = ( + pair_wise_cls_loss + + 3.0 * pair_wise_iou_loss + ) + + matching_matrix = torch.zeros_like(cost) + + for gt_idx in range(num_gt): + _, pos_idx = torch.topk( + cost[gt_idx], k=dynamic_ks[gt_idx].item(), largest=False + ) + matching_matrix[gt_idx][pos_idx] = 1.0 + + del top_k, dynamic_ks + anchor_matching_gt = matching_matrix.sum(0) + if (anchor_matching_gt > 1).sum() > 0: + _, cost_argmin = torch.min(cost[:, anchor_matching_gt > 1], dim=0) + matching_matrix[:, anchor_matching_gt > 1] *= 0.0 + matching_matrix[cost_argmin, anchor_matching_gt > 1] = 1.0 + fg_mask_inboxes = matching_matrix.sum(0) > 0.0 + matched_gt_inds = matching_matrix[:, fg_mask_inboxes].argmax(0) + + from_which_layer = from_which_layer[fg_mask_inboxes] + all_b = all_b[fg_mask_inboxes] + all_a = all_a[fg_mask_inboxes] + all_gj = all_gj[fg_mask_inboxes] + all_gi = all_gi[fg_mask_inboxes] + all_anch = all_anch[fg_mask_inboxes] + + this_target = this_target[matched_gt_inds] + + for i in range(nl): + layer_idx = from_which_layer == i + matching_bs[i].append(all_b[layer_idx]) + matching_as[i].append(all_a[layer_idx]) + matching_gjs[i].append(all_gj[layer_idx]) + matching_gis[i].append(all_gi[layer_idx]) + matching_targets[i].append(this_target[layer_idx]) + matching_anchs[i].append(all_anch[layer_idx]) + + for i in range(nl): + if matching_targets[i] != []: + matching_bs[i] = torch.cat(matching_bs[i], dim=0) + matching_as[i] = torch.cat(matching_as[i], dim=0) + matching_gjs[i] = torch.cat(matching_gjs[i], dim=0) + matching_gis[i] = torch.cat(matching_gis[i], dim=0) + matching_targets[i] = torch.cat(matching_targets[i], dim=0) + matching_anchs[i] = torch.cat(matching_anchs[i], dim=0) + else: + matching_bs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_as[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gjs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_gis[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_targets[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + matching_anchs[i] = torch.tensor([], device='cuda:0', dtype=torch.int64) + + return matching_bs, matching_as, matching_gjs, matching_gis, matching_targets, matching_anchs + + def find_5_positive(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + indices, anch = [], [] + gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 1.0 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(self.nl): + anchors = self.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices + anch.append(anchors[a]) # anchors + + return indices, anch + + def find_3_positive(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + indices, anch = [], [] + gain = torch.ones(7, device=targets.device).long() # normalized to gridspace gain + ai = torch.arange(na, device=targets.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[:, :, None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor([[0, 0], + [1, 0], [0, 1], [-1, 0], [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], device=targets.device).float() * g # offsets + + for i in range(self.nl): + anchors = self.anchors[i] + gain[2:6] = torch.tensor(p[i].shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain + if nt: + # Matches + r = t[:, :, 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1. / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1. < g) & (gxy > 1.)).T + l, m = ((gxi % 1. < g) & (gxi > 1.)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + b, c = t[:, :2].long().T # image, class + gxy = t[:, 2:4] # grid xy + gwh = t[:, 4:6] # grid wh + gij = (gxy - offsets).long() + gi, gj = gij.T # grid xy indices + + # Append + a = t[:, 6].long() # anchor indices + indices.append((b, a, gj.clamp_(0, gain[3] - 1), gi.clamp_(0, gain[2] - 1))) # image, anchor, grid indices + anch.append(anchors[a]) # anchors + + return indices, anch \ No newline at end of file diff --git a/test/yolov7-tracker/utils/metrics.py b/test/yolov7-tracker/utils/metrics.py new file mode 100644 index 0000000..5a406a5 --- /dev/null +++ b/test/yolov7-tracker/utils/metrics.py @@ -0,0 +1,225 @@ +# Model validation metrics + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch + +from . import general + + +def fitness(x): + # Model fitness as a weighted combination of metrics + w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95] + return (x[:, :4] * w).sum(1) + + +def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=()): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + # Arguments + tp: True positives (nparray, nx1 or nx10). + conf: Objectness value from 0-1 (nparray). + pred_cls: Predicted object classes (nparray). + target_cls: True object classes (nparray). + plot: Plot precision-recall curve at mAP@0.5 + save_dir: Plot save directory + # Returns + The average precision as computed in py-faster-rcnn. + """ + + # Sort by objectness + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + + # Find unique classes + unique_classes = np.unique(target_cls) + nc = unique_classes.shape[0] # number of classes, number of detections + + # Create Precision-Recall curve and compute AP for each class + px, py = np.linspace(0, 1, 1000), [] # for plotting + ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_l = (target_cls == c).sum() # number of labels + n_p = i.sum() # number of predictions + + if n_p == 0 or n_l == 0: + continue + else: + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) + + # Recall + recall = tpc / (n_l + 1e-16) # recall curve + r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases + + # Precision + precision = tpc / (tpc + fpc) # precision curve + p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score + + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) + if plot and j == 0: + py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5 + + # Compute F1 (harmonic mean of precision and recall) + f1 = 2 * p * r / (p + r + 1e-16) + if plot: + plot_pr_curve(px, py, ap, Path(save_dir) / 'PR_curve.png', names) + plot_mc_curve(px, f1, Path(save_dir) / 'F1_curve.png', names, ylabel='F1') + plot_mc_curve(px, p, Path(save_dir) / 'P_curve.png', names, ylabel='Precision') + plot_mc_curve(px, r, Path(save_dir) / 'R_curve.png', names, ylabel='Recall') + + i = f1.mean(0).argmax() # max F1 index + return p[:, i], r[:, i], ap, f1[:, i], unique_classes.astype('int32') + + +def compute_ap(recall, precision): + """ Compute the average precision, given the recall and precision curves + # Arguments + recall: The recall curve (list) + precision: The precision curve (list) + # Returns + Average precision, precision curve, recall curve + """ + + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.], recall, [recall[-1] + 0.01])) + mpre = np.concatenate(([1.], precision, [0.])) + + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + + # Integrate area under curve + method = 'interp' # methods: 'continuous', 'interp' + if method == 'interp': + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve + + return ap, mpre, mrec + + +class ConfusionMatrix: + # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, nc, conf=0.25, iou_thres=0.45): + self.matrix = np.zeros((nc + 1, nc + 1)) + self.nc = nc # number of classes + self.conf = conf + self.iou_thres = iou_thres + + def process_batch(self, detections, labels): + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + detections (Array[N, 6]), x1, y1, x2, y2, conf, class + labels (Array[M, 5]), class, x1, y1, x2, y2 + Returns: + None, updates confusion matrix accordingly + """ + detections = detections[detections[:, 4] > self.conf] + gt_classes = labels[:, 0].int() + detection_classes = detections[:, 5].int() + iou = general.box_iou(labels[:, 1:], detections[:, :4]) + + x = torch.where(iou > self.iou_thres) + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + else: + matches = np.zeros((0, 3)) + + n = matches.shape[0] > 0 + m0, m1, _ = matches.transpose().astype(np.int16) + for i, gc in enumerate(gt_classes): + j = m0 == i + if n and sum(j) == 1: + self.matrix[gc, detection_classes[m1[j]]] += 1 # correct + else: + self.matrix[self.nc, gc] += 1 # background FP + + if n: + for i, dc in enumerate(detection_classes): + # if dc > self.nc + 1: # ????为什么会出现dc大于nc+1 + # continue + if not any(m1 == i): + self.matrix[dc, self.nc] += 1 # background FN + + def matrix(self): + return self.matrix + + def plot(self, save_dir='', names=()): + try: + import seaborn as sn + + array = self.matrix / (self.matrix.sum(0).reshape(1, self.nc + 1) + 1E-6) # normalize + array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) + + fig = plt.figure(figsize=(12, 9), tight_layout=True) + sn.set(font_scale=1.0 if self.nc < 50 else 0.8) # for label size + labels = (0 < len(names) < 99) and len(names) == self.nc # apply names to ticklabels + sn.heatmap(array, annot=self.nc < 30, annot_kws={"size": 8}, cmap='Blues', fmt='.2f', square=True, + xticklabels=names + ['background FP'] if labels else "auto", + yticklabels=names + ['background FN'] if labels else "auto").set_facecolor((1, 1, 1)) + fig.axes[0].set_xlabel('True') + fig.axes[0].set_ylabel('Predicted') + fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + except Exception as e: + pass + + def print(self): + for i in range(self.nc + 1): + print(' '.join(map(str, self.matrix[i]))) + + +# Plots ---------------------------------------------------------------------------------------------------------------- + +def plot_pr_curve(px, py, ap, save_dir='pr_curve.png', names=()): + # Precision-recall curve + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + py = np.stack(py, axis=1) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py.T): + ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision) + else: + ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision) + + ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean()) + ax.set_xlabel('Recall') + ax.set_ylabel('Precision') + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left") + fig.savefig(Path(save_dir), dpi=250) + + +def plot_mc_curve(px, py, save_dir='mc_curve.png', names=(), xlabel='Confidence', ylabel='Metric'): + # Metric-confidence curve + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py): + ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric) + else: + ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric) + + y = py.mean(0) + ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + plt.legend(bbox_to_anchor=(1.04, 1), loc="upper left") + fig.savefig(Path(save_dir), dpi=250) diff --git a/test/yolov7-tracker/utils/plots.py b/test/yolov7-tracker/utils/plots.py new file mode 100644 index 0000000..e75bc7b --- /dev/null +++ b/test/yolov7-tracker/utils/plots.py @@ -0,0 +1,433 @@ +# Plotting utils + +import glob +import math +import os +import random +from copy import copy +from pathlib import Path + +import cv2 +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +import torch +import yaml +from PIL import Image, ImageDraw, ImageFont +from scipy.signal import butter, filtfilt + +from utils.general import xywh2xyxy, xyxy2xywh +from utils.metrics import fitness + +# Settings +matplotlib.rc('font', **{'size': 11}) +matplotlib.use('Agg') # for writing to files only + + +def color_list(): + # Return first 10 plt colors as (r,g,b) https://stackoverflow.com/questions/51350872/python-from-color-name-to-rgb + def hex2rgb(h): + return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4)) + + return [hex2rgb(h) for h in matplotlib.colors.TABLEAU_COLORS.values()] # or BASE_ (8), CSS4_ (148), XKCD_ (949) + + +def hist2d(x, y, n=100): + # 2d histogram used in labels.png and evolve.png + xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n) + hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges)) + xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1) + yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1) + return np.log(hist[xidx, yidx]) + + +def butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5): + # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy + def butter_lowpass(cutoff, fs, order): + nyq = 0.5 * fs + normal_cutoff = cutoff / nyq + return butter(order, normal_cutoff, btype='low', analog=False) + + b, a = butter_lowpass(cutoff, fs, order=order) + return filtfilt(b, a, data) # forward-backward filter + + +def plot_one_box(x, img, color=None, label=None, line_thickness=3): + # Plots one bounding box on image img + tl = line_thickness or round(0.002 * (img.shape[0] + img.shape[1]) / 2) + 1 # line/font thickness + color = color or [random.randint(0, 255) for _ in range(3)] + c1, c2 = (int(x[0]), int(x[1])), (int(x[2]), int(x[3])) + cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA) + if label: + tf = max(tl - 1, 1) # font thickness + t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] + c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3 + cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled + cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA) + + +def plot_one_box_PIL(box, img, color=None, label=None, line_thickness=None): + img = Image.fromarray(img) + draw = ImageDraw.Draw(img) + line_thickness = line_thickness or max(int(min(img.size) / 200), 2) + draw.rectangle(box, width=line_thickness, outline=tuple(color)) # plot + if label: + fontsize = max(round(max(img.size) / 40), 12) + font = ImageFont.truetype("Arial.ttf", fontsize) + txt_width, txt_height = font.getsize(label) + draw.rectangle([box[0], box[1] - txt_height + 4, box[0] + txt_width, box[1]], fill=tuple(color)) + draw.text((box[0], box[1] - txt_height + 1), label, fill=(255, 255, 255), font=font) + return np.asarray(img) + + +def plot_wh_methods(): # from utils.plots import *; plot_wh_methods() + # Compares the two methods for width-height anchor multiplication + # https://github.com/ultralytics/yolov3/issues/168 + x = np.arange(-4.0, 4.0, .1) + ya = np.exp(x) + yb = torch.sigmoid(torch.from_numpy(x)).numpy() * 2 + + fig = plt.figure(figsize=(6, 3), tight_layout=True) + plt.plot(x, ya, '.-', label='YOLOv3') + plt.plot(x, yb ** 2, '.-', label='YOLOR ^2') + plt.plot(x, yb ** 1.6, '.-', label='YOLOR ^1.6') + plt.xlim(left=-4, right=4) + plt.ylim(bottom=0, top=6) + plt.xlabel('input') + plt.ylabel('output') + plt.grid() + plt.legend() + fig.savefig('comparison.png', dpi=200) + + +def output_to_target(output): + # Convert model output to target format [batch_id, class_id, x, y, w, h, conf] + targets = [] + for i, o in enumerate(output): + for *box, conf, cls in o.cpu().numpy(): + targets.append([i, cls, *list(*xyxy2xywh(np.array(box)[None])), conf]) + return np.array(targets) + + +def plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16): + # Plot image grid with labels + + if isinstance(images, torch.Tensor): + images = images.cpu().float().numpy() + if isinstance(targets, torch.Tensor): + targets = targets.cpu().numpy() + + # un-normalise + if np.max(images[0]) <= 1: + images *= 255 + + tl = 3 # line thickness + tf = max(tl - 1, 1) # font thickness + bs, _, h, w = images.shape # batch size, _, height, width + bs = min(bs, max_subplots) # limit plot images + ns = np.ceil(bs ** 0.5) # number of subplots (square) + + # Check if we should resize + scale_factor = max_size / max(h, w) + if scale_factor < 1: + h = math.ceil(scale_factor * h) + w = math.ceil(scale_factor * w) + + colors = color_list() # list of colors + mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init + for i, img in enumerate(images): + if i == max_subplots: # if last batch has fewer images than we expect + break + + block_x = int(w * (i // ns)) + block_y = int(h * (i % ns)) + + img = img.transpose(1, 2, 0) + if scale_factor < 1: + img = cv2.resize(img, (w, h)) + + mosaic[block_y:block_y + h, block_x:block_x + w, :] = img + if len(targets) > 0: + image_targets = targets[targets[:, 0] == i] + boxes = xywh2xyxy(image_targets[:, 2:6]).T + classes = image_targets[:, 1].astype('int') + labels = image_targets.shape[1] == 6 # labels if no conf column + conf = None if labels else image_targets[:, 6] # check for confidence presence (label vs pred) + + if boxes.shape[1]: + if boxes.max() <= 1.01: # if normalized with tolerance 0.01 + boxes[[0, 2]] *= w # scale to pixels + boxes[[1, 3]] *= h + elif scale_factor < 1: # absolute coords need scale if image scales + boxes *= scale_factor + boxes[[0, 2]] += block_x + boxes[[1, 3]] += block_y + for j, box in enumerate(boxes.T): + cls = int(classes[j]) + color = colors[cls % len(colors)] + cls = names[cls] if names else cls + if labels or conf[j] > 0.25: # 0.25 conf thresh + label = '%s' % cls if labels else '%s %.1f' % (cls, conf[j]) + plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl) + + # Draw image filename labels + if paths: + label = Path(paths[i]).name[:40] # trim to 40 char + t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0] + cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf, + lineType=cv2.LINE_AA) + + # Image border + cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3) + + if fname: + r = min(1280. / max(h, w) / ns, 1.0) # ratio to limit image size + mosaic = cv2.resize(mosaic, (int(ns * w * r), int(ns * h * r)), interpolation=cv2.INTER_AREA) + # cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) # cv2 save + Image.fromarray(mosaic).save(fname) # PIL save + return mosaic + + +def plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''): + # Plot LR simulating training for full epochs + optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals + y = [] + for _ in range(epochs): + scheduler.step() + y.append(optimizer.param_groups[0]['lr']) + plt.plot(y, '.-', label='LR') + plt.xlabel('epoch') + plt.ylabel('LR') + plt.grid() + plt.xlim(0, epochs) + plt.ylim(0) + plt.savefig(Path(save_dir) / 'LR.png', dpi=200) + plt.close() + + +def plot_test_txt(): # from utils.plots import *; plot_test() + # Plot test.txt histograms + x = np.loadtxt('test.txt', dtype=np.float32) + box = xyxy2xywh(x[:, :4]) + cx, cy = box[:, 0], box[:, 1] + + fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True) + ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0) + ax.set_aspect('equal') + plt.savefig('hist2d.png', dpi=300) + + fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True) + ax[0].hist(cx, bins=600) + ax[1].hist(cy, bins=600) + plt.savefig('hist1d.png', dpi=200) + + +def plot_targets_txt(): # from utils.plots import *; plot_targets_txt() + # Plot targets.txt histograms + x = np.loadtxt('targets.txt', dtype=np.float32).T + s = ['x targets', 'y targets', 'width targets', 'height targets'] + fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True) + ax = ax.ravel() + for i in range(4): + ax[i].hist(x[i], bins=100, label='%.3g +/- %.3g' % (x[i].mean(), x[i].std())) + ax[i].legend() + ax[i].set_title(s[i]) + plt.savefig('targets.jpg', dpi=200) + + +def plot_study_txt(path='', x=None): # from utils.plots import *; plot_study_txt() + # Plot study.txt generated by test.py + fig, ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True) + # ax = ax.ravel() + + fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True) + # for f in [Path(path) / f'study_coco_{x}.txt' for x in ['yolor-p6', 'yolor-w6', 'yolor-e6', 'yolor-d6']]: + for f in sorted(Path(path).glob('study*.txt')): + y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T + x = np.arange(y.shape[1]) if x is None else np.array(x) + s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_inference (ms/img)', 't_NMS (ms/img)', 't_total (ms/img)'] + # for i in range(7): + # ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8) + # ax[i].set_title(s[i]) + + j = y[3].argmax() + 1 + ax2.plot(y[6, 1:j], y[3, 1:j] * 1E2, '.-', linewidth=2, markersize=8, + label=f.stem.replace('study_coco_', '').replace('yolo', 'YOLO')) + + ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5], + 'k.-', linewidth=2, markersize=8, alpha=.25, label='EfficientDet') + + ax2.grid(alpha=0.2) + ax2.set_yticks(np.arange(20, 60, 5)) + ax2.set_xlim(0, 57) + ax2.set_ylim(30, 55) + ax2.set_xlabel('GPU Speed (ms/img)') + ax2.set_ylabel('COCO AP val') + ax2.legend(loc='lower right') + plt.savefig(str(Path(path).name) + '.png', dpi=300) + + +def plot_labels(labels, names=(), save_dir=Path(''), loggers=None): + # plot dataset labels + print('Plotting labels... ') + c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes + nc = int(c.max() + 1) # number of classes + colors = color_list() + x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height']) + + # seaborn correlogram + sns.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9)) + plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200) + plt.close() + + # matplotlib labels + matplotlib.use('svg') # faster + ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel() + ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8) + ax[0].set_ylabel('instances') + if 0 < len(names) < 30: + ax[0].set_xticks(range(len(names))) + ax[0].set_xticklabels(names, rotation=90, fontsize=10) + else: + ax[0].set_xlabel('classes') + sns.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9) + sns.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9) + + # rectangles + labels[:, 1:3] = 0.5 # center + labels[:, 1:] = xywh2xyxy(labels[:, 1:]) * 2000 + img = Image.fromarray(np.ones((2000, 2000, 3), dtype=np.uint8) * 255) + for cls, *box in labels[:1000]: + ImageDraw.Draw(img).rectangle(box, width=1, outline=colors[int(cls) % 10]) # plot + ax[1].imshow(img) + ax[1].axis('off') + + for a in [0, 1, 2, 3]: + for s in ['top', 'right', 'left', 'bottom']: + ax[a].spines[s].set_visible(False) + + plt.savefig(save_dir / 'labels.jpg', dpi=200) + matplotlib.use('Agg') + plt.close() + + # loggers + for k, v in loggers.items() or {}: + if k == 'wandb' and v: + v.log({"Labels": [v.Image(str(x), caption=x.name) for x in save_dir.glob('*labels*.jpg')]}, commit=False) + + +def plot_evolution(yaml_file='data/hyp.finetune.yaml'): # from utils.plots import *; plot_evolution() + # Plot hyperparameter evolution results in evolve.txt + with open(yaml_file) as f: + hyp = yaml.load(f, Loader=yaml.SafeLoader) + x = np.loadtxt('evolve.txt', ndmin=2) + f = fitness(x) + # weights = (f - f.min()) ** 2 # for weighted results + plt.figure(figsize=(10, 12), tight_layout=True) + matplotlib.rc('font', **{'size': 8}) + for i, (k, v) in enumerate(hyp.items()): + y = x[:, i + 7] + # mu = (y * weights).sum() / weights.sum() # best weighted result + mu = y[f.argmax()] # best single result + plt.subplot(6, 5, i + 1) + plt.scatter(y, f, c=hist2d(y, f, 20), cmap='viridis', alpha=.8, edgecolors='none') + plt.plot(mu, f.max(), 'k+', markersize=15) + plt.title('%s = %.3g' % (k, mu), fontdict={'size': 9}) # limit to 40 characters + if i % 5 != 0: + plt.yticks([]) + print('%15s: %.3g' % (k, mu)) + plt.savefig('evolve.png', dpi=200) + print('\nPlot saved as evolve.png') + + +def profile_idetection(start=0, stop=0, labels=(), save_dir=''): + # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection() + ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel() + s = ['Images', 'Free Storage (GB)', 'RAM Usage (GB)', 'Battery', 'dt_raw (ms)', 'dt_smooth (ms)', 'real-world FPS'] + files = list(Path(save_dir).glob('frames*.txt')) + for fi, f in enumerate(files): + try: + results = np.loadtxt(f, ndmin=2).T[:, 90:-30] # clip first and last rows + n = results.shape[1] # number of rows + x = np.arange(start, min(stop, n) if stop else n) + results = results[:, x] + t = (results[0] - results[0].min()) # set t0=0s + results[0] = x + for i, a in enumerate(ax): + if i < len(results): + label = labels[fi] if len(labels) else f.stem.replace('frames_', '') + a.plot(t, results[i], marker='.', label=label, linewidth=1, markersize=5) + a.set_title(s[i]) + a.set_xlabel('time (s)') + # if fi == len(files) - 1: + # a.set_ylim(bottom=0) + for side in ['top', 'right']: + a.spines[side].set_visible(False) + else: + a.remove() + except Exception as e: + print('Warning: Plotting error for %s; %s' % (f, e)) + + ax[1].legend() + plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200) + + +def plot_results_overlay(start=0, stop=0): # from utils.plots import *; plot_results_overlay() + # Plot training 'results*.txt', overlaying train and val losses + s = ['train', 'train', 'train', 'Precision', 'mAP@0.5', 'val', 'val', 'val', 'Recall', 'mAP@0.5:0.95'] # legends + t = ['Box', 'Objectness', 'Classification', 'P-R', 'mAP-F1'] # titles + for f in sorted(glob.glob('results*.txt') + glob.glob('../../Downloads/results*.txt')): + results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T + n = results.shape[1] # number of rows + x = range(start, min(stop, n) if stop else n) + fig, ax = plt.subplots(1, 5, figsize=(14, 3.5), tight_layout=True) + ax = ax.ravel() + for i in range(5): + for j in [i, i + 5]: + y = results[j, x] + ax[i].plot(x, y, marker='.', label=s[j]) + # y_smooth = butter_lowpass_filtfilt(y) + # ax[i].plot(x, np.gradient(y_smooth), marker='.', label=s[j]) + + ax[i].set_title(t[i]) + ax[i].legend() + ax[i].set_ylabel(f) if i == 0 else None # add filename + fig.savefig(f.replace('.txt', '.png'), dpi=200) + + +def plot_results(start=0, stop=0, bucket='', id=(), labels=(), save_dir=''): + # Plot training 'results*.txt'. from utils.plots import *; plot_results(save_dir='runs/train/exp') + fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True) + ax = ax.ravel() + s = ['Box', 'Objectness', 'Classification', 'Precision', 'Recall', + 'val Box', 'val Objectness', 'val Classification', 'mAP@0.5', 'mAP@0.5:0.95'] + if bucket: + # files = ['https://storage.googleapis.com/%s/results%g.txt' % (bucket, x) for x in id] + files = ['results%g.txt' % x for x in id] + c = ('gsutil cp ' + '%s ' * len(files) + '.') % tuple('gs://%s/results%g.txt' % (bucket, x) for x in id) + os.system(c) + else: + files = list(Path(save_dir).glob('results*.txt')) + assert len(files), 'No results.txt files found in %s, nothing to plot.' % os.path.abspath(save_dir) + for fi, f in enumerate(files): + try: + results = np.loadtxt(f, usecols=[2, 3, 4, 8, 9, 12, 13, 14, 10, 11], ndmin=2).T + n = results.shape[1] # number of rows + x = range(start, min(stop, n) if stop else n) + for i in range(10): + y = results[i, x] + if i in [0, 1, 2, 5, 6, 7]: + y[y == 0] = np.nan # don't show zero loss values + # y /= y[0] # normalize + label = labels[fi] if len(labels) else f.stem + ax[i].plot(x, y, marker='.', label=label, linewidth=2, markersize=8) + ax[i].set_title(s[i]) + # if i in [5, 6, 7]: # share train and val loss y axes + # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) + except Exception as e: + print('Warning: Plotting error for %s; %s' % (f, e)) + + ax[1].legend() + fig.savefig(Path(save_dir) / 'results.png', dpi=200) diff --git a/test/yolov7-tracker/utils/torch_utils.py b/test/yolov7-tracker/utils/torch_utils.py new file mode 100644 index 0000000..b52b6cb --- /dev/null +++ b/test/yolov7-tracker/utils/torch_utils.py @@ -0,0 +1,374 @@ +# YOLOR PyTorch utils + +import datetime +import logging +import math +import os +import platform +import subprocess +import time +from contextlib import contextmanager +from copy import deepcopy +from pathlib import Path + +import torch +import torch.backends.cudnn as cudnn +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +try: + import thop # for FLOPS computation +except ImportError: + thop = None +logger = logging.getLogger(__name__) + + +@contextmanager +def torch_distributed_zero_first(local_rank: int): + """ + Decorator to make all processes in distributed training wait for each local_master to do something. + """ + if local_rank not in [-1, 0]: + torch.distributed.barrier() + yield + if local_rank == 0: + torch.distributed.barrier() + + +def init_torch_seeds(seed=0): + # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html + torch.manual_seed(seed) + if seed == 0: # slower, more reproducible + cudnn.benchmark, cudnn.deterministic = False, True + else: # faster, less reproducible + cudnn.benchmark, cudnn.deterministic = True, False + + +def date_modified(path=__file__): + # return human-readable file modification date, i.e. '2021-3-26' + t = datetime.datetime.fromtimestamp(Path(path).stat().st_mtime) + return f'{t.year}-{t.month}-{t.day}' + + +def git_describe(path=Path(__file__).parent): # path must be a directory + # return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe + s = f'git -C {path} describe --tags --long --always' + try: + return subprocess.check_output(s, shell=True, stderr=subprocess.STDOUT).decode()[:-1] + except subprocess.CalledProcessError as e: + return '' # not a git repository + + +def select_device(device='', batch_size=None): + # device = 'cpu' or '0' or '0,1,2,3' + s = f'YOLOR 🚀 {git_describe() or date_modified()} torch {torch.__version__} ' # string + cpu = device.lower() == 'cpu' + if cpu: + os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False + elif device: # non-cpu device requested + os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable + assert torch.cuda.is_available(), f'CUDA unavailable, invalid device {device} requested' # check availability + + cuda = not cpu and torch.cuda.is_available() + if cuda: + n = torch.cuda.device_count() + if n > 1 and batch_size: # check that batch_size is compatible with device_count + assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}' + space = ' ' * len(s) + for i, d in enumerate(device.split(',') if device else range(n)): + p = torch.cuda.get_device_properties(i) + s += f"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / 1024 ** 2}MB)\n" # bytes to MB + else: + s += 'CPU\n' + + logger.info(s.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else s) # emoji-safe + return torch.device('cuda:0' if cuda else 'cpu') + + +def time_synchronized(): + # pytorch-accurate time + if torch.cuda.is_available(): + torch.cuda.synchronize() + return time.time() + + +def profile(x, ops, n=100, device=None): + # profile a pytorch module or list of modules. Example usage: + # x = torch.randn(16, 3, 640, 640) # input + # m1 = lambda x: x * torch.sigmoid(x) + # m2 = nn.SiLU() + # profile(x, [m1, m2], n=100) # profile speed over 100 iterations + + device = device or torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + x = x.to(device) + x.requires_grad = True + print(torch.__version__, device.type, torch.cuda.get_device_properties(0) if device.type == 'cuda' else '') + print(f"\n{'Params':>12s}{'GFLOPS':>12s}{'forward (ms)':>16s}{'backward (ms)':>16s}{'input':>24s}{'output':>24s}") + for m in ops if isinstance(ops, list) else [ops]: + m = m.to(device) if hasattr(m, 'to') else m # device + m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m # type + dtf, dtb, t = 0., 0., [0., 0., 0.] # dt forward, backward + try: + flops = thop.profile(m, inputs=(x,), verbose=False)[0] / 1E9 * 2 # GFLOPS + except: + flops = 0 + + for _ in range(n): + t[0] = time_synchronized() + y = m(x) + t[1] = time_synchronized() + try: + _ = y.sum().backward() + t[2] = time_synchronized() + except: # no backward method + t[2] = float('nan') + dtf += (t[1] - t[0]) * 1000 / n # ms per op forward + dtb += (t[2] - t[1]) * 1000 / n # ms per op backward + + s_in = tuple(x.shape) if isinstance(x, torch.Tensor) else 'list' + s_out = tuple(y.shape) if isinstance(y, torch.Tensor) else 'list' + p = sum(list(x.numel() for x in m.parameters())) if isinstance(m, nn.Module) else 0 # parameters + print(f'{p:12}{flops:12.4g}{dtf:16.4g}{dtb:16.4g}{str(s_in):>24s}{str(s_out):>24s}') + + +def is_parallel(model): + return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel) + + +def intersect_dicts(da, db, exclude=()): + # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values + return {k: v for k, v in da.items() if k in db and not any(x in k for x in exclude) and v.shape == db[k].shape} + + +def initialize_weights(model): + for m in model.modules(): + t = type(m) + if t is nn.Conv2d: + pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif t is nn.BatchNorm2d: + m.eps = 1e-3 + m.momentum = 0.03 + elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6]: + m.inplace = True + + +def find_modules(model, mclass=nn.Conv2d): + # Finds layer indices matching module class 'mclass' + return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)] + + +def sparsity(model): + # Return global model sparsity + a, b = 0., 0. + for p in model.parameters(): + a += p.numel() + b += (p == 0).sum() + return b / a + + +def prune(model, amount=0.3): + # Prune model to requested global sparsity + import torch.nn.utils.prune as prune + print('Pruning model... ', end='') + for name, m in model.named_modules(): + if isinstance(m, nn.Conv2d): + prune.l1_unstructured(m, name='weight', amount=amount) # prune + prune.remove(m, 'weight') # make permanent + print(' %.3g global sparsity' % sparsity(model)) + + +def fuse_conv_and_bn(conv, bn): + # Fuse convolution and batchnorm layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + fusedconv = nn.Conv2d(conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + groups=conv.groups, + bias=True).requires_grad_(False).to(conv.weight.device) + + # prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # prepare spatial bias + b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def model_info(model, verbose=False, img_size=640): + # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320] + n_p = sum(x.numel() for x in model.parameters()) # number parameters + n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients + if verbose: + print('%5s %40s %9s %12s %20s %10s %10s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma')) + for i, (name, p) in enumerate(model.named_parameters()): + name = name.replace('module_list.', '') + print('%5g %40s %9s %12g %20s %10.3g %10.3g' % + (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std())) + + try: # FLOPS + from thop import profile + stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32 + img = torch.zeros((1, model.yaml.get('ch', 3), stride, stride), device=next(model.parameters()).device) # input + flops = profile(deepcopy(model), inputs=(img,), verbose=False)[0] / 1E9 * 2 # stride GFLOPS + img_size = img_size if isinstance(img_size, list) else [img_size, img_size] # expand if int/float + fs = ', %.1f GFLOPS' % (flops * img_size[0] / stride * img_size[1] / stride) # 640x640 GFLOPS + except (ImportError, Exception): + fs = '' + + logger.info(f"Model Summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}") + + +def load_classifier(name='resnet101', n=2): + # Loads a pretrained model reshaped to n-class output + model = torchvision.models.__dict__[name](pretrained=True) + + # ResNet model properties + # input_size = [3, 224, 224] + # input_space = 'RGB' + # input_range = [0, 1] + # mean = [0.485, 0.456, 0.406] + # std = [0.229, 0.224, 0.225] + + # Reshape output to n classes + filters = model.fc.weight.shape[1] + model.fc.bias = nn.Parameter(torch.zeros(n), requires_grad=True) + model.fc.weight = nn.Parameter(torch.zeros(n, filters), requires_grad=True) + model.fc.out_features = n + return model + + +def scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416) + # scales img(bs,3,y,x) by ratio constrained to gs-multiple + if ratio == 1.0: + return img + else: + h, w = img.shape[2:] + s = (int(h * ratio), int(w * ratio)) # new size + img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize + if not same_shape: # pad/crop img + h, w = [math.ceil(x * ratio / gs) * gs for x in (h, w)] + return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean + + +def copy_attr(a, b, include=(), exclude=()): + # Copy attributes from b to a, options to only include [...] and to exclude [...] + for k, v in b.__dict__.items(): + if (len(include) and k not in include) or k.startswith('_') or k in exclude: + continue + else: + setattr(a, k, v) + + +class ModelEMA: + """ Model Exponential Moving Average from https://github.com/rwightman/pytorch-image-models + Keep a moving average of everything in the model state_dict (parameters and buffers). + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + A smoothed version of the weights is necessary for some training schemes to perform well. + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, updates=0): + # Create EMA + self.ema = deepcopy(model.module if is_parallel(model) else model).eval() # FP32 EMA + # if next(model.parameters()).device.type != 'cpu': + # self.ema.half() # FP16 EMA + self.updates = updates # number of EMA updates + self.decay = lambda x: decay * (1 - math.exp(-x / 2000)) # decay exponential ramp (to help early epochs) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def update(self, model): + # Update EMA parameters + with torch.no_grad(): + self.updates += 1 + d = self.decay(self.updates) + + msd = model.module.state_dict() if is_parallel(model) else model.state_dict() # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: + v *= d + v += (1. - d) * msd[k].detach() + + def update_attr(self, model, include=(), exclude=('process_group', 'reducer')): + # Update EMA attributes + copy_attr(self.ema, model, include, exclude) + + +class BatchNormXd(torch.nn.modules.batchnorm._BatchNorm): + def _check_input_dim(self, input): + # The only difference between BatchNorm1d, BatchNorm2d, BatchNorm3d, etc + # is this method that is overwritten by the sub-class + # This original goal of this method was for tensor sanity checks + # If you're ok bypassing those sanity checks (eg. if you trust your inference + # to provide the right dimensional inputs), then you can just use this method + # for easy conversion from SyncBatchNorm + # (unfortunately, SyncBatchNorm does not store the original class - if it did + # we could return the one that was originally created) + return + +def revert_sync_batchnorm(module): + # this is very similar to the function that it is trying to revert: + # https://github.com/pytorch/pytorch/blob/c8b3686a3e4ba63dc59e5dcfe5db3430df256833/torch/nn/modules/batchnorm.py#L679 + module_output = module + if isinstance(module, torch.nn.modules.batchnorm.SyncBatchNorm): + new_cls = BatchNormXd + module_output = BatchNormXd(module.num_features, + module.eps, module.momentum, + module.affine, + module.track_running_stats) + if module.affine: + with torch.no_grad(): + module_output.weight = module.weight + module_output.bias = module.bias + module_output.running_mean = module.running_mean + module_output.running_var = module.running_var + module_output.num_batches_tracked = module.num_batches_tracked + if hasattr(module, "qconfig"): + module_output.qconfig = module.qconfig + for name, child in module.named_children(): + module_output.add_module(name, revert_sync_batchnorm(child)) + del module + return module_output + + +class TracedModel(nn.Module): + + def __init__(self, model=None, device=None, img_size=(640,640)): + super(TracedModel, self).__init__() + + print(" Convert model to Traced-model... ") + self.stride = model.stride + self.names = model.names + self.model = model + + self.model = revert_sync_batchnorm(self.model) + self.model.to('cpu') + self.model.eval() + + self.detect_layer = self.model.model[-1] + self.model.traced = True + + rand_example = torch.rand(1, 3, img_size, img_size) + + traced_script_module = torch.jit.trace(self.model, rand_example, strict=False) + # traced_script_module = torch.jit.script(self.model) + # traced_script_module.save("traced_model.pt") + print(" traced_script_module saved! ") + self.model = traced_script_module + self.model.to(device) + self.detect_layer.to(device) + print(" model is traced! \n") + + def forward(self, x, augment=False, profile=False): + out = self.model(x) + out = self.detect_layer(out) + return out \ No newline at end of file diff --git a/test/yolov7-tracker/utils/wandb_logging/__init__.py b/test/yolov7-tracker/utils/wandb_logging/__init__.py new file mode 100644 index 0000000..84952a8 --- /dev/null +++ b/test/yolov7-tracker/utils/wandb_logging/__init__.py @@ -0,0 +1 @@ +# init \ No newline at end of file diff --git a/test/yolov7-tracker/utils/wandb_logging/log_dataset.py b/test/yolov7-tracker/utils/wandb_logging/log_dataset.py new file mode 100644 index 0000000..74cd6c6 --- /dev/null +++ b/test/yolov7-tracker/utils/wandb_logging/log_dataset.py @@ -0,0 +1,24 @@ +import argparse + +import yaml + +from wandb_utils import WandbLogger + +WANDB_ARTIFACT_PREFIX = 'wandb-artifact://' + + +def create_dataset_artifact(opt): + with open(opt.data) as f: + data = yaml.load(f, Loader=yaml.SafeLoader) # data dict + logger = WandbLogger(opt, '', None, data, job_type='Dataset Creation') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, default='data/coco.yaml', help='data.yaml path') + parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset') + parser.add_argument('--project', type=str, default='YOLOR', help='name of W&B Project') + opt = parser.parse_args() + opt.resume = False # Explicitly disallow resume check for dataset upload job + + create_dataset_artifact(opt) diff --git a/test/yolov7-tracker/utils/wandb_logging/wandb_utils.py b/test/yolov7-tracker/utils/wandb_logging/wandb_utils.py new file mode 100644 index 0000000..aec7c5f --- /dev/null +++ b/test/yolov7-tracker/utils/wandb_logging/wandb_utils.py @@ -0,0 +1,306 @@ +import json +import sys +from pathlib import Path + +import torch +import yaml +from tqdm import tqdm + +sys.path.append(str(Path(__file__).parent.parent.parent)) # add utils/ to path +from utils.datasets import LoadImagesAndLabels +from utils.datasets import img2label_paths +from utils.general import colorstr, xywh2xyxy, check_dataset + +try: + import wandb + from wandb import init, finish +except ImportError: + wandb = None + +WANDB_ARTIFACT_PREFIX = 'wandb-artifact://' + + +def remove_prefix(from_string, prefix=WANDB_ARTIFACT_PREFIX): + return from_string[len(prefix):] + + +def check_wandb_config_file(data_config_file): + wandb_config = '_wandb.'.join(data_config_file.rsplit('.', 1)) # updated data.yaml path + if Path(wandb_config).is_file(): + return wandb_config + return data_config_file + + +def get_run_info(run_path): + run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX)) + run_id = run_path.stem + project = run_path.parent.stem + model_artifact_name = 'run_' + run_id + '_model' + return run_id, project, model_artifact_name + + +def check_wandb_resume(opt): + process_wandb_config_ddp_mode(opt) if opt.global_rank not in [-1, 0] else None + if isinstance(opt.resume, str): + if opt.resume.startswith(WANDB_ARTIFACT_PREFIX): + if opt.global_rank not in [-1, 0]: # For resuming DDP runs + run_id, project, model_artifact_name = get_run_info(opt.resume) + api = wandb.Api() + artifact = api.artifact(project + '/' + model_artifact_name + ':latest') + modeldir = artifact.download() + opt.weights = str(Path(modeldir) / "last.pt") + return True + return None + + +def process_wandb_config_ddp_mode(opt): + with open(opt.data) as f: + data_dict = yaml.load(f, Loader=yaml.SafeLoader) # data dict + train_dir, val_dir = None, None + if isinstance(data_dict['train'], str) and data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX): + api = wandb.Api() + train_artifact = api.artifact(remove_prefix(data_dict['train']) + ':' + opt.artifact_alias) + train_dir = train_artifact.download() + train_path = Path(train_dir) / 'data/images/' + data_dict['train'] = str(train_path) + + if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX): + api = wandb.Api() + val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias) + val_dir = val_artifact.download() + val_path = Path(val_dir) / 'data/images/' + data_dict['val'] = str(val_path) + if train_dir or val_dir: + ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml') + with open(ddp_data_path, 'w') as f: + yaml.dump(data_dict, f) + opt.data = ddp_data_path + + +class WandbLogger(): + def __init__(self, opt, name, run_id, data_dict, job_type='Training'): + # Pre-training routine -- + self.job_type = job_type + self.wandb, self.wandb_run, self.data_dict = wandb, None if not wandb else wandb.run, data_dict + # It's more elegant to stick to 1 wandb.init call, but useful config data is overwritten in the WandbLogger's wandb.init call + if isinstance(opt.resume, str): # checks resume from artifact + if opt.resume.startswith(WANDB_ARTIFACT_PREFIX): + run_id, project, model_artifact_name = get_run_info(opt.resume) + model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name + assert wandb, 'install wandb to resume wandb runs' + # Resume wandb-artifact:// runs here| workaround for not overwriting wandb.config + self.wandb_run = wandb.init(id=run_id, project=project, resume='allow') + opt.resume = model_artifact_name + elif self.wandb: + self.wandb_run = wandb.init(config=opt, + resume="allow", + project='YOLOR' if opt.project == 'runs/train' else Path(opt.project).stem, + name=name, + job_type=job_type, + id=run_id) if not wandb.run else wandb.run + if self.wandb_run: + if self.job_type == 'Training': + if not opt.resume: + wandb_data_dict = self.check_and_upload_dataset(opt) if opt.upload_dataset else data_dict + # Info useful for resuming from artifacts + self.wandb_run.config.opt = vars(opt) + self.wandb_run.config.data_dict = wandb_data_dict + self.data_dict = self.setup_training(opt, data_dict) + if self.job_type == 'Dataset Creation': + self.data_dict = self.check_and_upload_dataset(opt) + else: + prefix = colorstr('wandb: ') + print(f"{prefix}Install Weights & Biases for YOLOR logging with 'pip install wandb' (recommended)") + + def check_and_upload_dataset(self, opt): + assert wandb, 'Install wandb to upload dataset' + check_dataset(self.data_dict) + config_path = self.log_dataset_artifact(opt.data, + opt.single_cls, + 'YOLOR' if opt.project == 'runs/train' else Path(opt.project).stem) + print("Created dataset config file ", config_path) + with open(config_path) as f: + wandb_data_dict = yaml.load(f, Loader=yaml.SafeLoader) + return wandb_data_dict + + def setup_training(self, opt, data_dict): + self.log_dict, self.current_epoch, self.log_imgs = {}, 0, 16 # Logging Constants + self.bbox_interval = opt.bbox_interval + if isinstance(opt.resume, str): + modeldir, _ = self.download_model_artifact(opt) + if modeldir: + self.weights = Path(modeldir) / "last.pt" + config = self.wandb_run.config + opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp = str( + self.weights), config.save_period, config.total_batch_size, config.bbox_interval, config.epochs, \ + config.opt['hyp'] + data_dict = dict(self.wandb_run.config.data_dict) # eliminates the need for config file to resume + if 'val_artifact' not in self.__dict__: # If --upload_dataset is set, use the existing artifact, don't download + self.train_artifact_path, self.train_artifact = self.download_dataset_artifact(data_dict.get('train'), + opt.artifact_alias) + self.val_artifact_path, self.val_artifact = self.download_dataset_artifact(data_dict.get('val'), + opt.artifact_alias) + self.result_artifact, self.result_table, self.val_table, self.weights = None, None, None, None + if self.train_artifact_path is not None: + train_path = Path(self.train_artifact_path) / 'data/images/' + data_dict['train'] = str(train_path) + if self.val_artifact_path is not None: + val_path = Path(self.val_artifact_path) / 'data/images/' + data_dict['val'] = str(val_path) + self.val_table = self.val_artifact.get("val") + self.map_val_table_path() + if self.val_artifact is not None: + self.result_artifact = wandb.Artifact("run_" + wandb.run.id + "_progress", "evaluation") + self.result_table = wandb.Table(["epoch", "id", "prediction", "avg_confidence"]) + if opt.bbox_interval == -1: + self.bbox_interval = opt.bbox_interval = (opt.epochs // 10) if opt.epochs > 10 else 1 + return data_dict + + def download_dataset_artifact(self, path, alias): + if isinstance(path, str) and path.startswith(WANDB_ARTIFACT_PREFIX): + dataset_artifact = wandb.use_artifact(remove_prefix(path, WANDB_ARTIFACT_PREFIX) + ":" + alias) + assert dataset_artifact is not None, "'Error: W&B dataset artifact doesn\'t exist'" + datadir = dataset_artifact.download() + return datadir, dataset_artifact + return None, None + + def download_model_artifact(self, opt): + if opt.resume.startswith(WANDB_ARTIFACT_PREFIX): + model_artifact = wandb.use_artifact(remove_prefix(opt.resume, WANDB_ARTIFACT_PREFIX) + ":latest") + assert model_artifact is not None, 'Error: W&B model artifact doesn\'t exist' + modeldir = model_artifact.download() + epochs_trained = model_artifact.metadata.get('epochs_trained') + total_epochs = model_artifact.metadata.get('total_epochs') + assert epochs_trained < total_epochs, 'training to %g epochs is finished, nothing to resume.' % ( + total_epochs) + return modeldir, model_artifact + return None, None + + def log_model(self, path, opt, epoch, fitness_score, best_model=False): + model_artifact = wandb.Artifact('run_' + wandb.run.id + '_model', type='model', metadata={ + 'original_url': str(path), + 'epochs_trained': epoch + 1, + 'save period': opt.save_period, + 'project': opt.project, + 'total_epochs': opt.epochs, + 'fitness_score': fitness_score + }) + model_artifact.add_file(str(path / 'last.pt'), name='last.pt') + wandb.log_artifact(model_artifact, + aliases=['latest', 'epoch ' + str(self.current_epoch), 'best' if best_model else '']) + print("Saving model artifact on epoch ", epoch + 1) + + def log_dataset_artifact(self, data_file, single_cls, project, overwrite_config=False): + with open(data_file) as f: + data = yaml.load(f, Loader=yaml.SafeLoader) # data dict + nc, names = (1, ['item']) if single_cls else (int(data['nc']), data['names']) + names = {k: v for k, v in enumerate(names)} # to index dictionary + self.train_artifact = self.create_dataset_table(LoadImagesAndLabels( + data['train']), names, name='train') if data.get('train') else None + self.val_artifact = self.create_dataset_table(LoadImagesAndLabels( + data['val']), names, name='val') if data.get('val') else None + if data.get('train'): + data['train'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'train') + if data.get('val'): + data['val'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'val') + path = data_file if overwrite_config else '_wandb.'.join(data_file.rsplit('.', 1)) # updated data.yaml path + data.pop('download', None) + with open(path, 'w') as f: + yaml.dump(data, f) + + if self.job_type == 'Training': # builds correct artifact pipeline graph + self.wandb_run.use_artifact(self.val_artifact) + self.wandb_run.use_artifact(self.train_artifact) + self.val_artifact.wait() + self.val_table = self.val_artifact.get('val') + self.map_val_table_path() + else: + self.wandb_run.log_artifact(self.train_artifact) + self.wandb_run.log_artifact(self.val_artifact) + return path + + def map_val_table_path(self): + self.val_table_map = {} + print("Mapping dataset") + for i, data in enumerate(tqdm(self.val_table.data)): + self.val_table_map[data[3]] = data[0] + + def create_dataset_table(self, dataset, class_to_id, name='dataset'): + # TODO: Explore multiprocessing to slpit this loop parallely| This is essential for speeding up the the logging + artifact = wandb.Artifact(name=name, type="dataset") + img_files = tqdm([dataset.path]) if isinstance(dataset.path, str) and Path(dataset.path).is_dir() else None + img_files = tqdm(dataset.img_files) if not img_files else img_files + for img_file in img_files: + if Path(img_file).is_dir(): + artifact.add_dir(img_file, name='data/images') + labels_path = 'labels'.join(dataset.path.rsplit('images', 1)) + artifact.add_dir(labels_path, name='data/labels') + else: + artifact.add_file(img_file, name='data/images/' + Path(img_file).name) + label_file = Path(img2label_paths([img_file])[0]) + artifact.add_file(str(label_file), + name='data/labels/' + label_file.name) if label_file.exists() else None + table = wandb.Table(columns=["id", "train_image", "Classes", "name"]) + class_set = wandb.Classes([{'id': id, 'name': name} for id, name in class_to_id.items()]) + for si, (img, labels, paths, shapes) in enumerate(tqdm(dataset)): + height, width = shapes[0] + labels[:, 2:] = (xywh2xyxy(labels[:, 2:].view(-1, 4))) * torch.Tensor([width, height, width, height]) + box_data, img_classes = [], {} + for cls, *xyxy in labels[:, 1:].tolist(): + cls = int(cls) + box_data.append({"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, + "class_id": cls, + "box_caption": "%s" % (class_to_id[cls]), + "scores": {"acc": 1}, + "domain": "pixel"}) + img_classes[cls] = class_to_id[cls] + boxes = {"ground_truth": {"box_data": box_data, "class_labels": class_to_id}} # inference-space + table.add_data(si, wandb.Image(paths, classes=class_set, boxes=boxes), json.dumps(img_classes), + Path(paths).name) + artifact.add(table, name) + return artifact + + def log_training_progress(self, predn, path, names): + if self.val_table and self.result_table: + class_set = wandb.Classes([{'id': id, 'name': name} for id, name in names.items()]) + box_data = [] + total_conf = 0 + for *xyxy, conf, cls in predn.tolist(): + if conf >= 0.25: + box_data.append( + {"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]}, + "class_id": int(cls), + "box_caption": "%s %.3f" % (names[cls], conf), + "scores": {"class_score": conf}, + "domain": "pixel"}) + total_conf = total_conf + conf + boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space + id = self.val_table_map[Path(path).name] + self.result_table.add_data(self.current_epoch, + id, + wandb.Image(self.val_table.data[id][1], boxes=boxes, classes=class_set), + total_conf / max(1, len(box_data)) + ) + + def log(self, log_dict): + if self.wandb_run: + for key, value in log_dict.items(): + self.log_dict[key] = value + + def end_epoch(self, best_result=False): + if self.wandb_run: + wandb.log(self.log_dict) + self.log_dict = {} + if self.result_artifact: + train_results = wandb.JoinedTable(self.val_table, self.result_table, "id") + self.result_artifact.add(train_results, 'result') + wandb.log_artifact(self.result_artifact, aliases=['latest', 'epoch ' + str(self.current_epoch), + ('best' if best_result else '')]) + self.result_table = wandb.Table(["epoch", "id", "prediction", "avg_confidence"]) + self.result_artifact = wandb.Artifact("run_" + wandb.run.id + "_progress", "evaluation") + + def finish_run(self): + if self.wandb_run: + if self.log_dict: + wandb.log(self.log_dict) + wandb.run.finish() diff --git a/test/yolov7-tracker/weights/DHN.pth b/test/yolov7-tracker/weights/DHN.pth new file mode 100644 index 0000000..338346f Binary files /dev/null and b/test/yolov7-tracker/weights/DHN.pth differ diff --git a/test/yolov7-tracker/weights/ckpt.t7 b/test/yolov7-tracker/weights/ckpt.t7 new file mode 100644 index 0000000..d253aae Binary files /dev/null and b/test/yolov7-tracker/weights/ckpt.t7 differ diff --git a/test/yolov7-tracker/weights/osnet_x0_25.pth b/test/yolov7-tracker/weights/osnet_x0_25.pth new file mode 100644 index 0000000..f80a348 Binary files /dev/null and b/test/yolov7-tracker/weights/osnet_x0_25.pth differ