# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:ft=tcl:et:sw=4:ts=4:sts=4

PortSystem          1.0
PortGroup           python 1.0
PortGroup           rust 1.0

name                py-tokenizers
version             0.23.2
revision            0
license             Apache-2
maintainers         nomaintainer

description         Fast and Customizable Tokenizers
long_description    Tokenizers provides an implementation of today's \
                    most used tokenizers, with a focus on performance \
                    and versatility. Includes BPE, WordPiece, and \
                    Unigram tokenizer implementations.

homepage            https://github.com/huggingface/tokenizers

checksums           ${distname}${extract.suffix} \
                    rmd160  e13841b7038c42f1af2fc2e188ac8ed7a5e1cb4a \
                    sha256  7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac \
                    size    385745

python.versions     310 311 312 313 314
python.pep517_backend \
                    maturin

if {${name} ne ${subport}} {
    depends_lib-append \
                    port:py${python.version}-huggingface_hub

    # Work around PyO3 extension-module builds losing `-undefined dynamic_lookup`
    # when MacPorts' rust PortGroup also configures the linker. Without this,
    # linking fails with a wall of undefined _Py* symbols, because those symbols
    # are meant to resolve at runtime against the host CPython, not at link time.
    # Only known to be needed on macOS 14+ (Darwin 23+), where Apple's newer
    # default linker enforces stricter undefined-symbol checking.
    if {${os.platform} eq "darwin" && ${os.major} >= 23} {
        build.env-append \
            "RUSTFLAGS=-C link-arg=-undefined -C link-arg=dynamic_lookup"
    }

    # cd ${worksrcpath}/bindings/python
    # awk -F'"' '/^name = /{name=$2} /^version = /{ver=$2} /^checksum = /{print name, ver, $2, "\\"}' Cargo.lock | pbcopy
    cargo.crates \
        ahash 0.8.12 5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75 \
        aho-corasick 1.1.5 c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba \
        anstream 1.0.0 824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d \
        anstyle 1.0.14 940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000 \
        anstyle-parse 1.0.0 52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e \
        anstyle-query 1.1.5 40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc \
        anstyle-wincon 3.0.11 291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d \
        autocfg 1.5.1 f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53 \
        base64 0.13.1 9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8 \
        bitflags 1.3.2 bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a \
        bitflags 2.13.1 b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da \
        bumpalo 3.20.3 72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649 \
        castaway 0.2.4 dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a \
        cc 1.4.4 0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273 \
        cfg-if 1.0.4 9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801 \
        colorchoice 1.0.5 1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570 \
        compact_str 0.9.1 9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab \
        console 0.16.4 4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c \
        crossbeam-deque 0.8.7 5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb \
        crossbeam-epoch 0.9.20 2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f \
        crossbeam-utils 0.8.22 61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17 \
        daachorse 3.0.3 5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d \
        darling 0.20.11 fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee \
        darling_core 0.20.11 0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e \
        darling_macro 0.20.11 fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead \
        dary_heap 0.3.9 8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe \
        defmt 1.1.1 e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1 \
        defmt-macros 1.1.1 bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8 \
        defmt-parser 1.0.0 10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e \
        derive_builder 0.20.2 507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947 \
        derive_builder_core 0.20.2 2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8 \
        derive_builder_macro 0.20.2 ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c \
        either 1.18.0 252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34 \
        encode_unicode 1.0.0 34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0 \
        env_filter 2.0.0 900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217 \
        env_logger 0.11.11 de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6 \
        errno 0.3.14 39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb \
        esaxx-rs 0.1.10 d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6 \
        fastrand 2.5.0 da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223 \
        find-msvc-tools 0.1.11 d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890 \
        fnv 1.0.7 3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1 \
        futures-channel 0.3.34 b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4 \
        futures-core 0.3.34 92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e \
        futures-macro 0.3.34 9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44 \
        futures-task 0.3.34 cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd \
        futures-util 0.3.34 0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc \
        getrandom 0.3.4 899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd \
        getrandom 0.4.3 300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099 \
        heck 0.5.0 2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea \
        ident_case 1.0.1 b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39 \
        indicatif 0.18.6 9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c \
        is_terminal_polyfill 1.70.2 a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695 \
        itertools 0.14.0 2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285 \
        itoa 1.0.18 8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682 \
        jiff 0.2.35 668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc \
        jiff-core 0.1.0 7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09 \
        jiff-static 0.2.35 3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204 \
        js-sys 0.3.104 0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a \
        libc 0.2.189 3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2 \
        linux-raw-sys 0.12.1 32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53 \
        log 0.4.34 f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6 \
        macro_rules_attribute 0.2.3 b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c \
        macro_rules_attribute-proc_macro 0.2.3 fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c \
        matrixmultiply 0.3.11 3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7 \
        memchr 2.8.3 cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98 \
        minimal-lexical 0.2.1 68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a \
        mio 1.2.3 4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8 \
        monostate 0.1.18 3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67 \
        monostate-impl 0.1.18 e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9 \
        ndarray 0.16.1 882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841 \
        ndarray 0.17.2 520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d \
        nom 7.1.3 d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a \
        num-complex 0.4.6 73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495 \
        num-integer 0.1.47 7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b \
        num-traits 0.2.19 071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841 \
        numpy 0.29.0 6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4 \
        once_cell 1.21.4 9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50 \
        once_cell_polyfill 1.70.2 384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe \
        onig 6.5.3 0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2 \
        onig_sys 69.9.3 1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7 \
        paste 1.0.15 57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a \
        pastey 0.2.3 2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4 \
        pin-project-lite 0.2.17 a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd \
        pkg-config 0.3.34 f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548 \
        portable-atomic 1.15.0 05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85 \
        portable-atomic-util 0.2.7 c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618 \
        ppv-lite86 0.2.21 85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9 \
        proc-macro2 1.0.107 985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9 \
        pyo3 0.29.2 4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b \
        pyo3-async-runtimes 0.29.0 b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046 \
        pyo3-build-config 0.29.2 f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9 \
        pyo3-ffi 0.29.2 e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327 \
        pyo3-macros 0.29.2 73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c \
        pyo3-macros-backend 0.29.2 571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952 \
        quote 1.0.47 1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001 \
        r-efi 5.3.0 69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f \
        r-efi 6.0.0 f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf \
        rand 0.9.5 b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41 \
        rand_chacha 0.9.0 d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb \
        rand_core 0.9.5 76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c \
        rawpointer 0.2.1 60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3 \
        rayon 1.12.0 fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d \
        rayon-cond 0.4.0 2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f \
        rayon-core 1.13.0 22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91 \
        regex 1.13.1 f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d \
        regex-automata 0.4.18 ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2 \
        regex-syntax 0.8.11 d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4 \
        rustc-hash 2.1.3 6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d \
        rustix 1.1.4 b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190 \
        rustversion 1.0.23 cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f \
        ryu 1.0.23 9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f \
        serde 1.0.229 4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba \
        serde_core 1.0.229 67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48 \
        serde_derive 1.0.229 e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348 \
        serde_json 1.0.151 c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14 \
        shlex 2.0.1 f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba \
        signal-hook-registry 1.4.8 c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b \
        slab 0.4.12 0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5 \
        smallvec 1.16.0 b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f \
        spm_precompiled 0.1.4 5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326 \
        static_assertions 1.1.0 a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f \
        strsim 0.11.1 7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f \
        syn 2.0.119 872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297 \
        syn 3.0.4 e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f \
        target-lexicon 0.13.5 adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca \
        tempfile 3.27.0 32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd \
        thiserror 2.0.20 ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f \
        thiserror-impl 2.0.20 bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af \
        tokio 1.53.1 202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed \
        tokio-macros 2.7.2 78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e \
        unicode-ident 1.0.24 e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75 \
        unicode-normalization-alignments 0.1.12 43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de \
        unicode-segmentation 1.13.3 c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8 \
        unicode-width 0.2.2 b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254 \
        unicode_categories 0.1.1 39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e \
        unit-prefix 0.5.2 81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3 \
        utf8parse 0.2.2 06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821 \
        version_check 0.9.5 0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a \
        wasi 0.11.1+wasi-snapshot-preview1 ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b \
        wasip2 1.0.4+wasi-0.2.12 b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487 \
        wasm-bindgen 0.2.127 1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70 \
        wasm-bindgen-macro 0.2.127 77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1 \
        wasm-bindgen-macro-support 0.2.127 e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284 \
        wasm-bindgen-shared 0.2.127 7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf \
        web-time 1.1.0 5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb \
        windows-link 0.2.1 f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5 \
        windows-sys 0.61.2 ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc \
        wit-bindgen 0.57.1 1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e \
        zerocopy 0.8.56 556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb \
        zerocopy-derive 0.8.56 f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1 \
        zmij 1.0.23 29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b
}
