youtokentome: drop 1.0.6, add 1.0.7, fix tests

patch tests, add dependencies
This commit is contained in:
rick 2025-06-22 13:20:13 +02:00
parent ee37d1cbb5
commit 212169a7f7
No known key found for this signature in database
GPG key ID: A2E7B6CA577C8F07
4 changed files with 68 additions and 26 deletions

View file

@ -1 +1 @@
DIST youtokentome-1.0.6.tar.gz 86742 BLAKE2B 6c024fbce7130378d43c771ba0610842be42b74e2bea31385c2a8a8a9d1f9398273a9dd1185ea2f4c9ad7b2bf7ed61f6e43c4ffad509342f0a17060764fd4fb3 SHA512 4e9a35ec233c91946aa01d8a7f354d8227e5ae4065738b884c2a2522e5c5ebd1283c51a2fb99ce4b144cff1bef9762d38e2c6ed484a1e21256c1c4e8dcada3f9
DIST youtokentome-1.0.7.gh.tar.gz 49480 BLAKE2B edc016a1286dce6bd7daae4ce16578e7866fc4a41c2830b481641c16c305ed51ed33ca36550939906342d92594cd528051992e292090f95c886a6888367f59e4 SHA512 23dce3e6165e37821e4aed5e11708ab47dd22494ed6a66775b5d1a309ced6e04e4995d29a11a389e9f861cddba993a96f6bd51ff712d3e8d0ed9f8938a0a869e

View file

@ -0,0 +1,27 @@
Update import and change path for tests
Fix by rick <rick@gnous.eu>
--- a/tests/speed_test/speed_test.py
+++ b/tests/speed_test/speed_test.py
@@ -4,7 +4,7 @@ from pathlib import Path
from time import time
from tabulate import tabulate
-from tokenizers import BPETokenizer as HuggingFaceBPETokenizer
+from tokenizers import Tokenizer as HuggingFaceBPETokenizer
MODEL_FILE_NAME = "bpe.model"
MODEL_SUFFIX = ".model"
--- a/tests/unit_tests/test_stress.py
+++ b/tests/unit_tests/test_stress.py
@@ -4,8 +4,8 @@ from subprocess import run
def compile_test():
build_files = ["bpe.cpp", "utils.cpp", "utf8.cpp"]
- files = ["../../youtokentome/cpp/" + file_name for file_name in build_files]
- files.append("stress_test.cpp")
+ files = ["youtokentome/cpp/" + file_name for file_name in build_files]
+ files.append("tests/unit_tests/stress_test.cpp")
print("compiling stress test ...")

View file

@ -1,25 +0,0 @@
# Copyright 1999-2023 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
EAPI=8
DISTUTILS_USE_PEP517=setuptools
DISTUTILS_EXT=1
PYTHON_COMPAT=( python3_{11..13} )
inherit distutils-r1 pypi
DESCRIPTION="Unsupervised text tokenizer focused on computational efficiency"
HOMEPAGE="https://github.com/vkcom/youtokentome"
LICENSE="MIT"
SLOT="0"
KEYWORDS="amd64 ~arm64 x86"
IUSE="test"
RDEPEND="dev-python/click[${PYTHON_USEDEP}]"
DEPEND="${RDEPEND}"
BDEPEND="dev-python/cython[${PYTHON_USEDEP}]"
REQUIRED_USE="${PYTHON_REQUIRED_USE}"
distutils_enable_tests pytest

View file

@ -0,0 +1,40 @@
# Copyright 1999-2025 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
EAPI=8
DISTUTILS_USE_PEP517=setuptools
DISTUTILS_EXT=1
PYTHON_COMPAT=( python3_{11..13} )
inherit distutils-r1
DESCRIPTION="Unsupervised text tokenizer focused on computational efficiency"
HOMEPAGE="https://github.com/vkcom/youtokentome"
SRC_URI="https://github.com/VKCOM/YouTokenToMe/archive/refs/tags/v${PV}.tar.gz -> ${P}.gh.tar.gz"
S="${WORKDIR}/YouTokenToMe-${PV}"
LICENSE="MIT"
SLOT="0"
KEYWORDS="amd64 ~arm64 x86"
IUSE="test"
RESTRICT="!test? ( test )"
PATCHES=(
"${FILESDIR}/${P}-fix-test.patch"
)
RDEPEND="
dev-python/click[${PYTHON_USEDEP}]
"
BDEPEND="
${RDEPEND}
dev-python/cython[${PYTHON_USEDEP}]
test? (
dev-python/tabulate[${PYTHON_USEDEP}]
sci-ml/tokenizers
)
"
distutils_enable_tests pytest