mirror of
https://github.com/pentoo/pentoo-overlay
synced 2026-05-08 12:30:44 +02:00
youtokentome: drop 1.0.6, add 1.0.7, fix tests
patch tests, add dependencies
This commit is contained in:
parent
ee37d1cbb5
commit
212169a7f7
4 changed files with 68 additions and 26 deletions
|
|
@ -1 +1 @@
|
|||
DIST youtokentome-1.0.6.tar.gz 86742 BLAKE2B 6c024fbce7130378d43c771ba0610842be42b74e2bea31385c2a8a8a9d1f9398273a9dd1185ea2f4c9ad7b2bf7ed61f6e43c4ffad509342f0a17060764fd4fb3 SHA512 4e9a35ec233c91946aa01d8a7f354d8227e5ae4065738b884c2a2522e5c5ebd1283c51a2fb99ce4b144cff1bef9762d38e2c6ed484a1e21256c1c4e8dcada3f9
|
||||
DIST youtokentome-1.0.7.gh.tar.gz 49480 BLAKE2B edc016a1286dce6bd7daae4ce16578e7866fc4a41c2830b481641c16c305ed51ed33ca36550939906342d92594cd528051992e292090f95c886a6888367f59e4 SHA512 23dce3e6165e37821e4aed5e11708ab47dd22494ed6a66775b5d1a309ced6e04e4995d29a11a389e9f861cddba993a96f6bd51ff712d3e8d0ed9f8938a0a869e
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
Update import and change path for tests
|
||||
Fix by rick <rick@gnous.eu>
|
||||
|
||||
--- a/tests/speed_test/speed_test.py
|
||||
+++ b/tests/speed_test/speed_test.py
|
||||
@@ -4,7 +4,7 @@ from pathlib import Path
|
||||
from time import time
|
||||
|
||||
from tabulate import tabulate
|
||||
-from tokenizers import BPETokenizer as HuggingFaceBPETokenizer
|
||||
+from tokenizers import Tokenizer as HuggingFaceBPETokenizer
|
||||
|
||||
MODEL_FILE_NAME = "bpe.model"
|
||||
MODEL_SUFFIX = ".model"
|
||||
--- a/tests/unit_tests/test_stress.py
|
||||
+++ b/tests/unit_tests/test_stress.py
|
||||
@@ -4,8 +4,8 @@ from subprocess import run
|
||||
|
||||
def compile_test():
|
||||
build_files = ["bpe.cpp", "utils.cpp", "utf8.cpp"]
|
||||
- files = ["../../youtokentome/cpp/" + file_name for file_name in build_files]
|
||||
- files.append("stress_test.cpp")
|
||||
+ files = ["youtokentome/cpp/" + file_name for file_name in build_files]
|
||||
+ files.append("tests/unit_tests/stress_test.cpp")
|
||||
|
||||
print("compiling stress test ...")
|
||||
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Copyright 1999-2023 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
DISTUTILS_USE_PEP517=setuptools
|
||||
DISTUTILS_EXT=1
|
||||
PYTHON_COMPAT=( python3_{11..13} )
|
||||
|
||||
inherit distutils-r1 pypi
|
||||
|
||||
DESCRIPTION="Unsupervised text tokenizer focused on computational efficiency"
|
||||
HOMEPAGE="https://github.com/vkcom/youtokentome"
|
||||
|
||||
LICENSE="MIT"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64 ~arm64 x86"
|
||||
IUSE="test"
|
||||
|
||||
RDEPEND="dev-python/click[${PYTHON_USEDEP}]"
|
||||
DEPEND="${RDEPEND}"
|
||||
BDEPEND="dev-python/cython[${PYTHON_USEDEP}]"
|
||||
REQUIRED_USE="${PYTHON_REQUIRED_USE}"
|
||||
|
||||
distutils_enable_tests pytest
|
||||
40
dev-python/youtokentome/youtokentome-1.0.7.ebuild
Normal file
40
dev-python/youtokentome/youtokentome-1.0.7.ebuild
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Copyright 1999-2025 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
DISTUTILS_USE_PEP517=setuptools
|
||||
DISTUTILS_EXT=1
|
||||
PYTHON_COMPAT=( python3_{11..13} )
|
||||
|
||||
inherit distutils-r1
|
||||
|
||||
DESCRIPTION="Unsupervised text tokenizer focused on computational efficiency"
|
||||
HOMEPAGE="https://github.com/vkcom/youtokentome"
|
||||
SRC_URI="https://github.com/VKCOM/YouTokenToMe/archive/refs/tags/v${PV}.tar.gz -> ${P}.gh.tar.gz"
|
||||
|
||||
S="${WORKDIR}/YouTokenToMe-${PV}"
|
||||
|
||||
LICENSE="MIT"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64 ~arm64 x86"
|
||||
IUSE="test"
|
||||
RESTRICT="!test? ( test )"
|
||||
|
||||
PATCHES=(
|
||||
"${FILESDIR}/${P}-fix-test.patch"
|
||||
)
|
||||
|
||||
RDEPEND="
|
||||
dev-python/click[${PYTHON_USEDEP}]
|
||||
"
|
||||
BDEPEND="
|
||||
${RDEPEND}
|
||||
dev-python/cython[${PYTHON_USEDEP}]
|
||||
test? (
|
||||
dev-python/tabulate[${PYTHON_USEDEP}]
|
||||
sci-ml/tokenizers
|
||||
)
|
||||
"
|
||||
|
||||
distutils_enable_tests pytest
|
||||
Loading…
Reference in a new issue