improve split

This commit is contained in:
andrew (from workstation) 2020-10-16 22:22:07 +02:00
parent b804585a38
commit 969aeea86f

View File

@ -37,6 +37,10 @@ def split_docs(docs: typing.List[str]) -> typing.List[str]:
else:
result.append(token)
if len(result) > 1 and len(result[1]) < 3:
result[-2] += f" {result[-1]}"
del result[-1]
return result