From 1b84af4e24c6cc80dd62c780876c0605f33845f0 Mon Sep 17 00:00:00 2001 From: Osamaali313 Date: Tue, 14 Jul 2026 23:28:27 +0300 Subject: [PATCH] fix(util): add missing comma between "never" and "nevertheless" stop words In Utilities.get_stop_words_master_list the entries "never" and "nevertheless" had no comma between them, so Python implicit string concatenation merged them into a single junk token "nevernevertheless". As a result neither "never" nor "nevertheless" is in the stop-word list, and a token that can never match any real word is. This list feeds remove_stop_words / CorpTokenizer used in the tokenization/pruning paths (prompts, retrieval, models, agents), so both words are treated as meaningful tokens instead of being filtered. Add the comma. --- llmware/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llmware/util.py b/llmware/util.py index b6299c03..081c7ea2 100755 --- a/llmware/util.py +++ b/llmware/util.py @@ -499,7 +499,7 @@ def get_stop_words_master_list(): "meantime" ,"meanwhile" ,"merely" ,"mg" ,"might","miss", "ml" ,"more" ,"moreover", "most" ,"mostly" ,"mr" ,"mr." ,"mrs" ,"mrs." ,"ms", "ms." ,"much" ,"mug","must" ,"my" ,"myself", "n" ,"na" ,"name" ,"namely" ,"nay" ,"nd" ,"near" ,"nearly" ,"necessarily" ,"necessary" ,"need" - ,"needs", "neither" ,"never""nevertheless" ,"new" ,"next" ,"nine" ,"ninety" ,"no" ,"nobody", + ,"needs", "neither" ,"never" ,"nevertheless" ,"new" ,"next" ,"nine" ,"ninety" ,"no" ,"nobody", "non" ,"none","nonetheless","noone" ,"nor" ,"normally" ,"nos" ,"not" ,"note" ,"noted" , "nothing" ,"now" ,"nowhere" ,"o" ,"obtain","obtained", "obviously" ,"of" ,"off" ,"often", "oh" ,"ok" ,"okay" ,"old" ,"omit" ,"omitted" ,"on" ,"once" ,"one","ones","only" ,"onto" ,"or",