Add french, italian, portuguese, spanish, german (#155)

The french 6 layers variant wasn't included because it's not good enough
yet. Use the `french_24l` variant for now.

tests/test_split_sentences.py

@@ -13,14 +13,22 @@ def tokenizer():
 
 def test_short_text_single_chunk(tokenizer):
     """Short text should produce a single chunk."""
-    chunks = split_into_best_sentences(tokenizer, "Hello world.", 50)
+    chunks = split_into_best_sentences(
+        tokenizer,
+        "Hello world.",
+        50,
+        pad_with_spaces_for_short_inputs=False,
+        remove_semicolons=False,
+    )
     assert len(chunks) == 1
 
 
 def test_multiple_sentences_split(tokenizer):
     """Multiple sentences should be split when they exceed max_tokens."""
     text = "First sentence here. Second sentence here. Third sentence here. Fourth sentence here."
-    chunks = split_into_best_sentences(tokenizer, text, 10)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 10, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) > 1
 
 
@@ -34,7 +42,9 @@ def test_long_sentence_with_commas_is_split(tokenizer):
         "it was the season of Light, it was the season of Darkness, "
         "it was the spring of hope, it was the winter of despair"
     )
-    chunks = split_into_best_sentences(tokenizer, text, 50)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 50, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) > 1, "Long comma-separated text should be split into multiple chunks"
 
     # Verify all content is preserved (no words should be lost in splitting)
@@ -51,7 +61,9 @@ def test_long_sentence_with_commas_respects_max_tokens(tokenizer):
         "it was the epoch of belief, it was the epoch of incredulity"
     )
     max_tokens = 20
-    chunks = split_into_best_sentences(tokenizer, text, max_tokens)
+    chunks = split_into_best_sentences(
+        tokenizer, text, max_tokens, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     for chunk in chunks:
         token_count = len(tokenizer(chunk.strip()).tokens[0].tolist())
         # Allow some tolerance since comma clauses may vary in size
@@ -69,14 +81,18 @@ def test_mixed_sentences_and_commas(tokenizer):
         "until it finally reaches a period. "
         "Another short one."
     )
-    chunks = split_into_best_sentences(tokenizer, text, 20)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 20, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) >= 3
 
 
 def test_no_commas_no_periods_stays_single_chunk(tokenizer):
     """Text with no splitting characters stays as a single chunk."""
     text = "one two three four five six seven eight nine ten eleven twelve"
-    chunks = split_into_best_sentences(tokenizer, text, 5)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 5, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     # Should be 1 chunk since there are no split points
     assert len(chunks) == 1
 
@@ -87,14 +103,18 @@ def test_semicolons_and_colons_also_split(tokenizer):
         "First clause here; second clause here; third clause here; "
         "fourth clause here; fifth clause here; sixth clause here"
     )
-    chunks = split_into_best_sentences(tokenizer, text, 15)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 15, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) > 1
 
 
 def test_short_sentence_not_affected_by_comma_splitting(tokenizer):
     """Sentences under max_tokens should not be affected by comma logic."""
     text = "Hello, world."
-    chunks = split_into_best_sentences(tokenizer, text, 50)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 50, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) == 1
     assert "hello" in chunks[0].lower()
     assert "world" in chunks[0].lower()
@@ -103,14 +123,18 @@ def test_short_sentence_not_affected_by_comma_splitting(tokenizer):
 def test_empty_string_raises(tokenizer):
     """Empty input should raise ValueError from prepare_text_prompt."""
     with pytest.raises(ValueError, match="empty"):
-        split_into_best_sentences(tokenizer, "", 50)
+        split_into_best_sentences(
+            tokenizer, "", 50, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+        )
 
 
 def test_oversized_clause_without_commas_still_returns(tokenizer):
     """A long clause with no split points should still be returned (not dropped)."""
     # 20 words with no punctuation at all - no way to split
     text = " ".join(f"word{i}" for i in range(20))
-    chunks = split_into_best_sentences(tokenizer, text, 5)
+    chunks = split_into_best_sentences(
+        tokenizer, text, 5, pad_with_spaces_for_short_inputs=False, remove_semicolons=False
+    )
     assert len(chunks) == 1
     # prepare_text_prompt capitalizes the first char and adds a trailing period,
     # so compare case-insensitively and strip punctuation